From 316a42bacb746b34004729f14d635f971e2cb70e Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 9 Sep 2024 20:11:28 +0100 Subject: [PATCH 001/166] Updating roof recommendations to allow room roof recommendations to be included in non-invasive recommendations --- .../functions/energy_assessment_functions.py | 4 + .../app/db/functions/property_functions.py | 9 +- backend/app/db/models/portfolio.py | 1 + backend/app/plan/router.py | 30 ++++-- etl/customers/vectis/outputs.py | 91 +++++++++++++++---- recommendations/LightingRecommendations.py | 4 + recommendations/Recommendations.py | 7 +- recommendations/RoofRecommendations.py | 47 +++++++--- 8 files changed, 154 insertions(+), 39 deletions(-) diff --git a/backend/app/db/functions/energy_assessment_functions.py b/backend/app/db/functions/energy_assessment_functions.py index ca2f721c..bbdaaac7 100644 --- a/backend/app/db/functions/energy_assessment_functions.py +++ b/backend/app/db/functions/energy_assessment_functions.py @@ -71,6 +71,10 @@ def get_latest_assessment_by_uprn(session: Session, uprn: int) -> Optional[Energ :param uprn: The unique property reference number :return: The latest EnergyAssessment object or None if not found """ + + if not uprn: + return EnergyAssessment.empty_response() + try: # Query the EnergyAssessment model, filter by uprn, order by inspection_date in descending order latest_assessment = session.query(EnergyAssessment).filter_by(uprn=uprn).order_by( diff --git a/backend/app/db/functions/property_functions.py b/backend/app/db/functions/property_functions.py index 88b4e87d..b17d8e53 100644 --- a/backend/app/db/functions/property_functions.py +++ b/backend/app/db/functions/property_functions.py @@ -11,7 +11,8 @@ from backend.app.db.models.portfolio import ( from sqlalchemy.orm.exc import NoResultFound -def create_property(session: Session, portfolio_id: int, address: str, postcode: str, uprn: str) -> (int, bool): +def create_property(session: Session, portfolio_id: int, address: str, postcode: str, uprn: str, + energy_assessment: dict) -> (int, bool): """ This function will create a record for the property in the database if it does not exist. If it does exist, it will just update the updated_at field. @@ -39,13 +40,17 @@ def create_property(session: Session, portfolio_id: int, address: str, postcode: except NoResultFound: # Property doesn't exist, create a new one + + status = PortfolioStatus.ASSESSMENT.value if len(energy_assessment["epc"]) == 0 \ + else PortfolioStatus.SURVEY.value + new_property = PropertyModel( address=address, postcode=postcode, portfolio_id=portfolio_id, uprn=uprn, creation_status=PropertyCreationStatus.LOADING, - status=PortfolioStatus.ASSESSMENT.value, + status=status, has_pre_condition_report=False, has_recommendations=False ) diff --git a/backend/app/db/models/portfolio.py b/backend/app/db/models/portfolio.py index 7580a27d..e2b258f4 100644 --- a/backend/app/db/models/portfolio.py +++ b/backend/app/db/models/portfolio.py @@ -11,6 +11,7 @@ Base = declarative_base() class PortfolioStatus(enum.Enum): SCOPING = "scoping" ASSESSMENT = "assessment" + SURVEY = "survey" TENDERING = "tendering" PROJECT_UNDERWAY = "project underway" COMPLETION_ON_TRACK = "completion; status: on track" diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 929ce7fa..e925fe00 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -235,10 +235,14 @@ def create_epc_records(epc_searcher: SearchEpc, energy_assessment: dict): EnergyAssessment.empty_response() method """ + newest_epc = epc_searcher.newest_epc.copy() + if newest_epc["uprn"] == "" and epc_searcher.uprn: + newest_epc["uprn"] = epc_searcher.uprn + if not energy_assessment["epc"]: energy_assessment_is_newer = False return { - 'original_epc': epc_searcher.newest_epc.copy(), + 'original_epc': newest_epc, 'full_sap_epc': epc_searcher.full_sap_epc.copy(), 'old_data': epc_searcher.older_epcs.copy(), }, energy_assessment_is_newer @@ -249,22 +253,22 @@ def create_epc_records(epc_searcher: SearchEpc, energy_assessment: dict): # We insert county into the epc, since right now this isn't something that we pull out from the energy # assessment for col in ["county", "constituency", "constituency-label", "local-authority", "local-authority-label"]: - epc[col] = epc_searcher.newest_epc[col] + epc[col] = newest_epc[col] # We check if the energy assessment is newer than the newest EPC - if pd.to_datetime(energy_assessment_date) > pd.to_datetime(epc_searcher.newest_epc["inspection-date"]): + if pd.to_datetime(energy_assessment_date) > pd.to_datetime(newest_epc["inspection-date"]): # In this case, our energy assessment is newer than the EPCs available for this property energy_assessment_is_newer = True return { "original_epc": epc, "full_sap_epc": epc_searcher.full_sap_epc.copy(), - "old_data": epc_searcher.older_epcs.copy() + [epc_searcher.newest_epc.copy()] + "old_data": epc_searcher.older_epcs.copy() + [newest_epc] }, energy_assessment_is_newer # We check if the EPC we have produced is contained in the set of EPCs done for the property # We do this based on inspection-date and SAP epc_in_historicals = [ - x for x in epc_searcher.older_epcs + [epc_searcher.newest_epc] + x for x in epc_searcher.older_epcs + [newest_epc] if x["inspection-date"] == energy_assessment_date and x["current-energy-efficiency"] == epc["current-energy-efficiency"] ] @@ -273,7 +277,7 @@ def create_epc_records(epc_searcher: SearchEpc, energy_assessment: dict): if epc_in_historicals: # Then the EPC we have produced is already in the set of EPCs, and our EPC is older than the newest return { - "original_epc": epc_searcher.newest_epc.copy(), + "original_epc": newest_epc, "full_sap_epc": epc_searcher.full_sap_epc.copy(), "old_data": epc_searcher.older_epcs.copy() }, energy_assessment_is_newer @@ -281,7 +285,7 @@ def create_epc_records(epc_searcher: SearchEpc, energy_assessment: dict): # In this case, our EPC is older than the newest publically avaible one, but is not contained in # the historicals, so it can't have been lodged, so we include it in the old data return { - 'original_epc': epc_searcher.newest_epc.copy(), + 'original_epc': newest_epc, 'full_sap_epc': epc_searcher.full_sap_epc.copy(), 'old_data': epc_searcher.older_epcs.copy() + [epc], }, energy_assessment_is_newer @@ -412,7 +416,8 @@ async def trigger_plan(body: PlanTriggerRequest): # Create a record in db property_id, is_new = create_property( - session, body.portfolio_id, epc_searcher.address_clean, epc_searcher.postcode_clean, epc_searcher.uprn + session, body.portfolio_id, epc_searcher.address_clean, epc_searcher.postcode_clean, epc_searcher.uprn, + energy_assessment ) if not is_new and not body.multi_plan: continue @@ -799,6 +804,15 @@ async def trigger_plan(body: PlanTriggerRequest): if ventilation_rec: selected_recommendations.add(ventilation_rec["recommendation_id"]) + # If we have a trickle vents recommendation, we also switch it on. We don't just check the solution + trickle_vents_rec = next( + (r[0] for r in recommendations[p.id] if r[0]["type"] == "trickle_vents"), + None + ) + # If a matching recommendation was found, add its ID to the selected recommendations + if trickle_vents_rec: + selected_recommendations.add(trickle_vents_rec["recommendation_id"]) + # We'll use the set of selected recommendations to filter the recommendations to upload final_recommendations = [ [ diff --git a/etl/customers/vectis/outputs.py b/etl/customers/vectis/outputs.py index c6d0905f..67e27bd9 100644 --- a/etl/customers/vectis/outputs.py +++ b/etl/customers/vectis/outputs.py @@ -13,7 +13,7 @@ def app(): "surveyor": "JAFFERSONS ENERGY CONSULTANTS", "project_code": "VEC001", } - + # 5 Grove Mansions # These are the recommendations based on the on-site survey of the property. non_intrusive_recommendations = [ { @@ -22,17 +22,17 @@ def app(): "recommendations": [ { "type": "draught_proofing", - "cost": 123, + "cost": 100, "survey": True, "sap_points": 1 }, { - "type": "mixed_glazing", "cost": 12345, "survey": True, + "type": "mixed_glazing", "cost": 14632, "survey": True, "description": "Install double glazing to north facing windows and secondary glazing to the " "remaining windows at the front of the building", "sap_points": 3 }, - {"type": "trickle_vents", "cost": 500, "survey": True}, + {"type": "trickle_vents", "cost": 1000, "survey": True}, {"type": "suspended_floor_insulation", "cost": None, "survey": True, "sap_points": 2}, {"type": "internal_wall_insulation", "cost": None, "survey": True, "sap_points": 5}, ] @@ -41,14 +41,14 @@ def app(): # 8 Grove Mansions "uprn": 10024087855, "recommendations": [ - {"type": "draught_proofing", "cost": 123, "survey": True, "sap_points": 2}, + {"type": "draught_proofing", "cost": 100, "survey": True, "sap_points": 2}, { - "type": "mixed_glazing", "cost": 12345, "survey": True, + "type": "mixed_glazing", "cost": 7814, "survey": True, "description": "Install double glazing to north facing windows and secondary glazing to the " "remaining windows at the front of the building", "sap_points": 4 }, - {"type": "trickle_vents", "cost": 500, "survey": True}, + {"type": "trickle_vents", "cost": 700, "survey": True}, {"type": "low_energy_lighting", "cost": None, "survey": True, "sap_points": 0}, {"type": "internal_wall_insulation", "cost": None, "survey": True, 'sap_points': 5}, ] @@ -57,14 +57,14 @@ def app(): # 9 Grove Mansions "uprn": 121016128, "recommendations": [ - {"type": "draught_proofing", "cost": 123, "survey": True, "sap_points": 1}, + {"type": "draught_proofing", "cost": 100, "survey": True, "sap_points": 1}, { - "type": "mixed_glazing", "cost": 12345, "survey": True, + "type": "mixed_glazing", "cost": 9740, "survey": True, "description": "Install double glazing to north facing windows and secondary glazing to the " "remaining windows at the front of the building", "sap_points": 3 }, - {"type": "trickle_vents", "cost": 500, "survey": True}, + {"type": "trickle_vents", "cost": 1000, "survey": True}, {"type": "low_energy_lighting", "cost": None, "survey": True, "sap_points": 1}, {"type": "suspended_floor_insulation", "cost": None, "sap_points": 1}, {"type": "internal_wall_insulation", "cost": None, "survey": True, "sap_points": 6}, @@ -75,12 +75,12 @@ def app(): "uprn": 121016124, "recommendations": [ { - "type": "mixed_glazing", "cost": 12345, "survey": True, + "type": "mixed_glazing", "cost": 12662, "survey": True, "description": "Install double glazing to north facing windows and secondary glazing to the " "remaining windows at the front of the building", "sap_points": 5 }, - {"type": "trickle_vents", "cost": 500, "survey": True}, + {"type": "trickle_vents", "cost": 1300, "survey": True}, {"type": "low_energy_lighting", "cost": None, "survey": True, "sap_points": 2}, {"type": "internal_wall_insulation", "cost": None, "survey": True, "sap_points": 8}, ] @@ -89,14 +89,14 @@ def app(): # 14 Grove Mansions "uprn": 121016117, "recommendations": [ - {"type": "draught_proofing", "cost": 123, "survey": True, "sap_points": 1}, + {"type": "draught_proofing", "cost": 100, "survey": True, "sap_points": 1}, { - "type": "mixed_glazing", "cost": 12345, "survey": True, + "type": "mixed_glazing", "cost": 10736, "survey": True, "description": "Install double glazing to north facing windows and secondary glazing to the " "remaining windows at the front of the building", "sap_points": 4 }, - {"type": "trickle_vents", "cost": 500, "survey": True}, + {"type": "trickle_vents", "cost": 1000, "survey": True}, {"type": "low_energy_lighting", "cost": None, "survey": True, "sap_points": 1}, {"type": "internal_wall_insulation", "cost": None, "survey": True, "sap_points": 6}, ] @@ -113,6 +113,7 @@ def app(): ] asset_list = [ + # These are properties where we've done a survey { "uprn": 121016121, "address": "", "postcode": "" }, @@ -131,6 +132,63 @@ def app(): { "uprn": 10024087902, "address": "", "postcode": "" }, + # These properties we just model with default data + # Flat 1 + { + "uprn": 121016113, "address": "", "postcode": "" + }, + # Flat 10 + { + "uprn": 121016114, "address": "", "postcode": "" + }, + # Flat 11 + { + "uprn": 121016115, "address": "", "postcode": "" + }, + # Flat 12 + { + "uprn": 121016116, "address": "", "postcode": "" + }, + # Flat 15 + { + "uprn": 121016118, "address": "", "postcode": "" + }, + # Flat 16 + { + "uprn": 121016119, "address": "", "postcode": "" + }, + # Flat 17 + { + "address": "Flat 17 Grove Mansions", "postcode": "SW4 9SL" + }, + # Flat 18 + { + "uprn": 10024087901, "address": "", "postcode": "" + }, + # Flat 3 + { + "uprn": 121016122, "address": "", "postcode": "" + }, + # Flat 4 + { + "uprn": 121016123, "address": "", "postcode": "" + }, + # Flat 6 + { + "uprn": 121016125, "address": "", "postcode": "" + }, + # Flat 7 + { + "uprn": 10024087854, "address": "", "postcode": "" + }, + # Flat 7A + { + "uprn": 10024087840, "address": "", "postcode": "" + }, + # Flat 8A + { + "uprn": 10024087841, "address": "", "postcode": "" + }, ] asset_list = pd.DataFrame(asset_list) @@ -185,7 +243,8 @@ def app(): "trickle_vents", "low_energy_lighting", "suspended_floor_insulation", - "internal_wall_insulation" + "internal_wall_insulation", + "room_roof_insulation" ], "budget": None, "scenario_name": "Do when void", diff --git a/recommendations/LightingRecommendations.py b/recommendations/LightingRecommendations.py index b9456f8d..92394c11 100644 --- a/recommendations/LightingRecommendations.py +++ b/recommendations/LightingRecommendations.py @@ -9,6 +9,9 @@ class LightingRecommendations: # worth more than 2 points, but this is unlikely in the context of other upgrades that can be made to the property SAP_LIMIT = 2 + # If more than 50% of the lighting is LEDs already, the limit is 1 SAP point + SAP_LOWER_LIMIT = 1 + def __init__(self, property_instance: Property, materials: List): """ :param property_instance: Instance of the Property class, for the home associated to property_id @@ -128,6 +131,7 @@ class LightingRecommendations: "description_simulation": { "lighting-energy-eff": "Very Good", "lighting-description": "Low energy lighting in all fixed outlets", + "low-energy-lighting": 100, }, **cost_result, "survey": leds_recommendation_config.get("survey", False) diff --git a/recommendations/Recommendations.py b/recommendations/Recommendations.py index 45498a8a..db9a6320 100644 --- a/recommendations/Recommendations.py +++ b/recommendations/Recommendations.py @@ -529,7 +529,12 @@ class Recommendations: # For the moment, we cap the number of SAP points that can be achieved by LEDs at 2 if rec["type"] == "low_energy_lighting": - property_phase_impact["sap"] = min(property_phase_impact["sap"], LightingRecommendations.SAP_LIMIT) + + if property_instance.data["low-energy-lighting"] < 50: + lighting_sap_limit = LightingRecommendations.SAP_LIMIT + else: + lighting_sap_limit = LightingRecommendations.SAP_LOWER_LIMIT + property_phase_impact["sap"] = min(property_phase_impact["sap"], lighting_sap_limit) property_phase_impact["carbon"] = min( property_phase_impact["carbon"], rec["co2_equivalent_savings"] ) diff --git a/recommendations/RoofRecommendations.py b/recommendations/RoofRecommendations.py index fe027371..6635dd51 100644 --- a/recommendations/RoofRecommendations.py +++ b/recommendations/RoofRecommendations.py @@ -155,23 +155,44 @@ class RoofRecommendations: ) self.estimated_u_value = u_value - if (u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE) or ( - "loft_insulation" not in measures + if (u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE) or all( + m not in measures for m in MEASURE_MAP["roof_insulation"] ): # The Roof is already compliant return - if (self.property.roof["is_pitched"] and "loft_insulation" in measures) or ( - self.property.roof["is_flat"] and "flat_roof_insulation" in measures + non_invasive_recommendations = self.property.non_invasive_recommendations + + # We firstly handle non-intrusive recommendations, which may override the normal roof insulation recommendations + if ("loft_insulation" in [x["type"] for x in non_invasive_recommendations]) or ( + self.property.roof["is_pitched"] and "loft_insulation" in measures ): - insulation_thickness = 0 if "loft_insulation" not in measures else self.insulation_thickness - self.recommend_roof_insulation(u_value, insulation_thickness, self.property.roof, phase) + self.recommend_roof_insulation( + u_value=u_value, + insulation_thickness=self.insulation_thickness, + phase=phase, + is_flat=False, + is_pitched=True + ) + return + + if ( + (self.property.roof["is_flat"] and "flat_roof_insulation" in measures) or + "flat_roof_insulation" in [x["type"] for x in non_invasive_recommendations] + ): + self.recommend_roof_insulation( + u_value=u_value, + insulation_thickness=0, + phase=phase, + is_flat=True, + is_pitched=False + ) return # There are cases where the property might have a room roof as the second roof, but we have a recommendation for # it, so we allow this override if self.property.roof["is_roof_room"] and ("room_roof_insulation" in measures) or ( - "room_roof_insulation" in [x["type"] for x in self.property.non_invasive_recommendations] + "room_roof_insulation" in [x["type"] for x in non_invasive_recommendations] ): self.recommend_room_roof_insulation(u_value, phase) return @@ -195,7 +216,7 @@ class RoofRecommendations: raise ValueError("Invalid material type") def recommend_roof_insulation( - self, u_value, insulation_thickness, roof, phase + self, u_value, insulation_thickness, phase, is_pitched, is_flat ): """ @@ -218,7 +239,9 @@ class RoofRecommendations: :param u_value: U-value of the roof before any retrofit measures have been installed :param insulation_thickness: Existing Insulation thickness of the loft - :param roof: dictionary describing the make-up of the roof + :param phase: Phase of the recommendation + :param is_pitched: Is the roof pitched + :param is_flat: Is the roof flat :return: """ @@ -226,10 +249,10 @@ class RoofRecommendations: # Therefore the price is 100mm + whatever thickness is rolled on top, rolled at a 90 degree angle # from the base layer - if roof["is_pitched"]: + if is_pitched: insulation_materials = self.loft_insulation_materials non_insulation_materials = self.loft_non_insulation_materials - elif roof["is_flat"]: + elif is_flat: insulation_materials = self.flat_roof_insulation_materials non_insulation_materials = self.flat_roof_non_insulation_materials else: @@ -251,7 +274,7 @@ class RoofRecommendations: # Note: This requirement is only for loft insulation if ( (material["depth"] + insulation_thickness) < self.MINIMUM_RECOMMENDED_LOFT_INSULATION - ) and roof["is_pitched"]: + ) and is_pitched: continue part_u_value = r_value_per_mm_to_u_value(material["depth"], material["r_value_per_mm"]) From 910d8a5f384e479b6851ef3ac236fc5b3002cb42 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 9 Sep 2024 22:34:45 +0100 Subject: [PATCH 002/166] minor modifications to enable vectis portfolio --- etl/customers/vectis/outputs.py | 5 +++-- recommendations/Costs.py | 5 ++--- recommendations/Recommendations.py | 6 ++++-- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/etl/customers/vectis/outputs.py b/etl/customers/vectis/outputs.py index 67e27bd9..333d2494 100644 --- a/etl/customers/vectis/outputs.py +++ b/etl/customers/vectis/outputs.py @@ -220,7 +220,7 @@ def app(): "patches_file_path": "", "non_invasive_recommendations_file_path": non_invasive_recommendations_filename, "inclusions": [ - "draught_proofing", "mixed_glazing", "trickle_vents", "low_energy_lighting", + "draught_proofing", "mixed_glazing", "trickle_vents", "low_energy_lighting", "windows" ], "budget": None, "scenario_name": "Quick wins - do now while tenanted", @@ -244,7 +244,8 @@ def app(): "low_energy_lighting", "suspended_floor_insulation", "internal_wall_insulation", - "room_roof_insulation" + "room_roof_insulation", + "windows" ], "budget": None, "scenario_name": "Do when void", diff --git a/recommendations/Costs.py b/recommendations/Costs.py index 8deed75a..908a409a 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -168,9 +168,8 @@ class Costs: # https://www.greenmatch.co.uk/windows/double-glazing/cost SASH_WINDOW_INFLATION_FACTOR = 1.5 - # Typically, secondary glazing can be installed for 25% of the cost of double glazed windows - to be conservative, - # we scale the cost by half - SECONDARY_GLAZING_SCALING_FACTOR = 0.5 + # Based on relative costs from SCIS + SECONDARY_GLAZING_SCALING_FACTOR = 0.85 def __init__(self, property_instance): """ diff --git a/recommendations/Recommendations.py b/recommendations/Recommendations.py index db9a6320..d2c1db1b 100644 --- a/recommendations/Recommendations.py +++ b/recommendations/Recommendations.py @@ -102,6 +102,7 @@ class Recommendations: property_recommendations = [] phase = 0 measures = self.find_included_measures() + non_invasive_recommendation_types = [r["type"] for r in self.property_instance.non_invasive_recommendations] # Building Fabric self.wall_recomender.recommend(phase=phase, measures=measures) @@ -149,7 +150,8 @@ class Recommendations: property_recommendations.append(self.floor_recommender.recommendations) phase += 1 - if "windows" in measures: + if "windows" in measures and "mixed_glazing" not in non_invasive_recommendation_types: + # If we have a mixed glazing recommendation, we prioritise this over the windows recommendation self.windows_recommender.recommend(phase=phase) if self.windows_recommender.recommendation: property_recommendations.append(self.windows_recommender.recommendation) @@ -529,7 +531,7 @@ class Recommendations: # For the moment, we cap the number of SAP points that can be achieved by LEDs at 2 if rec["type"] == "low_energy_lighting": - + if property_instance.data["low-energy-lighting"] < 50: lighting_sap_limit = LightingRecommendations.SAP_LIMIT else: From ceb34979e4f82a4f77699c4592e36506d5bd4dfb Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 12 Sep 2024 18:10:27 +0100 Subject: [PATCH 003/166] rough analysis for funding eligibility --- .../Cleethorpes Portfolio/epc data.py | 97 +++++++ etl/customers/bcc_tender/app.py | 10 +- etl/customers/newhaven/slides.py | 239 ++++++++++++++++++ etl/sfr/midlands_portfolio_est_funding.py | 159 ++++++++++++ 4 files changed, 504 insertions(+), 1 deletion(-) create mode 100644 etl/customers/Cleethorpes Portfolio/epc data.py create mode 100644 etl/sfr/midlands_portfolio_est_funding.py diff --git a/etl/customers/Cleethorpes Portfolio/epc data.py b/etl/customers/Cleethorpes Portfolio/epc data.py new file mode 100644 index 00000000..a3ccbb2a --- /dev/null +++ b/etl/customers/Cleethorpes Portfolio/epc data.py @@ -0,0 +1,97 @@ +import os +import pandas as pd +from backend.SearchEpc import SearchEpc +from dotenv import load_dotenv +from tqdm import tqdm + +load_dotenv(dotenv_path="backend/.env") +EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN") + + +def app(): + """ + Simple script to pull the EPC data for the Cleethorpes Portfolio + :return: + """ + + asset_list = pd.read_excel( + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/sfr/Cleethorpes Portoflio/Updated Tenancy Schedule " + "Portfolio.xlsx", + ) + asset_list["row_id"] = asset_list.index + asset_list[" Street No."] = asset_list[" Street No."].astype(str) + + epc_data = [] + for _, property in tqdm(asset_list.iterrows(), total=len(asset_list)): + + if property[" Street No."] == "Ground Floor Commercial": + continue + uprn = property["Uprn"] + if not pd.isnull(uprn): + searcher = SearchEpc( + address1="", + postcode="", + auth_token=EPC_AUTH_TOKEN, + os_api_key="", + uprn=int(uprn) + ) + searcher.find_property(skip_os=True) + else: + + if not pd.isnull(property[" Flat No."]) and property[" Flat No."] not in ["", " "]: + address1 = property[" Flat No."].strip() + ", " + property[" Street No."].strip() + else: + address1 = property[" Street No."].strip() + + if address1 == "1a Mews House 30": + address1 = "1a Rear of" + searcher = SearchEpc( + address1=address1, + postcode=property[" Postcode"].strip(), + auth_token=EPC_AUTH_TOKEN, + os_api_key="", + uprn=None, + ) + searcher.get_epc() + # Get the newest record on lodgement-date + sorted_epcs = sorted( + searcher.data["rows"], key=lambda x: x["lodgement-date"] + ) + searcher.newest_epc = sorted_epcs[-1] + + if searcher.newest_epc is None: + raise ValueError(f"No EPC found for UPRN: {uprn}") + + epc_data.append( + { + "row_id": property["row_id"], + **searcher.newest_epc + } + ) + + epc_df = pd.DataFrame(epc_data) + + # Merge on data + asset_list_with_epc = asset_list.merge( + epc_df[["row_id", "address", "current-energy-rating", "current-energy-efficiency", "lodgement-date"]], + how="left", + left_on="row_id", + right_on="row_id", + ).rename( + columns={ + "address": "EPC Address", + "current-energy-rating": "Current EPC Rating", + "current-energy-efficiency": "Current SAP Score", + "lodgement-date": "EPC Date" + } + ) + + asset_list_with_epc.to_excel( + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/sfr/Cleethorpes Portoflio/Portfolio with EPCs.xlsx", + index=False + ) + + epc_df.to_csv( + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/sfr/Cleethorpes Portoflio/epc_data.csv", + index=False + ) diff --git a/etl/customers/bcc_tender/app.py b/etl/customers/bcc_tender/app.py index 8cdc6e13..898db949 100644 --- a/etl/customers/bcc_tender/app.py +++ b/etl/customers/bcc_tender/app.py @@ -102,7 +102,7 @@ analysis_epcs = analysis_epcs[ [ "UPRN", "TENURE", "CURRENT_ENERGY_RATING", "WALLS_DESCRIPTION", "ROOF_DESCRIPTION", "CONSTRUCTION_AGE_BAND", "TOTAL_FLOOR_AREA", "PROPERTY_TYPE", "BUILT_FORM", "MAINHEAT_DESCRIPTION", - "eligibility_type", + "eligibility_type", "PHOTO_SUPPLY", "ADDRESS1", "POSTCODE" ] ] analysis_epcs["grouped_epc_band"] = np.where( @@ -110,6 +110,14 @@ analysis_epcs["grouped_epc_band"] = np.where( "EPC D", "EPC E-G" ) + +analysis_epcs[pd.isnull(analysis_epcs["PHOTO_SUPPLY"])][["ADDRESS1", "POSTCODE"]].sample(1) + +analysis_epcs["PHOTO_SUPPLY"] = analysis_epcs["PHOTO_SUPPLY"].fillna(0) +analysis_epcs["PHOTO_SUPPLY"] = analysis_epcs["PHOTO_SUPPLY"].astype(float) +analysis_epcs["has_solar"] = np.where(analysis_epcs["PHOTO_SUPPLY"] > 0, 1, 0) +analysis_epcs["has_solar"].value_counts() + analysis_epcs.to_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/bcc tender/analysis_epcs.csv", index=False) # Create aggregations and we store this information diff --git a/etl/customers/newhaven/slides.py b/etl/customers/newhaven/slides.py index 2fe914e2..61ed89cc 100644 --- a/etl/customers/newhaven/slides.py +++ b/etl/customers/newhaven/slides.py @@ -5,6 +5,7 @@ from sqlalchemy.orm import sessionmaker from backend.app.db.connection import db_engine from backend.app.db.models.recommendations import Recommendation, Plan, PlanRecommendations, Scenario from backend.app.db.models.portfolio import PropertyModel, PropertyDetailsEpcModel +from utils.s3 import read_csv_from_s3 def get_data(portfolio_id, scenario_ids): @@ -415,3 +416,241 @@ def slides(): pd.set_option('display.max_rows', None) # Show more characters in a column pd.set_option('display.max_colwidth', None) + + # preparing of this data for the following 2 needs: + # 1) dataset to share with Nextgen heating + # 2) Breakdown of results by property type + + # get the asset list + asset_list = read_csv_from_s3(bucket_name="retrofit-plan-inputs-dev", filepath="8/90/pilot.csv") + asset_list = pd.DataFrame(asset_list) + # Get non-invasive recommendations + non_intrusive_recommendations = read_csv_from_s3( + bucket_name="retrofit-plan-inputs-dev", + filepath="8/90/non_invasive_recommendations.csv" + ) + non_intrusive_recommendations = pd.DataFrame(non_intrusive_recommendations) + + # Unnest this + import ast + survey_recs = [] + for _, row in non_intrusive_recommendations.iterrows(): + recs = ast.literal_eval(row["recommendations"]) + ashp_rec = next((r for r in recs if r["type"] == "air_source_heat_pump"), None) + solar_rec = next((r for r in recs if r["type"] == "solar_pv"), None) + to_append = { + "uprn": row["uprn"] + } + if ashp_rec["suitable"]: + to_append = { + **to_append, + "ashp_suitable": True, + "ashp_size_kw": ashp_rec["size"], + "ashp_cost": ashp_rec["cost"], + } + + if solar_rec["suitable"]: + to_append = { + **to_append, + "solar_suitable": True, + "solar_size_kwp": solar_rec["array_wattage"], + "solar_cost": solar_rec["cost"], + } + survey_recs.append(to_append) + survey_recs = pd.DataFrame(survey_recs) + + asset_list["uprn"] = asset_list["uprn"].astype(int) + survey_recs["uprn"] = survey_recs["uprn"].astype(int) + + vital_kwh = 7597 + domna_kwh = 10850 + scaling_factor = vital_kwh / domna_kwh + + next_gen_dataset = properties_df[[ + "uprn", "address", "postcode", + "property_type", "built_form", "current_energy_demand_heating_hotwater", + "mainfuel", "total_floor_area", "floor_height" + ]].rename( + columns={ + "mainfuel": "primary_fuel_type", + "total_floor_area": "gross_floor_area", + "current_energy_demand_heating_hotwater": "estimated_heating_hotwater_kwh" + } + ).merge( + asset_list[["uprn", "number_of_floors"]], + how="left", + on="uprn" + ).merge( + survey_recs, + how="left", + on="uprn" + ) + next_gen_dataset["estimated_heating_hotwater_kwh_scaled"] = ( + next_gen_dataset["estimated_heating_hotwater_kwh"] * scaling_factor + ) + + next_gen_dataset["ashp_suitable"] = next_gen_dataset["ashp_suitable"].fillna(False) + next_gen_dataset["solar_suitable"] = next_gen_dataset["solar_suitable"].fillna(False) + + # We prepare the scenario outputs by property type + grouped_data = next_gen_dataset.copy() + grouped_data["property_sub_type"] = grouped_data["built_form"].copy() + # If a property is a flat, re-map sub_type just to flat + grouped_data.loc[grouped_data["property_type"] == "Flat", "property_sub_type"] = "Flat" + # Same for maisonettes + grouped_data.loc[grouped_data["property_type"] == "Maisonette", "property_sub_type"] = "Maisonette" + + # We now pull out the recommendations impact by property type and sub type + + property_scenario_impact = [] + for scenario_id in scenario_ids: + # Get the recommendations for the scenario, default + scenario_recommendations = recommendations_df[ + (recommendations_df["Scenario ID"] == scenario_id) & + (recommendations_df["default"] == True) + ].copy() + + scenario_recommendations['ligting_kwh'] = scenario_recommendations.apply( + lambda x: x['kwh_savings'] if x['type'] == 'low_energy_lighting' else 0, + axis=1) + scenario_recommendations['solar_kwh'] = scenario_recommendations.apply( + lambda x: x['kwh_savings'] if x['type'] == 'solar_pv' else 0, axis=1) + + # Set 'Estimated Kwh Savings' to zero where specific kwh columns are used + scenario_recommendations['Estimated Kwh Savings'] = scenario_recommendations.apply( + lambda x: 0 if x['type'] in ['low_energy_lighting', 'solar_pv'] else x[ + 'kwh_savings'], axis=1) + + scenario_grouped_data = scenario_recommendations.groupby(['property_id']).agg({ + 'Estimated Kwh Savings': 'sum', + "estimated_cost": "sum" + }).reset_index() + + comparison = properties_df.drop_duplicates()[ + ["uprn", "property_id", "current_energy_demand_heating_hotwater"] + ].merge( + scenario_grouped_data, on=["property_id"], how="left" + ) + comparison["Estimated Kwh Savings"] = comparison["Estimated Kwh Savings"].fillna(0) + comparison["estimated_cost"] = comparison["estimated_cost"].fillna(0) + + comparison["post_scenario_heating_hotwater_kwh"] = ( + comparison["current_energy_demand_heating_hotwater"] - comparison["Estimated Kwh Savings"] + ) + comparison["scenario_id"] = scenario_id + + property_scenario_impact.append(comparison) + + property_scenario_impact = pd.concat(property_scenario_impact) + property_scenario_impact = property_scenario_impact.drop(columns=["property_id", "Estimated Kwh Savings"]) + + # Scale + property_scenario_impact["post_scenario_heating_hotwater_kwh_scaled"] = ( + property_scenario_impact["post_scenario_heating_hotwater_kwh"] * scaling_factor + ) + + grouped_data = grouped_data.merge( + property_scenario_impact, how="left", on="uprn" + ) + + # Agg the data + grouped_data = grouped_data.groupby(["property_type", "property_sub_type", "scenario_id"]).agg({ + "estimated_heating_hotwater_kwh": "mean", + "estimated_heating_hotwater_kwh_scaled": "mean", + "estimated_cost": "mean", + "post_scenario_heating_hotwater_kwh": "mean", + "post_scenario_heating_hotwater_kwh_scaled": "mean" + }).reset_index() + + scenario_names = pd.DataFrame( + [ + { + "scenario_id": 47, + "scenario": "Demand Reduction – cavity & roof insulation", + }, + { + "scenario_id": 48, + "scenario": "Demand reduction – no solid wall, floors or heating/renewables", + }, + { + "scenario_id": 49, + "scenario": "Demand reduction – no decant" + }, + { + "scenario_id": 50, + "scenario": "Demand reduction – no decant + heating & solar", + }, + { + "scenario_id": 51, + "scenario": "Whole house retrofit" + } + ] + + ) + + grouped_data = grouped_data.merge( + scenario_names, how="left", on="scenario_id" + ) + + if not grouped_data[ + grouped_data["estimated_heating_hotwater_kwh"] < grouped_data["post_scenario_heating_hotwater_kwh"]].empty: + raise Exception("someting went wrong") + + if not grouped_data[grouped_data["estimated_heating_hotwater_kwh_scaled"] < grouped_data[ + "post_scenario_heating_hotwater_kwh_scaled"]].empty: + raise Exception("someting went wrong") + + # Reorder the columns + grouped_data = grouped_data[ + [ + 'property_type', + 'property_sub_type', + 'scenario', + 'estimated_heating_hotwater_kwh', + 'post_scenario_heating_hotwater_kwh', + 'estimated_heating_hotwater_kwh_scaled', + 'post_scenario_heating_hotwater_kwh_scaled', + 'estimated_cost', + ] + ] + + grouped_data = grouped_data.rename( + columns={ + "property_type": "Property Type", + "property_sub_type": "Property Sub Type", + "scenario": "Scenario", + "estimated_heating_hotwater_kwh": "Estimated Heating & Hot Water kwh", + "post_scenario_heating_hotwater_kwh": "Post Scenario Heating & Hot Water kwh", + "estimated_heating_hotwater_kwh_scaled": "Estimated Heating & Hot Water kwh (scaled)", + "post_scenario_heating_hotwater_kwh_scaled": "Post Scenario Heating & Hot Water kwh (scaled)", + "estimated_cost": "Estimated Cost or Retrofit", + } + ) + + grouped_data.to_excel( + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Newhaven/outputs/Scenario kWh Impact by Property " + "Type.xlsx", + index=False + ) + + property_scenario_impact = property_scenario_impact.merge( + scenario_names, how="left", on="scenario_id" + ) + + df_pivot = property_scenario_impact.pivot_table(index='uprn', columns='scenario', + values=['post_scenario_heating_hotwater_kwh', + 'post_scenario_heating_hotwater_kwh_scaled']) + + # Flattening multi-index columns + df_pivot.columns = [f'{col[0]}_{col[1]}' for col in df_pivot.columns] + + # Reset the index to have a clean dataframe + df_pivot.reset_index(inplace=True) + + next_gen_dataset = next_gen_dataset.merge( + df_pivot, how="left", on="uprn" + ) + + next_gen_dataset.to_csv( + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Newhaven/outputs/next_gen_dataset.csv", index=False + ) diff --git a/etl/sfr/midlands_portfolio_est_funding.py b/etl/sfr/midlands_portfolio_est_funding.py new file mode 100644 index 00000000..09102cfb --- /dev/null +++ b/etl/sfr/midlands_portfolio_est_funding.py @@ -0,0 +1,159 @@ +import msgpack + +import pandas as pd +from utils.s3 import read_from_s3 +from recommendations.recommendation_utils import ( + estimate_number_of_floors, esimtate_pitched_roof_area, estimate_external_wall_area, estimate_perimeter +) + + +def app(): + """ + Aims to estimate the amount of GBIS funding eligible + :return: + """ + + cleaned = read_from_s3( + s3_file_name="cleaned_epc_data/cleaned.bson", + bucket_name="retrofit-data-dev" + ) + + cleaned = msgpack.unpackb(cleaned, raw=False) + + epc_data = pd.read_excel( + "/Users/khalimconn-kowlessar/Downloads/20240820 portfolio_epc_data.xlsx" + ) + + # For simplicity, get roofs or cavities + epc_data = epc_data.merge( + pd.DataFrame(cleaned["roof-description"]), + how="left", + left_on="ROOF_DESCRIPTION", + right_on="original_description" + ) + + epc_data["needs_roof_work"] = epc_data["insulation_thickness"].isin( + [ + None, + "100", + '150', + '50', + '75', + 'below average', + '25', + '12' + ] + ) & (epc_data["is_flat"] | epc_data["is_pitched"]) + + epc_data = epc_data.merge( + pd.DataFrame(cleaned["walls-description"]), + how="left", + left_on="WALLS_DESCRIPTION", + right_on="original_description", + suffixes=("", "_wall") + ) + + epc_data["needs_cavity_done"] = epc_data["is_cavity_wall"] & epc_data["insulation_thickness_wall"].isin( + ['none', "below average"] + ) + + loft_insulation_per_m2 = 16.07 + flat_roof_insulation_per_m2 = 195 + cwi_per_m2 = 14.21 + gbis_abs = 30 + + # We assume the work will take the home from a high D to a low D + def get_abs(floor_area): + if floor_area <= 72: + return 155 + + if floor_area <= 97: + return 169 + + if floor_area <= 199: + return 196.4 + + return 350.1 + + estimated_costs = [] + for _, home in epc_data.iterrows(): + to_append = { + "uprn": home["UPRN"], + "address": home["ADDRESS"], + "postcode": home["POSTCODE"], + } + + project_abs = get_abs(home["TOTAL_FLOOR_AREA"]) + available_funding = project_abs * gbis_abs + + n_floors = estimate_number_of_floors(home["PROPERTY_TYPE"]) + floor_height = float(home["FLOOR_HEIGHT"]) if not pd.isnull(home["FLOOR_HEIGHT"]) else 2.5 + + # Check if it needs the walls done + if home["needs_cavity_done"]: + # We estimate the amount of insulation required + est_perimeter = estimate_perimeter( + floor_area=float(home["TOTAL_FLOOR_AREA"]) / n_floors, + num_rooms=float(home["NUMBER_HABITABLE_ROOMS"]) / n_floors + ) + + insulation_needed = estimate_external_wall_area( + num_floors=n_floors, + floor_height=floor_height, + perimeter=est_perimeter, + built_form=home["BUILT_FORM"], + ) + cost_of_insulation = insulation_needed * cwi_per_m2 + + if available_funding > cost_of_insulation: + available_funding = cost_of_insulation + + to_append = { + **to_append, + "available_funding": available_funding, + "measure": "Cavity Wall Insulation", + "project_abs": project_abs + } + + estimated_costs.append(to_append) + continue + + if home["needs_roof_work"]: + # We estimate how much the cost of insulation would be + if home["is_pitched"]: + measure = "Loft Insulation" + + roof_area = float(home["TOTAL_FLOOR_AREA"]) / n_floors + cost_of_insulation = roof_area * loft_insulation_per_m2 + else: + measure = "Flat Roof Insulation" + roof_area = float(home["TOTAL_FLOOR_AREA"]) / n_floors + cost_of_insulation = roof_area * flat_roof_insulation_per_m2 + + if available_funding > cost_of_insulation: + available_funding = cost_of_insulation + + to_append = { + **to_append, + "available_funding": available_funding, + "measure": measure, + "project_abs": project_abs + } + + estimated_costs.append(to_append) + continue + + estimated_costs = pd.DataFrame(estimated_costs) + + estimated_costs.groupby("measure")["available_funding"].mean() + estimated_costs["measure"].value_counts() + + estimated_costs.to_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/sfr/estimated_costs_gbis.csv") + + epc_data[["UPRN", "ADDRESS", "POSTCODE"]].to_csv( + "/Users/khalimconn-kowlessar/Documents/hestia/sfr/council_tax_bands_sample.csv") + + n_properties_for_ashp = epc_data[ + (epc_data["PROPERTY_TYPE"] == "House") & + (epc_data["BUILT_FORM"].isin(["Detached", "Semi-Detached"])) + ].shape[0] From 15f55c021f694492a925204f54ca975bd52b0702 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 13 Sep 2024 15:31:43 +0100 Subject: [PATCH 004/166] AIHA data review WIP --- backend/SearchEpc.py | 92 ++++++- etl/customers/aiha/epc_data_pull.py | 363 ++++++++++++++++++++++++++++ 2 files changed, 454 insertions(+), 1 deletion(-) create mode 100644 etl/customers/aiha/epc_data_pull.py diff --git a/backend/SearchEpc.py b/backend/SearchEpc.py index 5f101d81..b5ec8c46 100644 --- a/backend/SearchEpc.py +++ b/backend/SearchEpc.py @@ -7,6 +7,9 @@ import pandas as pd import numpy as np from epc_api.client import EpcClient from backend.OrdnanceSurvey import OrdnanceSuveyClient +from etl.epc_clean.epc_attributes.WallAttributes import WallAttributes +from etl.epc_clean.epc_attributes.FloorAttributes import FloorAttributes +from etl.epc_clean.epc_attributes.RoofAttributes import RoofAttributes from BaseUtility import Definitions from utils.logger import setup_logger from typing import List @@ -181,6 +184,7 @@ class SearchEpc: self.newest_epc = None self.older_epcs = None self.full_sap_epc = None + self.metadata = None # These are the address and postcode values, which we store in the database self.address_clean = None @@ -306,7 +310,10 @@ class SearchEpc: if (property_type is None) and (address is None): return rows - if len(uprns) == 1: + unique_property_types = {r["property-type"] for r in rows} + + # We allow for variation in property type across flats/maisonettes + if (len(uprns) == 1) and ((len(unique_property_types) == 1) or unique_property_types == {"Flat", "Maisonette"}): return rows if property_type is not None: @@ -784,3 +791,86 @@ class SearchEpc: self.address_clean = self.ordnance_survey_client.address_os self.postcode_clean = self.ordnance_survey_client.postcode_os return + + def check_attribute_variations(self): + attribute_map = { + "walls-description": { + "cleaner": WallAttributes, + "attribute": [ + "is_cavity_wall", "is_solid_brick", "is_system_built", "is_timber_frame", + "is_granite_or_whinstone", "is_cob", "is_sandstone_or_limestone", "is_park_home" + ], + "name": "has_wall_type_ever_varied" + }, + "roof-description": { + "cleaner": RoofAttributes, + "attribute": [ + "is_flat", "is_pitched", "is_roof_room", "is_thatched", "has_dwelling_above" + ], + "name": "has_roof_type_ever_varied" + }, + "floor-description": { + "cleaner": FloorAttributes, + "attribute": [ + "is_to_unheated_space", "is_to_external_air", "is_suspended", "is_solid", "is_to_external_air", + ], + "name": "has_floor_type_ever_varied" + } + } + + attribute_variations = {} + for attribute, attribute_objs in attribute_map.items(): + attribute_variations[attribute_objs["name"]] = False + cleaner = attribute_objs["cleaner"] + type_timeline = pd.DataFrame([cleaner(epc[attribute]).process() for epc in self.older_epcs] + [ + cleaner(self.newest_epc[attribute]).process() + ]) + # For eac col in attribute_objs["attribute"] we check if the timeline has ever varied, i.e has gone + # from true to false + for col in attribute_objs["attribute"]: + if type_timeline[col].nunique() > 1: + attribute_variations[attribute_objs["name"]] = True + break + + return attribute_variations + + def identify_flat_floor(self): + # If there is no dwelling above, it is a top floor flat + processed_roof = RoofAttributes(self.newest_epc["roof-description"]).process() + if not processed_roof["has_dwelling_above"]: + return "top" + + # We know that there is a dwelling above. If there's also a drwelling below, it is a mid floor flat + processed_floor = FloorAttributes(self.newest_epc["floor-description"]).process() + if processed_floor["another_property_below"]: + return "mid" + + # Otherwise ground floor + return "ground" + + def get_metadata(self): + if self.newest_epc is None: + raise ValueError("No EPC data available") + + # We check if the property has ever been downgraded on SAP + has_sap_ever_downgraded = False + sap_timeline = [int(epc["current-energy-efficiency"]) for epc in self.older_epcs] + [ + int(self.newest_epc["current-energy-efficiency"]) + ] + # We check if there has ever been a decrease by differencing + has_sap_ever_downgraded = any(np.diff(sap_timeline) < 0) + + # We check if the wall type has ever varied over time + attribute_varations = self.check_attribute_variations() + + # If the property is a flat, we distinguish between top, mid, ground floor + floor = None + if self.newest_epc["property-type"] == "Flat": + floor = self.identify_flat_floor() + + self.metadata = { + "days_since_last_epc": (pd.Timestamp.now() - pd.Timestamp(self.newest_epc["lodgement-date"])).days, + "has_sap_ever_downgraded": has_sap_ever_downgraded, + "floor": floor, + **attribute_varations + } diff --git a/etl/customers/aiha/epc_data_pull.py b/etl/customers/aiha/epc_data_pull.py new file mode 100644 index 00000000..8aaaf5ba --- /dev/null +++ b/etl/customers/aiha/epc_data_pull.py @@ -0,0 +1,363 @@ +import os +from tqdm import tqdm +from dotenv import load_dotenv +import pandas as pd +from backend.SearchEpc import SearchEpc +from etl.spatial.OpenUprnClient import OpenUprnClient + +load_dotenv(dotenv_path="backend/.env") +EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN") + +pd.set_option('display.max_rows', 500) +pd.set_option('display.max_columns', 500) +pd.set_option('display.width', 1000) + + +def app(): + # Retrieve EPC data for the SHDF AIHA portfolio + + data = pd.read_excel( + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/AIHA/Khalim Review - 240902 - KSQ - AIHA - SHDF Wave " + "3 bid - Supplementary information.xlsx", + sheet_name="All units information", + header=3 + ) + + # Remove the .eg row + data = data.tail(-1) + + # Remove the bottom 2 rows + data = data.head(-2) + data = data.reset_index(drop=True) + data["row_id"] = data.index + + ammendments = { + "12 11-18 Schonfeld Square": "12 Schonfeld Square", + "35 35-37 Schonfeld Square": "35 Schonfeld Square", + '77 Schonfeld Square': '77 Lordship Road', + "83 Lordship Road (Schonfeld Square)": "83 Lordship Road", + "A 80 Bethune Road": "80A Bethune Road", + "86B Bethune Road": "Flat B, 86 Bethune Road", + "22 Glendale Road": "22 Glendale Avenue", + "121 Southbourne Road": "121 Southbourne Grove", + } + + no_epc = [ + "80B Bethune Road", + "89B Manor Road", + "12 Monkville Avenue", + "9 Greenview", + ] + + property_type_map = { + "House, mid-terrace": "House", + "House, end terrace": "House", + "House, semi-detached": "House", + "House, detached": "House", + "Flat": "Flat", + } + + epc_data = [] + epc_metadata = [] + for _, home in tqdm(data.iterrows(), total=len(data)): + + # Build address 1 based on if there is: + # 1) Address letter or number + # 2) Street address + + modified = False + address1 = "" + address1_backup = "" + + if home["Address letter or number"] in ["A", "B", "C"]: + + house_no = home['Street address'].split(' ')[0] + street = ' '.join(home['Street address'].split(' ')[1:]) + address1 = f"{house_no}{home['Address letter or number']} {street}" + + address1_backup = f"Flat {home['Address letter or number']} {house_no} {street}" + modified = True + + else: + if not pd.isnull(home["Address letter or number"]): + address1 += f"{home['Address letter or number']} " + if not pd.isnull(home["Street address"]): + address1 += f"{home['Street address']}" + address1 = address1.strip() + + if address1.split(" ")[-1].lower() == "rd": + # Replace with road + address1 = address1.lower().replace(" rd", " road") + + # Specific ammendments + if address1 in ammendments: + address1 = ammendments[address1] + + if address1 in no_epc: + continue + + searcher = SearchEpc( + address1=address1, + postcode=home["Postcode"], + auth_token=EPC_AUTH_TOKEN, + os_api_key="", + property_type=property_type_map[home["Property type"]] + ) + searcher.find_property(skip_os=True) + + if searcher.newest_epc is None and modified: + searcher = SearchEpc( + address1=address1_backup, + postcode=home["Postcode"], + auth_token=EPC_AUTH_TOKEN, + os_api_key="", + property_type=property_type_map[home["Property type"]] + ) + searcher.find_property(skip_os=True) + + if searcher.newest_epc is None: + raise Exception("Not found") + + epc_data.append( + { + "row_id": home["row_id"], + **searcher.newest_epc + } + ) + + searcher.get_metadata() + + epc_metadata.append( + { + "row_id": home["row_id"], + "address": address1, + "postcode": home["Postcode"], + **searcher.metadata + } + ) + + epc_metadata = pd.DataFrame(epc_metadata) + epc_data = pd.DataFrame(epc_data) + + # Check matched addresses + matched_addresses = epc_metadata[["row_id", "address", "postcode"]].copy() + matched_addresses = matched_addresses.merge( + data[["row_id", "Address letter or number", "Street address"]], on="row_id", how="inner" + ) + + # We look for differences between the asset list and the EPC data + comparison_cols = { + "Property type": [ + { + "epc_col": "property-type", + "map": property_type_map + }, + { + "epc_col": "built-form", + "map": { + "House, mid-terrace": "Mid-Terrace", + "House, end terrace": "End-Terrace", + "House, semi-detached": "Semi-Detached", + "House, detached": "Detached", + "Flat": "Flat", + } + } + ], + "Energy starting band (EPC)": [ + { + "epc_col": "current-energy-rating", + "map": {} + } + ], + "Wall type": [ + { + "epc_col": "walls-description", + "search_terms": { + "solid": "Solid brick", + "cavity": "Cavity wall", + "solid - internal lining": "Solid brick", + } + } + ], + "Roof type": [ + { + "epc_col": "roof-description", + "search_terms": { + "pitched": "Pitched", + "n/a - (flat above)": "another dwelling above" + } + } + ], + "Floor type": [ + { + "epc_col": "floor-description", + "search_terms": { + "solid": "Solid", + "suspended": "Suspended", + "solid - floating floor for services": "Solid" + } + } + ], + } + + import re + differences = [] + for asset_list_col, list_of_configs in comparison_cols.items(): + + if asset_list_col in ["Wall type", "Roof type", "Floor type"]: + config = list_of_configs[0] + # We handle this differently + remapped = data[["row_id", asset_list_col]].copy() + # Strip the asset list col incase of leading/trailing spaces + remapped[asset_list_col] = remapped[asset_list_col].str.strip() + remapped[asset_list_col] = remapped[asset_list_col].str.lower() + remapped = remapped.merge(epc_data[["row_id", config["epc_col"]]], on="row_id", how="inner") + # We do a search term check + remapped["Match"] = None + for search_term, epc_term in config["search_terms"].items(): + if "/" in search_term: + escaped_search_term = re.escape(search_term) + remapped.loc[remapped[asset_list_col].str.contains(escaped_search_term), "Match"] = ( + remapped.loc[ + remapped[asset_list_col].str.contains(escaped_search_term), config["epc_col"] + ].str.contains(epc_term) + ) + else: + remapped.loc[remapped[asset_list_col].str.contains(search_term), "Match"] = ( + remapped.loc[ + remapped[asset_list_col].str.contains(search_term), config["epc_col"] + ].str.contains(epc_term) + ) + + if pd.isnull(remapped["Match"]).sum(): + raise Exception("Not all matched") + + remapped["Match"] = remapped["Match"].astype(bool) + + if not all(remapped["Match"]): + differences.append( + { + "Column": asset_list_col, + "Differences": remapped[~remapped["Match"]], + } + ) + + continue + + for config in list_of_configs: + + remapped = data[["row_id", asset_list_col]].copy() + if config["map"]: + remapped[asset_list_col] = remapped[asset_list_col].map(config["map"]) + + # Merge on + remapped = remapped.merge(epc_data[["row_id", config["epc_col"]]], on="row_id", how="inner") + remapped["Match"] = remapped[asset_list_col] == remapped[config["epc_col"]] + if not all(remapped["Match"]): + differences.append( + { + "Column": asset_list_col, + "Differences": remapped[~remapped["Match"]], + } + ) + + # Check for property type + property_type_differences = differences[0]["Differences"].copy() + property_type_differences = property_type_differences.merge( + data[["row_id", "Address letter or number", "Street address"]], on="row_id", how="inner" + ) + print(property_type_differences) + + # Check for built form + built_form_differences = differences[1]["Differences"].copy() + built_form_differences = built_form_differences[built_form_differences["Property type"] != "Flat"] + built_form_differences = built_form_differences.merge( + data[["row_id", "Address letter or number", "Street address"]], on="row_id", how="inner" + ) + print(built_form_differences) + + # Check for energy rating + energy_rating_differences = differences[2]["Differences"].copy() + energy_rating_differences = energy_rating_differences.merge( + data[["row_id", "Address letter or number", "Street address"]], on="row_id", how="inner" + ).merge( + epc_data[["row_id", "uprn"]], on="row_id", how="inner" + ) + print(energy_rating_differences) + + # Check for wall type + wall_type_differences = differences[3]["Differences"].copy() + wall_type_differences = wall_type_differences.merge( + data[["row_id", "Address letter or number", "Street address"]], on="row_id", how="inner" + ).merge( + epc_data[["row_id", "uprn"]], on="row_id", how="inner" + ) + print(wall_type_differences) # Many wall type differences + + # Check for roof type + roof_type_differences = differences[4]["Differences"].copy() + roof_type_differences = roof_type_differences.merge( + data[["row_id", "Address letter or number", "Street address"]], on="row_id", how="inner" + ).merge( + epc_data[["row_id", "uprn"]], on="row_id", how="inner" + ) + print(roof_type_differences) # Many roof type differences + + # Check for floor type + floor_type_differences = differences[5]["Differences"].copy() + floor_type_differences = floor_type_differences.merge( + data[["row_id", "Address letter or number", "Street address"]], on="row_id", how="inner" + ).merge( + epc_data[["row_id", "uprn"]], on="row_id", how="inner" + ) + print(floor_type_differences) # Many floor type differences + + # TODO: 47 Ashtead Road [100021024699] shows solid brick wall on EPC - is probably cavity wall + + # We have the EPC data. Let's check conservation area/historic/listed building status + portfolio_spatial_data = OpenUprnClient.get_spatial_data( + epc_data["uprn"].unique().tolist(), bucket_name="retrofit-data-dev" + ) + + portfolio_spatial_data["UPRN"] = portfolio_spatial_data["UPRN"].astype(str) + + spatial_data = data[["row_id", "Planning constraints"]].merge( + epc_data[["row_id", "uprn"]], on="row_id", how="left", + + ).merge( + portfolio_spatial_data[["UPRN", "conservation_status", "is_listed_building", "is_heritage_building"]], + left_on="uprn", + right_on="UPRN", how="left" + ) + + spatial_data[ + (spatial_data["Planning constraints"] == "None") + ]["conservation_status"].value_counts() + + # One property is in a conservation area, that was not picked up in the asset data + print(spatial_data[ + (spatial_data["Planning constraints"] == "None") & + (spatial_data["conservation_status"] == True) + ].merge( + data[["row_id", "Address letter or number", "Street address", "Postcode"]], on="row_id", how="left" + )) + + # All properties match up apart from one where the asset data indicates it's in a conservation area, however + # the sparital data indicates it's not. There do not appear to be any listed/heritage buildings in the portfolio + + # Draft archetyping + archetyping_data = data[ + [ + "row_id", + "Energy starting band (EPC)", + "Property type", + "Property year built", + "Gross internal area (sqm)", + "Current heating system type", + "Wall type", + "Floor type", + "Roof type", + "Window type", + "Location (Floor)", + ] + ] From 391c6f5cf0e07eca36e8d2ecf8c075e475df95b5 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 13 Sep 2024 18:05:02 +0100 Subject: [PATCH 005/166] Adding to archetyping --- etl/customers/aiha/epc_data_pull.py | 408 ++++++++++++++++++++++++++++ 1 file changed, 408 insertions(+) diff --git a/etl/customers/aiha/epc_data_pull.py b/etl/customers/aiha/epc_data_pull.py index 8aaaf5ba..5e7c6714 100644 --- a/etl/customers/aiha/epc_data_pull.py +++ b/etl/customers/aiha/epc_data_pull.py @@ -2,6 +2,9 @@ import os from tqdm import tqdm from dotenv import load_dotenv import pandas as pd +import numpy as np +import msgpack +from utils.s3 import read_from_s3 from backend.SearchEpc import SearchEpc from etl.spatial.OpenUprnClient import OpenUprnClient @@ -345,7 +348,63 @@ def app(): # All properties match up apart from one where the asset data indicates it's in a conservation area, however # the sparital data indicates it's not. There do not appear to be any listed/heritage buildings in the portfolio + ################################################################ # Draft archetyping + ################################################################ + + cleaned = read_from_s3( + s3_file_name="cleaned_epc_data/cleaned.bson", + bucket_name="retrofit-data-dev" + ) + cleaned = msgpack.unpackb(cleaned, raw=False) + + epc_data = epc_data.merge( + pd.DataFrame(cleaned["walls-description"])[ + ['original_description', + 'is_cavity_wall', 'is_filled_cavity', 'is_solid_brick', 'is_system_built', 'is_timber_frame', + 'is_as_built', 'is_assumed', 'insulation_thickness'] + + ].rename( + columns={ + "is_solid_brick": "is_solid_brick_wall", + "is_system_built": "is_system_built_wall", + "is_timber_frame": "is_timber_frame_wall", + "is_assumed": "is_assumed_wall", + "insulation_thickness": "insulation_thickness_wall" + } + ), + left_on="walls-description", + right_on="original_description" + ).merge( + pd.DataFrame(cleaned["roof-description"])[ + [ + 'original_description', 'is_pitched', 'is_roof_room', 'is_loft', + 'is_flat', 'is_thatched', 'is_at_rafters', 'is_assumed', + 'has_dwelling_above', 'insulation_thickness' + ] + ].rename( + columns={ + "is_assumed": "is_assumed_roof", + } + ), + left_on="roof-description", + right_on="original_description" + ).merge( + pd.DataFrame(cleaned["floor-description"])[ + [ + 'original_description', 'is_solid', 'is_suspended', 'is_assumed', + 'insulation_thickness' + ] + ].rename( + columns={ + "is_assumed": "is_assumed_floor", + "insulation_thickness": "insulation_thickness_floor" + } + ), + left_on="floor-description", + right_on="original_description" + ) + archetyping_data = data[ [ "row_id", @@ -360,4 +419,353 @@ def app(): "Window type", "Location (Floor)", ] + ].merge( + epc_metadata[["row_id", "floor"]], + how="left", + on="row_id" + ).merge( + epc_data[ + [ + "row_id", "uprn", "current-energy-rating", "property-type", "built-form", "total-floor-area", + 'is_cavity_wall', 'is_filled_cavity', 'is_solid_brick_wall', 'is_system_built_wall', + 'is_timber_frame_wall', 'is_as_built', 'is_assumed_wall', 'insulation_thickness_wall', + 'is_solid', 'is_suspended', 'is_assumed_floor', 'insulation_thickness_floor', + 'is_pitched', 'is_roof_room', 'is_loft', + 'is_flat', 'is_thatched', 'is_at_rafters', 'is_assumed_roof', + 'has_dwelling_above', 'insulation_thickness', "mainheat-description", + "local-authority-label" + ] + ], + how="left", + on="row_id" + ).merge( + spatial_data[["row_id", "conservation_status", ]], + on="row_id", + how="left" + ) + + if archetyping_data.shape[0] != data.shape[0]: + raise Exception("Mismatch in data") + + # We create groups analogous to the Energy Company Obligation + # 0 - 72, 73 - 97, 98 - 199, 200+ + archetyping_data["Floor_area_category"] = pd.cut( + archetyping_data["Gross internal area (sqm)"], + bins=[0, 72, 97, 199, 1000], + labels=["0-72", "73-97", "98-199", "200+"] + ) + archetyping_data["Floor_area_category_backup"] = pd.cut( + archetyping_data["total-floor-area"].astype(float), + bins=[0, 72, 97, 199, 1000], + labels=["0-72", "73-97", "98-199", "200+"] + ) + archetyping_data["Floor_area_category"] = archetyping_data["Floor_area_category"].fillna( + archetyping_data["Floor_area_category_backup"] + ) + archetyping_data["Floor_area_category"] = archetyping_data["Floor_area_category"].astype(str) + archetyping_data["Floor_area_category"] = np.where( + pd.isnull(archetyping_data["Floor_area_category"]), + "Unknown", + archetyping_data["Floor_area_category"] + ) + archetyping_data = archetyping_data.drop(columns=["Floor_area_category_backup"]) + + archetyping_data["property-type-reduced"] = np.where( + archetyping_data["property-type"].isin(["Flat", "Maisionette"]), + "Flat/Maisonette", + archetyping_data["property-type"] + ) + + archetyping_data["built-form-reduced"] = np.where( + archetyping_data["built-form"].isin(["End-Terrace", "Semi-Detached"]), + "End-Terrace/Semi-Detached", + archetyping_data["built-form"] + ) + archetyping_data["built-form-reduced"] = np.where( + archetyping_data["property-type-reduced"] == "Flat/Maisonette", + "Flat/Maisonette", + archetyping_data["built-form-reduced"] + ) + + archetyping_data["Wall type"] = np.where( + archetyping_data["Wall type"].isin(['Solid ', 'Solid - internal lining ']), + "Solid", + archetyping_data["Wall type"] + ) + archetyping_data["Wall type"] = np.where( + archetyping_data["Wall type"].isin(['Cavity ', 'cavity ']), + "Cavity", + archetyping_data["Wall type"] + ) + + # Proposed remaps based on discoveries + value_remaps = { + # 8 Filey Avenue + "100021040744": { + "variable": "Property type", + "newvalue": "House, mid-terrace", + }, + # 7 Yetev Lev Court + "100021032043": { + "variable": "Wall type", + "newvalue": "Cavity", + }, + # 14 Yetev Lev Court + "100021032050": { + "variable": "Wall type", + "newvalue": "Cavity", + }, + # 23 Yetev Lev Court + "100021032059": { + "variable": "Wall type", + "newvalue": "Cavity", + }, + # 30 Yetev Lev Court + "100021032066": { + "variable": "Wall type", + "newvalue": "Cavity", + }, + # 34 Yetev Lev Court + "100021032070": { + "variable": "Wall type", + "newvalue": "Cavity", + }, + # B 86 Bethune Road + "100021026285": { + "variable": "Wall type", + "newvalue": "Solid", + }, + # A 80 Bethune Road + "100021026277": { + "variable": "Wall type", + "newvalue": "Solid", + }, + # 140 Kyverdale Road + "100021052262": { + "variable": "Property type", + "newvalue": "House, mid-terrace", + }, + # 6 Leabourne Road + "100021053799": { + "variable": "Wall type", + "newvalue": "Solid", + }, + # 22 Britannia Gardens - needs confirmation + # 7 Satanita Road - needs confirmation + # 12 Cheltenham Crescent + "100011402969": { + "variable": "Wall type", + "newvalue": "Cavity", + }, + "100021031752": { + "variable": "Roof type", + "newvalue": "Room Roof" + }, + # 79 Craven Park Road + "100021169682": { + "variable": "Roof type", + "newvalue": "Room Roof" + }, + # 88 Darenth Road + "100021036148": { + "variable": "Roof type", + "newvalue": "Room Roof" + }, + "100021036165": { + "variable": "Roof type", + "newvalue": "Room Roof" + }, + "100021036167": { + "variable": "Roof type", + "newvalue": "Room Roof" + }, + "100021053849": { + "variable": "Roof type", + "newvalue": "Room Roof" + }, + "100021054353": { + "variable": "Roof type", + "newvalue": "Room Roof" + }, + "100021054560": { + "variable": "Roof type", + "newvalue": "Room Roof" + }, + "100021059839": { + "variable": "Roof type", + "newvalue": "Room Roof" + }, + "100021059848": { + "variable": "Roof type", + "newvalue": "Room Roof" + } + } + + # Perform the remaps + for uprn, config in value_remaps.items(): + archetyping_data[config["variable"]] = np.where( + archetyping_data["uprn"].astype(str) == uprn, config["newvalue"], archetyping_data[config["variable"]] + ) + + # row_id = data[ + # # (data["Address letter or number"] == "C") & + # (data["Street address"].str.strip() == "41 Moresby Road") + # ]["row_id"] + # if len(row_id) != 1: + # raise Exception("Fail") + # print(epc_data[epc_data["row_id"] == row_id.values[0]]["uprn"]) + + # Map the year to the age band + def categorize_year(year): + if isinstance(year, str): + # Handle the case where year is in the format '1930s' + if 's' in year: + year = int(year[:4]) + else: + year = int(year) + else: + year = int(year) + + # Categorize based on year ranges + if year < 1900: + return 'A' + elif 1900 <= year <= 1929: + return 'B' + elif 1930 <= year <= 1949: + return 'C' + elif 1950 <= year <= 1966: + return 'D' + elif 1967 <= year <= 1975: + return 'E' + elif 1976 <= year <= 1982: + return 'F' + elif 1983 <= year <= 1990: + return 'G' + elif 1991 <= year <= 1995: + return 'H' + elif 1996 <= year <= 2002: + return 'I' + elif 2003 <= year <= 2006: + return 'J' + elif 2007 <= year <= 2011: + return 'K' + else: # year >= 2012 + return 'L' + + archetyping_data["SAP_age_band"] = archetyping_data["Property year built"].apply( + categorize_year + ) + + # Flag if the property is in London/Manchester + archetyping_data["Location"] = np.where( + archetyping_data["local-authority-label"].isin( + ["Hackney", "Barnet", "Haringey"] + ), + "London", + np.where( + archetyping_data["local-authority-label"].isin( + ["Salford", "Bury"] + ), + "Manchester", + "Southend" + ) + ) + # 9 Greenview is in manchester + archetyping_data["Location"] = np.where( + archetyping_data["row_id"] == data[data["Street address"] == "9 Greenview"]["row_id"].values[0], + "Manchester", + archetyping_data["Location"] + ) + + # Hackney 73 - London + # Southend-on-Sea 6 - Southend + # Barnet 4 - London + # Castle Point 4 - Southend + # Haringey 3 - London + # Salford 2 - Manchester + # Bury 1 - Manchester + + primary_archetyping_cols = [ + 'Property type', + "Location (Floor)", + 'Current heating system type', + 'Wall type', + 'Roof type', + "Location", + # 'current-energy-rating', 'property-type-reduced', 'built-form-reduced', 'is_cavity_wall', + # 'is_solid_brick_wall', 'is_system_built_wall', 'is_timber_frame_wall', 'is_as_built', + # 'is_solid', 'is_roof_room', + # 'is_loft', 'is_flat', 'is_thatched', + # 'is_at_rafters', 'has_dwelling_above', + # 'conservation_status', ] + + secondary_cols = [ + 'SAP_age_band', + 'is_filled_cavity', + 'insulation_thickness_wall' + 'insulation_thickness_floor' + 'insulation_thickness', + 'is_assumed_wall', + 'is_assumed_roof', + 'Floor_area_category' + ] + + archetypes = archetyping_data[primary_archetyping_cols].drop_duplicates() + # Hash the variables + archetypes["archetype_hash"] = archetypes.apply( + lambda x: hash(tuple(x.values)), + axis=1 + ) + archetypes = archetypes.sort_values("archetype_hash", ascending=True) + archetypes = archetypes.reset_index(drop=True) + archetypes["archetype_id"] = archetypes.index + + archetypes.to_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/AIHA/basic-archetypes.csv", index=False) + + # We match properties to archetypes + archetyping_data = archetyping_data.merge( + archetypes, + on=primary_archetyping_cols, + how="left" + ) + + # We should choose a representative property for each archetype + archetyping_data = archetyping_data.merge( + epc_metadata[["row_id", "days_since_last_epc"]], + how="left", + on="row_id" + ) + + # Mark the property with the oldest EPC as the representative property + representative_properties = archetyping_data.sort_values( + ["archetype_id", "days_since_last_epc"], ascending=[True, False] + ).drop_duplicates("archetype_id") + + archetyping_data["for_sample"] = np.where( + archetyping_data["row_id"].isin(representative_properties["row_id"]), + True, + False + ) + + # We save the archetyping data + archetyping_data.to_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/AIHA/archetyping_data.csv", + index=False) + # Save the EPC data + epc_data.to_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/AIHA/epc_data.csv", index=False) + # Save the spatial data + spatial_data = data[["row_id", "Address letter or number", "Street address", "Postcode"]].merge( + spatial_data, + on="row_id", + how="left" + ) + spatial_data.to_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/AIHA/spatial_data.csv", index=False) + + # Save archetyping data + archetyping_data = data[["row_id", "Address letter or number", "Street address", "Postcode"]].merge( + archetyping_data, + on="row_id", + how="left" + ) + + archetyping_data = archetyping_data.drop(columns=["row_id"]) From 9b08d49b85fceca164e70e9eaf8c32f1c7442e78 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Sun, 15 Sep 2024 13:58:59 +0100 Subject: [PATCH 006/166] Adding another heating test --- etl/customers/aiha/epc_data_pull.py | 4 +- recommendations/HeatingRecommender.py | 19 ++++--- .../test_data/heating_recommendations_data.py | 54 +++++++++++++++++++ 3 files changed, 69 insertions(+), 8 deletions(-) diff --git a/etl/customers/aiha/epc_data_pull.py b/etl/customers/aiha/epc_data_pull.py index 5e7c6714..16081205 100644 --- a/etl/customers/aiha/epc_data_pull.py +++ b/etl/customers/aiha/epc_data_pull.py @@ -767,5 +767,5 @@ def app(): on="row_id", how="left" ) - - archetyping_data = archetyping_data.drop(columns=["row_id"]) + archetyping_data.to_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/AIHA/archetyping_data.csv", + index=False) diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index dc433806..103fa7b1 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -9,6 +9,11 @@ from recommendations.HeatingControlRecommender import HeatingControlRecommender class HeatingRecommender: + ASSUMED_ELECTRIC_HEATING = [ + "Portable electric heaters assumed for most rooms", + "No system present, electric heaters assumed" + ] + ELECTRIC_HEATING_DESCRIPTIONS = [ "Room heaters, electric", "Electric storage heaters", @@ -16,6 +21,10 @@ class HeatingRecommender: "Portable electric heaters assumed for most rooms", ] + ROOM_HEATERS_DESCRIPTIONS = [ + "Room heaters, mains gas", "Room heaters, electric", "Portable electric heaters assumed for most rooms", + ] + high_heat_retention_contols_desc = "Controls for high heat retention storage heaters" def __init__(self, property_instance: Property): @@ -25,8 +34,8 @@ class HeatingRecommender: self.heating_recommendations = [] self.heating_control_recommendations = [] - self.has_electric_heating_description = ( - self.property.main_heating["clean_description"] in self.ELECTRIC_HEATING_DESCRIPTIONS + self.has_electric_heating_description = self.property.main_heating["clean_description"] in ( + self.ELECTRIC_HEATING_DESCRIPTIONS + self.ASSUMED_ELECTRIC_HEATING ) def is_high_heat_retention_valid(self, ashp_only_heating_recommendation, measures): @@ -37,9 +46,7 @@ class HeatingRecommender: # If the property has assumed electric heating, regardless of whether or not it has a mains connection, we # can consider hhr storage heaters - electric_heating_assumed = ( - self.property.main_heating["clean_description"] in ["No system present, electric heaters assumed"] - ) + electric_heating_assumed = self.property.main_heating["clean_description"] in self.ASSUMED_ELECTRIC_HEATING has_electric = self.has_electric_heating_description or electric_heating_assumed @@ -64,7 +71,7 @@ class HeatingRecommender: # The property is using portable heaters and has access to gas mains has_room_heaters = ( - self.property.main_heating["clean_description"] in ["Room heaters, mains gas", "Room heaters, electric"] and + self.property.main_heating["clean_description"] in self.ROOM_HEATERS_DESCRIPTIONS and self.property.data["mains-gas-flag"] ) diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index f283050b..c64aab6f 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -348,6 +348,50 @@ testing_examples = [ "Programmer and room thermostat for heating controls so we'd expect an ASHP heating recommendation" "as the only option, and heating controls recommendations for programmer, room thermostats and trvs" "as well as ttzc" + }, + { + "epc": { + 'lmk-key': '977006769242013072314202915172178', 'address1': '40, Elswick Road', 'address2': None, + 'address3': None, 'postcode': 'B44 0JQ', 'building-reference-number': 7277661178, + 'current-energy-rating': 'G', 'potential-energy-rating': 'B', 'current-energy-efficiency': 18, + 'potential-energy-efficiency': 88, 'property-type': 'House', 'built-form': 'Mid-Terrace', + 'inspection-date': '2013-07-23', 'local-authority': 'E08000025', 'constituency': 'E14000561', + 'county': None, + 'lodgement-date': '2013-07-23', 'transaction-type': 'none of the above', 'environment-impact-current': 31, + 'environment-impact-potential': 90, 'energy-consumption-current': 536, 'energy-consumption-potential': 59, + 'co2-emissions-current': 7.0, 'co2-emiss-curr-per-floor-area': 96, 'co2-emissions-potential': 0.9, + 'lighting-cost-current': 48, 'lighting-cost-potential': 48, 'heating-cost-current': 1395, + 'heating-cost-potential': 353, 'hot-water-cost-current': 457, 'hot-water-cost-potential': 69, + 'total-floor-area': 73.0, 'energy-tariff': 'Unknown', 'mains-gas-flag': 'Y', 'floor-level': 'NODATA!', + 'flat-top-storey': None, 'flat-storey-count': None, 'main-heating-controls': 2601.0, + 'multi-glaze-proportion': 100.0, 'glazed-type': 'double glazing, unknown install date', + 'glazed-area': 'Normal', 'extension-count': 0, 'number-habitable-rooms': 4, 'number-heated-rooms': 1, + 'low-energy-lighting': 90, 'number-open-fireplaces': 1, + 'hotwater-description': 'Electric immersion, standard tariff', 'hot-water-energy-eff': 'Very Poor', + 'hot-water-env-eff': 'Very Poor', 'floor-description': 'Solid, no insulation (assumed)', + 'floor-energy-eff': None, 'windows-description': 'Fully double glazed', 'windows-energy-eff': 'Average', + 'windows-env-eff': 'Average', 'walls-description': 'Solid brick, as built, no insulation (assumed)', + 'walls-energy-eff': 'Very Poor', 'walls-env-eff': 'Very Poor', + 'secondheat-description': 'Room heaters, mains gas', 'roof-description': 'Pitched, 75 mm loft insulation', + 'roof-energy-eff': 'Average', 'roof-env-eff': 'Average', + 'mainheat-description': 'Portable electric heaters assumed for most rooms', + 'mainheat-energy-eff': 'Very Poor', 'mainheat-env-eff': 'Very Poor', + 'mainheatcont-description': 'No thermostatic control of room temperature', 'mainheatc-energy-eff': 'Poor', + 'mainheatc-env-eff': 'Poor', 'lighting-description': 'Low energy lighting in 90% of fixed outlets', + 'lighting-energy-eff': 'Very Good', 'lighting-env-eff': 'Very Good', + 'main-fuel': 'mains gas (not community)', 'wind-turbine-count': 0, 'heat-loss-corridor': 'NO DATA!', + 'unheated-corridor-length': None, 'floor-height': 2.5, 'photo-supply': 0.0, + 'solar-water-heating-flag': None, + 'mechanical-ventilation': 'natural', 'address': '40, Elswick Road', 'local-authority-label': 'Birmingham', + 'constituency-label': 'Birmingham, Erdington', 'posttown': 'BIRMINGHAM', + 'construction-age-band': 'England and Wales: 1930-1949', + 'lodgement-datetime': '2013-07-23 14:20:29', 'tenure': 'owner-occupied', + 'fixed-lighting-outlets-count': 10.0, 'low-energy-fixed-light-count': 9.0, 'uprn': 100070358594, + 'uprn-source': 'Address Matched' + }, + "heating_recommendation_descriptions": [], + "heating_controls_recommendation_descriptions": [], + "notes": "" } ] @@ -389,3 +433,13 @@ print(eg["mainheat-energy-eff"]) print(eg["property-type"]) print(eg["built-form"]) print(eg["mainheatcont-description"]) + +### We also use the Midlands EPC F/G portfolio to get examples to create tests +portfolio = pd.read_excel( + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/sfr/20240820 portfolio_epc_data.xlsx" +) +portfolio.columns = [c.replace("_", "-").lower() for c in portfolio.columns] +eg = portfolio[ + (portfolio["mainheat-description"] == "Portable electric heaters assumed for most rooms") +].sample(1) +eg = eg.squeeze().to_dict() From c4ab7f5a2c6599f10290d20e1b8dc6f279f375ac Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 16 Sep 2024 13:11:08 +0100 Subject: [PATCH 007/166] Adding new heating recommendation test --- etl/customers/aiha/epc_data_pull.py | 6 ++-- recommendations/HeatingControlRecommender.py | 4 +-- recommendations/HeatingRecommender.py | 26 ++++++++++----- .../test_data/heating_recommendations_data.py | 32 +++++++++++++++---- .../tests/test_heating_recommendations.py | 7 ---- 5 files changed, 49 insertions(+), 26 deletions(-) diff --git a/etl/customers/aiha/epc_data_pull.py b/etl/customers/aiha/epc_data_pull.py index 16081205..f7f4631c 100644 --- a/etl/customers/aiha/epc_data_pull.py +++ b/etl/customers/aiha/epc_data_pull.py @@ -767,5 +767,7 @@ def app(): on="row_id", how="left" ) - archetyping_data.to_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/AIHA/archetyping_data.csv", - index=False) + archetyping_data.to_csv( + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/AIHA/archetyping_data.csv", + index=False + ) diff --git a/recommendations/HeatingControlRecommender.py b/recommendations/HeatingControlRecommender.py index 3e47c355..6f848441 100644 --- a/recommendations/HeatingControlRecommender.py +++ b/recommendations/HeatingControlRecommender.py @@ -121,7 +121,7 @@ class HeatingControlRecommender: self.recommendation.append( { - "description": "upgrade heating controls to High Heat Retention Storage Heater Controls", + "description": "Upgrade heating controls to High Heat Retention Storage Heater Controls", **self.costs.celect_type_controls(), "simulation_config": simulation_config, "description_simulation": description_simulation @@ -192,7 +192,7 @@ class HeatingControlRecommender: has_trvs=has_trvs ) - description = "upgrade heating controls to Room thermostat, programmer and TRVs" + description = "Upgrade heating controls to Room thermostat, programmer and TRVs" already_installed = "heating_control" in self.property.already_installed if already_installed: diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index 103fa7b1..dc2bf1b8 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -137,8 +137,6 @@ class HeatingRecommender: if hhr_valid: # Recommend high heat retention storage heaters - # TODO: We need to allow for the possibility that the property aleady has storage heaters, but just - # needs the controls self.recommend_hhr_storage_heaters(phase=phase, system_change=True, heating_controls_only=False) gas_boiler_suitable, has_boiler = self.is_boiler_upgrade_suitable( @@ -474,12 +472,8 @@ class HeatingRecommender: } controls_description = controls_recommendations[0]['description'] - # Make the first letter of the description lowercase - controls_description = ( - controls_description[0].lower() + controls_description[1:] - ) - recommendation_description = f"{description} and {controls_description}" + recommendation_description = f"{description} {controls_description}" already_installed = "heating_controls" in self.property.already_installed if already_installed: @@ -555,6 +549,14 @@ class HeatingRecommender: We will recommend upgrading to a high heat retention storage system, if the current system is not already high heat retention storage + If the property currently has electric storage heaters, with automatic charge control, we allow for a high + heat retention stoarage heaters recommendation. This is because the automatic charge control is not the same + as the high heat retention storage heaters. HHR storage heaters aren't guaranteed to be more efficient but + we can at least present the option to the end user and they can decide if they want to go ahead with the + recommendation or not. There's a useful guide by quidos, describing the differences between some of the + different storage heater options: + https://www.quidos.co.uk/wp-content/uploads/2017/04/Technical-Bulletin-010417-Storage-Heatersv2.pdf + :param phase: The phase of the recommendation :param system_change: Indicates if we are recommending a different type of heating system, compared to the current system @@ -600,7 +602,15 @@ class HeatingRecommender: costs = self.costs.high_heat_electric_storage_heaters( number_heated_rooms=number_heated_rooms ) - description = "Install high heat retention electric storage heaters" + description = "Install high heat retention electric storage heaters." + + # We check the existing heating system and controls + if ( + self.property.main_heating["has_electric_storage_heaters"] and + self.property.main_heating_controls["charging_system"] in ["automatic charge control"] + ): + description += (" The current electric heaters may be retrofit with high heat retention storage controls" + " however this is dependent on the existing system and may not be possible.") heating_description_simulation = { "mainheat-description": new_heating_description, diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index c64aab6f..0281a26d 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -144,11 +144,20 @@ testing_examples = [ 'tenure': 'owner-occupied', 'fixed-lighting-outlets-count': 6.0, 'low-energy-fixed-light-count': 4.0, 'uprn': 100090311351.0, 'uprn-source': 'Address Matched', 'property-type_y': None, 'built-form_y': None, }, - "heating_recommendation_descriptions": [], + "heating_recommendation_descriptions": [ + 'Install high heat retention electric storage heaters. The current electric heaters may be retrofit with ' + 'high heat retention storage controls however this is dependent on the existing system and may not be ' + 'possible. Upgrade heating controls to High Heat Retention Storage Heater Controls', + 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' + 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' + 'scheme grant' + + ], "heating_controls_recommendation_descriptions": [], - "notes": "This test has electric storage heaters with automatic charge control - this case should be researched" - "and checked that a high heat retention storage recommendation is actually sensible. If it's not, " - "we should adjust accordingly or perhaps have just a control recommendation" + "notes": "This test has electric storage heaters with automatic charge control - we recommend hhr storage" + "heaters in this case, but because there are already electic storage heaters in place, we " + "note, in the description of the recommendation, that this upgrade may be possible by retrofitting" + "the existing storage heaters, but that dependes on the model of the existing heaters" }, { "epc": { @@ -188,9 +197,18 @@ testing_examples = [ 'tenure': 'owner-occupied', 'fixed-lighting-outlets-count': None, 'low-energy-fixed-light-count': None, 'uprn': 100021560521.0, 'uprn-source': 'Address Matched', }, - "heating_recommendation_descriptions": [], - "heating_controls_recommendation_descriptions": [], - "notes": "" + "heating_recommendation_descriptions": [ + 'Upgrade to a new condensing boiler' + ], + "heating_controls_recommendation_descriptions": [ + 'Upgrade heating controls to Room thermostat, programmer and TRVs', + 'Upgrade heating controls to Smart Thermostats, room sensors and smart radiator valves (time & ' + 'temperature zone control)' + ], + "notes": "Because of this property is a maisonette, which already has a boiler (but an inefficient one due to " + "the current water heating efficiency) the only recommendation we expect is for " + "a boiler upgrade. The heating controls are programmer and thermostat, so we can also recommend" + "better heating controls" }, { "epc": { diff --git a/recommendations/tests/test_heating_recommendations.py b/recommendations/tests/test_heating_recommendations.py index 968583e4..4351623b 100644 --- a/recommendations/tests/test_heating_recommendations.py +++ b/recommendations/tests/test_heating_recommendations.py @@ -54,13 +54,6 @@ class TestHeatingRecommendations: :return: """ - if test_case["epc"]["uprn"] == 100090311351: - raise Exception( - "This test has electric storage heaters with automatic charge control - this case should be researched" - "and checked that a high heat retention storage recommendation is actually sensible. If it's not, " - "we should adjust accordingly or perhaps have just a control recommendation" - ) - if test_case["epc"]["uprn"] == 100021560521: raise Exception("Finish this test - could do so while on the train") From 342f926415f5c4239386b1081a3b43e37fb5a99a Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 16 Sep 2024 13:57:36 +0100 Subject: [PATCH 008/166] Added another unit test for heating recommendations --- backend/Property.py | 10 +++++++++- .../tests/test_data/heating_recommendations_data.py | 13 ++++++++++--- .../tests/test_heating_recommendations.py | 3 --- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index 704e4f0a..b15c9f25 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -1224,7 +1224,15 @@ class Property: if "air_source_heat_pump" not in measures: return False - suitable_property_type = self.data["property-type"] in ["House", "Bungalow"] + suitable_house = self.data["property-type"] == "House" and self.data["built-form"] in [ + "Detached", "Semi-Detached", + ] + + suitable_bungalow = self.data["property-type"] == "Bungalow" and self.data["built-form"] in [ + "Detached", "Semi-Detached" + ] + + suitable_property_type = suitable_house or suitable_bungalow has_air_source_heat_pump = self.main_heating["has_air_source_heat_pump"] return suitable_property_type and not has_air_source_heat_pump diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index 0281a26d..78f6c7bf 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -405,11 +405,18 @@ testing_examples = [ 'construction-age-band': 'England and Wales: 1930-1949', 'lodgement-datetime': '2013-07-23 14:20:29', 'tenure': 'owner-occupied', 'fixed-lighting-outlets-count': 10.0, 'low-energy-fixed-light-count': 9.0, 'uprn': 100070358594, - 'uprn-source': 'Address Matched' + 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None }, - "heating_recommendation_descriptions": [], + "heating_recommendation_descriptions": [ + 'Upgrade to a new condensing boiler upgrade heating controls to Room thermostat, programmer and TRVs', + 'Install high heat retention electric storage heaters. upgrade heating controls to High Heat Retention ' + 'Storage Heater Controls', + 'Upgrade to a new condensing boiler upgrade heating controls to Smart Thermostats, room sensors and smart ' + 'radiator valves (time & temperature zone control)' + ], "heating_controls_recommendation_descriptions": [], - "notes": "" + "notes": "This property has assumed electric heating and is mid-terrace house. It has a mains gas connection." + "We can recommend a boiler upgrade and high heat retention storage heaters" } ] diff --git a/recommendations/tests/test_heating_recommendations.py b/recommendations/tests/test_heating_recommendations.py index 4351623b..35373729 100644 --- a/recommendations/tests/test_heating_recommendations.py +++ b/recommendations/tests/test_heating_recommendations.py @@ -54,9 +54,6 @@ class TestHeatingRecommendations: :return: """ - if test_case["epc"]["uprn"] == 100021560521: - raise Exception("Finish this test - could do so while on the train") - epc_records = {"original_epc": test_case["epc"].copy(), "full_sap_epc": {}, "old_data": []} epc_record = EPCRecord( From 7e11584407904e9bc6936e73762211315ac6da3d Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 16 Sep 2024 14:08:28 +0100 Subject: [PATCH 009/166] Added coverage for oil boiler --- recommendations/HeatingRecommender.py | 14 ++++- .../test_data/heating_recommendations_data.py | 56 ++++++++++++++++++- 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index dc2bf1b8..3a5e0c2c 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -27,6 +27,11 @@ class HeatingRecommender: high_heat_retention_contols_desc = "Controls for high heat retention storage heaters" + # These are descriptions for boilers that are not gas boilers + NON_GAS_BOILERS = [ + "Boiler and radiators, oil", + ] + def __init__(self, property_instance: Property): self.property = property_instance self.costs = Costs(self.property) @@ -84,13 +89,20 @@ class HeatingRecommender: self.property.data["mains-gas-flag"] ) + # The next condition is if the home has a non-gas boiler, such as an oil boiler + non_gas_boiler = ( + self.property.main_heating["clean_description"] in self.NON_GAS_BOILERS and + self.property.data["mains-gas-flag"] + ) + is_valid = ( ( has_boiler or no_heating_has_mains or electic_heating_has_mains or has_room_heaters or - portable_heaters_has_mains + portable_heaters_has_mains or + non_gas_boiler ) and (not ashp_only_heating_recommendation) and ("boiler_upgrade" in measures) diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index 78f6c7bf..343c0600 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -417,6 +417,53 @@ testing_examples = [ "heating_controls_recommendation_descriptions": [], "notes": "This property has assumed electric heating and is mid-terrace house. It has a mains gas connection." "We can recommend a boiler upgrade and high heat retention storage heaters" + }, + { + "epc": { + 'lmk-key': '1162853989402014062718391220442948', 'address1': '145, Darley Green Road', 'address2': 'Knowle', + 'address3': None, 'postcode': 'B93 8PU', 'building-reference-number': 4475684278, + 'current-energy-rating': 'F', 'potential-energy-rating': 'D', 'current-energy-efficiency': 23, + 'potential-energy-efficiency': 58, 'property-type': 'House', 'built-form': 'Semi-Detached', + 'inspection-date': '2014-06-24', 'local-authority': 'E08000029', 'constituency': 'E14000812', + 'county': None, + 'lodgement-date': '2014-06-27', 'transaction-type': 'none of the above', 'environment-impact-current': 17, + 'environment-impact-potential': 45, 'energy-consumption-current': 382, 'energy-consumption-potential': 194, + 'co2-emissions-current': 27.0, 'co2-emiss-curr-per-floor-area': 94, 'co2-emissions-potential': 14.0, + 'lighting-cost-current': 175, 'lighting-cost-potential': 106, 'heating-cost-current': 5477, + 'heating-cost-potential': 3001, 'hot-water-cost-current': 267, 'hot-water-cost-potential': 120, + 'total-floor-area': 293.0, 'energy-tariff': 'Single', 'mains-gas-flag': 'N', 'floor-level': 'NODATA!', + 'flat-top-storey': None, 'flat-storey-count': None, 'main-heating-controls': 2106.0, + 'multi-glaze-proportion': 0.0, 'glazed-type': 'not defined', 'glazed-area': 'Normal', 'extension-count': 2, + 'number-habitable-rooms': 12, 'number-heated-rooms': 12, 'low-energy-lighting': 31, + 'number-open-fireplaces': 2, 'hotwater-description': 'From main system', 'hot-water-energy-eff': 'Average', + 'hot-water-env-eff': 'Poor', 'floor-description': 'Suspended, no insulation (assumed)', + 'floor-energy-eff': None, 'windows-description': 'Single glazed', 'windows-energy-eff': 'Very Poor', + 'windows-env-eff': 'Very Poor', 'walls-description': 'Solid brick, as built, no insulation (assumed)', + 'walls-energy-eff': 'Very Poor', 'walls-env-eff': 'Very Poor', + 'secondheat-description': 'Room heaters, dual fuel (mineral and wood)', + 'roof-description': 'Pitched, no insulation (assumed)', 'roof-energy-eff': 'Very Poor', + 'roof-env-eff': 'Very Poor', 'mainheat-description': 'Boiler and radiators, oil', + 'mainheat-energy-eff': 'Average', 'mainheat-env-eff': 'Average', + 'mainheatcont-description': 'Programmer, room thermostat and TRVs', 'mainheatc-energy-eff': 'Good', + 'mainheatc-env-eff': 'Good', 'lighting-description': 'Low energy lighting in 31% of fixed outlets', + 'lighting-energy-eff': 'Average', 'lighting-env-eff': 'Average', 'main-fuel': 'oil (not community)', + 'wind-turbine-count': 0, 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, + 'floor-height': 2.5, 'photo-supply': 0.0, 'solar-water-heating-flag': None, + 'mechanical-ventilation': 'natural', 'address': '145, Darley Green Road, Knowle', + 'local-authority-label': 'Solihull', 'constituency-label': 'Meriden', 'posttown': 'SOLIHULL', + 'construction-age-band': 'England and Wales: before 1900', + 'lodgement-datetime': '2014-06-27 18:39:12', 'tenure': 'owner-occupied', + 'fixed-lighting-outlets-count': 42.0, 'low-energy-fixed-light-count': 13.0, 'uprn': 100070985545, + 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None + }, + "heating_recommendation_descriptions": [ + 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' + 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' + 'scheme grant' + ], + "heating_controls_recommendation_descriptions": [], + "notes": "This property has an oil boiler and doesn't have a mains gas connection so we can only recommend" + "an air source heat pump" } ] @@ -460,11 +507,18 @@ print(eg["built-form"]) print(eg["mainheatcont-description"]) ### We also use the Midlands EPC F/G portfolio to get examples to create tests + +completed_descriptions = [ + "Portable electric heaters assumed for most rooms" +] + portfolio = pd.read_excel( "/Users/khalimconn-kowlessar/Documents/hestia/Customers/sfr/20240820 portfolio_epc_data.xlsx" ) portfolio.columns = [c.replace("_", "-").lower() for c in portfolio.columns] +portfolio = portfolio[~portfolio["mainheat-description"].isin(completed_descriptions)] + eg = portfolio[ - (portfolio["mainheat-description"] == "Portable electric heaters assumed for most rooms") + (portfolio["mainheat-description"] == "Boiler and radiators, oil") ].sample(1) eg = eg.squeeze().to_dict() From f86a7cb97d97bdc903509c18a3c5d784df00b3b9 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 16 Sep 2024 14:43:21 +0100 Subject: [PATCH 010/166] adding heating unit test coverage --- recommendations/HeatingRecommender.py | 19 +- .../test_data/heating_recommendations_data.py | 171 +++++++++++++++++- 2 files changed, 171 insertions(+), 19 deletions(-) diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index 3a5e0c2c..5d68ebcf 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -14,13 +14,6 @@ class HeatingRecommender: "No system present, electric heaters assumed" ] - ELECTRIC_HEATING_DESCRIPTIONS = [ - "Room heaters, electric", - "Electric storage heaters", - "Electric storage heaters, radiators", - "Portable electric heaters assumed for most rooms", - ] - ROOM_HEATERS_DESCRIPTIONS = [ "Room heaters, mains gas", "Room heaters, electric", "Portable electric heaters assumed for most rooms", ] @@ -39,9 +32,7 @@ class HeatingRecommender: self.heating_recommendations = [] self.heating_control_recommendations = [] - self.has_electric_heating_description = self.property.main_heating["clean_description"] in ( - self.ELECTRIC_HEATING_DESCRIPTIONS + self.ASSUMED_ELECTRIC_HEATING - ) + self.has_electric_heating_description = self.property.main_heating["has_electric"] def is_high_heat_retention_valid(self, ashp_only_heating_recommendation, measures): """ @@ -49,14 +40,8 @@ class HeatingRecommender: :return: """ - # If the property has assumed electric heating, regardless of whether or not it has a mains connection, we - # can consider hhr storage heaters - electric_heating_assumed = self.property.main_heating["clean_description"] in self.ASSUMED_ELECTRIC_HEATING - - has_electric = self.has_electric_heating_description or electric_heating_assumed - return ( - has_electric and (not ashp_only_heating_recommendation) and + self.has_electric_heating_description and (not ashp_only_heating_recommendation) and ("high_heat_retention_storage_heater" in measures) ) diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index 343c0600..eec7703e 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -464,6 +464,168 @@ testing_examples = [ "heating_controls_recommendation_descriptions": [], "notes": "This property has an oil boiler and doesn't have a mains gas connection so we can only recommend" "an air source heat pump" + }, + { + "epc": { + 'lmk-key': '351990902052009082517013406210567', 'address1': '56, Collingham Road', 'address2': None, + 'address3': None, 'postcode': 'LE3 2BA', 'building-reference-number': 5783266668, + 'current-energy-rating': 'F', 'potential-energy-rating': 'F', 'current-energy-efficiency': 28, + 'potential-energy-efficiency': 33, 'property-type': 'House', 'built-form': 'Semi-Detached', + 'inspection-date': '2009-08-25', 'local-authority': 'E06000016', 'constituency': 'E14000784', + 'county': None, + 'lodgement-date': '2009-08-25', 'transaction-type': 'marketed sale', 'environment-impact-current': 31, + 'environment-impact-potential': 33, 'energy-consumption-current': 579, 'energy-consumption-potential': 549, + 'co2-emissions-current': 7.4, 'co2-emiss-curr-per-floor-area': 95, 'co2-emissions-potential': 7.1, + 'lighting-cost-current': 78, 'lighting-cost-potential': 39, 'heating-cost-current': 985, + 'heating-cost-potential': 1015, 'hot-water-cost-current': 381, 'hot-water-cost-potential': 281, + 'total-floor-area': 87.8, 'energy-tariff': 'Single', 'mains-gas-flag': 'Y', 'floor-level': 'NO DATA!', + 'flat-top-storey': None, 'flat-storey-count': None, 'main-heating-controls': 2601.0, + 'multi-glaze-proportion': 35.0, 'glazed-type': 'double glazing installed before 2002', + 'glazed-area': 'Normal', 'extension-count': 0, 'number-habitable-rooms': 5, 'number-heated-rooms': 2, + 'low-energy-lighting': 0, 'number-open-fireplaces': 0, + 'hotwater-description': 'Electric immersion, standard tariff', 'hot-water-energy-eff': 'Very Poor', + 'hot-water-env-eff': 'Poor', 'floor-description': 'Suspended, no insulation (assumed)', + 'floor-energy-eff': None, 'windows-description': 'Partial double glazing', 'windows-energy-eff': 'Poor', + 'windows-env-eff': 'Poor', 'walls-description': 'Solid brick, as built, no insulation (assumed)', + 'walls-energy-eff': 'Very Poor', 'walls-env-eff': 'Very Poor', + 'secondheat-description': 'Room heaters, mains gas', 'roof-description': 'Pitched, no insulation (assumed)', + 'roof-energy-eff': 'Very Poor', 'roof-env-eff': 'Very Poor', + 'mainheat-description': 'Room heaters, mains gas', 'mainheat-energy-eff': 'Average', + 'mainheat-env-eff': 'Average', 'mainheatcont-description': 'No thermostatic control of room temperature', + 'mainheatc-energy-eff': 'Poor', 'mainheatc-env-eff': 'Poor', + 'lighting-description': 'No low energy lighting', 'lighting-energy-eff': 'Very Poor', + 'lighting-env-eff': 'Very Poor', + 'main-fuel': 'mains gas - this is for backwards compatibility only and should not be used', + 'wind-turbine-count': 0, 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, + 'floor-height': 2.48, 'photo-supply': 0.0, 'solar-water-heating-flag': 'N', + 'mechanical-ventilation': 'natural', 'address': '56, Collingham Road', 'local-authority-label': 'Leicester', + 'constituency-label': 'Leicester West', 'posttown': 'LEICESTER', + 'construction-age-band': 'England and Wales: 1930-1949', + 'lodgement-datetime': '2009-08-25 17:01:34', 'tenure': 'owner-occupied', + 'fixed-lighting-outlets-count': None, 'low-energy-fixed-light-count': None, 'uprn': 2465031849, + 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None + }, + "heating_recommendation_descriptions": [ + 'Upgrade to a new condensing boiler upgrade heating controls to Room thermostat, programmer and TRVs', + 'Upgrade to a new condensing boiler Upgrade heating controls to Smart Thermostats, room sensors and smart ' + 'radiator valves (time & temperature zone control)', + 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' + 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' + 'scheme grant' + ], + "heating_controls_recommendation_descriptions": [], + "notes": "This property has room heaters, from the mains gas supply. We recommend a boiler upgrade as" + "well as an air source heat pump" + }, + { + "epc": { + 'lmk-key': 'f9997a382dca2a1b5dc916a21cf1a28327fc6ffe32fa4c5eeb7a859fe73cabf4', + 'address1': '39 Parkes Street', 'address2': None, 'address3': None, 'postcode': 'WV13 2LR', + 'building-reference-number': 10005271458, 'current-energy-rating': 'G', 'potential-energy-rating': 'B', + 'current-energy-efficiency': 17, 'potential-energy-efficiency': 89, 'property-type': 'House', + 'built-form': 'Mid-Terrace', 'inspection-date': '2023-11-10', 'local-authority': 'E08000030', + 'constituency': 'E14001011', 'county': None, 'lodgement-date': '2023-11-10', 'transaction-type': 'rental', + 'environment-impact-current': 29, 'environment-impact-potential': 88, 'energy-consumption-current': 582, + 'energy-consumption-potential': 71, 'co2-emissions-current': 7.4, 'co2-emiss-curr-per-floor-area': 98, + 'co2-emissions-potential': 1.0, 'lighting-cost-current': 193, 'lighting-cost-potential': 121, + 'heating-cost-current': 3789, 'heating-cost-potential': 774, 'hot-water-cost-current': 1241, + 'hot-water-cost-potential': 165, 'total-floor-area': 75.0, 'energy-tariff': 'Single', 'mains-gas-flag': 'Y', + 'floor-level': None, 'flat-top-storey': None, 'flat-storey-count': None, 'main-heating-controls': None, + 'multi-glaze-proportion': 100.0, 'glazed-type': 'double glazing, unknown install date', + 'glazed-area': 'Normal', 'extension-count': 0, 'number-habitable-rooms': 5, 'number-heated-rooms': 0, + 'low-energy-lighting': 40, 'number-open-fireplaces': 0, + 'hotwater-description': 'No system present: electric immersion assumed', + 'hot-water-energy-eff': 'Very Poor', 'hot-water-env-eff': 'Poor', + 'floor-description': 'Suspended, no insulation (assumed)', 'floor-energy-eff': None, + 'windows-description': 'Fully double glazed', 'windows-energy-eff': 'Average', 'windows-env-eff': 'Average', + 'walls-description': 'Solid brick, as built, no insulation (assumed)', 'walls-energy-eff': 'Very Poor', + 'walls-env-eff': 'Very Poor', 'secondheat-description': 'None', + 'roof-description': 'Pitched, 200 mm loft insulation', 'roof-energy-eff': 'Good', 'roof-env-eff': 'Good', + 'mainheat-description': 'No system present: electric heaters assumed', 'mainheat-energy-eff': 'Very Poor', + 'mainheat-env-eff': 'Poor', 'mainheatcont-description': 'None', 'mainheatc-energy-eff': 'Very Poor', + 'mainheatc-env-eff': 'Very Poor', 'lighting-description': 'Low energy lighting in 40% of fixed outlets', + 'lighting-energy-eff': 'Average', 'lighting-env-eff': 'Average', + 'main-fuel': 'To be used only when there is no heating/hot-water system or data is from a community ' + 'network', + 'wind-turbine-count': 0, 'heat-loss-corridor': None, 'unheated-corridor-length': None, 'floor-height': 2.5, + 'photo-supply': 0.0, 'solar-water-heating-flag': 'N', 'mechanical-ventilation': 'natural', + 'address': '39 Parkes Street', 'local-authority-label': 'Walsall', 'constituency-label': 'Walsall North', + 'posttown': 'WILLENHALL', 'construction-age-band': 'England and Wales: 1900-1929', + 'lodgement-datetime': '2023-11-10 18:06:18', 'tenure': 'Rented (social)', + 'fixed-lighting-outlets-count': 10.0, 'low-energy-fixed-light-count': None, 'uprn': 100071113763, + 'uprn-source': 'Energy Assessor', 'sheating-energy-eff': None, 'sheating-env-eff': None + + }, + "heating_recommendation_descriptions": [ + 'Upgrade to a new condensing boiler upgrade heating controls to Room thermostat, programmer and TRVs', + 'Upgrade to a new condensing boiler Upgrade heating controls to Smart Thermostats, room sensors and smart ' + 'radiator valves (time & temperature zone control)', + 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' + 'Storage Heater Controls', + 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' + 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' + 'scheme grant' + ], + "heating_controls_recommendation_descriptions": [], + "notes": "This property has assumed electric heaters. Boiler upgrade, HHR and ASHP are all recommended" + }, + { + "epc": { + 'lmk-key': 'dca62e7f9e21ac21d9c8af1029102cbe47c0509b8e6fc302cd7df079f8fc3a53', + 'address1': '58 Telford Road', 'address2': None, 'address3': None, 'postcode': 'WS2 7LD', + 'building-reference-number': 10003473159, 'current-energy-rating': 'F', 'potential-energy-rating': 'C', + 'current-energy-efficiency': 37, 'potential-energy-efficiency': 77, 'property-type': 'House', + 'built-form': 'Mid-Terrace', 'inspection-date': '2022-10-26', 'local-authority': 'E08000030', + 'constituency': 'E14001011', 'county': None, 'lodgement-date': '2022-10-26', + 'transaction-type': 'marketed sale', 'environment-impact-current': 47, 'environment-impact-potential': 80, + 'energy-consumption-current': 380, 'energy-consumption-potential': 130, 'co2-emissions-current': 4.5, + 'co2-emiss-curr-per-floor-area': 65, 'co2-emissions-potential': 1.6, 'lighting-cost-current': 101, + 'lighting-cost-potential': 63, 'heating-cost-current': 1267, 'heating-cost-potential': 733, + 'hot-water-cost-current': 386, 'hot-water-cost-potential': 67, 'total-floor-area': 69.0, + 'energy-tariff': 'Single', 'mains-gas-flag': 'Y', 'floor-level': None, 'flat-top-storey': None, + 'flat-storey-count': None, 'main-heating-controls': None, 'multi-glaze-proportion': 90.0, + 'glazed-type': 'double glazing, unknown install date', 'glazed-area': 'Normal', 'extension-count': 0, + 'number-habitable-rooms': 4, 'number-heated-rooms': 1, 'low-energy-lighting': 40, + 'number-open-fireplaces': 0, 'hotwater-description': 'Electric immersion, standard tariff', + 'hot-water-energy-eff': 'Very Poor', 'hot-water-env-eff': 'Poor', + 'floor-description': 'Solid, no insulation (assumed)', 'floor-energy-eff': None, + 'windows-description': 'Mostly double glazing', 'windows-energy-eff': 'Average', + 'windows-env-eff': 'Average', 'walls-description': 'Cavity wall, filled cavity', + 'walls-energy-eff': 'Average', 'walls-env-eff': 'Average', + 'secondheat-description': 'Room heaters, mains gas', 'roof-description': 'Pitched, 100 mm loft insulation', + 'roof-energy-eff': 'Average', 'roof-env-eff': 'Average', + 'mainheat-description': 'Portable electric heaters assumed for most rooms, Room heaters, electric', + 'mainheat-energy-eff': 'Very Poor', 'mainheat-env-eff': 'Poor', + 'mainheatcont-description': 'No thermostatic control of room temperature', 'mainheatc-energy-eff': 'Poor', + 'mainheatc-env-eff': 'Poor', 'lighting-description': 'Low energy lighting in 40% of fixed outlets', + 'lighting-energy-eff': 'Average', 'lighting-env-eff': 'Average', 'main-fuel': 'mains gas (not community)', + 'wind-turbine-count': 0, 'heat-loss-corridor': None, 'unheated-corridor-length': None, 'floor-height': 2.46, + 'photo-supply': 0.0, 'solar-water-heating-flag': 'N', 'mechanical-ventilation': 'natural', + 'address': '58 Telford Road', 'local-authority-label': 'Walsall', 'constituency-label': 'Walsall North', + 'posttown': 'WALSALL', 'construction-age-band': 'England and Wales: 1950-1966', + 'lodgement-datetime': '2022-10-26 13:46:05', 'tenure': 'Owner-occupied', + 'fixed-lighting-outlets-count': 10.0, 'low-energy-fixed-light-count': None, 'uprn': 100071089116, + 'uprn-source': 'Energy Assessor', 'sheating-energy-eff': None, 'sheating-env-eff': None + }, + "heating_recommendation_descriptions": [ + 'Upgrade to a new condensing boiler upgrade heating controls to Room thermostat, programmer and TRVs', + 'Upgrade to a new condensing boiler Upgrade heating controls to Smart Thermostats, room sensors and smart ' + 'radiator valves (time & temperature zone control)', + 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' + 'Storage Heater Controls', + ], + "heating_controls_recommendation_descriptions": [ + 'Upgrade to a new condensing boiler upgrade heating controls to Room thermostat, programmer and TRVs', + 'Upgrade to a new condensing boiler Upgrade heating controls to Smart Thermostats, room sensors and smart ' + 'radiator valves (time & temperature zone control)', + 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' + 'Storage Heater Controls', + 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' + 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' + 'scheme grant' + ], + "notes": "This has a form of assumed electric heating and has a mains connection so we recommend HHR, boiler" + "upgrade and ASHP" } ] @@ -509,7 +671,11 @@ print(eg["mainheatcont-description"]) ### We also use the Midlands EPC F/G portfolio to get examples to create tests completed_descriptions = [ - "Portable electric heaters assumed for most rooms" + "Portable electric heaters assumed for most rooms", + "Boiler and radiators, oil", + "Boiler and radiators, mains gas", + "Room heaters, mains gas", + "No system present: electric heaters assumed" ] portfolio = pd.read_excel( @@ -517,8 +683,9 @@ portfolio = pd.read_excel( ) portfolio.columns = [c.replace("_", "-").lower() for c in portfolio.columns] portfolio = portfolio[~portfolio["mainheat-description"].isin(completed_descriptions)] +portfolio["mainheat-description"].value_counts() eg = portfolio[ - (portfolio["mainheat-description"] == "Boiler and radiators, oil") + (portfolio["mainheat-description"] == "Portable electric heaters assumed for most rooms, Room heaters, electric") ].sample(1) eg = eg.squeeze().to_dict() From e31adcb55c48f66195a9aa31a532a48ff4183a61 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 16 Sep 2024 14:50:52 +0100 Subject: [PATCH 011/166] Added addition mainheat unit tests --- recommendations/HeatingRecommender.py | 3 +- .../test_data/heating_recommendations_data.py | 56 ++++++++++++++++++- 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index 5d68ebcf..901add15 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -604,7 +604,8 @@ class HeatingRecommender: # We check the existing heating system and controls if ( self.property.main_heating["has_electric_storage_heaters"] and - self.property.main_heating_controls["charging_system"] in ["automatic charge control"] + self.property.main_heating_controls["charging_system"] in + ["automatic charge control", "manual charge control"] ): description += (" The current electric heaters may be retrofit with high heat retention storage controls" " however this is dependent on the existing system and may not be possible.") diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index eec7703e..5b2b4aa0 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -626,6 +626,57 @@ testing_examples = [ ], "notes": "This has a form of assumed electric heating and has a mains connection so we recommend HHR, boiler" "upgrade and ASHP" + }, + { + "epc": { + 'lmk-key': '594183609042011021816283787499688', 'address1': '96, Richmond Road', 'address2': None, + 'address3': None, 'postcode': 'DE23 8PX', 'building-reference-number': 54104868, + 'current-energy-rating': 'F', 'potential-energy-rating': 'F', 'current-energy-efficiency': 30, + 'potential-energy-efficiency': 31, 'property-type': 'House', 'built-form': 'Mid-Terrace', + 'inspection-date': '2011-02-18', 'local-authority': 'E06000015', 'constituency': 'E14000663', + 'county': None, + 'lodgement-date': '2011-02-18', 'transaction-type': 'rental (social)', 'environment-impact-current': 25, + 'environment-impact-potential': 26, 'energy-consumption-current': 709, 'energy-consumption-potential': 693, + 'co2-emissions-current': 8.7, 'co2-emiss-curr-per-floor-area': 107, 'co2-emissions-potential': 8.5, + 'lighting-cost-current': 56, 'lighting-cost-potential': 56, 'heating-cost-current': 1118, + 'heating-cost-potential': 1089, 'hot-water-cost-current': 164, 'hot-water-cost-potential': 164, + 'total-floor-area': 80.98, 'energy-tariff': 'dual', 'mains-gas-flag': 'Y', 'floor-level': 'NO DATA!', + 'flat-top-storey': None, 'flat-storey-count': None, 'main-heating-controls': 2401.0, + 'multi-glaze-proportion': 100.0, 'glazed-type': 'double glazing installed before 2002', + 'glazed-area': 'Normal', 'extension-count': 1, 'number-habitable-rooms': 4, 'number-heated-rooms': 4, + 'low-energy-lighting': 88, 'number-open-fireplaces': 0, + 'hotwater-description': 'Electric immersion, off-peak', 'hot-water-energy-eff': 'Poor', + 'hot-water-env-eff': 'Poor', 'floor-description': 'Suspended, no insulation (assumed)', + 'floor-energy-eff': None, 'windows-description': 'Fully double glazed', 'windows-energy-eff': 'Average', + 'windows-env-eff': 'Average', 'walls-description': 'Solid brick, as built, no insulation (assumed)', + 'walls-energy-eff': 'Very Poor', 'walls-env-eff': 'Very Poor', + 'secondheat-description': 'Room heaters, electric', 'roof-description': 'Pitched, 150 mm loft insulation', + 'roof-energy-eff': 'Good', 'roof-env-eff': 'Good', 'mainheat-description': 'Electric storage heaters', + 'mainheat-energy-eff': 'Poor', 'mainheat-env-eff': 'Very Poor', + 'mainheatcont-description': 'Manual charge control', 'mainheatc-energy-eff': 'Poor', + 'mainheatc-env-eff': 'Poor', 'lighting-description': 'Low energy lighting in 88% of fixed outlets', + 'lighting-energy-eff': 'Very Good', 'lighting-env-eff': 'Very Good', + 'main-fuel': 'electricity - this is for backwards compatibility only and should not be used', + 'wind-turbine-count': 0, 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, + 'floor-height': 2.72, 'photo-supply': 0.0, 'solar-water-heating-flag': 'N', + 'mechanical-ventilation': 'natural', 'address': '96, Richmond Road', 'local-authority-label': 'Derby', + 'constituency-label': 'Derby South', 'posttown': 'DERBY', + 'construction-age-band': 'England and Wales: 1900-1929', + 'lodgement-datetime': '2011-02-18 16:28:37', 'tenure': 'rental (social)', + 'fixed-lighting-outlets-count': None, 'low-energy-fixed-light-count': None, 'uprn': 100030352255, + 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None + }, + "heating_recommendation_descriptions": [ + 'Upgrade to a new condensing boiler Upgrade heating controls to Room thermostat, programmer and TRVs', + 'Upgrade to a new condensing boiler Upgrade heating controls to Smart Thermostats, room sensors and smart ' + 'radiator valves (time & temperature zone control)', + 'Install high heat retention electric storage heaters. The current electric heaters may be retrofit with ' + 'high heat retention storage controls however this is dependent on the existing system and may not be ' + 'possible. Upgrade heating controls to High Heat Retention Storage Heater Controls' + ], + "heating_controls_recommendation_descriptions": [], + "notes": "This property already has storage heaters with manual charge control. The home is mid terrace so" + "the ashp is not suitable" } ] @@ -675,7 +726,8 @@ completed_descriptions = [ "Boiler and radiators, oil", "Boiler and radiators, mains gas", "Room heaters, mains gas", - "No system present: electric heaters assumed" + "No system present: electric heaters assumed", + "Room heaters, electric", ] portfolio = pd.read_excel( @@ -686,6 +738,6 @@ portfolio = portfolio[~portfolio["mainheat-description"].isin(completed_descript portfolio["mainheat-description"].value_counts() eg = portfolio[ - (portfolio["mainheat-description"] == "Portable electric heaters assumed for most rooms, Room heaters, electric") + (portfolio["mainheat-description"] == "Electric storage heaters") ].sample(1) eg = eg.squeeze().to_dict() From 9156349ee592cfa3adde1c2b03d94d720338eab4 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 16 Sep 2024 17:34:28 +0100 Subject: [PATCH 012/166] Increasing heating system reommendations --- recommendations/HeatingRecommender.py | 9 +- .../test_data/heating_recommendations_data.py | 104 +++++++++++++++++- 2 files changed, 111 insertions(+), 2 deletions(-) diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index 901add15..5f632567 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -23,6 +23,8 @@ class HeatingRecommender: # These are descriptions for boilers that are not gas boilers NON_GAS_BOILERS = [ "Boiler and radiators, oil", + "Boiler and radiators, lpg", + "Boiler and radiators, electric" ] def __init__(self, property_instance: Property): @@ -40,8 +42,13 @@ class HeatingRecommender: :return: """ + # We can also recommend hhr if the property doesn't have a mains has connection + no_mains = not self.property.data["mains-gas-flag"] + + hhr_suitable = no_mains or self.has_electric_heating_description + return ( - self.has_electric_heating_description and (not ashp_only_heating_recommendation) and + hhr_suitable and (not ashp_only_heating_recommendation) and ("high_heat_retention_storage_heater" in measures) ) diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index 5b2b4aa0..bebbfec9 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -677,6 +677,106 @@ testing_examples = [ "heating_controls_recommendation_descriptions": [], "notes": "This property already has storage heaters with manual charge control. The home is mid terrace so" "the ashp is not suitable" + }, + { + "epc": { + 'lmk-key': '206883665252008121709314507989954', 'address1': '2 Upper Gardens Hazelhurst', + 'address2': 'Bishopswood', 'address3': None, 'postcode': 'HR9 5QX', 'building-reference-number': 9443575568, + 'current-energy-rating': 'F', 'potential-energy-rating': 'E', 'current-energy-efficiency': 32, + 'potential-energy-efficiency': 42, 'property-type': 'Bungalow', 'built-form': 'Semi-Detached', + 'inspection-date': '2008-12-11', 'local-authority': 'E06000019', 'constituency': 'E14000743', + 'county': None, + 'lodgement-date': '2008-12-17', 'transaction-type': 'rental (private)', 'environment-impact-current': 54, + 'environment-impact-potential': 64, 'energy-consumption-current': 290, 'energy-consumption-potential': 231, + 'co2-emissions-current': 3.7, 'co2-emiss-curr-per-floor-area': 58, 'co2-emissions-potential': 2.9, + 'lighting-cost-current': 39, 'lighting-cost-potential': 39, 'heating-cost-current': 654, + 'heating-cost-potential': 515, 'hot-water-cost-current': 204, 'hot-water-cost-potential': 169, + 'total-floor-area': 63.94, 'energy-tariff': 'dual', 'mains-gas-flag': 'N', 'floor-level': 'NO DATA!', + 'flat-top-storey': None, 'flat-storey-count': None, 'main-heating-controls': 2106.0, + 'multi-glaze-proportion': 100.0, 'glazed-type': 'double glazing installed before 2002', + 'glazed-area': 'Normal', 'extension-count': 0, 'number-habitable-rooms': 3, 'number-heated-rooms': 3, + 'low-energy-lighting': 75, 'number-open-fireplaces': 0, 'hotwater-description': 'From main system', + 'hot-water-energy-eff': 'Poor', 'hot-water-env-eff': 'Average', + 'floor-description': 'Solid, no insulation (assumed)', 'floor-energy-eff': None, + 'windows-description': 'Fully double glazed', 'windows-energy-eff': 'Average', 'windows-env-eff': 'Average', + 'walls-description': 'Cavity wall, as built, insulated (assumed)', 'walls-energy-eff': 'Good', + 'walls-env-eff': 'Good', 'secondheat-description': 'Room heaters, electric', + 'roof-description': 'Pitched, 100 mm loft insulation', 'roof-energy-eff': 'Average', + 'roof-env-eff': 'Average', 'mainheat-description': 'Boiler and radiators, LPG', + 'mainheat-energy-eff': 'Poor', 'mainheat-env-eff': 'Average', + 'mainheatcont-description': 'Programmer, room thermostat and TRVs', 'mainheatc-energy-eff': 'Average', + 'mainheatc-env-eff': 'Average', 'lighting-description': 'Low energy lighting in 75% of fixed outlets', + 'lighting-energy-eff': 'Very Good', 'lighting-env-eff': 'Very Good', + 'main-fuel': 'LPG - this is for backwards compatibility only and should not be used', + 'wind-turbine-count': 0, 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, + 'floor-height': 2.4, 'photo-supply': 0.0, 'solar-water-heating-flag': 'N', + 'mechanical-ventilation': 'natural', 'address': '2 Upper Gardens Hazelhurst, Bishopswood', + 'local-authority-label': 'Herefordshire, County of', + 'constituency-label': 'Hereford and South Herefordshire', 'posttown': 'ROSS-ON-WYE', + 'construction-age-band': 'England and Wales: 1983-1990', + 'lodgement-datetime': '2008-12-17 09:31:45', 'tenure': 'rental (private)', + 'fixed-lighting-outlets-count': None, 'low-energy-fixed-light-count': None, 'uprn': 10009573249, + 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None + }, + "heating_recommendation_descriptions": [ + 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' + 'Storage Heater Controls', + 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' + 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' + 'scheme grant' + ], + "heating_controls_recommendation_descriptions": [], + "notes": "This property has an LFG boiler but it doesn't have a mains gas connection so we can only recommend" + "an air source heat pump and hhr storage" + }, + { + "epc": { + 'lmk-key': '749e6ae968c0d5c6491ee1ee82a591733568c981e08ffde7e92b6e4172f3fb0f', + 'address1': '10 Small Holdings', 'address2': 'Stoneleigh Road', 'address3': 'Baginton', + 'postcode': 'CV8 3BA', 'building-reference-number': 10005704940, 'current-energy-rating': 'G', + 'potential-energy-rating': 'E', 'current-energy-efficiency': 12, 'potential-energy-efficiency': 46, + 'property-type': 'House', 'built-form': 'Semi-Detached', 'inspection-date': '2024-03-04', + 'local-authority': 'E07000222', 'constituency': 'E14000767', 'county': 'Warwickshire', + 'lodgement-date': '2024-03-08', 'transaction-type': 'not sale or rental', 'environment-impact-current': 14, + 'environment-impact-potential': 39, 'energy-consumption-current': 728, 'energy-consumption-potential': 379, + 'co2-emissions-current': 11.0, 'co2-emiss-curr-per-floor-area': 137, 'co2-emissions-potential': 5.9, + 'lighting-cost-current': 169, 'lighting-cost-potential': 169, 'heating-cost-current': 4435, + 'heating-cost-potential': 2718, 'hot-water-cost-current': 405, 'hot-water-cost-potential': 227, + 'total-floor-area': 81.0, 'energy-tariff': 'dual', 'mains-gas-flag': 'N', 'floor-level': None, + 'flat-top-storey': None, 'flat-storey-count': None, 'main-heating-controls': None, + 'multi-glaze-proportion': 100.0, 'glazed-type': 'not defined', 'glazed-area': 'Much More Than Typical', + 'extension-count': 0, 'number-habitable-rooms': 4, 'number-heated-rooms': 4, 'low-energy-lighting': 63, + 'number-open-fireplaces': 1, 'hotwater-description': 'From main system', 'hot-water-energy-eff': 'Poor', + 'hot-water-env-eff': 'Poor', 'floor-description': 'Solid, no insulation (assumed)', + 'floor-energy-eff': None, + 'windows-description': 'Fully double glazed', 'windows-energy-eff': 'Average', 'windows-env-eff': 'Average', + 'walls-description': 'Solid brick, as built, no insulation (assumed)', 'walls-energy-eff': 'Very Poor', + 'walls-env-eff': 'Very Poor', 'secondheat-description': 'Room heaters, smokeless fuel', + 'roof-description': 'Pitched, no insulation (assumed)', 'roof-energy-eff': 'Very Poor', + 'roof-env-eff': 'Very Poor', 'mainheat-description': 'Boiler and radiators, electric', + 'mainheat-energy-eff': 'Very Poor', 'mainheat-env-eff': 'Poor', + 'mainheatcont-description': 'Programmer, room thermostat and TRVs', 'mainheatc-energy-eff': 'Good', + 'mainheatc-env-eff': 'Good', 'lighting-description': 'Low energy lighting in 63% of fixed outlets', + 'lighting-energy-eff': 'Good', 'lighting-env-eff': 'Good', 'main-fuel': 'electricity (not community)', + 'wind-turbine-count': 0, 'heat-loss-corridor': None, 'unheated-corridor-length': None, 'floor-height': 2.31, + 'photo-supply': 0.0, 'solar-water-heating-flag': 'N', 'mechanical-ventilation': 'natural', + 'address': '10 Small Holdings, Stoneleigh Road, Baginton', 'local-authority-label': 'Warwick', + 'constituency-label': 'Kenilworth and Southam', 'posttown': 'COVENTRY', + 'construction-age-band': 'England and Wales: 1930-1949', + 'lodgement-datetime': '2024-03-08 10:43:35', 'tenure': 'Rented (social)', + 'fixed-lighting-outlets-count': 8.0, 'low-energy-fixed-light-count': None, 'uprn': 10013181470, + 'uprn-source': 'Energy Assessor', 'sheating-energy-eff': None, 'sheating-env-eff': None + }, + "heating_recommendation_descriptions": [ + 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' + 'Storage Heater Controls', + 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' + 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' + 'scheme grant' + ], + "heating_controls_recommendation_descriptions": [], + "notes": "This property has electric boilers in place, but does not have a mains connection so we don't " + "recommend a boiler upgrade. We recommend HHR and ASHP" } ] @@ -728,6 +828,8 @@ completed_descriptions = [ "Room heaters, mains gas", "No system present: electric heaters assumed", "Room heaters, electric", + "Electric storage heaters", + "Boiler and radiators, LPG", ] portfolio = pd.read_excel( @@ -738,6 +840,6 @@ portfolio = portfolio[~portfolio["mainheat-description"].isin(completed_descript portfolio["mainheat-description"].value_counts() eg = portfolio[ - (portfolio["mainheat-description"] == "Electric storage heaters") + (portfolio["mainheat-description"] == "Boiler and radiators, electric") ].sample(1) eg = eg.squeeze().to_dict() From c086ecae5285836858bc941482d3c9a2e7319798 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 16 Sep 2024 19:32:02 +0100 Subject: [PATCH 013/166] Additional heating recommendations covered and tests added --- etl/customers/aiha/epc_data_pull.py | 20 +++- recommendations/HeatingRecommender.py | 17 +-- .../test_data/heating_recommendations_data.py | 106 +++++++++++++++++- 3 files changed, 129 insertions(+), 14 deletions(-) diff --git a/etl/customers/aiha/epc_data_pull.py b/etl/customers/aiha/epc_data_pull.py index f7f4631c..8259578d 100644 --- a/etl/customers/aiha/epc_data_pull.py +++ b/etl/customers/aiha/epc_data_pull.py @@ -676,6 +676,24 @@ def app(): "Manchester", archetyping_data["Location"] ) + # We fix the location for B 80 Bethune Road + archetyping_data["Location"] = np.where( + ( + archetyping_data["row_id"].isin( + data[ + data["Street address"] == "80 Bethune Road" + ]["row_id"].values.tolist() + ) + ) & ( + archetyping_data["row_id"].isin( + data[ + data["Address letter or number"] == "B" + ]["row_id"].values.tolist() + ) + ), + "London", + archetyping_data["Location"] + ) # Hackney 73 - London # Southend-on-Sea 6 - Southend @@ -691,7 +709,7 @@ def app(): 'Current heating system type', 'Wall type', 'Roof type', - "Location", + # "Location", # 'current-energy-rating', 'property-type-reduced', 'built-form-reduced', 'is_cavity_wall', # 'is_solid_brick_wall', 'is_system_built_wall', 'is_timber_frame_wall', 'is_as_built', # 'is_solid', 'is_roof_room', diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index 5f632567..64a2e285 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -20,13 +20,6 @@ class HeatingRecommender: high_heat_retention_contols_desc = "Controls for high heat retention storage heaters" - # These are descriptions for boilers that are not gas boilers - NON_GAS_BOILERS = [ - "Boiler and radiators, oil", - "Boiler and radiators, lpg", - "Boiler and radiators, electric" - ] - def __init__(self, property_instance: Property): self.property = property_instance self.costs = Costs(self.property) @@ -59,7 +52,8 @@ class HeatingRecommender: """ # 1) if the property has mains heating with boiler and radiators, we recommend optimal heating controls - has_boiler = self.property.main_heating["clean_description"] in ["Boiler and radiators, mains gas"] + # If it's NOT a gas boiler, we'll potentially recommend a boiler + has_gas_boiler = self.property.main_heating["has_boiler"] and self.property.main_heating["has_mains_gas"] # 2) If the property doesn't have a heating system, but it has access to the mains gas no_heating_has_mains = self.property.main_heating["clean_description"] in [ @@ -81,15 +75,16 @@ class HeatingRecommender: self.property.data["mains-gas-flag"] ) - # The next condition is if the home has a non-gas boiler, such as an oil boiler + # The next condition is if the home has a non-gas boiler, such as an oil boiler, with a mains gas connection non_gas_boiler = ( - self.property.main_heating["clean_description"] in self.NON_GAS_BOILERS and + self.property.main_heating["has_boiler"] and + not self.property.main_heating["has_mains_gas"] and self.property.data["mains-gas-flag"] ) is_valid = ( ( - has_boiler or + has_gas_boiler or no_heating_has_mains or electic_heating_has_mains or has_room_heaters or diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index bebbfec9..2fc47e13 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -777,6 +777,105 @@ testing_examples = [ "heating_controls_recommendation_descriptions": [], "notes": "This property has electric boilers in place, but does not have a mains connection so we don't " "recommend a boiler upgrade. We recommend HHR and ASHP" + }, + { + "epc": { + 'lmk-key': '683441359142011092814474999092088', 'address1': '20, Haybridge Avenue', 'address2': 'Hadley', + 'address3': None, 'postcode': 'TF1 5JR', 'building-reference-number': 3100250968, + 'current-energy-rating': 'F', 'potential-energy-rating': 'E', 'current-energy-efficiency': 34, + 'potential-energy-efficiency': 41, 'property-type': 'House', 'built-form': 'Semi-Detached', + 'inspection-date': '2011-09-28', 'local-authority': 'E06000020', 'constituency': 'E14000992', + 'county': None, + 'lodgement-date': '2011-09-28', 'transaction-type': 'rental (social)', 'environment-impact-current': 29, + 'environment-impact-potential': 34, 'energy-consumption-current': 495, 'energy-consumption-potential': 435, + 'co2-emissions-current': 8.3, 'co2-emiss-curr-per-floor-area': 97, 'co2-emissions-potential': 7.3, + 'lighting-cost-current': 61, 'lighting-cost-potential': 45, 'heating-cost-current': 1273, + 'heating-cost-potential': 1101, 'hot-water-cost-current': 214, 'hot-water-cost-potential': 214, + 'total-floor-area': 85.1, 'energy-tariff': 'Single', 'mains-gas-flag': 'N', 'floor-level': 'NODATA!', + 'flat-top-storey': None, 'flat-storey-count': None, 'main-heating-controls': 2101.0, + 'multi-glaze-proportion': 0.0, 'glazed-type': 'not defined', 'glazed-area': 'Normal', 'extension-count': 1, + 'number-habitable-rooms': 5, 'number-heated-rooms': 5, 'low-energy-lighting': 64, + 'number-open-fireplaces': 1, 'hotwater-description': 'From main system, no cylinder thermostat', + 'hot-water-energy-eff': 'Poor', 'hot-water-env-eff': 'Poor', + 'floor-description': 'Solid, no insulation (assumed)', 'floor-energy-eff': None, + 'windows-description': 'Single glazed', 'windows-energy-eff': 'Very Poor', 'windows-env-eff': 'Very Poor', + 'walls-description': 'Solid brick, as built, no insulation (assumed)', 'walls-energy-eff': 'Very Poor', + 'walls-env-eff': 'Very Poor', 'secondheat-description': 'Room heaters, dual fuel (mineral and wood)', + 'roof-description': 'Pitched, 250mm loft insulation', 'roof-energy-eff': 'Good', 'roof-env-eff': 'Good', + 'mainheat-description': 'Boiler and radiators, dual fuel (mineral and wood)', + 'mainheat-energy-eff': 'Average', 'mainheat-env-eff': 'Average', + 'mainheatcont-description': 'No time or thermostatic control of room temperature', + 'mainheatc-energy-eff': 'Very Poor', 'mainheatc-env-eff': 'Very Poor', + 'lighting-description': 'Low energy lighting in 64% of fixed outlets', 'lighting-energy-eff': 'Good', + 'lighting-env-eff': 'Good', 'main-fuel': 'dual fuel - mineral + wood', 'wind-turbine-count': 0, + 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, 'floor-height': 2.5, + 'photo-supply': 0.0, + 'solar-water-heating-flag': None, 'mechanical-ventilation': 'natural', + 'address': '20, Haybridge Avenue, Hadley', 'local-authority-label': 'Telford and Wrekin', + 'constituency-label': 'The Wrekin', 'posttown': 'TELFORD', + 'construction-age-band': 'England and Wales: 1900-1929', + 'lodgement-datetime': '2011-09-28 14:47:49', 'tenure': 'rental (social)', + 'fixed-lighting-outlets-count': 11.0, 'low-energy-fixed-light-count': 7.0, 'uprn': 452047507, + 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None + }, + "heating_recommendation_descriptions": [ + 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' + 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' + 'scheme grant', + 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' + 'Storage Heater Controls' + ], + "heating_controls_recommendation_descriptions": [], + "notes": "This property has a dual fuel boiler and no mains gas connection. We recommend ASHP and HHR, but" + "no gas condensing boiler" + }, + { + "epc": { + 'lmk-key': 'ba1de1b99f30546d7c6654af44c74fa4511611f9283502b77efb825d8566023c', + 'address1': '19 DORSET STREET', 'address2': 'DERBY', 'address3': None, 'postcode': 'DE21 6BE', + 'building-reference-number': 10000116666, 'current-energy-rating': 'F', 'potential-energy-rating': 'C', + 'current-energy-efficiency': 29, 'potential-energy-efficiency': 78, 'property-type': 'House', + 'built-form': 'Semi-Detached', 'inspection-date': '2021-01-09', 'local-authority': 'E06000015', + 'constituency': 'E14000662', 'county': None, 'lodgement-date': '2021-01-11', + 'transaction-type': 'ECO assessment', 'environment-impact-current': 1, 'environment-impact-potential': 40, + 'energy-consumption-current': 532, 'energy-consumption-potential': 153, 'co2-emissions-current': 14.0, + 'co2-emiss-curr-per-floor-area': 198, 'co2-emissions-potential': 5.1, 'lighting-cost-current': 105, + 'lighting-cost-potential': 60, 'heating-cost-current': 1361, 'heating-cost-potential': 545, + 'hot-water-cost-current': 242, 'hot-water-cost-potential': 132, 'total-floor-area': 72.0, + 'energy-tariff': 'off-peak 7 hour', 'mains-gas-flag': 'N', 'floor-level': None, 'flat-top-storey': None, + 'flat-storey-count': None, 'main-heating-controls': None, 'multi-glaze-proportion': 100.0, + 'glazed-type': 'double glazing, unknown install date', 'glazed-area': 'Normal', 'extension-count': 0, + 'number-habitable-rooms': 4, 'number-heated-rooms': 4, 'low-energy-lighting': 25, + 'number-open-fireplaces': 1, 'hotwater-description': 'From main system', 'hot-water-energy-eff': 'Average', + 'hot-water-env-eff': 'Very Poor', 'floor-description': 'Solid, no insulation (assumed)', + 'floor-energy-eff': None, 'windows-description': 'Fully double glazed', 'windows-energy-eff': 'Average', + 'windows-env-eff': 'Average', 'walls-description': 'Solid brick, as built, no insulation (assumed)', + 'walls-energy-eff': 'Very Poor', 'walls-env-eff': 'Very Poor', 'secondheat-description': 'None', + 'roof-description': 'Pitched, 200 mm loft insulation', 'roof-energy-eff': 'Good', 'roof-env-eff': 'Good', + 'mainheat-description': 'Boiler and radiators, coal', 'mainheat-energy-eff': 'Poor', + 'mainheat-env-eff': 'Very Poor', + 'mainheatcont-description': 'No time or thermostatic control of room temperature', + 'mainheatc-energy-eff': 'Very Poor', 'mainheatc-env-eff': 'Very Poor', + 'lighting-description': 'Low energy lighting in 25% of fixed outlets', 'lighting-energy-eff': 'Average', + 'lighting-env-eff': 'Average', 'main-fuel': 'house coal (not community)', 'wind-turbine-count': 0, + 'heat-loss-corridor': None, 'unheated-corridor-length': None, 'floor-height': 2.37, 'photo-supply': 0.0, + 'solar-water-heating-flag': 'N', 'mechanical-ventilation': 'natural', 'address': '19 DORSET STREET, DERBY', + 'local-authority-label': 'Derby', 'constituency-label': 'Derby North', 'posttown': 'DERBY', + 'construction-age-band': 'England and Wales: 1950-1966', + 'lodgement-datetime': '2021-01-11 00:00:00', 'tenure': 'Owner-occupied', + 'fixed-lighting-outlets-count': 16.0, 'low-energy-fixed-light-count': 4.0, 'uprn': 100030309413, + 'uprn-source': 'Energy Assessor', 'sheating-energy-eff': None, 'sheating-env-eff': None + }, + "heating_recommendation_descriptions": [ + 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' + 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' + 'scheme grant', + 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' + 'Storage Heater Controls' + ], + "heating_controls_recommendation_descriptions": [], + "notes": "This property has a coal boiler and no mains gas connection. We recommend ASHP and HHR, but" + "no gas condensing boiler" } ] @@ -830,6 +929,8 @@ completed_descriptions = [ "Room heaters, electric", "Electric storage heaters", "Boiler and radiators, LPG", + "Boiler and radiators, electric", + "Boiler and radiators, dual fuel (mineral and wood)" ] portfolio = pd.read_excel( @@ -837,9 +938,10 @@ portfolio = pd.read_excel( ) portfolio.columns = [c.replace("_", "-").lower() for c in portfolio.columns] portfolio = portfolio[~portfolio["mainheat-description"].isin(completed_descriptions)] -portfolio["mainheat-description"].value_counts() +print(portfolio["mainheat-description"].value_counts()) eg = portfolio[ - (portfolio["mainheat-description"] == "Boiler and radiators, electric") + (portfolio["mainheat-description"] == "Boiler and radiators, coal") ].sample(1) eg = eg.squeeze().to_dict() +print(eg) From 64d19defbb7c3e6c9aff990ba533bffa26cfd787 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 16 Sep 2024 19:36:44 +0100 Subject: [PATCH 014/166] more heating tests --- .../test_data/heating_recommendations_data.py | 112 +++++++++++++++++- 1 file changed, 110 insertions(+), 2 deletions(-) diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index 2fc47e13..982eb280 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -876,6 +876,108 @@ testing_examples = [ "heating_controls_recommendation_descriptions": [], "notes": "This property has a coal boiler and no mains gas connection. We recommend ASHP and HHR, but" "no gas condensing boiler" + }, + { + "epc": { + 'lmk-key': '1139832199022019020816400153188351', 'address1': '1 Green Gates', 'address2': 'Bridstow', + 'address3': None, 'postcode': 'HR9 6QJ', 'building-reference-number': 5576913278, + 'current-energy-rating': 'F', 'potential-energy-rating': 'A', 'current-energy-efficiency': 37, + 'potential-energy-efficiency': 93, 'property-type': 'House', 'built-form': 'Semi-Detached', + 'inspection-date': '2019-02-08', 'local-authority': 'E06000019', 'constituency': 'E14000743', + 'county': None, + 'lodgement-date': '2019-02-08', 'transaction-type': 'ECO assessment', 'environment-impact-current': 11, + 'environment-impact-potential': 115, 'energy-consumption-current': 377, 'energy-consumption-potential': 28, + 'co2-emissions-current': 14.0, 'co2-emiss-curr-per-floor-area': 129, 'co2-emissions-potential': -1.9, + 'lighting-cost-current': 75, 'lighting-cost-potential': 75, 'heating-cost-current': 1512, + 'heating-cost-potential': 700, 'hot-water-cost-current': 258, 'hot-water-cost-potential': 113, + 'total-floor-area': 111.0, 'energy-tariff': 'Single', 'mains-gas-flag': 'N', 'floor-level': 'NODATA!', + 'flat-top-storey': None, 'flat-storey-count': None, 'main-heating-controls': 2101.0, + 'multi-glaze-proportion': 100.0, 'glazed-type': 'double glazing, unknown install date', + 'glazed-area': 'Normal', 'extension-count': 1, 'number-habitable-rooms': 5, 'number-heated-rooms': 5, + 'low-energy-lighting': 100, 'number-open-fireplaces': 0, 'hotwater-description': 'From main system', + 'hot-water-energy-eff': 'Average', 'hot-water-env-eff': 'Very Poor', + 'floor-description': 'Solid, no insulation (assumed)', 'floor-energy-eff': 'NO DATA!', + 'windows-description': 'Fully double glazed', 'windows-energy-eff': 'Average', 'windows-env-eff': 'Average', + 'walls-description': 'Cavity wall, as built, no insulation (assumed)', 'walls-energy-eff': 'Poor', + 'walls-env-eff': 'Poor', 'secondheat-description': 'None', + 'roof-description': 'Pitched, 270 mm loft insulation', 'roof-energy-eff': 'Good', 'roof-env-eff': 'Good', + 'mainheat-description': 'Boiler and radiators, smokeless fuel', 'mainheat-energy-eff': 'Poor', + 'mainheat-env-eff': 'Very Poor', + 'mainheatcont-description': 'No time or thermostatic control of room temperature', + 'mainheatc-energy-eff': 'Very Poor', 'mainheatc-env-eff': 'Very Poor', + 'lighting-description': 'Low energy lighting in all fixed outlets', 'lighting-energy-eff': 'Very Good', + 'lighting-env-eff': 'Very Good', 'main-fuel': 'smokeless coal', 'wind-turbine-count': 0, + 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, 'floor-height': None, + 'photo-supply': None, + 'solar-water-heating-flag': 'N', 'mechanical-ventilation': 'natural', 'address': '1 Green Gates, Bridstow', + 'local-authority-label': 'Herefordshire, County of', + 'constituency-label': 'Hereford and South Herefordshire', 'posttown': 'ROSS-ON-WYE', + 'construction-age-band': 'England and Wales: 1950-1966', + 'lodgement-datetime': '2019-02-08 16:40:01', 'tenure': 'rental (social)', + 'fixed-lighting-outlets-count': None, 'low-energy-fixed-light-count': None, 'uprn': 10007366417, + 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None + }, + "heating_recommendation_descriptions": [ + 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' + 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' + 'scheme grant', + 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' + 'Storage Heater Controls' + + ], + "heating_controls_recommendation_descriptions": [], + "notes": "This property has a smokeless fuel boiler and no mains gas connection. We recommend ASHP and HHR, but" + "no gas condensing boiler" + }, + { + "epc": { + 'lmk-key': '1253529329242015021115045635159198', 'address1': '143', 'address2': 'Shortheath', + 'address3': None, 'postcode': 'DE12 6BL', 'building-reference-number': 212621378, + 'current-energy-rating': 'F', 'potential-energy-rating': 'D', 'current-energy-efficiency': 22, + 'potential-energy-efficiency': 59, 'property-type': 'House', 'built-form': 'Semi-Detached', + 'inspection-date': '2015-02-11', 'local-authority': 'E07000039', 'constituency': 'E14000935', + 'county': 'Derbyshire', 'lodgement-date': '2015-02-11', 'transaction-type': 'RHI application', + 'environment-impact-current': 71, 'environment-impact-potential': 91, 'energy-consumption-current': 500, + 'energy-consumption-potential': 233, 'co2-emissions-current': 3.0, 'co2-emiss-curr-per-floor-area': 31, + 'co2-emissions-potential': 0.8, 'lighting-cost-current': 104, 'lighting-cost-potential': 59, + 'heating-cost-current': 1746, 'heating-cost-potential': 1010, 'hot-water-cost-current': 253, + 'hot-water-cost-potential': 151, 'total-floor-area': 96.0, 'energy-tariff': 'Single', 'mains-gas-flag': 'N', + 'floor-level': 'NODATA!', 'flat-top-storey': None, 'flat-storey-count': None, + 'main-heating-controls': 2111.0, + 'multi-glaze-proportion': 100.0, 'glazed-type': 'double glazing, unknown install date', + 'glazed-area': 'Normal', 'extension-count': 2, 'number-habitable-rooms': 5, 'number-heated-rooms': 5, + 'low-energy-lighting': 23, 'number-open-fireplaces': 1, 'hotwater-description': 'From main system', + 'hot-water-energy-eff': 'Poor', 'hot-water-env-eff': 'Very Good', + 'floor-description': 'Suspended, no insulation (assumed)', 'floor-energy-eff': 'NO DATA!', + 'windows-description': 'Fully double glazed', 'windows-energy-eff': 'Average', 'windows-env-eff': 'Average', + 'walls-description': 'Solid brick, as built, no insulation (assumed)', 'walls-energy-eff': 'Very Poor', + 'walls-env-eff': 'Very Poor', 'secondheat-description': 'Room heaters, dual fuel (mineral and wood)', + 'roof-description': 'Pitched, 250 mm loft insulation', 'roof-energy-eff': 'Good', 'roof-env-eff': 'Good', + 'mainheat-description': 'Boiler and radiators, wood pellets', 'mainheat-energy-eff': 'Poor', + 'mainheat-env-eff': 'Very Good', 'mainheatcont-description': 'TRVs and bypass', + 'mainheatc-energy-eff': 'Average', 'mainheatc-env-eff': 'Average', + 'lighting-description': 'Low energy lighting in 23% of fixed outlets', 'lighting-energy-eff': 'Poor', + 'lighting-env-eff': 'Poor', 'main-fuel': 'bulk wood pellets', 'wind-turbine-count': 0, + 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, 'floor-height': None, + 'photo-supply': None, + 'solar-water-heating-flag': 'N', 'mechanical-ventilation': 'natural', 'address': '143, Shortheath', + 'local-authority-label': 'South Derbyshire', 'constituency-label': 'South Derbyshire', + 'posttown': 'SWADLINCOTE', 'construction-age-band': 'England and Wales: 1900-1929', + 'lodgement-datetime': '2015-02-11 15:04:56', 'tenure': 'owner-occupied', + 'fixed-lighting-outlets-count': None, 'low-energy-fixed-light-count': None, 'uprn': 100030256931, + 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None + }, + "heating_recommendation_descriptions": [ + 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' + 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' + 'scheme grant', + 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' + 'Storage Heater Controls' + + ], + "heating_controls_recommendation_descriptions": [], + "notes": "This property has a wood pellets boiler and no mains gas connection. We recommend ASHP and HHR, but" + "no gas condensing boiler" } ] @@ -930,7 +1032,9 @@ completed_descriptions = [ "Electric storage heaters", "Boiler and radiators, LPG", "Boiler and radiators, electric", - "Boiler and radiators, dual fuel (mineral and wood)" + "Boiler and radiators, dual fuel (mineral and wood)", + "Boiler and radiators, coal", + "Boiler and radiators, smokeless fuel" ] portfolio = pd.read_excel( @@ -938,10 +1042,14 @@ portfolio = pd.read_excel( ) portfolio.columns = [c.replace("_", "-").lower() for c in portfolio.columns] portfolio = portfolio[~portfolio["mainheat-description"].isin(completed_descriptions)] +portfolio['sheating-energy-eff'] = None +portfolio['sheating-env-eff'] = None +portfolio["lodgement-datetime"] = portfolio["lodgement-datetime"].astype(str) + print(portfolio["mainheat-description"].value_counts()) eg = portfolio[ - (portfolio["mainheat-description"] == "Boiler and radiators, coal") + (portfolio["mainheat-description"] == "Boiler and radiators, wood pellets") ].sample(1) eg = eg.squeeze().to_dict() print(eg) From 942f1958db942513ea44e50e07191148fec6f38b Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 16 Sep 2024 19:42:30 +0100 Subject: [PATCH 015/166] another heating recommendation test --- recommendations/HeatingRecommender.py | 20 +++----- .../test_data/heating_recommendations_data.py | 51 ++++++++++++++++++- 2 files changed, 56 insertions(+), 15 deletions(-) diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index 64a2e285..8fbcec86 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -9,15 +9,6 @@ from recommendations.HeatingControlRecommender import HeatingControlRecommender class HeatingRecommender: - ASSUMED_ELECTRIC_HEATING = [ - "Portable electric heaters assumed for most rooms", - "No system present, electric heaters assumed" - ] - - ROOM_HEATERS_DESCRIPTIONS = [ - "Room heaters, mains gas", "Room heaters, electric", "Portable electric heaters assumed for most rooms", - ] - high_heat_retention_contols_desc = "Controls for high heat retention storage heaters" def __init__(self, property_instance: Property): @@ -62,7 +53,10 @@ class HeatingRecommender: # The property is using portable heaters and has access to gas mains has_room_heaters = ( - self.property.main_heating["clean_description"] in self.ROOM_HEATERS_DESCRIPTIONS and + ( + self.property.main_heating["has_room_heaters"] or + self.property.main_heating["has_portable_electric_heaters"] + ) and self.property.data["mains-gas-flag"] ) @@ -95,7 +89,7 @@ class HeatingRecommender: ("boiler_upgrade" in measures) ) - return is_valid, has_boiler + return is_valid, has_gas_boiler def recommend(self, has_cavity_or_loft_recommendations, phase=0, measures=None): """ @@ -138,14 +132,14 @@ class HeatingRecommender: # Recommend high heat retention storage heaters self.recommend_hhr_storage_heaters(phase=phase, system_change=True, heating_controls_only=False) - gas_boiler_suitable, has_boiler = self.is_boiler_upgrade_suitable( + gas_boiler_suitable, has_gas_boiler = self.is_boiler_upgrade_suitable( measures=measures, ashp_only_heating_recommendation=ashp_only_heating_recommendation ) if gas_boiler_suitable: # This indicates that the home previously did not have a boiler in place and so would require # an overhaul to the system - right now, this is all reasons, apart from if there is an existing boiler - system_change = not has_boiler + system_change = not has_gas_boiler exising_room_heaters = self.property.main_heating["clean_description"] in [ "Room heaters, electric", "Room heaters, mains gas" ] diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index 982eb280..c6751784 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -978,6 +978,52 @@ testing_examples = [ "heating_controls_recommendation_descriptions": [], "notes": "This property has a wood pellets boiler and no mains gas connection. We recommend ASHP and HHR, but" "no gas condensing boiler" + }, + { + "epc": { + 'lmk-key': '1125990659062017030307353552528423', 'address1': '3 Manor Farm Cottage', 'address2': 'Halse', + 'address3': None, 'postcode': 'NN13 6DY', 'building-reference-number': 2529522278, + 'current-energy-rating': 'F', 'potential-energy-rating': 'C', 'current-energy-efficiency': 29, + 'potential-energy-efficiency': 80, 'property-type': 'House', 'built-form': 'End-Terrace', + 'inspection-date': '2017-03-02', 'local-authority': 'E07000155', 'constituency': 'E14000942', + 'county': 'Northamptonshire', 'lodgement-date': '2017-03-03', 'transaction-type': 'rental (private)', + 'environment-impact-current': 26, 'environment-impact-potential': 74, 'energy-consumption-current': 511, + 'energy-consumption-potential': 130, 'co2-emissions-current': 8.0, 'co2-emiss-curr-per-floor-area': 108, + 'co2-emissions-potential': 2.2, 'lighting-cost-current': 84, 'lighting-cost-potential': 56, + 'heating-cost-current': 1333, 'heating-cost-potential': 498, 'hot-water-cost-current': 285, + 'hot-water-cost-potential': 144, 'total-floor-area': 74.0, 'energy-tariff': 'dual', 'mains-gas-flag': 'N', + 'floor-level': 'NODATA!', 'flat-top-storey': None, 'flat-storey-count': None, + 'main-heating-controls': 2601.0, + 'multi-glaze-proportion': 0.0, 'glazed-type': 'not defined', 'glazed-area': 'Normal', 'extension-count': 1, + 'number-habitable-rooms': 4, 'number-heated-rooms': 1, 'low-energy-lighting': 50, + 'number-open-fireplaces': 0, 'hotwater-description': 'Electric immersion, off-peak', + 'hot-water-energy-eff': 'Very Poor', 'hot-water-env-eff': 'Poor', + 'floor-description': 'Solid, no insulation (assumed)', 'floor-energy-eff': 'NO DATA!', + 'windows-description': 'Single glazed', 'windows-energy-eff': 'Very Poor', 'windows-env-eff': 'Very Poor', + 'walls-description': 'Sandstone or limestone, as built, no insulation (assumed)', + 'walls-energy-eff': 'Very Poor', 'walls-env-eff': 'Very Poor', + 'secondheat-description': 'Room heaters, dual fuel (mineral and wood)', + 'roof-description': 'Pitched, 75 mm loft insulation', 'roof-energy-eff': 'Average', + 'roof-env-eff': 'Average', 'mainheat-description': 'Room heaters, dual fuel (mineral and wood)', + 'mainheat-energy-eff': 'Poor', 'mainheat-env-eff': 'Average', + 'mainheatcont-description': 'No thermostatic control of room temperature', 'mainheatc-energy-eff': 'Poor', + 'mainheatc-env-eff': 'Poor', 'lighting-description': 'Low energy lighting in 50% of fixed outlets', + 'lighting-energy-eff': 'Good', 'lighting-env-eff': 'Good', 'main-fuel': 'dual fuel - mineral + wood', + 'wind-turbine-count': 0, 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, + 'floor-height': None, 'photo-supply': None, 'solar-water-heating-flag': 'N', + 'mechanical-ventilation': 'natural', 'address': '3 Manor Farm Cottage, Halse', + 'local-authority-label': 'South Northamptonshire', 'constituency-label': 'South Northamptonshire', + 'posttown': 'BRACKLEY', 'construction-age-band': 'England and Wales: before 1900', + 'lodgement-datetime': '2017-03-03 07:35:35', 'tenure': 'rental (private)', + 'fixed-lighting-outlets-count': None, 'low-energy-fixed-light-count': None, 'uprn': 10000460605, + 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None + }, + "heating_recommendation_descriptions": [ + 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' + 'Storage Heater Controls' + ], + "heating_controls_recommendation_descriptions": [], + "notes": "This is an end-terrace house, without mains gas connection, so all we recommend is HHR" } ] @@ -1034,7 +1080,8 @@ completed_descriptions = [ "Boiler and radiators, electric", "Boiler and radiators, dual fuel (mineral and wood)", "Boiler and radiators, coal", - "Boiler and radiators, smokeless fuel" + "Boiler and radiators, smokeless fuel", + "Boiler and radiators, wood pellets" ] portfolio = pd.read_excel( @@ -1049,7 +1096,7 @@ portfolio["lodgement-datetime"] = portfolio["lodgement-datetime"].astype(str) print(portfolio["mainheat-description"].value_counts()) eg = portfolio[ - (portfolio["mainheat-description"] == "Boiler and radiators, wood pellets") + (portfolio["mainheat-description"] == "Room heaters, dual fuel (mineral and wood)") ].sample(1) eg = eg.squeeze().to_dict() print(eg) From 76154c51729aaf2b6da490f0f123226c462e47e1 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 16 Sep 2024 19:48:25 +0100 Subject: [PATCH 016/166] another unit test covered --- recommendations/HeatingRecommender.py | 14 ++++-- .../test_data/heating_recommendations_data.py | 47 ++++++++++++++++++- 2 files changed, 56 insertions(+), 5 deletions(-) diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index 8fbcec86..bb074407 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -19,6 +19,7 @@ class HeatingRecommender: self.heating_control_recommendations = [] self.has_electric_heating_description = self.property.main_heating["has_electric"] + self.has_ashp = self.property.main_heating["has_air_source_heat_pump"] def is_high_heat_retention_valid(self, ashp_only_heating_recommendation, measures): """ @@ -31,8 +32,10 @@ class HeatingRecommender: hhr_suitable = no_mains or self.has_electric_heating_description + # If there's already an ASHP in place, we don't recommend HHR + return ( - hhr_suitable and (not ashp_only_heating_recommendation) and + hhr_suitable and (not ashp_only_heating_recommendation) and not self.has_ashp and ("high_heat_retention_storage_heater" in measures) ) @@ -86,7 +89,8 @@ class HeatingRecommender: non_gas_boiler ) and (not ashp_only_heating_recommendation) and - ("boiler_upgrade" in measures) + ("boiler_upgrade" in measures) and + (not self.has_ashp) ) return is_valid, has_gas_boiler @@ -155,7 +159,11 @@ class HeatingRecommender: # In the future, we'll allow overrides, so that non-intrusive surveys can contradict these conditions # and either allow or prevent the recommendation of an air source heat pump - if self.property.is_ashp_valid(measures=measures) and non_invasive_ashp_recommendation["suitable"]: + if ( + self.property.is_ashp_valid(measures=measures) and + non_invasive_ashp_recommendation["suitable"] and + not self.has_ashp + ): self.recommend_air_source_heat_pump( phase=phase, has_cavity_or_loft_recommendations=has_cavity_or_loft_recommendations, diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index c6751784..51d0636e 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -1024,6 +1024,48 @@ testing_examples = [ ], "heating_controls_recommendation_descriptions": [], "notes": "This is an end-terrace house, without mains gas connection, so all we recommend is HHR" + }, + { + "epc": { + 'lmk-key': '1281510829102015021321472533359578', 'address1': '6, Nags Head Lane', 'address2': 'Hargrave', + 'address3': None, 'postcode': 'NN9 6BJ', 'building-reference-number': 134423378, + 'current-energy-rating': 'F', 'potential-energy-rating': 'B', 'current-energy-efficiency': 38, + 'potential-energy-efficiency': 84, 'property-type': 'House', 'built-form': 'End-Terrace', + 'inspection-date': '2015-02-13', 'local-authority': 'E07000152', 'constituency': 'E14000648', + 'county': 'Northamptonshire', 'lodgement-date': '2015-02-13', + 'transaction-type': 'assessment for green deal', 'environment-impact-current': 45, + 'environment-impact-potential': 85, 'energy-consumption-current': 400, 'energy-consumption-potential': 96, + 'co2-emissions-current': 5.0, 'co2-emiss-curr-per-floor-area': 68, 'co2-emissions-potential': 1.2, + 'lighting-cost-current': 87, 'lighting-cost-potential': 48, 'heating-cost-current': 1094, + 'heating-cost-potential': 423, 'hot-water-cost-current': 240, 'hot-water-cost-potential': 144, + 'total-floor-area': 74.0, 'energy-tariff': 'Single', 'mains-gas-flag': 'N', 'floor-level': 'NODATA!', + 'flat-top-storey': None, 'flat-storey-count': None, 'main-heating-controls': 2204.0, + 'multi-glaze-proportion': 100.0, 'glazed-type': 'double glazing installed before 2002', + 'glazed-area': 'Normal', 'extension-count': 0, 'number-habitable-rooms': 5, 'number-heated-rooms': 5, + 'low-energy-lighting': 18, 'number-open-fireplaces': 0, 'hotwater-description': 'From main system', + 'hot-water-energy-eff': 'Poor', 'hot-water-env-eff': 'Good', + 'floor-description': 'Solid, no insulation (assumed)', 'floor-energy-eff': 'NO DATA!', + 'windows-description': 'Fully double glazed', 'windows-energy-eff': 'Average', 'windows-env-eff': 'Average', + 'walls-description': 'Solid brick, as built, no insulation (assumed)', 'walls-energy-eff': 'Poor', + 'walls-env-eff': 'Poor', 'secondheat-description': 'Room heaters, electric', + 'roof-description': 'Pitched, 300 mm loft insulation', 'roof-energy-eff': 'Very Good', + 'roof-env-eff': 'Very Good', 'mainheat-description': 'Air source heat pump, radiators, electric', + 'mainheat-energy-eff': 'Poor', 'mainheat-env-eff': 'Good', + 'mainheatcont-description': 'Programmer and room thermostat', 'mainheatc-energy-eff': 'Average', + 'mainheatc-env-eff': 'Average', 'lighting-description': 'Low energy lighting in 18% of fixed outlets', + 'lighting-energy-eff': 'Poor', 'lighting-env-eff': 'Poor', 'main-fuel': 'electricity (not community)', + 'wind-turbine-count': 0, 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, + 'floor-height': None, 'photo-supply': None, 'solar-water-heating-flag': 'N', + 'mechanical-ventilation': 'natural', 'address': '6, Nags Head Lane, Hargrave', + 'local-authority-label': 'East Northamptonshire', 'constituency-label': 'Corby', + 'posttown': 'WELLINGBOROUGH', 'construction-age-band': 'England and Wales: 1930-1949', + 'lodgement-datetime': '2015-02-13 21:47:25', 'tenure': 'rental (social)', + 'fixed-lighting-outlets-count': None, 'low-energy-fixed-light-count': None, 'uprn': 100031045596, + 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None + }, + "heating_recommendation_descriptions": [], + "heating_controls_recommendation_descriptions": [], + "notes": "This property already has an ashp. We don't recommend any heating upgrades" } ] @@ -1081,7 +1123,8 @@ completed_descriptions = [ "Boiler and radiators, dual fuel (mineral and wood)", "Boiler and radiators, coal", "Boiler and radiators, smokeless fuel", - "Boiler and radiators, wood pellets" + "Boiler and radiators, wood pellets", + "Room heaters, dual fuel (mineral and wood)", ] portfolio = pd.read_excel( @@ -1096,7 +1139,7 @@ portfolio["lodgement-datetime"] = portfolio["lodgement-datetime"].astype(str) print(portfolio["mainheat-description"].value_counts()) eg = portfolio[ - (portfolio["mainheat-description"] == "Room heaters, dual fuel (mineral and wood)") + (portfolio["mainheat-description"] == "Air source heat pump, radiators, electric") ].sample(1) eg = eg.squeeze().to_dict() print(eg) From 68fc8b8cbbba3f12e9081af92c2fd9ad0c4f0a94 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 17 Sep 2024 09:15:31 +0100 Subject: [PATCH 017/166] Added dual heating combined recommendation --- recommendations/HeatingControlRecommender.py | 38 ++- recommendations/HeatingRecommender.py | 224 +++++++++++++++--- recommendations/recommendation_utils.py | 41 ++++ .../test_data/heating_recommendations_data.py | 97 +++++++- .../tests/test_heating_recommendations.py | 2 + 5 files changed, 360 insertions(+), 42 deletions(-) diff --git a/recommendations/HeatingControlRecommender.py b/recommendations/HeatingControlRecommender.py index 6f848441..62e292df 100644 --- a/recommendations/HeatingControlRecommender.py +++ b/recommendations/HeatingControlRecommender.py @@ -12,8 +12,11 @@ class HeatingControlRecommender: self.recommendation = [] - def recommend(self, heating_description): + def recommend(self, heating_description, description_prefix="", description_suffix=""): + # TODO: Many of these functions are quite similar. We can possibly create a single wrapper function that + # takes in the heating description and the description prefix/suffix, and then creates the appropriate + # output # Reset the recommendations self.recommendation = [] @@ -24,14 +27,14 @@ class HeatingControlRecommender: return if heating_description in ["Electric storage heaters", "Electric storage heaters, radiators"]: - self.recommend_high_heat_retention_controls() + self.recommend_high_heat_retention_controls(description_prefix=description_prefix) return if heating_description in ["Boiler and radiators, mains gas"]: # We can recommend roomstat programmer trvs - self.recommend_roomstat_programmer_trvs() + self.recommend_roomstat_programmer_trvs(description_suffix=description_suffix) # We can also recommend time and temperature zone controls - self.recommend_time_temperature_zone_controls() + self.recommend_time_temperature_zone_controls(description_suffix=description_suffix) return @@ -94,16 +97,22 @@ class HeatingControlRecommender: # We don't implement any other recommendations right now return - def recommend_high_heat_retention_controls(self): + def recommend_high_heat_retention_controls(self, description_prefix=""): """ When applicable, we recommend upgrading the heating controls to high heat retention controls. This is a specific type of control system that is designed to work with electric storage heaters. It is a more efficient control system than the standard controls that come with electric storage heaters. We can then consider the heating system itself + + If there is a description prefix, this means there is a dual heating system and so we need to add this to the + description + :return: """ new_description = "Controls for high heat retention storage heaters" + if description_prefix: + new_description = f"{description_prefix}, {new_description}" # We recommend upgrading to Celect type controls ending_config = MainheatControlAttributes(new_description).process() @@ -112,7 +121,10 @@ class HeatingControlRecommender: new_config=ending_config, old_config=self.property.main_heating_controls ) # This upgrade will only take the heating system to average energy efficiency - simulation_config["mainheatc_energy_eff_ending"] = "Good" + if self.property.data["mainheatc-energy-eff"] in ["Poor", "Very Poor", "Average"]: + simulation_config["mainheatc_energy_eff_ending"] = "Good" + else: + simulation_config["mainheatc_energy_eff_ending"] = self.property.data["mainheatc-energy-eff"] description_simulation = { "mainheatcont-description": new_description, @@ -131,7 +143,7 @@ class HeatingControlRecommender: # We don't implement any other recommendations right now return - def recommend_roomstat_programmer_trvs(self): + def recommend_roomstat_programmer_trvs(self, description_suffix=""): """ If the home has a boiler and radiators, mains gas, we start by identifying potential heating controls that could be upgraded, that would provide a practical impact. @@ -163,6 +175,8 @@ class HeatingControlRecommender: return new_controls_description = "Programmer, room thermostat and TRVS" + if description_suffix: + new_controls_description = f"{new_controls_description}, {description_suffix}" ending_config = MainheatControlAttributes(new_controls_description).process() # We use this to determine how we should be updating the config @@ -216,7 +230,7 @@ class HeatingControlRecommender: return - def recommend_time_temperature_zone_controls(self): + def recommend_time_temperature_zone_controls(self, description_suffix=""): """ If the home has a boiler, we can recommend time and temperature zone controls. This is a more advanced and more efficient control system than the standard controls that come with a boiler. However, it may come @@ -238,6 +252,8 @@ class HeatingControlRecommender: return new_controls_description = "Time and temperature zone control" + if description_suffix: + new_controls_description = f"{new_controls_description}, {description_suffix}" ending_config = MainheatControlAttributes(new_controls_description).process() @@ -260,8 +276,10 @@ class HeatingControlRecommender: number_heated_rooms=int(self.property.data["number-heated-rooms"]) ) - description = ("Upgrade heating controls to Smart Thermostats, room sensors and smart radiator valves (time & " - "temperature zone control)") + description = ( + "Upgrade heating controls to Smart Thermostats, room sensors and smart radiator valves (time & " + "temperature zone control)" + ) already_installed = "heating_control" in self.property.already_installed if already_installed: diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index bb074407..23b9bf7d 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -1,5 +1,7 @@ from recommendations.Costs import Costs, BOILER_UPGRADE_SCHEME_ASHP_VALUE -from recommendations.recommendation_utils import check_simulation_difference, override_costs +from recommendations.recommendation_utils import ( + check_simulation_difference, override_costs, combine_recommendation_configs +) from backend.Property import Property from backend.app.plan.schemas import MEASURE_MAP from etl.epc_clean.epc_attributes.MainheatAttributes import MainHeatAttributes @@ -11,6 +13,37 @@ from recommendations.HeatingControlRecommender import HeatingControlRecommender class HeatingRecommender: high_heat_retention_contols_desc = "Controls for high heat retention storage heaters" + DUAL_HEATING_DESCRIPTIONS = { + "Boiler and radiators, mains gas, electric storage heaters": { + "hhr": { + "mainheating_description": "Boiler and radiators, mains gas, Electric storage heaters", + "recommendation_description": "Install high heat retention electric storage heaters alongside the " + "boiler. The current electric heaters may be retrofit with high heat " + "retention storage controls" + " however this is dependent on the existing system and may not be " + "possible.", + "controls_prefix": "current_controls" + }, + "boiler": { + "mainheating_description": "Boiler and radiators, mains gas, electric storage heaters", + "recommendation_description": "Upgrade the existing boiler to a new, more efficient condensing " + "boiler. ", + "controls_suffix": "Manual charge controls" + }, + # These are the heating types we need to produce a dual heating recommendation + "dual": { + "recommendation_description": "Upgrade both the existing boiler to a new condensing boiler and" + "upgrade storage heaters to high heat retention storage heaters.", + "types": [ + # type 1 + "boiler_upgrade", + # type 2 + "high_heat_retention_storage_heater", + ] + } + } + } + def __init__(self, property_instance: Property): self.property = property_instance self.costs = Costs(self.property) @@ -20,10 +53,34 @@ class HeatingRecommender: self.has_electric_heating_description = self.property.main_heating["has_electric"] self.has_ashp = self.property.main_heating["has_air_source_heat_pump"] + self.has_room_heaters = ( + self.property.main_heating["has_room_heaters"] or + self.property.main_heating["has_portable_electric_heaters"] + ) + self.has_boiler = self.property.main_heating["has_boiler"] + + self.dual_heating = self.identify_dual_heating() + + def identify_dual_heating(self): + # All heat systems are in here so we identify whether two of these are true + # MainHeatAttributes.HEAT_SYSTEMS + + n_trues = 0 + for heat_system in MainHeatAttributes.HEAT_SYSTEMS: + if self.property.main_heating[f"has_{heat_system.replace(' ', '_')}"]: + n_trues += 1 + + if n_trues > 2 or n_trues == 0: + raise Exception("Implement me") + if n_trues == 1: + return False + + return True def is_high_heat_retention_valid(self, ashp_only_heating_recommendation, measures): """ Check conditions if high heat retention storage is valid + If there's already an ASHP in place, we don't recommend HHR :return: """ @@ -32,8 +89,6 @@ class HeatingRecommender: hhr_suitable = no_mains or self.has_electric_heating_description - # If there's already an ASHP in place, we don't recommend HHR - return ( hhr_suitable and (not ashp_only_heating_recommendation) and not self.has_ashp and ("high_heat_retention_storage_heater" in measures) @@ -47,7 +102,7 @@ class HeatingRecommender: # 1) if the property has mains heating with boiler and radiators, we recommend optimal heating controls # If it's NOT a gas boiler, we'll potentially recommend a boiler - has_gas_boiler = self.property.main_heating["has_boiler"] and self.property.main_heating["has_mains_gas"] + has_gas_boiler = self.has_boiler and self.property.main_heating["has_mains_gas"] # 2) If the property doesn't have a heating system, but it has access to the mains gas no_heating_has_mains = self.property.main_heating["clean_description"] in [ @@ -55,21 +110,13 @@ class HeatingRecommender: ] and self.property.data["mains-gas-flag"] # The property is using portable heaters and has access to gas mains - has_room_heaters = ( - ( - self.property.main_heating["has_room_heaters"] or - self.property.main_heating["has_portable_electric_heaters"] - ) and - self.property.data["mains-gas-flag"] - ) + has_room_heaters = self.has_room_heaters and self.property.data["mains-gas-flag"] # We also check if the property has electric heating, but it has access to the mains gas electic_heating_has_mains = self.has_electric_heating_description and self.property.data["mains-gas-flag"] portable_heaters_has_mains = ( - self.property.main_heating["clean_description"] in ["Portable electric heaters assumed for most rooms"] - and - self.property.data["mains-gas-flag"] + self.property.main_heating["has_portable_electric_heaters"] and self.property.data["mains-gas-flag"] ) # The next condition is if the home has a non-gas boiler, such as an oil boiler, with a mains gas connection @@ -95,6 +142,55 @@ class HeatingRecommender: return is_valid, has_gas_boiler + def recommend_dual_heating(self): + + if self.property.main_heating["clean_description"] not in self.DUAL_HEATING_DESCRIPTIONS: + return + + dual_heating_description = self.DUAL_HEATING_DESCRIPTIONS[ + self.property.main_heating["clean_description"] + ]["dual"]["types"] + + recommendation_system_types = list(set([x["system_type"] for x in self.heating_recommendations])) + + # We check if we have the required type + if not any([x in recommendation_system_types for x in dual_heating_description]): + return + + type_1_recommendations = [ + x for x in self.heating_recommendations if x["system_type"] == dual_heating_description[0] + ] + type_2_recommendations = [ + x for x in self.heating_recommendations if x["system_type"] == dual_heating_description[1] + ] + # we combine the two recommendations + combined_recommendations = [] + for rec in type_1_recommendations: + for rec2 in type_2_recommendations: + combined_rec = rec.copy() + # Update the description + combined_rec["description"] = self.DUAL_HEATING_DESCRIPTIONS[ + self.property.main_heating["clean_description"] + ]["dual"]["recommendation_description"] + + # Combine simulation_config + # Make sure we end up with the best efficiecy values + combined_rec["simulation_config"] = combine_recommendation_configs( + rec["simulation_config"], rec2["simulation_config"] + ) + # Combine description_simulation + combined_rec["description_simulation"] = combine_recommendation_configs( + rec["description_simulation"], rec2["description_simulation"] + ) + + # Combine costs + for k in ["total", "subtotal", "vat", "labour_hours", "labour_days"]: + combined_rec[k] = rec[k] + rec2[k] + + combined_recommendations.append(combined_rec) + + self.heating_recommendations.extend(combined_recommendations) + def recommend(self, has_cavity_or_loft_recommendations, phase=0, measures=None): """ Produces heating recommendations @@ -144,14 +240,16 @@ class HeatingRecommender: # This indicates that the home previously did not have a boiler in place and so would require # an overhaul to the system - right now, this is all reasons, apart from if there is an existing boiler system_change = not has_gas_boiler - exising_room_heaters = self.property.main_heating["clean_description"] in [ - "Room heaters, electric", "Room heaters, mains gas" - ] + exising_room_heaters = self.property.main_heating["has_room_heaters"] self.recommend_boiler_upgrades( phase=phase, system_change=system_change, exising_room_heaters=exising_room_heaters ) + # If we have dual heating and we allow for a combined recommendation, to upgrade both systems + if self.dual_heating: + self.recommend_dual_heating(phase=phase, measures=measures) + # We recommend air source heat pumps # Heat pumps are suitable for all property types: # https://energysavingtrust.org.uk/from-flats-to-terraced-houses-heat-pumps-are-suitable-for-all-property-types/ @@ -424,7 +522,8 @@ class HeatingRecommender: description, phase, heating_controls_only, - system_change + system_change, + system_type ): """ Given a recommendation for heating controls, and a recommendation for the heating system, we combine the two @@ -439,6 +538,7 @@ class HeatingRecommender: :param system_change: Indicates if we are recommending a different type of heating system, compared to the current system. If we have a system change and we have a heat control recommendation, we only recommend both heating and controls together + :param system_type: The type of heating system we are recommending :return: """ @@ -494,7 +594,9 @@ class HeatingRecommender: "already_installed": already_installed, **total_costs, "simulation_config": recommendation_simulation_config, - "description_simulation": recommendation_description_simulation + "description_simulation": recommendation_description_simulation, + # We insert the heating system type here + "system_type": system_type } output.append(recommendation) @@ -572,7 +674,19 @@ class HeatingRecommender: # We only recommend Celect-type controls if the current heating system is not Celect-type controls if self.property.main_heating_controls["clean_description"] != self.high_heat_retention_contols_desc: - controls_recommender.recommend(heating_description="Electric storage heaters, radiators") + if self.dual_heating: + if self.DUAL_HEATING_DESCRIPTIONS[self.property.main_heating["clean_description"]]["hhr"][ + "controls_prefix" + ] == "current_controls": + description_prefix = self.property.main_heating_controls["clean_description"] + else: + raise NotImplementedError("Implement me") + else: + description_prefix = "" + + controls_recommender.recommend( + heating_description="Electric storage heaters, radiators", description_prefix=description_prefix + ) has_hhr = self.is_hhr_already_installed() # Conditions for not recommending electric storage heaters @@ -580,7 +694,13 @@ class HeatingRecommender: # No recommendation needed return - new_heating_description = "Electric storage heaters, radiators" + # We check if the property has dual heating in place with a boiler and storage heaters + if self.dual_heating: + new_heating_description = self.DUAL_HEATING_DESCRIPTIONS[ + self.property.main_heating["clean_description"] + ]["hhr"]["mainheating_description"] + else: + new_heating_description = "Electric storage heaters, radiators" # Set up artefacts, suitable for the simulation and regardless of controls heating_ending_config = MainHeatAttributes(new_heating_description).process() @@ -588,7 +708,10 @@ class HeatingRecommender: new_config=heating_ending_config, old_config=self.property.main_heating ) # This upgrade will only take the heating system to average energy efficiency - heating_simulation_config["mainheat_energy_eff_ending"] = "Average" + if self.property.data["mainheat-energy-eff"] in ["Very Poor", "Poor"]: + heating_simulation_config["mainheat_energy_eff_ending"] = "Average" + else: + heating_simulation_config["mainheat_energy_eff_ending"] = self.property.data["mainheat-energy-eff"] # If the property is off-gas and has no heating system in place, the number of heated rooms will actually # be 0, so we use the number of rooms as the figure @@ -603,7 +726,13 @@ class HeatingRecommender: costs = self.costs.high_heat_electric_storage_heaters( number_heated_rooms=number_heated_rooms ) - description = "Install high heat retention electric storage heaters." + if self.dual_heating: + description = self.DUAL_HEATING_DESCRIPTIONS[ + self.property.main_heating["clean_description"] + ]["hhr"]["recommendation_description"] + + else: + description = "Install high heat retention electric storage heaters." # We check the existing heating system and controls if ( @@ -627,7 +756,8 @@ class HeatingRecommender: description=description, phase=phase, heating_controls_only=heating_controls_only, - system_change=system_change + system_change=system_change, + system_type="high_heat_retention_storage_heater" ) if _return: return recommendations @@ -721,11 +851,26 @@ class HeatingRecommender: num_heated_rooms=self.property.data["number-heated-rooms"], ) - description = "Upgrade to a new condensing boiler" + if self.dual_heating: + description = self.DUAL_HEATING_DESCRIPTIONS[ + self.property.main_heating["clean_description"] + ]["boiler"]["recommendation_description"] + else: + description = "Upgrade to a new condensing boiler" + + new_heating_eff = ( + "Good" if self.property.data["mainheat-energy-eff"] in ["Very Poor", "Poor", "Average"] + else self.property.data["mainheat-energy-eff"] + ) + + new_hotwater_eff = ( + "Good" if self.property.data["hot-water-energy-eff"] in ["Very Poor", "Poor", "Average"] + else self.property.data["hot-water-energy-eff"] + ) simulation_config = { - "mainheat_energy_eff_ending": "Good", - "hot_water_energy_eff_ending": "Good" + "mainheat_energy_eff_ending": new_heating_eff, + "hot_water_energy_eff_ending": new_hotwater_eff } description_simulation = { @@ -736,7 +881,13 @@ class HeatingRecommender: if system_change: # Installation of a boiler improves the hot water system so we need to reflect this in # the outcome of the recommendation - new_heating_description = "Boiler and radiators, mains gas" + if self.dual_heating: + new_heating_description = self.DUAL_HEATING_DESCRIPTIONS[ + self.property.main_heating["clean_description"] + ]["boiler"]["mainheating_description"] + else: + new_heating_description = "Boiler and radiators, mains gas" + new_hotwater_description = "From main system" new_fuel_description = "mains gas (not community)" @@ -794,13 +945,23 @@ class HeatingRecommender: "already_installed": already_installed, "simulation_config": simulation_config, "description_simulation": description_simulation, - **boiler_costs + **boiler_costs, + "system_type": "boiler_upgrade", } # We recommend the heating controls # If the property did not previously have a boiler, we combine controls_recommender = HeatingControlRecommender(self.property) - controls_recommender.recommend(heating_description="Boiler and radiators, mains gas") + if self.dual_heating: + description_suffix = self.DUAL_HEATING_DESCRIPTIONS[ + self.property.main_heating["clean_description"] + ]["boiler"]["controls_suffix"] + else: + description_suffix = "" + controls_recommender.recommend( + heating_description="Boiler and radiators, mains gas", + description_suffix=description_suffix + ) # We may have 2 recommendations from the heating controls if not controls_recommender.recommendation and not boiler_recommendation: @@ -822,7 +983,8 @@ class HeatingRecommender: description=boiler_recommendation["description"], phase=recommendation_phase, heating_controls_only=False, - system_change=True + system_change=True, + system_type="boiler_upgrade" ) combined_recommendations.extend(combined_recommendation) diff --git a/recommendations/recommendation_utils.py b/recommendations/recommendation_utils.py index ce32e061..883a387b 100644 --- a/recommendations/recommendation_utils.py +++ b/recommendations/recommendation_utils.py @@ -800,3 +800,44 @@ def override_costs(costs): costs[k] = 0 return costs + + +def combine_recommendation_configs(recommendation_config1, recommendation_config2): + """ + Given two simulation configs, this function will combine them into one + :param recommendation_config1: + :param recommendation_config2: + :return: + """ + # Efficiency values - keys which contain _energy_eff_ending + eff_1 = { + k: v for k, v in recommendation_config1.items() if ("_energy_eff_ending" in k) or ("-energy-eff" in k) + } + eff_2 = { + k: v for k, v in recommendation_config2.items() if ("_energy_eff_ending" in k) or ("-energy-eff" in k) + } + + # We combine the simulation configs + combined = { + **recommendation_config1, + **recommendation_config2 + } + + # Find overlapping keys + overlapping_keys = set(eff_1.keys()).intersection(set(eff_2.keys())) + if overlapping_keys: + # We make sure we take the best value - map efficiency values to numbers + numerical_embedding = { + "Very poor": 1, + "Poor": 2, + "Average": 3, + "Good": 4, + "Very good": 5, + } + for key in overlapping_keys: + if numerical_embedding[eff_1[key]] >= numerical_embedding[eff_2[key]]: + combined[key] = eff_1[key] + else: + combined[key] = eff_2[key] + + return combined diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index 51d0636e..821e79c6 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -1066,6 +1066,99 @@ testing_examples = [ "heating_recommendation_descriptions": [], "heating_controls_recommendation_descriptions": [], "notes": "This property already has an ashp. We don't recommend any heating upgrades" + }, + { + "epc": { + 'lmk-key': '1dd9aa80d6e5bae3e0e4892d9ed1a83b53f3af848568f4a928c9f7a63d8825ea', + 'address1': '49 Ridgeway Road', 'address2': 'Wordsley', 'address3': None, 'postcode': 'DY8 5UD', + 'building-reference-number': 10003464876, 'current-energy-rating': 'F', 'potential-energy-rating': 'D', + 'current-energy-efficiency': 35, 'potential-energy-efficiency': 64, 'property-type': 'House', + 'built-form': 'Semi-Detached', 'inspection-date': '2021-11-17', 'local-authority': 'E08000027', + 'constituency': 'E14000672', 'county': None, 'lodgement-date': '2022-10-10', 'transaction-type': 'rental', + 'environment-impact-current': 41, 'environment-impact-potential': 67, 'energy-consumption-current': 401, + 'energy-consumption-potential': 207, 'co2-emissions-current': 6.1, 'co2-emiss-curr-per-floor-area': 69, + 'co2-emissions-potential': 3.2, 'lighting-cost-current': 61, 'lighting-cost-potential': 61, + 'heating-cost-current': 1488, 'heating-cost-potential': 1015, 'hot-water-cost-current': 114, + 'hot-water-cost-potential': 77, 'total-floor-area': 89.0, 'energy-tariff': 'Single', 'mains-gas-flag': 'Y', + 'floor-level': None, 'flat-top-storey': None, 'flat-storey-count': None, 'main-heating-controls': None, + 'multi-glaze-proportion': 100.0, 'glazed-type': 'double glazing installed during or after 2002', + 'glazed-area': 'Normal', 'extension-count': 2, 'number-habitable-rooms': 5, 'number-heated-rooms': 5, + 'low-energy-lighting': 91, 'number-open-fireplaces': 0, 'hotwater-description': 'From main system', + 'hot-water-energy-eff': 'Good', 'hot-water-env-eff': 'Good', + 'floor-description': 'Solid, no insulation (assumed)', 'floor-energy-eff': None, + 'windows-description': 'Fully double glazed', 'windows-energy-eff': 'Good', 'windows-env-eff': 'Good', + 'walls-description': 'Cavity wall, as built, no insulation (assumed)', 'walls-energy-eff': 'Poor', + 'walls-env-eff': 'Poor', 'secondheat-description': 'Room heaters, electric', + 'roof-description': 'Pitched, 200 mm loft insulation', 'roof-energy-eff': 'Good', 'roof-env-eff': 'Good', + 'mainheat-description': 'Boiler and radiators, mains gas, Electric storage heaters', + 'mainheat-energy-eff': 'Good', 'mainheat-env-eff': 'Good', + 'mainheatcont-description': 'Programmer, room thermostat and TRVs', 'mainheatc-energy-eff': 'Good', + 'mainheatc-env-eff': 'Good', 'lighting-description': 'Low energy lighting in 91% of fixed outlets', + 'lighting-energy-eff': 'Very Good', 'lighting-env-eff': 'Very Good', + 'main-fuel': 'mains gas (not community)', 'wind-turbine-count': 0, 'heat-loss-corridor': None, + 'unheated-corridor-length': None, 'floor-height': 2.53, 'photo-supply': 0.0, + 'solar-water-heating-flag': 'N', + 'mechanical-ventilation': 'natural', 'address': '49 Ridgeway Road, Wordsley', + 'local-authority-label': 'Dudley', 'constituency-label': 'Dudley South', 'posttown': 'Stourbridge', + 'construction-age-band': 'England and Wales: 1950-1966', 'lodgement-datetime': '2022-10-10 16:41:36', + 'tenure': 'Rented (social)', 'fixed-lighting-outlets-count': 11.0, 'low-energy-fixed-light-count': None, + 'uprn': 90041166, 'uprn-source': 'Energy Assessor', 'sheating-energy-eff': None, 'sheating-env-eff': None + }, + "heating_recommendation_descriptions": [ + 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' + 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' + 'scheme grant', + 'Install high heat retention electric storage heaters alongside the boiler. The current electric heaters ' + 'may be retrofit with high heat retention storage controls however this is dependent on the existing ' + 'system and may not be possible. Upgrade heating controls to High Heat Retention Storage Heater Controls' + ], + "heating_controls_recommendation_descriptions": [], + "notes": "This property has dual heating. A boiler and electric storage heaters. The heating is efficient so" + "we recommend ASHP and HHR" + }, + { + "epc": { + 'lmk-key': '1dd9aa80d6e5bae3e0e4892d9ed1a83b53f3af848568f4a928c9f7a63d8825ea', + 'address1': '49 Ridgeway Road', 'address2': 'Wordsley', 'address3': None, 'postcode': 'DY8 5UD', + 'building-reference-number': 10003464876, 'current-energy-rating': 'F', 'potential-energy-rating': 'D', + 'current-energy-efficiency': 35, 'potential-energy-efficiency': 64, 'property-type': 'House', + 'built-form': 'Semi-Detached', 'inspection-date': '2021-11-17', 'local-authority': 'E08000027', + 'constituency': 'E14000672', 'county': None, 'lodgement-date': '2022-10-10', 'transaction-type': 'rental', + 'environment-impact-current': 41, 'environment-impact-potential': 67, 'energy-consumption-current': 401, + 'energy-consumption-potential': 207, 'co2-emissions-current': 6.1, 'co2-emiss-curr-per-floor-area': 69, + 'co2-emissions-potential': 3.2, 'lighting-cost-current': 61, 'lighting-cost-potential': 61, + 'heating-cost-current': 1488, 'heating-cost-potential': 1015, 'hot-water-cost-current': 114, + 'hot-water-cost-potential': 77, 'total-floor-area': 89.0, 'energy-tariff': 'Single', 'mains-gas-flag': 'Y', + 'floor-level': None, 'flat-top-storey': None, 'flat-storey-count': None, 'main-heating-controls': None, + 'multi-glaze-proportion': 100.0, 'glazed-type': 'double glazing installed during or after 2002', + 'glazed-area': 'Normal', 'extension-count': 2, 'number-habitable-rooms': 5, 'number-heated-rooms': 5, + 'low-energy-lighting': 91, 'number-open-fireplaces': 0, 'hotwater-description': 'From main system', + 'hot-water-energy-eff': 'Good', 'hot-water-env-eff': 'Good', + 'floor-description': 'Solid, no insulation (assumed)', 'floor-energy-eff': None, + 'windows-description': 'Fully double glazed', 'windows-energy-eff': 'Good', 'windows-env-eff': 'Good', + 'walls-description': 'Cavity wall, as built, no insulation (assumed)', 'walls-energy-eff': 'Poor', + 'walls-env-eff': 'Poor', 'secondheat-description': 'Room heaters, electric', + 'roof-description': 'Pitched, 200 mm loft insulation', 'roof-energy-eff': 'Good', 'roof-env-eff': 'Good', + 'mainheat-description': 'Boiler and radiators, mains gas, Electric storage heaters', + 'mainheat-energy-eff': 'Average', 'mainheat-env-eff': 'Good', + 'mainheatcont-description': 'Programmer, room thermostat and TRVs', 'mainheatc-energy-eff': 'Good', + 'mainheatc-env-eff': 'Good', 'lighting-description': 'Low energy lighting in 91% of fixed outlets', + 'lighting-energy-eff': 'Very Good', 'lighting-env-eff': 'Very Good', + 'main-fuel': 'mains gas (not community)', 'wind-turbine-count': 0, 'heat-loss-corridor': None, + 'unheated-corridor-length': None, 'floor-height': 2.53, 'photo-supply': 0.0, + 'solar-water-heating-flag': 'N', + 'mechanical-ventilation': 'natural', 'address': '49 Ridgeway Road, Wordsley', + 'local-authority-label': 'Dudley', 'constituency-label': 'Dudley South', 'posttown': 'Stourbridge', + 'construction-age-band': 'England and Wales: 1950-1966', 'lodgement-datetime': '2022-10-10 16:41:36', + 'tenure': 'Rented (social)', 'fixed-lighting-outlets-count': 11.0, 'low-energy-fixed-light-count': None, + 'uprn': 90041166, 'uprn-source': 'Energy Assessor', 'sheating-energy-eff': None, 'sheating-env-eff': None + }, + "heating_recommendation_descriptions": [ + ], + "heating_controls_recommendation_descriptions": [], + "notes": "This property is a modified version of the previous dual heating property, where we lower the" + "starting heating efficiency so that we a combined heating upgrade to both the boiler and the electric" + "storage heaters" } ] @@ -1125,6 +1218,8 @@ completed_descriptions = [ "Boiler and radiators, smokeless fuel", "Boiler and radiators, wood pellets", "Room heaters, dual fuel (mineral and wood)", + "Air source heat pump, radiators, electric", + "Portable electric heaters assumed for most rooms, Room heaters, electric", ] portfolio = pd.read_excel( @@ -1139,7 +1234,7 @@ portfolio["lodgement-datetime"] = portfolio["lodgement-datetime"].astype(str) print(portfolio["mainheat-description"].value_counts()) eg = portfolio[ - (portfolio["mainheat-description"] == "Air source heat pump, radiators, electric") + (portfolio["mainheat-description"] == "Boiler and radiators, mains gas, Electric storage heaters") ].sample(1) eg = eg.squeeze().to_dict() print(eg) diff --git a/recommendations/tests/test_heating_recommendations.py b/recommendations/tests/test_heating_recommendations.py index 35373729..b80780d9 100644 --- a/recommendations/tests/test_heating_recommendations.py +++ b/recommendations/tests/test_heating_recommendations.py @@ -53,6 +53,8 @@ class TestHeatingRecommendations: we retrieve alongside them :return: """ + if test_case["epc"]["uprn"] == 90041166: + raise Exception("Finish the second test case with this uprn") epc_records = {"original_epc": test_case["epc"].copy(), "full_sap_epc": {}, "old_data": []} From 3b325395b9fd5ab61fe76c4af07bb29806ba64f4 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 17 Sep 2024 09:28:01 +0100 Subject: [PATCH 018/166] added dual heating recommendations --- recommendations/HeatingRecommender.py | 4 ++-- .../tests/test_data/heating_recommendations_data.py | 9 +++++++++ recommendations/tests/test_heating_recommendations.py | 2 -- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index 23b9bf7d..db83508b 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -33,7 +33,7 @@ class HeatingRecommender: # These are the heating types we need to produce a dual heating recommendation "dual": { "recommendation_description": "Upgrade both the existing boiler to a new condensing boiler and" - "upgrade storage heaters to high heat retention storage heaters.", + " upgrade storage heaters to high heat retention storage heaters.", "types": [ # type 1 "boiler_upgrade", @@ -248,7 +248,7 @@ class HeatingRecommender: # If we have dual heating and we allow for a combined recommendation, to upgrade both systems if self.dual_heating: - self.recommend_dual_heating(phase=phase, measures=measures) + self.recommend_dual_heating() # We recommend air source heat pumps # Heat pumps are suitable for all property types: diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index 821e79c6..76b8b218 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -1154,6 +1154,15 @@ testing_examples = [ 'uprn': 90041166, 'uprn-source': 'Energy Assessor', 'sheating-energy-eff': None, 'sheating-env-eff': None }, "heating_recommendation_descriptions": [ + 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' + 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' + 'scheme grant', + 'Upgrade the existing boiler to a new, more efficient condensing boiler. ', + 'Upgrade both the existing boiler to a new condensing boiler and upgrade storage heaters to high heat ' + 'retention storage heaters.', + 'Install high heat retention electric storage heaters alongside the boiler. The current electric heaters ' + 'may be retrofit with high heat retention storage controls however this is dependent on the existing ' + 'system and may not be possible. Upgrade heating controls to High Heat Retention Storage Heater Controls' ], "heating_controls_recommendation_descriptions": [], "notes": "This property is a modified version of the previous dual heating property, where we lower the" diff --git a/recommendations/tests/test_heating_recommendations.py b/recommendations/tests/test_heating_recommendations.py index b80780d9..35373729 100644 --- a/recommendations/tests/test_heating_recommendations.py +++ b/recommendations/tests/test_heating_recommendations.py @@ -53,8 +53,6 @@ class TestHeatingRecommendations: we retrieve alongside them :return: """ - if test_case["epc"]["uprn"] == 90041166: - raise Exception("Finish the second test case with this uprn") epc_records = {"original_epc": test_case["epc"].copy(), "full_sap_epc": {}, "old_data": []} From 59864bc8e8d3b64b1a091a7a2db4a22889238b00 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 17 Sep 2024 10:05:56 +0100 Subject: [PATCH 019/166] added new room heatings unit tests --- .../test_data/heating_recommendations_data.py | 53 ++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index 76b8b218..77126c7b 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -1168,6 +1168,55 @@ testing_examples = [ "notes": "This property is a modified version of the previous dual heating property, where we lower the" "starting heating efficiency so that we a combined heating upgrade to both the boiler and the electric" "storage heaters" + }, + { + "epc": { + 'lmk-key': '670443469402019100713595382910638', 'address1': '2 Crabs Castle', 'address2': 'Pontrilas', + 'address3': None, 'postcode': 'HR2 0BN', 'building-reference-number': 6466069868, + 'current-energy-rating': 'G', 'potential-energy-rating': 'C', 'current-energy-efficiency': 20, + 'potential-energy-efficiency': 80, 'property-type': 'House', 'built-form': 'Semi-Detached', + 'inspection-date': '2019-10-07', 'local-authority': 'E06000019', 'constituency': 'E14000743', + 'county': None, + 'lodgement-date': '2019-10-07', 'transaction-type': 'rental (social)', 'environment-impact-current': 1, + 'environment-impact-potential': 103, 'energy-consumption-current': 618, 'energy-consumption-potential': 110, + 'co2-emissions-current': 15.0, 'co2-emiss-curr-per-floor-area': 206, 'co2-emissions-potential': -0.3, + 'lighting-cost-current': 125, 'lighting-cost-potential': 63, 'heating-cost-current': 1371, + 'heating-cost-potential': 473, 'hot-water-cost-current': 524, 'hot-water-cost-potential': 101, + 'total-floor-area': 73.0, 'energy-tariff': 'dual', 'mains-gas-flag': 'N', 'floor-level': 'NODATA!', + 'flat-top-storey': None, 'flat-storey-count': None, 'main-heating-controls': 2601.0, + 'multi-glaze-proportion': 100.0, 'glazed-type': 'double glazing installed during or after 2002', + 'glazed-area': 'Normal', 'extension-count': 1, 'number-habitable-rooms': 4, 'number-heated-rooms': 4, + 'low-energy-lighting': 0, 'number-open-fireplaces': 1, + 'hotwater-description': 'Electric immersion, off-peak', 'hot-water-energy-eff': 'Very Poor', + 'hot-water-env-eff': 'Poor', 'floor-description': 'Solid, no insulation (assumed)', + 'floor-energy-eff': 'NO DATA!', 'windows-description': 'Fully double glazed', 'windows-energy-eff': 'Good', + 'windows-env-eff': 'Good', 'walls-description': 'Cavity wall, as built, no insulation (assumed)', + 'walls-energy-eff': 'Poor', 'walls-env-eff': 'Poor', 'secondheat-description': 'None', + 'roof-description': 'Pitched, 150 mm loft insulation', 'roof-energy-eff': 'Good', 'roof-env-eff': 'Good', + 'mainheat-description': 'Room heaters, anthracite', 'mainheat-energy-eff': 'Very Poor', + 'mainheat-env-eff': 'Very Poor', 'mainheatcont-description': 'No thermostatic control of room temperature', + 'mainheatc-energy-eff': 'Poor', 'mainheatc-env-eff': 'Poor', + 'lighting-description': 'No low energy lighting', 'lighting-energy-eff': 'Very Poor', + 'lighting-env-eff': 'Very Poor', 'main-fuel': 'anthracite', 'wind-turbine-count': 0, + 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, 'floor-height': None, + 'photo-supply': None, + 'solar-water-heating-flag': 'N', 'mechanical-ventilation': 'natural', + 'address': '2 Crabs Castle, Pontrilas', 'local-authority-label': 'Herefordshire, County of', + 'constituency-label': 'Hereford and South Herefordshire', 'posttown': 'HEREFORD', + 'construction-age-band': 'England and Wales: 1930-1949', 'lodgement-datetime': '2019-10-07 13:59:53', + 'tenure': 'rental (social)', 'fixed-lighting-outlets-count': None, 'low-energy-fixed-light-count': None, + 'uprn': 10009574286, 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None + }, + "heating_recommendation_descriptions": [ + 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' + 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' + 'scheme grant', + 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' + 'Storage Heater Controls' + ], + "heating_controls_recommendation_descriptions": [], + "notes": "This property has anthracite heating without mains. " + "We recommend ASHP and HHR, but no gas condensing boiler" } ] @@ -1229,6 +1278,8 @@ completed_descriptions = [ "Room heaters, dual fuel (mineral and wood)", "Air source heat pump, radiators, electric", "Portable electric heaters assumed for most rooms, Room heaters, electric", + "Boiler and radiators, mains gas, Electric storage heaters", + "Room heaters, anthracite", ] portfolio = pd.read_excel( @@ -1243,7 +1294,7 @@ portfolio["lodgement-datetime"] = portfolio["lodgement-datetime"].astype(str) print(portfolio["mainheat-description"].value_counts()) eg = portfolio[ - (portfolio["mainheat-description"] == "Boiler and radiators, mains gas, Electric storage heaters") + (portfolio["mainheat-description"] == "Room heaters, anthracite") ].sample(1) eg = eg.squeeze().to_dict() print(eg) From 185406026666c12b37a29488edb5b47e9c3d9569 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 17 Sep 2024 10:27:51 +0100 Subject: [PATCH 020/166] added additional heating test case --- recommendations/HeatingRecommender.py | 5 +- .../test_data/heating_recommendations_data.py | 52 ++++++++++++++++++- 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index db83508b..e4107205 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -87,7 +87,10 @@ class HeatingRecommender: # We can also recommend hhr if the property doesn't have a mains has connection no_mains = not self.property.data["mains-gas-flag"] - hhr_suitable = no_mains or self.has_electric_heating_description + # If the property already has room heaters then we recommend HHR as an option since the home already has + # a variation of room heaters + + hhr_suitable = no_mains or self.has_electric_heating_description or self.has_room_heaters return ( hhr_suitable and (not ashp_only_heating_recommendation) and not self.has_ashp and diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index 77126c7b..58f3fad4 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -1217,6 +1217,55 @@ testing_examples = [ "heating_controls_recommendation_descriptions": [], "notes": "This property has anthracite heating without mains. " "We recommend ASHP and HHR, but no gas condensing boiler" + }, + { + "epc": { + 'lmk-key': '298ecac47f69461257e582e9dd78cb4c8bbb8c05bee1af1530f052134ecd5044', 'address1': '9 HENDON RISE', + 'address2': None, 'address3': None, 'postcode': 'NG3 3AN', 'building-reference-number': 10001202381, + 'current-energy-rating': 'G', 'potential-energy-rating': 'D', 'current-energy-efficiency': 12, + 'potential-energy-efficiency': 66, 'property-type': 'House', 'built-form': 'Mid-Terrace', + 'inspection-date': '2021-03-31', 'local-authority': 'E06000018', 'constituency': 'E14000865', + 'county': None, + 'lodgement-date': '2021-05-27', 'transaction-type': 'rental', 'environment-impact-current': 15, + 'environment-impact-potential': 59, 'energy-consumption-current': 630, 'energy-consumption-potential': 187, + 'co2-emissions-current': 11.0, 'co2-emiss-curr-per-floor-area': 122, 'co2-emissions-potential': 3.6, + 'lighting-cost-current': 99, 'lighting-cost-potential': 99, 'heating-cost-current': 2107, + 'heating-cost-potential': 860, 'hot-water-cost-current': 458, 'hot-water-cost-potential': 73, + 'total-floor-area': 91.0, 'energy-tariff': 'Single', 'mains-gas-flag': 'Y', 'floor-level': None, + 'flat-top-storey': None, 'flat-storey-count': None, 'main-heating-controls': None, + 'multi-glaze-proportion': 100.0, 'glazed-type': 'double glazing installed before 2002', + 'glazed-area': 'Normal', 'extension-count': 1, 'number-habitable-rooms': 5, 'number-heated-rooms': 2, + 'low-energy-lighting': 63, 'number-open-fireplaces': 1, + 'hotwater-description': 'Electric immersion, standard tariff', 'hot-water-energy-eff': 'Very Poor', + 'hot-water-env-eff': 'Poor', 'floor-description': 'Suspended, no insulation (assumed)', + 'floor-energy-eff': None, 'windows-description': 'Fully double glazed', 'windows-energy-eff': 'Average', + 'windows-env-eff': 'Average', 'walls-description': 'Solid brick, as built, no insulation (assumed)', + 'walls-energy-eff': 'Very Poor', 'walls-env-eff': 'Very Poor', + 'secondheat-description': 'Portable electric heaters (assumed)', + 'roof-description': 'Pitched, no insulation (assumed)', 'roof-energy-eff': 'Very Poor', + 'roof-env-eff': 'Very Poor', + 'mainheat-description': 'Room heaters, mains gas, Room heaters, dual fuel (mineral and wood)', + 'mainheat-energy-eff': 'Average', 'mainheat-env-eff': 'Average', + 'mainheatcont-description': 'No thermostatic control of room temperature', 'mainheatc-energy-eff': 'Poor', + 'mainheatc-env-eff': 'Poor', 'lighting-description': 'Low energy lighting in 63% of fixed outlets', + 'lighting-energy-eff': 'Good', 'lighting-env-eff': 'Good', 'main-fuel': 'mains gas (not community)', + 'wind-turbine-count': 0, 'heat-loss-corridor': None, 'unheated-corridor-length': None, 'floor-height': 2.79, + 'photo-supply': 0.0, 'solar-water-heating-flag': 'N', 'mechanical-ventilation': 'natural', + 'address': '9 HENDON RISE', 'local-authority-label': 'Nottingham', 'constituency-label': 'Nottingham East', + 'posttown': 'NOTTINGHAM', 'construction-age-band': 'England and Wales: 1900-1929', + 'lodgement-datetime': '2021-05-27 13:52:20', 'tenure': 'Rented (private)', + 'fixed-lighting-outlets-count': 8.0, 'low-energy-fixed-light-count': None, 'uprn': 100031556691, + 'uprn-source': 'Energy Assessor', 'sheating-energy-eff': None, 'sheating-env-eff': None + }, + "heating_recommendation_descriptions": [ + 'Upgrade to a new condensing boiler Upgrade heating controls to Room thermostat, programmer and TRVs', + 'Upgrade to a new condensing boiler Upgrade heating controls to Smart Thermostats, room sensors and smart ' + 'radiator valves (time & temperature zone control)', + 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' + 'Storage Heater Controls' + ], + "heating_controls_recommendation_descriptions": [], + "notes": "This property has room heaters with two different fuel sources" } ] @@ -1280,6 +1329,7 @@ completed_descriptions = [ "Portable electric heaters assumed for most rooms, Room heaters, electric", "Boiler and radiators, mains gas, Electric storage heaters", "Room heaters, anthracite", + "Room heaters, mains gas, Room heaters, dual fuel (mineral and wood)" ] portfolio = pd.read_excel( @@ -1294,7 +1344,7 @@ portfolio["lodgement-datetime"] = portfolio["lodgement-datetime"].astype(str) print(portfolio["mainheat-description"].value_counts()) eg = portfolio[ - (portfolio["mainheat-description"] == "Room heaters, anthracite") + (portfolio["mainheat-description"] == "Room heaters, mains gas, Room heaters, dual fuel (mineral and wood)") ].sample(1) eg = eg.squeeze().to_dict() print(eg) From 5243e64e4128656390e4c7ffad862f4e6c319b9e Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 17 Sep 2024 11:08:57 +0100 Subject: [PATCH 021/166] Added additional heating unit test --- recommendations/HeatingRecommender.py | 4 +- .../test_data/heating_recommendations_data.py | 105 +++++++++++++++++- 2 files changed, 105 insertions(+), 4 deletions(-) diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index e4107205..e40c1736 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -51,7 +51,9 @@ class HeatingRecommender: self.heating_recommendations = [] self.heating_control_recommendations = [] - self.has_electric_heating_description = self.property.main_heating["has_electric"] + self.has_electric_heating_description = ( + self.property.main_heating["has_electric"] or self.property.main_heating["has_electricaire"] + ) self.has_ashp = self.property.main_heating["has_air_source_heat_pump"] self.has_room_heaters = ( self.property.main_heating["has_room_heaters"] or diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index 58f3fad4..fea53e2b 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -1265,7 +1265,105 @@ testing_examples = [ 'Storage Heater Controls' ], "heating_controls_recommendation_descriptions": [], - "notes": "This property has room heaters with two different fuel sources" + "notes": "This property has room heaters with two different fuel sources, so we recommend HHR, ASHP, and a " + "boiler upgrade" + }, + { + "epc": { + 'lmk-key': '717779210932011102812033766268396', 'address1': '28, Overdale Road', 'address2': None, + 'address3': None, 'postcode': 'CV5 8AL', 'building-reference-number': 1616392968, + 'current-energy-rating': 'F', 'potential-energy-rating': 'E', 'current-energy-efficiency': 28, + 'potential-energy-efficiency': 40, 'property-type': 'Flat', 'built-form': 'NO DATA!', + 'inspection-date': '2011-10-27', 'local-authority': 'E08000026', 'constituency': 'E14000650', + 'county': None, + 'lodgement-date': '2011-10-28', 'transaction-type': 'marketed sale', 'environment-impact-current': 37, + 'environment-impact-potential': 47, 'energy-consumption-current': 544, 'energy-consumption-potential': 431, + 'co2-emissions-current': 5.1, 'co2-emiss-curr-per-floor-area': 96, 'co2-emissions-potential': 4.0, + 'lighting-cost-current': 54, 'lighting-cost-potential': 31, 'heating-cost-current': 576, + 'heating-cost-potential': 711, 'hot-water-cost-current': 598, 'hot-water-cost-potential': 232, + 'total-floor-area': 52.93, 'energy-tariff': 'Single', 'mains-gas-flag': 'N', 'floor-level': '1st', + 'flat-top-storey': 'N', 'flat-storey-count': None, 'main-heating-controls': 2703.0, + 'multi-glaze-proportion': 100.0, 'glazed-type': 'double glazing, unknown install date', + 'glazed-area': 'Normal', 'extension-count': 0, 'number-habitable-rooms': 3, 'number-heated-rooms': 3, + 'low-energy-lighting': 25, 'number-open-fireplaces': 0, + 'hotwater-description': 'Electric immersion, standard tariff', 'hot-water-energy-eff': 'Very Poor', + 'hot-water-env-eff': 'Very Poor', 'floor-description': '(other premises below)', 'floor-energy-eff': None, + 'windows-description': 'Fully double glazed', 'windows-energy-eff': 'Average', 'windows-env-eff': 'Average', + 'walls-description': 'Cavity wall, as built, no insulation (assumed)', 'walls-energy-eff': 'Poor', + 'walls-env-eff': 'Poor', 'secondheat-description': 'None', 'roof-description': '(another dwelling above)', + 'roof-energy-eff': None, 'roof-env-eff': None, 'mainheat-description': 'Electric underfloor heating', + 'mainheat-energy-eff': 'Very Poor', 'mainheat-env-eff': 'Very Poor', + 'mainheatcont-description': 'Room thermostat only', 'mainheatc-energy-eff': 'Poor', + 'mainheatc-env-eff': 'Poor', 'lighting-description': 'Low energy lighting in 25% of fixed outlets', + 'lighting-energy-eff': 'Average', 'lighting-env-eff': 'Average', 'main-fuel': 'electricity (not community)', + 'wind-turbine-count': 0, 'heat-loss-corridor': 'unheated corridor', 'unheated-corridor-length': 4.44, + 'floor-height': 2.28, 'photo-supply': 0.0, 'solar-water-heating-flag': None, + 'mechanical-ventilation': 'natural', 'address': '28, Overdale Road', 'local-authority-label': 'Coventry', + 'constituency-label': 'Coventry North West', 'posttown': 'COVENTRY', + 'construction-age-band': 'England and Wales: 1950-1966', 'lodgement-datetime': '2011-10-28 12:03:37', + 'tenure': 'owner-occupied', 'fixed-lighting-outlets-count': 8.0, 'low-energy-fixed-light-count': 2.0, + 'uprn': 100070685908, 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, + 'sheating-env-eff': None + }, + "heating_recommendation_descriptions": [ + 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' + 'Storage Heater Controls' + ], + "heating_controls_recommendation_descriptions": [], + "notes": "This property is a flag, without mains gas connection. Currently has underfloor electric heating" + "so we recommend HHR" + }, + { + "epc": { + 'lmk-key': '800099229502014050214593392940598', 'address1': '16, Woodside', 'address2': None, + 'address3': None, 'postcode': 'WV11 2PT', 'building-reference-number': 4556129968, + 'current-energy-rating': 'F', 'potential-energy-rating': 'E', 'current-energy-efficiency': 21, + 'potential-energy-efficiency': 39, 'property-type': 'House', 'built-form': 'Semi-Detached', + 'inspection-date': '2014-05-01', 'local-authority': 'E08000031', 'constituency': 'E14001049', + 'county': None, + 'lodgement-date': '2014-05-02', 'transaction-type': 'none of the above', 'environment-impact-current': 32, + 'environment-impact-potential': 46, 'energy-consumption-current': 479, 'energy-consumption-potential': 340, + 'co2-emissions-current': 8.1, 'co2-emiss-curr-per-floor-area': 85, 'co2-emissions-potential': 5.7, + 'lighting-cost-current': 111, 'lighting-cost-potential': 55, 'heating-cost-current': 2021, + 'heating-cost-potential': 1655, 'hot-water-cost-current': 408, 'hot-water-cost-potential': 189, + 'total-floor-area': 96.0, 'energy-tariff': 'Unknown', 'mains-gas-flag': 'Y', 'floor-level': 'NODATA!', + 'flat-top-storey': None, 'flat-storey-count': None, 'main-heating-controls': 2504.0, + 'multi-glaze-proportion': 100.0, 'glazed-type': 'double glazing installed before 2002', + 'glazed-area': 'Normal', 'extension-count': 0, 'number-habitable-rooms': 5, 'number-heated-rooms': 5, + 'low-energy-lighting': 0, 'number-open-fireplaces': 0, + 'hotwater-description': 'Electric immersion, standard tariff', 'hot-water-energy-eff': 'Very Poor', + 'hot-water-env-eff': 'Very Poor', 'floor-description': 'Solid, no insulation (assumed)', + 'floor-energy-eff': None, 'windows-description': 'Fully double glazed', 'windows-energy-eff': 'Average', + 'windows-env-eff': 'Average', 'walls-description': 'System built, as built, no insulation (assumed)', + 'walls-energy-eff': 'Very Poor', 'walls-env-eff': 'Very Poor', 'secondheat-description': 'None', + 'roof-description': 'Pitched, 200 mm loft insulation', 'roof-energy-eff': 'Good', 'roof-env-eff': 'Good', + 'mainheat-description': 'Warm air, Electricaire', 'mainheat-energy-eff': 'Very Poor', + 'mainheat-env-eff': 'Very Poor', 'mainheatcont-description': 'Programmer and room thermostat', + 'mainheatc-energy-eff': 'Average', 'mainheatc-env-eff': 'Average', + 'lighting-description': 'No low energy lighting', 'lighting-energy-eff': 'Very Poor', + 'lighting-env-eff': 'Very Poor', 'main-fuel': 'electricity (not community)', 'wind-turbine-count': 0, + 'heat-loss-corridor': 'NO DATA!', 'unheated-corridor-length': None, 'floor-height': None, + 'photo-supply': 50.0, 'solar-water-heating-flag': None, 'mechanical-ventilation': 'natural', + 'address': '16, Woodside', 'local-authority-label': 'Wolverhampton', + 'constituency-label': 'Wolverhampton North East', 'posttown': 'WOLVERHAMPTON', + 'construction-age-band': 'England and Wales: 1967-1975', 'lodgement-datetime': '2014-05-02 14:59:33', + 'tenure': 'owner-occupied', 'fixed-lighting-outlets-count': 20.0, 'low-energy-fixed-light-count': 0.0, + 'uprn': 100071209105, 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, + 'sheating-env-eff': None + }, + "heating_recommendation_descriptions": [ + 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' + 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' + 'scheme grant', + 'Upgrade to a new condensing boiler Upgrade heating controls to Room thermostat, programmer and TRVs', + 'Upgrade to a new condensing boiler Upgrade heating controls to Smart Thermostats, room sensors and smart ' + 'radiator valves (time & temperature zone control)', + 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' + 'Storage Heater Controls' + ], + "heating_controls_recommendation_descriptions": [], + "notes": "The property has warm air electricaire heating, so we recommend ASHP and HHR. It also has a mains" + "connection so we recommend a gas condensing boiler" } ] @@ -1329,7 +1427,8 @@ completed_descriptions = [ "Portable electric heaters assumed for most rooms, Room heaters, electric", "Boiler and radiators, mains gas, Electric storage heaters", "Room heaters, anthracite", - "Room heaters, mains gas, Room heaters, dual fuel (mineral and wood)" + "Room heaters, mains gas, Room heaters, dual fuel (mineral and wood)", + "Electric underfloor heating", ] portfolio = pd.read_excel( @@ -1344,7 +1443,7 @@ portfolio["lodgement-datetime"] = portfolio["lodgement-datetime"].astype(str) print(portfolio["mainheat-description"].value_counts()) eg = portfolio[ - (portfolio["mainheat-description"] == "Room heaters, mains gas, Room heaters, dual fuel (mineral and wood)") + (portfolio["mainheat-description"] == "Warm air, Electricaire") ].sample(1) eg = eg.squeeze().to_dict() print(eg) From 53e68f8d76b3f8146166ba575c7507908e3b1643 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 17 Sep 2024 15:26:38 +0100 Subject: [PATCH 022/166] added tests for final property --- etl/sfr/midlands_portfolio_est_funding.py | 100 ++++++--- recommendations/HeatingRecommender.py | 19 +- .../test_data/heating_recommendations_data.py | 209 +++++++++++------- 3 files changed, 218 insertions(+), 110 deletions(-) diff --git a/etl/sfr/midlands_portfolio_est_funding.py b/etl/sfr/midlands_portfolio_est_funding.py index 09102cfb..017fd223 100644 --- a/etl/sfr/midlands_portfolio_est_funding.py +++ b/etl/sfr/midlands_portfolio_est_funding.py @@ -57,10 +57,18 @@ def app(): ['none', "below average"] ) + epc_data["needs_solid_wall"] = (epc_data["is_solid_brick"] | epc_data["is_system_built"]) & epc_data[ + "insulation_thickness_wall"].isin(['none', "below average"]) + + epc_data["could_take_solar"] = (epc_data["is_flat"] | epc_data["is_pitched"]) + loft_insulation_per_m2 = 16.07 flat_roof_insulation_per_m2 = 195 cwi_per_m2 = 14.21 + ewi_per_m2 = 200 gbis_abs = 30 + eco4_abs = 24 + solar_pv_cost = 4009 # We assume the work will take the home from a high D to a low D def get_abs(floor_area): @@ -75,6 +83,19 @@ def app(): return 350.1 + # We assume the work will take the home from a high E to a high C + def get_eco4_abs(floor_area): + if floor_area <= 72: + return 596.6 + + if floor_area <= 97: + return 650.2 + + if floor_area <= 199: + return 755.8 + + return 1347.1 + estimated_costs = [] for _, home in epc_data.iterrows(): to_append = { @@ -89,30 +110,59 @@ def app(): n_floors = estimate_number_of_floors(home["PROPERTY_TYPE"]) floor_height = float(home["FLOOR_HEIGHT"]) if not pd.isnull(home["FLOOR_HEIGHT"]) else 2.5 - # Check if it needs the walls done - if home["needs_cavity_done"]: - # We estimate the amount of insulation required - est_perimeter = estimate_perimeter( - floor_area=float(home["TOTAL_FLOOR_AREA"]) / n_floors, - num_rooms=float(home["NUMBER_HABITABLE_ROOMS"]) / n_floors - ) + # We estimate the amount of insulation required + est_perimeter = estimate_perimeter( + floor_area=float(home["TOTAL_FLOOR_AREA"]) / n_floors, + num_rooms=float(home["NUMBER_HABITABLE_ROOMS"]) / n_floors + ) - insulation_needed = estimate_external_wall_area( - num_floors=n_floors, - floor_height=floor_height, - perimeter=est_perimeter, - built_form=home["BUILT_FORM"], - ) - cost_of_insulation = insulation_needed * cwi_per_m2 + insulation_needed = estimate_external_wall_area( + num_floors=n_floors, + floor_height=floor_height, + perimeter=est_perimeter, + built_form=home["BUILT_FORM"], + ) - if available_funding > cost_of_insulation: - available_funding = cost_of_insulation + # At the very least we'll need solid wall + solar + if home["needs_solid_wall"] and home["could_take_solar"]: + measure = "EWI + Solar" + + total_cost = insulation_needed * ewi_per_m2 + solar_pv_cost + + eco4_project_abs = get_eco4_abs(home["TOTAL_FLOOR_AREA"]) + eco4_available_funding = eco4_project_abs * eco4_abs + + cost_of_work_after_funding = total_cost - eco4_available_funding + cost_of_work_after_funding = 0 if cost_of_work_after_funding < 0 else cost_of_work_after_funding to_append = { **to_append, + "scheme": "eco4", + "available_funding": eco4_available_funding, + "measure": measure, + "project_abs": eco4_project_abs, + "cost_of_work": total_cost, + "cost_of_work_after_funding": cost_of_work_after_funding, + } + + estimated_costs.append(to_append) + continue + + # Check if it needs the walls done + if home["needs_cavity_done"]: + cost_of_insulation = insulation_needed * cwi_per_m2 + + cost_of_work_after_funding = cost_of_insulation - available_funding + cost_of_work_after_funding = 0 if cost_of_work_after_funding < 0 else cost_of_work_after_funding + + to_append = { + **to_append, + "scheme": "gbis", "available_funding": available_funding, "measure": "Cavity Wall Insulation", - "project_abs": project_abs + "project_abs": project_abs, + "cost_of_work": cost_of_insulation, + "cost_of_work_after_funding": cost_of_work_after_funding } estimated_costs.append(to_append) @@ -130,14 +180,17 @@ def app(): roof_area = float(home["TOTAL_FLOOR_AREA"]) / n_floors cost_of_insulation = roof_area * flat_roof_insulation_per_m2 - if available_funding > cost_of_insulation: - available_funding = cost_of_insulation + cost_of_work_after_funding = cost_of_insulation - available_funding + cost_of_work_after_funding = 0 if cost_of_work_after_funding < 0 else cost_of_work_after_funding to_append = { **to_append, + "scheme": "gbis", "available_funding": available_funding, "measure": measure, - "project_abs": project_abs + "project_abs": project_abs, + "cost_of_work": cost_of_insulation, + "cost_of_work_after_funding": cost_of_work_after_funding } estimated_costs.append(to_append) @@ -145,13 +198,10 @@ def app(): estimated_costs = pd.DataFrame(estimated_costs) - estimated_costs.groupby("measure")["available_funding"].mean() - estimated_costs["measure"].value_counts() - estimated_costs.to_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/sfr/estimated_costs_gbis.csv") - epc_data[["UPRN", "ADDRESS", "POSTCODE"]].to_csv( - "/Users/khalimconn-kowlessar/Documents/hestia/sfr/council_tax_bands_sample.csv") + # epc_data[["UPRN", "ADDRESS", "POSTCODE"]].to_csv( + # "/Users/khalimconn-kowlessar/Documents/hestia/sfr/council_tax_bands_sample.csv") n_properties_for_ashp = epc_data[ (epc_data["PROPERTY_TYPE"] == "House") & diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index e40c1736..fea2d8db 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -91,7 +91,7 @@ class HeatingRecommender: # If the property already has room heaters then we recommend HHR as an option since the home already has # a variation of room heaters - + hhr_suitable = no_mains or self.has_electric_heating_description or self.has_room_heaters return ( @@ -130,6 +130,13 @@ class HeatingRecommender: not self.property.main_heating["has_mains_gas"] and self.property.data["mains-gas-flag"] ) + # Additionally, if the property has a gas connection, is using gas heating but doesn't have a boiler, + # we recommend a boiler + non_boiler_gas_heating = ( + self.property.data["mains-gas-flag"] and + self.property.main_heating["has_mains_gas"] and + not self.property.main_heating["has_boiler"] + ) is_valid = ( ( @@ -138,7 +145,8 @@ class HeatingRecommender: electic_heating_has_mains or has_room_heaters or portable_heaters_has_mains or - non_gas_boiler + non_gas_boiler or + non_boiler_gas_heating ) and (not ashp_only_heating_recommendation) and ("boiler_upgrade" in measures) and @@ -842,12 +850,13 @@ class HeatingRecommender: has_inefficient_space_heating = self.property.data["mainheat-energy-eff"] in ["Very Poor", "Poor", "Average"] - has_inefficient_mains_water = ( - self.property.hotwater["clean_description"] in ["From main system"] and + # We check if there's a mains connection and the hot water is inefficient, as this will improve with a boiler + has_inefficient_water = ( + self.property.data["mains-gas-flag"] and self.property.data["hot-water-energy-eff"] in ["Very Poor", "Poor", "Average"] ) - if has_inefficient_space_heating or has_inefficient_mains_water: + if has_inefficient_space_heating or has_inefficient_water: boiler_size = self.estimate_boiler_size( property_type=self.property.data["property-type"], built_form=self.property.data["built-form"], diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index fea53e2b..e6225299 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -1364,86 +1364,135 @@ testing_examples = [ "heating_controls_recommendation_descriptions": [], "notes": "The property has warm air electricaire heating, so we recommend ASHP and HHR. It also has a mains" "connection so we recommend a gas condensing boiler" + }, + { + "epc": { + 'lmk-key': '272170070262009042917361440218801', 'address1': '52, Chiswick Walk', 'address2': None, + 'address3': None, 'postcode': 'B37 6TA', 'building-reference-number': 479790668, + 'current-energy-rating': 'F', 'potential-energy-rating': 'E', 'current-energy-efficiency': 31, + 'potential-energy-efficiency': 50, 'property-type': 'Flat', 'built-form': 'End-Terrace', + 'inspection-date': '2009-04-29', 'local-authority': 'E08000029', 'constituency': 'E14000812', + 'county': None, + 'lodgement-date': '2009-04-29', 'transaction-type': 'marketed sale', 'environment-impact-current': 37, + 'environment-impact-potential': 42, 'energy-consumption-current': 548, 'energy-consumption-potential': 459, + 'co2-emissions-current': 5.8, 'co2-emiss-curr-per-floor-area': 89, 'co2-emissions-potential': 5.0, + 'lighting-cost-current': 60, 'lighting-cost-potential': 30, 'heating-cost-current': 751, + 'heating-cost-potential': 601, 'hot-water-cost-current': 239, 'hot-water-cost-potential': 129, + 'total-floor-area': 65.04, 'energy-tariff': 'Single', 'mains-gas-flag': 'Y', 'floor-level': '1st', + 'flat-top-storey': 'Y', 'flat-storey-count': 1.0, 'main-heating-controls': 2504.0, + 'multi-glaze-proportion': 100.0, 'glazed-type': 'double glazing installed during or after 2002', + 'glazed-area': 'Normal', 'extension-count': 0, 'number-habitable-rooms': 3, 'number-heated-rooms': 2, + 'low-energy-lighting': 0, 'number-open-fireplaces': 0, + 'hotwater-description': 'Electric immersion, standard tariff', 'hot-water-energy-eff': 'Very Poor', + 'hot-water-env-eff': 'Poor', 'floor-description': 'To external air, no insulation (assumed)', + 'floor-energy-eff': None, 'windows-description': 'Fully double glazed', 'windows-energy-eff': 'Good', + 'windows-env-eff': 'Good', 'walls-description': 'System built, as built, no insulation (assumed)', + 'walls-energy-eff': 'Very Poor', 'walls-env-eff': 'Very Poor', + 'secondheat-description': 'Portable electric heaters', + 'roof-description': 'Pitched, limited insulation (assumed)', 'roof-energy-eff': 'Very Poor', + 'roof-env-eff': 'Very Poor', 'mainheat-description': 'Warm air, mains gas', 'mainheat-energy-eff': 'Good', + 'mainheat-env-eff': 'Good', 'mainheatcont-description': 'Programmer and room thermostat', + 'mainheatc-energy-eff': 'Average', 'mainheatc-env-eff': 'Average', + 'lighting-description': 'No low energy lighting', 'lighting-energy-eff': 'Very Poor', + 'lighting-env-eff': 'Very Poor', + 'main-fuel': 'mains gas - this is for backwards compatibility only and should not be used', + 'wind-turbine-count': 0, 'heat-loss-corridor': 'unheated corridor', 'unheated-corridor-length': 5.63, + 'floor-height': 2.32, 'photo-supply': 0.0, 'solar-water-heating-flag': 'N', + 'mechanical-ventilation': 'natural', 'address': '52, Chiswick Walk', 'local-authority-label': 'Solihull', + 'constituency-label': 'Meriden', 'posttown': 'BIRMINGHAM', + 'construction-age-band': 'England and Wales: 1967-1975', 'lodgement-datetime': '2009-04-29 17:36:14', + 'tenure': 'owner-occupied', 'fixed-lighting-outlets-count': None, 'low-energy-fixed-light-count': None, + 'uprn': 100070955137, 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, + 'sheating-env-eff': None + }, + "heating_recommendation_descriptions": [ + 'Upgrade to a new condensing boiler Upgrade heating controls to Room thermostat, programmer and TRVs', + 'Upgrade to a new condensing boiler Upgrade heating controls to Smart Thermostats, room sensors and smart ' + 'radiator valves (time & temperature zone control)' + ], + "heating_controls_recommendation_descriptions": [], + "notes": "This property has warm air mains gas heating, so we recommend a gas condensing boiler" } ] -import random -from pathlib import Path -import inspect -import pandas as pd - -# this can be used to get example data to build the test cases -src_file_path = inspect.getfile(lambda: None) -EPC_DIRECTORY = Path(src_file_path).parent / "local_data" / "all-domestic-certificates" -epc_directories = [entry for entry in EPC_DIRECTORY.iterdir() if entry.is_dir()] -directory = random.sample(epc_directories, 1)[0] -data = pd.read_csv(directory / "certificates.csv", low_memory=False) -# Rename the columns to the same format as the api returns -data.columns = [c.replace("_", "-").lower() for c in data.columns] -data["floor-height"] = data["floor-height"].fillna(2.45) - -used_examples = pd.DataFrame( - [ - { - "mainheat-description": x["epc"]["mainheat-description"], - "mainheat-energy-eff": x["epc"]["mainheat-energy-eff"], - "property-type": x["epc"]["property-type"], - "built-form": x["epc"]["built-form"], - "used": True - } for x in testing_examples - ] -) - -data = data.merge( - used_examples, how="left", on=["mainheat-description", "mainheat-energy-eff", "built-form", "property-type"] -) -data = data[pd.isnull(data["used"])].drop(columns=["used"]) - -eg = data.sample(1).to_dict("records")[0] -print(eg["mainheat-description"]) -print(eg["mainheat-energy-eff"]) -print(eg["property-type"]) -print(eg["built-form"]) -print(eg["mainheatcont-description"]) - -### We also use the Midlands EPC F/G portfolio to get examples to create tests - -completed_descriptions = [ - "Portable electric heaters assumed for most rooms", - "Boiler and radiators, oil", - "Boiler and radiators, mains gas", - "Room heaters, mains gas", - "No system present: electric heaters assumed", - "Room heaters, electric", - "Electric storage heaters", - "Boiler and radiators, LPG", - "Boiler and radiators, electric", - "Boiler and radiators, dual fuel (mineral and wood)", - "Boiler and radiators, coal", - "Boiler and radiators, smokeless fuel", - "Boiler and radiators, wood pellets", - "Room heaters, dual fuel (mineral and wood)", - "Air source heat pump, radiators, electric", - "Portable electric heaters assumed for most rooms, Room heaters, electric", - "Boiler and radiators, mains gas, Electric storage heaters", - "Room heaters, anthracite", - "Room heaters, mains gas, Room heaters, dual fuel (mineral and wood)", - "Electric underfloor heating", -] - -portfolio = pd.read_excel( - "/Users/khalimconn-kowlessar/Documents/hestia/Customers/sfr/20240820 portfolio_epc_data.xlsx" -) -portfolio.columns = [c.replace("_", "-").lower() for c in portfolio.columns] -portfolio = portfolio[~portfolio["mainheat-description"].isin(completed_descriptions)] -portfolio['sheating-energy-eff'] = None -portfolio['sheating-env-eff'] = None -portfolio["lodgement-datetime"] = portfolio["lodgement-datetime"].astype(str) - -print(portfolio["mainheat-description"].value_counts()) - -eg = portfolio[ - (portfolio["mainheat-description"] == "Warm air, Electricaire") -].sample(1) -eg = eg.squeeze().to_dict() -print(eg) +# import random +# from pathlib import Path +# import inspect +# import pandas as pd +# +# # this can be used to get example data to build the test cases +# src_file_path = inspect.getfile(lambda: None) +# EPC_DIRECTORY = Path(src_file_path).parent / "local_data" / "all-domestic-certificates" +# epc_directories = [entry for entry in EPC_DIRECTORY.iterdir() if entry.is_dir()] +# directory = random.sample(epc_directories, 1)[0] +# data = pd.read_csv(directory / "certificates.csv", low_memory=False) +# # Rename the columns to the same format as the api returns +# data.columns = [c.replace("_", "-").lower() for c in data.columns] +# data["floor-height"] = data["floor-height"].fillna(2.45) +# +# used_examples = pd.DataFrame( +# [ +# { +# "mainheat-description": x["epc"]["mainheat-description"], +# "mainheat-energy-eff": x["epc"]["mainheat-energy-eff"], +# "property-type": x["epc"]["property-type"], +# "built-form": x["epc"]["built-form"], +# "used": True +# } for x in testing_examples +# ] +# ) +# +# data = data.merge( +# used_examples, how="left", on=["mainheat-description", "mainheat-energy-eff", "built-form", "property-type"] +# ) +# data = data[pd.isnull(data["used"])].drop(columns=["used"]) +# +# eg = data.sample(1).to_dict("records")[0] +# print(eg["mainheat-description"]) +# print(eg["mainheat-energy-eff"]) +# print(eg["property-type"]) +# print(eg["built-form"]) +# print(eg["mainheatcont-description"]) +# +# ### We also use the Midlands EPC F/G portfolio to get examples to create tests +# +# completed_descriptions = [ +# "Portable electric heaters assumed for most rooms", +# "Boiler and radiators, oil", +# "Boiler and radiators, mains gas", +# "Room heaters, mains gas", +# "No system present: electric heaters assumed", +# "Room heaters, electric", +# "Electric storage heaters", +# "Boiler and radiators, LPG", +# "Boiler and radiators, electric", +# "Boiler and radiators, dual fuel (mineral and wood)", +# "Boiler and radiators, coal", +# "Boiler and radiators, smokeless fuel", +# "Boiler and radiators, wood pellets", +# "Room heaters, dual fuel (mineral and wood)", +# "Air source heat pump, radiators, electric", +# "Portable electric heaters assumed for most rooms, Room heaters, electric", +# "Boiler and radiators, mains gas, Electric storage heaters", +# "Room heaters, anthracite", +# "Room heaters, mains gas, Room heaters, dual fuel (mineral and wood)", +# "Electric underfloor heating", +# "Warm air, Electricaire" +# ] +# +# portfolio = pd.read_excel( +# "/Users/khalimconn-kowlessar/Documents/hestia/Customers/sfr/20240820 portfolio_epc_data.xlsx" +# ) +# portfolio.columns = [c.replace("_", "-").lower() for c in portfolio.columns] +# portfolio = portfolio[~portfolio["mainheat-description"].isin(completed_descriptions)] +# portfolio['sheating-energy-eff'] = None +# portfolio['sheating-env-eff'] = None +# portfolio["lodgement-datetime"] = portfolio["lodgement-datetime"].astype(str) +# +# print(portfolio["mainheat-description"].value_counts()) +# +# eg = portfolio[ +# (portfolio["mainheat-description"] == "Warm air, mains gas") +# ].sample(1) +# eg = eg.squeeze().to_dict() +# print(eg) From b34f1faca0e10fbabd00a01b07af6909391dbf37 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 17 Sep 2024 16:28:07 +0100 Subject: [PATCH 023/166] adding new costs to backend --- etl/costs/app.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/etl/costs/app.py b/etl/costs/app.py index 59852cc5..85c2410e 100644 --- a/etl/costs/app.py +++ b/etl/costs/app.py @@ -11,7 +11,7 @@ import inspect src_file_path = inspect.getfile(lambda: None) -DATA_DIRECTORY = Path(src_file_path).parent / "local_data" / "20240626 Hestia Materials.xlsx" +DATA_DIRECTORY = Path(src_file_path).parent / "local_data" / "20240917 Hestia Materials.xlsx" # Environment file is at the same level as this file ENV_FILE = Path(src_file_path).parent / "etl" / "costs" / ".env" dotenv.load_dotenv(ENV_FILE) @@ -46,6 +46,17 @@ def push_costs_to_db(engine, costs_df): session.commit() +def set_current_costs_inactive(engine): + """ + Set all current costs to inactive in the database. + + :param engine: The SQLAlchemy engine connected to your database. + """ + with Session(engine) as session: + session.query(Material).update({Material.is_active: False}) + session.commit() + + def app(): """ This application uploads the cost data to our database @@ -108,6 +119,11 @@ def app(): costs[col] = costs[col].fillna(0) # Push the costs to the database + # Since this is just uploading all of the new costs to the database, we make all of the current costs inactive + print("Setting all current costs to inactive") + set_current_costs_inactive(db_engine) + + print("Pushing costs to db") push_costs_to_db(db_engine, costs) From 335823d3af3e34db42624b00d46353457282eb25 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 18 Sep 2024 10:54:52 +0100 Subject: [PATCH 024/166] fixing incorrect descriptions and removing test expecations for homes that shouldn't have an ashp --- recommendations/HeatingRecommender.py | 39 +++++++++++-- .../test_data/heating_recommendations_data.py | 58 ++++++++----------- 2 files changed, 57 insertions(+), 40 deletions(-) diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index fea2d8db..3daf0268 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -41,6 +41,20 @@ class HeatingRecommender: "high_heat_retention_storage_heater", ] } + }, + "Portable electric heaters assumed for most rooms, room heaters, electric": { + "hhr": { + "mainheating_description": "Electric storage heaters, radiators", + "recommendation_description": "Install high heat retention electric storage heaters.", + "controls_prefix": "" + }, + "boiler": { + "mainheating_description": "Boiler and radiators, mains gas", + "recommendation_description": "Upgrade to a new condensing boiler.", + "controls_suffix": "" + }, + # These are the heating types we need to produce a dual heating recommendation + "dual": None } } @@ -91,7 +105,7 @@ class HeatingRecommender: # If the property already has room heaters then we recommend HHR as an option since the home already has # a variation of room heaters - + hhr_suitable = no_mains or self.has_electric_heating_description or self.has_room_heaters return ( @@ -160,6 +174,12 @@ class HeatingRecommender: if self.property.main_heating["clean_description"] not in self.DUAL_HEATING_DESCRIPTIONS: return + # if we have set dual to None, we do not produce a dual heating recommendation + if self.DUAL_HEATING_DESCRIPTIONS[ + self.property.main_heating["clean_description"] + ]["dual"] is None: + return + dual_heating_description = self.DUAL_HEATING_DESCRIPTIONS[ self.property.main_heating["clean_description"] ]["dual"]["types"] @@ -688,10 +708,15 @@ class HeatingRecommender: # We only recommend Celect-type controls if the current heating system is not Celect-type controls if self.property.main_heating_controls["clean_description"] != self.high_heat_retention_contols_desc: if self.dual_heating: - if self.DUAL_HEATING_DESCRIPTIONS[self.property.main_heating["clean_description"]]["hhr"][ - "controls_prefix" - ] == "current_controls": + + controls_prefix = self.DUAL_HEATING_DESCRIPTIONS[ + self.property.main_heating["clean_description"] + ]["hhr"]["controls_prefix"] + + if controls_prefix == "current_controls": description_prefix = self.property.main_heating_controls["clean_description"] + elif controls_prefix == "": + description_prefix = "" else: raise NotImplementedError("Implement me") else: @@ -735,6 +760,10 @@ class HeatingRecommender: self.property.number_of_rooms ) ) + # To be conservative, we adjust if we still have 1 room + if (number_heated_rooms == 1) and (self.property.number_of_rooms > 2): + number_heated_rooms = self.property.number_of_rooms - 1 + # Upgrade to electric storage heaters costs = self.costs.high_heat_electric_storage_heaters( number_heated_rooms=number_heated_rooms @@ -870,7 +899,7 @@ class HeatingRecommender: self.property.main_heating["clean_description"] ]["boiler"]["recommendation_description"] else: - description = "Upgrade to a new condensing boiler" + description = "Upgrade to a new condensing boiler." new_heating_eff = ( "Good" if self.property.data["mainheat-energy-eff"] in ["Very Poor", "Poor", "Average"] diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index e6225299..e073ac99 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -94,7 +94,7 @@ testing_examples = [ 'uprn-source': 'Address Matched', }, "heating_recommendation_descriptions": [ - "Install high heat retention electric storage heaters and upgrade heating controls to High Heat Retention " + "Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention " "Storage Heater Controls" ], "heating_controls_recommendation_descriptions": [], @@ -198,7 +198,7 @@ testing_examples = [ 'uprn': 100021560521.0, 'uprn-source': 'Address Matched', }, "heating_recommendation_descriptions": [ - 'Upgrade to a new condensing boiler' + 'Upgrade to a new condensing boiler.' ], "heating_controls_recommendation_descriptions": [ 'Upgrade heating controls to Room thermostat, programmer and TRVs', @@ -251,17 +251,14 @@ testing_examples = [ 'uprn': 100021936225.0, 'uprn-source': 'Address Matched', }, "heating_recommendation_descriptions": [ - 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' - 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' - 'scheme grant', ], "heating_controls_recommendation_descriptions": [ 'upgrade heating controls to Room thermostat, programmer and TRVs', 'Upgrade heating controls to Smart Thermostats, room sensors and smart radiator valves (time & ' 'temperature zone control)' ], - "notes": "Because this property already has a boiler, we don't recommend HHR. We only have a " - "heating recommendation for an air source heat pump. Because the heating controls are " + "notes": "Because this property already has a boiler, we don't recommend HHR. We don't recommend an ashp " + "because the home is mid-terraced. Because the heating controls are " "Programmer, no room thermostat, we have a programmer, room thermostat and trvs recommendation" "for heating controls and for TTZC." }, @@ -306,7 +303,7 @@ testing_examples = [ 'uprn': 43088770.0, 'uprn-source': 'Address Matched', }, "heating_recommendation_descriptions": [ - 'Install high heat retention electric storage heaters and upgrade heating controls to High Heat Retention ' + 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' 'Storage Heater Controls' ], "heating_controls_recommendation_descriptions": [], @@ -408,10 +405,10 @@ testing_examples = [ 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None }, "heating_recommendation_descriptions": [ - 'Upgrade to a new condensing boiler upgrade heating controls to Room thermostat, programmer and TRVs', + 'Upgrade to a new condensing boiler. upgrade heating controls to Room thermostat, programmer and TRVs', 'Install high heat retention electric storage heaters. upgrade heating controls to High Heat Retention ' 'Storage Heater Controls', - 'Upgrade to a new condensing boiler upgrade heating controls to Smart Thermostats, room sensors and smart ' + 'Upgrade to a new condensing boiler. upgrade heating controls to Smart Thermostats, room sensors and smart ' 'radiator valves (time & temperature zone control)' ], "heating_controls_recommendation_descriptions": [], @@ -506,8 +503,8 @@ testing_examples = [ 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None }, "heating_recommendation_descriptions": [ - 'Upgrade to a new condensing boiler upgrade heating controls to Room thermostat, programmer and TRVs', - 'Upgrade to a new condensing boiler Upgrade heating controls to Smart Thermostats, room sensors and smart ' + 'Upgrade to a new condensing boiler. upgrade heating controls to Room thermostat, programmer and TRVs', + 'Upgrade to a new condensing boiler. Upgrade heating controls to Smart Thermostats, room sensors and smart ' 'radiator valves (time & temperature zone control)', 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' @@ -557,8 +554,8 @@ testing_examples = [ }, "heating_recommendation_descriptions": [ - 'Upgrade to a new condensing boiler upgrade heating controls to Room thermostat, programmer and TRVs', - 'Upgrade to a new condensing boiler Upgrade heating controls to Smart Thermostats, room sensors and smart ' + 'Upgrade to a new condensing boiler. upgrade heating controls to Room thermostat, programmer and TRVs', + 'Upgrade to a new condensing boiler. Upgrade heating controls to Smart Thermostats, room sensors and smart ' 'radiator valves (time & temperature zone control)', 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' 'Storage Heater Controls', @@ -608,22 +605,13 @@ testing_examples = [ 'uprn-source': 'Energy Assessor', 'sheating-energy-eff': None, 'sheating-env-eff': None }, "heating_recommendation_descriptions": [ - 'Upgrade to a new condensing boiler upgrade heating controls to Room thermostat, programmer and TRVs', - 'Upgrade to a new condensing boiler Upgrade heating controls to Smart Thermostats, room sensors and smart ' - 'radiator valves (time & temperature zone control)', + 'Upgrade to a new condensing boiler. Upgrade heating controls to Room thermostat, programmer and TRVs', 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' 'Storage Heater Controls', + 'Upgrade to a new condensing boiler. Upgrade heating controls to Smart Thermostats, room sensors and ' + 'smart radiator valves (time & temperature zone control)' ], - "heating_controls_recommendation_descriptions": [ - 'Upgrade to a new condensing boiler upgrade heating controls to Room thermostat, programmer and TRVs', - 'Upgrade to a new condensing boiler Upgrade heating controls to Smart Thermostats, room sensors and smart ' - 'radiator valves (time & temperature zone control)', - 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' - 'Storage Heater Controls', - 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' - 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' - 'scheme grant' - ], + "heating_controls_recommendation_descriptions": [], "notes": "This has a form of assumed electric heating and has a mains connection so we recommend HHR, boiler" "upgrade and ASHP" }, @@ -667,8 +655,8 @@ testing_examples = [ 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None }, "heating_recommendation_descriptions": [ - 'Upgrade to a new condensing boiler Upgrade heating controls to Room thermostat, programmer and TRVs', - 'Upgrade to a new condensing boiler Upgrade heating controls to Smart Thermostats, room sensors and smart ' + 'Upgrade to a new condensing boiler. Upgrade heating controls to Room thermostat, programmer and TRVs', + 'Upgrade to a new condensing boiler. Upgrade heating controls to Smart Thermostats, room sensors and smart ' 'radiator valves (time & temperature zone control)', 'Install high heat retention electric storage heaters. The current electric heaters may be retrofit with ' 'high heat retention storage controls however this is dependent on the existing system and may not be ' @@ -1258,8 +1246,8 @@ testing_examples = [ 'uprn-source': 'Energy Assessor', 'sheating-energy-eff': None, 'sheating-env-eff': None }, "heating_recommendation_descriptions": [ - 'Upgrade to a new condensing boiler Upgrade heating controls to Room thermostat, programmer and TRVs', - 'Upgrade to a new condensing boiler Upgrade heating controls to Smart Thermostats, room sensors and smart ' + 'Upgrade to a new condensing boiler. Upgrade heating controls to Room thermostat, programmer and TRVs', + 'Upgrade to a new condensing boiler. Upgrade heating controls to Smart Thermostats, room sensors and smart ' 'radiator valves (time & temperature zone control)', 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' 'Storage Heater Controls' @@ -1355,8 +1343,8 @@ testing_examples = [ 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' 'scheme grant', - 'Upgrade to a new condensing boiler Upgrade heating controls to Room thermostat, programmer and TRVs', - 'Upgrade to a new condensing boiler Upgrade heating controls to Smart Thermostats, room sensors and smart ' + 'Upgrade to a new condensing boiler. Upgrade heating controls to Room thermostat, programmer and TRVs', + 'Upgrade to a new condensing boiler. Upgrade heating controls to Smart Thermostats, room sensors and smart ' 'radiator valves (time & temperature zone control)', 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' 'Storage Heater Controls' @@ -1406,8 +1394,8 @@ testing_examples = [ 'sheating-env-eff': None }, "heating_recommendation_descriptions": [ - 'Upgrade to a new condensing boiler Upgrade heating controls to Room thermostat, programmer and TRVs', - 'Upgrade to a new condensing boiler Upgrade heating controls to Smart Thermostats, room sensors and smart ' + 'Upgrade to a new condensing boiler. Upgrade heating controls to Room thermostat, programmer and TRVs', + 'Upgrade to a new condensing boiler. Upgrade heating controls to Smart Thermostats, room sensors and smart ' 'radiator valves (time & temperature zone control)' ], "heating_controls_recommendation_descriptions": [], From 8ecd3e4bafebcbcc224c77383a9ed4aa3d95eb10 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 18 Sep 2024 11:18:38 +0100 Subject: [PATCH 025/166] fixed the heating and heating controls unit tests --- .../test_data/heating_recommendations_data.py | 49 +++++++++++-------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index e073ac99..8697e095 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -253,7 +253,7 @@ testing_examples = [ "heating_recommendation_descriptions": [ ], "heating_controls_recommendation_descriptions": [ - 'upgrade heating controls to Room thermostat, programmer and TRVs', + 'Upgrade heating controls to Room thermostat, programmer and TRVs', 'Upgrade heating controls to Smart Thermostats, room sensors and smart radiator valves (time & ' 'temperature zone control)' ], @@ -354,7 +354,7 @@ testing_examples = [ 'scheme grant' ], "heating_controls_recommendation_descriptions": [ - 'upgrade heating controls to Room thermostat, programmer and TRVs', + 'Upgrade heating controls to Room thermostat, programmer and TRVs', 'Upgrade heating controls to Smart Thermostats, room sensors and smart radiator valves (time & ' 'temperature zone control)' @@ -405,10 +405,10 @@ testing_examples = [ 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None }, "heating_recommendation_descriptions": [ - 'Upgrade to a new condensing boiler. upgrade heating controls to Room thermostat, programmer and TRVs', - 'Install high heat retention electric storage heaters. upgrade heating controls to High Heat Retention ' + 'Upgrade to a new condensing boiler. Upgrade heating controls to Room thermostat, programmer and TRVs', + 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' 'Storage Heater Controls', - 'Upgrade to a new condensing boiler. upgrade heating controls to Smart Thermostats, room sensors and smart ' + 'Upgrade to a new condensing boiler. Upgrade heating controls to Smart Thermostats, room sensors and smart ' 'radiator valves (time & temperature zone control)' ], "heating_controls_recommendation_descriptions": [], @@ -456,11 +456,14 @@ testing_examples = [ "heating_recommendation_descriptions": [ 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' - 'scheme grant' + 'scheme grant', + 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' + 'Storage Heater Controls' + ], "heating_controls_recommendation_descriptions": [], "notes": "This property has an oil boiler and doesn't have a mains gas connection so we can only recommend" - "an air source heat pump" + "an air source heat pump and HHR (since if the home has a non-gas boiler, we recommend HHR)" }, { "epc": { @@ -503,16 +506,18 @@ testing_examples = [ 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None }, "heating_recommendation_descriptions": [ - 'Upgrade to a new condensing boiler. upgrade heating controls to Room thermostat, programmer and TRVs', - 'Upgrade to a new condensing boiler. Upgrade heating controls to Smart Thermostats, room sensors and smart ' - 'radiator valves (time & temperature zone control)', + 'Upgrade to a new condensing boiler. Upgrade heating controls to Room thermostat, programmer and TRVs', + 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' + 'Storage Heater Controls', 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' - 'scheme grant' + 'scheme grant', + 'Upgrade to a new condensing boiler. Upgrade heating controls to Smart Thermostats, room sensors and ' + 'smart radiator valves (time & temperature zone control)' ], "heating_controls_recommendation_descriptions": [], "notes": "This property has room heaters, from the mains gas supply. We recommend a boiler upgrade as" - "well as an air source heat pump" + "well as an air source heat pump and HHR (since the home has a room heater set up)" }, { "epc": { @@ -554,17 +559,15 @@ testing_examples = [ }, "heating_recommendation_descriptions": [ - 'Upgrade to a new condensing boiler. upgrade heating controls to Room thermostat, programmer and TRVs', + 'Upgrade to a new condensing boiler. Upgrade heating controls to Room thermostat, programmer and TRVs', 'Upgrade to a new condensing boiler. Upgrade heating controls to Smart Thermostats, room sensors and smart ' 'radiator valves (time & temperature zone control)', 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' 'Storage Heater Controls', - 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' - 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' - 'scheme grant' ], "heating_controls_recommendation_descriptions": [], - "notes": "This property has assumed electric heaters. Boiler upgrade, HHR and ASHP are all recommended" + "notes": "This property has assumed electric heaters. Boiler upgrade, HHR are recommended. We don't recommend" + "an ASHP off of the bat because it's mid-terrace." }, { "epc": { @@ -1100,9 +1103,12 @@ testing_examples = [ 'may be retrofit with high heat retention storage controls however this is dependent on the existing ' 'system and may not be possible. Upgrade heating controls to High Heat Retention Storage Heater Controls' ], - "heating_controls_recommendation_descriptions": [], + "heating_controls_recommendation_descriptions": [ + 'Upgrade heating controls to Smart Thermostats, room sensors and smart radiator valves (time & ' + 'temperature zone control)' + ], "notes": "This property has dual heating. A boiler and electric storage heaters. The heating is efficient so" - "we recommend ASHP and HHR" + "we recommend ASHP and HHR. We also recommend upgrading the heating controls for the boiler" }, { "epc": { @@ -1152,7 +1158,10 @@ testing_examples = [ 'may be retrofit with high heat retention storage controls however this is dependent on the existing ' 'system and may not be possible. Upgrade heating controls to High Heat Retention Storage Heater Controls' ], - "heating_controls_recommendation_descriptions": [], + "heating_controls_recommendation_descriptions": [ + 'Upgrade heating controls to Smart Thermostats, room sensors and smart radiator valves (time & ' + 'temperature zone control)' + ], "notes": "This property is a modified version of the previous dual heating property, where we lower the" "starting heating efficiency so that we a combined heating upgrade to both the boiler and the electric" "storage heaters" From 781e19be992c17db3c57cc5da81b27491c846717 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 18 Sep 2024 11:51:08 +0100 Subject: [PATCH 026/166] fixed some basic unit tests --- backend/Property.py | 3 +++ etl/sfr/midlands_portfolio_asset_list.py | 11 +++++++++++ 2 files changed, 14 insertions(+) create mode 100644 etl/sfr/midlands_portfolio_asset_list.py diff --git a/backend/Property.py b/backend/Property.py index b15c9f25..0d194a79 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -187,6 +187,9 @@ class Property: # This additional condition data should change how we pass kwargs to this. We should no longer need to pass # kwargs to this class, but instead, we should pass the energy assessment condition data + energy_assessment = ( + {"condition": {}, "energy_assessment_is_newer": False} if energy_assessment is None else energy_assessment + ) self.energy_assessment_condition_data = energy_assessment["condition"] self.energy_assessment_is_newer = energy_assessment["energy_assessment_is_newer"] diff --git a/etl/sfr/midlands_portfolio_asset_list.py b/etl/sfr/midlands_portfolio_asset_list.py new file mode 100644 index 00000000..0434b45a --- /dev/null +++ b/etl/sfr/midlands_portfolio_asset_list.py @@ -0,0 +1,11 @@ +import pandas as pd + + +def app(): + """ + This script sets up + :return: + """ + + # Read in the portfolio EPC data + epc_data = pd.read_excel() From c693c6a633b09ce7a7464164f915c8f047cc6c9d Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 18 Sep 2024 12:19:28 +0100 Subject: [PATCH 027/166] updating write to db for solar api, wip --- backend/app/db/functions/solar_functions.py | 108 ++++++++++++-------- backend/app/plan/router.py | 1 - etl/sfr/midlands_portfolio_asset_list.py | 43 +++++++- 3 files changed, 108 insertions(+), 44 deletions(-) diff --git a/backend/app/db/functions/solar_functions.py b/backend/app/db/functions/solar_functions.py index 59243f01..3ead1551 100644 --- a/backend/app/db/functions/solar_functions.py +++ b/backend/app/db/functions/solar_functions.py @@ -1,5 +1,6 @@ import datetime import pytz +from sqlalchemy import select from sqlalchemy.orm import Session from sqlalchemy.orm.exc import NoResultFound from backend.app.db.models.solar import Solar, SolarScenario @@ -38,57 +39,80 @@ def get_solar_data(session: Session, longitude: float = None, latitude: float = def store_batch_data(session: Session, api_data: dict, uprns_to_location: list, scenarios_data: list): """ This function will store the API data to the solar table against all of the UPRNs with longitude and latitude. + If a record already exists in the Solar table by UPRN, it will be updated instead of creating a new one. + Similarly, if a scenario exists in SolarScenario by number_panels, it will also be updated. + :param session: The database session :param api_data: The API data to store - :param uprns_to_location: A list of dictionaries containing uprn, longitude, and latitude + :param uprns_to_location: A list of dictionaries containing UPRN, longitude, and latitude :param scenarios_data: A list of dictionaries containing scenario data for each UPRN """ try: - - # Insert data into the Solar table and get the IDs - solar_records = [] + # Insert or update data into the Solar table for data in uprns_to_location: - solar_record = Solar( - uprn=data['uprn'], - longitude=data['longitude'], - latitude=data['latitude'], - google_api_response=api_data, - updated_at=datetime.datetime.now(pytz.utc) - ) - solar_records.append(solar_record) - session.add(solar_record) + existing_solar = session.execute(select(Solar).where(Solar.uprn == data['uprn'])).scalar_one_or_none() - session.flush() # Flush to get the IDs generated - - for record in solar_records: - session.refresh(record) # Refresh to populate the ID fields - - # Retrieve the IDs of the inserted records - inserted_ids = {record.uprn: record.id for record in solar_records} - - # Prepare the data for SolarScenario - scenario_records = [] - for data in uprns_to_location: - solar_id = inserted_ids.get(data['uprn']) - for scenario in scenarios_data: - scenario_record = SolarScenario( - solar_id=solar_id, - scenario_type=scenario['scenario_type'], - number_panels=scenario['number_panels'], - array_kwhp=scenario['array_kwhp'], - lifetime_dc_kwh=scenario['lifetime_dc_kwh'], - yearly_dc_kwh=scenario['yearly_dc_kwh'], - lifetime_ac_kwh=scenario.get('lifetime_ac_kwh'), # Optional field - yearly_ac_kwh=scenario.get('yearly_ac_kwh'), # Optional field - cost=scenario['cost'], - expected_payback_years=scenario.get('expected_payback_years'), # Optional field - panelled_roof_area=scenario['panelled_roof_area'], - is_default=scenario['is_default'] + if existing_solar: + # Update the existing record + existing_solar.longitude = data['longitude'] + existing_solar.latitude = data['latitude'] + existing_solar.google_api_response = api_data + existing_solar.updated_at = datetime.datetime.now(pytz.utc) + solar_id = existing_solar.id + else: + # Insert a new record + solar_record = Solar( + uprn=data['uprn'], + longitude=data['longitude'], + latitude=data['latitude'], + google_api_response=api_data, + updated_at=datetime.datetime.now(pytz.utc) ) - scenario_records.append(scenario_record) + session.add(solar_record) + session.flush() # Flush to get the IDs generated + session.refresh(solar_record) # Refresh to populate the ID field + solar_id = solar_record.id - # Insert data into the SolarScenario table - session.bulk_save_objects(scenario_records) + # Insert or update data in the SolarScenario table + for scenario in scenarios_data: + existing_scenario = session.execute( + select(SolarScenario).where( + SolarScenario.solar_id == solar_id, + SolarScenario.number_panels == scenario['number_panels'] + ) + ).scalar_one_or_none() + + if existing_scenario: + # Update the existing scenario record + existing_scenario.scenario_type = scenario['scenario_type'] + existing_scenario.array_kwhp = scenario['array_kwhp'] + existing_scenario.lifetime_dc_kwh = scenario['lifetime_dc_kwh'] + existing_scenario.yearly_dc_kwh = scenario['yearly_dc_kwh'] + existing_scenario.lifetime_ac_kwh = scenario.get('lifetime_ac_kwh') # Optional field + existing_scenario.yearly_ac_kwh = scenario.get('yearly_ac_kwh') # Optional field + existing_scenario.cost = scenario['cost'] + existing_scenario.expected_payback_years = scenario.get('expected_payback_years') # Optional field + existing_scenario.panelled_roof_area = scenario['panelled_roof_area'] + existing_scenario.is_default = scenario['is_default'] + else: + # Insert a new scenario record + scenario_record = SolarScenario( + solar_id=solar_id, + scenario_type=scenario['scenario_type'], + number_panels=scenario['number_panels'], + array_kwhp=scenario['array_kwhp'], + lifetime_dc_kwh=scenario['lifetime_dc_kwh'], + yearly_dc_kwh=scenario['yearly_dc_kwh'], + lifetime_ac_kwh=scenario.get('lifetime_ac_kwh'), # Optional field + yearly_ac_kwh=scenario.get('yearly_ac_kwh'), # Optional field + cost=scenario['cost'], + expected_payback_years=scenario.get('expected_payback_years'), # Optional field + panelled_roof_area=scenario['panelled_roof_area'], + is_default=scenario['is_default'] + ) + session.add(scenario_record) + + # Commit the changes after all operations session.commit() except Exception as e: diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index e925fe00..b5cb96f6 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -30,7 +30,6 @@ from backend.app.plan.utils import get_cleaned from backend.app.utils import epc_to_sap_lower_bound, sap_to_epc from backend.ml_models.api import ModelApi -from backend.ml_models.AnnualBillSavings import AnnualBillSavings from backend.Property import Property from backend.apis.GoogleSolarApi import GoogleSolarApi diff --git a/etl/sfr/midlands_portfolio_asset_list.py b/etl/sfr/midlands_portfolio_asset_list.py index 0434b45a..01a01907 100644 --- a/etl/sfr/midlands_portfolio_asset_list.py +++ b/etl/sfr/midlands_portfolio_asset_list.py @@ -1,4 +1,5 @@ import pandas as pd +from utils.s3 import save_csv_to_s3 def app(): @@ -7,5 +8,45 @@ def app(): :return: """ + portfolio_id = 108 + # Read in the portfolio EPC data - epc_data = pd.read_excel() + epc_data = pd.read_excel( + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/sfr/20240820 portfolio_epc_data.xlsx" + ) + + asset_list = epc_data[ + [ + "ADDRESS1", "POSTCODE", "UPRN" + ] + ].copy().rename( + columns={ + "ADDRESS1": "address", + "POSTCODE": "postcode", + "UPRN": "uprn" + } + ) + + # Store data and prepare payload + + filename = f"{8}/{portfolio_id}/asset_list.csv" + save_csv_to_s3( + dataframe=asset_list, + bucket_name="retrofit-plan-inputs-dev", + file_name=filename + ) + + body = { + "portfolio_id": str(portfolio_id), + "housing_type": "Private", + "goal": "Increasing EPC", + "goal_value": "C", + "trigger_file_path": filename, + "already_installed_file_path": "", + "patches_file_path": "", + "non_invasive_recommendations_file_path": "", + "budget": None, + "scenario_name": "EPC C Package", + "multi_plan": True, + } + print(body) From ef6aad6425d9b2b4c1c77e25eb5a037eb1da35eb Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 18 Sep 2024 12:23:33 +0100 Subject: [PATCH 028/166] overwrite solar and solar scenario storage to db instead of creating a new record --- backend/apis/GoogleSolarApi.py | 3 --- backend/app/plan/router.py | 2 -- 2 files changed, 5 deletions(-) diff --git a/backend/apis/GoogleSolarApi.py b/backend/apis/GoogleSolarApi.py index 41ec7c11..a99d96c9 100644 --- a/backend/apis/GoogleSolarApi.py +++ b/backend/apis/GoogleSolarApi.py @@ -192,8 +192,6 @@ class GoogleSolarApi: if not self.need_to_store: return - logger.info("Storing to database") - scenarios_data = self.panel_performance.head(1)[ [ "n_panels", @@ -221,7 +219,6 @@ class GoogleSolarApi: scenarios_data["scenario_type"] = scenario_type scenarios_data = scenarios_data.to_dict(orient="records") - # TODO: Rather than just doing a straight insert, we should overwrite what's already there if it exists store_batch_data( session=session, api_data=self.insights_data, diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index b5cb96f6..d1578cc1 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -651,8 +651,6 @@ async def trigger_plan(body: PlanTriggerRequest): ) # Store the data in the database - # TODO: Rather than just doing a straight insert, we should overwrite what's already there if it - # exists solar_api_client.save_to_db( session=session, uprns_to_location=[ From 7d907ce8c09117ae787d9e85d40b67426a02b836 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 18 Sep 2024 13:32:42 +0100 Subject: [PATCH 029/166] fixing bug to allow all measures when no inclusions or exclusions are specified --- recommendations/Recommendations.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/recommendations/Recommendations.py b/recommendations/Recommendations.py index d2c1db1b..fbaf0f9b 100644 --- a/recommendations/Recommendations.py +++ b/recommendations/Recommendations.py @@ -79,7 +79,9 @@ class Recommendations: inclusions_full = [MEASURE_MAP[x] if x in MEASURE_MAP else x for x in self.inclusions] exclusions_full = [MEASURE_MAP[x] if x in MEASURE_MAP else x for x in self.exclusions] - if inclusions_full and exclusions_full: + # If inclusions and exclusions are empty, it means that nothing was specified, so we allow + # all recommendation types + if not inclusions_full and not exclusions_full: # All typical measures return self.all_specific_measures From 8a09a29956f2294d9d25855a378086ca682f9795 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 18 Sep 2024 18:53:04 +0100 Subject: [PATCH 030/166] adding new installer costs for solar pv to cost class --- backend/app/plan/schemas.py | 15 ++++- recommendations/Costs.py | 77 +++++++++++++++++++---- recommendations/Recommendations.py | 27 ++++---- recommendations/SolarPvRecommendations.py | 3 +- 4 files changed, 96 insertions(+), 26 deletions(-) diff --git a/backend/app/plan/schemas.py b/backend/app/plan/schemas.py index 2968babf..68f8bbf5 100644 --- a/backend/app/plan/schemas.py +++ b/backend/app/plan/schemas.py @@ -20,7 +20,7 @@ SPECIFIC_MEASURES = [ # Walls "internal_wall_insulation", "external_wall_insulation", - "cavity_wall_insulation" + "cavity_wall_insulation", # Roof "loft_insulation", "flat_roof_insulation", @@ -32,7 +32,20 @@ SPECIFIC_MEASURES = [ "boiler_upgrade", "high_heat_retention_storage_heater", "air_source_heat_pump", + "secondary_heating", + # Solar + "solar_pv", + # Windows Glazing + "windows", + # Mechanical ventilation + "ventilation", + # Other + "low_energy_lighting", + "fireplace", + "hot_water", +] +NON_INVASIVE_SPECIFIC_MEASURES = [ # Specific measures that will typically come from an energy assessment "trickle_vents", "draught_proofing", diff --git a/recommendations/Costs.py b/recommendations/Costs.py index 908a409a..71d20855 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -37,6 +37,30 @@ MCS_SOLAR_PV_COST_DATA = { "average_cost_per_kwh-Northern Ireland": 1347, } +INSTALLER_SOLAR_COSTS = [ + {'n_panels': 4, 'array_kwp': 1.6, 'cost': 3040.00, 'installer': 'CEG'}, + {'n_panels': 5, 'array_kwp': 2.1, 'cost': 3201.00, 'installer': 'CEG'}, + {'n_panels': 6, 'array_kwp': 2.5, 'cost': 3363.00, 'installer': 'CEG'}, + {'n_panels': 7, 'array_kwp': 2.9, 'cost': 3524.00, 'installer': 'CEG'}, + {'n_panels': 8, 'array_kwp': 3.3, 'cost': 3686.00, 'installer': 'CEG'}, + {'n_panels': 9, 'array_kwp': 3.7, 'cost': 3847.00, 'installer': 'CEG'}, + {'n_panels': 10, 'array_kwp': 4.1, 'cost': 4009.00, 'installer': 'CEG'}, + {'n_panels': 11, 'array_kwp': 4.5, 'cost': 4170.00, 'installer': 'CEG'}, + {'n_panels': 12, 'array_kwp': 4.9, 'cost': 4332.00, 'installer': 'CEG'}, + {'n_panels': 13, 'array_kwp': 5.3, 'cost': 4835.00, 'installer': 'CEG'}, + {'n_panels': 14, 'array_kwp': 5.7, 'cost': 5015.00, 'installer': 'CEG'}, + {'n_panels': 15, 'array_kwp': 6.2, 'cost': 5176.00, 'installer': 'CEG'}, + {'n_panels': 16, 'array_kwp': 6.6, 'cost': 5338.00, 'installer': 'CEG'}, + {'n_panels': 17, 'array_kwp': 7.0, 'cost': 5500.00, 'installer': 'CEG'}, + {'n_panels': 18, 'array_kwp': 7.4, 'cost': 6021.00, 'installer': 'CEG'} +] + +INSTALLER_SCAFFOLDING_COSTS = [ + {'stories': 1, 'description': '1 Story Scaffold', 'cost': 531.00, 'installer': 'CEG'}, + {'stories': 2, 'description': '2 Story Scaffold', 'cost': 841.00, 'installer': 'CEG'}, + {'stories': 3, 'description': '3 Story Scaffold', 'cost': 1077.00, 'installer': 'CEG'} +] + # This data is based on the MCS database, We use the larger figure between the 2023 and 2024 average, # to be conservative MCS_AIR_SOURCE_HEAT_PUMP_COST_DATA = { @@ -54,10 +78,27 @@ MCS_AIR_SOURCE_HEAT_PUMP_COST_DATA = { "Scotland": 12586, "Northern Ireland": 12000, # There are hardly any air source heat pump installs going on in Northern Ireland } + +INSTALLER_ASHP_COSTS = [ + {'capacity_kw': 5.0, 'brand': 'Mitsubishi', 'tank_size_liters': 150, 'cost': 10149.53, 'installer': 'CEG'}, + {'capacity_kw': 6.0, 'brand': 'Mitsubishi', 'tank_size_liters': 170, 'cost': 10823.48, 'installer': 'CEG'}, + {'capacity_kw': 8.5, 'brand': 'Mitsubishi', 'tank_size_liters': 200, 'cost': 11312.43, 'installer': 'CEG'}, + {'capacity_kw': 11.2, 'brand': 'Mitsubishi', 'tank_size_liters': 250, 'cost': 12156.75, 'installer': 'CEG'}, + {'capacity_kw': 14.0, 'brand': 'Mitsubishi', 'tank_size_liters': 300, 'cost': 14405.54, 'installer': 'CEG'}, + {'capacity_kw': 14.0, 'brand': 'Mitsubishi', 'tank_size_liters': 300, 'cost': 14405.54, 'installer': 'CEG'}, + {'capacity_kw': 17.0, 'brand': 'Grant', 'tank_size_liters': 300, 'cost': 14445.00, 'installer': 'CEG'}, + {'capacity_kw': 20.0, 'brand': 'Ecoforest', 'tank_size_liters': 400, 'cost': 21189.41, 'installer': 'CEG'}, + {'capacity_kw': None, 'brand': '2 x cascaded ASHPs', 'tank_size_liters': 500, 'cost': 22950.00, 'installer': 'CEG'} +] + BOILER_UPGRADE_SCHEME_ASHP_VALUE = 7500 -# This is based on quotes from installers -BATTERY_COST = 3500 +INSTALLER_SOLAR_BATTERY_COSTS = [ + {'capacity_kwh': 5, 'description': 'Battery Add on', 'cost': 2700.00, 'installer': 'CEG'}, + {'capacity_kwh': 10, 'description': 'Battery Add on', 'cost': 4300.00, 'installer': 'CEG'}, + {'capacity_kwh': 5, 'description': 'Battery Retrofit existing system', 'cost': 4250.00, 'installer': 'CEG'}, + {'capacity_kwh': 10, 'description': 'Battery Retrofit Existing system', 'cost': 5950.00, 'installer': 'CEG'} +] # This is based on https://www.checkatrade.com/blog/cost-guides/cost-smart-thermostat/ SMART_APPLIANCE_THERMOSTAT_COST = 400 @@ -1013,7 +1054,14 @@ class Costs: "labour_days": labour_days } - def solar_pv(self, wattage: float, has_battery: bool = False, array_cost=None): + def solar_pv( + self, wattage: float, + n_panels: int | float, + has_battery: bool = False, + array_cost=None, + n_floors: int = 1, + battery_kwh: int = 5, + ): """ Calculates the total cost for solar PV based data provided by the MCS dashboard, which contains @@ -1025,23 +1073,26 @@ class Costs: Price can also be benchmarked against this checkatrade article: https://www.checkatrade.com/blog/cost-guides/cost-of-solar-panel-installation/ - :param wattage: Peak wattage of the solar PV system] + :param wattage: Peak wattage of the solar PV system + :param n_panels: Number of solar panels :param has_battery: Bool, whether the system includes a battery :param array_cost: float, containing the cost of the solar PV array + :param n_floors: int, number of floors in the property, used to estimate the cost of scaffolding + :param battery_kwh: int, capacity of the battery in kWh. Defaulted to 5 """ - # Get the cost data relevant to the region - regional_cost = MCS_SOLAR_PV_COST_DATA["-".join(["average_cost_per_kwh", self.region])] + system_cost = [c for c in INSTALLER_SOLAR_COSTS if c["n_panels"] == n_panels][0]["cost"] - if array_cost is not None: - total_cost = array_cost - else: - kw = wattage / 1000 - total_cost = kw * regional_cost + total_cost = array_cost if array_cost is not None else system_cost if has_battery: - # The battery cost is based on the £3500 quote, recieved from installers - total_cost += BATTERY_COST + battery_cost = [c for c in INSTALLER_SOLAR_BATTERY_COSTS if c["capacity_kwh"] == battery_kwh][0]["cost"] + total_cost += battery_cost + + scaffolding_cost = [c for c in INSTALLER_SCAFFOLDING_COSTS if c["stories"] == n_floors][0]["cost"] + total_cost += scaffolding_cost + + # We add an additional cost for scaffolding subtotal_before_vat = total_cost / (1 + self.VAT_RATE) diff --git a/recommendations/Recommendations.py b/recommendations/Recommendations.py index fbaf0f9b..5037f450 100644 --- a/recommendations/Recommendations.py +++ b/recommendations/Recommendations.py @@ -18,9 +18,8 @@ from recommendations.DraughtProofingRecommendations import DraughtProofingRecomm from backend.ml_models.AnnualBillSavings import AnnualBillSavings from backend.apis.GoogleSolarApi import GoogleSolarApi import backend.app.assumptions as assumptions -from backend.app.plan.schemas import TYPICAL_MEASURE_TYPES, SPECIFIC_MEASURES, MEASURE_MAP +from backend.app.plan.schemas import SPECIFIC_MEASURES, MEASURE_MAP, NON_INVASIVE_SPECIFIC_MEASURES -ASHP_COP = 3 STARTING_DUMMY_ID_VALUE = -9999 @@ -50,8 +49,11 @@ class Recommendations: self.exclusions = exclusions if exclusions else [] self.inclusions = inclusions if inclusions else [] - self.all_typical_measures = TYPICAL_MEASURE_TYPES self.all_specific_measures = SPECIFIC_MEASURES + self.all_non_invase_measures = NON_INVASIVE_SPECIFIC_MEASURES + self.non_invasive_recommendation_types = [ + r["type"] for r in self.property_instance.non_invasive_recommendations + ] self.floor_recommender = FloorRecommendations(property_instance=property_instance, materials=materials) self.wall_recomender = WallRecommendations(property_instance=property_instance, materials=materials) @@ -82,14 +84,18 @@ class Recommendations: # If inclusions and exclusions are empty, it means that nothing was specified, so we allow # all recommendation types if not inclusions_full and not exclusions_full: - # All typical measures - return self.all_specific_measures + # All typical measures - this does not include non-invasive measures inless they are specified + return self.all_specific_measures + self.non_invasive_recommendation_types if inclusions_full: return inclusions_full if exclusions_full: - return [m for m in self.all_specific_measures if m not in exclusions_full] + measures = [ + m for m in self.all_specific_measures + self.non_invasive_recommendation_types + if m not in exclusions_full + ] + return measures def recommend(self): @@ -146,11 +152,10 @@ class Recommendations: if self.draught_proofing_recommender.recommendation: property_recommendations.append(self.draught_proofing_recommender.recommendation) - if "floor_insulation" in measures: - self.floor_recommender.recommend(phase=phase, measures=measures) - if self.floor_recommender.recommendations: - property_recommendations.append(self.floor_recommender.recommendations) - phase += 1 + self.floor_recommender.recommend(phase=phase, measures=measures) + if self.floor_recommender.recommendations: + property_recommendations.append(self.floor_recommender.recommendations) + phase += 1 if "windows" in measures and "mixed_glazing" not in non_invasive_recommendation_types: # If we have a mixed glazing recommendation, we prioritise this over the windows recommendation diff --git a/recommendations/SolarPvRecommendations.py b/recommendations/SolarPvRecommendations.py index d0d555c9..bbaffdda 100644 --- a/recommendations/SolarPvRecommendations.py +++ b/recommendations/SolarPvRecommendations.py @@ -196,7 +196,8 @@ class SolarPvRecommendations: cost_result = self.costs.solar_pv( wattage=recommendation_config["array_wattage"], has_battery=has_battery, - array_cost=non_invasive_recommendation.get("cost", None) + array_cost=non_invasive_recommendation.get("cost", None), + n_panels=recommendation_config["n_panels"], ) kw = np.floor(recommendation_config["array_wattage"] / 100) / 10 if has_battery: From 503a19291dcacf7d1a4ab9b09107bd5a59c49b95 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 18 Sep 2024 19:12:02 +0100 Subject: [PATCH 031/166] updating solar pv costs --- recommendations/Costs.py | 16 ++++++++++++++-- recommendations/SolarPvRecommendations.py | 11 ++++++++--- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/recommendations/Costs.py b/recommendations/Costs.py index 71d20855..671c4db7 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -55,6 +55,11 @@ INSTALLER_SOLAR_COSTS = [ {'n_panels': 18, 'array_kwp': 7.4, 'cost': 6021.00, 'installer': 'CEG'} ] +# CEG uses use Solshare as an inverter to provide solar PV to multiple flats. This costs £7500 for the inverter alone +# https://midsummerwholesale.co.uk/buy/solshare +INSTALLER_SOLAR_PV_INVERTER_COST = 7500 +INSTALLER_SOLAR_PV_INVERTER_LABOUR_COST = 500 # Just a rough guess to labour costs + INSTALLER_SCAFFOLDING_COSTS = [ {'stories': 1, 'description': '1 Story Scaffold', 'cost': 531.00, 'installer': 'CEG'}, {'stories': 2, 'description': '2 Story Scaffold', 'cost': 841.00, 'installer': 'CEG'}, @@ -1055,12 +1060,13 @@ class Costs: } def solar_pv( - self, wattage: float, + self, n_panels: int | float, has_battery: bool = False, array_cost=None, n_floors: int = 1, battery_kwh: int = 5, + needs_inverter=False ): """ @@ -1073,12 +1079,13 @@ class Costs: Price can also be benchmarked against this checkatrade article: https://www.checkatrade.com/blog/cost-guides/cost-of-solar-panel-installation/ - :param wattage: Peak wattage of the solar PV system :param n_panels: Number of solar panels :param has_battery: Bool, whether the system includes a battery :param array_cost: float, containing the cost of the solar PV array :param n_floors: int, number of floors in the property, used to estimate the cost of scaffolding :param battery_kwh: int, capacity of the battery in kWh. Defaulted to 5 + :param needs_inverter: Bool, whether the system needs an inverter, where the solar panels are feeding multiple + units """ system_cost = [c for c in INSTALLER_SOLAR_COSTS if c["n_panels"] == n_panels][0]["cost"] @@ -1092,6 +1099,11 @@ class Costs: scaffolding_cost = [c for c in INSTALLER_SCAFFOLDING_COSTS if c["stories"] == n_floors][0]["cost"] total_cost += scaffolding_cost + if needs_inverter: + total_cost += INSTALLER_SOLAR_PV_INVERTER_COST + # We also add an additional labour cost + total_cost += INSTALLER_SOLAR_PV_INVERTER_LABOUR_COST + # We add an additional cost for scaffolding subtotal_before_vat = total_cost / (1 + self.VAT_RATE) diff --git a/recommendations/SolarPvRecommendations.py b/recommendations/SolarPvRecommendations.py index bbaffdda..dc11ce4a 100644 --- a/recommendations/SolarPvRecommendations.py +++ b/recommendations/SolarPvRecommendations.py @@ -104,8 +104,13 @@ class SolarPvRecommendations: roof_coverage_percent = round(recommendation_config["panneled_roof_area"] / total_roof_area * 100) else: raise Exception("IMPLEMENT ME") - # Spread the cost to the individual units - adding a 20% contingency - total_cost = recommendation_config["total_cost"] / n_units + total_cost = self.costs.solar_pv( + array_cost=recommendation_config.get("cost", None), + n_panels=recommendation_config["n_panels"], + n_floors=self.property.number_of_storeys["number_of_storeys"], + needs_inverter=True, + )["total"] / n_units + kw = np.floor(recommendation_config["array_wattage"] / 100) / 10 # Default to a weeks work for a team of 3 people doing 8 hour days labour_days = 5 @@ -194,10 +199,10 @@ class SolarPvRecommendations: roof_coverage_percent = np.ceil(roof_coverage_percent / 10) * 10 for has_battery in [False, True]: cost_result = self.costs.solar_pv( - wattage=recommendation_config["array_wattage"], has_battery=has_battery, array_cost=non_invasive_recommendation.get("cost", None), n_panels=recommendation_config["n_panels"], + n_floors=self.property.number_of_floors ) kw = np.floor(recommendation_config["array_wattage"] / 100) / 10 if has_battery: From 767d0d3132a77549f06a64e7c7ee52bcf81923df Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 23 Sep 2024 15:55:07 +0100 Subject: [PATCH 032/166] updating costing methodology for new installer costs --- backend/apis/GoogleSolarApi.py | 18 ++++-- recommendations/Costs.py | 77 +++++++------------------- recommendations/WallRecommendations.py | 15 ----- 3 files changed, 32 insertions(+), 78 deletions(-) diff --git a/backend/apis/GoogleSolarApi.py b/backend/apis/GoogleSolarApi.py index a99d96c9..f1e3d2e9 100644 --- a/backend/apis/GoogleSolarApi.py +++ b/backend/apis/GoogleSolarApi.py @@ -42,6 +42,9 @@ class GoogleSolarApi: # your area installation_life_span = 20 + MIN_UNIT_PANELS = 4 # Minimum number of panels we allow for a domestic building + MIN_BUILDING_PANELS = 10 # Minimum number of panels we allow for a block of flats + def __init__(self, api_key, max_retries=5): """ Initialize the GoogleSolarApi class with the provided API key and maximum retries. @@ -250,6 +253,9 @@ class GoogleSolarApi: Optimise the solar panel configuration for the building. :return: """ + # If we look at the building level, we don't include any projects fewer than 10 panels, otherwise the + # minimum is 4 + min_panels = self.MIN_BUILDING_PANELS if is_building else self.MIN_UNIT_PANELS cost_instance = Costs(property_instance=property_instance) if property_instance is not None else None @@ -264,6 +270,10 @@ class GoogleSolarApi: roi_summary = [] for segment in roof_segment_summaries: + + if segment["panelsCount"] < min_panels: + continue + wattage = segment["panelsCount"] * self.insights_data["solarPotential"]["panelCapacityWatts"] generated_dc_energy = segment["yearlyEnergyDcKwh"] ratio = generated_dc_energy / wattage @@ -272,7 +282,9 @@ class GoogleSolarApi: cost = MCS_SOLAR_PV_COST_DATA["average_cost_per_kwh"] * (wattage / 1000) else: cost = cost_instance.solar_pv( - wattage=wattage, has_battery=False + n_panels=segment["panelsCount"], + has_battery=False, + n_floors=property_instance.number_of_floors, )["total"] roi_summary.append( @@ -330,10 +342,6 @@ class GoogleSolarApi: # We can have duplicate configurations panel_performance = panel_performance.drop_duplicates() - # If we look at the building level, we don't include any projects fewer than 10 panels, otherwise the - # minimum is 4 - min_panels = 10 if is_building else 4 - panel_performance = panel_performance[panel_performance["n_panels"] >= min_panels] if panel_performance.empty: self.panel_performance = pd.DataFrame( diff --git a/recommendations/Costs.py b/recommendations/Costs.py index 671c4db7..c71316ad 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -54,6 +54,8 @@ INSTALLER_SOLAR_COSTS = [ {'n_panels': 17, 'array_kwp': 7.0, 'cost': 5500.00, 'installer': 'CEG'}, {'n_panels': 18, 'array_kwp': 7.4, 'cost': 6021.00, 'installer': 'CEG'} ] +# This is the maximum number of panels that we have a cost from the installers for +INSTALLER_MAX_PANELS = 18 # CEG uses use Solshare as an inverter to provide solar PV to multiple flats. This costs £7500 for the inverter alone # https://midsummerwholesale.co.uk/buy/solshare @@ -362,7 +364,7 @@ class Costs: "labour_days": labour_days } - def internal_wall_insulation(self, wall_area, material, non_insulation_materials): + def internal_wall_insulation(self, wall_area, material): """ Broadly speaking, the high level steps to an internal wall insulation job are the following: @@ -401,74 +403,25 @@ class Costs: "labour_days": labour_days, } - # Extract and check the different types of data we'll need - demolition_data = [x for x in non_insulation_materials if x["type"] == "iwi_wall_demolition"] - vapour_barrier_data = [x for x in non_insulation_materials if x["type"] == "iwi_vapour_barrier"] - redecoration_data = [x for x in non_insulation_materials if x["type"] == "iwi_redecoration"] - if not demolition_data: - raise ValueError("No data found for iwi_wall_demolition") - - if (len(vapour_barrier_data) != 1) or (len(redecoration_data) != 3): - raise ValueError("Incorrect number of data entries for non-insulation materials") - # Break out the individual material costs # Since we don't know the exact wall construction, we take an average for demolition costs, since # the cost will depend on the type of wall construction - demolition_material_costs = np.mean([x["material_cost"] * wall_area for x in demolition_data]) - insulation_material_costs = material["material_cost"] * wall_area - vapour_barrier_material_costs = vapour_barrier_data[0]["material_cost"] * wall_area - redecoration_material_costs = sum([x["material_cost"] * wall_area for x in redecoration_data]) - demolition_plant_costs = np.mean([x["plant_cost"] * wall_area for x in demolition_data]) - - # Again for demolition, we average since we aren't sure which demolition process will be used - demolition_labour_costs = np.mean([x["labour_cost"] * wall_area for x in demolition_data]) - insulation_labour_costs = material["labour_cost"] * wall_area - vapour_barrier_labour_costs = vapour_barrier_data[0]["labour_cost"] * wall_area - redecoration_labour_costs = sum([x["labour_cost"] * wall_area for x in redecoration_data]) - - labour_costs = (demolition_labour_costs + insulation_labour_costs + vapour_barrier_labour_costs + - redecoration_labour_costs) - - labour_costs = labour_costs * self.labour_adjustment_factor - - materials_costs = (demolition_material_costs + insulation_material_costs + vapour_barrier_material_costs + - redecoration_material_costs) - - subtotal_before_profit = labour_costs + materials_costs + demolition_plant_costs - - contingency_cost = subtotal_before_profit * self.IWI_CONTINGENCY - preliminaries_cost = subtotal_before_profit * self.PRELIMINARIES - profit_cost = subtotal_before_profit * self.PROFIT_MARGIN - - subtotal_before_vat = subtotal_before_profit + contingency_cost + preliminaries_cost + profit_cost - - vat_cost = subtotal_before_vat * self.VAT_RATE - - total_cost = subtotal_before_vat + vat_cost - - demolition_labour_hours = np.mean([x["labour_hours_per_unit"] * wall_area for x in demolition_data]) - insulation_labour_hours = material["labour_hours_per_unit"] * wall_area - vapour_barrier_labour_hours = vapour_barrier_data[0]["labour_hours_per_unit"] * wall_area - redecoration_labour_hours = sum([x["labour_hours_per_unit"] * wall_area for x in redecoration_data]) - - labour_hours = (demolition_labour_hours + insulation_labour_hours + vapour_barrier_labour_hours + - redecoration_labour_hours) + total_including_vat = material["total_cost"] * wall_area + total_excluding_vat = total_including_vat / (1 + self.VAT_RATE) + vat_cost = total_including_vat - total_excluding_vat + # We estimate 1 weeks worth of work + labour_hours = 160 # To install internal wall insulation, a small to medium size project might be conducted by a team of 3-5 people labour_days = (labour_hours / 8) / 4 return { - "total": total_cost, - "subtotal": subtotal_before_vat, + "total": total_including_vat, + "subtotal": total_excluding_vat, "vat": vat_cost, - "contingency": contingency_cost, - "preliminaries": preliminaries_cost, - "material": materials_costs, - "profit": profit_cost, "labour_hours": labour_hours, "labour_days": labour_days, - "labour_cost": labour_costs } def suspended_floor_insulation(self, insulation_floor_area, material, non_insulation_materials): @@ -1088,7 +1041,15 @@ class Costs: units """ - system_cost = [c for c in INSTALLER_SOLAR_COSTS if c["n_panels"] == n_panels][0]["cost"] + if n_panels > INSTALLER_MAX_PANELS: + base_cost = [c for c in INSTALLER_SOLAR_COSTS if c["n_panels"] == INSTALLER_MAX_PANELS][0]["cost"] + cost_per_panel = [ + c for c in INSTALLER_SOLAR_COSTS if c["n_panels"] == (INSTALLER_MAX_PANELS - 1) + ][0]["cost"] + cost_per_panel = base_cost - cost_per_panel + system_cost = base_cost + (n_panels - INSTALLER_MAX_PANELS) * cost_per_panel + else: + system_cost = [c for c in INSTALLER_SOLAR_COSTS if c["n_panels"] == n_panels][0]["cost"] total_cost = array_cost if array_cost is not None else system_cost diff --git a/recommendations/WallRecommendations.py b/recommendations/WallRecommendations.py index a0c71860..0ddd7b0b 100644 --- a/recommendations/WallRecommendations.py +++ b/recommendations/WallRecommendations.py @@ -106,23 +106,10 @@ class WallRecommendations(Definitions): part for part in materials if part["type"] == "internal_wall_insulation" ] - self.internal_wall_non_insulation_materials = [ - part - for part in materials - if part["type"] - in ["iwi_wall_demolition", "iwi_vapour_barrier", "iwi_redecoration"] - ] - self.external_wall_insulation_materials = [ part for part in materials if part["type"] == "external_wall_insulation" ] - self.external_wall_non_insulation_materials = [ - part - for part in materials - if part["type"] in ["ewi_wall_demolition", "ewi_wall_preparation", "ewi_wall_redecoration"] - ] - def ewi_valid(self): """ This method check available data, to determine if a property is suitable for external wall insulation @@ -508,7 +495,6 @@ class WallRecommendations(Definitions): cost_result = self.costs.internal_wall_insulation( wall_area=self.property.insulation_wall_area, material=material.to_dict(), - non_insulation_materials=non_insulation_materials, ) already_installed = ( "internal_wall_insulation" @@ -617,7 +603,6 @@ class WallRecommendations(Definitions): iwi_recommendations = self._find_insulation( u_value=u_value, insulation_materials=pd.DataFrame(self.internal_wall_insulation_materials), - non_insulation_materials=self.internal_wall_non_insulation_materials, phase=phase, ) From fe86886adcef6225b31457fbaf5379745b30da82 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 23 Sep 2024 15:57:06 +0100 Subject: [PATCH 033/166] refactoring costing function for solid wall insulation| --- recommendations/Costs.py | 166 +------------------------ recommendations/WallRecommendations.py | 20 ++- 2 files changed, 9 insertions(+), 177 deletions(-) diff --git a/recommendations/Costs.py b/recommendations/Costs.py index c71316ad..101de0dd 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -364,26 +364,9 @@ class Costs: "labour_days": labour_days } - def internal_wall_insulation(self, wall_area, material): + def solid_wall_insulation(self, wall_area, material): """ - Broadly speaking, the high level steps to an internal wall insulation job are the following: - - 1) Demolition: This involves removing existing wall linings, fittings, and any other obstacles. - It's important to factor in the disposal of debris and the potential need for additional protective - measures to ensure the safety of the work area. - - 2) Insulation Installation: This is the core part of the process where the chosen insulation material is - applied. The choice of insulation material will depend on several factors including thermal performance, - wall construction, and space constraints. - - 3) Vapour Barrier Installation: This is crucial for preventing moisture from penetrating the insulation, - which can compromise its effectiveness and lead to mold growth. - - 4) Re-decoration: This involves applying plaster to the wall and then painting. - The quality of finish here is important for both aesthetic and functional reasons. - - 5) Trim and Finishing Work: Post-insulation, tasks such as re-installing skirting boards, door frames, - or window sills might be necessary. + Implements costing methodology now that we have direct quotes from installers. :return: """ @@ -638,151 +621,6 @@ class Costs: "labour_cost": labour_costs } - def external_wall_insulation(self, wall_area, material, non_insulation_materials): - """ - We characterise external wall insulation as the following steps: - - 1) Preparation of the Area: Tidying up the surroundings, trimming back foliage, and laying down protective - sheets to protect the flooring and landscaping around the work area. - - 2) Scaffolding Setup (if needed): Erecting scaffolding for safe access to the walls of semi-detached or - detached houses. For terraced houses or lower-level work, scaffolding might not be necessary. - - 3) Wall Surface Preparation: Cleaning the wall surface, removing any loose or flaking material, - and possibly applying a primer. If the existing wall is weak or damaged, partial or full replacement - of the top surface may be necessary. - - 4) Applying Primer: If the existing wall is suitable, applying a primer to improve adhesion of the insulation - boards and stabilize the wall surface, especially if it's old or weathered. - - 5) Insulation Application: Attaching insulation boards to the primed wall using adhesive, mechanical fixings, - or a combination of both. - - 6) Basecoat and Mesh Application: Applying a basecoat embedded with a reinforcing mesh over the insulation. - This layer provides strength and helps prevent cracking. - - 7) Decorative Finish: Applying a decorative finish, such as render or cladding, which protects the insulation - and provides an aesthetic look. - - 8) Reinstalling Fixtures: Reattaching any fixtures like downpipes, satellite dishes, or lighting fixtures that - were removed during preparation. Extensions or adjustments may be required due to the increased wall thickness. - - 9) Inspection and Cleanup: Conducting a thorough inspection to ensure quality and integrity of the EWI system, - followed by cleaning up the site to remove all debris and materials. - - In the actual materials data, at this point, we have costing for: - - wall preparation, hacking off existing wall finishes, linings, etc (ewi_wall_demolition) - - wall surface cleaning and priming (ewi_wall_preparation) - - insulation (external_wall_insulation) - - basecoat and mesh with decorative render topcoat finish (ewi_basecoat_and_mesh) - - All of this data comes from SPONS, however there are some clear features missing. Because we could not find - suitable cost records in SPONS for steps like cleaning the area, setting up small scale scaffolding, - re-attaching any fitings and cleaning up the area afterwards, instead we have accounted for these steps by - increasing the preliminaries rate. It is acknowldeged though, that this is not ideal and that the cost of these - steps should be included in the materials data. We will look to improve this in the future, with data from - installers - - :param wall_area: - :param material: - :param non_insulation_materials: - :return: - """ - - if material["is_installer_quote"]: - total_cost = material["total_cost"] * wall_area - # Add on a buffer for scaffolding - if self.property.data["property-type"] == "House": - total_cost += self.EWI_SCAFFOLDING_PRELIMINARIES * total_cost - - labour_hours = material["labour_hours_per_unit"] * wall_area - - # To install internal wall insulation, a small to medium size project might be conducted by a team of 3-5 - # people - labour_days = (labour_hours / 8) / 4 - - return { - "total": total_cost, - "labour_hours": labour_hours, - "labour_days": labour_days, - } - - # For semi detatched and detatched houses, as well as maisonettes, we price for scaffolding - - if self.property.data["property-type"] == "House": - if self.property.data["built-form"] in ['Semi-Detached', 'Detached', "End-Terrace"]: - preliminaries_rate = self.EWI_SCAFFOLDING_PRELIMINARIES - else: - preliminaries_rate = self.EWI_NO_SCAFFOLDING_PRELIMINARIES - elif self.property.data["property-type"] in ["Maisonette", "Flat"]: - preliminaries_rate = self.EWI_SCAFFOLDING_PRELIMINARIES - elif self.property.data["property-type"] == "Bungalow": - preliminaries_rate = self.EWI_NO_SCAFFOLDING_PRELIMINARIES - - demolition_data = [x for x in non_insulation_materials if x["type"] == "ewi_wall_demolition"] - preparation_data = [x for x in non_insulation_materials if x["type"] == "ewi_wall_preparation"] - redecoration_data = [x for x in non_insulation_materials if x["type"] == "ewi_wall_redecoration"] - - if (len(demolition_data) != 3) or (len(preparation_data) != 1) or (len(redecoration_data) != 1): - raise ValueError("Incorrect number of data entries for non-insulation materials") - - # Break out the individual material costs - # Since we don't know the exact wall construction, we take an average for demolition costs, since - # the cost will depend on the type of wall construction - demolition_material_costs = np.mean([x["material_cost"] * wall_area for x in demolition_data]) - insulation_material_costs = material["material_cost"] * wall_area - preparation_material_costs = preparation_data[0]["material_cost"] * wall_area - redecoration_material_costs = redecoration_data[0]["material_cost"] * wall_area - - demolition_plant_costs = np.mean([x["plant_cost"] * wall_area for x in demolition_data]) - - demolition_labour_costs = np.mean([x["labour_cost"] * wall_area for x in demolition_data]) - insulation_labour_costs = material["labour_cost"] * wall_area - preparation_labour_costs = preparation_data[0]["labour_cost"] * wall_area - redecoration_labour_costs = redecoration_data[0]["labour_cost"] * wall_area - - labour_costs = (demolition_labour_costs + insulation_labour_costs + redecoration_labour_costs + - preparation_labour_costs) - - labour_costs = labour_costs * self.labour_adjustment_factor - - materials_costs = (demolition_material_costs + insulation_material_costs + preparation_material_costs + - redecoration_material_costs) - - subtotal_before_profit = labour_costs + materials_costs + demolition_plant_costs - - contingency_cost = subtotal_before_profit * self.CONTINGENCY - preliminaries_cost = subtotal_before_profit * preliminaries_rate - profit_cost = subtotal_before_profit * self.PROFIT_MARGIN - - subtotal_before_vat = subtotal_before_profit + contingency_cost + preliminaries_cost + profit_cost - vat_cost = subtotal_before_vat * self.VAT_RATE - total_cost = subtotal_before_vat + vat_cost - - demolition_labour_hours = np.mean([x["labour_hours_per_unit"] * wall_area for x in demolition_data]) - insulation_labour_hours = material["labour_hours_per_unit"] * wall_area - preparation_labour_hours = preparation_data[0]["labour_hours_per_unit"] * wall_area - redecoration_labour_hours = redecoration_data[0]["labour_hours_per_unit"] * wall_area - - labour_hours = (demolition_labour_hours + insulation_labour_hours + redecoration_labour_hours + - preparation_labour_hours) - - # Assume a team of 3-5 people for a small to medium size project - labour_days = (labour_hours / 8) / 4 - - return { - "total": total_cost, - "subtotal": subtotal_before_vat, - "vat": vat_cost, - "contingency": contingency_cost, - "preliminaries": preliminaries_cost, - "material": materials_costs, - "profit": profit_cost, - "labour_hours": labour_hours, - "labour_days": labour_days, - "labour_cost": labour_costs - } - def low_energy_lighting(self, number_of_lights, number_current_lel_lights, material): """ diff --git a/recommendations/WallRecommendations.py b/recommendations/WallRecommendations.py index 0ddd7b0b..1c483bff 100644 --- a/recommendations/WallRecommendations.py +++ b/recommendations/WallRecommendations.py @@ -172,7 +172,6 @@ class WallRecommendations(Definitions): ewi_recommendations = self._find_insulation( u_value=u_value, insulation_materials=pd.DataFrame(self.external_wall_insulation_materials), - non_insulation_materials=self.external_wall_non_insulation_materials, phase=phase ) @@ -437,7 +436,7 @@ class WallRecommendations(Definitions): return simulation_config - def _find_insulation(self, u_value, insulation_materials, non_insulation_materials, phase): + def _find_insulation(self, u_value, insulation_materials, phase): lowest_selected_u_value = None recommendations = [] @@ -482,6 +481,11 @@ class WallRecommendations(Definitions): lowest_selected_u_value, new_u_value ) + cost_result = self.costs.solid_wall_insulation( + wall_area=self.property.insulation_wall_area, + material=material.to_dict(), + ) + if material["type"] == "internal_wall_insulation": if iwi_non_invasive_recommendations.get("cost") is not None: @@ -492,10 +496,6 @@ class WallRecommendations(Definitions): sap_points = iwi_non_invasive_recommendations.get("sap_points", None) survey = iwi_non_invasive_recommendations.get("survey", False) - cost_result = self.costs.internal_wall_insulation( - wall_area=self.property.insulation_wall_area, - material=material.to_dict(), - ) already_installed = ( "internal_wall_insulation" in self.property.already_installed @@ -511,12 +511,7 @@ class WallRecommendations(Definitions): sap_points = ewi_non_invasive_recommendations.get("sap_points", None) survey = ewi_non_invasive_recommendations.get("survey", False) - - cost_result = self.costs.external_wall_insulation( - wall_area=self.property.insulation_wall_area, - material=material.to_dict(), - non_insulation_materials=non_insulation_materials, - ) + already_installed = ( "external_wall_insulation" in self.property.already_installed @@ -594,7 +589,6 @@ class WallRecommendations(Definitions): insulation_materials=pd.DataFrame( self.external_wall_insulation_materials ), - non_insulation_materials=self.external_wall_non_insulation_materials, phase=phase, ) From d0f7c7f63a850cd830ca5716f6b7054e261a744f Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 23 Sep 2024 16:08:06 +0100 Subject: [PATCH 034/166] updating cwi costs --- recommendations/Costs.py | 37 +++++++------------------- recommendations/WallRecommendations.py | 2 +- 2 files changed, 10 insertions(+), 29 deletions(-) diff --git a/recommendations/Costs.py b/recommendations/Costs.py index 101de0dd..e1b16899 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -257,7 +257,6 @@ class Costs: :return: A dictionary containing detailed cost breakdown. """ - # CWI usually takes 1 day labour_hours = 8 labour_days = 1 @@ -272,40 +271,22 @@ class Costs: "labour_days": labour_days, } - material_cost_per_m2 = material["material_cost"] - - base_material_cost = material_cost_per_m2 * wall_area - labour_cost = material["labour_cost"] * wall_area * self.labour_adjustment_factor - - subtotal_before_profit = base_material_cost + labour_cost - - contingency_cost = subtotal_before_profit * self.CONTINGENCY - preliminaries_cost = subtotal_before_profit * self.PRELIMINARIES - profit_cost = subtotal_before_profit * self.PROFIT_MARGIN - - subtotal_before_vat = subtotal_before_profit + contingency_cost + preliminaries_cost + profit_cost - - vat_cost = subtotal_before_vat * self.VAT_RATE - - total_cost = subtotal_before_vat + vat_cost + total_including_vat = material["total_cost"] * wall_area if is_extraction_and_refill: - # bump up the cost of the work - total_cost = total_cost + CAVITY_EXTRACTION_COST * wall_area + total_including_vat = CAVITY_EXTRACTION_COST * wall_area # Additional 2 days work - labour_hours = labour_hours + (2 * 8) - labour_days = labour_days + 2 + labour_hours += + (2 * 8) + labour_days += + 2 + + total_excluding_vat = total_including_vat / (1 + self.VAT_RATE) + vat_cost = total_including_vat - total_excluding_vat return { - "total": total_cost, - "subtotal": subtotal_before_vat, + "total": total_including_vat, + "subtotal": total_excluding_vat, "vat": vat_cost, - "contingency": contingency_cost, - "preliminaries": preliminaries_cost, - "material": base_material_cost, - "profit": profit_cost, "labour_hours": labour_hours, - "labour_cost": labour_cost, "labour_days": labour_days } diff --git a/recommendations/WallRecommendations.py b/recommendations/WallRecommendations.py index 1c483bff..69bfdfb4 100644 --- a/recommendations/WallRecommendations.py +++ b/recommendations/WallRecommendations.py @@ -511,7 +511,7 @@ class WallRecommendations(Definitions): sap_points = ewi_non_invasive_recommendations.get("sap_points", None) survey = ewi_non_invasive_recommendations.get("survey", False) - + already_installed = ( "external_wall_insulation" in self.property.already_installed From 0c6d8121c2ccaf0075c5570c6e721af15b7fbf0b Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 24 Sep 2024 10:58:59 +0100 Subject: [PATCH 035/166] updating costing for loft insulation --- recommendations/Costs.py | 44 ++++++-------------------- recommendations/RoofRecommendations.py | 4 +-- 2 files changed, 11 insertions(+), 37 deletions(-) diff --git a/recommendations/Costs.py b/recommendations/Costs.py index e1b16899..69ff6073 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -298,51 +298,25 @@ class Costs: :return: A dictionary containing detailed cost breakdown. """ - labour_hours = material["labour_hours_per_unit"] * floor_area - # Assume a team of 1 person - labour_days = labour_hours / 8 - if material["is_installer_quote"]: total_cost = material["total_cost"] * floor_area return { "total": total_cost, - "labour_hours": labour_hours, - "labour_days": labour_days, + "labour_hours": 8, + "labour_days": 1, } - material_cost_per_m2 = material["material_cost"] - - # We inflate material costs due to recent price increases - material_cost_per_m2 = material_cost_per_m2 * 1.5 - - base_material_cost = material_cost_per_m2 * floor_area - labour_cost = material["labour_cost"] * floor_area * self.labour_adjustment_factor - - subtotal_before_profit = base_material_cost + labour_cost - - # We use high risk contingency because of the possibility of access issues and clearing existing insulation - contingency_cost = subtotal_before_profit * self.HIGH_RISK_CONTINGENCY - preliminaries_cost = subtotal_before_profit * self.PRELIMINARIES - profit_cost = subtotal_before_profit * self.PROFIT_MARGIN - - subtotal_before_vat = subtotal_before_profit + contingency_cost + preliminaries_cost + profit_cost - - vat_cost = subtotal_before_vat * self.VAT_RATE - - total_cost = subtotal_before_vat + vat_cost + total_including_vat = material["total_cost"] * floor_area + total_excluding_vat = total_including_vat / (1 + self.VAT_RATE) + vat_cost = total_including_vat - total_excluding_vat return { - "total": total_cost, - "subtotal": subtotal_before_vat, + "total": total_including_vat, + "subtotal": total_excluding_vat, "vat": vat_cost, - "contingency": contingency_cost, - "preliminaries": preliminaries_cost, - "material": base_material_cost, - "profit": profit_cost, - "labour_hours": labour_hours, - "labour_cost": labour_cost, - "labour_days": labour_days + "labour_hours": 8, + "labour_days": 1 } def solid_wall_insulation(self, wall_area, material): diff --git a/recommendations/RoofRecommendations.py b/recommendations/RoofRecommendations.py index 6635dd51..8c7e2291 100644 --- a/recommendations/RoofRecommendations.py +++ b/recommendations/RoofRecommendations.py @@ -44,10 +44,11 @@ class RoofRecommendations: self.recommendations = [] self.loft_insulation_materials = [ - part for part in materials if part["type"] == "loft_insulation" + part for part in materials if (part["type"] == "loft_insulation") and (part["is_installer_quote"]) ] self.loft_non_insulation_materials = [] + # We don't have proper installer quotes for flat roof insulation self.flat_roof_insulation_materials = [ part for part in materials if part["type"] == "flat_roof_insulation" ] @@ -266,7 +267,6 @@ class RoofRecommendations: lowest_selected_u_value = None recommendations = [] for _, insulation_material_group in insulation_materials.groupby("description"): - for _, material in insulation_material_group.iterrows(): # We make sure we hit a depth of 270mm. We should factor in any existing insulation if the From e6fc34741c9b58cb2de1d5091b3200c4f17ff3cb Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 24 Sep 2024 14:35:35 +0100 Subject: [PATCH 036/166] update flat roof insulation recommendations --- recommendations/Costs.py | 90 +-------------------- recommendations/FireplaceRecommendations.py | 2 +- recommendations/RoofRecommendations.py | 33 +++----- 3 files changed, 12 insertions(+), 113 deletions(-) diff --git a/recommendations/Costs.py b/recommendations/Costs.py index 69ff6073..08b05a8a 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -290,9 +290,9 @@ class Costs: "labour_days": labour_days } - def loft_insulation(self, floor_area, material): + def loft_and_flat_insulation(self, floor_area, material): """ - Calculates the total cost for cavity wall insulation based on material and labor costs, + Calculates the total cost for loft/flat roof insulation based on material and labor costs, including contingency, preliminaries, profit, and VAT. :return: A dictionary containing detailed cost breakdown. @@ -624,92 +624,6 @@ class Costs: "labour_cost": labour_cost } - def flat_roof_insulation(self, floor_area, material, non_insulation_materials): - """ - A model of a warm, flat roof construction can be seen in this video: - https://www.youtube.com/watch?v=WZ6Ng6YI9OA - Warm, flat roof insulation will normally be 100-125mm in depth - - We break this measure down into the following jobs to be done - 1) Preparation of the room. This involves cleaning the existing roof surface, removing any debris and repairing - any damage. Additionally, an edge barrier will likely need to be installed, to protect the sides of the - roof from water ingress. - 2) Primer Application. A layer of primer is applied to the clean roof surface to enhance the adhestia of - subsequent layers, and seal the existing roof surface. - 3) Vapour Proof Layer Installation. Lay a vapour control layer to prevent moisture ingress from inside the - building, which is essential in warm roof construction. - 4) Insulation Layer Application. Place and securely fix insulation boards over the roof. These could be rigid - boards like PIR (Polyisocyanurate). - 5) Waterproofing Membrane Installation: Cover the insulation (and timber layer, if used) with a - waterproofing membrane, like EPDM, PVC, or bituminous felt. Carefully seal all joints, edges, and around any - roof penetrations to ensure water tightness - - :param floor_area: Area of the flat roof to be insulated, based on the area of the floor - :param material: Selected insulation material - :param non_insulation_materials: Non-insulation materials required for the job - :return: - """ - - preparation_data_m2 = [ - x for x in non_insulation_materials if - (x["type"] == "flat_roof_preparation") and (x["cost_unit"] == "gbp_per_m2") - ] - vapour_barrier_data = [x for x in non_insulation_materials if x["type"] == "flat_roof_vapour_barrier"] - waterproofing_data = [x for x in non_insulation_materials if x["type"] == "flat_roof_waterproofing"] - - if (len(preparation_data_m2) != 2) or (len(vapour_barrier_data) != 1) or ( - len(waterproofing_data) != 1): - raise ValueError("Incorrect number of data entries for non-insulation materials") - - # Break out the individual material costs - preparation_m2_material_costs = sum([x["material_cost"] * floor_area for x in preparation_data_m2]) - vapour_barrier_material_costs = vapour_barrier_data[0]["material_cost"] * floor_area - insulation_material_costs = material["material_cost"] * floor_area - - preparation_m2_labour_costs = sum([x["labour_cost"] * floor_area for x in preparation_data_m2]) - vapour_barrier_labour_costs = vapour_barrier_data[0]["labour_cost"] * floor_area - - # For waterproofing and upstand, we only have a total cost - waterproofing_total_costs = waterproofing_data[0]["total_cost"] * floor_area - - labour_costs = preparation_m2_labour_costs + vapour_barrier_labour_costs - labour_costs = labour_costs * self.labour_adjustment_factor - - materials_costs = preparation_m2_material_costs + vapour_barrier_material_costs + insulation_material_costs - - subtotal_before_profit = labour_costs + materials_costs + waterproofing_total_costs - - contingency_cost = subtotal_before_profit * self.FLAT_ROOF_CONTINGENCY - preliminaries_cost = subtotal_before_profit * self.PRELIMINARIES - profit_cost = subtotal_before_profit * self.PROFIT_MARGIN - - subtotal_before_vat = subtotal_before_profit + contingency_cost + preliminaries_cost + profit_cost - vat_cost = subtotal_before_vat * self.VAT_RATE - total_cost = subtotal_before_vat + vat_cost - - preparation_m2_labour_hours = sum([x["labour_hours_per_unit"] * floor_area for x in preparation_data_m2]) - vapour_barrier_labour_hours = vapour_barrier_data[0]["labour_hours_per_unit"] * floor_area - waterproofing_labour_hours = waterproofing_data[0]["labour_hours_per_unit"] * floor_area - - labour_hours = preparation_m2_labour_hours + vapour_barrier_labour_hours + waterproofing_labour_hours - - # To install flat roof insulation, assume a small/medium project might be conducted by a team of 2-4. - # We'll assume a team of 2 since a lot of the roofs will be on the smaller side and will review this later - labour_days = (labour_hours / 8) / 2 - - return { - "total": total_cost, - "subtotal": subtotal_before_vat, - "vat": vat_cost, - "contingency": contingency_cost, - "preliminaries": preliminaries_cost, - "material": materials_costs, - "profit": profit_cost, - "labour_hours": labour_hours, - "labour_days": labour_days, - "labour_cost": labour_costs - } - def window_glazing(self, number_of_windows, material, is_secondary_glazing=False): """ We characterise the jobs to be done for window glazing as the following: diff --git a/recommendations/FireplaceRecommendations.py b/recommendations/FireplaceRecommendations.py index 9a9d7f76..163728dd 100644 --- a/recommendations/FireplaceRecommendations.py +++ b/recommendations/FireplaceRecommendations.py @@ -9,7 +9,7 @@ class FireplaceRecommendations(Definitions): """ # This is our base assumption for the cost of the work - COST_OF_WORK = 300 + COST_OF_WORK = 235 def __init__( self, diff --git a/recommendations/RoofRecommendations.py b/recommendations/RoofRecommendations.py index 8c7e2291..89b8205f 100644 --- a/recommendations/RoofRecommendations.py +++ b/recommendations/RoofRecommendations.py @@ -46,19 +46,12 @@ class RoofRecommendations: self.loft_insulation_materials = [ part for part in materials if (part["type"] == "loft_insulation") and (part["is_installer_quote"]) ] - self.loft_non_insulation_materials = [] # We don't have proper installer quotes for flat roof insulation self.flat_roof_insulation_materials = [ part for part in materials if part["type"] == "flat_roof_insulation" ] - self.flat_roof_non_insulation_materials = [ - part for part in materials if part["type"] in [ - "flat_roof_preparation", "flat_roof_vapour_barrier", "flat_roof_waterproofing" - ] - ] - # Extract the insulation thickness from the roof, which is used throughout this method self.insulation_thickness = convert_thickness_to_numeric( self.property.roof["insulation_thickness"], @@ -252,10 +245,8 @@ class RoofRecommendations: if is_pitched: insulation_materials = self.loft_insulation_materials - non_insulation_materials = self.loft_non_insulation_materials elif is_flat: insulation_materials = self.flat_roof_insulation_materials - non_insulation_materials = self.flat_roof_non_insulation_materials else: raise ValueError("Roof is not pitched or flat") @@ -297,14 +288,16 @@ class RoofRecommendations: if new_u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE: lowest_selected_u_value = update_lowest_selected_u_value(lowest_selected_u_value, new_u_value) + cost_result = self.costs.loft_and_flat_insulation( + floor_area=self.property.insulation_floor_area, + material=material + ) + + already_installed = material["type"] in self.property.already_installed + if already_installed: + cost_result = override_costs(cost_result) + if material["type"] == "loft_insulation": - cost_result = self.costs.loft_insulation( - floor_area=self.property.insulation_floor_area, - material=material - ) - already_installed = "loft_insulation" in self.property.already_installed - if already_installed: - cost_result = override_costs(cost_result) new_thickness = insulation_thickness + material["depth"] # This is based on the values we have in the training data @@ -341,14 +334,6 @@ class RoofRecommendations: new_description = f"Pitched, {int(proposed_depth)}mm loft insulation" elif material["type"] == "flat_roof_insulation": - cost_result = self.costs.flat_roof_insulation( - floor_area=self.property.insulation_floor_area, - material=material, - non_insulation_materials=non_insulation_materials - ) - already_installed = "flat_roof_insulation" in self.property.already_installed - if already_installed: - cost_result = override_costs(cost_result) new_description = "Flat, insulated" new_efficiency = "Good" else: From 4a216fb42301b781b8f95ba6aee55bb816ed1759 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 24 Sep 2024 15:43:33 +0100 Subject: [PATCH 037/166] handling property with u-values in survey which is not a new dwelling --- etl/costs/app.py | 2 ++ recommendations/HeatingRecommender.py | 17 ++++++++--------- recommendations/RoofRecommendations.py | 2 +- recommendations/VentilationRecommendations.py | 2 +- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/etl/costs/app.py b/etl/costs/app.py index 85c2410e..797191d2 100644 --- a/etl/costs/app.py +++ b/etl/costs/app.py @@ -82,6 +82,7 @@ def app(): db_engine = create_engine(db_string, pool_size=5, max_overflow=5) cwi_costs = pd.read_excel(DATA_DIRECTORY, sheet_name="cavity_wall_insulation", header=0) + ventilation_costs = pd.read_excel(DATA_DIRECTORY, sheet_name="Ventilation", header=0) loft_insulation_costs = pd.read_excel(DATA_DIRECTORY, sheet_name="loft_insulation", header=0) iwi_costs = pd.read_excel(DATA_DIRECTORY, sheet_name="internal_wall_insulation", header=0) suspended_floor_costs = pd.read_excel(DATA_DIRECTORY, sheet_name="suspended_floor_insulation", header=0) @@ -95,6 +96,7 @@ def app(): costs = pd.concat( [ cwi_costs, + ventilation_costs, loft_insulation_costs, iwi_costs, suspended_floor_costs, diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index 3daf0268..b54f89bb 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -1034,17 +1034,16 @@ class HeatingRecommender: # Overwrite the existing boiler recommendation self.heating_recommendations.extend(combined_recommendations) else: - # We increment the recommendation phase, since the heating controls are separate from the boiler upgrade - # but we'll only upgrade if we have a heating recommendation - has_heating_recommendation = any( - rec["type"] == "heating" for rec in self.heating_recommendations - ) - if has_heating_recommendation: - recommendation_phase += 1 - # The heating controls recommendation is distrinct from the boiler upgrade recommendation - # We insert phase into the recommendations for heating controls + # We consider a heating control upgrade as a measure which occures in the same phase as a boiler upgrade + # Namely, we have the following options within this phase + # 1) Boiler + heating controls + # 2) Boiler only + # 3) Heating controls only + # But they are options that are not mutually exclusive + # So, we actually set heating controls as a heating recommendation for recommendation in controls_recommender.recommendation: recommendation["phase"] = recommendation_phase + # recommendation["type"] = "heating" self.heating_control_recommendations.extend(controls_recommender.recommendation) diff --git a/recommendations/RoofRecommendations.py b/recommendations/RoofRecommendations.py index 89b8205f..fbd99d67 100644 --- a/recommendations/RoofRecommendations.py +++ b/recommendations/RoofRecommendations.py @@ -132,7 +132,7 @@ class RoofRecommendations: # The Roof is already compliant return - if self.property.data["transaction-type"] == "new dwelling": + if self.property.data["transaction-type"] in ["new dwelling", "not sale or rental"]: return raise NotImplementedError("Implement me") diff --git a/recommendations/VentilationRecommendations.py b/recommendations/VentilationRecommendations.py index 34439827..5913ab9c 100644 --- a/recommendations/VentilationRecommendations.py +++ b/recommendations/VentilationRecommendations.py @@ -52,7 +52,7 @@ class VentilationRecommendations(Definitions): already_installed = "cavity_wall_insulation" in self.property.already_installed - estimated_cost = n_units * part[0]["cost"] if not already_installed else 0 + estimated_cost = n_units * part[0]["total_cost"] if not already_installed else 0 labour_hours = 4 * n_units if not already_installed else 0 labour_days = 4 * n_units / 8.0 if not already_installed else 0 From 7236ff2ed2dfd86fe2ab7b7ade4d6636003dd9c1 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 24 Sep 2024 16:47:21 +0100 Subject: [PATCH 038/166] ensure panel performance meets minimum number of panels in the case of a double roof segment --- backend/apis/GoogleSolarApi.py | 1 + recommendations/WallRecommendations.py | 20 ++++++-------------- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/backend/apis/GoogleSolarApi.py b/backend/apis/GoogleSolarApi.py index f1e3d2e9..c82c9c9a 100644 --- a/backend/apis/GoogleSolarApi.py +++ b/backend/apis/GoogleSolarApi.py @@ -491,6 +491,7 @@ class GoogleSolarApi: panel_performance["n_panels"] = panel_performance["n_panels_halved"] panel_performance = panel_performance.drop(columns=["n_panels_halved"]) + panel_performance = panel_performance[panel_performance["n_panels"] >= min_panels] self.panel_performance = panel_performance diff --git a/recommendations/WallRecommendations.py b/recommendations/WallRecommendations.py index 69bfdfb4..4902ae03 100644 --- a/recommendations/WallRecommendations.py +++ b/recommendations/WallRecommendations.py @@ -69,6 +69,7 @@ class WallRecommendations(Definitions): "Timber frame, as built, no insulation": "Timber frame, with external insulation", 'Timber frame, as built, partial insulation': 'Timber frame, with external insulation', "Sandstone or limestone, as built, no insulation": "Sandstone or limestone, with external insulation", + "Sandstone, as built, no insulation": "Sandstone, with external insulation", } # These are the ending descriptions we consider for walls with internal insulation @@ -83,6 +84,7 @@ class WallRecommendations(Definitions): "Timber frame, as built, no insulation": "Timber frame, with internal insulation", 'Timber frame, as built, partial insulation': 'Timber frame, with internal insulation', "Sandstone or limestone, as built, no insulation": "Sandstone or limestone, with internal insulation", + "Sandstone, as built, no insulation": "Sandstone, with internal insulation", } def __init__( @@ -486,6 +488,10 @@ class WallRecommendations(Definitions): material=material.to_dict(), ) + already_installed = material["type"] in self.property.already_installed + if already_installed: + cost_result = override_costs(cost_result) + if material["type"] == "internal_wall_insulation": if iwi_non_invasive_recommendations.get("cost") is not None: @@ -496,13 +502,6 @@ class WallRecommendations(Definitions): sap_points = iwi_non_invasive_recommendations.get("sap_points", None) survey = iwi_non_invasive_recommendations.get("survey", False) - already_installed = ( - "internal_wall_insulation" - in self.property.already_installed - ) - if already_installed: - cost_result = override_costs(cost_result) - new_description = self.get_internal_external_wall_description( self.INTERNALLY_INSULATED_WALL_DESCRIPTIONS, new_u_value ) @@ -512,13 +511,6 @@ class WallRecommendations(Definitions): sap_points = ewi_non_invasive_recommendations.get("sap_points", None) survey = ewi_non_invasive_recommendations.get("survey", False) - already_installed = ( - "external_wall_insulation" - in self.property.already_installed - ) - if already_installed: - cost_result = override_costs(cost_result) - new_description = self.get_internal_external_wall_description( self.EXTERNALLY_INSULATED_WALL_DESCRIPTIONS, new_u_value ) From cf5f69d6f09b53d7a505cd436c7daab2adf5e517 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 24 Sep 2024 18:31:50 +0100 Subject: [PATCH 039/166] debugging cleaning class for examples that hadn't been covered previously --- etl/epc_clean/app.py | 3 ++- etl/epc_clean/epc_attributes/MainheatAttributes.py | 6 ++++++ etl/epc_clean/epc_attributes/MainheatControlAttributes.py | 1 + 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/etl/epc_clean/app.py b/etl/epc_clean/app.py index 1d833b72..1dfdd452 100644 --- a/etl/epc_clean/app.py +++ b/etl/epc_clean/app.py @@ -44,7 +44,8 @@ def app(): # Rename the columns to the same format as the api returns data.columns = [c.replace("_", "-").lower() for c in data.columns] # Take just date before the date threshold - data = data[data["lodgement-date"] >= EARLIEST_EPC_DATE] + # For this cleaning dataset, let's try and use all EPCs + # data = data[data["lodgement-date"] >= EARLIEST_EPC_DATE] # Convert to list of dictioaries as returned by the api data = data.to_dict("records") diff --git a/etl/epc_clean/epc_attributes/MainheatAttributes.py b/etl/epc_clean/epc_attributes/MainheatAttributes.py index 56115dca..a7b4305e 100644 --- a/etl/epc_clean/epc_attributes/MainheatAttributes.py +++ b/etl/epc_clean/epc_attributes/MainheatAttributes.py @@ -66,6 +66,7 @@ class MainHeatAttributes(Definitions): "electric heat pumps": "electric heat pump", "solar-assisted heat pump": "solar assisted heat pump", "portable electric heating": "portable electric heaters", + "portable electric heating assumed for most rooms": "portable electric heaters assumed for most rooms", } edge_case_result = {} @@ -138,6 +139,11 @@ class MainHeatAttributes(Definitions): self.is_edge_case = True return + if self.description == ', electric': + self.edge_case_result['has_electric'] = True + self.is_edge_case = True + return + def process(self) -> Dict[str, Union[str, bool]]: result: Dict[str, Union[str, bool]] = {f'has_{ds.replace(" ", "_")}': False for ds in self.DISTRIBUTION_SYSTEMS} diff --git a/etl/epc_clean/epc_attributes/MainheatControlAttributes.py b/etl/epc_clean/epc_attributes/MainheatControlAttributes.py index 46fff6d8..b3cc4df4 100644 --- a/etl/epc_clean/epc_attributes/MainheatControlAttributes.py +++ b/etl/epc_clean/epc_attributes/MainheatControlAttributes.py @@ -75,6 +75,7 @@ class MainheatControlAttributes(Definitions): TO_REMAP = { "celect control": 'celect-type control', "celect controls": 'celect-type control', + "trv's, program & flow switch": 'trvs, programmer & flow switch', } WELSH_TEXT = { From 3957f0fcf8344367dc2fcd2c71e50900a1ef37aa Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 24 Sep 2024 18:41:18 +0100 Subject: [PATCH 040/166] fixing cleaning descriptions --- etl/epc_clean/epc_attributes/MainheatAttributes.py | 5 +++++ etl/epc_clean/epc_attributes/MainheatControlAttributes.py | 1 + 2 files changed, 6 insertions(+) diff --git a/etl/epc_clean/epc_attributes/MainheatAttributes.py b/etl/epc_clean/epc_attributes/MainheatAttributes.py index a7b4305e..16897133 100644 --- a/etl/epc_clean/epc_attributes/MainheatAttributes.py +++ b/etl/epc_clean/epc_attributes/MainheatAttributes.py @@ -144,6 +144,11 @@ class MainHeatAttributes(Definitions): self.is_edge_case = True return + if self.description == 'community, community': + self.edge_case_result['has_community_scheme'] = True + self.is_edge_case = True + return + def process(self) -> Dict[str, Union[str, bool]]: result: Dict[str, Union[str, bool]] = {f'has_{ds.replace(" ", "_")}': False for ds in self.DISTRIBUTION_SYSTEMS} diff --git a/etl/epc_clean/epc_attributes/MainheatControlAttributes.py b/etl/epc_clean/epc_attributes/MainheatControlAttributes.py index b3cc4df4..2759096d 100644 --- a/etl/epc_clean/epc_attributes/MainheatControlAttributes.py +++ b/etl/epc_clean/epc_attributes/MainheatControlAttributes.py @@ -76,6 +76,7 @@ class MainheatControlAttributes(Definitions): "celect control": 'celect-type control', "celect controls": 'celect-type control', "trv's, program & flow switch": 'trvs, programmer & flow switch', + 'appliance thermostat': 'appliance thermostats', } WELSH_TEXT = { From e942d6e70094d862255911c95af245c4673abdd8 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 24 Sep 2024 18:57:10 +0100 Subject: [PATCH 041/166] adding some additional case coverage to epc cleaner --- etl/epc_clean/app.py | 3 +-- etl/epc_clean/epc_attributes/HotWaterAttributes.py | 1 + etl/epc_clean/epc_attributes/RoofAttributes.py | 7 +++---- .../tests/test_data/test_roof_attributes_cases.py | 2 +- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/etl/epc_clean/app.py b/etl/epc_clean/app.py index 1dfdd452..a3c1018f 100644 --- a/etl/epc_clean/app.py +++ b/etl/epc_clean/app.py @@ -44,8 +44,7 @@ def app(): # Rename the columns to the same format as the api returns data.columns = [c.replace("_", "-").lower() for c in data.columns] # Take just date before the date threshold - # For this cleaning dataset, let's try and use all EPCs - # data = data[data["lodgement-date"] >= EARLIEST_EPC_DATE] + data = data[data["lodgement-date"] >= "2011-01-01"] # Convert to list of dictioaries as returned by the api data = data.to_dict("records") diff --git a/etl/epc_clean/epc_attributes/HotWaterAttributes.py b/etl/epc_clean/epc_attributes/HotWaterAttributes.py index f9cec48b..67f5bebd 100644 --- a/etl/epc_clean/epc_attributes/HotWaterAttributes.py +++ b/etl/epc_clean/epc_attributes/HotWaterAttributes.py @@ -96,6 +96,7 @@ class HotWaterAttributes(Definitions): WELSH_TEXT = { "ogçör brif system": "from main system", + "o r brif system": "from main system", "ogçör brif system, adfer gwres nwyon ffliw": "from main system, flue gas heat recovery", "bwyler/cylchredydd nwy": "gas boiler/circulator", "ogçör brif system, dim thermostat ar y silindr": "from main system, no cylinder thermostat", diff --git a/etl/epc_clean/epc_attributes/RoofAttributes.py b/etl/epc_clean/epc_attributes/RoofAttributes.py index 84d1f3e9..154fe41b 100644 --- a/etl/epc_clean/epc_attributes/RoofAttributes.py +++ b/etl/epc_clean/epc_attributes/RoofAttributes.py @@ -6,7 +6,7 @@ from etl.epc_clean.epc_attributes.attribute_utils import extract_component_types class RoofAttributes(Definitions): ROOF_TYPES = ['pitched', 'roof room', 'loft', 'flat', 'thatched', 'at rafters', 'assumed'] - DWELLING_ABOVE = ["another dwelling above", "other premises above"] + DWELLING_ABOVE = ["another dwelling above", "other premises above", "other dwelling above"] WELSH_TEXT = { "ar oleddf, dim inswleiddio": "pitched, no insulation", @@ -113,9 +113,8 @@ class RoofAttributes(Definitions): # roof type result, description = extract_component_types(result, description, list_of_components=self.ROOF_TYPES) - result["has_dwelling_above"] = ( - "another dwelling above" in description or "other premises above" in description - ) + result["has_dwelling_above"] = any([x in description for x in self.DWELLING_ABOVE]) + for dwelling_above in self.DWELLING_ABOVE: description = description.replace(dwelling_above, "") diff --git a/etl/epc_clean/tests/test_data/test_roof_attributes_cases.py b/etl/epc_clean/tests/test_data/test_roof_attributes_cases.py index 6b719afd..06c1f078 100644 --- a/etl/epc_clean/tests/test_data/test_roof_attributes_cases.py +++ b/etl/epc_clean/tests/test_data/test_roof_attributes_cases.py @@ -397,7 +397,7 @@ clean_roof_test_cases = [ 'is_thatched': False, 'is_at_rafters': False, 'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True, 'insulation_thickness': 'none'}, {'original_description': 'Average thermal transmittance 0.80 W/m+é-¦K', 'thermal_transmittance': 0.8, - 'thermal_transmittance_unit': 'w/m+é-¦k', 'is_pitched': False, 'is_roof_room': False, + 'thermal_transmittance_unit': 'w/m-¦k', 'is_pitched': False, 'is_roof_room': False, 'is_loft': False, 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False, 'is_assumed': False, 'has_dwelling_above': False, 'is_valid': True, 'insulation_thickness': None} ] From f3f04de3444ec0cdd1367b498e08b3330dbe7ce4 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 24 Sep 2024 19:01:35 +0100 Subject: [PATCH 042/166] adding welsh translation --- etl/epc_clean/epc_attributes/RoofAttributes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/etl/epc_clean/epc_attributes/RoofAttributes.py b/etl/epc_clean/epc_attributes/RoofAttributes.py index 154fe41b..a67a6029 100644 --- a/etl/epc_clean/epc_attributes/RoofAttributes.py +++ b/etl/epc_clean/epc_attributes/RoofAttributes.py @@ -16,6 +16,7 @@ class RoofAttributes(Definitions): "ar oleddf, inswleiddio cyfyngedig (rhagdybiaeth)": "pitched, limited insulation (assumed)", "ar oleddf, inswleiddio cyfyngedig": "pitched, limited insulation", "ar oleddf, wedigçöi inswleiddio wrth y trawstiau": 'pitched, insulated at rafters', + "ar oleddf, wedi?i inswleiddio wrth y trawstiau": 'pitched, insulated at rafters', "yn wastad, inswleiddio cyfyngedig (rhagdybiaeth)": "flat, limited insulation (assumed)", "yn wastad, inswleiddio cyfyngedig": "flat, limited insulation", "yn wastad, dim inswleiddio (rhagdybiaeth)": "flat, no insulation (assumed)", From 98272b9de45b59a7a5ff6cd2561c07d57d8c3a77 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 24 Sep 2024 19:06:42 +0100 Subject: [PATCH 043/166] Adding floor edge case --- etl/epc_clean/epc_attributes/FloorAttributes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etl/epc_clean/epc_attributes/FloorAttributes.py b/etl/epc_clean/epc_attributes/FloorAttributes.py index 817c2b43..c9c2abab 100644 --- a/etl/epc_clean/epc_attributes/FloorAttributes.py +++ b/etl/epc_clean/epc_attributes/FloorAttributes.py @@ -11,7 +11,7 @@ class FloorAttributes(Definitions): # For the short term, while we are still exploring the data, we maintain a list of error cases which # we want to ignore and consider as no data. - OBSERVED_ERRORS = ["Conservatory"] + OBSERVED_ERRORS = ["Conservatory", "insulated"] WELSH_TEXT = { "(anheddiad arall islaw)": "(another dwelling below)", From 18dc0c109fb50a55baeae83bc5e592e22b3ceadc Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 24 Sep 2024 19:20:40 +0100 Subject: [PATCH 044/166] Added welsh translations --- etl/epc_clean/epc_attributes/LightingAttributes.py | 1 + etl/epc_clean/epc_attributes/RoofAttributes.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/etl/epc_clean/epc_attributes/LightingAttributes.py b/etl/epc_clean/epc_attributes/LightingAttributes.py index 18475b2d..52baa033 100644 --- a/etl/epc_clean/epc_attributes/LightingAttributes.py +++ b/etl/epc_clean/epc_attributes/LightingAttributes.py @@ -7,6 +7,7 @@ from etl.epc_clean.utils import correct_spelling class LightingAttributes(Definitions): WELSH_TEXT = { "goleuadau ynni-isel ym mhob un ogçör mannau gosod": "low energy lighting in all fixed outlets", + "goleuadau ynni-isel ym mhob un o r mannau gosod": "low energy lighting in all fixed outlets", "dim goleuadau ynni-isel": "no low energy lighting", "goleuadau ynni-isel ym mhob un o'r mannau gosod": 'Low energy lighting in all fixed outlets' } diff --git a/etl/epc_clean/epc_attributes/RoofAttributes.py b/etl/epc_clean/epc_attributes/RoofAttributes.py index a67a6029..453ada18 100644 --- a/etl/epc_clean/epc_attributes/RoofAttributes.py +++ b/etl/epc_clean/epc_attributes/RoofAttributes.py @@ -12,6 +12,7 @@ class RoofAttributes(Definitions): "ar oleddf, dim inswleiddio": "pitched, no insulation", "ar oleddf, dim inswleiddio (rhagdybiaeth)": "pitched, no insulation (assumed)", "ar oleddf, wedigçöi inswleiddio (rhagdybiaeth)": "pitched, insulated (assumed)", + "ar oleddf, wedigçöi hinswleiddio (rhagdybiaeth)": "pitched, insulated (assumed)", "ar oleddf, wedigçöi inswleiddio": "pitched, insulated", "ar oleddf, inswleiddio cyfyngedig (rhagdybiaeth)": "pitched, limited insulation (assumed)", "ar oleddf, inswleiddio cyfyngedig": "pitched, limited insulation", @@ -26,6 +27,7 @@ class RoofAttributes(Definitions): "(eiddo arall uwchben)": "(another dwelling above)", "(annedd arall uwchben)": "(another dwelling above)", "ystafell(oedd) to, wedigçöi hinswleiddio": "roof room(s), insulated", + "ystafell(oedd) to, wedi?i hinswleiddio (rhagdybiaeth)": "roof room(s), insulated (assumed)", "ystafell(oedd) to, wedigçöi hinswleiddio (rhagdybiaeth)": "roof room(s), insulated (assumed)", "ystafell(oedd) to, inswleiddio cyfyngedig (rhagdybiaeth)": "roof room(s), limited insulation (assumed)", "ystafell(oedd) to, inswleiddio cyfyngedig": "roof room(s), limited insulation", From 9fe9586d06170195d2aadd95c3ac3963d602716b Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 24 Sep 2024 19:40:02 +0100 Subject: [PATCH 045/166] adding new mainfuel community heating description --- etl/epc_clean/epc_attributes/MainFuelAttributes.py | 3 ++- etl/epc_clean/epc_attributes/MainheatAttributes.py | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/etl/epc_clean/epc_attributes/MainFuelAttributes.py b/etl/epc_clean/epc_attributes/MainFuelAttributes.py index 72b86482..9bb53ff1 100644 --- a/etl/epc_clean/epc_attributes/MainFuelAttributes.py +++ b/etl/epc_clean/epc_attributes/MainFuelAttributes.py @@ -50,7 +50,8 @@ class MainFuelAttributes(Definitions): NO_INDIVIDUAL_HEATING_OR_COMMUNITY_NETWORK = [ 'to be used only when there is no heatinghotwater system or data is from a community network', - 'to be used only when there is no heatinghotwater system' + 'to be used only when there is no heatinghotwater system', + 'community heating schemes waste heat from power stations', ] def __init__(self, description: str): diff --git a/etl/epc_clean/epc_attributes/MainheatAttributes.py b/etl/epc_clean/epc_attributes/MainheatAttributes.py index 16897133..2d482125 100644 --- a/etl/epc_clean/epc_attributes/MainheatAttributes.py +++ b/etl/epc_clean/epc_attributes/MainheatAttributes.py @@ -67,6 +67,7 @@ class MainHeatAttributes(Definitions): "solar-assisted heat pump": "solar assisted heat pump", "portable electric heating": "portable electric heaters", "portable electric heating assumed for most rooms": "portable electric heaters assumed for most rooms", + "electric storage, electric": "electric storage heaters", } edge_case_result = {} @@ -98,6 +99,10 @@ class MainHeatAttributes(Definitions): self.description = remapped + backup_remap = self.REMAP.get(self.description) + if backup_remap: + self.description = backup_remap + self.process_edge_cases() if not self.nodata: From 78f5532dd9236507accf96416afa9e99dc0552ad Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 24 Sep 2024 19:59:28 +0100 Subject: [PATCH 046/166] additional hot water system handled --- etl/epc_clean/epc_attributes/HotWaterAttributes.py | 1 + .../tests/test_data/test_hot_water_attributes_cases.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/etl/epc_clean/epc_attributes/HotWaterAttributes.py b/etl/epc_clean/epc_attributes/HotWaterAttributes.py index 67f5bebd..bd98ec50 100644 --- a/etl/epc_clean/epc_attributes/HotWaterAttributes.py +++ b/etl/epc_clean/epc_attributes/HotWaterAttributes.py @@ -19,6 +19,7 @@ class HotWaterAttributes(Definitions): 'solid fuel boiler', # burns solid materials to generate heat for water heating and/or space heating 'solid fuel range cooker', 'room heaters', # Generic/unspecified category + 'electric multipoint', ] # SYSTEM_TYPES refer to the larger system within which the heater operates. diff --git a/etl/epc_clean/tests/test_data/test_hot_water_attributes_cases.py b/etl/epc_clean/tests/test_data/test_hot_water_attributes_cases.py index 2d6f10a0..ae5348be 100644 --- a/etl/epc_clean/tests/test_data/test_hot_water_attributes_cases.py +++ b/etl/epc_clean/tests/test_data/test_hot_water_attributes_cases.py @@ -219,4 +219,9 @@ hotwater_cases = [ 'heater_type': 'electric instantaneous', 'system_type': None, 'thermostat_characteristics': None, 'heating_scope': None, 'energy_recovery': 'waste water heat recovery', 'tariff_type': None, 'extra_features': None, 'chp_systems': None, 'distribution_system': None, 'no_system_present': None, 'assumed': False, "appliance": None}, + {'original_description': 'Electric multipoint', 'heater_type': 'electric multipoint', 'system_type': None, + 'thermostat_characteristics': None, + 'heating_scope': None, 'energy_recovery': None, 'tariff_type': None, 'extra_features': None, 'chp_systems': None, + 'distribution_system': None, 'no_system_present': None, 'appliance': None, 'assumed': False} + ] From 504d46c929cf7050c4660682498429e7a2c0a533 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 24 Sep 2024 20:29:56 +0100 Subject: [PATCH 047/166] handling additional error cases --- etl/epc_clean/epc_attributes/HotWaterAttributes.py | 1 + etl/epc_clean/epc_attributes/MainheatAttributes.py | 1 + etl/epc_clean/epc_attributes/RoofAttributes.py | 1 + .../test_data/test_mainheat_attributes_cases.py | 14 ++++++++++++++ 4 files changed, 17 insertions(+) diff --git a/etl/epc_clean/epc_attributes/HotWaterAttributes.py b/etl/epc_clean/epc_attributes/HotWaterAttributes.py index bd98ec50..78ee5f7d 100644 --- a/etl/epc_clean/epc_attributes/HotWaterAttributes.py +++ b/etl/epc_clean/epc_attributes/HotWaterAttributes.py @@ -101,6 +101,7 @@ class HotWaterAttributes(Definitions): "ogçör brif system, adfer gwres nwyon ffliw": "from main system, flue gas heat recovery", "bwyler/cylchredydd nwy": "gas boiler/circulator", "ogçör brif system, dim thermostat ar y silindr": "from main system, no cylinder thermostat", + "o r brif system, dim thermostat ar y silindr": "from main system, no cylinder thermostat", "twymwr tanddwr, an-frig": "electric immersion, off-peak", "ogçör brif system, gydag ynnigçör haul": "from main system, plus solar", "twymwr tanddwr, tarriff safonol": "electric immersion, standard tariff", diff --git a/etl/epc_clean/epc_attributes/MainheatAttributes.py b/etl/epc_clean/epc_attributes/MainheatAttributes.py index 2d482125..bf86f573 100644 --- a/etl/epc_clean/epc_attributes/MainheatAttributes.py +++ b/etl/epc_clean/epc_attributes/MainheatAttributes.py @@ -68,6 +68,7 @@ class MainHeatAttributes(Definitions): "portable electric heating": "portable electric heaters", "portable electric heating assumed for most rooms": "portable electric heaters assumed for most rooms", "electric storage, electric": "electric storage heaters", + "radiator heating, electric": "room heaters, electric", } edge_case_result = {} diff --git a/etl/epc_clean/epc_attributes/RoofAttributes.py b/etl/epc_clean/epc_attributes/RoofAttributes.py index 453ada18..f3a4ee49 100644 --- a/etl/epc_clean/epc_attributes/RoofAttributes.py +++ b/etl/epc_clean/epc_attributes/RoofAttributes.py @@ -12,6 +12,7 @@ class RoofAttributes(Definitions): "ar oleddf, dim inswleiddio": "pitched, no insulation", "ar oleddf, dim inswleiddio (rhagdybiaeth)": "pitched, no insulation (assumed)", "ar oleddf, wedigçöi inswleiddio (rhagdybiaeth)": "pitched, insulated (assumed)", + "ar oleddf, wedi?i inswleiddio (rhagdybiaeth)": "pitched, insulated (assumed)", "ar oleddf, wedigçöi hinswleiddio (rhagdybiaeth)": "pitched, insulated (assumed)", "ar oleddf, wedigçöi inswleiddio": "pitched, insulated", "ar oleddf, inswleiddio cyfyngedig (rhagdybiaeth)": "pitched, limited insulation (assumed)", diff --git a/etl/epc_clean/tests/test_data/test_mainheat_attributes_cases.py b/etl/epc_clean/tests/test_data/test_mainheat_attributes_cases.py index 558b176e..86175d4e 100644 --- a/etl/epc_clean/tests/test_data/test_mainheat_attributes_cases.py +++ b/etl/epc_clean/tests/test_data/test_mainheat_attributes_cases.py @@ -1664,5 +1664,19 @@ mainheat_cases = [ 'has_mains_gas': False, 'has_wood_logs': False, 'has_coal': False, 'has_oil': False, 'has_wood_pellets': False, 'has_anthracite': False, 'has_dual_fuel_mineral_and_wood': False, 'has_smokeless_fuel': False, 'has_lpg': False, 'has_b30k': False, 'has_assumed': True, 'has_electricaire': False, 'has_assumed_for_most_rooms': True, + 'has_underfloor_heating': False}, + {'original_description': 'Radiator heating, electric', 'has_radiators': False, 'has_fan_coil_units': False, + 'has_pipes_in_screed_above_insulation': False, + 'has_pipes_in_insulated_timber_floor': False, 'has_pipes_in_concrete_slab': False, 'has_boiler': False, + 'has_air_source_heat_pump': False, 'has_room_heaters': True, 'has_electric_storage_heaters': False, + 'has_warm_air': False, 'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False, + 'has_community_scheme': False, 'has_ground_source_heat_pump': False, 'has_no_system_present': False, + 'has_portable_electric_heaters': False, 'has_water_source_heat_pump': False, 'has_electric_heat_pump': False, + 'has_micro-cogeneration': False, 'has_solar_assisted_heat_pump': False, 'has_exhaust_source_heat_pump': False, + 'has_community_heat_pump': False, 'has_electric': True, 'has_mains_gas': False, 'has_wood_logs': False, + 'has_coal': False, 'has_oil': False, 'has_wood_pellets': False, 'has_anthracite': False, + 'has_dual_fuel_mineral_and_wood': False, 'has_smokeless_fuel': False, 'has_lpg': False, 'has_b30k': False, + 'has_assumed': False, 'has_electricaire': False, 'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False} + ] From 3ebda6277e1c31feef898ffb20482948f5979bf7 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 24 Sep 2024 20:53:24 +0100 Subject: [PATCH 048/166] fixing multiple translations and issues with epc descriptions --- .../epc_attributes/MainheatAttributes.py | 2 ++ .../epc_attributes/RoofAttributes.py | 27 ++++++++++--------- .../test_mainheat_attributes_cases.py | 17 +++++++++++- 3 files changed, 32 insertions(+), 14 deletions(-) diff --git a/etl/epc_clean/epc_attributes/MainheatAttributes.py b/etl/epc_clean/epc_attributes/MainheatAttributes.py index bf86f573..c9f9fbe3 100644 --- a/etl/epc_clean/epc_attributes/MainheatAttributes.py +++ b/etl/epc_clean/epc_attributes/MainheatAttributes.py @@ -69,6 +69,8 @@ class MainHeatAttributes(Definitions): "portable electric heating assumed for most rooms": "portable electric heaters assumed for most rooms", "electric storage, electric": "electric storage heaters", "radiator heating, electric": "room heaters, electric", + "hot-water-only systems, gas": "no system present, electric heaters assumed", + "gas-fired heat pumps, electric": "air source heat pump, electric", } edge_case_result = {} diff --git a/etl/epc_clean/epc_attributes/RoofAttributes.py b/etl/epc_clean/epc_attributes/RoofAttributes.py index f3a4ee49..f36d445f 100644 --- a/etl/epc_clean/epc_attributes/RoofAttributes.py +++ b/etl/epc_clean/epc_attributes/RoofAttributes.py @@ -66,10 +66,18 @@ class RoofAttributes(Definitions): search for regular expressions and translate """ - loft_insulation_thickness_match = re.search(r"ar oleddf, (\d+ mm) o inswleiddio yn y llofft", self.description) - loft_insulation_thickness_match2 = re.search(r"ar oleddf, (\d+ mm) lo inswleiddio yn y llof", self.description) - loft_insulation_thickness_match3 = re.search(r"ar oleddf, (\d+\+ mm) lo inswleiddio yn y llof", - self.description) + loft_insulation_regexes = [ + r"ar oleddf, (\d+ mm) o inswleiddio yn y llofft", + r"ar oleddf, (\d+ mm) lo inswleiddio yn y llof", + r"ar oleddf, (\d+\+ mm) lo inswleiddio yn y llof", + r"ar oleddf, (\d+mm) o inswleiddio yn y llofft", + r"ar oleddf, (\d+\+ mm) o inswleiddio yn y llofft" + ] + li_thickness_match = None + for regex in loft_insulation_regexes: + li_thickness_match = re.search(regex, self.description) + if li_thickness_match: + break uvalue_search = re.search(r"trawsyriannedd thermol cyfartalog (\d+(\.\d+)?)\s*w/m-¦k", self.description) uvalue_search2 = re.search( @@ -77,15 +85,8 @@ class RoofAttributes(Definitions): ) # Step 2: Generalized translation with placeholder - if (loft_insulation_thickness_match is not None) | \ - (loft_insulation_thickness_match2 is not None) | \ - (loft_insulation_thickness_match3 is not None): - if loft_insulation_thickness_match is not None: - insulation_thickness = loft_insulation_thickness_match.group(1) - elif loft_insulation_thickness_match2 is not None: - insulation_thickness = loft_insulation_thickness_match2.group(1) - else: - insulation_thickness = loft_insulation_thickness_match3.group(1) + if li_thickness_match is not None: + insulation_thickness = li_thickness_match.group(1) self.description = f"pitched, {insulation_thickness} loft insulation" elif uvalue_search is not None or uvalue_search2 is not None: diff --git a/etl/epc_clean/tests/test_data/test_mainheat_attributes_cases.py b/etl/epc_clean/tests/test_data/test_mainheat_attributes_cases.py index 86175d4e..82675a74 100644 --- a/etl/epc_clean/tests/test_data/test_mainheat_attributes_cases.py +++ b/etl/epc_clean/tests/test_data/test_mainheat_attributes_cases.py @@ -1677,6 +1677,21 @@ mainheat_cases = [ 'has_coal': False, 'has_oil': False, 'has_wood_pellets': False, 'has_anthracite': False, 'has_dual_fuel_mineral_and_wood': False, 'has_smokeless_fuel': False, 'has_lpg': False, 'has_b30k': False, 'has_assumed': False, 'has_electricaire': False, 'has_assumed_for_most_rooms': False, - 'has_underfloor_heating': False} + 'has_underfloor_heating': False}, + { + 'original_description': 'Hot-Water-Only Systems, gas', + 'has_radiators': False, 'has_fan_coil_units': False, 'has_pipes_in_screed_above_insulation': False, + 'has_pipes_in_insulated_timber_floor': False, 'has_pipes_in_concrete_slab': False, 'has_boiler': False, + 'has_air_source_heat_pump': False, 'has_room_heaters': False, 'has_electric_storage_heaters': False, + 'has_warm_air': False, 'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False, + 'has_community_scheme': False, 'has_ground_source_heat_pump': False, 'has_no_system_present': True, + 'has_portable_electric_heaters': False, 'has_water_source_heat_pump': False, 'has_electric_heat_pump': False, + 'has_micro-cogeneration': False, 'has_solar_assisted_heat_pump': False, 'has_exhaust_source_heat_pump': False, + 'has_community_heat_pump': False, 'has_electric': True, 'has_mains_gas': False, 'has_wood_logs': False, + 'has_coal': False, 'has_oil': False, 'has_wood_pellets': False, 'has_anthracite': False, + 'has_dual_fuel_mineral_and_wood': False, 'has_smokeless_fuel': False, 'has_lpg': False, 'has_b30k': False, + 'has_assumed': True, 'has_electricaire': False, 'has_assumed_for_most_rooms': False, + 'has_underfloor_heating': False + } ] From f0e7aa6d6b12855fdf0e3a2f09872445f365ba9c Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 25 Sep 2024 09:07:50 +0100 Subject: [PATCH 049/166] added additional welsh translation --- etl/epc_clean/epc_attributes/MainheatAttributes.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/etl/epc_clean/epc_attributes/MainheatAttributes.py b/etl/epc_clean/epc_attributes/MainheatAttributes.py index c9f9fbe3..6382238f 100644 --- a/etl/epc_clean/epc_attributes/MainheatAttributes.py +++ b/etl/epc_clean/epc_attributes/MainheatAttributes.py @@ -34,6 +34,8 @@ class MainHeatAttributes(Definitions): "gwresogyddion ystafell, trydan": "room heaters, electric", "pwmp gwres sygçön tarddu yn yr awyr, dan y llawr, trydan": "air source heat pump, underfloor heating, " "electric", + "pwmp gwres sygçön tarddu yn yr awyr, dan y llawr, trydan, pwmp gwres sygçön tarddu yn yr awyr, dan y llawr, " + "trydan": "air source heat pump, underfloor heating, electric", "cynllun cymunedol": "community scheme", "bwyler a gwres dan y llawr, nwy prif gyflenwad": "boiler and underfloor heating, mains gas", "bwyler a rheiddiaduron, logiau coed": 'boiler and radiators, wood logs', From e0f0042086b13307f441503b707927fe4def6f9d Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 25 Sep 2024 09:23:42 +0100 Subject: [PATCH 050/166] welsh translations --- etl/epc_clean/epc_attributes/FloorAttributes.py | 1 + etl/epc_clean/epc_attributes/MainheatAttributes.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/etl/epc_clean/epc_attributes/FloorAttributes.py b/etl/epc_clean/epc_attributes/FloorAttributes.py index c9c2abab..9d0f514d 100644 --- a/etl/epc_clean/epc_attributes/FloorAttributes.py +++ b/etl/epc_clean/epc_attributes/FloorAttributes.py @@ -30,6 +30,7 @@ class FloorAttributes(Definitions): "i ofod heb ei wresogi, wedigçöi inswleiddio": "to unheated space, insulated", "solet, wedigçöi inswleiddio (rhagdybiaeth)": "solid, insulated (assumed)", "solet, wedigçöi inswleiddio": "solid, insulated", + "solet, wedi???i inswleiddio (rhagdybiaeth)": "solid, insulated (assumed)", "i ofod heb ei wresogi, dim inswleiddio (rhagdybiaeth)": "to unheated space, no insulation (assumed)", "i ofod heb ei wresogi, dim inswleiddio": "to unheated space, no insulation" } diff --git a/etl/epc_clean/epc_attributes/MainheatAttributes.py b/etl/epc_clean/epc_attributes/MainheatAttributes.py index 6382238f..430b418d 100644 --- a/etl/epc_clean/epc_attributes/MainheatAttributes.py +++ b/etl/epc_clean/epc_attributes/MainheatAttributes.py @@ -61,6 +61,8 @@ class MainHeatAttributes(Definitions): "bwyler a rheiddiaduron, olew, st+¦r wresogyddion trydan": "boiler and radiators, oil, electric storage " "heaters", "pwmp gwres sygçön tarddu yn yr awyr, awyr gynnes, trydan": "air source heat pump, warm air, electric", + # Should be handled by edge cases + ", trydan": ", electric", } REMAP = { From a27d664a2f3ed141473c759c54df112142ad07cc Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 25 Sep 2024 11:24:55 +0100 Subject: [PATCH 051/166] Handling windows cleaning edge case --- etl/epc_clean/epc_attributes/WindowAttributes.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/etl/epc_clean/epc_attributes/WindowAttributes.py b/etl/epc_clean/epc_attributes/WindowAttributes.py index e9139510..a52977e6 100644 --- a/etl/epc_clean/epc_attributes/WindowAttributes.py +++ b/etl/epc_clean/epc_attributes/WindowAttributes.py @@ -33,12 +33,18 @@ class WindowAttributes(Definitions): "gwydrau lluosog ym mhobman": "multiple glazing throughout", } + # These are observed data anomalies that we want to ignore + NO_DATA_CASES = [ + "SAP05:Windows", + "Solid, no insulation (assumed)", # A description typically associated with floors, not windows + ] + def __init__(self, description: str): self.description: str = clean_description(description.lower()) # In the case of an empty description, we want to return a dictionary with all values set to False # and indicate there was no data - self.nodata = not description or description in self.DATA_ANOMALY_MATCHES or description == "SAP05:Windows" + self.nodata = not description or description in self.DATA_ANOMALY_MATCHES or description in self.NO_DATA_CASES translation = self.WELSH_TEXT.get(self.description) if translation: From b3053f8518727db1b00c2165b286988dbf5796e9 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 25 Sep 2024 16:10:34 +0100 Subject: [PATCH 052/166] handing edge cases for epc cleaning --- etl/epc_clean/epc_attributes/HotWaterAttributes.py | 9 ++++++++- etl/epc_clean/epc_attributes/MainheatAttributes.py | 1 + etl/epc_clean/epc_attributes/RoofAttributes.py | 3 +++ etl/epc_clean/epc_attributes/WindowAttributes.py | 1 + 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/etl/epc_clean/epc_attributes/HotWaterAttributes.py b/etl/epc_clean/epc_attributes/HotWaterAttributes.py index 78ee5f7d..b0105b10 100644 --- a/etl/epc_clean/epc_attributes/HotWaterAttributes.py +++ b/etl/epc_clean/epc_attributes/HotWaterAttributes.py @@ -127,13 +127,20 @@ class HotWaterAttributes(Definitions): "thermostat, flue gas heat recovery", "ogçör brif system, gydag ynnigçör haul, adfer gwres nwyon ffliw": "from main system, plus solar, flue gas " "heat recovery", + "o r brif system, gydag ynni r haul, dim thermostat ar y silindr": "from main system, plus solar, no cylinder " + "thermostat", } + NODATA_DESCRIPTIONS = [ + "sap05 hot-water", + "sap hot-water" + ] + def __init__(self, description: str): self.description: str = clean_description(description.lower()).strip() self.nodata = not self.description or description in self.DATA_ANOMALY_MATCHES or ( - self.description == "sap05 hot-water" + self.description in self.NODATA_DESCRIPTIONS ) translation = self.WELSH_TEXT.get(self.description) diff --git a/etl/epc_clean/epc_attributes/MainheatAttributes.py b/etl/epc_clean/epc_attributes/MainheatAttributes.py index 430b418d..46cbf52b 100644 --- a/etl/epc_clean/epc_attributes/MainheatAttributes.py +++ b/etl/epc_clean/epc_attributes/MainheatAttributes.py @@ -61,6 +61,7 @@ class MainHeatAttributes(Definitions): "bwyler a rheiddiaduron, olew, st+¦r wresogyddion trydan": "boiler and radiators, oil, electric storage " "heaters", "pwmp gwres sygçön tarddu yn yr awyr, awyr gynnes, trydan": "air source heat pump, warm air, electric", + "stor wresogyddion trydan": "electric storage heaters", # Should be handled by edge cases ", trydan": ", electric", } diff --git a/etl/epc_clean/epc_attributes/RoofAttributes.py b/etl/epc_clean/epc_attributes/RoofAttributes.py index f36d445f..75cb8af1 100644 --- a/etl/epc_clean/epc_attributes/RoofAttributes.py +++ b/etl/epc_clean/epc_attributes/RoofAttributes.py @@ -19,6 +19,8 @@ class RoofAttributes(Definitions): "ar oleddf, inswleiddio cyfyngedig": "pitched, limited insulation", "ar oleddf, wedigçöi inswleiddio wrth y trawstiau": 'pitched, insulated at rafters', "ar oleddf, wedi?i inswleiddio wrth y trawstiau": 'pitched, insulated at rafters', + "ar oleddf, wedi?i inswleiddio wrth y trawstia": 'pitched, insulated at rafters', + "ar oleddf, wedigçöi inswleiddio wrth y trawstia": 'pitched, insulated at rafters', "yn wastad, inswleiddio cyfyngedig (rhagdybiaeth)": "flat, limited insulation (assumed)", "yn wastad, inswleiddio cyfyngedig": "flat, limited insulation", "yn wastad, dim inswleiddio (rhagdybiaeth)": "flat, no insulation (assumed)", @@ -35,6 +37,7 @@ class RoofAttributes(Definitions): "ystafell(oedd) to, nenfwd wedigçöi inswleiddio": "roof room(s), ceiling insulated", "ystafell(oedd) to, dim inswleiddio (rhagdybiaeth)": "roof room(s), no insulation (assumed)", "ystafell(oedd) to, dim inswleiddio": "roof room(s), no insulation", + "to gwellt, gydag inswleiddio ychwanegol": "thatched, with additional insulation", } DEFAULT_KEYS = [ diff --git a/etl/epc_clean/epc_attributes/WindowAttributes.py b/etl/epc_clean/epc_attributes/WindowAttributes.py index a52977e6..8c4d0c45 100644 --- a/etl/epc_clean/epc_attributes/WindowAttributes.py +++ b/etl/epc_clean/epc_attributes/WindowAttributes.py @@ -37,6 +37,7 @@ class WindowAttributes(Definitions): NO_DATA_CASES = [ "SAP05:Windows", "Solid, no insulation (assumed)", # A description typically associated with floors, not windows + "Suspended, no insulation (assumed)", # A description typically associated with floors, not windows ] def __init__(self, description: str): From c1b3bc2ecec48a8c99e8eae0fea2a86b42b90f7c Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 26 Sep 2024 08:01:22 +0100 Subject: [PATCH 053/166] debugging descriptions --- backend/app/plan/router.py | 123 ++++++++++++++++++ backend/app/plan/schemas.py | 4 +- .../epc_attributes/HotWaterAttributes.py | 1 + .../MainheatControlAttributes.py | 1 + recommendations/FloorRecommendations.py | 3 +- recommendations/WindowsRecommendations.py | 3 +- 6 files changed, 131 insertions(+), 4 deletions(-) diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index d1578cc1..3b50d46d 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -729,6 +729,129 @@ async def trigger_plan(body: PlanTriggerRequest): scoring_epcs.extend(property_instance.updated_simulation_epcs) recommendations[property_id] = recommendations_with_impact + # For Debugging + recommendation_impact_df = [] + for property_id in recommendations.keys(): + for recs_by_type in recommendations[property_id]: + for rec in recs_by_type: + recommendation_impact_df.append( + { + "property_id": property_id, + "uprn": [p.uprn for p in input_properties if p.id == property_id][0], + "recommendation_id": rec["recommendation_id"], + "type": rec["type"], + "description": rec["description"], + "sap_points": rec["sap_points"], + "co2_equivalent_savings": rec["co2_equivalent_savings"], + "heat_demand": rec["heat_demand"] + } + ) + recommendation_impact_df = pd.DataFrame(recommendation_impact_df) + + surveyed_uprns = [ + 10024087855, 121016117, 121016124, + 10024087902, 121016121, 121016128 + ] + recommendation_impact_df = recommendation_impact_df[recommendation_impact_df["uprn"].isin(surveyed_uprns)] + recommendation_impact_df = recommendation_impact_df[recommendation_impact_df["type"] == "windows_glazing"] + + actual_impacts_df = pd.DataFrame( + [ + # 10024087855 + {"uprn": 10024087855, "type": "internal_wall_insulation", "actual_sap_points": 5}, + {"uprn": 10024087855, "type": "draught_proofing", "actual_sap_points": 2}, + {"uprn": 10024087855, "type": "low_energy_lighting", "actual_sap_points": 0}, + {"uprn": 10024087855, "type": "windows_glazing", "actual_sap_points": 4}, + # 121016117 + {"uprn": 121016117, "type": "internal_wall_insulation", "actual_sap_points": 6}, + {"uprn": 121016117, "type": "draught_proofing", "actual_sap_points": 1}, + {"uprn": 121016117, "type": "low_energy_lighting", "actual_sap_points": 1}, + {"uprn": 121016117, "type": "windows_glazing", "actual_sap_points": 4}, + # 121016124 + {"uprn": 121016124, "type": "internal_wall_insulation", "actual_sap_points": 8}, + {"uprn": 121016124, "type": "low_energy_lighting", "actual_sap_points": 2}, + {"uprn": 121016124, "type": "windows_glazing", "actual_sap_points": 5}, + # 10024087902 + {"uprn": 10024087902, "type": "room_roof_insulation", "actual_sap_points": 16}, + {"uprn": 10024087902, "type": "internal_wall_insulation", "actual_sap_points": 2}, + {"uprn": 10024087902, "type": "low_energy_lighting", "actual_sap_points": 0}, + # 121016121 + {"uprn": 121016121, "type": "internal_wall_insulation", "actual_sap_points": 5}, + {"uprn": 121016121, "type": "suspended_floor_insulation", "actual_sap_points": 2}, + {"uprn": 121016121, "type": "draught_proofing", "actual_sap_points": 1}, + {"uprn": 121016121, "type": "windows_glazing", "actual_sap_points": 3}, + # 121016128 + {"uprn": 121016128, "type": "internal_wall_insulation", "actual_sap_points": 6}, + {"uprn": 121016128, "type": "suspended_floor_insulation", "actual_sap_points": 1}, + {"uprn": 121016128, "type": "draught_proofing", "actual_sap_points": 1}, + {"uprn": 121016128, "type": "low_energy_lighting", "actual_sap_points": 1}, + {"uprn": 121016128, "type": "windows_glazing", "actual_sap_points": 3}, + ] + ) + + comparison = recommendation_impact_df.merge( + actual_impacts_df, how="inner", on=["uprn", "type"] + ) + + # from utils.s3 import read_dataframe_from_s3_parquet + # training_data = read_dataframe_from_s3_parquet( + # bucket_name="retrofit-data-dev", + # file_key="sap_change_model/2024-08-06-11-19-49/dataset_rooms.parquet" + # ) + # import pickle + # with open("delete_me.pkl", "wb") as f: + # pickle.dump(training_data, f) + + # Read in the pickle + import pickle + with open("delete_me.pkl", "rb") as f: + training_data = pickle.load(f) + + # How do we simulate windows: + ending_cols = [col for col in training_data.columns if col.endswith("_ending")] + starting = {} + for c in ending_cols: + starting_colname = c.replace("_ending", "_starting") + if starting_colname in training_data.columns: + starting[c] = starting_colname + else: + starting[c] = c.replace("_ending", "") + + allowed_to_change = [ + # Windows + "windows_energy_eff_ending", + "glazed_type_ending", + "glazing_type_ending", + "multi_glaze_proportion_ending", + + # Other + "sap_ending", + "heat_demand_ending", + "carbon_ending", + "estimated_perimeter_ending", + "lodgement_year_ending", + "lodgement_month_ending", + "days_to_ending", + "number_habitable_rooms_ending", + "number_heated_rooms_ending", + ] + fixed = [c for c in ending_cols if c not in allowed_to_change + ["uprn"]] + training_fixed = training_data.copy() + for col in fixed: + starting_col = starting[col] + training_fixed = training_fixed[training_fixed[col] == training_fixed[starting_col]] + + training_fixed = training_fixed.reset_index(drop=True) + + # Find the things that change + example = training_fixed.iloc[0] + things_that_change = [] + for c in ending_cols: + if example[c] != example[starting[c]]: + things_that_change.append(c) + # 100051011370 + example[] + # We call the API with the scoring epcs scoring_epcs = pd.DataFrame(scoring_epcs) scoring_epcs = kwh_client.transform(data=scoring_epcs, cleaned=cleaned) diff --git a/backend/app/plan/schemas.py b/backend/app/plan/schemas.py index 68f8bbf5..0ddd9761 100644 --- a/backend/app/plan/schemas.py +++ b/backend/app/plan/schemas.py @@ -90,13 +90,13 @@ class PlanTriggerRequest(BaseModel): # Validator to ensure exclusions are within the pre-defined possibilities @validator('exclusions', each_item=True) def check_exclusions(cls, v): - if v not in TYPICAL_MEASURE_TYPES + SPECIFIC_MEASURES: + if v not in TYPICAL_MEASURE_TYPES + SPECIFIC_MEASURES + NON_INVASIVE_SPECIFIC_MEASURES: raise ValueError(f"{v} is not an allowed exclusion") return v @validator('inclusions', each_item=True) def check_inclusions(cls, v): - if v not in TYPICAL_MEASURE_TYPES + SPECIFIC_MEASURES: + if v not in TYPICAL_MEASURE_TYPES + SPECIFIC_MEASURES + NON_INVASIVE_SPECIFIC_MEASURES: raise ValueError(f"{v} is not an allowed inclusion") return v diff --git a/etl/epc_clean/epc_attributes/HotWaterAttributes.py b/etl/epc_clean/epc_attributes/HotWaterAttributes.py index b0105b10..76b4e6fa 100644 --- a/etl/epc_clean/epc_attributes/HotWaterAttributes.py +++ b/etl/epc_clean/epc_attributes/HotWaterAttributes.py @@ -129,6 +129,7 @@ class HotWaterAttributes(Definitions): "heat recovery", "o r brif system, gydag ynni r haul, dim thermostat ar y silindr": "from main system, plus solar, no cylinder " "thermostat", + "o r brif system, gydag ynni r haul": "from main system, plus solar", } NODATA_DESCRIPTIONS = [ diff --git a/etl/epc_clean/epc_attributes/MainheatControlAttributes.py b/etl/epc_clean/epc_attributes/MainheatControlAttributes.py index 2759096d..eaa701da 100644 --- a/etl/epc_clean/epc_attributes/MainheatControlAttributes.py +++ b/etl/epc_clean/epc_attributes/MainheatControlAttributes.py @@ -115,6 +115,7 @@ class MainheatControlAttributes(Definitions): 't+ól un gyfradd, trvs': 'single rate heating, trvs', 'trvs a falf osgoi': 'trvs and bypass', 'rheolaeth celect': 'celect-type control', + 'rheoli r tal a llaw': 'manual charge control', } def __init__(self, description: str): diff --git a/recommendations/FloorRecommendations.py b/recommendations/FloorRecommendations.py index a1f63f96..db18a458 100644 --- a/recommendations/FloorRecommendations.py +++ b/recommendations/FloorRecommendations.py @@ -68,7 +68,8 @@ class FloorRecommendations(Definitions): measures = MEASURE_MAP["floor_insulation"] if measures is None else measures - if not measures: + # If we have no measures or none of the measures are relevant, we can't recommend anything + if not measures or not any(x in measures for x in MEASURE_MAP["floor_insulation"]): return u_value = self.property.floor["thermal_transmittance"] diff --git a/recommendations/WindowsRecommendations.py b/recommendations/WindowsRecommendations.py index ae7f7057..bc91f801 100644 --- a/recommendations/WindowsRecommendations.py +++ b/recommendations/WindowsRecommendations.py @@ -60,7 +60,8 @@ class WindowsRecommendations: return if windows_area is not None: - raise Exception("We have windows area, we should use this data for our recommendations!!!") + # TODO - we don't have a price for this so we can't recommend it + print("We have windows area, we should use this data for our recommendations!!!") # We scale the number of windows based on the proportion of existing glazing if self.property.data["multi-glaze-proportion"] != "": From 6a45789edd94aab73e9fa55c903e864c67f6d105 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 26 Sep 2024 08:47:53 +0100 Subject: [PATCH 054/166] investigating secondary glazing recommendations --- backend/Property.py | 5 +-- backend/app/plan/router.py | 44 ++++++++++++++++--- .../epc_attributes/RoofAttributes.py | 1 + .../epc_attributes/WindowAttributes.py | 1 + 4 files changed, 41 insertions(+), 10 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index 0d194a79..77415d0e 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -503,11 +503,10 @@ class Property: output["lighting_energy_eff_ending"] = "Very Good" if recommendation["type"] == "windows_glazing": + is_secondary_glazing = recommendation["is_secondary_glazing"] output["multi_glaze_proportion_ending"] = 100 if output["windows_energy_eff_ending"] not in ["Average", "Good", "Very Good"]: - output["windows_energy_eff_ending"] = "Average" - - is_secondary_glazing = recommendation["is_secondary_glazing"] + output["windows_energy_eff_ending"] = "Average" if not is_secondary_glazing else "Good" if output["glazing_type_ending"] == "multiple": pass diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 3b50d46d..05c79a22 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -843,14 +843,44 @@ async def trigger_plan(body: PlanTriggerRequest): training_fixed = training_fixed.reset_index(drop=True) + # Get the recommendation config for this uprn + uprn = 121016121 + property_instance = [p for p in input_properties if p.uprn == uprn][0] + property_recs = recommendations[property_instance.id] + window_recs = [r for r in property_recs if r[0]["type"] == "windows_glazing"][0] + window_recs[0].keys() + window_recs[0]["description_simulation"]["multi-glaze-proportion"] + # TODO: - In description_simulation for windows, we update glazed-type but in the model training data there + # is a column called "glazing-type". + # - We don't update glazed-area (should be "Much More Than Typical" most likely? Or Normal??) + # TODO: I think we update eveything that we actually need to, when simulating the recommendation impact for the + # ML models + # TODO: Secondary glazing appears to go to "Good", not "Average". Investigate why + # TODO: For the two properties, force recommendations for double glazing and check impact + + z = training_data[training_data["glazed_type_ending"] == "secondary glazing"] + z = z[z["multi_glaze_proportion_ending"] == 100] + z["windows_energy_eff_ending"].value_counts() + # Find the things that change - example = training_fixed.iloc[0] - things_that_change = [] - for c in ending_cols: - if example[c] != example[starting[c]]: - things_that_change.append(c) - # 100051011370 - example[] + example = training_fixed.iloc[3] + for _, example in training_fixed.iterrows(): + things_that_change = [] + for c in ending_cols: + if example[c] != example[starting[c]]: + things_that_change.append(c) + if len(things_that_change) > 4: + print(things_that_change) + print(example["uprn"]) + # blah + + # 100051011370 (doesn't change in actual glazing) + # example["glazed_type_ending"] + # double glazing installed before 2002 + # example["glazed_type_starting"] + # double glazing, unknown install date + + # 100040925015 # We call the API with the scoring epcs scoring_epcs = pd.DataFrame(scoring_epcs) diff --git a/etl/epc_clean/epc_attributes/RoofAttributes.py b/etl/epc_clean/epc_attributes/RoofAttributes.py index 75cb8af1..0fc2156e 100644 --- a/etl/epc_clean/epc_attributes/RoofAttributes.py +++ b/etl/epc_clean/epc_attributes/RoofAttributes.py @@ -15,6 +15,7 @@ class RoofAttributes(Definitions): "ar oleddf, wedi?i inswleiddio (rhagdybiaeth)": "pitched, insulated (assumed)", "ar oleddf, wedigçöi hinswleiddio (rhagdybiaeth)": "pitched, insulated (assumed)", "ar oleddf, wedigçöi inswleiddio": "pitched, insulated", + "ar oleddf, wedi?i inswleiddio": "pitched, insulated", "ar oleddf, inswleiddio cyfyngedig (rhagdybiaeth)": "pitched, limited insulation (assumed)", "ar oleddf, inswleiddio cyfyngedig": "pitched, limited insulation", "ar oleddf, wedigçöi inswleiddio wrth y trawstiau": 'pitched, insulated at rafters', diff --git a/etl/epc_clean/epc_attributes/WindowAttributes.py b/etl/epc_clean/epc_attributes/WindowAttributes.py index 8c4d0c45..2b1dc172 100644 --- a/etl/epc_clean/epc_attributes/WindowAttributes.py +++ b/etl/epc_clean/epc_attributes/WindowAttributes.py @@ -27,6 +27,7 @@ class WindowAttributes(Definitions): "gwydrau triphlyg llawn": "fully triple glazed", "gwydrau triphlyg rhannol": "partial triple glazed", "gwydrau triphlyg mwyaf": "mostly triple glazed", + "gwydrau triphlyg gan mwyaf": "mostly triple glazed", "gwydrau eilaidd llawn": "full secondary glazing", "gwydrau eilaidd mwyaf": "mostly secondary glazing", "gwydrau eilaidd rhannol": "partial secondary glazing", From 957d006cb8d7e43e4f73a6ccf89021840da05b99 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 26 Sep 2024 08:59:44 +0100 Subject: [PATCH 055/166] fixing cleaning of welsh descriptions --- etl/epc_clean/epc_attributes/MainheatAttributes.py | 1 + etl/epc_clean/epc_attributes/MainheatControlAttributes.py | 1 + 2 files changed, 2 insertions(+) diff --git a/etl/epc_clean/epc_attributes/MainheatAttributes.py b/etl/epc_clean/epc_attributes/MainheatAttributes.py index 46cbf52b..896f73b9 100644 --- a/etl/epc_clean/epc_attributes/MainheatAttributes.py +++ b/etl/epc_clean/epc_attributes/MainheatAttributes.py @@ -37,6 +37,7 @@ class MainHeatAttributes(Definitions): "pwmp gwres sygçön tarddu yn yr awyr, dan y llawr, trydan, pwmp gwres sygçön tarddu yn yr awyr, dan y llawr, " "trydan": "air source heat pump, underfloor heating, electric", "cynllun cymunedol": "community scheme", + "cynllun cymunedol, heat from boilers - mains gas": "community scheme", "bwyler a gwres dan y llawr, nwy prif gyflenwad": "boiler and underfloor heating, mains gas", "bwyler a rheiddiaduron, logiau coed": 'boiler and radiators, wood logs', "bwyler a rheiddiaduron, tanwydd di-fwg": "boiler and radiators, smokeless fuel", diff --git a/etl/epc_clean/epc_attributes/MainheatControlAttributes.py b/etl/epc_clean/epc_attributes/MainheatControlAttributes.py index eaa701da..d7509d47 100644 --- a/etl/epc_clean/epc_attributes/MainheatControlAttributes.py +++ b/etl/epc_clean/epc_attributes/MainheatControlAttributes.py @@ -116,6 +116,7 @@ class MainheatControlAttributes(Definitions): 'trvs a falf osgoi': 'trvs and bypass', 'rheolaeth celect': 'celect-type control', 'rheoli r tal a llaw': 'manual charge control', + 'tal un gyfradd, thermostat ystafell yn unig': 'flat rate charging, room thermostat only' } def __init__(self, description: str): From 0e8136d445f0f84c1b56b2611853aa1c3b157b7c Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 27 Sep 2024 16:18:31 +0100 Subject: [PATCH 056/166] debugging epc cleaning --- etl/epc_clean/epc_attributes/FloorAttributes.py | 3 ++- etl/epc_clean/epc_attributes/MainheatAttributes.py | 8 ++++++++ etl/epc_clean/epc_attributes/MainheatControlAttributes.py | 3 ++- etl/epc_clean/epc_attributes/RoofAttributes.py | 1 + 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/etl/epc_clean/epc_attributes/FloorAttributes.py b/etl/epc_clean/epc_attributes/FloorAttributes.py index 9d0f514d..bba33424 100644 --- a/etl/epc_clean/epc_attributes/FloorAttributes.py +++ b/etl/epc_clean/epc_attributes/FloorAttributes.py @@ -32,7 +32,8 @@ class FloorAttributes(Definitions): "solet, wedigçöi inswleiddio": "solid, insulated", "solet, wedi???i inswleiddio (rhagdybiaeth)": "solid, insulated (assumed)", "i ofod heb ei wresogi, dim inswleiddio (rhagdybiaeth)": "to unheated space, no insulation (assumed)", - "i ofod heb ei wresogi, dim inswleiddio": "to unheated space, no insulation" + "i ofod heb ei wresogi, heb ei inswleiddio (rhagdybiaeth)": "to unheated space, no insulation (assumed)", + "i ofod heb ei wresogi, dim inswleiddio": "to unheated space, no insulation", } def __init__(self, description: str): diff --git a/etl/epc_clean/epc_attributes/MainheatAttributes.py b/etl/epc_clean/epc_attributes/MainheatAttributes.py index 896f73b9..ea61c3b4 100644 --- a/etl/epc_clean/epc_attributes/MainheatAttributes.py +++ b/etl/epc_clean/epc_attributes/MainheatAttributes.py @@ -63,6 +63,13 @@ class MainHeatAttributes(Definitions): "heaters", "pwmp gwres sygçön tarddu yn yr awyr, awyr gynnes, trydan": "air source heat pump, warm air, electric", "stor wresogyddion trydan": "electric storage heaters", + # Not 100% certain - the translation is "bottled gas" + "bwyler a rheiddiaduron, nwy potel": "boiler and radiators, lpg", + "gwresogyddion trydan cludadwy wedi i ragdybio ar gyfer y rhan fwyaf o r ystafelloedd": "portable electric " + "heaters assumed for " + "most rooms", + "st r wresogyddion trydan": "electric storage heaters", + "dim system ar gael, rhagdybir bod gwresogyddion trydan, trydan": "no system present, electric heaters assumed", # Should be handled by edge cases ", trydan": ", electric", } @@ -77,6 +84,7 @@ class MainHeatAttributes(Definitions): "radiator heating, electric": "room heaters, electric", "hot-water-only systems, gas": "no system present, electric heaters assumed", "gas-fired heat pumps, electric": "air source heat pump, electric", + "radiator heating, heat from boilers - gas": "boiler and radiators, mains gas", } edge_case_result = {} diff --git a/etl/epc_clean/epc_attributes/MainheatControlAttributes.py b/etl/epc_clean/epc_attributes/MainheatControlAttributes.py index d7509d47..4a846498 100644 --- a/etl/epc_clean/epc_attributes/MainheatControlAttributes.py +++ b/etl/epc_clean/epc_attributes/MainheatControlAttributes.py @@ -116,7 +116,8 @@ class MainheatControlAttributes(Definitions): 'trvs a falf osgoi': 'trvs and bypass', 'rheolaeth celect': 'celect-type control', 'rheoli r tal a llaw': 'manual charge control', - 'tal un gyfradd, thermostat ystafell yn unig': 'flat rate charging, room thermostat only' + 'tal un gyfradd, thermostat ystafell yn unig': 'flat rate charging, room thermostat only', + "rheoli'r t l llaw": "manual charge control", } def __init__(self, description: str): diff --git a/etl/epc_clean/epc_attributes/RoofAttributes.py b/etl/epc_clean/epc_attributes/RoofAttributes.py index 0fc2156e..2eacc951 100644 --- a/etl/epc_clean/epc_attributes/RoofAttributes.py +++ b/etl/epc_clean/epc_attributes/RoofAttributes.py @@ -27,6 +27,7 @@ class RoofAttributes(Definitions): "yn wastad, dim inswleiddio (rhagdybiaeth)": "flat, no insulation (assumed)", "yn wastad, dim inswleiddio": "flat, no insulation", "yn wastad, wedigçöi inswleiddio (rhagdybiaeth)": "flat, insulated (assumed)", + "yn wastad, wedi?i hinswleiddio (rhagdybiaeth)": "flat, insulated (assumed)", "yn wastad, wedigçöi inswleiddio": "flat, insulated", "(eiddo arall uwchben)": "(another dwelling above)", "(annedd arall uwchben)": "(another dwelling above)", From 32d702e9306614f29d2d4a1268e6706bf99b9314 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 27 Sep 2024 17:10:04 +0100 Subject: [PATCH 057/166] moved leds recommendations earlier --- backend/app/plan/router.py | 11 ++++++++++- recommendations/Recommendations.py | 12 ++++++------ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 05c79a22..90353052 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -738,6 +738,7 @@ async def trigger_plan(body: PlanTriggerRequest): { "property_id": property_id, "uprn": [p.uprn for p in input_properties if p.id == property_id][0], + "address": [p.address for p in input_properties if p.id == property_id][0], "recommendation_id": rec["recommendation_id"], "type": rec["type"], "description": rec["description"], @@ -753,7 +754,9 @@ async def trigger_plan(body: PlanTriggerRequest): 10024087902, 121016121, 121016128 ] recommendation_impact_df = recommendation_impact_df[recommendation_impact_df["uprn"].isin(surveyed_uprns)] - recommendation_impact_df = recommendation_impact_df[recommendation_impact_df["type"] == "windows_glazing"] + # recommendation_impact_df = recommendation_impact_df[recommendation_impact_df["type"].isin( + # ["windows_glazing", "internal_wall_insulation"]) + # ] actual_impacts_df = pd.DataFrame( [ @@ -793,6 +796,12 @@ async def trigger_plan(body: PlanTriggerRequest): actual_impacts_df, how="inner", on=["uprn", "type"] ) + property_recs = recommendation_impact_df[recommendation_impact_df["uprn"] == 121016128] + property = [p for p in input_properties if p.uprn == 121016128][0] + print(property.data["current-energy-efficiency"]) + print(property_recs["sap_points"].sum()) + property_recs["address"] + # from utils.s3 import read_dataframe_from_s3_parquet # training_data = read_dataframe_from_s3_parquet( # bucket_name="retrofit-data-dev", diff --git a/recommendations/Recommendations.py b/recommendations/Recommendations.py index 5037f450..d5e37f8e 100644 --- a/recommendations/Recommendations.py +++ b/recommendations/Recommendations.py @@ -157,6 +157,12 @@ class Recommendations: property_recommendations.append(self.floor_recommender.recommendations) phase += 1 + if "low_energy_lighting" in measures: + self.lighting_recommender.recommend(phase=phase) + if self.lighting_recommender.recommendation: + property_recommendations.append(self.lighting_recommender.recommendation) + phase += 1 + if "windows" in measures and "mixed_glazing" not in non_invasive_recommendation_types: # If we have a mixed glazing recommendation, we prioritise this over the windows recommendation self.windows_recommender.recommend(phase=phase) @@ -233,12 +239,6 @@ class Recommendations: property_recommendations.append(self.hotwater_recommender.recommendations) phase += 1 - if "low_energy_lighting" in measures: - self.lighting_recommender.recommend(phase=phase) - if self.lighting_recommender.recommendation: - property_recommendations.append(self.lighting_recommender.recommendation) - phase += 1 - if "secondary_heating" in measures: self.secondary_heating_recommender.recommend(phase=phase) if self.secondary_heating_recommender.recommendation: From fb3fef5a4a6fdad9a39e665c46d931d9bf401bb9 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Sat, 28 Sep 2024 17:03:41 +0100 Subject: [PATCH 058/166] reviewing model performance for vectis - complete --- backend/app/plan/router.py | 143 +++++++++++---------- backend/app/plan/schemas.py | 4 +- recommendations/FloorRecommendations.py | 4 +- recommendations/LightingRecommendations.py | 33 +++++ recommendations/Recommendations.py | 19 ++- recommendations/WindowsRecommendations.py | 21 ++- 6 files changed, 142 insertions(+), 82 deletions(-) diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 90353052..6e4d8475 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -730,77 +730,80 @@ async def trigger_plan(body: PlanTriggerRequest): recommendations[property_id] = recommendations_with_impact # For Debugging - recommendation_impact_df = [] - for property_id in recommendations.keys(): - for recs_by_type in recommendations[property_id]: - for rec in recs_by_type: - recommendation_impact_df.append( - { - "property_id": property_id, - "uprn": [p.uprn for p in input_properties if p.id == property_id][0], - "address": [p.address for p in input_properties if p.id == property_id][0], - "recommendation_id": rec["recommendation_id"], - "type": rec["type"], - "description": rec["description"], - "sap_points": rec["sap_points"], - "co2_equivalent_savings": rec["co2_equivalent_savings"], - "heat_demand": rec["heat_demand"] - } - ) - recommendation_impact_df = pd.DataFrame(recommendation_impact_df) - - surveyed_uprns = [ - 10024087855, 121016117, 121016124, - 10024087902, 121016121, 121016128 - ] - recommendation_impact_df = recommendation_impact_df[recommendation_impact_df["uprn"].isin(surveyed_uprns)] - # recommendation_impact_df = recommendation_impact_df[recommendation_impact_df["type"].isin( - # ["windows_glazing", "internal_wall_insulation"]) + # recommendation_impact_df = [] + # for property_id in recommendations.keys(): + # for recs_by_type in recommendations[property_id]: + # for rec in recs_by_type: + # recommendation_impact_df.append( + # { + # "property_id": property_id, + # "uprn": [p.uprn for p in input_properties if p.id == property_id][0], + # "address": [p.address for p in input_properties if p.id == property_id][0], + # "recommendation_id": rec["recommendation_id"], + # "type": rec["type"], + # "description": rec["description"], + # "sap_points": rec["sap_points"], + # "co2_equivalent_savings": rec["co2_equivalent_savings"], + # "heat_demand": rec["heat_demand"] + # } + # ) + # recommendation_impact_df = pd.DataFrame(recommendation_impact_df) + # + # surveyed_uprns = [ + # 10024087855, 121016117, 121016124, + # 10024087902, 121016121, 121016128 # ] - - actual_impacts_df = pd.DataFrame( - [ - # 10024087855 - {"uprn": 10024087855, "type": "internal_wall_insulation", "actual_sap_points": 5}, - {"uprn": 10024087855, "type": "draught_proofing", "actual_sap_points": 2}, - {"uprn": 10024087855, "type": "low_energy_lighting", "actual_sap_points": 0}, - {"uprn": 10024087855, "type": "windows_glazing", "actual_sap_points": 4}, - # 121016117 - {"uprn": 121016117, "type": "internal_wall_insulation", "actual_sap_points": 6}, - {"uprn": 121016117, "type": "draught_proofing", "actual_sap_points": 1}, - {"uprn": 121016117, "type": "low_energy_lighting", "actual_sap_points": 1}, - {"uprn": 121016117, "type": "windows_glazing", "actual_sap_points": 4}, - # 121016124 - {"uprn": 121016124, "type": "internal_wall_insulation", "actual_sap_points": 8}, - {"uprn": 121016124, "type": "low_energy_lighting", "actual_sap_points": 2}, - {"uprn": 121016124, "type": "windows_glazing", "actual_sap_points": 5}, - # 10024087902 - {"uprn": 10024087902, "type": "room_roof_insulation", "actual_sap_points": 16}, - {"uprn": 10024087902, "type": "internal_wall_insulation", "actual_sap_points": 2}, - {"uprn": 10024087902, "type": "low_energy_lighting", "actual_sap_points": 0}, - # 121016121 - {"uprn": 121016121, "type": "internal_wall_insulation", "actual_sap_points": 5}, - {"uprn": 121016121, "type": "suspended_floor_insulation", "actual_sap_points": 2}, - {"uprn": 121016121, "type": "draught_proofing", "actual_sap_points": 1}, - {"uprn": 121016121, "type": "windows_glazing", "actual_sap_points": 3}, - # 121016128 - {"uprn": 121016128, "type": "internal_wall_insulation", "actual_sap_points": 6}, - {"uprn": 121016128, "type": "suspended_floor_insulation", "actual_sap_points": 1}, - {"uprn": 121016128, "type": "draught_proofing", "actual_sap_points": 1}, - {"uprn": 121016128, "type": "low_energy_lighting", "actual_sap_points": 1}, - {"uprn": 121016128, "type": "windows_glazing", "actual_sap_points": 3}, - ] - ) - - comparison = recommendation_impact_df.merge( - actual_impacts_df, how="inner", on=["uprn", "type"] - ) - - property_recs = recommendation_impact_df[recommendation_impact_df["uprn"] == 121016128] - property = [p for p in input_properties if p.uprn == 121016128][0] - print(property.data["current-energy-efficiency"]) - print(property_recs["sap_points"].sum()) - property_recs["address"] + # recommendation_impact_df = recommendation_impact_df[recommendation_impact_df["uprn"].isin(surveyed_uprns)] + # # recommendation_impact_df = recommendation_impact_df[recommendation_impact_df["type"].isin( + # # ["windows_glazing", "internal_wall_insulation"]) + # # ] + # + # actual_impacts_df = pd.DataFrame( + # [ + # # 10024087855 + # {"uprn": 10024087855, "type": "internal_wall_insulation", "actual_sap_points": 5}, + # {"uprn": 10024087855, "type": "draught_proofing", "actual_sap_points": 2}, + # {"uprn": 10024087855, "type": "low_energy_lighting", "actual_sap_points": 0}, + # {"uprn": 10024087855, "type": "windows_glazing", "actual_sap_points": 4}, + # # 121016117 + # {"uprn": 121016117, "type": "internal_wall_insulation", "actual_sap_points": 6}, + # {"uprn": 121016117, "type": "draught_proofing", "actual_sap_points": 1}, + # {"uprn": 121016117, "type": "low_energy_lighting", "actual_sap_points": 1}, + # {"uprn": 121016117, "type": "windows_glazing", "actual_sap_points": 4}, + # # 121016124 + # {"uprn": 121016124, "type": "internal_wall_insulation", "actual_sap_points": 8}, + # {"uprn": 121016124, "type": "low_energy_lighting", "actual_sap_points": 2}, + # {"uprn": 121016124, "type": "windows_glazing", "actual_sap_points": 5}, + # # 10024087902 + # {"uprn": 10024087902, "type": "room_roof_insulation", "actual_sap_points": 16}, + # {"uprn": 10024087902, "type": "internal_wall_insulation", "actual_sap_points": 2}, + # {"uprn": 10024087902, "type": "low_energy_lighting", "actual_sap_points": 0}, + # # 121016121 + # {"uprn": 121016121, "type": "internal_wall_insulation", "actual_sap_points": 5}, + # {"uprn": 121016121, "type": "suspended_floor_insulation", "actual_sap_points": 2}, + # {"uprn": 121016121, "type": "draught_proofing", "actual_sap_points": 1}, + # {"uprn": 121016121, "type": "windows_glazing", "actual_sap_points": 3}, + # # 121016128 + # {"uprn": 121016128, "type": "internal_wall_insulation", "actual_sap_points": 6}, + # {"uprn": 121016128, "type": "suspended_floor_insulation", "actual_sap_points": 1}, + # {"uprn": 121016128, "type": "draught_proofing", "actual_sap_points": 1}, + # {"uprn": 121016128, "type": "low_energy_lighting", "actual_sap_points": 1}, + # {"uprn": 121016128, "type": "windows_glazing", "actual_sap_points": 3}, + # ] + # ) + # + # comparison = recommendation_impact_df.merge( + # actual_impacts_df, how="inner", on=["uprn", "type"] + # ) + # + # print(recommendation_impact_df.groupby(["uprn"])["sap_points"].sum()) + # property_recs = recommendation_impact_df[recommendation_impact_df["uprn"] == 121016128] + # property = [p for p in input_properties if p.uprn == 121016128][0] + # print(property.data["current-energy-efficiency"]) + # print(property_recs["sap_points"].sum()) + # print(property_recs["type"]) + # print(float(property.data["current-energy-efficiency"]) + property_recs["sap_points"].sum()) + # recommendations[property.id][2][0]["simulation_config"] # from utils.s3 import read_dataframe_from_s3_parquet # training_data = read_dataframe_from_s3_parquet( diff --git a/backend/app/plan/schemas.py b/backend/app/plan/schemas.py index 0ddd9761..c08cdefc 100644 --- a/backend/app/plan/schemas.py +++ b/backend/app/plan/schemas.py @@ -36,7 +36,8 @@ SPECIFIC_MEASURES = [ # Solar "solar_pv", # Windows Glazing - "windows", + "double_glazing", + "secondary_glazing", # Mechanical ventilation "ventilation", # Other @@ -62,6 +63,7 @@ MEASURE_MAP = { "roof_insulation": ["loft_insulation", "flat_roof_insulation", "room_roof_insulation"], "floor_insulation": ["suspended_floor_insulation", "solid_floor_insulation"], "heating": ["boiler_upgrade", "high_heat_retention_storage_heater", "air_source_heat_pump"], + "windows": ["double_glazing", "secondary_glazing"], } diff --git a/recommendations/FloorRecommendations.py b/recommendations/FloorRecommendations.py index db18a458..d82162da 100644 --- a/recommendations/FloorRecommendations.py +++ b/recommendations/FloorRecommendations.py @@ -224,7 +224,9 @@ class FloorRecommendations(Definitions): simulation_config = { **floor_simulation_config, - "floor_thermal_transmittance_ending": new_u_value, + # We don't simulate the impact using this U-value, but rather the average because this + # variable is way too volatile. Will likely be removed from the model + "floor_thermal_transmittance_ending": 0.685593, } self.recommendations.append( diff --git a/recommendations/LightingRecommendations.py b/recommendations/LightingRecommendations.py index 92394c11..2b0e8724 100644 --- a/recommendations/LightingRecommendations.py +++ b/recommendations/LightingRecommendations.py @@ -1,3 +1,5 @@ +import pandas as pd + from backend.Property import Property from typing import List from recommendations.Costs import Costs @@ -30,6 +32,37 @@ class LightingRecommendations: self.material = material[0] self.recommendation = [] + @classmethod + def get_sap_limit(cls, lighting_energy_efficiency: str, lighting_proportion: float): + """ + Lighting seems to be a more straight forward measure to estimate SAP points for, based on the starting + energy efficiency rating. + + We seem to have the following brackes based on % of LEDs in outlets + Very poor: 0 - 9% + Poor: 10 - 24% + Average: 25 - 44% + Good: 45 - 69% + Very good: 70 - 100% + :return: + """ + + if lighting_energy_efficiency == "Very Good": + return 0 + + if lighting_energy_efficiency in ["Good", "Average"]: + return cls.SAP_LOWER_LIMIT + + # If lighting_energy_efficiency is missing, we'll use the proportion of low energy lighting + if not lighting_energy_efficiency or pd.isnull(lighting_energy_efficiency): + if lighting_proportion >= 0.7: + return 0 + if lighting_proportion >= 0.25: + return cls.SAP_LOWER_LIMIT + return cls.SAP_LIMIT + + return cls.SAP_LIMIT + @staticmethod def estimate_lighting_impact(number_of_bulbs: int): """ diff --git a/recommendations/Recommendations.py b/recommendations/Recommendations.py index d5e37f8e..526cb2a2 100644 --- a/recommendations/Recommendations.py +++ b/recommendations/Recommendations.py @@ -80,6 +80,13 @@ class Recommendations: inclusions_full = [MEASURE_MAP[x] if x in MEASURE_MAP else x for x in self.inclusions] exclusions_full = [MEASURE_MAP[x] if x in MEASURE_MAP else x for x in self.exclusions] + # We need to unlist any lists, but we should check if they're lists first + inclusions_full = [ + item for sublist in inclusions_full for item in (sublist if isinstance(sublist, list) else [sublist]) + ] + exclusions_full = [ + item for sublist in exclusions_full for item in (sublist if isinstance(sublist, list) else [sublist]) + ] # If inclusions and exclusions are empty, it means that nothing was specified, so we allow # all recommendation types @@ -163,9 +170,9 @@ class Recommendations: property_recommendations.append(self.lighting_recommender.recommendation) phase += 1 - if "windows" in measures and "mixed_glazing" not in non_invasive_recommendation_types: + if "mixed_glazing" not in non_invasive_recommendation_types: # If we have a mixed glazing recommendation, we prioritise this over the windows recommendation - self.windows_recommender.recommend(phase=phase) + self.windows_recommender.recommend(phase=phase, measures=measures) if self.windows_recommender.recommendation: property_recommendations.append(self.windows_recommender.recommendation) phase += 1 @@ -538,11 +545,11 @@ class Recommendations: # For the moment, we cap the number of SAP points that can be achieved by LEDs at 2 if rec["type"] == "low_energy_lighting": + lighting_sap_limit = LightingRecommendations.get_sap_limit( + property_instance.data["lighting-energy-eff"], + property_instance.lighting["low_energy_proportion"] + ) - if property_instance.data["low-energy-lighting"] < 50: - lighting_sap_limit = LightingRecommendations.SAP_LIMIT - else: - lighting_sap_limit = LightingRecommendations.SAP_LOWER_LIMIT property_phase_impact["sap"] = min(property_phase_impact["sap"], lighting_sap_limit) property_phase_impact["carbon"] = min( property_phase_impact["carbon"], rec["co2_equivalent_savings"] diff --git a/recommendations/WindowsRecommendations.py b/recommendations/WindowsRecommendations.py index bc91f801..235d9ee2 100644 --- a/recommendations/WindowsRecommendations.py +++ b/recommendations/WindowsRecommendations.py @@ -3,6 +3,7 @@ from typing import List import numpy as np from backend.Property import Property +from backend.app.plan.schemas import MEASURE_MAP from etl.epc_clean.epc_attributes.WindowAttributes import WindowAttributes from recommendations.Costs import Costs from recommendations.recommendation_utils import override_costs, check_simulation_difference @@ -32,7 +33,7 @@ class WindowsRecommendations: raise ValueError("There should only be one window glazing material") self.glazing_material = self.glazing_material[0] - def recommend(self, phase=0): + def recommend(self, measures=None, phase=0): """ This method will recommend the best possible glazing options for a property. @@ -41,14 +42,26 @@ class WindowsRecommendations: :return: """ + measures = MEASURE_MAP["windows"] if measures is None else measures + + # If we have no windows recs, leave + if not any(x in measures for x in MEASURE_MAP["windows"]): + return + # If the property is in a conservation area or is a listed building, it becomes more difficult to install # double glazing. Therefore, we don't recommend it. It is still possible but is not practical as it # requires planning permission and might require a more expensive window type, such as timber. number_of_windows = self.property.number_of_windows - is_secondary_glazing = self.property.restricted_measures or ( - self.property.windows["glazing_type"] == "secondary" - ) + + if "double_glazing" in measures and "secondary_glazing" not in measures: + is_secondary_glazing = False + elif "secondary_glazing" in measures and "double_glazing" not in measures: + is_secondary_glazing = True + else: + is_secondary_glazing = self.property.restricted_measures or ( + self.property.windows["glazing_type"] == "secondary" + ) windows_area = self.property.windows_area if not number_of_windows: From ceb003ec7a82b954c3c8b5f43292a4726b042e1e Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 30 Sep 2024 11:02:32 +0100 Subject: [PATCH 059/166] debugging epc descriptions --- etl/epc_clean/epc_attributes/MainheatAttributes.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/etl/epc_clean/epc_attributes/MainheatAttributes.py b/etl/epc_clean/epc_attributes/MainheatAttributes.py index ea61c3b4..52dc1bc7 100644 --- a/etl/epc_clean/epc_attributes/MainheatAttributes.py +++ b/etl/epc_clean/epc_attributes/MainheatAttributes.py @@ -166,6 +166,11 @@ class MainHeatAttributes(Definitions): self.is_edge_case = True return + if self.description == ', mains gas': + self.edge_case_result['has_mains_gas'] = True + self.is_edge_case = True + return + if self.description == 'community, community': self.edge_case_result['has_community_scheme'] = True self.is_edge_case = True From 591b9522251bcf138357b96128ff4cdebaa5607a Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 30 Sep 2024 11:53:04 +0100 Subject: [PATCH 060/166] adding the assessment information to retrieve_find_my_epc_data --- etl/bill_savings/data_collection.py | 42 +++++++++++++++++-- etl/customers/aiha/epc_surveyor_list.py | 41 ++++++++++++++++++ .../MainheatControlAttributes.py | 7 +++- 3 files changed, 86 insertions(+), 4 deletions(-) create mode 100644 etl/customers/aiha/epc_surveyor_list.py diff --git a/etl/bill_savings/data_collection.py b/etl/bill_savings/data_collection.py index 49bcff82..ee8a228f 100644 --- a/etl/bill_savings/data_collection.py +++ b/etl/bill_savings/data_collection.py @@ -100,9 +100,44 @@ def retrieve_find_my_epc_data(uprn: int, postcode: str, address: str, expected_e bills = address_res.find('div', {'id': 'bills-affected'}) bills_list = bills.find_all('li') if not bills_list: - return None - heating_text = bills_list[0].text - hot_water_text = bills_list[1].text + # If this is the case, it's usually becaue the EPC was very old. Early EPCs did not have this information + heating_text = None + hot_water_text = None + else: + heating_text = bills_list[0].text + hot_water_text = bills_list[1].text + + # Search for the assessment informaton + assessment_information = address_res.find('div', {'id': 'information'}) + # Parse this information + rows = assessment_information.find_all('div', class_='govuk-summary-list__row') + # Create a dictionary to hold the parsed information + assessment_data = {} + for row in rows: + key = row.find('dt').text.strip() + if key == "Type of assessment": + # We dont reliably extract this + continue + value_tag = row.find('dd') + + # Check if value contains a link (email) + if value_tag.find('a'): + value = value_tag.find('a').text.strip() + elif value_tag.find('summary'): + value = value_tag.find('span').text.strip() + else: + value = value_tag.text.strip() + + assessment_data[key] = value + + expected_keys = [ + 'Assessor’s name', 'Telephone', 'Email', 'Accreditation scheme', 'Assessor’s ID', 'Assessor’s declaration', + 'Date of assessment', 'Date of certificate' + ] + # Check we have all the expected keys + for key in expected_keys: + if key not in assessment_data: + raise ValueError(f"Missing key: {key}") resulting_data = { 'extracted_uprn': uprn, @@ -114,6 +149,7 @@ def retrieve_find_my_epc_data(uprn: int, postcode: str, address: str, expected_e "potential_epc_efficiency": int(potential_rating.split(' ')[-1]), "heating_text": heating_text, "hot_water_text": hot_water_text, + **assessment_data } return resulting_data diff --git a/etl/customers/aiha/epc_surveyor_list.py b/etl/customers/aiha/epc_surveyor_list.py new file mode 100644 index 00000000..b85139ae --- /dev/null +++ b/etl/customers/aiha/epc_surveyor_list.py @@ -0,0 +1,41 @@ +import pandas as pd +import numpy as np +import time +from tqdm import tqdm +from etl.bill_savings.data_collection import retrieve_find_my_epc_data, calculate_expiry_date + + +def main(): + """ + This script handles pulling the surveyor names and acreditation details for Surveyors who have completed + the newest EPC for AIHA's properties + """ + + epc_data = pd.read_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/AIHA/epc_data.csv") + epc_data = epc_data[["uprn", "address", "address1", "postcode", "lodgement-date"]] + + epc_collected_data = [] + for _, unit in tqdm(epc_data.iterrows(), total=len(epc_data)): + time.sleep(np.random.uniform(0.2, 1.5)) + uprn = int(unit["uprn"]) + address = unit["address1"] + postcode = unit["postcode"] + expected_expiry_date = calculate_expiry_date(unit["lodgement-date"]) + + response = retrieve_find_my_epc_data( + uprn=uprn, + postcode=postcode, + address=address, + expected_expiry_date=expected_expiry_date + ) + if response is None: + raise Exception("fix me") + epc_collected_data.append(response) + + epc_collected_data = pd.DataFrame(epc_collected_data) + + for x in epc_collected_data: + keys = x.keys() + # Check for None keys + if any(k is None for k in keys): + frew diff --git a/etl/epc_clean/epc_attributes/MainheatControlAttributes.py b/etl/epc_clean/epc_attributes/MainheatControlAttributes.py index 4a846498..a13823d2 100644 --- a/etl/epc_clean/epc_attributes/MainheatControlAttributes.py +++ b/etl/epc_clean/epc_attributes/MainheatControlAttributes.py @@ -120,10 +120,15 @@ class MainheatControlAttributes(Definitions): "rheoli'r t l llaw": "manual charge control", } + NO_DATA_DESCRIPTIONS = [ + "SAP05:Main-Heating-Controls", + "SAP:Main-Heating-Controls", + ] + def __init__(self, description: str): self.description: str = clean_description(description.lower()).strip() self.nodata = not self.description or description in self.DATA_ANOMALY_MATCHES or ( - description == "SAP05:Main-Heating-Controls" + description in self.NO_DATA_DESCRIPTIONS ) translation = self.WELSH_TEXT.get(self.description) From 986174b95d3aef7ad77650dc995df00435470872 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 30 Sep 2024 12:57:42 +0100 Subject: [PATCH 061/166] handling new EPC description --- etl/customers/aiha/epc_surveyor_list.py | 31 ++++++++++++++++--- .../epc_attributes/MainheatAttributes.py | 1 + .../test_mainheat_attributes_cases.py | 17 +++++++++- 3 files changed, 43 insertions(+), 6 deletions(-) diff --git a/etl/customers/aiha/epc_surveyor_list.py b/etl/customers/aiha/epc_surveyor_list.py index b85139ae..cec72928 100644 --- a/etl/customers/aiha/epc_surveyor_list.py +++ b/etl/customers/aiha/epc_surveyor_list.py @@ -34,8 +34,29 @@ def main(): epc_collected_data = pd.DataFrame(epc_collected_data) - for x in epc_collected_data: - keys = x.keys() - # Check for None keys - if any(k is None for k in keys): - frew + epc_collected_data = epc_data[["uprn", "address", "address1", "postcode"]].merge( + epc_collected_data, left_on="uprn", right_on="extracted_uprn" + ) + + elmhurst_surveys = epc_collected_data[ + epc_collected_data["Accreditation scheme"].isin( + ["NHER", "Stroma Certification Ltd", "Elmhurst Energy Systems Ltd"] + ) + ] + + quidos_surveys = epc_collected_data[ + epc_collected_data["Accreditation scheme"].isin( + ["Quidos Limited"] + ) + ] + + ecmk_surveys = epc_collected_data[ + epc_collected_data["Accreditation scheme"].isin( + ["ECMK"] + ) + ] + + # Store the data: + elmhurst_surveys.to_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/AIHA/Elmhurst Surveys.csv") + quidos_surveys.to_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/AIHA/Quidos Surveys.csv") + ecmk_surveys.to_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/AIHA/ECMK Surveys.csv") diff --git a/etl/epc_clean/epc_attributes/MainheatAttributes.py b/etl/epc_clean/epc_attributes/MainheatAttributes.py index 52dc1bc7..051db8c2 100644 --- a/etl/epc_clean/epc_attributes/MainheatAttributes.py +++ b/etl/epc_clean/epc_attributes/MainheatAttributes.py @@ -85,6 +85,7 @@ class MainHeatAttributes(Definitions): "hot-water-only systems, gas": "no system present, electric heaters assumed", "gas-fired heat pumps, electric": "air source heat pump, electric", "radiator heating, heat from boilers - gas": "boiler and radiators, mains gas", + "heat pump, warm air, mains gas": "air source heat pump, warm air, mains gas", } edge_case_result = {} diff --git a/etl/epc_clean/tests/test_data/test_mainheat_attributes_cases.py b/etl/epc_clean/tests/test_data/test_mainheat_attributes_cases.py index 82675a74..51fc3416 100644 --- a/etl/epc_clean/tests/test_data/test_mainheat_attributes_cases.py +++ b/etl/epc_clean/tests/test_data/test_mainheat_attributes_cases.py @@ -1692,6 +1692,21 @@ mainheat_cases = [ 'has_dual_fuel_mineral_and_wood': False, 'has_smokeless_fuel': False, 'has_lpg': False, 'has_b30k': False, 'has_assumed': True, 'has_electricaire': False, 'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False - } + }, + { + "original_description": "heat pump, warm air, mains gas", # This gets remapped to air source heat pump + 'has_radiators': False, 'has_fan_coil_units': False, 'has_pipes_in_screed_above_insulation': False, + 'has_pipes_in_insulated_timber_floor': False, 'has_pipes_in_concrete_slab': False, 'has_boiler': False, + 'has_air_source_heat_pump': True, 'has_room_heaters': False, 'has_electric_storage_heaters': False, + 'has_warm_air': True, 'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False, + 'has_community_scheme': False, 'has_ground_source_heat_pump': False, 'has_no_system_present': False, + 'has_portable_electric_heaters': False, 'has_water_source_heat_pump': False, 'has_electric_heat_pump': False, + 'has_micro-cogeneration': False, 'has_solar_assisted_heat_pump': False, 'has_exhaust_source_heat_pump': False, + 'has_community_heat_pump': False, 'has_electric': False, 'has_mains_gas': True, 'has_wood_logs': False, + 'has_coal': False, 'has_oil': False, 'has_wood_pellets': False, 'has_anthracite': False, + 'has_dual_fuel_mineral_and_wood': False, 'has_smokeless_fuel': False, 'has_lpg': False, 'has_b30k': False, + 'has_assumed': False, 'has_electricaire': False, 'has_assumed_for_most_rooms': False, + 'has_underfloor_heating': False + } ] From 0da3b7b9a8a37d49e2791fba442280520a21b289 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 30 Sep 2024 14:23:20 +0100 Subject: [PATCH 062/166] fixed mainheat attributes tests for init --- etl/epc_clean/tests/test_data/test_floor_attributes_cases.py | 2 +- etl/epc_clean/tests/test_mainheat_attributes.py | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/etl/epc_clean/tests/test_data/test_floor_attributes_cases.py b/etl/epc_clean/tests/test_data/test_floor_attributes_cases.py index 280e7459..81ec7a32 100644 --- a/etl/epc_clean/tests/test_data/test_floor_attributes_cases.py +++ b/etl/epc_clean/tests/test_data/test_floor_attributes_cases.py @@ -367,7 +367,7 @@ clean_floor_cases = [ 'thermal_transmittance_unit': None, 'is_assumed': True, 'is_to_unheated_space': True, 'is_to_external_air': False, 'is_suspended': False, 'is_solid': False, 'insulation_thickness': 'none', "another_property_below": False}, {'original_description': "Average thermal transmittance 1.10 W/m+é-¦K", 'thermal_transmittance': 1.1, - 'thermal_transmittance_unit': 'w/m+é-¦k', 'is_assumed': False, + 'thermal_transmittance_unit': 'w/m-¦k', 'is_assumed': False, 'is_to_unheated_space': False, 'is_to_external_air': False, 'is_suspended': False, 'is_solid': False, 'another_property_below': False, 'insulation_thickness': None}, { diff --git a/etl/epc_clean/tests/test_mainheat_attributes.py b/etl/epc_clean/tests/test_mainheat_attributes.py index f175e821..7e245b8d 100644 --- a/etl/epc_clean/tests/test_mainheat_attributes.py +++ b/etl/epc_clean/tests/test_mainheat_attributes.py @@ -11,10 +11,6 @@ class TestMainHeatAttributes: floor_attr = MainHeatAttributes(valid_description) assert floor_attr.description == valid_description.lower() - # Test initialization with an empty description - with pytest.raises(ValueError): - MainHeatAttributes('') - # Test initialization with a description that contains none of the keywords with pytest.raises(ValueError): MainHeatAttributes('description without keywords') From a0e4446340772c84d5543be852f0f48ca4a0b6b6 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 30 Sep 2024 14:26:55 +0100 Subject: [PATCH 063/166] fixed mainheat attriutes unit tests --- .../tests/test_data/test_mainheat_attributes_cases.py | 4 ++-- etl/epc_clean/tests/test_mainheat_attributes.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/etl/epc_clean/tests/test_data/test_mainheat_attributes_cases.py b/etl/epc_clean/tests/test_data/test_mainheat_attributes_cases.py index 51fc3416..16acdd37 100644 --- a/etl/epc_clean/tests/test_data/test_mainheat_attributes_cases.py +++ b/etl/epc_clean/tests/test_data/test_mainheat_attributes_cases.py @@ -1658,9 +1658,9 @@ mainheat_cases = [ 'has_air_source_heat_pump': False, 'has_room_heaters': False, 'has_electric_storage_heaters': False, 'has_warm_air': False, 'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False, 'has_community_scheme': False, 'has_ground_source_heat_pump': False, 'has_no_system_present': False, - 'has_portable_electric_heaters': False, 'has_water_source_heat_pump': False, 'has_electric_heat_pump': False, + 'has_portable_electric_heaters': True, 'has_water_source_heat_pump': False, 'has_electric_heat_pump': False, 'has_micro-cogeneration': False, 'has_solar_assisted_heat_pump': False, 'has_exhaust_source_heat_pump': False, - 'has_community_heat_pump': False, 'has_portable_electric_heating': True, 'has_electric': True, + 'has_community_heat_pump': False, 'has_electric': True, 'has_mains_gas': False, 'has_wood_logs': False, 'has_coal': False, 'has_oil': False, 'has_wood_pellets': False, 'has_anthracite': False, 'has_dual_fuel_mineral_and_wood': False, 'has_smokeless_fuel': False, 'has_lpg': False, 'has_b30k': False, 'has_assumed': True, 'has_electricaire': False, 'has_assumed_for_most_rooms': True, diff --git a/etl/epc_clean/tests/test_mainheat_attributes.py b/etl/epc_clean/tests/test_mainheat_attributes.py index 7e245b8d..d79c271a 100644 --- a/etl/epc_clean/tests/test_mainheat_attributes.py +++ b/etl/epc_clean/tests/test_mainheat_attributes.py @@ -34,7 +34,6 @@ class TestMainHeatAttributes: def test_invalid_description(self): # Test that invalid descriptions raise a ValueError invalid_descriptions = [ - "", "invalid description", "description with no known heating data_types", ] From 55d61395509d4e3dc58091a57ff81746d71e4c4d Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 30 Sep 2024 14:30:16 +0100 Subject: [PATCH 064/166] fixed all cleaning unit tests --- etl/epc_clean/tests/test_data/test_wall_attributes_cases.py | 4 ++-- etl/epc_clean/tests/test_wall_attributes.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/etl/epc_clean/tests/test_data/test_wall_attributes_cases.py b/etl/epc_clean/tests/test_data/test_wall_attributes_cases.py index 96c545c1..507449ab 100644 --- a/etl/epc_clean/tests/test_data/test_wall_attributes_cases.py +++ b/etl/epc_clean/tests/test_data/test_wall_attributes_cases.py @@ -1,5 +1,5 @@ wall_cases = [ - {'original_description': 'Average thermal transmittance -4.67 W/m-¦K', 'thermal_transmittance': -4.67, + {'original_description': 'Average thermal transmittance -4.67 W/m-¦K', 'thermal_transmittance': 4.67, 'thermal_transmittance_unit': 'w/m-¦k', 'is_cavity_wall': False, 'is_filled_cavity': False, 'is_solid_brick': False, 'is_system_built': False, 'is_timber_frame': False, 'is_granite_or_whinstone': False, 'is_as_built': False, 'is_cob': False, 'is_assumed': False, 'is_sandstone_or_limestone': False, @@ -692,7 +692,7 @@ wall_cases = [ 'is_cob': False, 'is_assumed': True, 'is_sandstone_or_limestone': False, 'insulation_thickness': 'none', 'external_insulation': False, 'internal_insulation': False}, {'original_description': 'Average thermal transmittance 1.60 W/m+é-¦K', - 'thermal_transmittance': 1.6, 'thermal_transmittance_unit': 'w/m+é-¦k', 'is_cavity_wall': False, + 'thermal_transmittance': 1.6, 'thermal_transmittance_unit': 'w/m-¦k', 'is_cavity_wall': False, 'is_filled_cavity': False, 'is_solid_brick': False, 'is_system_built': False, 'is_timber_frame': False, 'is_granite_or_whinstone': False, 'is_as_built': False, 'is_cob': False, 'is_assumed': False, 'is_sandstone_or_limestone': False, 'insulation_thickness': None, 'external_insulation': False, diff --git a/etl/epc_clean/tests/test_wall_attributes.py b/etl/epc_clean/tests/test_wall_attributes.py index 01a60615..970dbd98 100644 --- a/etl/epc_clean/tests/test_wall_attributes.py +++ b/etl/epc_clean/tests/test_wall_attributes.py @@ -16,7 +16,7 @@ class TestWallAttributes: description = 'average thermal transmittance -4.67 w/m-¦k' wa = wall_attr(description) result = wa.process() - assert result['thermal_transmittance'] == -4.67 + assert result['thermal_transmittance'] == 4.67 assert result['thermal_transmittance_unit'] == 'w/m-¦k' def test_wall_types(self, wall_attr): From 04df5743c239f0bcd695311326be29cf4eb76e9e Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 30 Sep 2024 14:55:13 +0100 Subject: [PATCH 065/166] deleted irrelevant tests for solar pv costs --- .../tests/test_air_source_heat_pump.py | 944 ------------------ recommendations/tests/test_costs.py | 336 +------ 2 files changed, 51 insertions(+), 1229 deletions(-) delete mode 100644 recommendations/tests/test_air_source_heat_pump.py diff --git a/recommendations/tests/test_air_source_heat_pump.py b/recommendations/tests/test_air_source_heat_pump.py deleted file mode 100644 index 0d69b10d..00000000 --- a/recommendations/tests/test_air_source_heat_pump.py +++ /dev/null @@ -1,944 +0,0 @@ -import pandas as pd -import msgpack -from datetime import datetime - -from utils.s3 import read_dataframe_from_s3_parquet, read_from_s3 -from backend.Property import Property -from recommendations.HeatingRecommender import HeatingRecommender -from recommendations.Recommendations import Recommendations -from etl.epc.Record import EPCRecord -from etl.solar.SolarPhotoSupply import SolarPhotoSupply -from backend.ml_models.api import ModelApi - - -def find_examples(): - """ Some scrappy helper code to find EPC examples""" - # Let's look for some testing data, where the only thing different pre and post is the installation of an - # air source heat pump - data = read_dataframe_from_s3_parquet( - bucket_name="retrofit-data-dev", - file_key="sap_change_model/2024-03-24-15-51-13/dataset_no_cleaning.parquet" - ) - - # Firstly, take records where before there was no air source heat pump and afterwards there was - data = data[ - data["has_air_source_heat_pump_ending"] & ~data["has_air_source_heat_pump"] - ] - - # Start with a property that has a boiler - data = data[data["has_boiler"]] - - static_columns = [ - # Walls - 'walls_thermal_transmittance_ending', - 'is_filled_cavity_ending', - 'is_park_home_ending', - 'walls_insulation_thickness_ending', - 'external_insulation_ending', - 'internal_insulation_ending', - # Floors - # 'floor_thermal_transmittance_ending', # Don't subset on this, because it changes based on floor area - 'floor_insulation_thickness_ending', - # Roof - 'roof_thermal_transmittance_ending', - 'is_at_rafters_ending', - 'roof_insulation_thickness_ending', - # Hot water - air source heat pump will shange the hot water system (probably from whatever it was -> main) - # 'heater_type_ending', - # 'system_type_ending', - # 'thermostat_characteristics_ending', - # 'heating_scope_ending', - # 'energy_recovery_ending', - # 'hotwater_tariff_type_ending', - # 'extra_features_ending', - # 'chp_systems_ending', - # 'distribution_system_ending', - # 'no_system_present_ending', - # 'appliance_ending', - # Heating - Will change when installing an ASHP - # 'has_radiators_ending', - # 'has_fan_coil_units_ending', - # 'has_pipes_in_screed_above_insulation_ending', - # 'has_pipes_in_insulated_timber_floor_ending', - # 'has_pipes_in_concrete_slab_ending', - # 'has_boiler_ending', - # 'has_air_source_heat_pump_ending', # We want the air source heat pump to change - # 'has_room_heaters_ending', - # 'has_electric_storage_heaters_ending', - # 'has_warm_air_ending', - # 'has_electric_underfloor_heating_ending', - # 'has_electric_ceiling_heating_ending', - # 'has_community_scheme_ending', - # 'has_ground_source_heat_pump_ending', - # 'has_no_system_present_ending', - # 'has_portable_electric_heaters_ending', - # 'has_water_source_heat_pump_ending', - # 'has_electric_heat_pump_ending', - # 'has_micro-cogeneration_ending', - # 'has_solar_assisted_heat_pump_ending', - # 'has_exhaust_source_heat_pump_ending', - # 'has_community_heat_pump_ending', - # 'has_electric_ending', - # 'has_mains_gas_ending', - # 'has_wood_logs_ending', 'has_coal_ending', 'has_oil_ending', - # 'has_wood_pellets_ending', 'has_anthracite_ending', 'has_dual_fuel_mineral_and_wood_ending', - # 'has_smokeless_fuel_ending', 'has_lpg_ending', 'has_b30k_ending', 'has_electricaire_ending', - # 'has_assumed_for_most_rooms_ending', 'has_underfloor_heating_ending', - # 'thermostatic_control_ending', - # 'charging_system_ending', - # 'switch_system_ending', - # 'no_control_ending', - # 'dhw_control_ending', - # 'community_heating_ending', - # 'multiple_room_thermostats_ending', - # 'auxiliary_systems_ending', - # 'trvs_ending', - # 'rate_control_ending', - # Window - 'glazing_type_ending', - # Fuel - could change with ASHP - # 'fuel_type_ending', - # 'main-fuel_tariff_type_ending', - # 'is_community_ending', - # 'no_individual_heating_or_community_network_ending', - # 'complex_fuel_type_ending', - - 'mechanical_ventilation_ending', 'secondheat_description_ending', 'glazed_type_ending', - 'multi_glaze_proportion_ending', 'low_energy_lighting_ending', 'number_open_fireplaces_ending', - 'solar_water_heating_flag_ending', - 'photo_supply_ending', - 'energy_tariff_ending', - 'extension_count_ending', - 'total_floor_area_ending', - # 'hot_water_energy_eff_ending', - 'floor_energy_eff_ending', - 'windows_energy_eff_ending', - 'walls_energy_eff_ending', - 'sheating_energy_eff_ending', - 'roof_energy_eff_ending', - # 'mainheat_energy_eff_ending', - # 'mainheatc_energy_eff_ending', - 'lighting_energy_eff_ending', - 'number_habitable_rooms_ending', - 'number_heated_rooms_ending', - ] - - for col in static_columns: - - base_starting = col.split("_ending")[0] - if base_starting + "_starting" in data.columns: - starting_col = base_starting + "_starting" - else: - starting_col = base_starting - # Filter - print("Column: %s" % col) - print("Starting size: %s" % data.shape[0]) - data = data[data[starting_col] == data[col]] - print("Ending size: %s" % data.shape[0]) - - z = data[['uprn', col, starting_col]] - - # Great example UPRNs - # 100030969273 - # 10034685399 - Completely transforms the heating and hot water systems in the home (goes from oil -> electricity) - # 100091200828 - goes from a liquid petroleum gas boiler to ashp - - # Look for starting with a gas boiler - data[ - data["has_boiler"] & data["has_radiators"] & data["has_mains_gas"] & ~data["has_boiler_ending"] - ] - - # UPRN: 100011776843 - - -class TestAirSourceHeatPump: - - def test_eligible(self): - # This tests a house, which will be suitable for an air source heat pump - epc_record = EPCRecord() - epc_record.prepared_epc = { - "county": "Broxbourne", - "mainheat-energy-eff": "Good", - "hot-water-energy-eff": "Good", - "mainheatc-energy-eff": "Good", - "number-heated-rooms": 5, - "property-type": "House", - "built-form": "Semi-Detached" - } - - property_instance = Property(id=0, address="fake", postcode="fake", epc_record=epc_record) - property_instance.main_heating = { - 'original_description': 'Boiler and radiators, mains gas', - "clean_description": "Boiler and radiators, mains gas", - 'has_radiators': True, - 'has_fan_coil_units': False, 'has_pipes_in_screed_above_insulation': False, - 'has_pipes_in_insulated_timber_floor': False, 'has_pipes_in_concrete_slab': False, 'has_boiler': True, - 'has_air_source_heat_pump': False, - 'has_room_heaters': False, 'has_electric_storage_heaters': False, - 'has_warm_air': False, - 'has_electric_underfloor_heating': False, - 'has_electric_ceiling_heating': False, 'has_community_scheme': False, - 'has_ground_source_heat_pump': False, 'has_no_system_present': False, - 'has_portable_electric_heaters': False, - 'has_water_source_heat_pump': False, 'has_electric': False, - 'has_mains_gas': True, 'has_wood_logs': False, - 'has_coal': False, 'has_oil': False, 'has_wood_pellets': False, - 'has_anthracite': False, - 'has_dual_fuel_mineral_and_wood': False, 'has_smokeless_fuel': False, - 'has_lpg': False, 'has_assumed': False, - 'has_electricaire': False, 'has_assumed_for_most_rooms': False, - 'has_underfloor_heating': False, - "has_electric_heat_pumps": False, - "has_micro-cogeneration": False - } - property_instance.main_fuel = { - 'original_description': 'mains gas (not community)', 'fuel_type': 'mains gas', - 'tariff_type': None, - 'is_community': False, 'no_individual_heating_or_community_network': False, - 'complex_fuel_type': None - } - property_instance.hotwater = { - 'original_description': 'From main system', - 'clean_description': 'From main system', - 'heater_type': None, - 'system_type': 'from main system', - 'thermostat_characteristics': None, 'heating_scope': None, - 'energy_recovery': None, 'tariff_type': None, - 'extra_features': None, 'chp_systems': None, 'distribution_system': None, - 'no_system_present': None, - 'assumed': False, "appliance": None - } - property_instance.main_heating_controls = { - 'original_description': 'Programmer, room thermostat and TRVs', - 'thermostatic_control': 'room thermostat', 'charging_system': None, 'switch_system': 'programmer', - 'no_control': None, 'dhw_control': None, 'community_heating': None, 'multiple_room_thermostats': False, - 'auxiliary_systems': None, 'trvs': 'trvs', 'rate_control': None - - } - - recommender = HeatingRecommender(property_instance=property_instance) - - assert not recommender.heating_recommendations - - recommender.recommend(phase=0) - - assert recommender.recommendation is None - - def test_air_source_heat_pump_gas_boiler_starting(self): - starting_epc = { - 'low-energy-fixed-light-count': '', 'address': '430 Gidlow Lane', 'uprn-source': 'Energy Assessor', - 'floor-height': '2.62', 'heating-cost-potential': '599', 'unheated-corridor-length': '', - 'hot-water-cost-potential': '67', 'construction-age-band': 'England and Wales: 1950-1966', - 'potential-energy-rating': 'C', 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Good', - 'lighting-energy-eff': 'Very Good', 'environment-impact-potential': '72', - 'glazed-type': 'double glazing installed during or after 2002', 'heating-cost-current': '913', - 'address3': '', 'mainheatcont-description': 'Programmer, no room thermostat', 'sheating-energy-eff': 'N/A', - 'property-type': 'House', 'local-authority-label': 'Wigan', 'fixed-lighting-outlets-count': '9', - 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '210', - 'county': '', 'postcode': 'WN6 8RG', 'solar-water-heating-flag': 'N', 'constituency': 'E14001039', - 'co2-emissions-potential': '2.6', 'number-heated-rooms': '4', - 'floor-description': 'Solid, no insulation (assumed)', 'energy-consumption-potential': '180', - 'local-authority': 'E08000010', 'built-form': 'Mid-Terrace', 'number-open-fireplaces': '0', - 'windows-description': 'Fully double glazed', 'glazed-area': 'Normal', 'inspection-date': '2022-02-15', - 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '78', 'address1': '430 Gidlow Lane', - 'heat-loss-corridor': '', 'flat-storey-count': '', 'constituency-label': 'Wigan', - 'roof-energy-eff': 'Very Poor', 'total-floor-area': '80.0', 'building-reference-number': '10002334112', - 'environment-impact-current': '38', 'co2-emissions-current': '6.2', - 'roof-description': 'Pitched, no insulation (assumed)', 'floor-energy-eff': 'N/A', - 'number-habitable-rooms': '4', 'address2': '', 'hot-water-env-eff': 'Poor', 'posttown': 'WIGAN', - 'mainheatc-energy-eff': 'Very Poor', 'main-fuel': 'mains gas (not community)', - 'lighting-env-eff': 'Very Good', 'windows-energy-eff': 'Good', 'floor-env-eff': 'N/A', - 'sheating-env-eff': 'N/A', 'lighting-description': 'Low energy lighting in all fixed outlets', - 'roof-env-eff': 'Very Poor', 'walls-energy-eff': 'Average', 'photo-supply': '0.0', - 'lighting-cost-potential': '67', 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '100', - 'main-heating-controls': '', 'lodgement-datetime': '2022-02-23 16:39:41', 'flat-top-storey': '', - 'current-energy-rating': 'E', 'secondheat-description': 'Room heaters, mains gas', - 'walls-env-eff': 'Average', 'transaction-type': 'ECO assessment', 'uprn': '100011776843', - 'current-energy-efficiency': '45', 'energy-consumption-current': '441', - 'mainheat-description': 'Boiler and radiators, mains gas', 'lighting-cost-current': '67', - 'lodgement-date': '2022-02-23', 'extension-count': '1', 'mainheatc-env-eff': 'Very Poor', - 'lmk-key': '46cb404438a6d88ddff8965cab8b3027ec15c32d93e0b6a5f0381a5109b9bb0d', 'wind-turbine-count': '0', - 'tenure': 'Owner-occupied', 'floor-level': '', 'potential-energy-efficiency': '77', - 'hot-water-energy-eff': 'Poor', 'low-energy-lighting': '100', - 'walls-description': 'Cavity wall, filled cavity', - 'hotwater-description': 'From main system, no cylinder thermostat' - } - - ending_epc = { - 'low-energy-fixed-light-count': '', 'address': '430 Gidlow Lane', 'uprn-source': 'Energy Assessor', - 'floor-height': '2.62', 'heating-cost-potential': '803', 'unheated-corridor-length': '', - 'hot-water-cost-potential': '292', 'construction-age-band': 'England and Wales: 1950-1966', - 'potential-energy-rating': 'C', 'mainheat-energy-eff': 'Very Good', 'windows-env-eff': 'Good', - 'lighting-energy-eff': 'Very Good', 'environment-impact-potential': '78', - 'glazed-type': 'double glazing installed during or after 2002', 'heating-cost-current': '861', - 'address3': '', 'mainheatcont-description': 'Time and temperature zone control', - 'sheating-energy-eff': 'N/A', 'property-type': 'House', 'local-authority-label': 'Wigan', - 'fixed-lighting-outlets-count': '9', 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', - 'hot-water-cost-current': '434', 'county': '', 'postcode': 'WN6 8RG', 'solar-water-heating-flag': 'N', - 'constituency': 'E14001039', 'co2-emissions-potential': '2.0', 'number-heated-rooms': '4', - 'floor-description': 'Solid, no insulation (assumed)', 'energy-consumption-potential': '147', - 'local-authority': 'E08000010', 'built-form': 'Mid-Terrace', 'number-open-fireplaces': '0', - 'windows-description': 'Fully double glazed', 'glazed-area': 'Normal', 'inspection-date': '2022-05-11', - 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '43', 'address1': '430 Gidlow Lane', - 'heat-loss-corridor': '', 'flat-storey-count': '', 'constituency-label': 'Wigan', - 'roof-energy-eff': 'Very Poor', 'total-floor-area': '80.0', 'building-reference-number': '10002334112', - 'environment-impact-current': '63', 'co2-emissions-current': '3.4', - 'roof-description': 'Pitched, no insulation (assumed)', 'floor-energy-eff': 'N/A', - 'number-habitable-rooms': '4', 'address2': '', 'hot-water-env-eff': 'Poor', 'posttown': 'WIGAN', - 'mainheatc-energy-eff': 'Very Good', 'main-fuel': 'electricity (not community)', - 'lighting-env-eff': 'Very Good', 'windows-energy-eff': 'Good', 'floor-env-eff': 'N/A', - 'sheating-env-eff': 'N/A', 'lighting-description': 'Low energy lighting in all fixed outlets', - 'roof-env-eff': 'Very Poor', 'walls-energy-eff': 'Average', 'photo-supply': '0.0', - 'lighting-cost-potential': '67', 'mainheat-env-eff': 'Very Good', 'multi-glaze-proportion': '100', - 'main-heating-controls': '', 'lodgement-datetime': '2022-06-06 13:01:20', 'flat-top-storey': '', - 'current-energy-rating': 'E', 'secondheat-description': 'Room heaters, mains gas', - 'walls-env-eff': 'Average', 'transaction-type': 'ECO assessment', 'uprn': '100011776843', - 'current-energy-efficiency': '53', 'energy-consumption-current': '252', - 'mainheat-description': 'Air source heat pump, radiators, electric', 'lighting-cost-current': '67', - 'lodgement-date': '2022-06-06', 'extension-count': '1', 'mainheatc-env-eff': 'Very Good', - 'lmk-key': '672d5947f3d4a55d97255af71651d6127a939418fa66a687070af77e0ba90df2', 'wind-turbine-count': '0', - 'tenure': 'Owner-occupied', 'floor-level': '', 'potential-energy-efficiency': '70', - 'hot-water-energy-eff': 'Very Poor', 'low-energy-lighting': '100', - 'walls-description': 'Cavity wall, filled cavity', 'hotwater-description': 'From main system' - } - - # differences = [] - # for k, v in ending_epc.items(): - # if v != starting_epc[k]: - # differences.append( - # { - # "variable": k, - # "starting_value": starting_epc[k], - # "ending_value": v - # } - # ) - # differences = pd.DataFrame(differences) - # - # diffs = differences[ - # differences["variable"].isin( - # [ - # "mainheat-energy-eff", - # "mainheatcont-description", - # "mainheatc-energy-eff", - # "main-fuel", - # "mainheat-env-eff", - # "mainheat-description", - # "hot-water-energy-eff", - # "hotwater-description" - # ] - # ) - # ] - - cleaning_data = read_dataframe_from_s3_parquet( - bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet", - ) - - cleaned = read_from_s3( - s3_file_name="cleaned_epc_data/cleaned.bson", - bucket_name="retrofit-data-dev" - ) - cleaned = msgpack.unpackb(cleaned, raw=False) - - photo_supply_lookup, floor_area_decile_thresholds = SolarPhotoSupply.load(bucket="retrofit-data-dev") - - epc = EPCRecord( - epc_records={ - 'original_epc': starting_epc, - 'full_sap_epc': {}, - 'old_data': [] - }, - run_mode="newdata", - cleaning_data=cleaning_data - ) - - home = Property( - id=0, - address="", - postcode="", - epc_record=epc, - already_installed={}, - non_invasive_recommendations={}, - ) - home.in_conservation_area = False - home.is_listed = False - home.is_heritage = False - home.restricted_measures = True - home.get_components( - cleaned=cleaned, - photo_supply_lookup=photo_supply_lookup, - floor_area_decile_thresholds=floor_area_decile_thresholds - ) - - recommender = HeatingRecommender(property_instance=home) - recommender.recommend_air_source_heat_pump(phase=0, has_cavity_or_loft_recommendations=False) - - # Patch - for this property, the hot water energy efficiency is very poor. it's not clear why this is, - # but we insert this for this test - recommender.heating_recommendations[0]["simulation_config"]["hot_water_energy_eff_ending"] = "Very Poor" - - property_recommendations = Recommendations.insert_temp_recommendation_id([recommender.heating_recommendations]) - - assert len(recommender.heating_recommendations) == 1 - - home.create_base_difference_epc_record(cleaned_lookup=cleaned) - home.adjust_difference_record_with_recommendations( - property_recommendations, [] - ) - - scoring_data = pd.DataFrame(home.recommendations_scoring_data).drop( - columns=["rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending", - "carbon_ending"] - ) - - model_api = ModelApi(portfolio_id="ashp-test", timestamp=datetime.now().isoformat()) - model_api.MODEL_PREFIXES = ["sap_change_predictions"] - - predictions_dict = model_api.predict_all( - df=scoring_data, - bucket="retrofit-data-dev", - prediction_buckets={ - "sap_change_predictions": "retrofit-sap-predictions-dev", - } - ) - assert predictions_dict["sap_change_predictions"]["predictions"].values[0] == 52.2 - - def test_air_source_heat_pump_gas_boiler_starting_2(self): - """ - This property seems to have miniscule movement in SAP - just 2 poins - :return: - """ - - starting_epc = { - 'low-energy-fixed-light-count': '', 'address': '31 Whinney Hill Park', 'uprn-source': 'Energy Assessor', - 'floor-height': '2.3', 'heating-cost-potential': '394', 'unheated-corridor-length': '', - 'hot-water-cost-potential': '48', 'construction-age-band': 'England and Wales: 1967-1975', - 'potential-energy-rating': 'B', 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Average', - 'lighting-energy-eff': 'Good', 'environment-impact-potential': '87', - 'glazed-type': 'double glazing, unknown install date', 'heating-cost-current': '487', 'address3': '', - 'mainheatcont-description': 'Programmer, room thermostat and TRVs', 'sheating-energy-eff': 'N/A', - 'property-type': 'Bungalow', 'local-authority-label': 'Calderdale', 'fixed-lighting-outlets-count': '5', - 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '86', - 'county': '', 'postcode': 'HD6 2PX', 'solar-water-heating-flag': 'N', 'constituency': 'E14000614', - 'co2-emissions-potential': '0.8', 'number-heated-rooms': '2', - 'floor-description': 'Solid, no insulation (assumed)', 'energy-consumption-potential': '105', - 'local-authority': 'E08000033', 'built-form': 'End-Terrace', 'number-open-fireplaces': '0', - 'windows-description': 'Fully double glazed', 'glazed-area': 'Normal', 'inspection-date': '2021-11-25', - 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '56', 'address1': '31 Whinney Hill Park', - 'heat-loss-corridor': '', 'flat-storey-count': '', 'constituency-label': 'Calder Valley', - 'roof-energy-eff': 'Good', 'total-floor-area': '44.0', 'building-reference-number': '10001772583', - 'environment-impact-current': '62', 'co2-emissions-current': '2.5', - 'roof-description': 'Pitched, 250 mm loft insulation', 'floor-energy-eff': 'N/A', - 'number-habitable-rooms': '2', 'address2': '', 'hot-water-env-eff': 'Good', 'posttown': 'BRIGHOUSE', - 'mainheatc-energy-eff': 'Good', 'main-fuel': 'mains gas (not community)', 'lighting-env-eff': 'Good', - 'windows-energy-eff': 'Average', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A', - 'lighting-description': 'Low energy lighting in 60% of fixed outlets', 'roof-env-eff': 'Good', - 'walls-energy-eff': 'Average', 'photo-supply': '0.0', 'lighting-cost-potential': '40', - 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '100', 'main-heating-controls': '', - 'lodgement-datetime': '2021-11-25 11:39:35', 'flat-top-storey': '', 'current-energy-rating': 'D', - 'secondheat-description': 'Room heaters, electric', 'walls-env-eff': 'Average', - 'transaction-type': 'rental', 'uprn': '100051304421', 'current-energy-efficiency': '62', - 'energy-consumption-current': '322', 'mainheat-description': 'Boiler and radiators, mains gas', - 'lighting-cost-current': '56', 'lodgement-date': '2021-11-25', 'extension-count': '0', - 'mainheatc-env-eff': 'Good', 'lmk-key': '077f70657e9c3f1f0ce5392798398398616b159493b2a8ca2338961596631c27', - 'wind-turbine-count': '0', 'tenure': 'Rented (social)', 'floor-level': '', - 'potential-energy-efficiency': '86', 'hot-water-energy-eff': 'Good', 'low-energy-lighting': '60', - 'walls-description': 'Cavity wall, filled cavity', 'hotwater-description': 'From main system' - } - - ending_epc = { - 'low-energy-fixed-light-count': '', 'address': '31 Whinney Hill Park', - 'uprn-source': 'Energy Assessor', 'floor-height': '2.3', 'heating-cost-potential': '277', - 'unheated-corridor-length': '', 'hot-water-cost-potential': '266', - 'construction-age-band': 'England and Wales: 1967-1975', 'potential-energy-rating': 'B', - 'mainheat-energy-eff': 'Very Good', 'windows-env-eff': 'Average', 'lighting-energy-eff': 'Good', - 'environment-impact-potential': '90', 'glazed-type': 'double glazing, unknown install date', - 'heating-cost-current': '331', 'address3': '', - 'mainheatcont-description': 'Programmer and room thermostat', 'sheating-energy-eff': 'N/A', - 'property-type': 'Bungalow', 'local-authority-label': 'Calderdale', - 'fixed-lighting-outlets-count': '5', 'energy-tariff': 'Single', - 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '404', 'county': '', - 'postcode': 'HD6 2PX', 'solar-water-heating-flag': 'N', 'constituency': 'E14000614', - 'co2-emissions-potential': '0.7', 'number-heated-rooms': '2', - 'floor-description': 'Solid, no insulation (assumed)', 'energy-consumption-potential': '92', - 'local-authority': 'E08000033', 'built-form': 'End-Terrace', 'number-open-fireplaces': '0', - 'windows-description': 'Fully double glazed', 'glazed-area': 'Normal', - 'inspection-date': '2021-11-25', 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '48', - 'address1': '31 Whinney Hill Park', 'heat-loss-corridor': '', 'flat-storey-count': '', - 'constituency-label': 'Calder Valley', 'roof-energy-eff': 'Good', 'total-floor-area': '44.0', - 'building-reference-number': '10001772583', 'environment-impact-current': '68', - 'co2-emissions-current': '2.1', 'roof-description': 'Pitched, 250 mm loft insulation', - 'floor-energy-eff': 'N/A', 'number-habitable-rooms': '2', 'address2': '', - 'hot-water-env-eff': 'Poor', 'posttown': 'BRIGHOUSE', 'mainheatc-energy-eff': 'Average', - 'main-fuel': 'electricity (not community)', 'lighting-env-eff': 'Good', - 'windows-energy-eff': 'Average', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A', - 'lighting-description': 'Low energy lighting in 60% of fixed outlets', 'roof-env-eff': 'Good', - 'walls-energy-eff': 'Average', 'photo-supply': '0.0', 'lighting-cost-potential': '40', - 'mainheat-env-eff': 'Very Good', 'multi-glaze-proportion': '100', 'main-heating-controls': '', - 'lodgement-datetime': '2022-03-23 16:06:21', 'flat-top-storey': '', 'current-energy-rating': 'D', - 'secondheat-description': 'Room heaters, electric', 'walls-env-eff': 'Average', - 'transaction-type': 'rental', 'uprn': '100051304421', 'current-energy-efficiency': '64', - 'energy-consumption-current': '283', - 'mainheat-description': 'Air source heat pump, radiators, electric', - 'lighting-cost-current': '57', 'lodgement-date': '2022-03-23', 'extension-count': '0', - 'mainheatc-env-eff': 'Average', - 'lmk-key': '6296248141447b53426a40f1c39da17dad5f4786485db55ee38737891111a4d4', - 'wind-turbine-count': '0', 'tenure': 'Rented (social)', 'floor-level': '', - 'potential-energy-efficiency': '89', 'hot-water-energy-eff': 'Very Poor', - 'low-energy-lighting': '60', 'walls-description': 'Cavity wall, filled cavity', - 'hotwater-description': 'From main system' - } - - # differences = [] - # for k, v in ending_epc.items(): - # if v != starting_epc[k]: - # differences.append( - # { - # "variable": k, - # "starting_value": starting_epc[k], - # "ending_value": v - # } - # ) - # differences = pd.DataFrame(differences) - # - # diffs = differences[ - # differences["variable"].isin( - # [ - # "mainheat-energy-eff", - # "mainheatcont-description", - # "mainheatc-energy-eff", - # "main-fuel", - # "mainheat-env-eff", - # "mainheat-description", - # "hot-water-energy-eff", - # "hotwater-description" - # ] - # ) - # ] - - cleaning_data = read_dataframe_from_s3_parquet( - bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet", - ) - - cleaned = read_from_s3( - s3_file_name="cleaned_epc_data/cleaned.bson", - bucket_name="retrofit-data-dev" - ) - cleaned = msgpack.unpackb(cleaned, raw=False) - - photo_supply_lookup, floor_area_decile_thresholds = SolarPhotoSupply.load(bucket="retrofit-data-dev") - - epc = EPCRecord( - epc_records={ - 'original_epc': starting_epc, - 'full_sap_epc': {}, - 'old_data': [] - }, - run_mode="newdata", - cleaning_data=cleaning_data - ) - - home = Property( - id=0, - address="", - postcode="", - epc_record=epc, - already_installed={}, - non_invasive_recommendations={}, - ) - home.in_conservation_area = False - home.is_listed = False - home.is_heritage = False - home.restricted_measures = True - home.get_components( - cleaned=cleaned, - photo_supply_lookup=photo_supply_lookup, - floor_area_decile_thresholds=floor_area_decile_thresholds - ) - - recommender = HeatingRecommender(property_instance=home) - recommender.recommend_air_source_heat_pump(phase=0, has_cavity_or_loft_recommendations=False) - property_recommendations = Recommendations.insert_temp_recommendation_id([recommender.heating_recommendations]) - - assert len(recommender.heating_recommendations) == 1 - - home.create_base_difference_epc_record(cleaned_lookup=cleaned) - home.adjust_difference_record_with_recommendations( - property_recommendations, [] - ) - - scoring_data = pd.DataFrame(home.recommendations_scoring_data).drop( - columns=["rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending", - "carbon_ending"] - ) - - model_api = ModelApi(portfolio_id="ashp-test", timestamp=datetime.now().isoformat()) - model_api.MODEL_PREFIXES = ["sap_change_predictions"] - - predictions_dict = model_api.predict_all( - df=scoring_data, - bucket="retrofit-data-dev", - prediction_buckets={ - "sap_change_predictions": "retrofit-sap-predictions-dev", - } - ) - assert predictions_dict["sap_change_predictions"]["predictions"].values[0] == 69.3 - - # In actuality with this property, the heating controls get downgraded, so we test a manual patch of this - patched_simulation_config = { - 'mainheat_energy_eff_ending': "Very Good", - 'hot_water_energy_eff_ending': 'Very Poor', - 'has_boiler_ending': False, - 'has_air_source_heat_pump_ending': True, - 'has_electric_ending': True, - 'has_mains_gas_ending': False, - 'fuel_type_ending': 'electricity', - 'trvs_ending': None, - "mainheatc_energy_eff_ending": 'Average' - } - - # PATCHING - property_recommendations_patch = Recommendations.insert_temp_recommendation_id( - [recommender.heating_recommendations] - ) - property_recommendations_patch[0][0]["simulation_config"] = patched_simulation_config - - home.create_base_difference_epc_record(cleaned_lookup=cleaned) - home.adjust_difference_record_with_recommendations( - property_recommendations_patch, [] - ) - - scoring_data_patch = pd.DataFrame(home.recommendations_scoring_data).drop( - columns=["rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending", - "carbon_ending"] - ) - - model_api = ModelApi(portfolio_id="ashp-test", timestamp=datetime.now().isoformat()) - model_api.MODEL_PREFIXES = ["sap_change_predictions"] - - predictions_dict_patch = model_api.predict_all( - df=scoring_data_patch, - bucket="retrofit-data-dev", - prediction_buckets={ - "sap_change_predictions": "retrofit-sap-predictions-dev", - } - ) - # The error is only 0.3, so the model is working - assert predictions_dict_patch["sap_change_predictions"]["predictions"].values[0] == 64.3 - assert ending_epc["current-energy-efficiency"] == '64' - - def test_air_source_heat_pump_lpg_boiler(self): - starting_epc = { - 'low-energy-fixed-light-count': '', 'address': 'Holly Lodge, The Drive, Perry', - 'uprn-source': 'Energy Assessor', 'floor-height': '2.8', 'heating-cost-potential': '1628', - 'unheated-corridor-length': '', 'hot-water-cost-potential': '175', - 'construction-age-band': 'England and Wales: 1950-1966', 'potential-energy-rating': 'D', - 'mainheat-energy-eff': 'Poor', 'windows-env-eff': 'Average', 'lighting-energy-eff': 'Average', - 'environment-impact-potential': '70', 'glazed-type': 'double glazing, unknown install date', - 'heating-cost-current': '2158', 'address3': 'Perry', - 'mainheatcont-description': 'No time or thermostatic control of room temperature', - 'sheating-energy-eff': 'N/A', 'property-type': 'Bungalow', 'local-authority-label': 'Huntingdonshire', - 'fixed-lighting-outlets-count': '12', 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', - 'hot-water-cost-current': '257', 'county': 'Cambridgeshire', 'postcode': 'PE28 0SX', - 'solar-water-heating-flag': 'N', 'constituency': 'E14000757', 'co2-emissions-potential': '3.3', - 'number-heated-rooms': '5', 'floor-description': 'Solid, no insulation (assumed)', - 'energy-consumption-potential': '128', 'local-authority': 'E07000011', 'built-form': 'Semi-Detached', - 'number-open-fireplaces': '0', 'windows-description': 'Fully double glazed', 'glazed-area': 'Normal', - 'inspection-date': '2023-08-31', 'mains-gas-flag': 'N', 'co2-emiss-curr-per-floor-area': '51', - 'address1': 'Holly Lodge', 'heat-loss-corridor': '', 'flat-storey-count': '', - 'constituency-label': 'Huntingdon', 'roof-energy-eff': 'Good', 'total-floor-area': '117.0', - 'building-reference-number': '10005199915', 'environment-impact-current': '50', - 'co2-emissions-current': '5.9', 'roof-description': 'Pitched, 270 mm loft insulation', - 'floor-energy-eff': 'N/A', 'number-habitable-rooms': '5', 'address2': 'The Drive', - 'hot-water-env-eff': 'Good', 'posttown': 'HUNTINGDON', 'mainheatc-energy-eff': 'Very Poor', - 'main-fuel': 'LPG (not community)', 'lighting-env-eff': 'Average', 'windows-energy-eff': 'Average', - 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A', - 'lighting-description': 'Low energy lighting in 33% of fixed outlets', 'roof-env-eff': 'Good', - 'walls-energy-eff': 'Average', 'photo-supply': '0.0', 'lighting-cost-potential': '166', - 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '100', 'main-heating-controls': '', - 'lodgement-datetime': '2023-10-30 13:46:54', 'flat-top-storey': '', 'current-energy-rating': 'F', - 'secondheat-description': 'Room heaters, electric', 'walls-env-eff': 'Average', - 'transaction-type': 'ECO assessment', 'uprn': '100091200828', 'current-energy-efficiency': '32', - 'energy-consumption-current': '243', 'mainheat-description': 'Boiler and radiators, LPG', - 'lighting-cost-current': '277', 'lodgement-date': '2023-10-30', 'extension-count': '0', - 'mainheatc-env-eff': 'Very Poor', - 'lmk-key': 'f1d3bd4b8b50bc9b006231ccb158537c408523b748b3f4ef7e98cd03b144afa5', 'wind-turbine-count': '0', - 'tenure': 'Owner-occupied', 'floor-level': '', 'potential-energy-efficiency': '56', - 'hot-water-energy-eff': 'Poor', 'low-energy-lighting': '33', - 'walls-description': 'Cavity wall, filled cavity', 'hotwater-description': 'From main system' - } - - ending_epc = { - 'low-energy-fixed-light-count': '', 'address': 'Holly Lodge, The Drive, Perry', - 'uprn-source': 'Energy Assessor', 'floor-height': '2.8', 'heating-cost-potential': '917', - 'unheated-corridor-length': '', 'hot-water-cost-potential': '328', - 'construction-age-band': 'England and Wales: 1950-1966', 'potential-energy-rating': 'A', - 'mainheat-energy-eff': 'Very Good', 'windows-env-eff': 'Average', 'lighting-energy-eff': 'Average', - 'environment-impact-potential': '96', 'glazed-type': 'double glazing, unknown install date', - 'heating-cost-current': '1098', 'address3': 'Perry', - 'mainheatcont-description': 'Programmer, TRVs and bypass', 'sheating-energy-eff': 'N/A', - 'property-type': 'Bungalow', 'local-authority-label': 'Huntingdonshire', - 'fixed-lighting-outlets-count': '12', 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', - 'hot-water-cost-current': '328', 'county': 'Cambridgeshire', 'postcode': 'PE28 0SX', - 'solar-water-heating-flag': 'N', 'constituency': 'E14000757', 'co2-emissions-potential': '0.3', - 'number-heated-rooms': '5', 'floor-description': 'Solid, no insulation (assumed)', - 'energy-consumption-potential': '16', 'local-authority': 'E07000011', 'built-form': 'Semi-Detached', - 'number-open-fireplaces': '0', 'windows-description': 'Fully double glazed', 'glazed-area': 'Normal', - 'inspection-date': '2023-10-05', 'mains-gas-flag': 'N', 'co2-emiss-curr-per-floor-area': '6', - 'address1': 'Holly Lodge', 'heat-loss-corridor': '', 'flat-storey-count': '', - 'constituency-label': 'Huntingdon', 'roof-energy-eff': 'Good', 'total-floor-area': '117.0', - 'building-reference-number': '10005199915', 'environment-impact-current': '92', - 'co2-emissions-current': '0.7', 'roof-description': 'Pitched, 270 mm loft insulation', - 'floor-energy-eff': 'N/A', 'number-habitable-rooms': '5', 'address2': 'The Drive', - 'hot-water-env-eff': 'Very Good', 'posttown': 'HUNTINGDON', 'mainheatc-energy-eff': 'Average', - 'main-fuel': 'electricity (not community)', 'lighting-env-eff': 'Average', 'windows-energy-eff': 'Average', - 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A', - 'lighting-description': 'Low energy lighting in 33% of fixed outlets', 'roof-env-eff': 'Good', - 'walls-energy-eff': 'Average', 'photo-supply': '', 'lighting-cost-potential': '166', - 'mainheat-env-eff': 'Very Good', 'multi-glaze-proportion': '100', 'main-heating-controls': '', - 'lodgement-datetime': '2023-11-01 16:29:16', 'flat-top-storey': '', 'current-energy-rating': 'A', - 'secondheat-description': 'Room heaters, electric', 'walls-env-eff': 'Average', - 'transaction-type': 'ECO assessment', 'uprn': '100091200828', 'current-energy-efficiency': '92', - 'energy-consumption-current': '37', 'mainheat-description': 'Air source heat pump, radiators, electric', - 'lighting-cost-current': '277', 'lodgement-date': '2023-11-01', 'extension-count': '0', - 'mainheatc-env-eff': 'Average', - 'lmk-key': 'cb7f2838b727907767c8c2a385cd22f722b1e4745463391d910d228e52124515', 'wind-turbine-count': '0', - 'tenure': 'Owner-occupied', 'floor-level': '', 'potential-energy-efficiency': '95', - 'hot-water-energy-eff': 'Good', 'low-energy-lighting': '33', - 'walls-description': 'Cavity wall, filled cavity', 'hotwater-description': 'From main system' - } - - cleaning_data = read_dataframe_from_s3_parquet( - bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet", - ) - - cleaned = read_from_s3( - s3_file_name="cleaned_epc_data/cleaned.bson", - bucket_name="retrofit-data-dev" - ) - cleaned = msgpack.unpackb(cleaned, raw=False) - - photo_supply_lookup, floor_area_decile_thresholds = SolarPhotoSupply.load(bucket="retrofit-data-dev") - - epc = EPCRecord( - epc_records={ - 'original_epc': starting_epc, - 'full_sap_epc': {}, - 'old_data': [] - }, - run_mode="newdata", - cleaning_data=cleaning_data - ) - - home = Property( - id=0, - address="", - postcode="", - epc_record=epc, - already_installed={}, - non_invasive_recommendations={}, - ) - home.in_conservation_area = False - home.is_listed = False - home.is_heritage = False - home.restricted_measures = True - home.get_components( - cleaned=cleaned, - photo_supply_lookup=photo_supply_lookup, - floor_area_decile_thresholds=floor_area_decile_thresholds - ) - - recommender = HeatingRecommender(property_instance=home) - recommender.recommend_air_source_heat_pump(phase=0, has_cavity_or_loft_recommendations=False) - property_recommendations = Recommendations.insert_temp_recommendation_id([recommender.heating_recommendations]) - - assert len(recommender.heating_recommendations) == 1 - - home.create_base_difference_epc_record(cleaned_lookup=cleaned) - home.adjust_difference_record_with_recommendations( - property_recommendations, [] - ) - - scoring_data = pd.DataFrame(home.recommendations_scoring_data).drop( - columns=["rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending", - "carbon_ending"] - ) - - model_api = ModelApi(portfolio_id="ashp-test", timestamp=datetime.now().isoformat()) - model_api.MODEL_PREFIXES = ["sap_change_predictions"] - - predictions_dict = model_api.predict_all( - df=scoring_data, - bucket="retrofit-data-dev", - prediction_buckets={ - "sap_change_predictions": "retrofit-sap-predictions-dev", - } - ) - # We predict a huge uplift but not quite as much as the EPC, due to some distinct differences between our - # recommendation and the EPC - assert predictions_dict["sap_change_predictions"]["predictions"].values[0] == 81.3 - assert ending_epc['current-energy-efficiency'] == '92' - - # PATCH - # We patch the simulation config, to reflect the ending EPC, to see if we get the ending EPC's config - patched_simulation_config = { - 'mainheat_energy_eff_ending': "Very Good", - 'hot_water_energy_eff_ending': 'Good', - 'has_boiler_ending': False, - 'has_air_source_heat_pump_ending': True, - 'has_electric_ending': True, - 'has_lpg_ending': False, - 'fuel_type_ending': 'electricity', - 'switch_system_ending': 'programmer', - 'no_control_ending': None, - 'auxiliary_systems_ending': 'bypass', - 'trvs_ending': 'trvs', - "mainheatc_energy_eff_ending": 'Average' - } - - # PATCHING - property_recommendations_patch = Recommendations.insert_temp_recommendation_id( - [recommender.heating_recommendations] - ) - property_recommendations_patch[0][0]["simulation_config"] = patched_simulation_config - - home.create_base_difference_epc_record(cleaned_lookup=cleaned) - home.adjust_difference_record_with_recommendations( - property_recommendations_patch, [] - ) - - scoring_data_patch = pd.DataFrame(home.recommendations_scoring_data).drop( - columns=["rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending", - "carbon_ending"] - ) - - model_api = ModelApi(portfolio_id="ashp-test", timestamp=datetime.now().isoformat()) - model_api.MODEL_PREFIXES = ["sap_change_predictions"] - - predictions_dict_patch = model_api.predict_all( - df=scoring_data_patch, - bucket="retrofit-data-dev", - prediction_buckets={ - "sap_change_predictions": "retrofit-sap-predictions-dev", - } - ) - - assert predictions_dict_patch["sap_change_predictions"]["predictions"].values[0] == 88.9 - # We still underpredict but the improvement is notable - - def test_offgrid(self): - """ - We test on a property we've worked with before, where we compare two options - a) Upgrading to a boiler - b) Upgrading to a heat pump - :return: - """ - - starting_epc = { - 'low-energy-fixed-light-count': '', 'address': '6 Beech Road', 'uprn-source': 'Energy Assessor', - 'floor-height': '2.4', 'heating-cost-potential': '612', 'unheated-corridor-length': '', - 'hot-water-cost-potential': '123', 'construction-age-band': 'England and Wales: 1930-1949', - 'potential-energy-rating': 'B', 'mainheat-energy-eff': 'Very Poor', 'windows-env-eff': 'Good', - 'lighting-energy-eff': 'Good', 'environment-impact-potential': '87', - 'glazed-type': 'double glazing installed during or after 2002', 'heating-cost-current': '2278', - 'address3': '', 'mainheatcont-description': 'Appliance thermostats', 'sheating-energy-eff': 'N/A', - 'property-type': 'House', 'local-authority-label': 'Dudley', 'fixed-lighting-outlets-count': '9', - 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '604', - 'county': '', 'postcode': 'DY1 4BP', 'solar-water-heating-flag': 'N', 'constituency': 'E14000671', - 'co2-emissions-potential': '1.0', 'number-heated-rooms': '4', - 'floor-description': 'Solid, no insulation (assumed)', 'energy-consumption-potential': '93', - 'local-authority': 'E08000027', 'built-form': 'End-Terrace', 'number-open-fireplaces': '0', - 'windows-description': 'Fully double glazed', 'glazed-area': 'Normal', 'inspection-date': '2024-03-13', - 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '83', 'address1': '6 Beech Road', - 'heat-loss-corridor': '', 'flat-storey-count': '', 'constituency-label': 'Dudley North', - 'roof-energy-eff': 'Very Poor', 'total-floor-area': '60.0', 'building-reference-number': '10005780080', - 'environment-impact-current': '41', 'co2-emissions-current': '5.0', - 'roof-description': 'Pitched, 12 mm loft insulation', 'floor-energy-eff': 'N/A', - 'number-habitable-rooms': '4', 'address2': '', 'hot-water-env-eff': 'Poor', 'posttown': 'DUDLEY', - 'mainheatc-energy-eff': 'Good', 'main-fuel': 'electricity (not community)', 'lighting-env-eff': 'Good', - 'windows-energy-eff': 'Good', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A', - 'lighting-description': 'Low energy lighting in 67% of fixed outlets', 'roof-env-eff': 'Very Poor', - 'walls-energy-eff': 'Average', 'photo-supply': '0.0', 'lighting-cost-potential': '113', - 'mainheat-env-eff': 'Poor', 'multi-glaze-proportion': '100', 'main-heating-controls': '', - 'lodgement-datetime': '2024-03-13 11:29:11', 'flat-top-storey': '', 'current-energy-rating': 'F', - 'secondheat-description': 'None', 'walls-env-eff': 'Average', 'transaction-type': 'rental', - 'uprn': '90055152', 'current-energy-efficiency': '32', 'energy-consumption-current': '491', - 'mainheat-description': 'Room heaters, electric', 'lighting-cost-current': '113', - 'lodgement-date': '2024-03-13', 'extension-count': '1', 'mainheatc-env-eff': 'Good', - 'lmk-key': '78ddf851b660e599a0894924d0e6b503980f5e0ad1aa711f8411718dc2989c44', 'wind-turbine-count': '0', - 'tenure': 'Rented (social)', 'floor-level': '', 'potential-energy-efficiency': '87', - 'hot-water-energy-eff': 'Very Poor', 'low-energy-lighting': '67', - 'walls-description': 'Cavity wall, filled cavity', - 'hotwater-description': 'Electric immersion, standard tariff' - } - - cleaning_data = read_dataframe_from_s3_parquet( - bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet", - ) - - cleaned = read_from_s3( - s3_file_name="cleaned_epc_data/cleaned.bson", - bucket_name="retrofit-data-dev" - ) - cleaned = msgpack.unpackb(cleaned, raw=False) - - photo_supply_lookup, floor_area_decile_thresholds = SolarPhotoSupply.load(bucket="retrofit-data-dev") - - epc = EPCRecord( - epc_records={ - 'original_epc': starting_epc, - 'full_sap_epc': {}, - 'old_data': [] - }, - run_mode="newdata", - cleaning_data=cleaning_data - ) - - home = Property( - id=0, - address="", - postcode="", - epc_record=epc, - already_installed={}, - non_invasive_recommendations={}, - ) - home.in_conservation_area = False - home.is_listed = False - home.is_heritage = False - home.restricted_measures = True - home.get_components( - cleaned=cleaned, - photo_supply_lookup=photo_supply_lookup, - floor_area_decile_thresholds=floor_area_decile_thresholds - ) - - recommender = HeatingRecommender(property_instance=home) - recommender.recommend_air_source_heat_pump(phase=0, has_cavity_or_loft_recommendations=False) - recommender.recommend_boiler_upgrades(phase=0, system_change=True, exising_room_heaters=False) - - assert len(recommender.heating_recommendations) == 3 - - property_recommendations = Recommendations.insert_temp_recommendation_id([recommender.heating_recommendations]) - - home.create_base_difference_epc_record(cleaned_lookup=cleaned) - home.adjust_difference_record_with_recommendations( - property_recommendations, [] - ) - - scoring_data = pd.DataFrame(home.recommendations_scoring_data).drop( - columns=["rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending", - "carbon_ending"] - ) - - model_api = ModelApi(portfolio_id="ashp-test", timestamp=datetime.now().isoformat()) - model_api.MODEL_PREFIXES = ["sap_change_predictions"] - - predictions_dict = model_api.predict_all( - df=scoring_data, - bucket="retrofit-data-dev", - prediction_buckets={ - "sap_change_predictions": "retrofit-sap-predictions-dev", - } - ) - - # The ASHP isn't better under SAP, compared to a gas boiler with good heat controls - assert predictions_dict["sap_change_predictions"]["predictions"].tolist() == [66.9, 65.5, 65.9] diff --git a/recommendations/tests/test_costs.py b/recommendations/tests/test_costs.py index 402e38eb..74a210c1 100644 --- a/recommendations/tests/test_costs.py +++ b/recommendations/tests/test_costs.py @@ -18,10 +18,9 @@ class TestCosts: "description": "cwi", "depth": 75, "thermal_conductivity": 0.037, - "prime_cost": 5.17, - "material_cost": 5.62, - "labour_cost": 1.125, + "total_cost": 14, "labour_hours_per_unit": 0.065, + "is_installer_quote": True } cwi_results = costs.cavity_wall_insulation( @@ -29,12 +28,7 @@ class TestCosts: material=cwi_material, ) - assert cwi_results == { - 'total': 1065.0661223512907, 'subtotal': 887.5551019594088, 'vat': 177.51102039188177, - 'contingency': 63.396792997100626, 'preliminaries': 63.396792997100626, 'material': 539.0166061175574, - 'profit': 126.79358599420125, 'labour_hours': 6.234177828761786, 'labour_cost': 94.95132385344874, - 'labour_days': 0.38963611429761164 - } + assert cwi_results == {'total': 1342.7459938871539, 'labour_hours': 8, 'labour_days': 1} def test_loft_insulation(self): mock_property = Mock() @@ -47,22 +41,17 @@ class TestCosts: "description": "Crown Loft Roll 44 glass fibre roll", "depth": 270, "thermal_conductivity": 0.044, - "prime_cost": None, - "material_cost": 5.91938, - "labour_cost": 1.96, - "labour_hours_per_unit": 0.11 + "total_cost": 11, + "labour_hours_per_unit": 0.11, + "is_installer_quote": True, } - loft_results = costs.loft_insulation( + loft_results = costs.loft_and_flat_insulation( floor_area=33.5, material=loft_material, ) - assert loft_results == { - 'total': 639.4133610000001, 'subtotal': 532.8444675000001, 'vat': 106.56889350000002, - 'contingency': 71.045929, 'preliminaries': 35.5229645, 'material': 297.448845, 'profit': 71.045929, - 'labour_hours': 3.685, 'labour_cost': 57.7808, 'labour_days': 0.460625 - } + assert loft_results == {'total': 368.5, 'labour_hours': 8, 'labour_days': 1} def test_internal_wall_insulation(self): mock_property = Mock() @@ -71,87 +60,6 @@ class TestCosts: } costs = Costs(mock_property) - iwi_non_insulation_materials = [ - {'type': 'iwi_wall_demolition', - 'description': 'Solid & Dry Lined walls: Hack of wall finishes with chipping hammer; plaster to walls.', - 'depth': 0.0, 'depth_unit': 0.0, 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.0, - 'thermal_conductivity_unit': 0.0, 'prime_material_cost': 0.0, 'material_cost': 0.0, 'labour_cost': 10.27, - 'labour_hours_per_unit': 0.33, 'plant_cost': 1.28, 'total_cost': 11.55, 'link': 'SPONs', 'Notes': 0.0}, - {'type': 'iwi_wall_demolition', - 'description': 'Stud walls: Remove wall linings including battening behind; plasterboard and skim', - 'depth': 0.0, 'depth_unit': 0.0, 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.0, - 'thermal_conductivity_unit': 0.0, 'prime_material_cost': 0.0, 'material_cost': 0.0, 'labour_cost': 6.23, - 'labour_hours_per_unit': 0.2, 'plant_cost': 1.25, 'total_cost': 7.48, 'link': 'SPONs', 'Notes': 0.0}, - {'type': 'iwi_wall_demolition', - 'description': 'Lathe and Plaster walls: Remove wall linings including battening behind; wood lath and ' - 'plaster', - 'depth': 0.0, 'depth_unit': 0.0, 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.0, - 'thermal_conductivity_unit': 0.0, 'prime_material_cost': 0.0, 'material_cost': 0.0, 'labour_cost': 6.85, - 'labour_hours_per_unit': 0.22, 'plant_cost': 2.09, 'total_cost': 8.94, 'link': 'SPONs', 'Notes': 0.0}, - {'Notes': "", - 'cost_unit': "", - 'depth': "", - 'depth_unit': "", - 'description': 'Visqueen High Performance Vapour Barrier', - 'labour_cost': 0.48, - 'labour_hours_per_unit': 0.02, - 'link': 'SPONs', - 'material_cost': 1.21, - 'plant_cost': 0, - 'prime_material_cost': 0.58, - 'thermal_conductivity': "", - 'thermal_conductivity_unit': "", - 'total_cost': 1.69, - 'type': 'iwi_vapour_barrier'}, - {'Notes': "", - 'cost_unit': "", - 'depth': "", - 'depth_unit': "", - 'description': 'Plaster; one coat Thistle board finish or other equal; steel trowelled; 3 mm thick work ' - 'to walls or ceilings; one coat; to plasterboard base; over 600mm wide', - 'labour_cost': 6.58, - 'labour_hours_per_unit': 0.25, - 'link': "", - 'material_cost': 0.06, - 'plant_cost': 0, - 'prime_material_cost': 0.0, - 'thermal_conductivity': "", - 'thermal_conductivity_unit': "", - 'total_cost': 6.64, - 'type': 'iwi_redecoration'}, - {'Notes': "", - 'cost_unit': "", - 'depth': "", - 'depth_unit': "", - 'description': 'Two coats emulsion paint on plaster, over 40mm girth; 3.5m - ' - '5m high', - 'labour_cost': 0.0, - 'labour_hours_per_unit': 0.21, - 'link': "", - 'material_cost': 0.41, - 'plant_cost': 0, - 'prime_material_cost': "", - 'thermal_conductivity': "", - 'thermal_conductivity_unit': "", - 'total_cost': 4.34, - 'type': 'iwi_redecoration'}, - {'Notes': "", - 'cost_unit': "", - 'depth': "", - 'depth_unit': "", - 'description': 'Fitting existing softwood skirting or architrave to new ' - 'frames; 150mm high', - 'labour_cost': 4.87, - 'labour_hours_per_unit': 0.01, - 'link': "", - 'material_cost': 4.86, - 'plant_cost': 0, - 'prime_material_cost': "", - 'thermal_conductivity': "", - 'thermal_conductivity_unit': "", - 'total_cost': 4.88, - 'type': 'iwi_redecoration'} - ] iwi_material = { "type": "internal_wall_insulation", @@ -161,26 +69,19 @@ class TestCosts: "cost_unit": "gbp_per_m2", "thermal_conductivity": 0.022, "thermal_conductivity_unit": "watt_per_meter_kelvin", - "prime_material_cost": "", - "material_cost": 11.68, - "labour_cost": 3.12, "labour_hours_per_unit": 0.18, - "plant_cost": "", - "total_cost": 14.8, - "link": "SPONs" + "total_cost": 200, + "link": "link", + "is_installer_quote": True } - iwi_results = costs.internal_wall_insulation( + iwi_results = costs.solid_wall_insulation( wall_area=95.9104281347967, material=iwi_material, - non_insulation_materials=iwi_non_insulation_materials ) assert iwi_results == { - 'total': 6880.2304726777775, 'subtotal': 5733.525393898148, 'vat': 1146.7050787796295, - 'contingency': 764.470052519753, 'preliminaries': 382.2350262598765, 'material': 1747.488000615996, - 'profit': 764.470052519753, 'labour_hours': 88.23759388401297, 'labour_days': 2.757424808875405, - 'labour_cost': 1927.1602026551818 + 'total': 19182.085626959342, 'labour_hours': 17.263877064263404, 'labour_days': 0.5394961582582314 } def test_suspended_floor_insulation(self): @@ -201,7 +102,8 @@ class TestCosts: 'total_cost': 13.46, 'link': 'SPONs', 'Notes': 'Spons did not contain labour costs so we use values for similar insulations. ' 'We use the ' - 'same values as in Crown loft roll 44, since it is also an insulation roll' + 'same values as in Crown loft roll 44, since it is also an insulation roll', + "is_installer_quote": False } sus_floor_non_insulation_materials = [ @@ -256,7 +158,7 @@ class TestCosts: 'depth': 100.0, 'depth_unit': 'mm', 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.033, 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'prime_material_cost': 0, 'material_cost': 12.02, 'labour_cost': 4.4, 'labour_hours_per_unit': 0.19, 'plant_cost': 0, - 'total_cost': 16.42, 'link': 'SPONs', 'Notes': 0 + 'total_cost': 16.42, 'link': 'SPONs', 'Notes': 0, "is_installer_quote": False } sol_floor_non_insulation_materials = [ @@ -342,81 +244,18 @@ class TestCosts: ewi_material = { 'type': 'external_wall_insulation', 'description': 'Ecotherm Eco-Versal PIR Insulation Board', 'depth': 150.0, 'depth_unit': 'mm', 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.022, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'prime_material_cost': 23.53, - 'material_cost': 34.62, 'labour_cost': 33.06, 'labour_hours_per_unit': 1.4, 'plant_cost': 0, - 'total_cost': 67.68, 'link': 'SPONs', 'Notes': 0 + 'thermal_conductivity_unit': 'watt_per_meter_kelvin', + 'labour_hours_per_unit': 1.4, + 'total_cost': 300, 'link': 'SPONs', 'Notes': 0, "is_installer_quote": True } - ewi_non_insulation_materials = [ - {'type': 'ewi_wall_demolition', - 'description': 'Solid & Dry Lined walls: Hack of wall finishes with chipping ' - 'hammer; plaster to walls.', - 'depth': 0, 'depth_unit': 0, 'cost_unit': 'gbp_per_m2', - 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, - 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 10.27, - 'labour_hours_per_unit': 0.33, 'plant_cost': 1.28, 'total_cost': 11.55, - 'link': 'SPONs', 'Notes': 0}, {'type': 'ewi_wall_demolition', - 'description': 'Stud walls: Remove wall linings ' - 'including battening behind; ' - 'plasterboard and skim', - 'depth': 0, 'depth_unit': 0, - 'cost_unit': 'gbp_per_m2', - 'thermal_conductivity': 0, - 'thermal_conductivity_unit': 0, - 'prime_material_cost': 0, 'material_cost': 0, - 'labour_cost': 6.23, 'labour_hours_per_unit': 0.2, - 'plant_cost': 1.25, 'total_cost': 7.48, - 'link': 'SPONs', 'Notes': 0}, - {'type': 'ewi_wall_demolition', - 'description': 'Lathe and Plaster walls: Remove wall linings including battening ' - 'behind; wood lath and plaster', - 'depth': 0, 'depth_unit': 0, 'cost_unit': 'gbp_per_m2', - 'thermal_conductivity': 0, 'thermal_conductivity_unit': 0, - 'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 6.85, - 'labour_hours_per_unit': 0.22, 'plant_cost': 2.09, 'total_cost': 8.94, - 'link': 'SPONs', 'Notes': 0}, {'type': 'ewi_wall_preparation', - 'description': 'Clean and prepare surfaces, ' - 'one coat Keim dilution, ' - 'one coat primer and two coats ' - 'of Keim Ecosil paint; Brick or ' - 'block walls; over 300 mm girth', - 'depth': 0, 'depth_unit': 0, 'cost_unit': 0, - 'thermal_conductivity': 0, - 'thermal_conductivity_unit': 0, - 'prime_material_cost': 0, 'material_cost': 7.3, - 'labour_cost': 5.62, 'labour_hours_per_unit': 0.3, - 'plant_cost': 0, 'total_cost': 12.92, - 'link': 'SPONs', - 'Notes': 'This work covers the preparation and ' - 'priming of the wall before insulating'}, - {'type': 'ewi_wall_redecoration', - 'description': 'EPS insulation fixed with adhesive to SFS structure (measured ' - 'separately) with horizontal PVC intermediate track and vertical ' - 'T-spines; with glassfibre mesh reinforcement embedded in Sto ' - 'Armat Classic Basecoat Render and Stolit K 1.5 Decorative ' - 'Topcoat Render (white)', - 'depth': 0, 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0, - 'thermal_conductivity_unit': 0, 'prime_material_cost': 0, 'material_cost': 0, - 'labour_cost': 0, 'labour_hours_per_unit': 0, 'plant_cost': 0, - 'total_cost': 69.94, 'link': 'SPONs', - 'Notes': 'This material in SPONs is for 70mm EPS insulation, which comes in at a ' - 'cost of 99.17 per meter square. This includes the cost of insulation. ' - 'To get the costing for just the works and not the insulation, ' - 'we subtract the cost of EPS insulation, using Ravathem 75mm insulation ' - 'as an example, which costs £29.23 per meter square, giving us the cost ' - 'of the remaining works without insulation. This material gives us a ' - 'cost for basecoat, mesh application and a render finish'}] - ewi_results = costs.external_wall_insulation( + ewi_results = costs.solid_wall_insulation( wall_area=95.9104281347967, material=ewi_material, - non_insulation_materials=ewi_non_insulation_materials ) assert ewi_results == { - 'total': 15047.078622131372, 'subtotal': 12539.232185109477, 'vat': 2507.8464370218953, - 'contingency': 808.9827216199662, 'preliminaries': 2022.4568040499155, 'material': 4020.565147410677, - 'profit': 1617.9654432399325, 'labour_hours': 187.02533486285358, 'labour_days': 5.8445417144641745, - 'labour_cost': 3921.5600094613983 + 'total': 28773.12844043901, 'labour_hours': 134.2745993887154, 'labour_days': 4.196081230897356 } def test_flat_roof_insulation(self): @@ -426,120 +265,47 @@ class TestCosts: } costs = Costs(mock_property) - flat_roof_material = {'id': 1225, 'type': 'flat_roof_insulation', - 'description': 'Kingspan Thermaroof TR21 zero OPD ' - 'urethene insulation board', - 'depth': 100.0, 'depth_unit': 'mm', 'cost': None, - 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.04, - 'r_value_unit': 'square_meter_kelvin_per_watt', - 'thermal_conductivity': 0.025, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', - 'link': 'SPONs', - 'created_at': "now", 'is_active': True, - 'prime_material_cost': None, 'material_cost': 50.95, - 'labour_cost': 10.66, 'labour_hours_per_unit': 0.48, - 'plant_cost': 0.0, 'total_cost': 61.61, - 'notes': "SPONs didn't have a labour hours so we use " - "0.48 which is similar to other materials"} + flat_roof_material = { + 'id': 1225, 'type': 'flat_roof_insulation', + 'description': 'Kingspan Thermaroof TR21 zero OPD ' + 'urethene insulation board', + 'depth': 100.0, 'depth_unit': 'mm', 'cost': None, + 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.04, + 'r_value_unit': 'square_meter_kelvin_per_watt', + 'thermal_conductivity': 0.025, + 'thermal_conductivity_unit': 'watt_per_meter_kelvin', + 'link': 'SPONs', + 'created_at': "now", 'is_active': True, + 'prime_material_cost': None, 'material_cost': 50.95, + 'labour_cost': 10.66, 'labour_hours_per_unit': 0.48, + 'plant_cost': 0.0, 'total_cost': 61.61, + 'notes': "SPONs didn't have a labour hours so we use " + "0.48 which is similar to other materials", + "is_installer_quote": False + } - flat_roof_non_insulation_materials = [ - {'id': 17, 'type': 'mechanical_ventilation', 'description': 'Mechanical Extract Ventilation', 'depth': None, - 'depth_unit': None, 'cost': 500, 'cost_unit': 'gbp_per_unit', 'r_value_per_mm': None, 'r_value_unit': None, - 'thermal_conductivity': None, 'thermal_conductivity_unit': None, 'link': None, - 'created_at': datetime.datetime(2023, 10, 18, 16, 39, 9, 827188), 'is_active': True, - 'prime_material_cost': None, - 'material_cost': None, 'labour_cost': None, 'labour_hours_per_unit': None, 'plant_cost': None, - 'total_cost': None, - 'notes': None}, - {'id': 1221, 'type': 'flat_roof_preparation', - 'description': 'clean surface to receive new damp-proof membrane', - 'depth': 0.0, 'depth_unit': None, 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': None, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None, - 'thermal_conductivity_unit': None, - 'link': 'SPONs', 'created_at': datetime.datetime(2023, 12, 4, 20, 1, 49, 298076), 'is_active': True, - 'prime_material_cost': None, 'material_cost': 0.0, 'labour_cost': 4.36, 'labour_hours_per_unit': 0.14, - 'plant_cost': 0.0, 'total_cost': 4.36, - 'notes': 'This data is based on concrete however forms a decent baseline for a Bituminous Felt flat roof'}, - {'id': 1223, 'type': 'flat_roof_preparation', - 'description': 'One coat primer; on wood surfaces before fixing; General surfaces; over 300 mm girth', - 'depth': 0.0, 'depth_unit': None, 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': None, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None, - 'thermal_conductivity_unit': None, - 'link': 'SPONs', 'created_at': datetime.datetime(2023, 12, 4, 20, 1, 49, 298076), 'is_active': True, - 'prime_material_cost': None, 'material_cost': 2.49, 'labour_cost': 1.5, 'labour_hours_per_unit': 0.08, - 'plant_cost': 0.0, 'total_cost': 3.99, 'notes': 'SPONs data gives us a baseline for a wood surface'}, - {'id': 1224, 'type': 'flat_roof_vapour_barrier', 'description': 'Visqueen High Performance Vapour Barrier', - 'depth': 0.0, 'depth_unit': None, 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': None, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None, - 'thermal_conductivity_unit': None, - 'link': 'SPONs', 'created_at': datetime.datetime(2023, 12, 4, 20, 1, 49, 298076), 'is_active': True, - 'prime_material_cost': 0.58, 'material_cost': 1.21, 'labour_cost': 0.48, 'labour_hours_per_unit': 0.02, - 'plant_cost': 0.0, 'total_cost': 1.69, 'notes': None}, - {'id': 1234, 'type': 'flat_roof_waterproofing', - 'description': '20 mm thick two coat coverings; felt isolating membrane; to concrete (or ' - 'timber) base; flat or to falls or slopes not exceeding 10° from horizontal', - 'depth': 0.0, 'depth_unit': None, 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': None, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None, - 'thermal_conductivity_unit': None, 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 12, 4, 20, 1, 49, 298076), 'is_active': True, - 'prime_material_cost': None, 'material_cost': 0.0, 'labour_cost': 0.0, - 'labour_hours_per_unit': 0.5, 'plant_cost': 0.0, 'total_cost': 31.13, 'notes': None} - ] - - flat_roof_floor_results = costs.flat_roof_insulation( + flat_roof_floor_results = costs.loft_and_flat_insulation( floor_area=33.5, material=flat_roof_material, - non_insulation_materials=flat_roof_non_insulation_materials ) - assert flat_roof_floor_results == {'total': 5325.327767999999, 'subtotal': 4437.773139999999, - 'vat': 887.5546279999999, 'contingency': 459.07998, - 'preliminaries': 306.05332, 'material': 1830.775, 'profit': 612.10664, - 'labour_hours': 24.79, 'labour_days': 1.549375, 'labour_cost': 186.9032} + assert flat_roof_floor_results == { + 'total': 2063.935, 'subtotal': 1719.9458333333334, 'vat': 343.9891666666665, 'labour_hours': 8, + 'labour_days': 1 + } assert costs.labour_adjustment_factor == 0.88 - # Mock property instance for regional tests - @pytest.fixture(params=[ - ("Northamptonshire", "East Midlands", 7927.44), - ("Greater London Authority", "Inner London", 10475.0), - ("Adur", "South East England", 8333.32), - ("Bournemouth", "South West England", 8452), - ("Basildon", "East of England", 7895.44), - ("Birmingham", "West Midlands", 7706.2), - ("County Durham", "North East England", 8113.96), - ("Allerdale", "North West England", 6481.68), - ("York", "Yorkshire and the Humber", 8243.6), - ("Cardiff", "Wales", 7595.32), - ("Glasgow City", "Scotland", 7871.88), - ("Belfast", "Northern Ireland", 8504.36) - ]) - def mock_property_with_region(self, request): - county, region, expected_cost = request.param - mock_property = Mock() - mock_property.data = {"county": county} - return mock_property, region, expected_cost - # Test for different wattages - @pytest.mark.parametrize("wattage, expected_cost", [ - (3000, 5945.58), - (4000, 7927.44), - (5000, 9909.3), - (6000, 11891.16), + @pytest.mark.parametrize("n_panels, expected_cost", [ + (7, 4055.0), + (10, 4540.0), + (12, 4863.0), + (15, 5707.0), ]) - def test_solar_pv_different_wattages(self, wattage, expected_cost): + def test_solar_pv_different_wattages(self, n_panels, expected_cost): mock_property = Mock() mock_property.data = {"county": "Mansfield"} costs = Costs(mock_property) - result = costs.solar_pv(wattage) - assert result['total'] == pytest.approx(expected_cost, rel=0.01) - - def test_solar_pv_regional_variation(self, mock_property_with_region): - # Test for regional cost variations - property_instance, expected_region, expected_cost = mock_property_with_region - costs = Costs(property_instance) - - assert costs.region == expected_region - - result = costs.solar_pv(4000) # Testing with a fixed wattage of 4000 + result = costs.solar_pv(n_panels) assert result['total'] == pytest.approx(expected_cost, rel=0.01) From 1d06653a7333f45ad46fea25b874ae9305c78fec Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 30 Sep 2024 15:28:37 +0100 Subject: [PATCH 066/166] updated cost of filling fireplaces --- recommendations/tests/test_fireplace_recommendations.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/recommendations/tests/test_fireplace_recommendations.py b/recommendations/tests/test_fireplace_recommendations.py index f21d6bc3..7eb55b21 100644 --- a/recommendations/tests/test_fireplace_recommendations.py +++ b/recommendations/tests/test_fireplace_recommendations.py @@ -40,7 +40,7 @@ class TestFirepaceRecommendations: assert recommender.recommendation assert recommender.recommendation[0]["type"] == "sealing_open_fireplace" - assert recommender.recommendation[0]["total"] == 300 + assert recommender.recommendation[0]["total"] == 235 def test_multiple_fireplaces(self): epc_record = EPCRecord() @@ -59,4 +59,4 @@ class TestFirepaceRecommendations: assert recommender.recommendation assert recommender.recommendation[0]["type"] == "sealing_open_fireplace" - assert recommender.recommendation[0]["total"] == 900 + assert recommender.recommendation[0]["total"] == 235 * 3 From 80579989fc5a0c234491869312acdef5e581da35 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 30 Sep 2024 15:42:44 +0100 Subject: [PATCH 067/166] fixed floor recommendations tests --- recommendations/tests/test_data/materials.py | 1154 ++++------------- .../tests/test_floor_recommendations.py | 12 +- 2 files changed, 267 insertions(+), 899 deletions(-) diff --git a/recommendations/tests/test_data/materials.py b/recommendations/tests/test_data/materials.py index 187d1401..194971e9 100644 --- a/recommendations/tests/test_data/materials.py +++ b/recommendations/tests/test_data/materials.py @@ -1,965 +1,327 @@ import datetime materials = [ - {'id': 17, 'type': 'mechanical_ventilation', 'description': 'Mechanical Extract Ventilation', 'depth': None, - 'depth_unit': None, 'cost': 500, 'cost_unit': 'gbp_per_unit', 'r_value_per_mm': None, 'r_value_unit': None, - 'thermal_conductivity': None, 'thermal_conductivity_unit': None, 'link': None, - 'created_at': datetime.datetime(2023, 10, 18, 16, 39, 9, 827188), 'is_active': True, 'prime_material_cost': None, - 'material_cost': None, 'labour_cost': None, 'labour_hours_per_unit': None, 'plant_cost': None, 'total_cost': None, - 'notes': None}, - {'id': 1221, 'type': 'flat_roof_preparation', 'description': 'clean surface to receive new damp-proof membrane', - 'depth': 0.0, 'depth_unit': None, 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': None, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None, 'thermal_conductivity_unit': None, - 'link': 'SPONs', 'created_at': datetime.datetime(2023, 12, 4, 20, 1, 49, 298076), 'is_active': True, - 'prime_material_cost': None, 'material_cost': 0.0, 'labour_cost': 4.36, 'labour_hours_per_unit': 0.14, - 'plant_cost': 0.0, 'total_cost': 4.36, - 'notes': 'This data is based on concrete however forms a decent baseline for a Bituminous Felt flat roof'}, - {'id': 1223, 'type': 'flat_roof_preparation', - 'description': 'One coat primer; on wood surfaces before fixing; General surfaces; over 300 mm girth', - 'depth': 0.0, 'depth_unit': None, 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': None, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None, 'thermal_conductivity_unit': None, - 'link': 'SPONs', 'created_at': datetime.datetime(2023, 12, 4, 20, 1, 49, 298076), 'is_active': True, - 'prime_material_cost': None, 'material_cost': 2.49, 'labour_cost': 1.5, 'labour_hours_per_unit': 0.08, - 'plant_cost': 0.0, 'total_cost': 3.99, 'notes': 'SPONs data gives us a baseline for a wood surface'}, - {'id': 1224, 'type': 'flat_roof_vapour_barrier', 'description': 'Visqueen High Performance Vapour Barrier', - 'depth': 0.0, 'depth_unit': None, 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': None, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None, 'thermal_conductivity_unit': None, - 'link': 'SPONs', 'created_at': datetime.datetime(2023, 12, 4, 20, 1, 49, 298076), 'is_active': True, - 'prime_material_cost': 0.58, 'material_cost': 1.21, 'labour_cost': 0.48, 'labour_hours_per_unit': 0.02, - 'plant_cost': 0.0, 'total_cost': 1.69, 'notes': None}, {'id': 1225, 'type': 'flat_roof_insulation', - 'description': 'Kingspan Thermaroof TR21 zero OPD ' - 'urethene insulation board', - 'depth': 100.0, 'depth_unit': 'mm', 'cost': None, - 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.04, - 'r_value_unit': 'square_meter_kelvin_per_watt', - 'thermal_conductivity': 0.025, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', - 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 12, 4, 20, 1, 49, - 298076), 'is_active': True, - 'prime_material_cost': None, 'material_cost': 50.95, - 'labour_cost': 10.66, 'labour_hours_per_unit': 0.48, - 'plant_cost': 0.0, 'total_cost': 61.61, - 'notes': "SPONs didn't have a labour hours so we use " - "0.48 which is similar to other materials"}, - {'id': 1226, 'type': 'flat_roof_insulation', 'description': 'Ravatherm XPS × 500 SL', 'depth': 100.0, - 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.03125, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.032, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 12, 4, 20, 1, 49, 298076), 'is_active': True, 'prime_material_cost': None, - 'material_cost': 22.14, 'labour_cost': 10.66, 'labour_hours_per_unit': 0.48, 'plant_cost': 0.0, 'total_cost': 32.8, - 'notes': None}, - {'id': 1227, 'type': 'flat_roof_insulation', 'description': 'Ravatherm XPS × 500 SL', 'depth': 120.0, - 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.03125, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.032, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', - 'link': 'https://www.panelsystems.co.uk/product/floormate-ravatherm-sb?attribute_pa_group=floormate-500a' - '&attribute_pa_product-name=ravatherm-xps-x-500-sl&attribute_pa_length=1250&attribute_pa_width=600' - '&attribute_pa_thickness=120&attribute_pa_unit-of-sale=pack-3-brds&attribute_pa_min-order-qty=10&gclid' - '=CjwKCAiAjrarBhAWEiwA2qWdCKJK2iqlzUZ-mBFOfCLy2f5TldAbOj7G3LrvYw5JLaigplJAajzYpRoCtB8QAvD_BwE', - 'created_at': datetime.datetime(2023, 12, 4, 20, 1, 49, 298076), 'is_active': True, 'prime_material_cost': None, - 'material_cost': 26.187656, 'labour_cost': 10.66, 'labour_hours_per_unit': 0.48, 'plant_cost': 0.0, - 'total_cost': 36.847656, - 'notes': "SPONs didn't have this thickness, so the material price is based on the fact that on the link, " - "the 120mm thickness is 18% more expensive per board than the 100mm thickness"}, - {'id': 1228, 'type': 'flat_roof_insulation', 'description': 'Ravatherm XPS × 500 SL', 'depth': 140.0, - 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.03125, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.032, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', - 'link': 'https://www.panelsystems.co.uk/product/floormate-ravatherm-sb?attribute_pa_group=floormate-500a' - '&attribute_pa_product-name=ravatherm-xps-x-500-sl&attribute_pa_length=1250&attribute_pa_width=600' - '&attribute_pa_thickness=120&attribute_pa_unit-of-sale=pack-3-brds&attribute_pa_min-order-qty=10&gclid' - '=CjwKCAiAjrarBhAWEiwA2qWdCKJK2iqlzUZ-mBFOfCLy2f5TldAbOj7G3LrvYw5JLaigplJAajzYpRoCtB8QAvD_BwE', - 'created_at': datetime.datetime(2023, 12, 4, 20, 1, 49, 298076), 'is_active': True, 'prime_material_cost': None, - 'material_cost': 31.114737, 'labour_cost': 10.66, 'labour_hours_per_unit': 0.48, 'plant_cost': 0.0, - 'total_cost': 41.77474, - 'notes': "SPONs didn't have this thickness, so the material price is based on the fact that on the link, " - "the 140mm thickness is 40% more expensive per board than the 100mm thickness"}, - {'id': 1229, 'type': 'flat_roof_insulation', 'description': 'Foamglas T3+ Flat Roof Insulation', 'depth': 100.0, - 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.027777778, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.036, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 12, 4, 20, 1, 49, 298076), 'is_active': True, 'prime_material_cost': 95.83, - 'material_cost': 109.09, 'labour_cost': 30.7, 'labour_hours_per_unit': 1.3, 'plant_cost': 0.0, - 'total_cost': 139.79, 'notes': None}, - {'id': 1230, 'type': 'flat_roof_insulation', 'description': 'Foamglas T4+ Flat Roof Insulation', 'depth': 100.0, - 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.024390243, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.041, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 12, 4, 20, 1, 49, 298076), 'is_active': True, 'prime_material_cost': 63.89, - 'material_cost': 76.19, 'labour_cost': 28.34, 'labour_hours_per_unit': 1.2, 'plant_cost': 0.0, - 'total_cost': 104.53, 'notes': None}, - {'id': 1231, 'type': 'flat_roof_insulation', 'description': 'Ecotherm Eco-Versal General Purpose Insulation Board', - 'depth': 100.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 12, 4, 20, 1, 49, 298076), 'is_active': True, 'prime_material_cost': 15.12, - 'material_cost': 25.96, 'labour_cost': 30.7, 'labour_hours_per_unit': 1.3, 'plant_cost': 0.0, 'total_cost': 56.66, - 'notes': None}, - {'id': 1232, 'type': 'flat_roof_insulation', 'description': 'Ecotherm Eco-Versal General Purpose Insulation Board', - 'depth': 120.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 12, 4, 20, 1, 49, 298076), 'is_active': True, 'prime_material_cost': 20.16, - 'material_cost': 34.613335, 'labour_cost': 30.7, 'labour_hours_per_unit': 1.3, 'plant_cost': 0.0, - 'total_cost': 65.31333, - 'notes': "SPONs didn't have this thickness, so the material price is based on the fact that on the link, " - "the 120mm thickness is 33% more expensive than the 100mm thickness"}, - {'id': 1233, 'type': 'flat_roof_insulation', 'description': 'Ecotherm Eco-Versal General Purpose Insulation Board', - 'depth': 150.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 12, 4, 20, 1, 49, 298076), 'is_active': True, 'prime_material_cost': 23.53, - 'material_cost': 34.62, 'labour_cost': 33.06, 'labour_hours_per_unit': 1.4, 'plant_cost': 0.0, 'total_cost': 67.68, - 'notes': None}, {'id': 1234, 'type': 'flat_roof_waterproofing', - 'description': '20 mm thick two coat coverings; felt isolating membrane; to concrete (or ' - 'timber) base; flat or to falls or slopes not exceeding 10° from horizontal', - 'depth': 0.0, 'depth_unit': None, 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': None, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None, - 'thermal_conductivity_unit': None, 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 12, 4, 20, 1, 49, 298076), 'is_active': True, - 'prime_material_cost': None, 'material_cost': 0.0, 'labour_cost': 0.0, - 'labour_hours_per_unit': 0.5, 'plant_cost': 0.0, 'total_cost': 31.13, 'notes': None}, - {'id': 1109, 'type': 'cavity_wall_insulation', 'description': 'Expanded Polystyrene Beads cavity wall insulation', - 'depth': 75.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.030303031, + {'id': 1997, 'type': 'cavity_wall_insulation', 'description': 'Imperial Bead cavity wall insulation', 'depth': 75.0, + 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.030303031, 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.033, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', - 'link': 'https://www.styrene.co.uk/downloads/Datasheets/Stylite_Cavity_Loose_Fill_Insulation_Datasheet_v20211.pdf', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': None, - 'material_cost': 18.875, 'labour_cost': 1.125, 'labour_hours_per_unit': 0.065, 'plant_cost': 0.0, - 'total_cost': 20.0, - 'notes': "It is hard to find materials online. To price this, we've used this article: " - "https://www.greenmatch.co.uk/blog/cavity-wall-insulation-cost It puts EPS beads at around £22 per " - "meter squared, blowing wool insulation at £18 per meter squared and Polyurethane Foam at £26 per meter " - "squared, when taking the most pessimistic prices. These rates have been used to adjust the price of " - "the mineral wool insulation to give us the other forms of insulation"}, - {'id': 1110, 'type': 'cavity_wall_insulation', 'description': 'Injected Polyurthane Foam cavity wall insulation', - 'depth': 75.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.030303031, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.033, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', - 'link': 'https://www.foaminstall.co.uk/wp-content/uploads/2017/04/Lapolla-Cavity-Fill-BBA-certificate-sheet1.pdf', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': None, - 'material_cost': 22.875, 'labour_cost': 1.125, 'labour_hours_per_unit': 0.065, 'plant_cost': 0.0, - 'total_cost': 24.0, 'notes': None}, - {'id': 1111, 'type': 'loft_insulation', 'description': 'Crown Loft Roll 44 glass fibre roll', 'depth': 100.0, - 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.022727273, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.044, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': 2.03, - 'material_cost': 2.1, 'labour_cost': 1.56, 'labour_hours_per_unit': 0.09, 'plant_cost': 0.0, 'total_cost': 3.66, - 'notes': None}, - {'id': 1112, 'type': 'loft_insulation', 'description': 'Crown Loft Roll 44 glass fibre roll', 'depth': 150.0, - 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.022727273, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.044, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': 3.06, - 'material_cost': 3.16, 'labour_cost': 1.78, 'labour_hours_per_unit': 0.1, 'plant_cost': 0.0, 'total_cost': 4.94, - 'notes': None}, - {'id': 1113, 'type': 'loft_insulation', 'description': 'Crown Loft Roll 44 glass fibre roll', 'depth': 170.0, - 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.022727273, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.044, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', - 'link': 'https://insulation4less.co.uk/products/knauf-170mm-combi-cut?variant=31671561257013&dfw_tracker=77750' - '-31671561257013&utm_source=google&utm_medium=shopping&utm_campaign=shoptimised&gad_source=1&gclid' - '=CjwKCAiAx_GqBhBQEiwAlDNAZi1LiTWKVn0W1vktOYAPPQU3hss5Tq2qNn6GNhodCQoRD_tvqCLdxhoCKnIQAvD_BwE', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': None, - 'material_cost': 3.81938, 'labour_cost': 1.71304, 'labour_hours_per_unit': 0.11, 'plant_cost': 0.0, - 'total_cost': 5.53242, - 'notes': "We don't have a 170mm in SPONs so the material cost is based on the fact that the 170mm insulation is " - "87.4% of the cost of the 200mm insulation"}, - {'id': 1114, 'type': 'loft_insulation', 'description': 'Crown Loft Roll 44 glass fibre roll', 'depth': 200.0, - 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.022727273, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.044, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': 4.25, - 'material_cost': 4.37, 'labour_cost': 1.96, 'labour_hours_per_unit': 0.11, 'plant_cost': 0.0, 'total_cost': 6.33, - 'notes': None}, - {'id': 1115, 'type': 'loft_insulation', 'description': 'Crown Loft Roll 44 glass fibre roll', 'depth': 270.0, - 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.022727273, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.044, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': None, - 'material_cost': 5.91938, 'labour_cost': 1.96, 'labour_hours_per_unit': 0.11, 'plant_cost': 0.0, - 'total_cost': 7.87938, 'notes': 'This is the 100mm product + the 170mm product'}, - {'id': 1116, 'type': 'loft_insulation', 'description': 'Crown Loft Roll 44 glass fibre roll', 'depth': 300.0, - 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.022727273, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.044, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': None, - 'material_cost': 6.47, 'labour_cost': 1.96, 'labour_hours_per_unit': 0.11, 'plant_cost': 0.0, 'total_cost': 8.43, - 'notes': 'This is the 100mm product + the 200mm product'}, - {'id': 1117, 'type': 'loft_insulation', 'description': 'Isover Mineral Wool Modular Roll', 'depth': 100.0, - 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.023255814, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.043, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': 1.99, - 'material_cost': 2.05, 'labour_cost': 1.6, 'labour_hours_per_unit': 0.09, 'plant_cost': 0.0, 'total_cost': 3.65, - 'notes': None}, - {'id': 1118, 'type': 'loft_insulation', 'description': 'Isover Mineral Wool Modular Roll', 'depth': 150.0, - 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.023255814, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.043, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': 2.96, - 'material_cost': 3.05, 'labour_cost': 1.78, 'labour_hours_per_unit': 0.1, 'plant_cost': 0.0, 'total_cost': 4.83, - 'notes': None}, - {'id': 1119, 'type': 'loft_insulation', 'description': 'Isover Mineral Wool Modular Roll', 'depth': 170.0, - 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.023255814, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.043, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', - 'link': 'https://flooringwarehousedirect.co.uk/product/isover-spacesaver-roll-170mm-x-1160mm-x-7-03m-8-15m2/', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': None, - 'material_cost': 3.8706238, 'labour_cost': 2.281361, 'labour_hours_per_unit': 0.12816635, 'plant_cost': 0.0, - 'total_cost': 6.1519847, - 'notes': "We don't have a 170mm in SPONs so the material cost is based on the fact that the 170mm insulation is " - "85.4% of the cost of the 200mm insulation"}, - {'id': 1120, 'type': 'loft_insulation', 'description': 'Isover Mineral Wool Modular Roll', 'depth': 200.0, - 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.023255814, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.043, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': 4.4, - 'material_cost': 4.53, 'labour_cost': 2.67, 'labour_hours_per_unit': 0.15, 'plant_cost': 0.0, 'total_cost': 7.2, - 'notes': None}, - {'id': 1121, 'type': 'loft_insulation', 'description': 'Isover Mineral Wool Modular Roll', 'depth': 270.0, - 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.023255814, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.043, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': None, - 'material_cost': 5.920624, 'labour_cost': 2.67, 'labour_hours_per_unit': 0.15, 'plant_cost': 0.0, - 'total_cost': 8.590624, 'notes': 'This is the 100mm product + the 170mm product'}, - {'id': 1122, 'type': 'loft_insulation', 'description': 'Isover Mineral Wool Modular Roll', 'depth': 300.0, - 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.023255814, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.043, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': None, - 'material_cost': 6.58, 'labour_cost': 2.67, 'labour_hours_per_unit': 0.15, 'plant_cost': 0.0, 'total_cost': 9.25, - 'notes': 'This is the 100mm product + the 200mm product'}, - {'id': 1123, 'type': 'loft_insulation', 'description': 'Isover Acoustic Partition Roll', 'depth': 100.0, - 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.023255814, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.043, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': 5.93, - 'material_cost': 6.4, 'labour_cost': 2.67, 'labour_hours_per_unit': 0.15, 'plant_cost': 0.0, 'total_cost': 9.07, - 'notes': 'This provides acoustic insulation as well'}, - {'id': 1124, 'type': 'loft_insulation', 'description': 'Isover Acoustic Partition Roll', 'depth': 300.0, - 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.023255814, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.043, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': 17.79, - 'material_cost': 19.2, 'labour_cost': 2.67, 'labour_hours_per_unit': 0.15, 'plant_cost': 0.0, 'total_cost': 21.87, - 'notes': 'This provides acoustic insulation as well'}, - {'id': 1125, 'type': 'loft_insulation', 'description': 'Thermafleece EcoRoll Insulation', 'depth': 300.0, - 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.025641026, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.039, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': None, - 'material_cost': 24.78, 'labour_cost': 2.67, 'labour_hours_per_unit': 0.15, 'plant_cost': 0.0, 'total_cost': 27.45, - 'notes': 'This material is based on installing 3 layers of the 100mm product'}, - {'id': 1126, 'type': 'loft_insulation', 'description': 'Thermafleece EcoRoll Insulation', 'depth': 280.0, - 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.025641026, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.039, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': None, - 'material_cost': 23.36, 'labour_cost': 3.12, 'labour_hours_per_unit': 0.18, 'plant_cost': 0.0, 'total_cost': 26.48, - 'notes': 'This material is based on installed 2 layers of the 140mm product'}, - {'id': 1127, 'type': 'iwi_wall_demolition', - 'description': 'Solid & Dry Lined walls: Hack of wall finishes with chipping hammer; plaster to walls.', - 'depth': 0.0, 'depth_unit': None, 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': None, + 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SCIS', + 'created_at': datetime.datetime(2024, 9, 24, 13, 42, 52, 584553), 'is_active': True, 'prime_material_cost': None, + 'material_cost': 0.0, 'labour_cost': 0.0, 'labour_hours_per_unit': 0.0, 'plant_cost': 0.0, 'total_cost': 14.21, + 'notes': None, 'is_installer_quote': True}, + {'id': 1998, 'type': 'mechanical_ventilation', 'description': 'Mechanical Extract Ventilation', 'depth': 0.0, + 'depth_unit': None, 'cost': None, 'cost_unit': 'gbp_per_unit', 'r_value_per_mm': None, 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None, 'thermal_conductivity_unit': None, - 'link': 'SPONs', 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, - 'prime_material_cost': None, 'material_cost': 0.0, 'labour_cost': 10.27, 'labour_hours_per_unit': 0.33, - 'plant_cost': 1.28, 'total_cost': 11.55, 'notes': None}, {'id': 1128, 'type': 'iwi_wall_demolition', - 'description': 'Stud walls: Remove wall linings ' - 'including battening behind; ' - 'plasterboard and skim', - 'depth': 0.0, 'depth_unit': None, 'cost': None, - 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': None, - 'r_value_unit': 'square_meter_kelvin_per_watt', - 'thermal_conductivity': None, - 'thermal_conductivity_unit': None, 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, - 244907), - 'is_active': True, 'prime_material_cost': None, - 'material_cost': 0.0, 'labour_cost': 6.23, - 'labour_hours_per_unit': 0.2, 'plant_cost': 1.25, - 'total_cost': 7.48, 'notes': None}, - {'id': 1129, 'type': 'iwi_wall_demolition', - 'description': 'Lathe and Plaster walls: Remove wall linings including battening behind; wood lath and plaster', - 'depth': 0.0, 'depth_unit': None, 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': None, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None, 'thermal_conductivity_unit': None, - 'link': 'SPONs', 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, - 'prime_material_cost': None, 'material_cost': 0.0, 'labour_cost': 6.85, 'labour_hours_per_unit': 0.22, - 'plant_cost': 2.09, 'total_cost': 8.94, 'notes': None}, - {'id': 1130, 'type': 'internal_wall_insulation', 'description': 'Foamglas Grade F Wall Insulation Slabs', - 'depth': 60.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.02631579, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.038, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': 41.69, - 'material_cost': 53.33, 'labour_cost': 29.52, 'labour_hours_per_unit': 1.25, 'plant_cost': 0.0, - 'total_cost': 82.85, 'notes': None}, - {'id': 1131, 'type': 'internal_wall_insulation', 'description': 'Foamglas Grade F Wall Insulation Slabs', - 'depth': 100.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.02631579, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.038, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': 86.86, - 'material_cost': 99.85, 'labour_cost': 29.52, 'labour_hours_per_unit': 1.25, 'plant_cost': 0.0, - 'total_cost': 129.37, 'notes': None}, - {'id': 1132, 'type': 'internal_wall_insulation', 'description': 'Foamglas Grade F Wall Insulation Slabs', - 'depth': 150.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.02631579, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.038, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, - 'prime_material_cost': 130.29, 'material_cost': 144.58, 'labour_cost': 29.52, 'labour_hours_per_unit': 1.25, - 'plant_cost': 0.0, 'total_cost': 174.1, 'notes': None}, - {'id': 1133, 'type': 'internal_wall_insulation', 'description': 'Ecotherm Eco-Versal PIR Insulation Board', - 'depth': 30.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547, + 'link': 'SCIS', 'created_at': datetime.datetime(2024, 9, 24, 13, 42, 52, 584553), 'is_active': True, + 'prime_material_cost': None, 'material_cost': 0.0, 'labour_cost': 0.0, 'labour_hours_per_unit': 0.0, + 'plant_cost': 0.0, 'total_cost': 535.5, 'notes': None, 'is_installer_quote': True}, + {'id': 2015, 'type': 'loft_insulation', 'description': 'Knauf Loft Roll 44 glass fibre roll', 'depth': 100.0, + 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.022727273, + 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.044, + 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SCIS', + 'created_at': datetime.datetime(2024, 9, 24, 13, 42, 52, 584553), 'is_active': True, 'prime_material_cost': None, + 'material_cost': 0.0, 'labour_cost': 0.0, 'labour_hours_per_unit': 0.09, 'plant_cost': 0.0, 'total_cost': 14.95, + 'notes': 'This is a placeholder cost until SCIS gives us a breakdown by thickness', 'is_installer_quote': True}, + {'id': 2016, 'type': 'loft_insulation', 'description': 'Knauf Loft Roll 44 glass fibre roll', 'depth': 200.0, + 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.022727273, + 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.044, + 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SCIS', + 'created_at': datetime.datetime(2024, 9, 24, 13, 42, 52, 584553), 'is_active': True, 'prime_material_cost': None, + 'material_cost': 0.0, 'labour_cost': 0.0, 'labour_hours_per_unit': 0.11, 'plant_cost': 0.0, 'total_cost': 15.525, + 'notes': 'This is a placeholder cost until SCIS gives us a breakdown by thickness', 'is_installer_quote': True}, + {'id': 2017, 'type': 'loft_insulation', 'description': 'Knauf Loft Roll 44 glass fibre roll', 'depth': 270.0, + 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.022727273, + 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.044, + 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SCIS', + 'created_at': datetime.datetime(2024, 9, 24, 13, 42, 52, 584553), 'is_active': True, 'prime_material_cost': None, + 'material_cost': 0.0, 'labour_cost': 0.0, 'labour_hours_per_unit': 0.11, 'plant_cost': 0.0, 'total_cost': 16.1, + 'notes': 'This is a placeholder cost until SCIS gives us a breakdown by thickness', 'is_installer_quote': True}, + {'id': 2018, 'type': 'loft_insulation', 'description': 'Knauf Loft Roll 44 glass fibre roll', 'depth': 300.0, + 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.022727273, + 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.044, + 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SCIS', + 'created_at': datetime.datetime(2024, 9, 24, 13, 42, 52, 584553), 'is_active': True, 'prime_material_cost': None, + 'material_cost': 0.0, 'labour_cost': 0.0, 'labour_hours_per_unit': 0.11, 'plant_cost': 0.0, 'total_cost': 16.53, + 'notes': 'This is a placeholder cost until SCIS gives us a breakdown by thickness', 'is_installer_quote': True}, + {'id': 2039, 'type': 'internal_wall_insulation', 'description': 'SWIP EcoBatt', 'depth': 95.0, 'depth_unit': 'mm', + 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.03125, 'r_value_unit': 'square_meter_kelvin_per_watt', + 'thermal_conductivity': 0.032, 'thermal_conductivity_unit': None, 'link': 'SCIS', + 'created_at': datetime.datetime(2024, 9, 24, 13, 42, 52, 584553), 'is_active': True, 'prime_material_cost': None, + 'material_cost': 0.0, 'labour_cost': 0.0, 'labour_hours_per_unit': 2.1, 'plant_cost': 0.0, 'total_cost': 244.8, + 'notes': 'We are awaiting further breakdown of costs by thickness and finishes', 'is_installer_quote': False}, + {'id': 2074, 'type': 'suspended_floor_insulation', 'description': 'Q-bot underfloor insulation', 'depth': 50.0, + 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547, 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': 6.16, - 'material_cost': 16.73, 'labour_cost': 28.34, 'labour_hours_per_unit': 1.2, 'plant_cost': 0.0, 'total_cost': 45.07, - 'notes': None}, - {'id': 1134, 'type': 'internal_wall_insulation', 'description': 'Ecotherm Eco-Versal PIR Insulation Board', - 'depth': 50.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547, + 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SCIS', + 'created_at': datetime.datetime(2024, 9, 24, 13, 42, 52, 584553), 'is_active': True, 'prime_material_cost': None, + 'material_cost': 0.0, 'labour_cost': 0.0, 'labour_hours_per_unit': 1.63, 'plant_cost': 0.0, 'total_cost': 75.0, + 'notes': 'Linearly interpolated based on Qbot costs', 'is_installer_quote': True}, + {'id': 2075, 'type': 'suspended_floor_insulation', 'description': 'Q-bot underfloor insulation', 'depth': 75.0, + 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547, 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': 8.46, - 'material_cost': 19.1, 'labour_cost': 28.34, 'labour_hours_per_unit': 1.2, 'plant_cost': 0.0, 'total_cost': 47.44, - 'notes': None}, - {'id': 1135, 'type': 'internal_wall_insulation', 'description': 'Ecotherm Eco-Versal PIR Insulation Board', - 'depth': 100.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547, + 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SCIS', + 'created_at': datetime.datetime(2024, 9, 24, 13, 42, 52, 584553), 'is_active': True, 'prime_material_cost': None, + 'material_cost': 0.0, 'labour_cost': 0.0, 'labour_hours_per_unit': 1.63, 'plant_cost': 0.0, 'total_cost': 93.75, + 'notes': 'Linearly interpolated based on Qbot costs', 'is_installer_quote': True}, + {'id': 2076, 'type': 'suspended_floor_insulation', 'description': 'Q-bot underfloor insulation', 'depth': 100.0, + 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547, 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': 15.12, - 'material_cost': 25.96, 'labour_cost': 30.7, 'labour_hours_per_unit': 1.3, 'plant_cost': 0.0, 'total_cost': 56.66, - 'notes': None}, - {'id': 1136, 'type': 'internal_wall_insulation', 'description': 'Kingspan Kooltherm K18 insulated plasterboard', - 'depth': 37.5, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.04761905, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.021, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': None, - 'material_cost': 26.86, 'labour_cost': 5.21, 'labour_hours_per_unit': 0.23, 'plant_cost': 0.0, 'total_cost': 32.07, - 'notes': None}, - {'id': 1137, 'type': 'internal_wall_insulation', 'description': 'Kingspan Kooltherm K18 insulated plasterboard', - 'depth': 42.5, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.04761905, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.021, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': None, - 'material_cost': 17.37, 'labour_cost': 5.21, 'labour_hours_per_unit': 0.23, 'plant_cost': 0.0, 'total_cost': 22.58, - 'notes': None}, - {'id': 1138, 'type': 'internal_wall_insulation', 'description': 'Kingspan Kooltherm K18 insulated plasterboard', - 'depth': 52.5, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.04761905, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.021, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': None, - 'material_cost': 21.74, 'labour_cost': 5.79, 'labour_hours_per_unit': 0.25, 'plant_cost': 0.0, 'total_cost': 27.53, - 'notes': None}, - {'id': 1139, 'type': 'internal_wall_insulation', 'description': 'Kingspan Kooltherm K18 insulated plasterboard', - 'depth': 62.5, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.04761905, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.021, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': None, - 'material_cost': 19.3, 'labour_cost': 5.79, 'labour_hours_per_unit': 0.25, 'plant_cost': 0.0, 'total_cost': 25.09, - 'notes': None}, - {'id': 1140, 'type': 'internal_wall_insulation', 'description': 'Kingspan Kooltherm K18 insulated plasterboard', - 'depth': 72.5, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.04761905, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.021, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': None, - 'material_cost': 23.15, 'labour_cost': 5.79, 'labour_hours_per_unit': 0.25, 'plant_cost': 0.0, 'total_cost': 28.94, - 'notes': None}, - {'id': 1141, 'type': 'iwi_vapour_barrier', 'description': 'Visqueen High Performance Vapour Barrier', 'depth': 0.0, + 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SCIS', + 'created_at': datetime.datetime(2024, 9, 24, 13, 42, 52, 584553), 'is_active': True, 'prime_material_cost': None, + 'material_cost': 0.0, 'labour_cost': 0.0, 'labour_hours_per_unit': 1.63, 'plant_cost': 0.0, 'total_cost': 112.5, + 'notes': 'Linearly interpolated based on Qbot costs', 'is_installer_quote': True}, + {'id': 2077, 'type': 'suspended_floor_insulation', 'description': 'Q-bot underfloor insulation', 'depth': 125.0, + 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547, + 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022, + 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SCIS', + 'created_at': datetime.datetime(2024, 9, 24, 13, 42, 52, 584553), 'is_active': True, 'prime_material_cost': None, + 'material_cost': 0.0, 'labour_cost': 0.0, 'labour_hours_per_unit': 1.63, 'plant_cost': 0.0, 'total_cost': 112.5, + 'notes': 'Linearly interpolated based on Qbot costs', 'is_installer_quote': True}, + {'id': 2078, 'type': 'suspended_floor_insulation', 'description': 'Q-bot underfloor insulation', 'depth': 150.0, + 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547, + 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022, + 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SCIS', + 'created_at': datetime.datetime(2024, 9, 24, 13, 42, 52, 584553), 'is_active': True, 'prime_material_cost': None, + 'material_cost': 0.0, 'labour_cost': 0.0, 'labour_hours_per_unit': 1.63, 'plant_cost': 0.0, 'total_cost': 150.0, + 'notes': 'Linearly interpolated based on Qbot costs', 'is_installer_quote': True}, + {'id': 2079, 'type': 'solid_floor_demolition', 'description': 'Removal of carpet and underfelt', 'depth': 0.0, 'depth_unit': None, 'cost': None, 'cost_unit': None, 'r_value_per_mm': None, 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None, 'thermal_conductivity_unit': None, - 'link': 'SPONs', 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, - 'prime_material_cost': 0.58, 'material_cost': 1.21, 'labour_cost': 0.48, 'labour_hours_per_unit': 0.02, - 'plant_cost': 0.0, 'total_cost': 1.69, 'notes': None}, {'id': 1142, 'type': 'iwi_redecoration', - 'description': 'Plaster; one coat Thistle board finish ' - 'or other equal; steel trowelled; 3 mm ' - 'thick work to walls or ceilings; one ' - 'coat; to plasterboard base; over 600mm ' - 'wide', - 'depth': 0.0, 'depth_unit': None, 'cost': None, - 'cost_unit': None, 'r_value_per_mm': None, - 'r_value_unit': 'square_meter_kelvin_per_watt', - 'thermal_conductivity': None, - 'thermal_conductivity_unit': None, 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, - 244907), 'is_active': True, - 'prime_material_cost': None, 'material_cost': 0.06, - 'labour_cost': 6.58, 'labour_hours_per_unit': 0.25, - 'plant_cost': 0.0, 'total_cost': 6.64, 'notes': None}, - {'id': 1143, 'type': 'iwi_redecoration', - 'description': 'Two coats emulsion paint on plaster, over 40mm girth; 3.5m - 5m high', 'depth': 0.0, - 'depth_unit': None, 'cost': None, 'cost_unit': None, 'r_value_per_mm': None, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None, 'thermal_conductivity_unit': None, - 'link': 'SPONs', 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, - 'prime_material_cost': None, 'material_cost': 0.41, 'labour_cost': 3.93, 'labour_hours_per_unit': 0.21, - 'plant_cost': 0.0, 'total_cost': 4.34, 'notes': None}, {'id': 1144, 'type': 'iwi_redecoration', - 'description': 'Fitting existing softwood skirting or ' - 'architrave to new frames; 150mm high', - 'depth': 0.0, 'depth_unit': None, 'cost': None, - 'cost_unit': None, 'r_value_per_mm': None, - 'r_value_unit': 'square_meter_kelvin_per_watt', - 'thermal_conductivity': None, - 'thermal_conductivity_unit': None, 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, - 244907), 'is_active': True, - 'prime_material_cost': None, 'material_cost': 0.01, - 'labour_cost': 4.87, 'labour_hours_per_unit': 0.12, - 'plant_cost': 0.0, 'total_cost': 4.88, 'notes': None}, - {'id': 1145, 'type': 'suspended_floor_demolition', 'description': 'Removal of carpet and underfelt', 'depth': 0.0, - 'depth_unit': None, 'cost': None, 'cost_unit': None, 'r_value_per_mm': None, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None, 'thermal_conductivity_unit': None, - 'link': 'SPONs', 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, + 'link': 'SPONs', 'created_at': datetime.datetime(2024, 9, 24, 13, 42, 52, 584553), 'is_active': True, 'prime_material_cost': None, 'material_cost': 0.0, 'labour_cost': 3.32, 'labour_hours_per_unit': 0.11, 'plant_cost': 0.0, 'total_cost': 3.32, 'notes': 'We ignore the plant cost that is in SPONs because we assume the carpet is not scrapped and therefore ' - 'there is no need for a skip'}, - {'id': 1146, 'type': 'suspended_floor_demolition', - 'description': 'Remove boarding; withdraw nails; set aside for reuse; ground level', 'depth': 0.0, - 'depth_unit': None, 'cost': None, 'cost_unit': None, 'r_value_per_mm': None, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None, 'thermal_conductivity_unit': None, - 'link': 'SPONs', 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, - 'prime_material_cost': None, 'material_cost': 0.0, 'labour_cost': 9.34, 'labour_hours_per_unit': 0.3, - 'plant_cost': 0.0, 'total_cost': 9.34, 'notes': None}, - {'id': 1147, 'type': 'suspended_floor_vapour_barrier', 'description': 'Visqueen High Performance Vapour Barrier', + 'there is no need for a skip', + 'is_installer_quote': False}, {'id': 2080, 'type': 'solid_floor_preparation', + 'description': 'clean surface of concrete to receive new damp-proof membrane', + 'depth': 0.0, 'depth_unit': None, 'cost': None, 'cost_unit': None, + 'r_value_per_mm': None, 'r_value_unit': 'square_meter_kelvin_per_watt', + 'thermal_conductivity': None, 'thermal_conductivity_unit': None, 'link': None, + 'created_at': datetime.datetime(2024, 9, 24, 13, 42, 52, 584553), 'is_active': True, + 'prime_material_cost': None, 'material_cost': 0.0, 'labour_cost': 4.36, + 'labour_hours_per_unit': 0.14, 'plant_cost': 0.0, 'total_cost': 4.36, 'notes': None, + 'is_installer_quote': False}, {'id': 2081, 'type': 'solid_floor_preparation', + 'description': 'Clean out crack to form a ' + '20mm×20mm groove and fill with ' + 'cement: mortar mixed with bonding ' + 'agent', + 'depth': 0.0, 'depth_unit': None, 'cost': None, + 'cost_unit': None, 'r_value_per_mm': None, + 'r_value_unit': 'square_meter_kelvin_per_watt', + 'thermal_conductivity': None, + 'thermal_conductivity_unit': None, 'link': None, + 'created_at': datetime.datetime(2024, 9, 24, 13, 42, + 52, 584553), + 'is_active': True, 'prime_material_cost': None, + 'material_cost': 6.91, 'labour_cost': 18.99, + 'labour_hours_per_unit': 0.61, 'plant_cost': 0.16, + 'total_cost': 26.06, + 'notes': 'This step is the assessment and repair ' + 'of any damage to the concrete floor such ' + 'as filling cracks or levelling uneven ' + 'areas', + 'is_installer_quote': False}, + {'id': 2082, 'type': 'solid_floor_vapour_barrier', 'description': 'Visqueen High Performance Vapour Barrier', 'depth': 0.0, 'depth_unit': None, 'cost': None, 'cost_unit': None, 'r_value_per_mm': None, 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None, 'thermal_conductivity_unit': None, - 'link': 'SPONs', 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, + 'link': 'SPONs', 'created_at': datetime.datetime(2024, 9, 24, 13, 42, 52, 584553), 'is_active': True, 'prime_material_cost': 0.58, 'material_cost': 1.21, 'labour_cost': 0.48, 'labour_hours_per_unit': 0.02, - 'plant_cost': 0.0, 'total_cost': 1.69, 'notes': None}, - {'id': 1148, 'type': 'suspended_floor_insulation', 'description': 'Thermafleece CosyWool Roll', 'depth': 50.0, - 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.025641026, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.039, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': None, - 'material_cost': 4.24, 'labour_cost': 1.56, 'labour_hours_per_unit': 0.09, 'plant_cost': 0.0, 'total_cost': 5.8, - 'notes': 'Spons did not contain labour costs so we use values for similar insulations. We use the same values as ' - 'in Crown loft roll 44, since it is also an insulation roll'}, - {'id': 1149, 'type': 'suspended_floor_insulation', 'description': 'Thermafleece CosyWool Roll', 'depth': 75.0, - 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.025641026, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.039, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': None, - 'material_cost': 6.31, 'labour_cost': 1.56, 'labour_hours_per_unit': 0.09, 'plant_cost': 0.0, 'total_cost': 7.87, - 'notes': 'Spons did not contain labour costs so we use values for similar insulations. We use the same values as ' - 'in Crown loft roll 44, since it is also an insulation roll'}, - {'id': 1150, 'type': 'suspended_floor_insulation', 'description': 'Thermafleece CosyWool Roll', 'depth': 100.0, - 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.025641026, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.039, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': None, - 'material_cost': 8.26, 'labour_cost': 1.56, 'labour_hours_per_unit': 0.1, 'plant_cost': 0.0, 'total_cost': 9.82, - 'notes': 'Spons did not contain labour costs so we use values for similar insulations. We use the same values as ' - 'in Crown loft roll 44, since it is also an insulation roll'}, - {'id': 1151, 'type': 'suspended_floor_insulation', 'description': 'Thermafleece CosyWool Roll', 'depth': 140.0, - 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.025641026, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.039, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': None, - 'material_cost': 11.68, 'labour_cost': 1.78, 'labour_hours_per_unit': 0.1, 'plant_cost': 0.0, 'total_cost': 13.46, - 'notes': 'Spons did not contain labour costs so we use values for similar insulations. We use the same values as ' - 'in Crown loft roll 44, since it is also an insulation roll'}, - {'id': 1152, 'type': 'suspended_floor_insulation', - 'description': 'Thermafleece TF35 high density wool insulating batts', 'depth': 50.0, 'depth_unit': 'mm', - 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.028571429, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.035, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': None, - 'material_cost': 6.63, 'labour_cost': 1.56, 'labour_hours_per_unit': 0.09, 'plant_cost': 0.0, 'total_cost': 8.19, - 'notes': 'Spons did not contain labour costs so we use values for similar insulations. We use the same values as ' - 'in Crown loft roll 44, since it is also an insulation roll'}, - {'id': 1153, 'type': 'suspended_floor_insulation', - 'description': 'Thermafleece TF35 high density wool insulating batts', 'depth': 75.0, 'depth_unit': 'mm', - 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.028571429, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.035, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': None, - 'material_cost': 10.31, 'labour_cost': 1.56, 'labour_hours_per_unit': 0.09, 'plant_cost': 0.0, 'total_cost': 11.87, - 'notes': 'Spons did not contain labour costs so we use values for similar insulations. We use the same values as ' - 'in Crown loft roll 44, since it is also an insulation roll'}, - {'id': 1154, 'type': 'suspended_floor_insulation', - 'description': 'Ecotherm Eco-Versal General Purpose Insulation Board', 'depth': 30.0, 'depth_unit': 'mm', - 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': 6.16, - 'material_cost': 16.73, 'labour_cost': 28.34, 'labour_hours_per_unit': 1.2, 'plant_cost': 0.0, 'total_cost': 45.07, - 'notes': None}, {'id': 1155, 'type': 'suspended_floor_insulation', - 'description': 'Ecotherm Eco-Versal General Purpose Insulation Board', 'depth': 50.0, - 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, - 'prime_material_cost': 8.46, 'material_cost': 19.1, 'labour_cost': 28.34, - 'labour_hours_per_unit': 1.2, 'plant_cost': 0.0, 'total_cost': 47.44, 'notes': None}, - {'id': 1156, 'type': 'suspended_floor_insulation', - 'description': 'Ecotherm Eco-Versal General Purpose Insulation Board', 'depth': 100.0, 'depth_unit': 'mm', - 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': 15.12, - 'material_cost': 25.96, 'labour_cost': 30.7, 'labour_hours_per_unit': 1.3, 'plant_cost': 0.0, 'total_cost': 56.66, - 'notes': None}, {'id': 1157, 'type': 'suspended_floor_insulation', - 'description': 'Ecotherm Eco-Versal General Purpose Insulation Board', 'depth': 150.0, - 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, - 'prime_material_cost': 23.53, 'material_cost': 34.62, 'labour_cost': 33.06, - 'labour_hours_per_unit': 1.4, 'plant_cost': 0.0, 'total_cost': 67.68, 'notes': None}, - {'id': 1158, 'type': 'suspended_floor_insulation', 'description': 'Crown Loft Roll 44 glass fibre roll', - 'depth': 100.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_unit', 'r_value_per_mm': 0.022727273, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.044, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': 2.03, - 'material_cost': 2.1, 'labour_cost': 1.56, 'labour_hours_per_unit': 0.09, 'plant_cost': 0.0, 'total_cost': 3.66, - 'notes': None}, - {'id': 1159, 'type': 'suspended_floor_insulation', 'description': 'Crown Loft Roll 44 glass fibre roll', - 'depth': 150.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_unit', 'r_value_per_mm': 0.022727273, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.044, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': 3.06, - 'material_cost': 3.16, 'labour_cost': 1.78, 'labour_hours_per_unit': 0.1, 'plant_cost': 0.0, 'total_cost': 4.94, - 'notes': None}, - {'id': 1160, 'type': 'suspended_floor_insulation', 'description': 'Crown Loft Roll 44 glass fibre roll', - 'depth': 200.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_unit', 'r_value_per_mm': 0.022727273, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.044, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': 4.25, - 'material_cost': 4.37, 'labour_cost': 1.96, 'labour_hours_per_unit': 0.11, 'plant_cost': 0.0, 'total_cost': 6.33, - 'notes': None}, - {'id': 1161, 'type': 'suspended_floor_insulation', 'description': 'Isover Mineral Wool Modular Roll', - 'depth': 100.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_unit', 'r_value_per_mm': 0.023255814, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.043, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': 1.99, - 'material_cost': 2.05, 'labour_cost': 1.6, 'labour_hours_per_unit': 0.09, 'plant_cost': 0.0, 'total_cost': 3.65, - 'notes': None}, - {'id': 1162, 'type': 'suspended_floor_insulation', 'description': 'Isover Mineral Wool Modular Roll', - 'depth': 150.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_unit', 'r_value_per_mm': 0.023255814, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.043, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': 2.96, - 'material_cost': 3.05, 'labour_cost': 1.78, 'labour_hours_per_unit': 0.1, 'plant_cost': 0.0, 'total_cost': 4.83, - 'notes': None}, - {'id': 1163, 'type': 'suspended_floor_insulation', 'description': 'Isover Mineral Wool Modular Roll', - 'depth': 200.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_unit', 'r_value_per_mm': 0.023255814, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.043, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': 4.4, - 'material_cost': 4.53, 'labour_cost': 2.67, 'labour_hours_per_unit': 0.15, 'plant_cost': 0.0, 'total_cost': 7.2, - 'notes': None}, - {'id': 1164, 'type': 'suspended_floor_insulation', 'description': 'Isover Acoustic Partition Roll', 'depth': 25.0, - 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_unit', 'r_value_per_mm': 0.025641026, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.039, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': 1.67, - 'material_cost': 2.01, 'labour_cost': 1.43, 'labour_hours_per_unit': 0.08, 'plant_cost': 0.0, 'total_cost': 3.44, - 'notes': None}, - {'id': 1165, 'type': 'suspended_floor_insulation', 'description': 'Isover Acoustic Partition Roll', 'depth': 50.0, - 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_unit', 'r_value_per_mm': 0.025641026, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.039, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': 2.74, - 'material_cost': 3.11, 'labour_cost': 1.6, 'labour_hours_per_unit': 0.09, 'plant_cost': 0.0, 'total_cost': 4.71, - 'notes': None}, - {'id': 1166, 'type': 'suspended_floor_insulation', 'description': 'Isover Acoustic Partition Roll', 'depth': 75.0, - 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_unit', 'r_value_per_mm': 0.023255814, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.043, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': 4.57, - 'material_cost': 5.01, 'labour_cost': 1.78, 'labour_hours_per_unit': 0.1, 'plant_cost': 0.0, 'total_cost': 6.79, - 'notes': None}, - {'id': 1167, 'type': 'suspended_floor_insulation', 'description': 'Isover Acoustic Partition Roll', 'depth': 100.0, - 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_unit', 'r_value_per_mm': 0.023255814, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.043, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': 5.93, - 'material_cost': 6.4, 'labour_cost': 2.67, 'labour_hours_per_unit': 0.15, 'plant_cost': 0.0, 'total_cost': 9.07, - 'notes': None}, - {'id': 1168, 'type': 'suspended_floor_insulation', 'description': 'Kay-Cel Expanded Polystyrene Board', - 'depth': 25.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.030303031, + 'plant_cost': 0.0, 'total_cost': 1.69, 'notes': None, 'is_installer_quote': False}, + {'id': 2083, 'type': 'solid_floor_insulation', 'description': 'Kay-Cel Expanded Polystyrene Board', 'depth': 25.0, + 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.030303031, 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.033, 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': None, + 'created_at': datetime.datetime(2024, 9, 24, 13, 42, 52, 584553), 'is_active': True, 'prime_material_cost': None, 'material_cost': 3.88, 'labour_cost': 3.24, 'labour_hours_per_unit': 0.14, 'plant_cost': 0.0, 'total_cost': 7.12, - 'notes': None}, - {'id': 1169, 'type': 'suspended_floor_insulation', 'description': 'Kay-Cel Expanded Polystyrene Board', - 'depth': 50.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.030303031, + 'notes': None, 'is_installer_quote': False}, + {'id': 2084, 'type': 'solid_floor_insulation', 'description': 'Kay-Cel Expanded Polystyrene Board', 'depth': 50.0, + 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.030303031, 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.033, 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': None, + 'created_at': datetime.datetime(2024, 9, 24, 13, 42, 52, 584553), 'is_active': True, 'prime_material_cost': None, 'material_cost': 6.62, 'labour_cost': 3.71, 'labour_hours_per_unit': 0.16, 'plant_cost': 0.0, 'total_cost': 10.33, - 'notes': None}, - {'id': 1170, 'type': 'suspended_floor_insulation', 'description': 'Kay-Cel Expanded Polystyrene Board', - 'depth': 75.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.030303031, + 'notes': None, 'is_installer_quote': False}, + {'id': 2085, 'type': 'solid_floor_insulation', 'description': 'Kay-Cel Expanded Polystyrene Board', 'depth': 75.0, + 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.030303031, 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.033, 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': None, + 'created_at': datetime.datetime(2024, 9, 24, 13, 42, 52, 584553), 'is_active': True, 'prime_material_cost': None, 'material_cost': 9.3, 'labour_cost': 4.17, 'labour_hours_per_unit': 0.18, 'plant_cost': 0.0, 'total_cost': 13.47, - 'notes': None}, - {'id': 1171, 'type': 'suspended_floor_insulation', 'description': 'Kay-Cel Expanded Polystyrene Board', - 'depth': 100.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.030303031, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.033, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': None, - 'material_cost': 12.02, 'labour_cost': 4.4, 'labour_hours_per_unit': 0.19, 'plant_cost': 0.0, 'total_cost': 16.42, - 'notes': None}, {'id': 1172, 'type': 'suspended_floor_insulation', - 'description': 'Kingspan Thermafloor TF70 High Performance Rigid Floor Insulation', 'depth': 50.0, - 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, - 'prime_material_cost': None, 'material_cost': 10.36, 'labour_cost': 4.06, - 'labour_hours_per_unit': 0.18, 'plant_cost': 0.0, 'total_cost': 14.42, 'notes': None}, - {'id': 1173, 'type': 'suspended_floor_insulation', + 'notes': None, 'is_installer_quote': False}, {'id': 2086, 'type': 'solid_floor_insulation', + 'description': 'Kingspan Thermafloor TF70 High Performance Rigid ' + 'Floor Insulation', + 'depth': 50.0, 'depth_unit': 'mm', 'cost': None, + 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547, + 'r_value_unit': 'square_meter_kelvin_per_watt', + 'thermal_conductivity': 0.022, + 'thermal_conductivity_unit': 'watt_per_meter_kelvin', + 'link': 'SPONs', + 'created_at': datetime.datetime(2024, 9, 24, 13, 42, 52, 584553), + 'is_active': True, 'prime_material_cost': None, + 'material_cost': 10.36, 'labour_cost': 4.06, + 'labour_hours_per_unit': 0.18, 'plant_cost': 0.0, + 'total_cost': 14.42, 'notes': None, 'is_installer_quote': False}, + {'id': 2087, 'type': 'solid_floor_insulation', 'description': 'Kingspan Thermafloor TF70 High Performance Rigid Floor Insulation', 'depth': 75.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547, 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022, 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': None, + 'created_at': datetime.datetime(2024, 9, 24, 13, 42, 52, 584553), 'is_active': True, 'prime_material_cost': None, 'material_cost': 15.35, 'labour_cost': 4.06, 'labour_hours_per_unit': 0.18, 'plant_cost': 0.0, 'total_cost': 19.41, - 'notes': None}, {'id': 1174, 'type': 'suspended_floor_insulation', - 'description': 'Kingspan Thermafloor TF70 High Performance Rigid Floor Insulation', - 'depth': 100.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', - 'r_value_per_mm': 0.045454547, 'r_value_unit': 'square_meter_kelvin_per_watt', - 'thermal_conductivity': 0.022, 'thermal_conductivity_unit': 'watt_per_meter_kelvin', - 'link': 'SPONs', 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), - 'is_active': True, 'prime_material_cost': None, 'material_cost': 19.17, 'labour_cost': 4.06, - 'labour_hours_per_unit': 0.18, 'plant_cost': 0.0, 'total_cost': 23.23, 'notes': None}, - {'id': 1175, 'type': 'suspended_floor_insulation', - 'description': 'Kingspan Thermafloor TF70 High Performance Rigid Floor Insulation', 'depth': 125.0, - 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': None, - 'material_cost': 26.59, 'labour_cost': 4.06, 'labour_hours_per_unit': 0.18, 'plant_cost': 0.0, 'total_cost': 30.65, - 'notes': None}, {'id': 1176, 'type': 'suspended_floor_insulation', - 'description': 'Kingspan Thermafloor TF70 High Performance Rigid Floor Insulation', - 'depth': 150.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', - 'r_value_per_mm': 0.045454547, 'r_value_unit': 'square_meter_kelvin_per_watt', - 'thermal_conductivity': 0.022, 'thermal_conductivity_unit': 'watt_per_meter_kelvin', - 'link': 'SPONs', 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), - 'is_active': True, 'prime_material_cost': None, 'material_cost': 31.13, 'labour_cost': 4.64, - 'labour_hours_per_unit': 0.2, 'plant_cost': 0.0, 'total_cost': 35.77, 'notes': None}, - {'id': 1177, 'type': 'suspended_floor_redecoration', 'description': 'refix floorboards previously set aside', - 'depth': 0.0, 'depth_unit': None, 'cost': None, 'cost_unit': None, 'r_value_per_mm': None, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None, 'thermal_conductivity_unit': None, - 'link': 'SPONs', 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, - 'prime_material_cost': None, 'material_cost': 1.54, 'labour_cost': 24.98, 'labour_hours_per_unit': 0.74, - 'plant_cost': 0.0, 'total_cost': 26.52, 'notes': None}, - {'id': 1178, 'type': 'suspended_floor_redecoration', 'description': 'Fitting carpet', 'depth': 0.0, - 'depth_unit': None, 'cost': None, 'cost_unit': None, 'r_value_per_mm': None, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None, 'thermal_conductivity_unit': None, - 'link': 'SPONs', 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, - 'prime_material_cost': None, 'material_cost': 0.0, 'labour_cost': 6.59, 'labour_hours_per_unit': 0.37, - 'plant_cost': 0.0, 'total_cost': 6.59, - 'notes': 'SPONs does not have data on re-fitting the carpet so we use the data in Fitted carpeting; Gradus woven ' - 'polypropylene tufted loop\n\n as a baseline. We assume re-use of carpets, therefore we need just ' - 'labour rates'}, - {'id': 1179, 'type': 'solid_floor_demolition', 'description': 'Removal of carpet and underfelt', 'depth': 0.0, - 'depth_unit': None, 'cost': None, 'cost_unit': None, 'r_value_per_mm': None, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None, 'thermal_conductivity_unit': None, - 'link': 'SPONs', 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, - 'prime_material_cost': None, 'material_cost': 0.0, 'labour_cost': 3.32, 'labour_hours_per_unit': 0.11, - 'plant_cost': 0.0, 'total_cost': 3.32, - 'notes': 'We ignore the plant cost that is in SPONs because we assume the carpet is not scrapped and therefore ' - 'there is no need for a skip'}, - {'id': 1180, 'type': 'solid_floor_preparation', - 'description': 'clean surface of concrete to receive new damp-proof membrane', 'depth': 0.0, 'depth_unit': None, - 'cost': None, 'cost_unit': None, 'r_value_per_mm': None, 'r_value_unit': 'square_meter_kelvin_per_watt', - 'thermal_conductivity': None, 'thermal_conductivity_unit': None, 'link': None, - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': None, - 'material_cost': 0.0, 'labour_cost': 4.36, 'labour_hours_per_unit': 0.14, 'plant_cost': 0.0, 'total_cost': 4.36, - 'notes': None}, {'id': 1181, 'type': 'solid_floor_preparation', - 'description': 'Clean out crack to form a 20mm×20mm groove and fill with cement: mortar mixed ' - 'with bonding agent', - 'depth': 0.0, 'depth_unit': None, 'cost': None, 'cost_unit': None, 'r_value_per_mm': None, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None, - 'thermal_conductivity_unit': None, 'link': None, - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, - 'prime_material_cost': None, 'material_cost': 6.91, 'labour_cost': 18.99, - 'labour_hours_per_unit': 0.61, 'plant_cost': 0.16, 'total_cost': 26.06, - 'notes': 'This step is the assessment and repair of any damage to the concrete floor such as ' - 'filling cracks or levelling uneven areas'}, - {'id': 1182, 'type': 'solid_floor_vapour_barrier', 'description': 'Visqueen High Performance Vapour Barrier', - 'depth': 0.0, 'depth_unit': None, 'cost': None, 'cost_unit': None, 'r_value_per_mm': None, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None, 'thermal_conductivity_unit': None, - 'link': 'SPONs', 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, - 'prime_material_cost': 0.58, 'material_cost': 1.21, 'labour_cost': 0.48, 'labour_hours_per_unit': 0.02, - 'plant_cost': 0.0, 'total_cost': 1.69, 'notes': None}, - {'id': 1183, 'type': 'solid_floor_insulation', 'description': 'Kay-Cel Expanded Polystyrene Board', 'depth': 25.0, - 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.030303031, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.033, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': None, - 'material_cost': 3.88, 'labour_cost': 3.24, 'labour_hours_per_unit': 0.14, 'plant_cost': 0.0, 'total_cost': 7.12, - 'notes': None}, - {'id': 1184, 'type': 'solid_floor_insulation', 'description': 'Kay-Cel Expanded Polystyrene Board', 'depth': 50.0, - 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.030303031, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.033, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': None, - 'material_cost': 6.62, 'labour_cost': 3.71, 'labour_hours_per_unit': 0.16, 'plant_cost': 0.0, 'total_cost': 10.33, - 'notes': None}, - {'id': 1185, 'type': 'solid_floor_insulation', 'description': 'Kay-Cel Expanded Polystyrene Board', 'depth': 75.0, - 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.030303031, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.033, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': None, - 'material_cost': 9.3, 'labour_cost': 4.17, 'labour_hours_per_unit': 0.18, 'plant_cost': 0.0, 'total_cost': 13.47, - 'notes': None}, {'id': 1186, 'type': 'solid_floor_insulation', - 'description': 'Kingspan Thermafloor TF70 High Performance Rigid Floor Insulation', 'depth': 50.0, - 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, - 'prime_material_cost': None, 'material_cost': 10.36, 'labour_cost': 4.06, - 'labour_hours_per_unit': 0.18, 'plant_cost': 0.0, 'total_cost': 14.42, 'notes': None}, - {'id': 1187, 'type': 'solid_floor_insulation', - 'description': 'Kingspan Thermafloor TF70 High Performance Rigid Floor Insulation', 'depth': 75.0, - 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': None, - 'material_cost': 15.35, 'labour_cost': 4.06, 'labour_hours_per_unit': 0.18, 'plant_cost': 0.0, 'total_cost': 19.41, - 'notes': None}, {'id': 1188, 'type': 'solid_floor_insulation', - 'description': 'Ecotherm Eco-Versal General Purpose Insulation Board', 'depth': 30.0, - 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, - 'prime_material_cost': 6.16, 'material_cost': 16.73, 'labour_cost': 28.34, - 'labour_hours_per_unit': 1.2, 'plant_cost': 0.0, 'total_cost': 45.07, 'notes': None}, - {'id': 1189, 'type': 'solid_floor_insulation', + 'notes': None, 'is_installer_quote': False}, {'id': 2088, 'type': 'solid_floor_insulation', + 'description': 'Ecotherm Eco-Versal General Purpose Insulation ' + 'Board', + 'depth': 30.0, 'depth_unit': 'mm', 'cost': None, + 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547, + 'r_value_unit': 'square_meter_kelvin_per_watt', + 'thermal_conductivity': 0.022, + 'thermal_conductivity_unit': 'watt_per_meter_kelvin', + 'link': 'SPONs', + 'created_at': datetime.datetime(2024, 9, 24, 13, 42, 52, 584553), + 'is_active': True, 'prime_material_cost': 6.16, + 'material_cost': 16.73, 'labour_cost': 28.34, + 'labour_hours_per_unit': 1.2, 'plant_cost': 0.0, 'total_cost': 45.07, + 'notes': None, 'is_installer_quote': False}, + {'id': 2089, 'type': 'solid_floor_insulation', 'description': 'Ecotherm Eco-Versal General Purpose Insulation Board', 'depth': 50.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547, 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022, 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': 8.46, + 'created_at': datetime.datetime(2024, 9, 24, 13, 42, 52, 584553), 'is_active': True, 'prime_material_cost': 8.46, 'material_cost': 19.1, 'labour_cost': 28.34, 'labour_hours_per_unit': 1.2, 'plant_cost': 0.0, 'total_cost': 47.44, - 'notes': None}, {'id': 1190, 'type': 'solid_floor_insulation', - 'description': 'Ecotherm Eco-Versal General Purpose Insulation Board', 'depth': 60.0, - 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', - 'link': 'https://londonbuildingsupplies.co.uk/products/60mm--ecotherm-eco-versal-general' - '-purpose-pir-insulation-board---2.4m-x-1.2m-x-60mm.html', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, - 'prime_material_cost': None, 'material_cost': 24.081198, 'labour_cost': 28.34, - 'labour_hours_per_unit': 1.2, 'plant_cost': 0.0, 'total_cost': 52.421196, - 'notes': "This material isn't in SPONs but checking online, is around 92% of the cost of the " - "100mm"}, - {'id': 1191, 'type': 'solid_floor_insulation', + 'notes': None, 'is_installer_quote': False}, {'id': 2090, 'type': 'solid_floor_insulation', + 'description': 'Ecotherm Eco-Versal General Purpose Insulation ' + 'Board', + 'depth': 60.0, 'depth_unit': 'mm', 'cost': None, + 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547, + 'r_value_unit': 'square_meter_kelvin_per_watt', + 'thermal_conductivity': 0.022, + 'thermal_conductivity_unit': 'watt_per_meter_kelvin', + 'link': 'https://londonbuildingsupplies.co.uk/products/60mm--ecotherm-eco-versal-general-purpose-pir-insulation-board---2.4m-x-1.2m-x-60mm.html', + 'created_at': datetime.datetime(2024, 9, 24, 13, 42, 52, 584553), + 'is_active': True, 'prime_material_cost': None, + 'material_cost': 24.081198, 'labour_cost': 28.34, + 'labour_hours_per_unit': 1.2, 'plant_cost': 0.0, + 'total_cost': 52.421196, + 'notes': "This material isn't in SPONs but checking online, " + "is around 92% of the cost of the 100mm", + 'is_installer_quote': False}, + {'id': 2091, 'type': 'solid_floor_insulation', 'description': 'Ecotherm Eco-Versal General Purpose Insulation Board', 'depth': 70.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547, 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022, 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'https://londonbuildingsupplies.co.uk/products/70mm--ecotherm-eco-versal-general-purpose-pir-insulation' '-board---2.4m-x-1.2m-x-70mm.html', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': None, + 'created_at': datetime.datetime(2024, 9, 24, 13, 42, 52, 584553), 'is_active': True, 'prime_material_cost': None, 'material_cost': 27.089088, 'labour_cost': 28.34, 'labour_hours_per_unit': 1.2, 'plant_cost': 0.0, 'total_cost': 55.42909, 'notes': "This material isn't in SPONs but checking online, is around 104% of the cost of the 100mm (more " - "expensive than 100mm)"}, - {'id': 1192, 'type': 'solid_floor_insulation', - 'description': 'Ecotherm Eco-Versal General Purpose Insulation Board', 'depth': 100.0, 'depth_unit': 'mm', - 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': 15.12, - 'material_cost': 25.96, 'labour_cost': 30.7, 'labour_hours_per_unit': 1.3, 'plant_cost': 0.0, 'total_cost': 56.66, - 'notes': None}, - {'id': 1193, 'type': 'solid_floor_insulation', 'description': 'Ravatherm XPS X 500 SL Polystyrene Foam', + "expensive than 100mm)", + 'is_installer_quote': False}, {'id': 2092, 'type': 'solid_floor_insulation', + 'description': 'Ecotherm Eco-Versal General Purpose Insulation Board', + 'depth': 100.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', + 'r_value_per_mm': 0.045454547, 'r_value_unit': 'square_meter_kelvin_per_watt', + 'thermal_conductivity': 0.022, 'thermal_conductivity_unit': 'watt_per_meter_kelvin', + 'link': 'SPONs', 'created_at': datetime.datetime(2024, 9, 24, 13, 42, 52, 584553), + 'is_active': True, 'prime_material_cost': 15.12, 'material_cost': 25.96, + 'labour_cost': 30.7, 'labour_hours_per_unit': 1.3, 'plant_cost': 0.0, + 'total_cost': 56.66, 'notes': None, 'is_installer_quote': False}, + {'id': 2093, 'type': 'solid_floor_insulation', 'description': 'Ravatherm XPS X 500 SL Polystyrene Foam', 'depth': 50.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.032258064, 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.031, 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': None, + 'created_at': datetime.datetime(2024, 9, 24, 13, 42, 52, 584553), 'is_active': True, 'prime_material_cost': None, 'material_cost': 11.07, 'labour_cost': 10.66, 'labour_hours_per_unit': 0.46, 'plant_cost': 0.0, 'total_cost': 21.73, 'notes': "In Spons, the thermal conductivity is 0.033 however the datasheet indicates it's 0.32: " "https://ravagobuildingsolutions.com/uk/wp-content/uploads/sites/30/2022/08/ravatherm-xps-x-500-sl-tds" - "-version-1-20210901.pdf"}, - {'id': 1194, 'type': 'solid_floor_insulation', 'description': 'Ravatherm XPS X 500 SL Polystyrene Foam', + "-version-1-20210901.pdf", + 'is_installer_quote': False}, + {'id': 2094, 'type': 'solid_floor_insulation', 'description': 'Ravatherm XPS X 500 SL Polystyrene Foam', 'depth': 75.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.03125, 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.032, 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': None, + 'created_at': datetime.datetime(2024, 9, 24, 13, 42, 52, 584553), 'is_active': True, 'prime_material_cost': None, 'material_cost': 16.28, 'labour_cost': 10.66, 'labour_hours_per_unit': 0.46, 'plant_cost': 0.0, - 'total_cost': 26.94, 'notes': None}, {'id': 1195, 'type': 'solid_floor_redecoration', - 'description': 'Screeded beds; protection to compressible formwork ' - 'exceeding 600mm wide', - 'depth': 0.0, 'depth_unit': None, 'cost': None, 'cost_unit': None, - 'r_value_per_mm': None, 'r_value_unit': 'square_meter_kelvin_per_watt', - 'thermal_conductivity': None, 'thermal_conductivity_unit': None, - 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), - 'is_active': True, 'prime_material_cost': 9.6, 'material_cost': 9.89, - 'labour_cost': 2.67, 'labour_hours_per_unit': 0.15, 'plant_cost': 0.0, - 'total_cost': 12.56, - 'notes': 'This is the screed layer, placed on top of the insulation'}, - {'id': 1196, 'type': 'solid_floor_redecoration', 'description': 'Fitting carpet', 'depth': 0.0, 'depth_unit': None, + 'total_cost': 26.94, 'notes': None, 'is_installer_quote': False}, {'id': 2095, 'type': 'solid_floor_redecoration', + 'description': 'Screeded beds; protection to ' + 'compressible formwork ' + 'exceeding 600mm wide', + 'depth': 0.0, 'depth_unit': None, 'cost': None, + 'cost_unit': None, 'r_value_per_mm': None, + 'r_value_unit': 'square_meter_kelvin_per_watt', + 'thermal_conductivity': None, + 'thermal_conductivity_unit': None, + 'link': 'SPONs', + 'created_at': datetime.datetime(2024, 9, 24, 13, + 42, 52, 584553), + 'is_active': True, 'prime_material_cost': 9.6, + 'material_cost': 9.89, 'labour_cost': 2.67, + 'labour_hours_per_unit': 0.15, + 'plant_cost': 0.0, 'total_cost': 12.56, + 'notes': 'This is the screed layer, ' + 'placed on top of the insulation', + 'is_installer_quote': False}, + {'id': 2096, 'type': 'solid_floor_redecoration', 'description': 'Fitting carpet', 'depth': 0.0, 'depth_unit': None, 'cost': None, 'cost_unit': None, 'r_value_per_mm': None, 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None, 'thermal_conductivity_unit': None, 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': None, + 'created_at': datetime.datetime(2024, 9, 24, 13, 42, 52, 584553), 'is_active': True, 'prime_material_cost': None, 'material_cost': 0.0, 'labour_cost': 6.59, 'labour_hours_per_unit': 0.37, 'plant_cost': 0.0, 'total_cost': 6.59, 'notes': 'SPONs does not have data on re-fitting the carpet so we use the data in Fitted carpeting; Gradus woven ' 'polypropylene tufted loop\n\n as a baseline. We assume re-use of carpets, therefore we need just ' - 'labour rates'}, - {'id': 1197, 'type': 'solid_floor_redecoration', - 'description': 'Fitting existing softwood skirting or architrave to new frames; 150mm high', 'depth': 0.0, - 'depth_unit': None, 'cost': None, 'cost_unit': None, 'r_value_per_mm': None, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None, 'thermal_conductivity_unit': None, - 'link': 'SPONs', 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, - 'prime_material_cost': None, 'material_cost': 0.01, 'labour_cost': 4.87, 'labour_hours_per_unit': 0.12, - 'plant_cost': 0.0, 'total_cost': 4.88, 'notes': None}, {'id': 1198, 'type': 'ewi_wall_demolition', - 'description': 'Solid & Dry Lined walls: Hack of wall ' - 'finishes with chipping hammer; plaster ' - 'to walls.', - 'depth': 0.0, 'depth_unit': None, 'cost': None, - 'cost_unit': None, 'r_value_per_mm': None, - 'r_value_unit': 'square_meter_kelvin_per_watt', - 'thermal_conductivity': None, - 'thermal_conductivity_unit': None, 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, - 244907), 'is_active': True, - 'prime_material_cost': None, 'material_cost': 0.0, - 'labour_cost': 10.27, 'labour_hours_per_unit': 0.33, - 'plant_cost': 1.28, 'total_cost': 11.55, 'notes': None}, - {'id': 1199, 'type': 'ewi_wall_demolition', - 'description': 'Stud walls: Remove wall linings including battening behind; plasterboard and skim', 'depth': 0.0, - 'depth_unit': None, 'cost': None, 'cost_unit': None, 'r_value_per_mm': None, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None, 'thermal_conductivity_unit': None, - 'link': 'SPONs', 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, - 'prime_material_cost': None, 'material_cost': 0.0, 'labour_cost': 6.23, 'labour_hours_per_unit': 0.2, - 'plant_cost': 1.25, 'total_cost': 7.48, 'notes': None}, {'id': 1200, 'type': 'ewi_wall_demolition', - 'description': 'Lathe and Plaster walls: Remove wall ' - 'linings including battening behind; ' - 'wood lath and plaster', - 'depth': 0.0, 'depth_unit': None, 'cost': None, - 'cost_unit': None, 'r_value_per_mm': None, - 'r_value_unit': 'square_meter_kelvin_per_watt', - 'thermal_conductivity': None, - 'thermal_conductivity_unit': None, 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, - 244907), - 'is_active': True, 'prime_material_cost': None, - 'material_cost': 0.0, 'labour_cost': 6.85, - 'labour_hours_per_unit': 0.22, 'plant_cost': 2.09, - 'total_cost': 8.94, 'notes': None}, - {'id': 1201, 'type': 'ewi_wall_preparation', - 'description': 'Clean and prepare surfaces, one coat Keim dilution, one coat primer and two coats of Keim Ecosil ' - 'paint; Brick or block walls; over 300 mm girth', - 'depth': 0.0, 'depth_unit': None, 'cost': None, 'cost_unit': None, 'r_value_per_mm': None, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None, 'thermal_conductivity_unit': None, - 'link': 'SPONs', 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, - 'prime_material_cost': None, 'material_cost': 7.3, 'labour_cost': 5.62, 'labour_hours_per_unit': 0.3, - 'plant_cost': 0.0, 'total_cost': 12.92, - 'notes': 'This work covers the preparation and priming of the wall before insulating'}, - {'id': 1202, 'type': 'external_wall_insulation', 'description': 'Ecotherm Eco-Versal PIR Insulation Board', - 'depth': 30.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': 6.16, - 'material_cost': 16.73, 'labour_cost': 28.34, 'labour_hours_per_unit': 1.2, 'plant_cost': 0.0, 'total_cost': 45.07, - 'notes': None}, - {'id': 1203, 'type': 'external_wall_insulation', 'description': 'Ecotherm Eco-Versal PIR Insulation Board', - 'depth': 50.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': 8.46, - 'material_cost': 19.1, 'labour_cost': 28.34, 'labour_hours_per_unit': 1.2, 'plant_cost': 0.0, 'total_cost': 47.44, - 'notes': None}, - {'id': 1204, 'type': 'external_wall_insulation', 'description': 'Ecotherm Eco-Versal PIR Insulation Board', - 'depth': 100.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': 15.12, - 'material_cost': 25.96, 'labour_cost': 30.7, 'labour_hours_per_unit': 1.3, 'plant_cost': 0.0, 'total_cost': 56.66, - 'notes': None}, - {'id': 1205, 'type': 'external_wall_insulation', 'description': 'Ecotherm Eco-Versal PIR Insulation Board', - 'depth': 150.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': 23.53, - 'material_cost': 34.62, 'labour_cost': 33.06, 'labour_hours_per_unit': 1.4, 'plant_cost': 0.0, 'total_cost': 67.68, - 'notes': None}, - {'id': 1206, 'type': 'external_wall_insulation', 'description': 'Foamglas Grade F Wall Insulation Slabs', - 'depth': 60.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.02631579, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.038, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': 41.69, - 'material_cost': 53.33, 'labour_cost': 29.52, 'labour_hours_per_unit': 1.25, 'plant_cost': 0.0, - 'total_cost': 82.85, 'notes': None}, - {'id': 1207, 'type': 'external_wall_insulation', 'description': 'Foamglas Grade F Wall Insulation Slabs', - 'depth': 100.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.02631579, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.038, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': 86.86, - 'material_cost': 99.85, 'labour_cost': 29.52, 'labour_hours_per_unit': 1.25, 'plant_cost': 0.0, - 'total_cost': 129.37, 'notes': None}, - {'id': 1208, 'type': 'external_wall_insulation', 'description': 'Foamglas Grade F Wall Insulation Slabs', - 'depth': 150.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.02631579, - 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.038, - 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, - 'prime_material_cost': 130.29, 'material_cost': 144.58, 'labour_cost': 29.52, 'labour_hours_per_unit': 1.25, - 'plant_cost': 0.0, 'total_cost': 174.1, 'notes': None}, {'id': 1209, 'type': 'ewi_wall_redecoration', - 'description': 'EPS insulation fixed with adhesive to ' - 'SFS structure (measured separately) ' - 'with horizontal PVC intermediate track ' - 'and vertical T-spines; with glassfibre ' - 'mesh reinforcement embedded in Sto ' - 'Armat Classic Basecoat Render and ' - 'Stolit K 1.5 Decorative Topcoat Render ' - '(white)', - 'depth': 0.0, 'depth_unit': None, 'cost': None, - 'cost_unit': None, 'r_value_per_mm': None, - 'r_value_unit': 'square_meter_kelvin_per_watt', - 'thermal_conductivity': None, - 'thermal_conductivity_unit': None, 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, - 244907), - 'is_active': True, 'prime_material_cost': None, - 'material_cost': 0.0, 'labour_cost': 0.0, - 'labour_hours_per_unit': 0.0, 'plant_cost': 0.0, - 'total_cost': 69.94, - 'notes': 'This material in SPONs is for 70mm EPS ' - 'insulation, which comes in at a cost of 99.17 ' - 'per meter square. This includes the cost of ' - 'insulation. To get the costing for just the ' - 'works and not the insulation, we subtract the ' - 'cost of EPS insulation, using Ravathem 75mm ' - 'insulation as an example, which costs £29.23 ' - 'per meter square, giving us the cost of the ' - 'remaining works without insulation. This ' - 'material gives us a cost for basecoat, ' - 'mesh application and a render finish'}, - {'id': 1210, 'type': 'low_energy_lighting_installation', 'description': 'Installation of fittings and cost of bub', + 'labour rates', + 'is_installer_quote': False}, {'id': 2097, 'type': 'solid_floor_redecoration', + 'description': 'Fitting existing softwood skirting or architrave to new frames; ' + '150mm high', + 'depth': 0.0, 'depth_unit': None, 'cost': None, 'cost_unit': None, + 'r_value_per_mm': None, 'r_value_unit': 'square_meter_kelvin_per_watt', + 'thermal_conductivity': None, 'thermal_conductivity_unit': None, 'link': 'SPONs', + 'created_at': datetime.datetime(2024, 9, 24, 13, 42, 52, 584553), 'is_active': True, + 'prime_material_cost': None, 'material_cost': 0.01, 'labour_cost': 4.87, + 'labour_hours_per_unit': 0.12, 'plant_cost': 0.0, 'total_cost': 4.88, 'notes': None, + 'is_installer_quote': False}, {'id': 2132, 'type': 'external_wall_insulation', + 'description': 'EWI Pro EPS external wall ' + 'insulation system with Brick Slip ' + 'finish', + 'depth': 150.0, 'depth_unit': 'mm', 'cost': None, + 'cost_unit': 'gbp_per_m2', + 'r_value_per_mm': 0.02631579, + 'r_value_unit': 'square_meter_kelvin_per_watt', + 'thermal_conductivity': 0.038, + 'thermal_conductivity_unit': 'watt_per_meter_kelvin', + 'link': 'SCIS', + 'created_at': datetime.datetime(2024, 9, 24, 13, 42, + 52, 584553), + 'is_active': True, 'prime_material_cost': None, + 'material_cost': 0.0, 'labour_cost': 0.0, + 'labour_hours_per_unit': 0.0, 'plant_cost': 0.0, + 'total_cost': 298.35, + 'notes': 'This is the quoted value from SCIS', + 'is_installer_quote': True}, + {'id': 2133, 'type': 'low_energy_lighting_installation', 'description': 'Installation of fittings and cost of bub', 'depth': 0.0, 'depth_unit': None, 'cost': None, 'cost_unit': 'gbp_per_unit', 'r_value_per_mm': None, 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None, 'thermal_conductivity_unit': None, 'link': 'https://www.checkatrade.com/blog/cost-guides/cost-install-downlights/ ' 'https://www.hamuch.com/cost/led-spot-light#:~:text=It%20costs%20an%20average%20of,' 'will%20drive%20up%20the%20cost.', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': None, - 'material_cost': 20.0, 'labour_cost': 15.0, 'labour_hours_per_unit': 0.8, 'plant_cost': 0.0, 'total_cost': 66.0, + 'created_at': datetime.datetime(2024, 9, 24, 13, 42, 52, 584553), 'is_active': True, 'prime_material_cost': None, + 'material_cost': 20.0, 'labour_cost': 15.0, 'labour_hours_per_unit': 0.8, 'plant_cost': 0.0, 'total_cost': 35.0, 'notes': 'We estimate the unit economics from the checkatrade article. We assume that the average job consists ' 'of installing 6 lights based on the hamuch article. We use the median value of 400 for a job of 6 ' - 'lights'}, - {'id': 1235, 'type': 'windows_glazing', - 'description': 'uPVC windows; Profile 22 or other equal and approved; reinforced where appropriate with ' - 'aluminium alloy; in refurbishment work, including standard ironmongery; sills and factory glazed ' - 'with low-e 24 mm double glazing; removing existing windows and fixing new in position; including ' - 'lugs plugged and screwed to brickwork or blockwork; Casement/fixed light; including vents; ' - 'e.p.d.m. glazing gaskets and weather seals; 1770 mm × 1200 mm; ref P312WW', - 'depth': 0.0, 'depth_unit': None, 'cost': None, 'cost_unit': 'gbp_per_unit', 'r_value_per_mm': None, + 'lights', + 'is_installer_quote': False}, + {'id': 2147, 'type': 'flat_roof_insulation', 'description': 'Ecotherm Eco-Versal General Purpose Insulation Board', + 'depth': 150.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547, + 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022, + 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SCIS', + 'created_at': datetime.datetime(2024, 9, 24, 13, 42, 52, 584553), 'is_active': True, 'prime_material_cost': None, + 'material_cost': 0.0, 'labour_cost': 0.0, 'labour_hours_per_unit': 0.0, 'plant_cost': 0.0, 'total_cost': 195.0, + 'notes': 'Rough estimate based on a quote from Nic on 30th May, but the cost is just a rough estimate', + 'is_installer_quote': True}, + {'id': 2149, 'type': 'windows_glazing', 'description': 'REHAU PVCu Casement Windows', 'depth': 0.0, + 'depth_unit': None, 'cost': None, 'cost_unit': 'gbp_per_unit', 'r_value_per_mm': None, 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None, 'thermal_conductivity_unit': None, - 'link': 'SPONs', - 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), - 'is_active': True, 'prime_material_cost': 176.55, - 'material_cost': 182.25, 'labour_cost': 163.36, 'labour_hours_per_unit': 6.5, 'plant_cost': 0.0, - 'total_cost': 345.61, - 'notes': 'This is the cost of removal of existing windows and installation of new windows. This is a casement ' - 'style window, which is the most common but also the cheapest style. In the cost estimation framework, ' - 'we can inflate prices for different finishes, to be conservative on price.'} + 'link': 'SCIS', 'created_at': datetime.datetime(2024, 9, 24, 13, 42, 52, 584553), 'is_active': True, + 'prime_material_cost': None, 'material_cost': 0.0, 'labour_cost': 0.0, 'labour_hours_per_unit': 0.0, + 'plant_cost': 0.0, 'total_cost': 1140.0, 'notes': None, 'is_installer_quote': True} ] diff --git a/recommendations/tests/test_floor_recommendations.py b/recommendations/tests/test_floor_recommendations.py index 555f9a27..b3b108b3 100644 --- a/recommendations/tests/test_floor_recommendations.py +++ b/recommendations/tests/test_floor_recommendations.py @@ -7,11 +7,15 @@ from recommendations.tests.test_data.materials import materials from backend.Property import Property +# import inspect +# +# file_path = inspect.getfile(lambda: None) # with open( -# os.path.abspath(os.path.dirname(__file__)) + "/recommendations/tests/test_data/input_properties.pkl", "rb" +# os.path.abspath(os.path.dirname(file_path)) + "/recommendations/tests/test_data/input_properties.pkl", "rb" # ) as f: # input_properties = pickle.load(f) + class TestFloorRecommendations: @pytest.fixture @@ -59,6 +63,7 @@ class TestFloorRecommendations: input_properties[2].floor_type = "suspended" input_properties[2].number_of_floors = 1 input_properties[2].floor_level = 0 + input_properties[2].already_installed = [] recommender = FloorRecommendations(property_instance=input_properties[2], materials=materials) assert recommender.estimated_u_value is None @@ -71,8 +76,8 @@ class TestFloorRecommendations: assert types == {"suspended_floor_insulation"} - assert len(recommender.recommendations) == 6 - assert recommender.recommendations[0]["total"] == 4925.205 + assert len(recommender.recommendations) == 1 + assert recommender.recommendations[0]["total"] == 4687.5 assert recommender.recommendations[0]["new_u_value"] == 0.21 def test_uvalue_0_12(self, input_properties): @@ -108,6 +113,7 @@ class TestFloorRecommendations: input_properties[4].floor_type = "solid" input_properties[4].number_of_floors = 1 input_properties[4].floor_level = 0 + input_properties[4].already_installed = [] # In this case, we have no county, so in this case, it should yse the local-authority-label if possible input_properties[4].data["county"] = "" From 1e5661bf3ac051eb5e893da70312a04c6df8f53f Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 30 Sep 2024 15:55:25 +0100 Subject: [PATCH 068/166] added back exposed floor recommendations --- .../tests/test_floor_recommendations.py | 249 +++++++++--------- 1 file changed, 129 insertions(+), 120 deletions(-) diff --git a/recommendations/tests/test_floor_recommendations.py b/recommendations/tests/test_floor_recommendations.py index b3b108b3..17f1f82e 100644 --- a/recommendations/tests/test_floor_recommendations.py +++ b/recommendations/tests/test_floor_recommendations.py @@ -5,6 +5,7 @@ from unittest.mock import Mock from recommendations.FloorRecommendations import FloorRecommendations from recommendations.tests.test_data.materials import materials from backend.Property import Property +from etl.epc.Record import EPCRecord # import inspect @@ -152,123 +153,131 @@ class TestFloorRecommendations: assert recommender.estimated_u_value is None assert not recommender.recommendations - # def test_exposed_floor_no_insulation(self): - # input_property = Property(id=1, postcode="F4k3 2", address1="223 fake street", epc_client=Mock()) - # input_property.floor = { - # 'original_description': 'To unheated space, no insulation (assumed)', - # 'clean_description': 'To unheated space, no insulation', 'thermal_transmittance': None, - # 'thermal_transmittance_unit': None, 'is_assumed': True, 'is_to_unheated_space': True, - # 'is_to_external_air': False, 'is_suspended': False, 'is_solid': False, 'another_property_below': False, - # 'insulation_thickness': 'none' - # } - # input_property.age_band = "L" - # input_property.set_floor_type() - # input_property.data = {"floor-level": 0, "property-type": "House"} - # input_property.floor_area = 100 - # input_property.number_of_floors = 1 - # - # recommender = FloorRecommendations( - # property_instance=input_property, - # materials=materials - # ) - # - # assert not recommender.recommendations - # - # recommender.recommend() - # - # # Because of age band L, this should have a u-value of 0.22 to begin with and no recommendation - # assert not len(recommender.recommendations) - # assert recommender.estimated_u_value == 0.22 - # - # # Now with an older age band - # - # input_property2 = Property(id=1, postcode="F4k3 2", address1="223 fake street", epc_client=Mock()) - # input_property2.floor = { - # 'original_description': 'To unheated space, no insulation (assumed)', - # 'clean_description': 'To unheated space, no insulation', 'thermal_transmittance': None, - # 'thermal_transmittance_unit': None, 'is_assumed': True, 'is_to_unheated_space': True, - # 'is_to_external_air': False, 'is_suspended': False, 'is_solid': False, 'another_property_below': False, - # 'insulation_thickness': 'none' - # } - # input_property2.age_band = "D" - # input_property2.set_floor_type() - # input_property2.data = {"floor-level": 0, "property-type": "House"} - # input_property2.floor_area = 100 - # input_property2.number_of_floors = 1 - # - # recommender2 = FloorRecommendations( - # property_instance=input_property2, - # materials=materials - # ) - # - # assert not recommender2.recommendations - # - # recommender2.recommend() - # - # assert len(recommender2.recommendations) == 1 - # - # assert recommender2.recommendations[0]["new_u_value"] == 0.23 - # assert recommender2.recommendations[0]["starting_u_value"] == 1.2 - # assert recommender2.recommendations[0]["cost"] == 1500 - # - # def test_exposed_floor_below_average_insulated(self): - # input_property3 = Property(id=1, postcode="F4k3 2", address1="223 fake street", epc_client=Mock()) - # input_property3.floor = { - # 'original_description': 'To unheated space, below average insulation (assumed)', - # 'clean_description': 'To unheated space, below average insulation', 'thermal_transmittance': None, - # 'thermal_transmittance_unit': None, 'is_assumed': True, 'is_to_unheated_space': True, - # 'is_to_external_air': False, 'is_suspended': False, 'is_solid': False, 'another_property_below': False, - # 'insulation_thickness': 'below average' - # } - # input_property3.age_band = "C" - # input_property3.set_floor_type() - # input_property3.data = {"floor-level": 0, "property-type": "House"} - # input_property3.floor_area = 100 - # input_property3.number_of_floors = 1 - # - # recommender3 = FloorRecommendations( - # property_instance=input_property3, - # materials=materials - # ) - # - # assert not recommender3.recommendations - # - # recommender3.recommend() - # - # assert recommender3.estimated_u_value == 0.5 - # - # assert len(recommender3.recommendations) == 1 - # - # assert recommender3.recommendations[0]["new_u_value"] == 0.22 - # assert recommender3.recommendations[0]["starting_u_value"] == 0.5 - # assert recommender3.recommendations[0]["cost"] == 1100 - # assert recommender3.recommendations[0]["parts"][0]["depths"] == [100] - # - # # With average insulation, no recommendations - # - # input_property4 = Property(id=1, postcode="F4k3 2", address1="223 fake street", epc_client=Mock()) - # input_property4.floor = { - # 'original_description': 'To unheated space, insulated (assumed)', - # 'clean_description': 'To unheated space, insulated', 'thermal_transmittance': None, - # 'thermal_transmittance_unit': None, 'is_assumed': True, 'is_to_unheated_space': True, - # 'is_to_external_air': False, 'is_suspended': False, 'is_solid': False, 'another_property_below': False, - # 'insulation_thickness': 'average' - # } - # input_property4.age_band = "C" - # input_property4.set_floor_type() - # input_property4.data = {"floor-level": 0, "property-type": "House"} - # input_property4.floor_area = 100 - # input_property4.number_of_floors = 1 - # - # recommender4 = FloorRecommendations( - # property_instance=input_property4, - # materials=materials - # ) - # - # assert not recommender4.recommendations - # - # recommender4.recommend() - # - # assert recommender4.estimated_u_value is None - # - # assert len(recommender4.recommendations) == 0 + def test_exposed_floor_no_insulation(self): + epc_record = EPCRecord() + epc_record.prepared_epc = {"county": "Greater London", "floor-level": 0, "property-type": "House"} + epc_record.full_sap_epc = {} + + input_property = Property(id=1, postcode="F4k3 2", address="223 fake street", epc_record=epc_record) + input_property.floor = { + 'original_description': 'To unheated space, no insulation (assumed)', + 'clean_description': 'To unheated space, no insulation', 'thermal_transmittance': None, + 'thermal_transmittance_unit': None, 'is_assumed': True, 'is_to_unheated_space': True, + 'is_to_external_air': False, 'is_suspended': False, 'is_solid': False, 'another_property_below': False, + 'insulation_thickness': 'none' + } + input_property.age_band = "L" + input_property.set_floor_type() + input_property.floor_area = 100 + input_property.number_of_floors = 1 + + recommender = FloorRecommendations( + property_instance=input_property, + materials=materials + ) + + assert not recommender.recommendations + + recommender.recommend() + + # Because of age band L, this should have a u-value of 0.22 to begin with and no recommendation + assert not len(recommender.recommendations) + assert recommender.estimated_u_value == 0.22 + + # Now with an older age band + epc_record2 = EPCRecord() + epc_record2.prepared_epc = {"county": "Greater London", "floor-level": 0, "property-type": "House"} + epc_record2.full_sap_epc = {} + + input_property2 = Property(id=1, postcode="F4k3 2", address="223 fake street", epc_record=epc_record2) + input_property2.floor = { + 'original_description': 'To unheated space, no insulation (assumed)', + 'clean_description': 'To unheated space, no insulation', 'thermal_transmittance': None, + 'thermal_transmittance_unit': None, 'is_assumed': True, 'is_to_unheated_space': True, + 'is_to_external_air': False, 'is_suspended': False, 'is_solid': False, 'another_property_below': False, + 'insulation_thickness': 'none' + } + input_property2.age_band = "D" + input_property2.set_floor_type() + input_property2.insulation_floor_area = 100 + input_property2.number_of_floors = 1 + + recommender2 = FloorRecommendations( + property_instance=input_property2, + materials=materials + ) + + assert not recommender2.recommendations + + recommender2.recommend() + + assert len(recommender2.recommendations) == 1 + + assert recommender2.recommendations[0]["new_u_value"] == 0.24 + assert recommender2.recommendations[0]["starting_u_value"] == 1.2 + assert recommender2.recommendations[0]["total"] == 9375 + + def test_exposed_floor_below_average_insulated(self): + epc_record3 = EPCRecord() + epc_record3.prepared_epc = {"county": "Greater London", "floor-level": 0, "property-type": "House"} + epc_record3.full_sap_epc = {} + input_property3 = Property(id=1, postcode="F4k3 2", address="223 fake street", epc_record=epc_record3) + input_property3.floor = { + 'original_description': 'To unheated space, below average insulation (assumed)', + 'clean_description': 'To unheated space, below average insulation', 'thermal_transmittance': None, + 'thermal_transmittance_unit': None, 'is_assumed': True, 'is_to_unheated_space': True, + 'is_to_external_air': False, 'is_suspended': False, 'is_solid': False, 'another_property_below': False, + 'insulation_thickness': 'below average' + } + input_property3.age_band = "C" + input_property3.set_floor_type() + input_property3.insulation_floor_area = 100 + input_property3.number_of_floors = 1 + + recommender3 = FloorRecommendations( + property_instance=input_property3, + materials=materials + ) + + assert not recommender3.recommendations + + recommender3.recommend() + + assert recommender3.estimated_u_value == 0.5 + + assert len(recommender3.recommendations) == 1 + + assert recommender3.recommendations[0]["new_u_value"] == 0.24 + assert recommender3.recommendations[0]["starting_u_value"] == 0.5 + assert recommender3.recommendations[0]["total"] == 7500 + assert recommender3.recommendations[0]["parts"][0]["depth"] == 50 + + # With average insulation, no recommendations + epc_record4 = EPCRecord() + epc_record4.prepared_epc = {"county": "Greater London", "floor-level": 0, "property-type": "House"} + epc_record4.full_sap_epc = {} + input_property4 = Property(id=1, postcode="F4k3 2", address="223 fake street", epc_record=epc_record4) + input_property4.floor = { + 'original_description': 'To unheated space, insulated (assumed)', + 'clean_description': 'To unheated space, insulated', 'thermal_transmittance': None, + 'thermal_transmittance_unit': None, 'is_assumed': True, 'is_to_unheated_space': True, + 'is_to_external_air': False, 'is_suspended': False, 'is_solid': False, 'another_property_below': False, + 'insulation_thickness': 'average' + } + input_property4.age_band = "C" + input_property4.set_floor_type() + input_property4.insulation_floor_area = 100 + input_property4.number_of_floors = 1 + + recommender4 = FloorRecommendations( + property_instance=input_property4, + materials=materials + ) + + assert not recommender4.recommendations + + recommender4.recommend() + + assert recommender4.estimated_u_value is None + + assert len(recommender4.recommendations) == 0 From 7263d83a42ba24c41ce1ec657c65c98c35323840 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 30 Sep 2024 16:00:30 +0100 Subject: [PATCH 069/166] fixed lighting recommendation test --- .../tests/test_lighting_recommendations.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/recommendations/tests/test_lighting_recommendations.py b/recommendations/tests/test_lighting_recommendations.py index 45213d70..96440c01 100644 --- a/recommendations/tests/test_lighting_recommendations.py +++ b/recommendations/tests/test_lighting_recommendations.py @@ -41,8 +41,12 @@ class TestLightingRecommendations: assert len(lr.recommendation) == 1 assert lr.recommendation == [ - {'parts': [], 'type': 'low_energy_lighting', 'description': 'Install low energy lighting in 4 outlets', - 'starting_u_value': None, 'new_u_value': None, 'sap_points': 0.4, 'total': 240.24, - 'subtotal': 200.20000000000002, 'vat': 40.040000000000006, 'contingency': 14.3, 'preliminaries': 14.3, - 'material': 80.0, 'profit': 28.6, 'labour_hours': 3.2, 'labour_days': 0.4, 'labour_cost': 63.0} + {'phase': 0, 'parts': [], 'type': 'low_energy_lighting', + 'description': 'Install low energy lighting in 4 outlets', 'starting_u_value': None, 'new_u_value': None, + 'already_installed': False, 'sap_points': 0.4, 'kwh_savings': 219.0, 'co2_equivalent_savings': 0.035478, + 'description_simulation': {'lighting-energy-eff': 'Very Good', + 'lighting-description': 'Low energy lighting in all fixed outlets', + 'low-energy-lighting': 100}, 'total': 240.24, 'subtotal': 200.20000000000002, + 'vat': 40.040000000000006, 'contingency': 14.3, 'preliminaries': 14.3, 'material': 80.0, 'profit': 28.6, + 'labour_hours': 3.2, 'labour_days': 0.4, 'labour_cost': 63.0, 'survey': False} ] From f06233eeef580be8accdf05cc31d947ba45f9c62 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 30 Sep 2024 16:03:57 +0100 Subject: [PATCH 070/166] fixed recommendation utils unit tests --- .../tests/test_recommendation_utils.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/recommendations/tests/test_recommendation_utils.py b/recommendations/tests/test_recommendation_utils.py index 559a51b2..543d5319 100644 --- a/recommendations/tests/test_recommendation_utils.py +++ b/recommendations/tests/test_recommendation_utils.py @@ -88,8 +88,8 @@ class TestRecommendationUtils: def test_get_roof_u_value_case_3(self): inputs = { - 'original_description': 'Room-in-roof, 200 mm insulation at rafters', - 'clean_description': 'Room-in-roof, 200 mm insulation at rafters', + 'original_description': 'Room-in-roof, insulated at rafters', + 'clean_description': 'Room-in-roof, insulated at rafters', 'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_pitched': False, @@ -101,12 +101,12 @@ class TestRecommendationUtils: 'is_assumed': False, 'has_dwelling_above': False, 'is_valid': True, - 'insulation_thickness': '200', + 'insulation_thickness': 'average', 'age_band': "J" } u_value = recommendation_utils.get_roof_u_value(**inputs) - assert u_value == 0.21, f"Expected 0.21, but got {u_value}" + assert u_value == 0.4, f"Expected 0.4, but got {u_value}" def test_get_roof_u_value_case_4(self): inputs = { @@ -179,8 +179,8 @@ class TestRecommendationUtils: def test_get_roof_u_value_case_7(self): # Test case where the roof has a room in it inputs = { - 'original_description': 'Pitched, room-in-roof, 100mm insulation', - 'clean_description': 'Pitched, room-in-roof, 100mm insulation', + 'original_description': 'Pitched, room-in-roof, above average insulation', + 'clean_description': 'Pitched, room-in-roof, above average insulation', 'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_pitched': True, @@ -192,12 +192,12 @@ class TestRecommendationUtils: 'is_assumed': False, 'has_dwelling_above': False, 'is_valid': True, - 'insulation_thickness': '100', + 'insulation_thickness': 'above average', 'age_band': "J" } u_value = recommendation_utils.get_roof_u_value(**inputs) - assert u_value == 0.40, f"Expected 0.40, but got {u_value}" + assert u_value == 0.16, f"Expected 0.16, but got {u_value}" def test_get_roof_u_value_case_8(self): # Test case where there is a dwelling above the roof, U-value should be 0 From f7c01951b12a2d1e82ff148bcac4b20ab4f8eeae Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 30 Sep 2024 16:09:47 +0100 Subject: [PATCH 071/166] fixing recommendation utils tests --- recommendations/tests/test_recommendation_utils.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/recommendations/tests/test_recommendation_utils.py b/recommendations/tests/test_recommendation_utils.py index 543d5319..c42655eb 100644 --- a/recommendations/tests/test_recommendation_utils.py +++ b/recommendations/tests/test_recommendation_utils.py @@ -437,7 +437,6 @@ def test_estimate_windows(): construction_age_band="England and Wales: 1976-1982", floor_area=37, number_habitable_rooms=2, - extension_count=0, ) assert windows_case_1 == 4, f"Expected 4 windows, got {windows_case_1}" @@ -450,7 +449,6 @@ def test_estimate_windows(): construction_age_band="England and Wales: 1950-1966", floor_area=69, number_habitable_rooms=4, - extension_count=0, ) assert windows_case_2 == 6, f"Expected 6 windows, got {windows_case_2}" @@ -463,7 +461,6 @@ def test_estimate_windows(): construction_age_band="England and Wales: 1967-1975", floor_area=56, number_habitable_rooms=3, - extension_count=0, ) assert windows_case_3 == 5, f"Expected 5 windows, got {windows_case_3}" @@ -476,7 +473,6 @@ def test_estimate_windows(): construction_age_band="England and Wales: 1967-1975", floor_area=77.28, number_habitable_rooms=4, - extension_count=0, ) assert windows_case_4 == 7, f"Expected 7 windows, got {windows_case_4}" @@ -489,7 +485,6 @@ def test_estimate_windows(): construction_age_band="England and Wales: 1950-1966", floor_area=88.4, number_habitable_rooms=5, - extension_count=0, ) assert windows_case_5 == 12, f"Expected 12 windows, got {windows_case_5}" @@ -502,7 +497,6 @@ def test_estimate_windows(): construction_age_band="", floor_area=100, number_habitable_rooms=3, - extension_count=0, ) assert windows_case_6 == 5, f"Expected 5 windows, got {windows_case_6}" @@ -514,7 +508,6 @@ def test_estimate_windows(): construction_age_band="England and Wales: 1967-1975", floor_area=85, number_habitable_rooms=4, - extension_count=0, ) assert windows_case_7 == 10, f"Expected 10 windows, got {windows_case_7}" @@ -526,7 +519,6 @@ def test_estimate_windows(): construction_age_band="", floor_area=50, number_habitable_rooms=3, - extension_count=0, ) assert windows_case_8 == 5, f"Expected 5 windows, got {windows_case_8}" From bb5e8376cd8e9c27fc058b1fecf0af7aae8a5814 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 30 Sep 2024 16:19:07 +0100 Subject: [PATCH 072/166] fixed roof recommendations tests --- .../tests/test_roof_recommendations.py | 43 ++++++++++--------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/recommendations/tests/test_roof_recommendations.py b/recommendations/tests/test_roof_recommendations.py index 3d555a4f..e7a4c540 100644 --- a/recommendations/tests/test_roof_recommendations.py +++ b/recommendations/tests/test_roof_recommendations.py @@ -28,9 +28,10 @@ class TestRoofRecommendations: assert not roof_recommender.recommendations - roof_recommender.recommend() + roof_recommender.recommend(phase=0) - assert len(roof_recommender.recommendations) + assert len(roof_recommender.recommendations) == 1 + assert roof_recommender.recommendations[0]["parts"][0]["depth"] == 300 def test_loft_insulation_recommendation_50mm_insulation(self): epc_record = EPCRecord() @@ -52,13 +53,14 @@ class TestRoofRecommendations: assert not roof_recommender2.recommendations - roof_recommender2.recommend() + roof_recommender2.recommend(phase=0) assert len(roof_recommender2.recommendations) == 1 - assert roof_recommender2.recommendations[0]["total"] == 1936.9206000000004 + assert roof_recommender2.recommendations[0]["total"] == 1610.0000000000002 assert roof_recommender2.recommendations[0]["new_u_value"] == 0.14 assert roof_recommender2.recommendations[0]["starting_u_value"] == 0.68 + assert roof_recommender2.recommendations[0]["parts"][0]["depth"] == 270 epc_record = EPCRecord() epc_record.prepared_epc = {"county": "Greater London Authority"} @@ -79,7 +81,7 @@ class TestRoofRecommendations: assert not roof_recommender3.recommendations - roof_recommender3.recommend() + roof_recommender3.recommend(phase=0) assert roof_recommender3.recommendations assert len(roof_recommender3.recommendations) == 1 @@ -105,14 +107,14 @@ class TestRoofRecommendations: assert not roof_recommender4.recommendations - roof_recommender4.recommend() + roof_recommender4.recommend(phase=0) - assert len(roof_recommender4.recommendations) == 4 + assert len(roof_recommender4.recommendations) == 1 - assert roof_recommender4.recommendations[0]["total"] == 1128.744 - assert roof_recommender4.recommendations[0]["new_u_value"] == 0.15 + assert roof_recommender4.recommendations[0]["total"] == 1552.5 + assert roof_recommender4.recommendations[0]["new_u_value"] == 0.13 assert roof_recommender4.recommendations[0]["starting_u_value"] == 0.3 - assert roof_recommender4.recommendations[0]["parts"][0]["depth"] == 150 + assert roof_recommender4.recommendations[0]["parts"][0]["depth"] == 200 epc_record = EPCRecord() epc_record.prepared_epc = {"county": "Somerset"} @@ -133,12 +135,11 @@ class TestRoofRecommendations: assert not roof_recommender5.recommendations - roof_recommender5.recommend() + roof_recommender5.recommend(phase=0) - # The 150mm insulation should be selected, since there it already 150mm assert roof_recommender5.recommendations - assert len(roof_recommender5.recommendations) == 4 - assert roof_recommender5.recommendations[0]["parts"][0]["depth"] == 150 + assert len(roof_recommender5.recommendations) == 1 + assert roof_recommender5.recommendations[0]["parts"][0]["depth"] == 200 def test_loft_insulation_recommendation_270mm_insulation(self): # We shouldn't recommend anything in this case @@ -161,7 +162,7 @@ class TestRoofRecommendations: assert not roof_recommender6.recommendations - roof_recommender6.recommend() + roof_recommender6.recommend(phase=0) assert len(roof_recommender6.recommendations) == 0 @@ -302,12 +303,12 @@ class TestRoofRecommendations: assert not roof_recommender11.recommendations - roof_recommender11.recommend() + roof_recommender11.recommend(phase=0) assert len(roof_recommender11.recommendations) == 1 assert roof_recommender11.recommendations[0]["parts"][0]["depth"] == 150 - assert roof_recommender11.recommendations[0]["total"] == 4380.84324 + assert roof_recommender11.recommendations[0]["total"] == 6532.5 assert roof_recommender11.recommendations[0]["new_u_value"] == 0.14 assert roof_recommender11.recommendations[0]["starting_u_value"] == 2.3 assert roof_recommender11.recommendations[0]["description"] == \ @@ -334,7 +335,7 @@ class TestRoofRecommendations: assert not roof_recommender12.recommendations - roof_recommender12.recommend() + roof_recommender12.recommend(phase=0) assert not roof_recommender12.recommendations @@ -358,13 +359,13 @@ class TestRoofRecommendations: assert not roof_recommender13.recommendations - roof_recommender13.recommend() + roof_recommender13.recommend(phase=0) assert len(roof_recommender13.recommendations) == 1 assert roof_recommender13.recommendations[0]["parts"][0]["depth"] == 150 - assert roof_recommender13.recommendations[0]["total"] == 5199.969120000002 + assert roof_recommender13.recommendations[0]["total"] == 7800 assert roof_recommender13.recommendations[0]["new_u_value"] == 0.14 assert roof_recommender13.recommendations[0]["starting_u_value"] == 2.3 @@ -390,6 +391,6 @@ class TestRoofRecommendations: assert not roof_recommender14.recommendations - roof_recommender14.recommend() + roof_recommender14.recommend(phase=0) assert not roof_recommender14.recommendations From 973d7ab6ee46572a0474d1473d89aff04dba9226 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 30 Sep 2024 16:26:36 +0100 Subject: [PATCH 073/166] fixing further roof recommendations tests --- recommendations/RoofRecommendations.py | 15 +---- .../tests/test_roof_recommendations.py | 58 +++++++++---------- 2 files changed, 28 insertions(+), 45 deletions(-) diff --git a/recommendations/RoofRecommendations.py b/recommendations/RoofRecommendations.py index fbd99d67..2a77a3a5 100644 --- a/recommendations/RoofRecommendations.py +++ b/recommendations/RoofRecommendations.py @@ -446,20 +446,7 @@ class RoofRecommendations: _, new_u_value = calculate_u_value_uplift(u_value, part_u_value) new_u_value = math.ceil(new_u_value * 100.0) / 100.0 - # If I have a lowest U value and my new u value is higher than that but lower than the - # diminishing returns threshold, it can be considered - - # If I have a lowest U value and my new u value is lower than the lowest value, it's - # further into the diminishing returns threshold and can shouldn't be - - # if is_diminishing_returns( - # recommendations, new_u_value, lowest_selected_u_value, self.DIMINISHING_RETURNS_U_VALUE - # ): - # continue - # We allow a small tolerance for error so we don't discount the recommendation entirely - # if new_u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE: - # lowest_selected_u_value = update_lowest_selected_u_value(lowest_selected_u_value, new_u_value) estimated_cost = ( cost_per_unit * self.property.insulation_floor_area if @@ -504,7 +491,7 @@ class RoofRecommendations: "type": "room_roof_insulation", "description": "Insulate room in roof at rafters and re-decorate", "starting_u_value": u_value, - "new_u_value": None, + "new_u_value": new_u_value, "sap_points": sap_points, "simulation_config": simulation_config, "description_simulation": { diff --git a/recommendations/tests/test_roof_recommendations.py b/recommendations/tests/test_roof_recommendations.py index e7a4c540..910aef02 100644 --- a/recommendations/tests/test_roof_recommendations.py +++ b/recommendations/tests/test_roof_recommendations.py @@ -166,37 +166,33 @@ class TestRoofRecommendations: assert len(roof_recommender6.recommendations) == 0 - # def test_uninsulated_room_in_roof(self): - # property_instance7 = Property(id=0, address1="fake", postcode="fake", epc_client=Mock()) - # property_instance7.age_band = "F" - # property_instance7.insulation_floor_area = 100 - # property_instance7.roof = { - # 'original_description': 'Roof room(s), no insulation (assumed)', - # 'clean_description': 'Roof room(s), no insulation', - # 'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_pitched': False, - # 'is_roof_room': True, 'is_loft': False, 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False, - # 'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True, 'insulation_thickness': 'none' - # } - # - # property_instance7.pitched_roof_area = 110 - # property_instance7.data = {"county": "Southampton"} - # - # roof_recommender7 = RoofRecommendations(property_instance=property_instance7, materials=materials) - # - # assert not roof_recommender7.recommendations - # - # roof_recommender7.recommend() - # - # # Even though we have 3 depths, we only end with 1 due to diminishin returns - # assert len(roof_recommender7.recommendations) == 1 - # - # assert roof_recommender7.recommendations[0]["parts"][0]["depths"] == [270] - # - # assert roof_recommender7.recommendations[0]["new_u_value"] == 0.14 - # assert roof_recommender7.recommendations[0]["starting_u_value"] == 0.8 - # assert roof_recommender7.recommendations[0]["description"] == \ - # "Insulate your room roof with 270mm of Example room roof insulation" - # + def test_uninsulated_room_in_roof(self): + epc_record = EPCRecord() + epc_record.prepared_epc = {"county": "Southampton", "roof-energy-eff": "Very Poor"} + property_instance7 = Property(id=0, address="fake", postcode="fake", epc_record=epc_record) + property_instance7.age_band = "F" + property_instance7.insulation_floor_area = 100 + property_instance7.roof = { + 'original_description': 'Roof room(s), no insulation (assumed)', + 'clean_description': 'Roof room(s), no insulation', + 'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_pitched': False, + 'is_roof_room': True, 'is_loft': False, 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False, + 'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True, 'insulation_thickness': 'none' + } + + property_instance7.pitched_roof_area = 110 + + roof_recommender7 = RoofRecommendations(property_instance=property_instance7, materials=materials) + + assert not roof_recommender7.recommendations + + roof_recommender7.recommend(phase=0) + + assert len(roof_recommender7.recommendations) == 1 + assert roof_recommender7.recommendations[0]["new_u_value"] == 0.23 + assert roof_recommender7.recommendations[0]["starting_u_value"] == 1.5 + assert roof_recommender7.recommendations[0]["description"] == "Insulate room in roof at rafters and re-decorate" + # def test_ceiling_insulated_room_in_roof(self): # property_instance8 = Property(id=8, address1="fake", postcode="fake", epc_client=Mock()) # property_instance8.age_band = "F" From a565d63836584c0a0da2c6d20fdcad2c28fb3c05 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 30 Sep 2024 16:31:57 +0100 Subject: [PATCH 074/166] Added addition rir recommendation unit tests --- .../tests/test_roof_recommendations.py | 170 +++++++++--------- 1 file changed, 84 insertions(+), 86 deletions(-) diff --git a/recommendations/tests/test_roof_recommendations.py b/recommendations/tests/test_roof_recommendations.py index 910aef02..139975bd 100644 --- a/recommendations/tests/test_roof_recommendations.py +++ b/recommendations/tests/test_roof_recommendations.py @@ -193,92 +193,90 @@ class TestRoofRecommendations: assert roof_recommender7.recommendations[0]["starting_u_value"] == 1.5 assert roof_recommender7.recommendations[0]["description"] == "Insulate room in roof at rafters and re-decorate" - # def test_ceiling_insulated_room_in_roof(self): - # property_instance8 = Property(id=8, address1="fake", postcode="fake", epc_client=Mock()) - # property_instance8.age_band = "F" - # property_instance8.insulation_floor_area = 100 - # property_instance8.roof = { - # 'original_description': 'Roof room(s), ceiling insulated', - # 'clean_description': 'Roof room(s), ceiling insulated', - # 'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_pitched': False, - # 'is_roof_room': True, 'is_loft': False, 'is_flat': False, 'is_thatched': False, - # 'is_at_rafters': False, - # 'is_assumed': False, 'has_dwelling_above': False, 'is_valid': True, - # 'insulation_thickness': 'average' - # } - # - # property_instance8.pitched_roof_area = 110 - # - # roof_recommender8 = RoofRecommendations(property_instance=property_instance8, materials=materials) - # - # assert not roof_recommender8.recommendations - # - # roof_recommender8.recommend() - # - # # No recommendations in this case - # assert not roof_recommender8.recommendations - # - # def test_insulated_room_in_roof(self): - # property_instance9 = Property(id=9, address1="fake", postcode="fake", epc_client=Mock()) - # property_instance9.age_band = "F" - # property_instance9.insulation_floor_area = 100 - # property_instance9.roof = { - # 'original_description': 'Roof room(s), insulated (assumed)', - # 'clean_description': 'Roof room(s), insulated', - # 'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_pitched': False, - # 'is_roof_room': True, 'is_loft': False, 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False, - # 'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True, 'insulation_thickness': 'average' - # } - # - # property_instance9.pitched_roof_area = 110 - # property_instance9.data = {"county": "Rutland"} - # - # roof_recommender9 = RoofRecommendations(property_instance=property_instance9, materials=materials) - # - # assert not roof_recommender9.recommendations - # - # roof_recommender9.recommend() - # - # # No recommendations in this case - # assert not roof_recommender9.recommendations - # - # def test_limited_insulated_room_in_roof(self): - # property_instance10 = Property(id=10, address1="fake", postcode="fake", epc_client=Mock()) - # property_instance10.age_band = "F" - # property_instance10.insulation_floor_area = 100 - # property_instance10.roof = { - # 'original_description': 'Roof room(s), limited insulation (assumed)', - # 'clean_description': 'Roof room(s), limited insulation', - # 'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_pitched': False, - # 'is_roof_room': True, 'is_loft': False, 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False, - # 'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True, - # 'insulation_thickness': 'below average' - # } - # - # property_instance10.pitched_roof_area = 110 - # property_instance10.data = {"county": "Westmorland"} - # - # roof_recommender10 = RoofRecommendations(property_instance=property_instance10, materials=materials) - # - # assert not roof_recommender10.recommendations - # - # roof_recommender10.recommend() - # - # assert len(roof_recommender10.recommendations) == 2 - # - # assert roof_recommender10.recommendations[0]["parts"][0]["depths"] == [220] - # assert roof_recommender10.recommendations[1]["parts"][0]["depths"] == [270] - # - # assert roof_recommender10.recommendations[0]["new_u_value"] == 0.16 - # assert roof_recommender10.recommendations[1]["new_u_value"] == 0.14 - # - # assert roof_recommender10.recommendations[0]["starting_u_value"] == 0.8 - # assert roof_recommender10.recommendations[1]["starting_u_value"] == 0.8 - # - # assert roof_recommender10.recommendations[0]["description"] == \ - # "Insulate your room roof with 220mm of Example room roof insulation" - # assert roof_recommender10.recommendations[1]["description"] == \ - # "Insulate your room roof with 270mm of Example room roof insulation" + def test_ceiling_insulated_room_in_roof(self): + epc_record = EPCRecord() + epc_record.prepared_epc = {"county": "Southampton", "roof-energy-eff": "Very Poor"} + property_instance8 = Property(id=8, address="fake", postcode="fake", epc_record=epc_record) + property_instance8.age_band = "F" + property_instance8.insulation_floor_area = 100 + property_instance8.roof = { + 'original_description': 'Roof room(s), ceiling insulated', + 'clean_description': 'Roof room(s), ceiling insulated', + 'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_pitched': False, + 'is_roof_room': True, 'is_loft': False, 'is_flat': False, 'is_thatched': False, + 'is_at_rafters': False, + 'is_assumed': False, 'has_dwelling_above': False, 'is_valid': True, + 'insulation_thickness': 'average' + } + + property_instance8.pitched_roof_area = 110 + + roof_recommender8 = RoofRecommendations(property_instance=property_instance8, materials=materials) + + assert not roof_recommender8.recommendations + + roof_recommender8.recommend(phase=0) + + # No recommendations in this case + assert not roof_recommender8.recommendations + + def test_insulated_room_in_roof(self): + epc_record = EPCRecord() + epc_record.prepared_epc = {"county": "Southampton", "roof-energy-eff": "Very Poor"} + property_instance9 = Property(id=9, address="fake", postcode="fake", epc_record=epc_record) + property_instance9.age_band = "F" + property_instance9.insulation_floor_area = 100 + property_instance9.roof = { + 'original_description': 'Roof room(s), insulated (assumed)', + 'clean_description': 'Roof room(s), insulated', + 'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_pitched': False, + 'is_roof_room': True, 'is_loft': False, 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False, + 'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True, 'insulation_thickness': 'average' + } + + property_instance9.pitched_roof_area = 110 + property_instance9.data = {"county": "Rutland"} + + roof_recommender9 = RoofRecommendations(property_instance=property_instance9, materials=materials) + + assert not roof_recommender9.recommendations + + roof_recommender9.recommend(phase=0) + + # No recommendations in this case + assert not roof_recommender9.recommendations + + def test_limited_insulated_room_in_roof(self): + epc_record = EPCRecord() + epc_record.prepared_epc = {"county": "Westmorland", "roof-energy-eff": "Poor"} + property_instance10 = Property(id=10, address="fake", postcode="fake", epc_record=epc_record) + property_instance10.age_band = "F" + property_instance10.insulation_floor_area = 100 + property_instance10.roof = { + 'original_description': 'Roof room(s), limited insulation (assumed)', + 'clean_description': 'Roof room(s), limited insulation', + 'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_pitched': False, + 'is_roof_room': True, 'is_loft': False, 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False, + 'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True, + 'insulation_thickness': 'below average' + } + + property_instance10.pitched_roof_area = 110 + + roof_recommender10 = RoofRecommendations(property_instance=property_instance10, materials=materials) + + assert not roof_recommender10.recommendations + + roof_recommender10.recommend(phase=0) + + assert len(roof_recommender10.recommendations) == 1 + + assert roof_recommender10.recommendations[0]["new_u_value"] == 0.19 + + assert roof_recommender10.recommendations[0]["starting_u_value"] == 0.68 + + assert (roof_recommender10.recommendations[0]["description"] == + 'Insulate room in roof at rafters and re-decorate') def test_flat_no_insulation(self): epc_record = EPCRecord() From 8410d04dad2dddc763c9448258ee124f1dd4eda2 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 30 Sep 2024 17:13:49 +0100 Subject: [PATCH 075/166] deleted broken solars --- .../tests/test_solar_pv_recommendations.py | 376 ++---------------- 1 file changed, 40 insertions(+), 336 deletions(-) diff --git a/recommendations/tests/test_solar_pv_recommendations.py b/recommendations/tests/test_solar_pv_recommendations.py index fbbfe3a1..05349f9c 100644 --- a/recommendations/tests/test_solar_pv_recommendations.py +++ b/recommendations/tests/test_solar_pv_recommendations.py @@ -50,360 +50,64 @@ class TestSolarPvRecommendations: epc_record = EPCRecord() epc_record.prepared_epc = {"property-type": "House", "photo-supply": None, "county": "Huntingdonshire"} property_instance_valid_all = Property(id=1, address="", postcode="", epc_record=epc_record) - property_instance_valid_all.solar_pv_roof_area = 20 - property_instance_valid_all.solar_pv_percentage = 40 + property_instance_valid_all.roof_area = 40 + property_instance_valid_all.number_of_floors = 2 property_instance_valid_all.roof = {"is_flat": True} + property_instance_valid_all.solar_panel_configuration = { + "panel_performance": pd.DataFrame( + [ + { + "panneled_roof_area": 20, + "n_panels": 10, + "array_wattage": 4000, + "initial_ac_kwh_per_year": 3800 + } + ] + ) + } + return property_instance_valid_all def test_invalid_property_type(self, property_instance_invalid_type): solar_pv = SolarPvRecommendations(property_instance_invalid_type) - solar_pv.recommend() + solar_pv.recommend(phase=0) assert not solar_pv.recommendation def test_invalid_roof_type(self, property_instance_invalid_roof): solar_pv = SolarPvRecommendations(property_instance_invalid_roof) - solar_pv.recommend() + solar_pv.recommend(phase=0) assert not solar_pv.recommendation def test_existing_solar_pv(self, property_instance_has_solar_pv): solar_pv = SolarPvRecommendations(property_instance_has_solar_pv) - solar_pv.recommend() + solar_pv.recommend(phase=0) assert not solar_pv.recommendation def test_valid_all_conditions(self, property_instance_valid_all): solar_pv = SolarPvRecommendations(property_instance_valid_all) - solar_pv.recommend() + solar_pv.recommend(phase=0) assert solar_pv.recommendation == [ { - 'parts': [], - 'type': 'solar_pv', - 'description': 'Install a 4 kilowatt-peak (kWp) solar photovoltaic (PV) panel system on the roof', - 'starting_u_value': None, - 'new_u_value': None, - 'sap_points': None, - 'total': 8527.0752, - 'subtotal': 7105.896, - 'vat': 1421.1791999999996, - 'labour_hours': 72, - 'labour_days': 2, - 'photo_supply': 4000 + 'phase': 0, 'parts': [], 'type': 'solar_pv', + 'description': 'Install a 4.0 kilowatt-peak (kWp) solar photovoltaic (PV) panel system on 50% the ' + 'roof.', + 'starting_u_value': None, 'new_u_value': None, 'sap_points': None, 'already_installed': False, + 'total': 4850.0, 'subtotal': 4041.666666666667, 'vat': 808.333333333333, 'labour_hours': 48, + 'labour_days': 2, 'photo_supply': 50.0, 'has_battery': False, 'initial_ac_kwh_per_year': 3800, + 'description_simulation': {'photo-supply': 50.0} + }, + { + 'phase': 0, 'parts': [], 'type': 'solar_pv', + 'description': 'Install a 4.0 kilowatt-peak (kWp) ' + 'solar photovoltaic (PV) panel system ' + 'on 50% the roof, with a battery ' + 'storage system.', + 'starting_u_value': None, 'new_u_value': None, + 'sap_points': None, 'already_installed': False, + 'total': 7550.0, 'subtotal': 6291.666666666667, + 'vat': 1258.333333333333, 'labour_hours': 48, + 'labour_days': 2, 'photo_supply': 50.0, + 'has_battery': True, 'initial_ac_kwh_per_year': 3800, + 'description_simulation': {'photo-supply': 50.0} } ] - - def test_model(self): - """ - This function tests the recommendation engine, in conjunction with the model - :return: - """ - - starting_epc = { - 'low-energy-fixed-light-count': '', 'address': '27 Cromwell Street', 'uprn-source': 'Energy Assessor', - 'floor-height': '2.5', 'heating-cost-potential': '443', 'unheated-corridor-length': '', - 'hot-water-cost-potential': '53', 'construction-age-band': 'England and Wales: before 1900', - 'potential-energy-rating': 'B', 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Average', - 'lighting-energy-eff': 'Very Poor', 'environment-impact-potential': '85', - 'glazed-type': 'double glazing installed before 2002', 'heating-cost-current': '904', 'address3': '', - 'mainheatcont-description': 'Programmer, room thermostat and TRVs', 'sheating-energy-eff': 'N/A', - 'property-type': 'House', 'local-authority-label': 'West Lindsey', 'fixed-lighting-outlets-count': '10', - 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '79', - 'county': 'Lincolnshire', 'postcode': 'DN21 1DH', 'solar-water-heating-flag': 'N', - 'constituency': 'E14000707', 'co2-emissions-potential': '1.5', 'number-heated-rooms': '5', - 'floor-description': 'Suspended, no insulation (assumed)', 'energy-consumption-potential': '92', - 'local-authority': 'E07000142', 'built-form': 'Mid-Terrace', 'number-open-fireplaces': '0', - 'windows-description': 'Fully double glazed', 'glazed-area': 'Normal', 'inspection-date': '2021-11-17', - 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '61', 'address1': '27 Cromwell Street', - 'heat-loss-corridor': '', 'flat-storey-count': '', 'constituency-label': 'Gainsborough', - 'roof-energy-eff': 'Very Poor', 'total-floor-area': '89.0', 'building-reference-number': '10001989430', - 'environment-impact-current': '47', 'co2-emissions-current': '5.4', - 'roof-description': 'Pitched, no insulation (assumed)', 'floor-energy-eff': 'N/A', - 'number-habitable-rooms': '5', 'address2': '', 'hot-water-env-eff': 'Good', 'posttown': 'GAINSBOROUGH', - 'mainheatc-energy-eff': 'Good', 'main-fuel': 'mains gas (not community)', 'lighting-env-eff': 'Very Poor', - 'windows-energy-eff': 'Average', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A', - 'lighting-description': 'No low energy lighting', 'roof-env-eff': 'Very Poor', - 'walls-energy-eff': 'Very Poor', 'photo-supply': '0.0', 'lighting-cost-potential': '72', - 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '100', 'main-heating-controls': '', - 'lodgement-datetime': '2021-12-01 10:12:23', 'flat-top-storey': '', 'current-energy-rating': 'E', - 'secondheat-description': 'Room heaters, mains gas', 'walls-env-eff': 'Very Poor', - 'transaction-type': 'ECO assessment', 'uprn': '100030949912', 'current-energy-efficiency': '54', - 'energy-consumption-current': '346', 'mainheat-description': 'Boiler and radiators, mains gas', - 'lighting-cost-current': '144', 'lodgement-date': '2021-12-01', 'extension-count': '2', - 'mainheatc-env-eff': 'Good', 'lmk-key': '3ec5533af02ec78361c1f9bea8dd2e878c2c6fa6cf59e5cc505c3eeb038e0f91', - 'wind-turbine-count': '0', 'tenure': 'Owner-occupied', 'floor-level': '', - 'potential-energy-efficiency': '86', 'hot-water-energy-eff': 'Good', 'low-energy-lighting': '0', - 'walls-description': 'Solid brick, as built, no insulation (assumed)', - 'hotwater-description': 'From main system' - } - - ending_epc = { - 'low-energy-fixed-light-count': '', 'address': '27 Cromwell Street', 'uprn-source': 'Energy Assessor', - 'floor-height': '2.5', 'heating-cost-potential': '443', 'unheated-corridor-length': '', - 'hot-water-cost-potential': '53', 'construction-age-band': 'England and Wales: before 1900', - 'potential-energy-rating': 'B', 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Average', - 'lighting-energy-eff': 'Very Poor', 'environment-impact-potential': '86', - 'glazed-type': 'double glazing installed before 2002', 'heating-cost-current': '904', 'address3': '', - 'mainheatcont-description': 'Programmer, room thermostat and TRVs', 'sheating-energy-eff': 'N/A', - 'property-type': 'House', 'local-authority-label': 'West Lindsey', 'fixed-lighting-outlets-count': '10', - 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '79', - 'county': 'Lincolnshire', 'postcode': 'DN21 1DH', 'solar-water-heating-flag': 'N', - 'constituency': 'E14000707', 'co2-emissions-potential': '1.4', 'number-heated-rooms': '5', - 'floor-description': 'Suspended, no insulation (assumed)', 'energy-consumption-potential': '84', - 'local-authority': 'E07000142', 'built-form': 'Mid-Terrace', 'number-open-fireplaces': '0', - 'windows-description': 'Fully double glazed', 'glazed-area': 'Normal', 'inspection-date': '2021-12-21', - 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '49', 'address1': '27 Cromwell Street', - 'heat-loss-corridor': '', 'flat-storey-count': '', 'constituency-label': 'Gainsborough', - 'roof-energy-eff': 'Very Poor', 'total-floor-area': '89.0', 'building-reference-number': '10001989430', - 'environment-impact-current': '55', 'co2-emissions-current': '4.4', - 'roof-description': 'Pitched, no insulation (assumed)', 'floor-energy-eff': 'N/A', - 'number-habitable-rooms': '5', 'address2': '', 'hot-water-env-eff': 'Good', 'posttown': 'GAINSBOROUGH', - 'mainheatc-energy-eff': 'Good', 'main-fuel': 'mains gas (not community)', 'lighting-env-eff': 'Very Poor', - 'windows-energy-eff': 'Average', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A', - 'lighting-description': 'No low energy lighting', 'roof-env-eff': 'Very Poor', - 'walls-energy-eff': 'Very Poor', 'photo-supply': '50.0', 'lighting-cost-potential': '72', - 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '100', 'main-heating-controls': '', - 'lodgement-datetime': '2021-12-21 17:33:09', 'flat-top-storey': '', 'current-energy-rating': 'D', - 'secondheat-description': 'Room heaters, mains gas', 'walls-env-eff': 'Very Poor', - 'transaction-type': 'ECO assessment', 'uprn': '100030949912', 'current-energy-efficiency': '65', - 'energy-consumption-current': '277', 'mainheat-description': 'Boiler and radiators, mains gas', - 'lighting-cost-current': '144', 'lodgement-date': '2021-12-21', 'extension-count': '2', - 'mainheatc-env-eff': 'Good', 'lmk-key': 'b0b19583c59afbc69db12f4d6c98cd8837e80da3214d577c426eb3e672d424fc', - 'wind-turbine-count': '0', 'tenure': 'Owner-occupied', 'floor-level': '', - 'potential-energy-efficiency': '88', 'hot-water-energy-eff': 'Good', 'low-energy-lighting': '0', - 'walls-description': 'Solid brick, as built, no insulation (assumed)', - 'hotwater-description': 'From main system' - } - - cleaning_data = read_dataframe_from_s3_parquet( - bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet", - ) - - cleaned = read_from_s3( - s3_file_name="cleaned_epc_data/cleaned.bson", - bucket_name="retrofit-data-dev" - ) - cleaned = msgpack.unpackb(cleaned, raw=False) - - photo_supply_lookup, floor_area_decile_thresholds = SolarPhotoSupply.load(bucket="retrofit-data-dev") - - epc = EPCRecord( - epc_records={ - 'original_epc': starting_epc, - 'full_sap_epc': {}, - 'old_data': [] - }, - run_mode="newdata", - cleaning_data=cleaning_data - ) - - home = Property( - id=0, - address="", - postcode="", - epc_record=epc, - already_installed={}, - non_invasive_recommendations={}, - ) - home.in_conservation_area = False - home.is_listed = False - home.is_heritage = False - home.restricted_measures = True - home.get_components( - cleaned=cleaned, - photo_supply_lookup=photo_supply_lookup, - floor_area_decile_thresholds=floor_area_decile_thresholds - ) - - recommender = SolarPvRecommendations(property_instance=home) - recommender.recommend(phase=0) - - coverage_50_percent = [x for x in recommender.recommendation if x["photo_supply"] == 50] - assert len(coverage_50_percent) == 2 - - property_recommendations = Recommendations.insert_temp_recommendation_id([coverage_50_percent]) - - home.create_base_difference_epc_record(cleaned_lookup=cleaned) - home.adjust_difference_record_with_recommendations( - property_recommendations, [] - ) - - scoring_data = pd.DataFrame(home.recommendations_scoring_data).drop( - columns=["rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending", - "carbon_ending"] - ) - - model_api = ModelApi(portfolio_id="ashp-test", timestamp=datetime.now().isoformat()) - model_api.MODEL_PREFIXES = ["sap_change_predictions"] - - predictions_dict = model_api.predict_all( - df=scoring_data, - bucket="retrofit-data-dev", - prediction_buckets={ - "sap_change_predictions": "retrofit-sap-predictions-dev", - } - ) - - assert predictions_dict["sap_change_predictions"]["predictions"].tolist() == [65.9, 65.9] - assert ending_epc["current-energy-efficiency"] == '65' - - def test_model2(self): - data[["uprn", "sap_ending"]] - # - - searcher = SearchEpc( - address1="", - postcode="", - auth_token="a2Nvbm5rb3dsZXNzYXJAZ21haWwuY29tOjY5MGJiMWM0NmIyOGI5ZDUxYzAxMzQzYzNiZGNlZGJjZDNmODQwMzA=", - os_api_key="", - full_address="", - uprn=100030952942, - ) - searcher.find_property(False) - - ending_epc = { - 'low-energy-fixed-light-count': '', 'address': '6 Kenmare Crescent', - 'uprn-source': 'Energy Assessor', 'floor-height': '2.49', 'heating-cost-potential': '464', - 'unheated-corridor-length': '', 'hot-water-cost-potential': '46', - 'construction-age-band': 'England and Wales: 1967-1975', 'potential-energy-rating': 'B', - 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Average', 'lighting-energy-eff': 'Very Good', - 'environment-impact-potential': '91', 'glazed-type': 'not defined', 'heating-cost-current': '535', - 'address3': '', 'mainheatcont-description': 'Programmer, room thermostat and TRVs', - 'sheating-energy-eff': 'N/A', 'property-type': 'Bungalow', - 'local-authority-label': 'West Lindsey', 'fixed-lighting-outlets-count': '9', - 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '69', - 'county': 'Lincolnshire', 'postcode': 'DN21 1PR', 'solar-water-heating-flag': 'N', - 'constituency': 'E14000707', 'co2-emissions-potential': '0.7', 'number-heated-rooms': '3', - 'floor-description': 'Suspended, no insulation (assumed)', 'energy-consumption-potential': '56', - 'local-authority': 'E07000142', 'built-form': 'Semi-Detached', 'number-open-fireplaces': '0', - 'windows-description': 'Fully double glazed', 'glazed-area': 'Much More Than Typical', - 'inspection-date': '2022-08-24', 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '18', - 'address1': '6 Kenmare Crescent', 'heat-loss-corridor': '', 'flat-storey-count': '', - 'constituency-label': 'Gainsborough', 'roof-energy-eff': 'Very Good', 'total-floor-area': '66.0', - 'building-reference-number': '10002845316', 'environment-impact-current': '85', - 'co2-emissions-current': '1.2', 'roof-description': 'Pitched, 300 mm loft insulation', - 'floor-energy-eff': 'N/A', 'number-habitable-rooms': '3', 'address2': '', - 'hot-water-env-eff': 'Good', 'posttown': 'GAINSBOROUGH', 'mainheatc-energy-eff': 'Good', - 'main-fuel': 'mains gas (not community)', 'lighting-env-eff': 'Very Good', - 'windows-energy-eff': 'Average', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A', - 'lighting-description': 'Low energy lighting in all fixed outlets', 'roof-env-eff': 'Very Good', - 'walls-energy-eff': 'Average', 'photo-supply': '40.0', 'lighting-cost-potential': '65', - 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '100', 'main-heating-controls': '', - 'lodgement-datetime': '2022-08-24 15:39:42', 'flat-top-storey': '', 'current-energy-rating': 'B', - 'secondheat-description': 'Room heaters, electric', 'walls-env-eff': 'Average', - 'transaction-type': 'ECO assessment', 'uprn': '100030952942', 'current-energy-efficiency': '87', - 'energy-consumption-current': '100', 'mainheat-description': 'Boiler and radiators, mains gas', - 'lighting-cost-current': '65', 'lodgement-date': '2022-08-24', 'extension-count': '0', - 'mainheatc-env-eff': 'Good', - 'lmk-key': 'e20be883431b1fed15db7fa1f52634fb7655d2b80c2fdad37df779f93ec4dafd', - 'wind-turbine-count': '0', 'tenure': 'Owner-occupied', 'floor-level': '', - 'potential-energy-efficiency': '91', 'hot-water-energy-eff': 'Good', 'low-energy-lighting': '100', - 'walls-description': 'Cavity wall, filled cavity', 'hotwater-description': 'From main system' - } - starting_epc = { - 'low-energy-fixed-light-count': '', 'address': '6 Kenmare Crescent', 'uprn-source': 'Energy Assessor', - 'floor-height': '2.49', 'heating-cost-potential': '464', 'unheated-corridor-length': '', - 'hot-water-cost-potential': '46', 'construction-age-band': 'England and Wales: 1967-1975', - 'potential-energy-rating': 'B', 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Average', - 'lighting-energy-eff': 'Very Good', 'environment-impact-potential': '85', 'glazed-type': 'not defined', - 'heating-cost-current': '535', 'address3': '', - 'mainheatcont-description': 'Programmer, room thermostat and TRVs', 'sheating-energy-eff': 'N/A', - 'property-type': 'Bungalow', 'local-authority-label': 'West Lindsey', 'fixed-lighting-outlets-count': '9', - 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '69', - 'county': 'Lincolnshire', 'postcode': 'DN21 1PR', 'solar-water-heating-flag': 'N', - 'constituency': 'E14000707', 'co2-emissions-potential': '1.2', 'number-heated-rooms': '3', - 'floor-description': 'Suspended, no insulation (assumed)', 'energy-consumption-potential': '102', - 'local-authority': 'E07000142', 'built-form': 'Semi-Detached', 'number-open-fireplaces': '0', - 'windows-description': 'Fully double glazed', 'glazed-area': 'Much More Than Typical', - 'inspection-date': '2022-05-31', 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '40', - 'address1': '6 Kenmare Crescent', 'heat-loss-corridor': '', 'flat-storey-count': '', - 'constituency-label': 'Gainsborough', 'roof-energy-eff': 'Very Good', 'total-floor-area': '66.0', - 'building-reference-number': '10002845316', 'environment-impact-current': '68', - 'co2-emissions-current': '2.6', 'roof-description': 'Pitched, 300 mm loft insulation', - 'floor-energy-eff': 'N/A', 'number-habitable-rooms': '3', 'address2': '', 'hot-water-env-eff': 'Good', - 'posttown': 'GAINSBOROUGH', 'mainheatc-energy-eff': 'Good', 'main-fuel': 'mains gas (not community)', - 'lighting-env-eff': 'Very Good', 'windows-energy-eff': 'Average', 'floor-env-eff': 'N/A', - 'sheating-env-eff': 'N/A', 'lighting-description': 'Low energy lighting in all fixed outlets', - 'roof-env-eff': 'Very Good', 'walls-energy-eff': 'Average', 'photo-supply': '0.0', - 'lighting-cost-potential': '65', 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '100', - 'main-heating-controls': '', 'lodgement-datetime': '2022-06-15 08:38:02', 'flat-top-storey': '', - 'current-energy-rating': 'D', 'secondheat-description': 'Room heaters, electric', - 'walls-env-eff': 'Average', 'transaction-type': 'ECO assessment', 'uprn': '100030952942', - 'current-energy-efficiency': '68', 'energy-consumption-current': '227', - 'mainheat-description': 'Boiler and radiators, mains gas', 'lighting-cost-current': '65', - 'lodgement-date': '2022-06-15', 'extension-count': '0', 'mainheatc-env-eff': 'Good', - 'lmk-key': 'ce181970b7077cb9b4626242bfb010b30a0e48541b5f22427e81f1adbeeec4f2', 'wind-turbine-count': '0', - 'tenure': 'Owner-occupied', 'floor-level': '', 'potential-energy-efficiency': '85', - 'hot-water-energy-eff': 'Good', 'low-energy-lighting': '100', - 'walls-description': 'Cavity wall, filled cavity', 'hotwater-description': 'From main system' - } - - cleaning_data = read_dataframe_from_s3_parquet( - bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet", - ) - - cleaned = read_from_s3( - s3_file_name="cleaned_epc_data/cleaned.bson", - bucket_name="retrofit-data-dev" - ) - cleaned = msgpack.unpackb(cleaned, raw=False) - - photo_supply_lookup, floor_area_decile_thresholds = SolarPhotoSupply.load(bucket="retrofit-data-dev") - - epc = EPCRecord( - epc_records={ - 'original_epc': starting_epc, - 'full_sap_epc': {}, - 'old_data': [] - }, - run_mode="newdata", - cleaning_data=cleaning_data - ) - - home = Property( - id=0, - address="", - postcode="", - epc_record=epc, - already_installed={}, - non_invasive_recommendations={}, - ) - home.in_conservation_area = False - home.is_listed = False - home.is_heritage = False - home.restricted_measures = True - home.get_components( - cleaned=cleaned, - photo_supply_lookup=photo_supply_lookup, - floor_area_decile_thresholds=floor_area_decile_thresholds - ) - - recommender = SolarPvRecommendations(property_instance=home) - recommender.recommend(phase=0) - - coverage_40_percent = [x for x in recommender.recommendation if x["photo_supply"] == 40] - assert len(coverage_40_percent) == 2 - - property_recommendations = Recommendations.insert_temp_recommendation_id([coverage_40_percent]) - - home.create_base_difference_epc_record(cleaned_lookup=cleaned) - home.adjust_difference_record_with_recommendations( - property_recommendations, [] - ) - - scoring_data = pd.DataFrame(home.recommendations_scoring_data).drop( - columns=["rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending", - "carbon_ending"] - ) - - model_api = ModelApi(portfolio_id="ashp-test", timestamp=datetime.now().isoformat()) - model_api.MODEL_PREFIXES = ["sap_change_predictions"] - - predictions_dict = model_api.predict_all( - df=scoring_data, - bucket="retrofit-data-dev", - prediction_buckets={ - "sap_change_predictions": "retrofit-sap-predictions-dev", - } - ) - - assert predictions_dict["sap_change_predictions"]["predictions"].tolist() == [87.1, 87.1] - assert ending_epc["current-energy-efficiency"] == '87' - assert starting_epc["current-energy-efficiency"] == '68' From 97061fd212999e703fae52e7af5a61eeb25441a7 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 30 Sep 2024 17:15:29 +0100 Subject: [PATCH 076/166] fixed ventilation recommendations tests --- recommendations/tests/test_ventilation_recommendations.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/recommendations/tests/test_ventilation_recommendations.py b/recommendations/tests/test_ventilation_recommendations.py index aa992253..441f9a22 100644 --- a/recommendations/tests/test_ventilation_recommendations.py +++ b/recommendations/tests/test_ventilation_recommendations.py @@ -22,7 +22,7 @@ class TestVentilationRecommendations: assert len(recommender.recommendation) == 1 - assert recommender.recommendation[0]["total"] == 1000 + assert recommender.recommendation[0]["total"] == 1071.0 assert recommender.recommendation[0]["type"] == "mechanical_ventilation" assert len(recommender.recommendation[0]["parts"]) == 1 assert recommender.recommendation[0]["parts"][0]["description"] == 'Mechanical Extract Ventilation' @@ -44,7 +44,7 @@ class TestVentilationRecommendations: assert len(recommender2.recommendation) == 1 - assert recommender2.recommendation[0]["total"] == 1000 + assert recommender2.recommendation[0]["total"] == 1071.0 assert recommender2.recommendation[0]["type"] == "mechanical_ventilation" assert len(recommender2.recommendation[0]["parts"]) == 1 assert recommender2.recommendation[0]["parts"][0]["description"] == 'Mechanical Extract Ventilation' @@ -66,7 +66,7 @@ class TestVentilationRecommendations: assert len(recommender3.recommendation) == 1 - assert recommender3.recommendation[0]["total"] == 1000 + assert recommender3.recommendation[0]["total"] == 1071.0 assert recommender3.recommendation[0]["type"] == "mechanical_ventilation" assert len(recommender3.recommendation[0]["parts"]) == 1 assert recommender3.recommendation[0]["parts"][0]["description"] == 'Mechanical Extract Ventilation' From 126232d5a53a7f8d913dde7e8eda8250660ea4ad Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 30 Sep 2024 17:42:35 +0100 Subject: [PATCH 077/166] walls recommendations fixed --- backend/app/plan/schemas.py | 2 +- .../tests/test_wall_recommendations.py | 158 +++++++++--------- 2 files changed, 81 insertions(+), 79 deletions(-) diff --git a/backend/app/plan/schemas.py b/backend/app/plan/schemas.py index c08cdefc..6f0f6327 100644 --- a/backend/app/plan/schemas.py +++ b/backend/app/plan/schemas.py @@ -58,7 +58,7 @@ NON_INVASIVE_SPECIFIC_MEASURES = [ # such as "external_wall_insulation", "internal_wall_insulation", "cavity_wall_insulation" MEASURE_MAP = { "wall_insulation": [ - "internal_wall_insulation", "external_wall_insulation", "cavity_wall_insulation", "cavity_extract_and_refill" + "internal_wall_insulation", "external_wall_insulation", "cavity_wall_insulation", ], "roof_insulation": ["loft_insulation", "flat_roof_insulation", "room_roof_insulation"], "floor_insulation": ["suspended_floor_insulation", "solid_floor_insulation"], diff --git a/recommendations/tests/test_wall_recommendations.py b/recommendations/tests/test_wall_recommendations.py index 580ebb91..a4093e58 100644 --- a/recommendations/tests/test_wall_recommendations.py +++ b/recommendations/tests/test_wall_recommendations.py @@ -10,8 +10,10 @@ from recommendations.tests.test_data.materials import materials from etl.epc.Record import EPCRecord +# import inspect +# file_path = inspect.getfile(lambda: None) # with open( -# os.path.abspath(os.path.dirname(__file__)) + "/recommendations/tests/test_data/input_properties.pkl", "rb" +# os.path.abspath(os.path.dirname(file_path)) + "/recommendations/tests/test_data/input_properties.pkl", "rb" # ) as f: # input_properties = pickle.load(f) @@ -86,17 +88,21 @@ class TestWallRecommendations: input_properties[1].walls["is_sandstone_or_limestone"] = False input_properties[1].age_band = "A" input_properties[1].restricted_measures = False + input_properties[1].already_installed = [] + input_properties[1].walls["is_park_home"] = False + input_properties[1].construction_age_band = "England and Wales: 1930-1949" + input_properties[1].non_invasive_recommendations = [] recommender = WallRecommendations( property_instance=input_properties[1], materials=materials ) assert recommender.property.walls["original_description"] == "Solid brick, as built, no insulation (assumed)" - assert not recommender.ewi_valid + assert not recommender.ewi_valid() assert recommender.property.in_conservation_area == "not_in_conservation_area" assert recommender.property.data["property-type"] == "Flat" - recommender.recommend() + recommender.recommend(phase=0) # This should result in some recommendations, all of which should be internal insulation assert recommender.recommendations @@ -131,7 +137,7 @@ class TestWallRecommendations: ) assert recommender.property.walls["original_description"] == "Solid brick, as built, insulated (assumed)" - assert not recommender.ewi_valid + assert not recommender.ewi_valid() assert recommender.property.in_conservation_area == "not_in_conservation_area" assert recommender.property.data["property-type"] == "Flat" assert recommender.estimated_u_value is None @@ -204,6 +210,11 @@ class TestWallRecommendationsBase: property_mock.restricted_measures = False property_mock.insulation_wall_area = 100 property_mock.data = {"county": "Derbyshire"} + property_mock.walls = { + "is_cob": False, + "is_sandstone_or_limestone": False, + "is_cavity_wall": False + } return property_mock @pytest.fixture @@ -216,24 +227,24 @@ class TestWallRecommendationsBase: def test_ewi_valid_in_conservation_area(self, wall_recommendations_instance): wall_recommendations_instance.property.in_conservation_area = "in_conversation_area" wall_recommendations_instance.property.restricted_measures = True - assert wall_recommendations_instance.ewi_valid is False + assert wall_recommendations_instance.ewi_valid() is False def test_ewi_valid_is_flat(self, wall_recommendations_instance): wall_recommendations_instance.property.data = {"property-type": "flat"} - assert wall_recommendations_instance.ewi_valid is False + assert wall_recommendations_instance.ewi_valid() is False def test_ewi_valid_not_in_conservation_area_and_not_flat(self, wall_recommendations_instance): wall_recommendations_instance.property.in_conservation_area = "not_in_conversation_area" wall_recommendations_instance.property.restricted_measures = False wall_recommendations_instance.property.data = {"property-type": "house"} - assert wall_recommendations_instance.ewi_valid is True + assert wall_recommendations_instance.ewi_valid() is True class TestCavityWallRecommensations: def test_fill_empty_cavity(self): epc_record = EPCRecord() - epc_record.prepared_epc = {"county": "Derbyshire"} + epc_record.prepared_epc = {"county": "Derbyshire", "walls-energy-eff": "Very Poor"} input_property = Property(id=1, postcode="F4k3", address="123 fake street", epc_record=epc_record) input_property.walls = { 'original_description': 'Cavity wall, as built, no insulation (assumed)', @@ -248,6 +259,7 @@ class TestCavityWallRecommensations: } input_property.age_band = "C" input_property.insulation_wall_area = 50 + input_property.construction_age_band = "England and Wales: 1930-1949" recommender = WallRecommendations( property_instance=input_property, @@ -261,14 +273,11 @@ class TestCavityWallRecommensations: assert recommender.recommendations assert recommender.estimated_u_value == 1.5 assert np.isclose(recommender.recommendations[0]["new_u_value"], 0.35) - assert np.isclose(recommender.recommendations[0]["total"], 1668.6600000000003) - - assert np.isclose(recommender.recommendations[1]["new_u_value"], 0.35) - assert np.isclose(recommender.recommendations[1]["total"], 2004.6600000000003) + assert np.isclose(recommender.recommendations[0]["total"], 710.5) def test_fill_partial_filled_cavity(self): epc_record = EPCRecord() - epc_record.prepared_epc = {"county": "County Durham"} + epc_record.prepared_epc = {"county": "County Durham", "walls-energy-eff": "Poor"} input_property = Property(id=1, postcode="F4k3", address="123 fake street", epc_record=epc_record) input_property.walls = { 'original_description': 'Cavity wall, as built, partial insulation (assumed)', @@ -283,6 +292,7 @@ class TestCavityWallRecommensations: } input_property.age_band = "C" input_property.insulation_wall_area = 50 + input_property.construction_age_band = "England and Wales: 1930-1949" recommender = WallRecommendations( property_instance=input_property, @@ -296,14 +306,13 @@ class TestCavityWallRecommensations: assert recommender.recommendations assert recommender.estimated_u_value == 1.3 assert np.isclose(recommender.recommendations[0]["new_u_value"], 0.41) - assert np.isclose(recommender.recommendations[0]["total"], 1663.9350000000002) - - assert np.isclose(recommender.recommendations[1]["new_u_value"], 0.41) - assert np.isclose(recommender.recommendations[1]["total"], 1999.9350000000002) + assert np.isclose(recommender.recommendations[0]["total"], 710.5) def test_system_built_wall(self): epc_record = EPCRecord() - epc_record.prepared_epc = {"property-type": "House", "county": "Derbyshire", "built-form": "Detached"} + epc_record.prepared_epc = { + "property-type": "House", "county": "Derbyshire", "built-form": "Detached", "walls-energy-eff": "Very Poor" + } input_property2 = Property(id=1, postcode="F4k3 2", address="223 fake street", epc_record=epc_record) input_property2.walls = { 'original_description': 'System built, as built, no insulation (assumed)', @@ -319,6 +328,7 @@ class TestCavityWallRecommensations: input_property2.age_band = "F" input_property2.insulation_wall_area = 120 input_property2.restricted_measures = False + input_property2.construction_age_band = "England and Wales: 1976-1982" assert input_property2.walls["is_system_built"] @@ -332,26 +342,24 @@ class TestCavityWallRecommensations: recommender2.recommend() assert recommender2.recommendations - assert len(recommender2.recommendations) == 9 + assert len(recommender2.recommendations) == 2 assert recommender2.estimated_u_value == 1 - assert np.isclose(recommender2.recommendations[0]["new_u_value"], 0.19) - assert np.isclose(recommender2.recommendations[0]["total"], 16429.960320000002) + assert np.isclose(recommender2.recommendations[0]["new_u_value"], 0.21) + assert np.isclose(recommender2.recommendations[0]["total"], 35802.0) assert recommender2.recommendations[0]["parts"][0]["type"] == "external_wall_insulation" - assert recommender2.recommendations[0]["parts"][0]["depth"] == 100 + assert recommender2.recommendations[0]["parts"][0]["depth"] == 150 - assert np.isclose(recommender2.recommendations[8]["new_u_value"], 0.23) - assert np.isclose(recommender2.recommendations[8]["total"], 11292.768) - assert recommender2.recommendations[8]["parts"][0]["type"] == "internal_wall_insulation" - assert recommender2.recommendations[8]["parts"][0]["depth"] == 72.5 - - assert np.isclose(recommender2.recommendations[6]["new_u_value"], 0.29) - assert np.isclose(recommender2.recommendations[6]["total"], 10988.208) - assert recommender2.recommendations[6]["parts"][0]["type"] == "internal_wall_insulation" - assert recommender2.recommendations[6]["parts"][0]["depth"] == 52.5 + assert np.isclose(recommender2.recommendations[1]["new_u_value"], 0.26) + assert np.isclose(recommender2.recommendations[1]["total"], 29376) + assert recommender2.recommendations[1]["parts"][0]["type"] == "internal_wall_insulation" + assert recommender2.recommendations[1]["parts"][0]["depth"] == 95 def test_timber_frame_wall(self): epc_record = EPCRecord() - epc_record.prepared_epc = {"property-type": "House", "county": "Derbyshire", "built-form": "Semi-Detached"} + epc_record.prepared_epc = { + "property-type": "House", "county": "Derbyshire", "built-form": "Semi-Detached", + "walls-energy-eff": "Very Poor" + } input_property3 = Property(id=1, postcode="F4k3 2", address="223 fake street", epc_record=epc_record) input_property3.walls = { 'original_description': 'Timber frame, as built, no insulation (assumed)', @@ -367,6 +375,7 @@ class TestCavityWallRecommensations: input_property3.age_band = "B" input_property3.insulation_wall_area = 99 input_property3.restricted_measures = False + input_property3.construction_age_band = "England and Wales: 1950-1966" assert input_property3.walls["is_timber_frame"] @@ -380,21 +389,24 @@ class TestCavityWallRecommensations: recommender3.recommend() assert recommender3.recommendations - assert len(recommender3.recommendations) == 6 + assert len(recommender3.recommendations) == 2 assert recommender3.estimated_u_value == 1.9 - assert np.isclose(recommender3.recommendations[0]["new_u_value"], 0.2) - assert np.isclose(recommender3.recommendations[0]["total"], 13554.717263999999) + assert np.isclose(recommender3.recommendations[0]["new_u_value"], 0.23) + assert np.isclose(recommender3.recommendations[0]["total"], 29536.65) assert recommender3.recommendations[0]["parts"][0]["type"] == "external_wall_insulation" - assert recommender3.recommendations[0]["parts"][0]["depth"] == 100.0 + assert recommender3.recommendations[0]["parts"][0]["depth"] == 150.0 - assert np.isclose(recommender3.recommendations[1]["new_u_value"], 0.23) - assert np.isclose(recommender3.recommendations[1]["total"], 35206.19308800001) - assert recommender3.recommendations[1]["parts"][0]["type"] == "external_wall_insulation" - assert recommender3.recommendations[1]["parts"][0]["depth"] == 150.0 + assert np.isclose(recommender3.recommendations[1]["new_u_value"], 0.29) + assert np.isclose(recommender3.recommendations[1]["total"], 24235.2) + assert recommender3.recommendations[1]["parts"][0]["type"] == "internal_wall_insulation" + assert recommender3.recommendations[1]["parts"][0]["depth"] == 95.0 def test_granite_or_whinstone_wall(self): epc_record = EPCRecord() - epc_record.prepared_epc = {"property-type": "Bungalow", "county": "Derbyshire", "built-form": "Detached"} + epc_record.prepared_epc = { + "property-type": "Bungalow", "county": "Derbyshire", "built-form": "Detached", + "walls-energy-eff": "Very Poor" + } input_property4 = Property(id=1, postcode="F4k3 2", address="223 fake street", epc_record=epc_record) input_property4.walls = { 'original_description': 'Granite or whinstone, as built, no insulation (assumed)', @@ -410,6 +422,7 @@ class TestCavityWallRecommensations: input_property4.age_band = "A" input_property4.insulation_wall_area = 223 input_property4.restricted_measures = False + input_property4.construction_age_band = "England and Wales: before 1900" assert input_property4.walls["is_granite_or_whinstone"] @@ -423,21 +436,24 @@ class TestCavityWallRecommensations: recommender4.recommend() assert recommender4.recommendations - assert len(recommender4.recommendations) == 6 + assert len(recommender4.recommendations) == 2 assert recommender4.estimated_u_value == 2.3 - assert np.isclose(recommender4.recommendations[0]["new_u_value"], 0.21) - assert np.isclose(recommender4.recommendations[0]["total"], 29547.42864) + assert np.isclose(recommender4.recommendations[0]["new_u_value"], 0.23) + assert np.isclose(recommender4.recommendations[0]["total"], 66532.05) assert recommender4.recommendations[0]["parts"][0]["type"] == "external_wall_insulation" - assert recommender4.recommendations[0]["parts"][0]["depth"] == 100 + assert recommender4.recommendations[0]["parts"][0]["depth"] == 150 - assert np.isclose(recommender4.recommendations[1]["new_u_value"], 0.23) - assert np.isclose(recommender4.recommendations[1]["total"], 76744.68288000001) - assert recommender4.recommendations[1]["parts"][0]["type"] == "external_wall_insulation" - assert recommender4.recommendations[1]["parts"][0]["depth"] == 150 + assert np.isclose(recommender4.recommendations[1]["new_u_value"], 0.3) + assert np.isclose(recommender4.recommendations[1]["total"], 54590.4) + assert recommender4.recommendations[1]["parts"][0]["type"] == "internal_wall_insulation" + assert recommender4.recommendations[1]["parts"][0]["depth"] == 95 def test_cob_wall(self): epc_record = EPCRecord() - epc_record.prepared_epc = {"property-type": "Bungalow", "county": "Derbyshire", "built-form": "Detached"} + epc_record.prepared_epc = { + "property-type": "Bungalow", "county": "Derbyshire", "built-form": "Detached", + "walls-energy-eff": "Very Poor" + } input_property5 = Property(id=1, postcode="F4k3 2", address="223 fake street", epc_record=epc_record) input_property5.walls = { 'original_description': 'Cob, as built', @@ -453,6 +469,7 @@ class TestCavityWallRecommensations: input_property5.age_band = "E" input_property5.insulation_wall_area = 77 input_property5.restricted_measures = False + input_property5.construction_age_band = "England and Wales: 1967-1975" assert input_property5.walls["is_cob"] @@ -465,22 +482,15 @@ class TestCavityWallRecommensations: recommender5.recommend() - assert recommender5.recommendations - assert len(recommender5.recommendations) == 5 - assert recommender5.estimated_u_value == 0.8 - assert np.isclose(recommender5.recommendations[0]["new_u_value"], 0.29) - assert np.isclose(recommender5.recommendations[0]["total"], 8963.834880000002) - assert recommender5.recommendations[0]["parts"][0]["type"] == "external_wall_insulation" - assert recommender5.recommendations[0]["parts"][0]["depth"] == 50 - - assert np.isclose(recommender5.recommendations[3]["new_u_value"], 0.26) - assert np.isclose(recommender5.recommendations[3]["total"], 20771.11344) - assert recommender5.recommendations[3]["parts"][0]["type"] == "internal_wall_insulation" - assert recommender5.recommendations[3]["parts"][0]["depth"] == 100 + # No insulation recommendations for cob walls + assert not recommender5.recommendations def test_sandstone_or_limestone_wall(self): epc_record = EPCRecord() - epc_record.prepared_epc = {"property-type": "House", "county": "Derbyshire", "built-form": "Mid-Terrace"} + epc_record.prepared_epc = { + "property-type": "House", "county": "Derbyshire", "built-form": "Mid-Terrace", + "walls-energy-eff": "Very Poor" + } input_property6 = Property(id=1, postcode="F4k3 6", address="623 fake street", epc_record=epc_record) input_property6.walls = { 'original_description': 'Sandstone or limestone, as built, no insulation (assumed)', @@ -496,6 +506,7 @@ class TestCavityWallRecommensations: input_property6.age_band = "F" input_property6.insulation_wall_area = 350 input_property6.restricted_measures = False + input_property6.construction_age_band = "England and Wales: 1976-1982" assert input_property6.walls["is_sandstone_or_limestone"] @@ -508,20 +519,11 @@ class TestCavityWallRecommensations: recommender6.recommend() + # For sandstone walls, we only recommend internal wall insulation assert recommender6.recommendations - assert len(recommender6.recommendations) == 9 + assert len(recommender6.recommendations) == 1 assert recommender6.estimated_u_value == 1 - assert np.isclose(recommender6.recommendations[0]["new_u_value"], 0.19) - assert np.isclose(recommender6.recommendations[0]["total"], 46374.888000000006) - assert recommender6.recommendations[0]["parts"][0]["type"] == "external_wall_insulation" - assert recommender6.recommendations[0]["parts"][0]["depth"] == 100 - - assert np.isclose(recommender6.recommendations[2]["new_u_value"], 0.21) - assert np.isclose(recommender6.recommendations[2]["total"], 120451.29600000002) - assert recommender6.recommendations[2]["parts"][0]["type"] == "external_wall_insulation" - assert recommender6.recommendations[2]["parts"][0]["depth"] == 150 - - assert np.isclose(recommender6.recommendations[4]["new_u_value"], 0.28) - assert np.isclose(recommender6.recommendations[4]["total"], 94414.15199999999) - assert recommender6.recommendations[4]["parts"][0]["type"] == "internal_wall_insulation" - assert recommender6.recommendations[4]["parts"][0]["depth"] == 100 + assert np.isclose(recommender6.recommendations[0]["new_u_value"], 0.26) + assert np.isclose(recommender6.recommendations[0]["total"], 85680.0) + assert recommender6.recommendations[0]["parts"][0]["type"] == "internal_wall_insulation" + assert recommender6.recommendations[0]["parts"][0]["depth"] == 95 From 643bfb5d1d7c9e77b09b755cf8340fac4ede3330 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 1 Oct 2024 11:56:03 +0100 Subject: [PATCH 078/166] adding simulation config to windows recommendation --- recommendations/WindowsRecommendations.py | 112 ++++++++++++++++-- .../tests/test_window_recommendations.py | 64 +++++----- 2 files changed, 133 insertions(+), 43 deletions(-) diff --git a/recommendations/WindowsRecommendations.py b/recommendations/WindowsRecommendations.py index 235d9ee2..a8fd2a87 100644 --- a/recommendations/WindowsRecommendations.py +++ b/recommendations/WindowsRecommendations.py @@ -48,6 +48,15 @@ class WindowsRecommendations: if not any(x in measures for x in MEASURE_MAP["windows"]): return + if self.property.windows["glazing_type"] in ["triple", "high performance"]: + # We don't make any recommendations in this case. The property already has outstanding glazing + return + + if self.property.windows["has_glazing"] & ( + self.property.windows["glazing_coverage"] == "full" + ): + return + # If the property is in a conservation area or is a listed building, it becomes more difficult to install # double glazing. Therefore, we don't recommend it. It is still possible but is not practical as it # requires planning permission and might require a more expensive window type, such as timber. @@ -67,11 +76,6 @@ class WindowsRecommendations: if not number_of_windows: raise ValueError("Number of windows not specified") - if self.property.windows["has_glazing"] & ( - self.property.windows["glazing_coverage"] == "full" - ): - return - if windows_area is not None: # TODO - we don't have a price for this so we can't recommend it print("We have windows area, we should use this data for our recommendations!!!") @@ -122,6 +126,95 @@ class WindowsRecommendations: ". Secondary glazing recommended due to conservation area status" ) + # Set up the simulation config + if self.property.windows["glazing_type"] == "multiple": + glazing_type_ending = "multiple" + glazed_type_ending = ( + "secondary glazing" if is_secondary_glazing else "double glazing installed during or after 2002" + ) + windows_energy_eff = "Good" + new_windows_description = "Multiple glazing throughout" + + elif self.property.windows["glazing_type"] == "single": + # We will only recommend either secondary or double glazing + glazing_type_ending = ( + "secondary" if is_secondary_glazing else "double" + ) + glazed_type_ending = ( + "secondary glazing" if is_secondary_glazing else "double glazing installed during or after 2002" + ) + + if is_secondary_glazing: + windows_energy_eff = "Good" + new_windows_description = "Full secondary glazing" + else: + windows_energy_eff = "Average" + new_windows_description = "Fully double glazed" + + elif self.property.windows["glazing_type"] == "double": + glazing_type_ending = ( + "multiple" if is_secondary_glazing else "double" + ) + + # We set glazed type depending on which window type is more prevalent. Since there is already double + # glazing in place, if we're recommending more double glazing, we set the glazed type to double glazing + # otherwise, if we're recommending secondary glazing and the proportion of glazing in place already that + # is double is less than 50% we set the glazed type to secondary glazing + + if not is_secondary_glazing: + glazed_type_ending = "double glazing installed during or after 2002" + new_windows_description = "Fully double glazed" + windows_energy_eff = "Average" + else: + if self.property.data["multi-glaze-proportion"] < 50: + glazed_type_ending = "secondary glazing" + else: + glazed_type_ending = "double glazing installed during or after 2002" + + new_windows_description = "Multiple glazing throughout" + windows_energy_eff = "Good" + + elif self.property.windows["glazing_type"] == "secondary": + glazing_type_ending = ( + "secondary" if is_secondary_glazing else "multiple" + ) + windows_energy_eff = "Good" + # This is the opposite. If there is secondary glazing in place, and we're recommending double + # we set glazed_type_ending, depending on the proportion of glazing in place + if is_secondary_glazing: + glazed_type_ending = "secondary glazing" + new_windows_description = "Full secondary glazing" + else: + if self.property.data["multi-glaze-proportion"] < 50: + glazed_type_ending = "double glazing installed during or after 2002" + else: + glazed_type_ending = "secondary glazing" + new_windows_description = "Multiple glazing throughout" + + else: + raise ValueError("Invalid glazing type - implement me") + + windows_ending_config = WindowAttributes(new_windows_description).process() + + windows_simulation_config = check_simulation_difference( + new_config=windows_ending_config, old_config=self.property.windows, prefix="windows_" + ) + + simulation_config = { + **windows_simulation_config, + "multi_glaze_proportion_ending": 100, + "windows_energy_eff_ending": windows_energy_eff, + "glazing_type_ending": glazing_type_ending, + "glazed_type_ending": glazed_type_ending, + } + + description_simulation = { + "multi-glaze-proportion": 100, + "windows-energy-eff": windows_energy_eff, + "windows-description": new_windows_description, + "glazed-type": glazed_type_ending, + } + self.recommendation = [ { "phase": phase, @@ -134,13 +227,8 @@ class WindowsRecommendations: "already_installed": already_installed, **cost_result, "is_secondary_glazing": is_secondary_glazing, - # TODO: Make this condition on is_secondary_glazing - "description_simulation": { - "multi-glaze-proportion": 100, - "windows-energy-eff": "Average", - "windows-description": "Fully double glazed", - "glazed-type": "double glazing installed during or after 2002", - } + "description_simulation": description_simulation, + "simulation_config": simulation_config, } ] diff --git a/recommendations/tests/test_window_recommendations.py b/recommendations/tests/test_window_recommendations.py index 36e70834..02978051 100644 --- a/recommendations/tests/test_window_recommendations.py +++ b/recommendations/tests/test_window_recommendations.py @@ -37,11 +37,13 @@ class TestWindowRecommendations: recommender.recommend() assert recommender.recommendation == [ - {'parts': [], 'type': 'windows_glazing', 'description': 'Install double glazing to all windows', - 'starting_u_value': None, 'new_u_value': None, 'sap_points': None, 'total': 5721.943248, - 'subtotal': 4768.28604, 'vat': 953.6572080000001, 'contingency': 340.59186, 'preliminaries': 340.59186, - 'material': 1275.75, 'profit': 681.18372, 'labour_hours': 45.5, 'labour_cost': 994.8624, - 'labour_days': 2.84375, 'is_secondary_glazing': False}] + {'phase': 0, 'parts': [], 'type': 'windows_glazing', 'description': 'Install double glazing to all windows', + 'starting_u_value': None, 'new_u_value': None, 'sap_points': None, 'already_installed': False, + 'total': 7980.0, 'labour_hours': 0.0, 'labour_days': 0.0, 'is_secondary_glazing': False, + 'description_simulation': {'multi-glaze-proportion': 100, 'windows-energy-eff': 'Average', + 'windows-description': 'Fully double glazed', + 'glazed-type': 'double glazing installed during or after 2002'}} + ] def test_partial_double_glazed(self): """ @@ -73,11 +75,14 @@ class TestWindowRecommendations: recommender2.recommend() assert recommender2.recommendation == [ - {'parts': [], 'type': 'windows_glazing', 'description': 'Install double glazing to the remaining windows', - 'starting_u_value': None, 'new_u_value': None, 'sap_points': None, 'total': 4087.10232, - 'subtotal': 3405.9186, 'vat': 681.18372, 'contingency': 243.2799, 'preliminaries': 243.2799, - 'material': 911.25, 'profit': 486.5598, 'labour_hours': 32.5, 'labour_cost': 710.6160000000001, - 'labour_days': 2.03125, 'is_secondary_glazing': False}] + {'phase': 0, 'parts': [], 'type': 'windows_glazing', + 'description': 'Install double glazing to the remaining windows', 'starting_u_value': None, + 'new_u_value': None, 'sap_points': None, 'already_installed': False, 'total': 5700.0, 'labour_hours': 0.0, + 'labour_days': 0.0, 'is_secondary_glazing': False, + 'description_simulation': {'multi-glaze-proportion': 100, 'windows-energy-eff': 'Average', + 'windows-description': 'Fully double glazed', + 'glazed-type': 'double glazing installed during or after 2002'}} + ] def test_fully_double_glazed(self): """ @@ -160,12 +165,14 @@ class TestWindowRecommendations: recommender5.recommend() assert recommender5.recommendation == [ - {'parts': [], 'type': 'windows_glazing', - 'description': 'Install secondary glazing to the remaining windows', - 'starting_u_value': None, 'new_u_value': None, 'sap_points': None, 'total': 1089.893952, - 'subtotal': 908.24496, 'vat': 181.64899200000002, 'contingency': 64.87464, 'preliminaries': 64.87464, - 'material': 729.0, 'profit': 129.74928, 'labour_hours': 13.0, 'labour_cost': 568.4928, - 'labour_days': 0.8125, 'is_secondary_glazing': True}] + {'phase': 0, 'parts': [], 'type': 'windows_glazing', + 'description': 'Install secondary glazing to the remaining windows', 'starting_u_value': None, + 'new_u_value': None, 'sap_points': None, 'already_installed': False, 'total': 4560.0, 'labour_hours': 0.0, + 'labour_days': 0.0, 'is_secondary_glazing': True, + 'description_simulation': {'multi-glaze-proportion': 100, 'windows-energy-eff': 'Good', + 'windows-description': 'Full secondary glazing', + 'glazed-type': 'secondary glazing'}} + ] def test_single_glazed_restricted_measures(self): epc_record = EPCRecord() @@ -195,14 +202,14 @@ class TestWindowRecommendations: recommender6.recommend() assert recommender6.recommendation == [ - {'parts': [], 'type': 'windows_glazing', - 'description': 'Install secondary glazing to all windows. Secondary ' - 'glazing recommended due to herigate building status', - 'starting_u_value': None, 'new_u_value': None, 'sap_points': None, - 'total': 1907.314416, 'subtotal': 1589.42868, 'vat': 317.885736, - 'contingency': 113.53062, 'preliminaries': 113.53062, - 'material': 1275.75, 'profit': 227.06124, 'labour_hours': 22.75, - 'labour_cost': 994.8624, 'labour_days': 1.421875, 'is_secondary_glazing': True} + {'phase': 0, 'parts': [], 'type': 'windows_glazing', + 'description': 'Install secondary glazing to all windows. Secondary glazing recommended due to herigate ' + 'building status', + 'starting_u_value': None, 'new_u_value': None, 'sap_points': None, 'already_installed': False, + 'total': 7980.0, 'labour_hours': 0.0, 'labour_days': 0.0, 'is_secondary_glazing': True, + 'description_simulation': {'multi-glaze-proportion': 100, 'windows-energy-eff': 'Good', + 'windows-description': 'Full secondary glazing', + 'glazed-type': 'secondary glazing'}} ] def test_full_triple_glazed(self): @@ -233,7 +240,7 @@ class TestWindowRecommendations: def test_partial_triple_glazed(self): """ - We should just recommend double glazing to the remaining windows, since it's a cheaper option + We don't recommend anything here """ epc_record = EPCRecord() epc_record.prepared_epc = { @@ -258,9 +265,4 @@ class TestWindowRecommendations: recommender8.recommend() - assert recommender8.recommendation == [ - {'parts': [], 'type': 'windows_glazing', 'description': 'Install double glazing to the remaining windows', - 'starting_u_value': None, 'new_u_value': None, 'sap_points': None, 'total': 1634.840928, - 'subtotal': 1362.36744, 'vat': 272.47348800000003, 'contingency': 97.31196, 'preliminaries': 97.31196, - 'material': 364.5, 'profit': 194.62392, 'labour_hours': 13.0, 'labour_cost': 284.2464, - 'labour_days': 0.8125, 'is_secondary_glazing': False}] + assert not recommender8.recommendation From 1ab9aaadf47e1684f2faf149acb8c79f43d1089a Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 1 Oct 2024 12:06:28 +0100 Subject: [PATCH 079/166] Updating windows recommender with simulation_config and fixed tests --- recommendations/WindowsRecommendations.py | 3 + .../tests/test_window_recommendations.py | 108 ++++++++++++------ 2 files changed, 79 insertions(+), 32 deletions(-) diff --git a/recommendations/WindowsRecommendations.py b/recommendations/WindowsRecommendations.py index a8fd2a87..cd1b982b 100644 --- a/recommendations/WindowsRecommendations.py +++ b/recommendations/WindowsRecommendations.py @@ -194,6 +194,9 @@ class WindowsRecommendations: else: raise ValueError("Invalid glazing type - implement me") + if (self.property.data["windows-energy-eff"] in ["Good", "Very Good"]) and (windows_energy_eff == "Average"): + windows_energy_eff = self.property.data["windows-energy-eff"] + windows_ending_config = WindowAttributes(new_windows_description).process() windows_simulation_config = check_simulation_difference( diff --git a/recommendations/tests/test_window_recommendations.py b/recommendations/tests/test_window_recommendations.py index 02978051..6478d87d 100644 --- a/recommendations/tests/test_window_recommendations.py +++ b/recommendations/tests/test_window_recommendations.py @@ -15,7 +15,8 @@ class TestWindowRecommendations: epc_record.prepared_epc = { "county": "Wychavon", "multi-glaze-proportion": 0, - "uprn": 0 + "uprn": 0, + "windows-energy-eff": "Very Poor" } property_1 = Property( id=1, @@ -36,13 +37,25 @@ class TestWindowRecommendations: recommender.recommend() + # The home is going from single glazing (v poor energy eff) -> double glazing (average energy eff) + assert recommender.recommendation == [ - {'phase': 0, 'parts': [], 'type': 'windows_glazing', 'description': 'Install double glazing to all windows', - 'starting_u_value': None, 'new_u_value': None, 'sap_points': None, 'already_installed': False, - 'total': 7980.0, 'labour_hours': 0.0, 'labour_days': 0.0, 'is_secondary_glazing': False, - 'description_simulation': {'multi-glaze-proportion': 100, 'windows-energy-eff': 'Average', - 'windows-description': 'Fully double glazed', - 'glazed-type': 'double glazing installed during or after 2002'}} + { + 'phase': 0, 'parts': [], 'type': 'windows_glazing', + 'description': 'Install double glazing to all windows', + 'starting_u_value': None, 'new_u_value': None, 'sap_points': None, 'already_installed': False, + 'total': 7980.0, 'labour_hours': 0.0, 'labour_days': 0.0, 'is_secondary_glazing': False, + 'description_simulation': { + 'multi-glaze-proportion': 100, 'windows-energy-eff': 'Average', + 'windows-description': 'Fully double glazed', + 'glazed-type': 'double glazing installed during or after 2002' + }, + 'simulation_config': { + 'has_glazing_ending': True, 'glazing_type_ending': 'double', + 'multi_glaze_proportion_ending': 100, 'windows_energy_eff_ending': 'Average', + 'glazed_type_ending': 'double glazing installed during or after 2002' + } + } ] def test_partial_double_glazed(self): @@ -55,7 +68,8 @@ class TestWindowRecommendations: epc_record.prepared_epc = { "county": "Wychavon", "multi-glaze-proportion": 33, - "uprn": 0 + "uprn": 0, + "windows-energy-eff": "Good" # This has been observed in the EPC data } property_2 = Property( id=1, @@ -75,13 +89,23 @@ class TestWindowRecommendations: recommender2.recommend() assert recommender2.recommendation == [ - {'phase': 0, 'parts': [], 'type': 'windows_glazing', - 'description': 'Install double glazing to the remaining windows', 'starting_u_value': None, - 'new_u_value': None, 'sap_points': None, 'already_installed': False, 'total': 5700.0, 'labour_hours': 0.0, - 'labour_days': 0.0, 'is_secondary_glazing': False, - 'description_simulation': {'multi-glaze-proportion': 100, 'windows-energy-eff': 'Average', - 'windows-description': 'Fully double glazed', - 'glazed-type': 'double glazing installed during or after 2002'}} + { + 'phase': 0, 'parts': [], 'type': 'windows_glazing', + 'description': 'Install double glazing to the remaining windows', 'starting_u_value': None, + 'new_u_value': None, 'sap_points': None, 'already_installed': False, 'total': 5700.0, + 'labour_hours': 0.0, + 'labour_days': 0.0, 'is_secondary_glazing': False, + 'description_simulation': { + 'multi-glaze-proportion': 100, 'windows-energy-eff': 'Good', + 'windows-description': 'Fully double glazed', + 'glazed-type': 'double glazing installed during or after 2002' + }, + 'simulation_config': { + 'glazing_coverage_ending': 'full', 'multi_glaze_proportion_ending': 100, + 'windows_energy_eff_ending': 'Good', 'glazing_type_ending': 'double', + 'glazed_type_ending': 'double glazing installed during or after 2002' + } + } ] def test_fully_double_glazed(self): @@ -145,7 +169,8 @@ class TestWindowRecommendations: epc_record.prepared_epc = { "county": "Wychavon", "multi-glaze-proportion": 50, - "uprn": 0 + "uprn": 0, + "windows-energy-eff": "Poor" # This has been observed in the EPC data } property_5 = Property( id=1, @@ -165,13 +190,22 @@ class TestWindowRecommendations: recommender5.recommend() assert recommender5.recommendation == [ - {'phase': 0, 'parts': [], 'type': 'windows_glazing', - 'description': 'Install secondary glazing to the remaining windows', 'starting_u_value': None, - 'new_u_value': None, 'sap_points': None, 'already_installed': False, 'total': 4560.0, 'labour_hours': 0.0, - 'labour_days': 0.0, 'is_secondary_glazing': True, - 'description_simulation': {'multi-glaze-proportion': 100, 'windows-energy-eff': 'Good', - 'windows-description': 'Full secondary glazing', - 'glazed-type': 'secondary glazing'}} + { + 'phase': 0, 'parts': [], 'type': 'windows_glazing', + 'description': 'Install secondary glazing to the remaining windows', 'starting_u_value': None, + 'new_u_value': None, 'sap_points': None, 'already_installed': False, 'total': 4560.0, + 'labour_hours': 0.0, 'labour_days': 0.0, 'is_secondary_glazing': True, + 'description_simulation': { + 'multi-glaze-proportion': 100, 'windows-energy-eff': 'Good', + 'windows-description': 'Full secondary glazing', + 'glazed-type': 'secondary glazing' + }, + 'simulation_config': { + 'glazing_coverage_ending': 'full', 'multi_glaze_proportion_ending': 100, + 'windows_energy_eff_ending': 'Good', 'glazing_type_ending': 'secondary', + 'glazed_type_ending': 'secondary glazing' + } + } ] def test_single_glazed_restricted_measures(self): @@ -179,7 +213,8 @@ class TestWindowRecommendations: epc_record.prepared_epc = { "county": "Wychavon", "multi-glaze-proportion": 0, - "uprn": 0 + "uprn": 0, + "windows-energy-eff": "Very Poor" } property_6 = Property( @@ -202,14 +237,23 @@ class TestWindowRecommendations: recommender6.recommend() assert recommender6.recommendation == [ - {'phase': 0, 'parts': [], 'type': 'windows_glazing', - 'description': 'Install secondary glazing to all windows. Secondary glazing recommended due to herigate ' - 'building status', - 'starting_u_value': None, 'new_u_value': None, 'sap_points': None, 'already_installed': False, - 'total': 7980.0, 'labour_hours': 0.0, 'labour_days': 0.0, 'is_secondary_glazing': True, - 'description_simulation': {'multi-glaze-proportion': 100, 'windows-energy-eff': 'Good', - 'windows-description': 'Full secondary glazing', - 'glazed-type': 'secondary glazing'}} + { + 'phase': 0, 'parts': [], 'type': 'windows_glazing', + 'description': 'Install secondary glazing to all windows. Secondary glazing recommended due to ' + 'herigate building status', + 'starting_u_value': None, 'new_u_value': None, 'sap_points': None, 'already_installed': False, + 'total': 7980.0, 'labour_hours': 0.0, 'labour_days': 0.0, 'is_secondary_glazing': True, + 'description_simulation': { + 'multi-glaze-proportion': 100, 'windows-energy-eff': 'Good', + 'windows-description': 'Full secondary glazing', + 'glazed-type': 'secondary glazing' + }, + 'simulation_config': { + 'has_glazing_ending': True, 'glazing_coverage_ending': 'full', + 'glazing_type_ending': 'secondary', 'multi_glaze_proportion_ending': 100, + 'windows_energy_eff_ending': 'Good', 'glazed_type_ending': 'secondary glazing' + } + } ] def test_full_triple_glazed(self): From 4118ad49f24cfa992ce753cccd4c418b8ad2e80f Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 1 Oct 2024 12:34:01 +0100 Subject: [PATCH 080/166] added new windows simulation test --- .../tests/test_window_recommendations.py | 354 ++++++++++++++++++ 1 file changed, 354 insertions(+) diff --git a/recommendations/tests/test_window_recommendations.py b/recommendations/tests/test_window_recommendations.py index 6478d87d..45b03a5e 100644 --- a/recommendations/tests/test_window_recommendations.py +++ b/recommendations/tests/test_window_recommendations.py @@ -2,6 +2,8 @@ from recommendations.WindowsRecommendations import WindowsRecommendations from backend.Property import Property from recommendations.tests.test_data.materials import materials from etl.epc.Record import EPCRecord +import msgpack +from utils.s3 import read_dataframe_from_s3_parquet, read_from_s3 class TestWindowRecommendations: @@ -310,3 +312,355 @@ class TestWindowRecommendations: recommender8.recommend() assert not recommender8.recommendation + + def test_simulating_outcome_single_glazed(self): + # Could move these to fixtures + cleaning_data = read_dataframe_from_s3_parquet( + bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet", + ) + cleaned = read_from_s3(s3_file_name="cleaned_epc_data/cleaned.bson", bucket_name="retrofit-data-dev") + cleaned = msgpack.unpackb(cleaned, raw=False) + + epc = { + 'lmk-key': 'f4cf43c90ab3140112a9d1c8cfb21ec1bf73f5a2ca3c75118f289d3447dddf15', 'address1': '3 The Green', + 'address2': 'Old Dalby', 'address3': None, 'postcode': 'LE14 3LL', + 'building-reference-number': 10006291833, 'current-energy-rating': 'E', 'potential-energy-rating': 'B', + 'current-energy-efficiency': 47, 'potential-energy-efficiency': 82, 'property-type': 'House', + 'built-form': 'Semi-Detached', 'inspection-date': '2024-07-19', 'local-authority': 'E07000133', + 'constituency': 'E14000909', 'county': 'Leicestershire', 'lodgement-date': '2024-07-21', + 'transaction-type': 'rental', 'environment-impact-current': 41, 'environment-impact-potential': 79, + 'energy-consumption-current': 478, 'energy-consumption-potential': 155.0, 'co2-emissions-current': 5.1, + 'co2-emiss-curr-per-floor-area': 85, 'co2-emissions-potential': 1.7, 'lighting-cost-current': 91.0, + 'lighting-cost-potential': 91.0, 'heating-cost-current': 1677.0, 'heating-cost-potential': 874.0, + 'hot-water-cost-current': 161.0, 'hot-water-cost-potential': 109.0, 'total-floor-area': 61.0, + 'energy-tariff': 'dual', 'mains-gas-flag': 'Y', 'floor-level': None, 'flat-top-storey': None, + 'flat-storey-count': None, 'main-heating-controls': None, 'multi-glaze-proportion': 0.0, + 'glazed-type': 'not defined', 'glazed-area': 'Normal', 'extension-count': 3.0, + 'number-habitable-rooms': 4.0, 'number-heated-rooms': 4.0, 'low-energy-lighting': 100.0, + 'number-open-fireplaces': 0.0, 'hotwater-description': 'From main system', + 'hot-water-energy-eff': 'Good', 'hot-water-env-eff': 'Good', + 'floor-description': 'Solid, no insulation (assumed)', 'floor-energy-eff': None, 'floor-env-eff': None, + 'windows-description': 'Single glazed', 'windows-energy-eff': 'Very Poor', + 'windows-env-eff': 'Very Poor', 'walls-description': 'Solid brick, as built, no insulation (assumed)', + 'walls-energy-eff': 'Very Poor', 'walls-env-eff': 'Very Poor', 'secondheat-description': 'None', + 'sheating-energy-eff': None, 'sheating-env-eff': None, + 'roof-description': 'Pitched, no insulation (assumed)', 'roof-energy-eff': 'Very Poor', + 'roof-env-eff': 'Very Poor', 'mainheat-description': 'Boiler and radiators, mains gas', + 'mainheat-energy-eff': 'Good', 'mainheat-env-eff': 'Good', + 'mainheatcont-description': 'Programmer and room thermostat', 'mainheatc-energy-eff': 'Average', + 'mainheatc-env-eff': 'Average', 'lighting-description': 'Low energy lighting in all fixed outlets', + 'lighting-energy-eff': 'Very Good', 'lighting-env-eff': 'Very Good', + 'main-fuel': 'mains gas (not community)', 'wind-turbine-count': 0.0, 'heat-loss-corridor': None, + 'unheated-corridor-length': None, 'floor-height': 2.37, 'photo-supply': 0.0, + 'solar-water-heating-flag': 'N', 'mechanical-ventilation': 'natural', + 'address': '3 The Green, Old Dalby', 'local-authority-label': 'Melton', + 'constituency-label': 'Rutland and Melton', 'posttown': 'MELTON MOWBRAY', + 'construction-age-band': 'England and Wales: before 1900', 'lodgement-datetime': '2024-07-21 19:29:04', + 'tenure': 'Rented (private)', 'fixed-lighting-outlets-count': 7.0, 'low-energy-fixed-light-count': None, + 'uprn': 200001041444.0, 'uprn-source': 'Energy Assessor' + } + + epc_records = { + "original_epc": epc, + "full_sap_epc": {}, + "old_data": [] + } + + epc_record = EPCRecord( + epc_records=epc_records, + run_mode="newdata", + cleaning_data=cleaning_data + ) + + property_9 = Property( + id=1, + postcode='1', + address='1', + epc_record=epc_record + ) + property_9.windows = { + 'original_description': 'Single glazed', 'has_glazing': False, 'glazing_coverage': None, + 'glazing_type': 'single', + 'no_data': False + } + + property_9.number_of_windows = 7 + property_9.restricted_measures = False + property_9.is_heritage = False + + recommender9 = WindowsRecommendations(property_instance=property_9, materials=materials) + + assert not recommender9.recommendation + + recommender9.recommend() + + assert recommender9.recommendation == [ + { + 'phase': 0, 'parts': [], 'type': 'windows_glazing', + 'description': 'Install double glazing to all windows', 'starting_u_value': None, 'new_u_value': None, + 'sap_points': None, 'already_installed': False, 'total': 7980.0, 'labour_hours': 0.0, + 'labour_days': 0.0, 'is_secondary_glazing': False, + 'description_simulation': { + 'multi-glaze-proportion': 100, 'windows-energy-eff': 'Average', + 'windows-description': 'Fully double glazed', + 'glazed-type': 'double glazing installed during or after 2002' + }, + 'simulation_config': { + 'has_glazing_ending': True, 'glazing_coverage_ending': 'full', + 'glazing_type_ending': 'double', 'multi_glaze_proportion_ending': 100, + 'windows_energy_eff_ending': 'Average', + 'glazed_type_ending': 'double glazing installed during or after 2002' + } + } + ] + + # We now simulate the outcome + windows_rec = recommender9.recommendation.copy() + windows_rec[0]["recommendation_id"] = 1 + property_recommendations = [windows_rec] + + property_9.create_base_difference_epc_record(cleaned_lookup=cleaned) + + starting_record = property_9.base_difference_record.df.to_dict("records")[0] + + expected_base_difference_record = { + 'uprn': 200001041444, 'rdsap_change': 0, 'heat_demand_change': 0, 'carbon_change': 0.0, + 'potential_energy_efficiency': 82.0, 'environment_impact_potential': 79.0, + 'energy_consumption_potential': 155.0, 'co2_emissions_potential': 1.7, 'property_type': 'House', + 'built_form': 'Semi-Detached', 'constituency': 'E14000909', 'number_habitable_rooms': 4.0, + 'number_heated_rooms': 4.0, 'construction_age_band': 'England and Wales: before 1900', + 'fixed_lighting_outlets_count': 7.0, 'walls_thermal_transmittance': 1.7, + 'walls_thermal_transmittance_unit': 'Unknown', 'is_cavity_wall': False, 'is_filled_cavity': False, + 'is_solid_brick': True, 'is_system_built': False, 'is_timber_frame': False, + 'is_granite_or_whinstone': False, 'is_as_built': True, 'is_cob': False, 'walls_is_assumed': True, + 'is_sandstone_or_limestone': False, 'is_park_home': False, 'walls_insulation_thickness': 'none', + 'external_insulation': False, 'internal_insulation': False, 'floor_thermal_transmittance': 0.96, + 'is_to_unheated_space': False, 'is_to_external_air': False, 'is_suspended': False, 'is_solid': True, + 'another_property_below': False, 'floor_insulation_thickness': 'none', 'roof_thermal_transmittance': 2.3, + 'is_pitched': True, 'is_roof_room': False, 'is_loft': False, 'is_flat': False, 'is_thatched': False, + 'is_at_rafters': False, 'has_dwelling_above': False, 'roof_insulation_thickness': 'none', + 'heater_type': 'Unknown', 'system_type': 'from main system', 'thermostat_characteristics': 'Unknown', + 'heating_scope': 'Unknown', 'energy_recovery': 'Unknown', 'hotwater_tariff_type': 'Unknown', + 'extra_features': 'Unknown', 'chp_systems': 'Unknown', 'distribution_system': 'Unknown', + 'no_system_present': 'Unknown', 'appliance': 'Unknown', 'has_radiators': True, 'has_fan_coil_units': False, + 'has_pipes_in_screed_above_insulation': False, 'has_pipes_in_insulated_timber_floor': False, + 'has_pipes_in_concrete_slab': False, 'has_boiler': True, 'has_air_source_heat_pump': False, + 'has_room_heaters': False, 'has_electric_storage_heaters': False, 'has_warm_air': False, + 'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False, + 'has_community_scheme': False, 'has_ground_source_heat_pump': False, 'has_no_system_present': False, + 'has_portable_electric_heaters': False, 'has_water_source_heat_pump': False, + 'has_electric_heat_pump': False, 'has_micro-cogeneration': False, 'has_solar_assisted_heat_pump': False, + 'has_exhaust_source_heat_pump': False, 'has_community_heat_pump': False, 'has_electric': False, + 'has_mains_gas': True, 'has_wood_logs': False, 'has_coal': False, 'has_oil': False, + 'has_wood_pellets': False, 'has_anthracite': False, 'has_dual_fuel_mineral_and_wood': False, + 'has_smokeless_fuel': False, 'has_lpg': False, 'has_b30k': False, 'has_electricaire': False, + 'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False, + 'thermostatic_control': 'room thermostat', 'charging_system': 'Unknown', 'switch_system': 'programmer', + 'no_control': 'Unknown', 'dhw_control': 'Unknown', 'community_heating': 'Unknown', + 'multiple_room_thermostats': False, 'auxiliary_systems': 'Unknown', 'trvs': 'Unknown', + 'rate_control': 'Unknown', 'glazing_type': 'single', 'fuel_type': 'mains gas', + 'main-fuel_tariff_type': 'Unknown', 'is_community': False, + 'no_individual_heating_or_community_network': False, 'complex_fuel_type': 'Unknown', + 'walls_thermal_transmittance_ending': 1.7, 'walls_thermal_transmittance_unit_ending': 'Unknown', + 'is_filled_cavity_ending': False, 'is_as_built_ending': True, 'walls_is_assumed_ending': True, + 'is_park_home_ending': False, 'walls_insulation_thickness_ending': 'none', + 'external_insulation_ending': False, 'internal_insulation_ending': False, + 'floor_thermal_transmittance_ending': 0.96, 'floor_insulation_thickness_ending': 'none', + 'roof_thermal_transmittance_ending': 2.3, 'is_at_rafters_ending': False, + 'roof_insulation_thickness_ending': 'none', 'heater_type_ending': 'Unknown', + 'system_type_ending': 'from main system', 'thermostat_characteristics_ending': 'Unknown', + 'heating_scope_ending': 'Unknown', 'energy_recovery_ending': 'Unknown', + 'hotwater_tariff_type_ending': 'Unknown', 'extra_features_ending': 'Unknown', + 'chp_systems_ending': 'Unknown', 'distribution_system_ending': 'Unknown', + 'no_system_present_ending': 'Unknown', 'appliance_ending': 'Unknown', 'has_radiators_ending': True, + 'has_fan_coil_units_ending': False, 'has_pipes_in_screed_above_insulation_ending': False, + 'has_pipes_in_insulated_timber_floor_ending': False, 'has_pipes_in_concrete_slab_ending': False, + 'has_boiler_ending': True, 'has_air_source_heat_pump_ending': False, 'has_room_heaters_ending': False, + 'has_electric_storage_heaters_ending': False, 'has_warm_air_ending': False, + 'has_electric_underfloor_heating_ending': False, 'has_electric_ceiling_heating_ending': False, + 'has_community_scheme_ending': False, 'has_ground_source_heat_pump_ending': False, + 'has_no_system_present_ending': False, 'has_portable_electric_heaters_ending': False, + 'has_water_source_heat_pump_ending': False, 'has_electric_heat_pump_ending': False, + 'has_micro-cogeneration_ending': False, 'has_solar_assisted_heat_pump_ending': False, + 'has_exhaust_source_heat_pump_ending': False, 'has_community_heat_pump_ending': False, + 'has_electric_ending': False, 'has_mains_gas_ending': True, 'has_wood_logs_ending': False, + 'has_coal_ending': False, 'has_oil_ending': False, 'has_wood_pellets_ending': False, + 'has_anthracite_ending': False, 'has_dual_fuel_mineral_and_wood_ending': False, + 'has_smokeless_fuel_ending': False, 'has_lpg_ending': False, 'has_b30k_ending': False, + 'has_electricaire_ending': False, 'has_assumed_for_most_rooms_ending': False, + 'has_underfloor_heating_ending': False, 'thermostatic_control_ending': 'room thermostat', + 'charging_system_ending': 'Unknown', 'switch_system_ending': 'programmer', 'no_control_ending': 'Unknown', + 'dhw_control_ending': 'Unknown', 'community_heating_ending': 'Unknown', + 'multiple_room_thermostats_ending': False, 'auxiliary_systems_ending': 'Unknown', 'trvs_ending': 'Unknown', + 'rate_control_ending': 'Unknown', 'glazing_type_ending': 'single', 'fuel_type_ending': 'mains gas', + 'main-fuel_tariff_type_ending': 'Unknown', 'is_community_ending': False, + 'no_individual_heating_or_community_network_ending': False, 'complex_fuel_type_ending': 'Unknown', + 'sap_starting': 47, 'sap_ending': 47, 'heat_demand_starting': 478, 'heat_demand_ending': 478, + 'carbon_starting': 5.1, 'carbon_ending': 5.1, 'lighting_cost_starting': 91.0, 'lighting_cost_ending': 91.0, + 'heating_cost_starting': 1677.0, 'heating_cost_ending': 1677.0, 'hot_water_cost_starting': 161.0, + 'hot_water_cost_ending': 161.0, 'mechanical_ventilation_starting': 'natural', + 'mechanical_ventilation_ending': 'natural', 'secondheat_description_starting': 'None', + 'secondheat_description_ending': 'None', 'glazed_type_starting': 'not defined', + 'glazed_type_ending': 'not defined', 'multi_glaze_proportion_starting': 0.0, + 'multi_glaze_proportion_ending': 0.0, 'low_energy_lighting_starting': 100.0, + 'low_energy_lighting_ending': 100.0, 'number_open_fireplaces_starting': 0.0, + 'number_open_fireplaces_ending': 0.0, 'solar_water_heating_flag_starting': 'N', + 'solar_water_heating_flag_ending': 'N', 'photo_supply_starting': 0.0, 'photo_supply_ending': 0.0, + 'transaction_type_starting': 'rental', 'transaction_type_ending': 'rental', + 'energy_tariff_starting': 'dual', 'energy_tariff_ending': 'dual', 'extension_count_starting': 3.0, + 'extension_count_ending': 3.0, 'total_floor_area_starting': 61.0, 'total_floor_area_ending': 61.0, + 'floor_height_starting': 2.37, 'floor_height_ending': 2.37, 'hot_water_energy_eff_starting': 'Good', + 'hot_water_energy_eff_ending': 'Good', 'floor_energy_eff_starting': 'NO_RATING', + 'floor_energy_eff_ending': 'NO_RATING', 'windows_energy_eff_starting': 'Very Poor', + 'windows_energy_eff_ending': 'Very Poor', 'walls_energy_eff_starting': 'Very Poor', + 'walls_energy_eff_ending': 'Very Poor', 'sheating_energy_eff_starting': 'NO_RATING', + 'sheating_energy_eff_ending': 'NO_RATING', 'roof_energy_eff_starting': 'Very Poor', + 'roof_energy_eff_ending': 'Very Poor', 'mainheat_energy_eff_starting': 'Good', + 'mainheat_energy_eff_ending': 'Good', 'mainheatc_energy_eff_starting': 'Average', + 'mainheatc_energy_eff_ending': 'Average', 'lighting_energy_eff_starting': 'Very Good', + 'lighting_energy_eff_ending': 'Very Good', 'number_habitable_rooms_starting': 4.0, + 'number_habitable_rooms_ending': 4.0, 'number_heated_rooms_starting': 4.0, + 'number_heated_rooms_ending': 4.0, 'days_to_starting': 3642, 'days_to_ending': 3642, + 'estimated_perimeter_starting': 23.430749027719962, 'estimated_perimeter_ending': 23.430749027719962 + } + + assert starting_record == expected_base_difference_record + + # Simulate outcome + property_9.adjust_difference_record_with_recommendations( + property_recommendations, windows_rec + ) + + simulated_data = property_9.recommendations_scoring_data.copy() + + assert len(simulated_data) == 1 + + expected_simulated_outcome = { + 'uprn': 200001041444, 'rdsap_change': 0, 'heat_demand_change': 0, 'carbon_change': 0.0, + 'potential_energy_efficiency': 82.0, 'environment_impact_potential': 79.0, + 'energy_consumption_potential': 155.0, 'co2_emissions_potential': 1.7, 'property_type': 'House', + 'built_form': 'Semi-Detached', 'constituency': 'E14000909', 'number_habitable_rooms': 4.0, + 'number_heated_rooms': 4.0, 'construction_age_band': 'England and Wales: before 1900', + 'fixed_lighting_outlets_count': 7.0, 'walls_thermal_transmittance': 1.7, + 'walls_thermal_transmittance_unit': 'Unknown', 'is_cavity_wall': False, 'is_filled_cavity': False, + 'is_solid_brick': True, 'is_system_built': False, 'is_timber_frame': False, + 'is_granite_or_whinstone': False, 'is_as_built': True, 'is_cob': False, 'walls_is_assumed': True, + 'is_sandstone_or_limestone': False, 'is_park_home': False, 'walls_insulation_thickness': 'none', + 'external_insulation': False, 'internal_insulation': False, 'floor_thermal_transmittance': 0.96, + 'is_to_unheated_space': False, 'is_to_external_air': False, 'is_suspended': False, 'is_solid': True, + 'another_property_below': False, 'floor_insulation_thickness': 'none', 'roof_thermal_transmittance': 2.3, + 'is_pitched': True, 'is_roof_room': False, 'is_loft': False, 'is_flat': False, 'is_thatched': False, + 'is_at_rafters': False, 'has_dwelling_above': False, 'roof_insulation_thickness': 'none', + 'heater_type': 'Unknown', 'system_type': 'from main system', 'thermostat_characteristics': 'Unknown', + 'heating_scope': 'Unknown', 'energy_recovery': 'Unknown', 'hotwater_tariff_type': 'Unknown', + 'extra_features': 'Unknown', 'chp_systems': 'Unknown', 'distribution_system': 'Unknown', + 'no_system_present': 'Unknown', 'appliance': 'Unknown', 'has_radiators': True, 'has_fan_coil_units': False, + 'has_pipes_in_screed_above_insulation': False, 'has_pipes_in_insulated_timber_floor': False, + 'has_pipes_in_concrete_slab': False, 'has_boiler': True, 'has_air_source_heat_pump': False, + 'has_room_heaters': False, 'has_electric_storage_heaters': False, 'has_warm_air': False, + 'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False, + 'has_community_scheme': False, 'has_ground_source_heat_pump': False, 'has_no_system_present': False, + 'has_portable_electric_heaters': False, 'has_water_source_heat_pump': False, + 'has_electric_heat_pump': False, 'has_micro-cogeneration': False, 'has_solar_assisted_heat_pump': False, + 'has_exhaust_source_heat_pump': False, 'has_community_heat_pump': False, 'has_electric': False, + 'has_mains_gas': True, 'has_wood_logs': False, 'has_coal': False, 'has_oil': False, + 'has_wood_pellets': False, 'has_anthracite': False, 'has_dual_fuel_mineral_and_wood': False, + 'has_smokeless_fuel': False, 'has_lpg': False, 'has_b30k': False, 'has_electricaire': False, + 'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False, + 'thermostatic_control': 'room thermostat', 'charging_system': 'Unknown', 'switch_system': 'programmer', + 'no_control': 'Unknown', 'dhw_control': 'Unknown', 'community_heating': 'Unknown', + 'multiple_room_thermostats': False, 'auxiliary_systems': 'Unknown', 'trvs': 'Unknown', + 'rate_control': 'Unknown', 'glazing_type': 'single', 'fuel_type': 'mains gas', + 'main-fuel_tariff_type': 'Unknown', 'is_community': False, + 'no_individual_heating_or_community_network': False, 'complex_fuel_type': 'Unknown', + 'walls_thermal_transmittance_ending': 1.7, 'walls_thermal_transmittance_unit_ending': 'Unknown', + 'is_filled_cavity_ending': False, 'is_as_built_ending': True, 'walls_is_assumed_ending': True, + 'is_park_home_ending': False, 'walls_insulation_thickness_ending': 'none', + 'external_insulation_ending': False, 'internal_insulation_ending': False, + 'floor_thermal_transmittance_ending': 0.96, 'floor_insulation_thickness_ending': 'none', + 'roof_thermal_transmittance_ending': 2.3, 'is_at_rafters_ending': False, + 'roof_insulation_thickness_ending': 'none', 'heater_type_ending': 'Unknown', + 'system_type_ending': 'from main system', 'thermostat_characteristics_ending': 'Unknown', + 'heating_scope_ending': 'Unknown', 'energy_recovery_ending': 'Unknown', + 'hotwater_tariff_type_ending': 'Unknown', 'extra_features_ending': 'Unknown', + 'chp_systems_ending': 'Unknown', 'distribution_system_ending': 'Unknown', + 'no_system_present_ending': 'Unknown', 'appliance_ending': 'Unknown', 'has_radiators_ending': True, + 'has_fan_coil_units_ending': False, 'has_pipes_in_screed_above_insulation_ending': False, + 'has_pipes_in_insulated_timber_floor_ending': False, 'has_pipes_in_concrete_slab_ending': False, + 'has_boiler_ending': True, 'has_air_source_heat_pump_ending': False, 'has_room_heaters_ending': False, + 'has_electric_storage_heaters_ending': False, 'has_warm_air_ending': False, + 'has_electric_underfloor_heating_ending': False, 'has_electric_ceiling_heating_ending': False, + 'has_community_scheme_ending': False, 'has_ground_source_heat_pump_ending': False, + 'has_no_system_present_ending': False, 'has_portable_electric_heaters_ending': False, + 'has_water_source_heat_pump_ending': False, 'has_electric_heat_pump_ending': False, + 'has_micro-cogeneration_ending': False, 'has_solar_assisted_heat_pump_ending': False, + 'has_exhaust_source_heat_pump_ending': False, 'has_community_heat_pump_ending': False, + 'has_electric_ending': False, 'has_mains_gas_ending': True, 'has_wood_logs_ending': False, + 'has_coal_ending': False, 'has_oil_ending': False, 'has_wood_pellets_ending': False, + 'has_anthracite_ending': False, 'has_dual_fuel_mineral_and_wood_ending': False, + 'has_smokeless_fuel_ending': False, 'has_lpg_ending': False, 'has_b30k_ending': False, + 'has_electricaire_ending': False, 'has_assumed_for_most_rooms_ending': False, + 'has_underfloor_heating_ending': False, 'thermostatic_control_ending': 'room thermostat', + 'charging_system_ending': 'Unknown', 'switch_system_ending': 'programmer', 'no_control_ending': 'Unknown', + 'dhw_control_ending': 'Unknown', 'community_heating_ending': 'Unknown', + 'multiple_room_thermostats_ending': False, 'auxiliary_systems_ending': 'Unknown', 'trvs_ending': 'Unknown', + 'rate_control_ending': 'Unknown', 'glazing_type_ending': 'double', 'fuel_type_ending': 'mains gas', + 'main-fuel_tariff_type_ending': 'Unknown', 'is_community_ending': False, + 'no_individual_heating_or_community_network_ending': False, 'complex_fuel_type_ending': 'Unknown', + 'sap_starting': 47, 'sap_ending': 47, 'heat_demand_starting': 478, 'heat_demand_ending': 478, + 'carbon_starting': 5.1, 'carbon_ending': 5.1, 'lighting_cost_starting': 91.0, 'lighting_cost_ending': 91.0, + 'heating_cost_starting': 1677.0, 'heating_cost_ending': 1677.0, 'hot_water_cost_starting': 161.0, + 'hot_water_cost_ending': 161.0, 'mechanical_ventilation_starting': 'natural', + 'mechanical_ventilation_ending': 'natural', 'secondheat_description_starting': 'None', + 'secondheat_description_ending': 'None', 'glazed_type_starting': 'not defined', + 'glazed_type_ending': 'double glazing installed during or after 2002', + 'multi_glaze_proportion_starting': 0.0, 'multi_glaze_proportion_ending': 100, + 'low_energy_lighting_starting': 100.0, 'low_energy_lighting_ending': 100.0, + 'number_open_fireplaces_starting': 0.0, 'number_open_fireplaces_ending': 0.0, + 'solar_water_heating_flag_starting': 'N', 'solar_water_heating_flag_ending': 'N', + 'photo_supply_starting': 0.0, 'photo_supply_ending': 0.0, 'transaction_type_starting': 'rental', + 'transaction_type_ending': 'rental', 'energy_tariff_starting': 'dual', 'energy_tariff_ending': 'dual', + 'extension_count_starting': 3.0, 'extension_count_ending': 3.0, 'total_floor_area_starting': 61.0, + 'total_floor_area_ending': 61.0, 'floor_height_starting': 2.37, 'floor_height_ending': 2.37, + 'hot_water_energy_eff_starting': 'Good', 'hot_water_energy_eff_ending': 'Good', + 'floor_energy_eff_starting': 'NO_RATING', 'floor_energy_eff_ending': 'NO_RATING', + 'windows_energy_eff_starting': 'Very Poor', 'windows_energy_eff_ending': 'Average', + 'walls_energy_eff_starting': 'Very Poor', 'walls_energy_eff_ending': 'Very Poor', + 'sheating_energy_eff_starting': 'NO_RATING', 'sheating_energy_eff_ending': 'NO_RATING', + 'roof_energy_eff_starting': 'Very Poor', 'roof_energy_eff_ending': 'Very Poor', + 'mainheat_energy_eff_starting': 'Good', 'mainheat_energy_eff_ending': 'Good', + 'mainheatc_energy_eff_starting': 'Average', 'mainheatc_energy_eff_ending': 'Average', + 'lighting_energy_eff_starting': 'Very Good', 'lighting_energy_eff_ending': 'Very Good', + 'number_habitable_rooms_starting': 4.0, 'number_habitable_rooms_ending': 4.0, + 'number_heated_rooms_starting': 4.0, 'number_heated_rooms_ending': 4.0, 'days_to_starting': 3642, + 'days_to_ending': 3713, 'estimated_perimeter_starting': 23.430749027719962, + 'estimated_perimeter_ending': 23.430749027719962, 'id': '1+1' + } + + assert simulated_data[0] == expected_simulated_outcome + + # Check which keys are different + different = [] + for k in simulated_data[0].keys(): + if k == "id": + continue + if simulated_data[0][k] != starting_record[k]: + different.append( + { + "variable": k, + "starting": starting_record[k], + "simulated": simulated_data[0][k], + + } + ) + + expected_different = [ + {'variable': 'glazing_type_ending', 'starting': 'single', 'simulated': 'double'}, + {'variable': 'glazed_type_ending', 'starting': 'not defined', + 'simulated': 'double glazing installed during or after 2002'}, + {'variable': 'multi_glaze_proportion_ending', 'starting': 0.0, 'simulated': 100}, + {'variable': 'windows_energy_eff_ending', 'starting': 'Very Poor', 'simulated': 'Average'}, + {'variable': 'days_to_ending', 'starting': 3642, 'simulated': 3713} + ] + + assert different == expected_different From 1b7599015c9a9fa72283b38bbc65dfc29f0bc940 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 1 Oct 2024 12:42:32 +0100 Subject: [PATCH 081/166] added new windows rec --- backend/Property.py | 35 ++----------------- .../tests/test_window_recommendations.py | 10 ++++-- 2 files changed, 10 insertions(+), 35 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index 77415d0e..584e1442 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -502,43 +502,12 @@ class Property: output["low_energy_lighting_ending"] = 100 output["lighting_energy_eff_ending"] = "Very Good" - if recommendation["type"] == "windows_glazing": - is_secondary_glazing = recommendation["is_secondary_glazing"] - output["multi_glaze_proportion_ending"] = 100 - if output["windows_energy_eff_ending"] not in ["Average", "Good", "Very Good"]: - output["windows_energy_eff_ending"] = "Average" if not is_secondary_glazing else "Good" - - if output["glazing_type_ending"] == "multiple": - pass - elif output["glazing_type_ending"] == "single": - output["glazing_type_ending"] = ( - "secondary" if is_secondary_glazing else "double" - ) - elif output["glazing_type_ending"] == "double": - output["glazing_type_ending"] = ( - "multiple" if is_secondary_glazing else "double" - ) - elif output["glazing_type_ending"] == "secondary": - output["glazing_type_ending"] = ( - "secondary" if is_secondary_glazing else "multiple" - ) - elif output["glazing_type_ending"] in ["triple", "high performance"]: - output["glazing_type_ending"] = "multiple" - else: - raise ValueError("Invalid glazing type - implement me") - - if is_secondary_glazing: - output["glazed_type_ending"] = "secondary glazing" - else: - output["glazed_type_ending"] = ( - "double glazing installed during or after 2002" - ) - if recommendation["type"] in [ "heating", "hot_water_tank_insulation", "heating_control", "secondary_heating", "internal_wall_insulation", "external_wall_insulation", "cavity_wall_insulation", "cylinder_thermostat", "loft_insulation", "room_roof_insulation", "flat_roof_insulation", - "solid_floor_insulation", "suspended_floor_insulation", "mixed_glazing" + "solid_floor_insulation", "suspended_floor_insulation", "mixed_glazing", + "windows_glazing" ]: # We update the data, as defined in the recommendaton for prefix in ["walls", "roof", "floor"]: diff --git a/recommendations/tests/test_window_recommendations.py b/recommendations/tests/test_window_recommendations.py index 45b03a5e..0e36d105 100644 --- a/recommendations/tests/test_window_recommendations.py +++ b/recommendations/tests/test_window_recommendations.py @@ -634,15 +634,21 @@ class TestWindowRecommendations: 'number_habitable_rooms_starting': 4.0, 'number_habitable_rooms_ending': 4.0, 'number_heated_rooms_starting': 4.0, 'number_heated_rooms_ending': 4.0, 'days_to_starting': 3642, 'days_to_ending': 3713, 'estimated_perimeter_starting': 23.430749027719962, - 'estimated_perimeter_ending': 23.430749027719962, 'id': '1+1' + 'estimated_perimeter_ending': 23.430749027719962, 'has_glazing_ending': True, + 'glazing_coverage_ending': 'full', 'id': '1+1' } assert simulated_data[0] == expected_simulated_outcome + # has_glazing_ending and glazing_coverage_ending are not in the starting record - test for this in case it + # changes + assert "has_glazing_ending" not in starting_record + assert "glazing_coverage_ending" not in starting_record + # Check which keys are different different = [] for k in simulated_data[0].keys(): - if k == "id": + if k in ["id", 'has_glazing_ending', 'glazing_coverage_ending']: continue if simulated_data[0][k] != starting_record[k]: different.append( From 97b9bf58ca897c284458aa674a05ec95a9404e16 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 1 Oct 2024 15:58:22 +0100 Subject: [PATCH 082/166] fixed unit tests --- backend/tests/test_property.py | 334 +++++++++++++++++---------------- 1 file changed, 171 insertions(+), 163 deletions(-) diff --git a/backend/tests/test_property.py b/backend/tests/test_property.py index 43149791..78f08f3c 100644 --- a/backend/tests/test_property.py +++ b/backend/tests/test_property.py @@ -1,9 +1,11 @@ +from datetime import datetime import pandas as pd import pytest from unittest.mock import Mock from backend.Property import Property from etl.epc_clean.EpcClean import EpcClean from etl.epc.Record import EPCRecord +from etl.bill_savings.KwhData import KwhData # Define some test data mock_epc_response = { @@ -17,12 +19,13 @@ mock_epc_response = { "built-form": "Detached", "inspection-date": "2023-06-01", 'lodgement-datetime': '2023-06-01 20:29:01', + 'lodgement-date': '2023-06-01', "some-other-key": "some-value", "roof-description": "pitched, no insulation", "walls-description": "Walls Description", - "windows-description": "Windows Description", - "mainheat-description": "Main Heating Description", - "hotwater-description": "Hot Water Description", + "windows-description": "Fully double glazed", + "mainheat-description": "Boiler and radiators, mains gas", + "hotwater-description": "From main system", "transaction-type": "rental", "lighting-description": "Good Lighting Efficiency", "energy-consumption-current": "50", @@ -39,7 +42,10 @@ mock_epc_response = { "total-floor-area": 100, "construction-age-band": "England and Wales: 1967-1975", "floor-description": "Floor Description", - "floor-level": "Ground" + "floor-level": "Ground", + "lighting-cost-current": 123, + "heating-cost-current": 800, + "hot-water-cost-current": 200 }, { "lmk-key": 2, @@ -49,12 +55,13 @@ mock_epc_response = { "built-form": "Detached", "inspection-date": "2023-05-01", 'lodgement-datetime': '2023-05-01 20:29:01', + 'lodgement-date': '2023-05-01', "some-other-key": "some-other-value", "roof-description": "Roof Description", "walls-description": "Walls Description", - "windows-description": "Windows Description", - "mainheat-description": "Main Heating Description", - "hotwater-description": "Hot Water Description", + "windows-description": "Fully double glazed", + "mainheat-description": "Boiler and radiators, mains gas", + "hotwater-description": "From main system", "transaction-type": "rental", "lighting-description": "Good Lighting Efficiency", "energy-consumption-current": "50", @@ -71,98 +78,10 @@ mock_epc_response = { "total-floor-area": 100, "construction-age-band": "England and Wales: 1967-1975", "floor-description": "Floor Description", - "floor-level": "Ground" - } - ] -} - -mock_epc_response_dupe = { - 'rows': [ - { - "lmk-key": 1, - "uprn": 1, - "number-habitable-rooms": 5, - "property-type": "House", - 'inspection-date': '2023-06-01', - 'lodgement-datetime': '2023-06-01 20:29:01', - 'some-other-key': 'some-value', 'roof-description': 'Roof Description', - 'walls-description': 'Walls Description', 'windows-description': 'Windows Description', - 'mainheat-description': 'Main Heating Description', 'hotwater-description': 'Hot Water Description', - "transaction-type": "rental", - "lighting-description": "Good Lighting Efficiency", - "energy-consumption-current": "50", - "co2-emissions-current": "123", - "mechanical-ventilation": "natural", - 'photo-supply': 0, - "solar-water-heating-flag": "N", - "wind-turbine-count": 0, - "extension-count": 0, - "heat-loss-corridor": "no corridor", - "unheated-corridor-length": 0, - "mains-gas-flag": "Y", - "floor-height": 2.5, - "total-floor-area": 100, - "construction-age-band": "England and Wales: 1967-1975", - "floor-description": "Floor Description", - "floor-level": "Ground" - }, - { - "lmk-key": 2, - "uprn": 2, - "number-habitable-rooms": 5, - "property-type": "House", - 'inspection-date': '2023-05-01', - 'lodgement-datetime': '2023-05-01 20:29:01', - 'some-other-key': 'some-other-value', - 'roof-description': 'Roof Description', 'walls-description': 'Walls Description', - 'windows-description': 'Windows Description', 'mainheat-description': 'Main Heating Description', - 'hotwater-description': 'Hot Water Description', - "transaction-type": "rental", - "lighting-description": "Good Lighting Efficiency", - "energy-consumption-current": "50", - "co2-emissions-current": "123", - "mechanical-ventilation": "natural", - 'photo-supply': 0, - "solar-water-heating-flag": "N", - "wind-turbine-count": 0, - "extension-count": 0, - "heat-loss-corridor": "no corridor", - "unheated-corridor-length": 0, - "mains-gas-flag": "Y", - "floor-height": 2.5, - "total-floor-area": 100, - "construction-age-band": "England and Wales: 1967-1975", - "floor-description": "Floor Description", - "floor-level": "Ground" - }, - { - "lmk-key": 3, - "uprn": 3, - "number-habitable-rooms": 5, - "property-type": "House", - 'inspection-date': '2023-06-01', - 'lodgement-datetime': '2023-06-01 20:29:01', - 'some-other-key': 'duplicate-date', - 'roof-description': 'Roof Description', - 'walls-description': 'Walls Description', 'windows-description': 'Windows Description', - 'mainheat-description': 'Main Heating Description', 'hotwater-description': 'Hot Water Description', - "transaction-type": "rental", - "lighting-description": "Good Lighting Efficiency", - "energy-consumption-current": "50", - "co2-emissions-current": "123", - "mechanical-ventilation": "natural", - 'photo-supply': 0, - "solar-water-heating-flag": "N", - "wind-turbine-count": 0, - "extension-count": 0, - "heat-loss-corridor": "no corridor", - "unheated-corridor-length": 0, - "mains-gas-flag": "Y", - "floor-height": 2.5, - "total-floor-area": 100, - "construction-age-band": "England and Wales: 1967-1975", - "floor-description": "Floor Description", - "floor-level": "Ground" + "floor-level": "Ground", + "lighting-cost-current": 123, + "heating-cost-current": 800, + "hot-water-cost-current": 200 } ] } @@ -170,34 +89,14 @@ mock_epc_response_dupe = { class TestProperty: - @pytest.fixture(autouse=True) - def mock_photo_supply_lookup(self): - return pd.DataFrame( - [ - dict( - tenure="rental (social)", - built_form="Detached", - property_type="House", - construction_age_band="England and Wales: 1967-1975", - is_flat=False, - is_pitched=True, - is_roof_room=False, - floor_area_decile=2, - photo_supply_median=40 - ) - ] - ) - - @pytest.fixture(autouse=True) - def mock_floor_area_decile_thresholds(self): - return pd.DataFrame( - {"floor_area_decile_thresholds": [0, 10, 30, 50]} - ) - @pytest.fixture(autouse=True) def property_instance(self, mock_cleaner): epc_record = EPCRecord() - epc_record.prepared_epc = mock_epc_response["rows"][0] + prepared_epc = mock_epc_response["rows"][0].copy() + # Replace hyphens with underscores + prepared_epc = {k.replace("-", "_"): v for k, v in prepared_epc.items()} + epc_record.prepared_epc = prepared_epc + epc_record.uprn = prepared_epc["uprn"] property_instance = Property(id=1, postcode="AB12CD", address="Test Address", epc_record=epc_record) property_instance.number_of_floors = 2 @@ -206,27 +105,6 @@ class TestProperty: property_instance.floor_height = 2.5 return property_instance - @pytest.fixture(autouse=True) - def property_instance_dupe_data(self): - epc_record = EPCRecord() - epc_record.prepared_epc = mock_epc_response_dupe["rows"][0] - property_instance_dupe_data = Property(id=2, postcode="AB12CD", address="Test Address", epc_record=epc_record) - return property_instance_dupe_data - - # @pytest.fixture - # def mock_epc_client(self): - # mock_epc_client = Mock(spec=EpcClient(auth_token="mocked_auth_token")) - # mock_epc_client.domestic.search.return_value = mock_epc_response.copy() - # mock_epc_client.auth_token = "mocked_auth_token" - # return mock_epc_client - # - # @pytest.fixture - # def mock_epc_client_dupe_data(self): - # mock_epc_client_dupe_data = Mock(spec=EpcClient(auth_token="mocked_auth_token")) - # mock_epc_client_dupe_data.domestic.search.return_value = mock_epc_response_dupe.copy() - # mock_epc_client_dupe_data.auth_token = "mocked_auth_token" - # return mock_epc_client_dupe_data - @pytest.fixture def mock_cleaner(self): lighting_averages = [ @@ -270,15 +148,59 @@ class TestProperty: "is_roof_room": False} ], "walls-description": [walls_data], - "windows-description": [{"original_description": "Windows Description"}], - "mainheat-description": [{"original_description": "Main Heating Description"}], - "hotwater-description": [{"original_description": "Hot Water Description"}], + "windows-description": [ + {'original_description': 'Fully double glazed', 'has_glazing': True, 'glazing_coverage': 'full', + 'glazing_type': 'double', 'no_data': False} + ], + "mainheat-description": [ + { + 'original_description': 'Boiler and radiators, mains gas', 'has_radiators': True, + 'has_fan_coil_units': False, + 'has_pipes_in_screed_above_insulation': False, 'has_pipes_in_insulated_timber_floor': False, + 'has_pipes_in_concrete_slab': False, 'has_boiler': True, 'has_air_source_heat_pump': False, + 'has_room_heaters': False, 'has_electric_storage_heaters': False, 'has_warm_air': False, + 'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False, + 'has_community_scheme': False, + 'has_ground_source_heat_pump': False, 'has_no_system_present': False, + 'has_portable_electric_heaters': False, + 'has_water_source_heat_pump': False, 'has_electric': False, 'has_mains_gas': True, + 'has_wood_logs': False, + 'has_coal': False, 'has_oil': False, 'has_wood_pellets': False, 'has_anthracite': False, + 'has_dual_fuel_mineral_and_wood': False, 'has_smokeless_fuel': False, 'has_lpg': False, + 'has_assumed': False, + 'has_electricaire': False, 'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False, + "has_electric_heat_pumps": False, + "has_micro-cogeneration": False + } + ], + "hotwater-description": [ + {'original_description': 'From main system', 'heater_type': None, 'system_type': 'from main system', + 'thermostat_characteristics': None, 'heating_scope': None, 'energy_recovery': None, + 'tariff_type': None, + 'extra_features': None, 'chp_systems': None, 'distribution_system': None, 'no_system_present': None, + 'assumed': False, "appliance": None} + ], "lighting-description": [{"original_description": "Good Lighting Efficiency"}], "floor-description": [ {"original_description": "Floor Description", "is_suspended": True, "another_property_below": False}] } return mock_cleaner + @pytest.fixture + def kwh_client(self): + kwh_client = KwhData(bucket="retrofit-data-dev", read_consumption_data=False) + # We fix this pricing table for these tests + kwh_client.retail_price_comparison = pd.DataFrame( + [ + { + "Date": datetime.today().strftime("%Y-%m-%d"), + 'Average standard variable tariff (Large legacy suppliers)': 1 + } + ] + ) + kwh_client.retail_price_comparison["Date"] = pd.to_datetime(kwh_client.retail_price_comparison["Date"]) + return kwh_client + def test_init(self): epc_record = EPCRecord() epc_record.prepared_epc = {"uprn": 1} @@ -292,13 +214,26 @@ class TestProperty: inst3 = Property(4, "AB12CD", "Test Address", epc_record=epc_record) assert inst3.data == {"uprn": 1} - def test_get_components( - self, property_instance, mock_cleaner, mock_photo_supply_lookup, mock_floor_area_decile_thresholds + def test_set_features( + self, property_instance, mock_cleaner, kwh_client, ): - property_instance.get_components( + kwh_predictions = { + "heating_kwh_predictions": pd.DataFrame( + [ + {"id": property_instance.uprn, "predictions": 12000} + ] + ), + "hotwater_kwh_predictions": pd.DataFrame( + [ + {"id": property_instance.uprn, "predictions": 3000} + ] + ), + } + + property_instance.set_features( mock_cleaner.cleaned, - photo_supply_lookup=mock_photo_supply_lookup, - floor_area_decile_thresholds=mock_floor_area_decile_thresholds + kwh_client, + kwh_predictions ) # Verify that the components are set correctly @@ -318,9 +253,32 @@ class TestProperty: "is_sandstone_or_limestone": False, "is_granite_or_whinstone": False, } - assert property_instance.windows == {"original_description": "Windows Description"} - assert property_instance.main_heating == {"original_description": "Main Heating Description"} - assert property_instance.hotwater == {"original_description": "Hot Water Description"} + assert property_instance.windows == { + 'original_description': 'Fully double glazed', 'has_glazing': True, 'glazing_coverage': 'full', + 'glazing_type': 'double', 'no_data': False + } + assert property_instance.main_heating == { + 'original_description': 'Boiler and radiators, mains gas', 'has_radiators': True, + 'has_fan_coil_units': False, 'has_pipes_in_screed_above_insulation': False, + 'has_pipes_in_insulated_timber_floor': False, 'has_pipes_in_concrete_slab': False, 'has_boiler': True, + 'has_air_source_heat_pump': False, 'has_room_heaters': False, 'has_electric_storage_heaters': False, + 'has_warm_air': False, 'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False, + 'has_community_scheme': False, 'has_ground_source_heat_pump': False, 'has_no_system_present': False, + 'has_portable_electric_heaters': False, 'has_water_source_heat_pump': False, 'has_electric': False, + 'has_mains_gas': True, 'has_wood_logs': False, 'has_coal': False, 'has_oil': False, + 'has_wood_pellets': False, 'has_anthracite': False, 'has_dual_fuel_mineral_and_wood': False, + 'has_smokeless_fuel': False, 'has_lpg': False, 'has_assumed': False, 'has_electricaire': False, + 'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False, 'has_electric_heat_pumps': False, + 'has_micro-cogeneration': False + } + + assert property_instance.hotwater == { + 'original_description': 'From main system', 'heater_type': None, + 'system_type': 'from main system', 'thermostat_characteristics': None, + 'heating_scope': None, 'energy_recovery': None, 'tariff_type': None, + 'extra_features': None, 'chp_systems': None, 'distribution_system': None, + 'no_system_present': None, 'assumed': False, 'appliance': None + } assert property_instance.wall_type == "cavity" @@ -330,11 +288,24 @@ class TestProperty: # Verify that ValueError is raised when EpcClean doesn't contain cleaned data with pytest.raises(ValueError, match="Cleaner does not contain cleaned data"): - property_instance.get_components(mock_cleaner.cleaned, pd.DataFrame(), pd.DataFrame()) + property_instance.set_features(mock_cleaner.cleaned, pd.DataFrame(), pd.DataFrame()) def test_get_components_no_attributes( - self, property_instance, mock_cleaner, mock_photo_supply_lookup, mock_floor_area_decile_thresholds + self, property_instance, mock_cleaner, kwh_client ): + kwh_predictions = { + "heating_kwh_predictions": pd.DataFrame( + [ + {"id": property_instance.uprn, "predictions": 12000} + ] + ), + "hotwater_kwh_predictions": pd.DataFrame( + [ + {"id": property_instance.uprn, "predictions": 3000} + ] + ), + } + # Modify the mock cleaner to have no attributes for a specific description mock_cleaner.cleaned = { "roof-description": [] @@ -351,23 +322,45 @@ class TestProperty: "is_sandstone_or_limestone": False, "is_granite_or_whinstone": False, } - property_instance.floor = { "is_suspended": False, "another_property_below": False, "is_solid": True } + property_instance.main_heating = { + 'original_description': 'Boiler and radiators, mains gas', 'has_radiators': True, + 'has_fan_coil_units': False, 'has_pipes_in_screed_above_insulation': False, + 'has_pipes_in_insulated_timber_floor': False, 'has_pipes_in_concrete_slab': False, 'has_boiler': True, + 'has_air_source_heat_pump': False, 'has_room_heaters': False, 'has_electric_storage_heaters': False, + 'has_warm_air': False, 'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False, + 'has_community_scheme': False, 'has_ground_source_heat_pump': False, 'has_no_system_present': False, + 'has_portable_electric_heaters': False, 'has_water_source_heat_pump': False, 'has_electric': False, + 'has_mains_gas': True, 'has_wood_logs': False, 'has_coal': False, 'has_oil': False, + 'has_wood_pellets': False, 'has_anthracite': False, 'has_dual_fuel_mineral_and_wood': False, + 'has_smokeless_fuel': False, 'has_lpg': False, 'has_assumed': False, 'has_electricaire': False, + 'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False, 'has_electric_heat_pumps': False, + 'has_micro-cogeneration': False + } + property_instance.hotwater = { + 'original_description': 'From main system', 'heater_type': None, 'system_type': 'from main system', + 'thermostat_characteristics': None, 'heating_scope': None, 'energy_recovery': None, + 'tariff_type': None, + 'extra_features': None, 'chp_systems': None, 'distribution_system': None, 'no_system_present': None, + 'assumed': False, "appliance": None + } # Assert backup cleaning has been applied - property_instance.get_components( - mock_cleaner.cleaned, mock_photo_supply_lookup, mock_floor_area_decile_thresholds + property_instance.set_features( + mock_cleaner.cleaned, + kwh_client, + kwh_predictions ) assert property_instance.roof["clean_description"] == "Pitched, no insulation" assert property_instance.roof["is_pitched"] def test_get_components_multiple_attributes( - self, property_instance, mock_cleaner, mock_photo_supply_lookup, mock_floor_area_decile_thresholds + self, property_instance, mock_cleaner, kwh_client ): # This shouldn't happen - it would mean a cleaning error property_instance.data["roof-description"] = "Roof Description" @@ -378,13 +371,27 @@ class TestProperty: ] } + kwh_predictions = { + "heating_kwh_predictions": pd.DataFrame( + [ + {"id": property_instance.uprn, "predictions": 12000} + ] + ), + "hotwater_kwh_predictions": pd.DataFrame( + [ + {"id": property_instance.uprn, "predictions": 3000} + ] + ), + } + # Verify that ValueError is raised when multiple attributes are found with pytest.raises(ValueError, match="Either No attributes or multiple found for roof-description"): - property_instance.get_components(cleaned, mock_photo_supply_lookup, mock_floor_area_decile_thresholds) + property_instance.set_features(cleaned, kwh_client, kwh_predictions) def test_set_spatial(self): epc_record = EPCRecord() epc_record.prepared_epc = mock_epc_response["rows"][0] + epc_record.uprn = mock_epc_response["rows"][0]["uprn"] prop = Property(1, postcode="AB12CD", address="Test Address", epc_record=epc_record) spatial1 = pd.DataFrame([{ @@ -418,6 +425,7 @@ class TestProperty: # floor, so we should set floor_level to 0 epc_record = EPCRecord() epc_record.prepared_epc = {'floor-level': '01', 'property-type': 'Flat'} + epc_record.uprn = 1 prop = Property(1, postcode="AB12CD", address="Test Address", epc_record=epc_record) prop.floor = { 'original_description': 'Solid, no insulation (assumed)', 'clean_description': 'Solid, no insulation', From dde8b2254a2c989119e8cf640a8d26535ca039a9 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 1 Oct 2024 16:03:46 +0100 Subject: [PATCH 083/166] adding backend install to actions file --- .github/workflows/unit_tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 39d285f2..339a3429 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -35,6 +35,7 @@ jobs: - name: Run tests with pytest run: | pip install -r model_data/requirements/dev.txt + pip install -r backend/requirements/base.txt pytest # - name: Upload coverage to Codecov # uses: codecov/codecov-action@v2 From c5b7775a2342f0a09e7acb49168472fe3a377611 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 1 Oct 2024 16:27:17 +0100 Subject: [PATCH 084/166] created hashing of uprn --- backend/SearchEpc.py | 3 +- etl/customers/eon/pilot_asset_list.py | 54 +++++++++++++-------------- 2 files changed, 29 insertions(+), 28 deletions(-) diff --git a/backend/SearchEpc.py b/backend/SearchEpc.py index b5ec8c46..e7e717ac 100644 --- a/backend/SearchEpc.py +++ b/backend/SearchEpc.py @@ -405,7 +405,8 @@ class SearchEpc: else: raise ValueError("Multiple UPRNs found - investigate me") - uprn = uprns.pop() if uprns else None + # If we do not have a UPRN, we create one based on a hash of the address & postcoce + uprn = uprns.pop() if uprns else hash(self.address1 + self.postcode) if self.fast: return newest_epc, [], {}, "", "", None diff --git a/etl/customers/eon/pilot_asset_list.py b/etl/customers/eon/pilot_asset_list.py index 05e459cb..298b34ba 100644 --- a/etl/customers/eon/pilot_asset_list.py +++ b/etl/customers/eon/pilot_asset_list.py @@ -170,39 +170,39 @@ def app(): # For each property, retrieve UPRN with from the Ordnance Survey API. To do this, I have created a free # trial with Ordnance Survey with my personal account as a temporary solution. # Let's just pull the full EPC data for this - asset_list_with_uprn = [] - for row, property_meta in tqdm(raw_asset_list_base.iterrows(), total=raw_asset_list_base.shape[0]): - if row <= 104: - continue - time.sleep(1.1) - searcher = SearchEpc( - address1=property_meta["address"], - postcode=property_meta["postcode"], - auth_token=EPC_AUTH_TOKEN, - os_api_key=ORDNANCE_SURVEY_API_KEY, - full_address=", ".join([property_meta["address"], property_meta["postcode"]]) - ) - - # Let's just find the UPRN - searcher.ordnance_survey_client.get_places_api() - - uprn = searcher.ordnance_survey_client.most_relevant_result["UPRN"] - - # searcher.find_property(skip_os=False) - - asset_list_with_uprn.append( - { - **property_meta, - "uprn": uprn, - } - ) + # asset_list_with_uprn = [] + # for row, property_meta in tqdm(raw_asset_list_base.iterrows(), total=raw_asset_list_base.shape[0]): + # if row <= 104: + # continue + # time.sleep(1.1) + # searcher = SearchEpc( + # address1=property_meta["address"], + # postcode=property_meta["postcode"], + # auth_token=EPC_AUTH_TOKEN, + # os_api_key=ORDNANCE_SURVEY_API_KEY, + # full_address=", ".join([property_meta["address"], property_meta["postcode"]]) + # ) + # + # # Let's just find the UPRN + # searcher.ordnance_survey_client.get_places_api() + # + # uprn = searcher.ordnance_survey_client.most_relevant_result["UPRN"] + # + # # searcher.find_property(skip_os=False) + # + # asset_list_with_uprn.append( + # { + # **property_meta, + # "uprn": uprn, + # } + # ) # Store this as a backup # import pandas as pd # asset_list_with_uprn_df = pd.DataFrame(asset_list_with_uprn) # asset_list_with_uprn_df.to_csv("eon_asset_list_with_uprn.csv", index=False) # Read in - # asset_list_with_uprn = pd.read_csv("eon_asset_list_with_uprn.csv").to_dict(orient="records") + asset_list_with_uprn = pd.read_csv("eon_asset_list_with_uprn.csv").to_dict(orient="records") # Store the asset list and create the portfolio payload asset_list_with_uprn_df = pd.DataFrame(asset_list_with_uprn) From fc1468efdac13a9331f64d7785917a7759b46b0c Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 1 Oct 2024 16:31:18 +0100 Subject: [PATCH 085/166] enforcing datetime is string --- etl/bill_savings/KwhData.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/etl/bill_savings/KwhData.py b/etl/bill_savings/KwhData.py index 6b5f594a..24ce9f2c 100644 --- a/etl/bill_savings/KwhData.py +++ b/etl/bill_savings/KwhData.py @@ -259,6 +259,9 @@ class KwhData: # Create new features: data['estimate_annual_kwh'] = data['energy-consumption-current'] * data['total-floor-area'] + # Ensure this is string, because we could have mixed types + data["lodgement-datetime"] = data["lodgement-datetime"].astype(str) + if save: self.model_training_data_filepath = f"energy_consumption/{self.run_date}/training_data.parquet" logger.info(f"Storing energy consumption dataset in s3 at {self.consumption_data_filepath}") From 6029aad07c1761dbcfaa1cb7231092f735f641f8 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 1 Oct 2024 19:32:46 +0100 Subject: [PATCH 086/166] fixing the example eon asset list --- backend/Property.py | 2 ++ backend/SearchEpc.py | 10 ++++-- etl/customers/eon/pilot_asset_list.py | 52 +++++++++++++-------------- 3 files changed, 34 insertions(+), 30 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index 584e1442..6577ba62 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -111,6 +111,8 @@ class Property: self.measures = ast.literal_eval(measures) if measures else None self.uprn = epc_record.get("uprn") + self.uprn_source = self.data["uprn-source"] + self.full_sap_epc = epc_record.get("full_sap_epc") self.in_conservation_area, self.is_listed, self.is_heritage = None, None, None self.restricted_measures = False diff --git a/backend/SearchEpc.py b/backend/SearchEpc.py index e7e717ac..367d8c85 100644 --- a/backend/SearchEpc.py +++ b/backend/SearchEpc.py @@ -126,6 +126,9 @@ class SearchEpc: combinations about the home to find the property """ + # If we create the uprn based on a hash, we mark it as simulated + UPRN_SOURCE_SIMULATED = "SIMULATED" + MAX_RETRIES = 5 SUCCESS = { @@ -405,8 +408,11 @@ class SearchEpc: else: raise ValueError("Multiple UPRNs found - investigate me") - # If we do not have a UPRN, we create one based on a hash of the address & postcoce - uprn = uprns.pop() if uprns else hash(self.address1 + self.postcode) + if uprns: + uprn = uprns.pop() + else: + newest_epc["uprn-source"] = self.UPRN_SOURCE_SIMULATED + uprn = hash(self.address1 + self.postcode) if self.fast: return newest_epc, [], {}, "", "", None diff --git a/etl/customers/eon/pilot_asset_list.py b/etl/customers/eon/pilot_asset_list.py index 298b34ba..afed61e4 100644 --- a/etl/customers/eon/pilot_asset_list.py +++ b/etl/customers/eon/pilot_asset_list.py @@ -170,39 +170,35 @@ def app(): # For each property, retrieve UPRN with from the Ordnance Survey API. To do this, I have created a free # trial with Ordnance Survey with my personal account as a temporary solution. # Let's just pull the full EPC data for this - # asset_list_with_uprn = [] - # for row, property_meta in tqdm(raw_asset_list_base.iterrows(), total=raw_asset_list_base.shape[0]): - # if row <= 104: - # continue - # time.sleep(1.1) - # searcher = SearchEpc( - # address1=property_meta["address"], - # postcode=property_meta["postcode"], - # auth_token=EPC_AUTH_TOKEN, - # os_api_key=ORDNANCE_SURVEY_API_KEY, - # full_address=", ".join([property_meta["address"], property_meta["postcode"]]) - # ) - # - # # Let's just find the UPRN - # searcher.ordnance_survey_client.get_places_api() - # - # uprn = searcher.ordnance_survey_client.most_relevant_result["UPRN"] - # - # # searcher.find_property(skip_os=False) - # - # asset_list_with_uprn.append( - # { - # **property_meta, - # "uprn": uprn, - # } - # ) + asset_list_with_uprn = [] + for row, property_meta in tqdm(raw_asset_list_base.iterrows(), total=raw_asset_list_base.shape[0]): + searcher = SearchEpc( + address1=property_meta["address"], + postcode=property_meta["postcode"], + auth_token=EPC_AUTH_TOKEN, + os_api_key=ORDNANCE_SURVEY_API_KEY, + full_address=", ".join([property_meta["address"], property_meta["postcode"]]) + ) + + searcher.find_property(skip_os=True) + uprn = searcher.uprn + # searcher.find_property(skip_os=False) + + asset_list_with_uprn.append( + { + **property_meta, + "uprn": uprn, + "matched_address": searcher.address1, + "matched_postcode": searcher.postcode + } + ) # Store this as a backup # import pandas as pd # asset_list_with_uprn_df = pd.DataFrame(asset_list_with_uprn) - # asset_list_with_uprn_df.to_csv("eon_asset_list_with_uprn.csv", index=False) + # asset_list_with_uprn_df.to_csv("eon_asset_list_with_uprn_2.csv", index=False) # Read in - asset_list_with_uprn = pd.read_csv("eon_asset_list_with_uprn.csv").to_dict(orient="records") + # asset_list_with_uprn = pd.read_csv("eon_asset_list_with_uprn.csv").to_dict(orient="records") # Store the asset list and create the portfolio payload asset_list_with_uprn_df = pd.DataFrame(asset_list_with_uprn) From 37bd18642b3b1ed54f880eb55558b9697d99c3a8 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 1 Oct 2024 19:39:20 +0100 Subject: [PATCH 087/166] debugging eon asset list --- etl/customers/eon/pilot_asset_list.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/etl/customers/eon/pilot_asset_list.py b/etl/customers/eon/pilot_asset_list.py index afed61e4..aca0884c 100644 --- a/etl/customers/eon/pilot_asset_list.py +++ b/etl/customers/eon/pilot_asset_list.py @@ -2,12 +2,11 @@ import time import pandas as pd -from utils.s3 import read_excel_from_s3 from backend.SearchEpc import SearchEpc from dotenv import load_dotenv import os from tqdm import tqdm -from utils.s3 import save_csv_to_s3 +from utils.s3 import save_csv_to_s3, read_excel_from_s3 # Read in the .env file in backend load_dotenv(dotenv_path="backend/.env") @@ -181,7 +180,10 @@ def app(): ) searcher.find_property(skip_os=True) - uprn = searcher.uprn + if searcher.newest_epc["uprn-source"] == SearchEpc.UPRN_SOURCE_SIMULATED: + uprn = None + else: + uprn = searcher.uprn # searcher.find_property(skip_os=False) asset_list_with_uprn.append( From e4aa4cbb2f1c363f56e588b5d5403fa5ca186908 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 2 Oct 2024 10:48:54 +0100 Subject: [PATCH 088/166] handling edge cases for solar api --- backend/Property.py | 13 +++++- backend/apis/GoogleSolarApi.py | 58 +++++++++++++++++++++++++ backend/app/plan/router.py | 15 +++++++ etl/spatial/OpenUprnClient.py | 22 +++++++++- recommendations/recommendation_utils.py | 2 +- 5 files changed, 107 insertions(+), 3 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index 6577ba62..eaa27359 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -18,6 +18,7 @@ from recommendations.recommendation_utils import ( get_wall_type, estimate_external_wall_area, estimate_windows, + estimate_pitched_roof_area ) from backend.ml_models.AnnualBillSavings import AnnualBillSavings from backend.app.utils import sap_to_epc @@ -631,7 +632,17 @@ class Property: self.solar_panel_configuration = solar_panel_configuration # We also set the roof area - self.roof_area = roof_area + if roof_area is None: + if self.roof["is_flat"]: + self.roof_area = estimate_pitched_roof_area( + floor_area=self.insulation_floor_area, + floor_height=self.floor_height + ) + else: + self.roof_area = self.insulation_floor_area + + else: + self.roof_area = roof_area def set_current_energy_bill(self, kwh_client, kwh_predictions): """ diff --git a/backend/apis/GoogleSolarApi.py b/backend/apis/GoogleSolarApi.py index c82c9c9a..4bb5ef37 100644 --- a/backend/apis/GoogleSolarApi.py +++ b/backend/apis/GoogleSolarApi.py @@ -589,3 +589,61 @@ class GoogleSolarApi: # we need to do is perform the solar analysis and then half the results. We set an indicator which # implies we should do this self.double_property = True + + @classmethod + def default_panel_performance(cls, property_instance): + """ + In a small number of cases, where properties have simulated uprns, we do not have a longitude and latitude + value and therefore we just return a default panel performance + :param property_instance: + :return: + """ + + cost_instance = Costs(property_instance=property_instance) + + # We return a 2.4 and 4 kwp system + panel_performance = pd.DataFrame( + [ + { + 'n_panels': 10, + 'yearly_dc_energy': 4000 * 0.99, # Assumed 99% efficient wattage -> dc + 'total_cost': cost_instance.solar_pv( + n_panels=10, has_battery=False, n_floors=property_instance.number_of_floors + )["total"], + 'weighted_ratio': None, + 'panneled_roof_area': 10 * 1.8, + 'array_wattage': 4000, + 'initial_ac_kwh_per_year': 4000 * 0.95, # Assumed 95% efficient wattage -> ac + 'lifetime_ac_kwh': None, + 'lifetime_dc_kwh': None, + 'roi': None, + 'generation_value': None, + 'generation_deficit': None, + 'expected_payback_years': None, + 'surplus': None, + 'combined_score': None, + 'rank': None + }, + { + 'n_panels': 6, + 'yearly_dc_energy': 2400 * 0.99, # Assumed 99% efficient wattage -> dc + 'total_cost': cost_instance.solar_pv( + n_panels=6, has_battery=False, n_floors=property_instance.number_of_floors + )["total"], + 'weighted_ratio': None, + 'panneled_roof_area': 6 * 1.8, + 'array_wattage': 2400, + 'initial_ac_kwh_per_year': 2400 * 0.95, # Assumed 95% efficient wattage -> ac + 'lifetime_ac_kwh': None, + 'lifetime_dc_kwh': None, + 'roi': None, + 'generation_value': None, + 'generation_deficit': None, + 'expected_payback_years': None, + 'surplus': None, + 'combined_score': None, + 'rank': None + }, + ] + ) + return panel_performance diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 6e4d8475..fb053ddb 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -636,6 +636,21 @@ async def trigger_plan(body: PlanTriggerRequest): if not property_instance.is_solar_pv_valid(): continue + if unit["longitude"] is None or unit["latitude"] is None: + # At this point, we've checked that solar PV is valid, and so we provide some defaults + + property_instance.set_solar_panel_configuration( + solar_panel_configuration={ + "insights_data": None, + "panel_performance": GoogleSolarApi.default_panel_performance( + property_instance=property_instance + ), + "unit_share_of_energy": 1 + }, + roof_area=None + ) + continue + # We check if we have a solar non-invasive recommendation if [r for r in property_instance.non_invasive_recommendations if r["type"] == "solar_pv"]: continue diff --git a/etl/spatial/OpenUprnClient.py b/etl/spatial/OpenUprnClient.py index 11827f8d..5c43347a 100644 --- a/etl/spatial/OpenUprnClient.py +++ b/etl/spatial/OpenUprnClient.py @@ -5,6 +5,7 @@ import geopandas as gpd from utils.logger import setup_logger from utils.s3 import read_io_from_s3, save_dataframe_to_s3_parquet, read_dataframe_from_s3_parquet from backend.Property import Property +from backend.SearchEpc import SearchEpc logger = setup_logger() @@ -151,7 +152,7 @@ class OpenUprnClient: bucket_name=bucket_name, file_key="spatial/filename_meta.parquet" ) - uprns = [p.uprn for p in input_properties] + uprns = [p.uprn for p in input_properties if p.uprn_source != SearchEpc.UPRN_SOURCE_SIMULATED] uprn_map = cls.make_uprn_map(uprns, uprn_filenames) for filename, associated_uprn in tqdm(uprn_map.items(), total=len(uprn_map)): @@ -165,6 +166,9 @@ class OpenUprnClient: if p.uprn in associated_uprn: p.set_spatial(spatial_df[spatial_df["UPRN"] == p.uprn]) + if p.uprn_source == SearchEpc.UPRN_SOURCE_SIMULATED: + p.set_spatial(cls.empty_spatial_df()) + # Perform a final check to ensure that all properties have spatial data for p in input_properties: if p.spatial is None: @@ -172,6 +176,22 @@ class OpenUprnClient: return input_properties + @staticmethod + def empty_spatial_df(): + return pd.DataFrame( + [ + { + "X_COORDINATE": None, + "Y_COORDINATE": None, + "LATITUDE": None, + "LONGITUDE": None, + "conservation_status": False, + "is_listed_building": False, + "is_heritage_building": False, + } + ] + ) + @classmethod def get_spatial_data(cls, uprns: list[int], bucket_name): """ diff --git a/recommendations/recommendation_utils.py b/recommendations/recommendation_utils.py index 883a387b..72707ded 100644 --- a/recommendations/recommendation_utils.py +++ b/recommendations/recommendation_utils.py @@ -655,7 +655,7 @@ def convert_thickness_to_numeric(string_thickness, is_pitched, is_flat): return int(string_thickness) -def esimtate_pitched_roof_area(floor_area: float, floor_height: float) -> float: +def estimate_pitched_roof_area(floor_area: float, floor_height: float) -> float: """ This function will estimate the area of a pitched roof, given the floor area below the roof and the floor height of the property. From 46f513a4f274db29208d5e660039cf046d6698da Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 2 Oct 2024 11:29:42 +0100 Subject: [PATCH 089/166] tidying up solar api code --- backend/Property.py | 22 ++- backend/apis/GoogleSolarApi.py | 229 +++++++++++++++++++++- backend/app/plan/router.py | 177 ++--------------- recommendations/SolarPvRecommendations.py | 4 +- 4 files changed, 249 insertions(+), 183 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index eaa27359..418b0368 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -631,17 +631,23 @@ class Property: """ self.solar_panel_configuration = solar_panel_configuration + if not self.roof["is_flat"]: + default_roof_area = estimate_pitched_roof_area( + floor_area=self.insulation_floor_area, + floor_height=self.floor_height + ) + else: + default_roof_area = self.insulation_floor_area + # We also set the roof area if roof_area is None: - if self.roof["is_flat"]: - self.roof_area = estimate_pitched_roof_area( - floor_area=self.insulation_floor_area, - floor_height=self.floor_height - ) - else: - self.roof_area = self.insulation_floor_area - + self.roof_area = default_roof_area else: + # Perform a comparison between the default_roof_area and roof_area + difference = abs(default_roof_area - roof_area) + if difference / default_roof_area > 0.1: + raise Exception("Investigate difference in roof area") + self.roof_area = roof_area def set_current_energy_bill(self, kwh_client, kwh_predictions): diff --git a/backend/apis/GoogleSolarApi.py b/backend/apis/GoogleSolarApi.py index 4bb5ef37..bf67a786 100644 --- a/backend/apis/GoogleSolarApi.py +++ b/backend/apis/GoogleSolarApi.py @@ -1,16 +1,22 @@ +import time +import requests import pandas as pd import numpy as np -from recommendations.Costs import MCS_SOLAR_PV_COST_DATA -from backend.ml_models.AnnualBillSavings import AnnualBillSavings -import requests +from typing import List from functools import lru_cache -import time -from backend.app.db.functions.solar_functions import get_solar_data, store_batch_data -from utils.logger import setup_logger from sklearn.preprocessing import MinMaxScaler -from recommendations.Costs import Costs +from tqdm import tqdm from math import sin, cos, sqrt, atan2, radians +from utils.logger import setup_logger +from recommendations.Costs import Costs, MCS_SOLAR_PV_COST_DATA +from etl.bill_savings.EnergyConsumptionModel import EnergyConsumptionModel +from backend.ml_models.AnnualBillSavings import AnnualBillSavings +from backend.Property import Property +from backend.app.db.functions.solar_functions import get_solar_data, store_batch_data +import backend.app.assumptions as assumptions +from backend.app.plan.schemas import PlanTriggerRequest + logger = setup_logger() @@ -590,6 +596,215 @@ class GoogleSolarApi: # implies we should do this self.double_property = True + @staticmethod + def prepare_input_data( + input_properties: List[Property], + energy_consumption_client: EnergyConsumptionModel, + body: PlanTriggerRequest + ): + """ + :param input_properties: List of properties + :param energy_consumption_client: EnergyConsumptionModel instance + :param body: PlanTriggerRequest instance + This sets up the data required to make the solar api request + :return: + """ + + building_solar_config = [ + { + "building_id": p.building_id, + "longitude": p.spatial["longitude"], + "latitude": p.spatial["latitude"], + # Energy consumption is adjusted for the property's expected post retrofit state + # We set the target rating to EPC C, which is the typical EPC rating we would expect the + # property to achieve post retrofit of just the fabric + "energy_consumption": energy_consumption_client.estimate_new_consumption( + current_energy_efficiency=p.data["current-energy-efficiency"], + target_efficiency="69", + current_consumption=p.estimate_electrical_consumption( + assumed_ashp_efficiency=assumptions.AVERAGE_ASHP_EFFICIENCY, exclusions=body.exclusions + ) + ), + "property_id": p.id, + "uprn": p.uprn + } for p in input_properties if p.building_id is not None + ] + unit_solar_config = [ + { + "longitude": p.spatial["longitude"], + "latitude": p.spatial["latitude"], + # Energy consumption is adjusted for the property's expected post retrofit state + # We set the target rating to EPC C, which is the typical EPC rating we would expect the + # property to achieve post retrofit of just the fabric + "energy_consumption": energy_consumption_client.estimate_new_consumption( + current_energy_efficiency=p.data["current-energy-efficiency"], + target_efficiency="69", + current_consumption=p.estimate_electrical_consumption( + assumed_ashp_efficiency=assumptions.AVERAGE_ASHP_EFFICIENCY, exclusions=body.exclusions + ), + ), + "property_id": p.id, + "uprn": p.uprn + } for p in input_properties if p.building_id is None + ] + + return building_solar_config, unit_solar_config + + @classmethod + def building_solar_analysis( + cls, building_solar_config: List, input_properties: List[Property], session, google_solar_api_key: str + ): + """ + Perform the solar analysis for the building level + :param building_solar_config: List of building solar configurations + :param input_properties: List of properties + :param session: Database session + :param google_solar_api_key: Google Solar API key + :return: + """ + + if not building_solar_config: + return input_properties + + # Find the unique longitude and latitude pairs for each building id + unique_coordinates = {} + building_uprns = {} + for entry in building_solar_config: + building_id = entry['building_id'] + coordinate_pair = {'longitude': entry['longitude'], 'latitude': entry['latitude']} + + if building_id not in unique_coordinates: + unique_coordinates[building_id] = [] + + if coordinate_pair not in unique_coordinates[building_id]: + unique_coordinates[building_id].append(coordinate_pair) + + if building_id not in building_uprns: + building_uprns[building_id] = [] + + if entry['uprn'] not in building_uprns[building_id]: + building_uprns[building_id].append( + { + "uprn": entry['uprn'], "longitude": entry['longitude'], "latitude": entry['latitude'] + } + ) + + solar_panel_configuration = {} + for building_id, coordinates in unique_coordinates.items(): + if len(coordinates) > 1: + raise NotImplementedError("more than one coordinate for a building - handle me") + + coordinates = coordinates[0] + energy_consumption = sum( + [entry['energy_consumption'] for entry in building_solar_config if entry['building_id'] == building_id] + ) + solar_api_client = cls(api_key=google_solar_api_key) + solar_api_client.get( + longitude=coordinates["longitude"], + latitude=coordinates["latitude"], + energy_consumption=energy_consumption, + is_building=True, + session=session + ) + solar_panel_configuration[building_id] = { + "insights_data": solar_api_client.insights_data, + "panel_performance": solar_api_client.panel_performance, + "n_units": len([entry for entry in building_solar_config if entry['building_id'] == building_id]) + } + + # Store the data in the database + # TODO: Rather than just doing a straight insert, we should overwrite what's already there if it + # exists + solar_api_client.save_to_db( + session=session, uprns_to_location=building_uprns[building_id], scenario_type="building" + ) + + # Insert this into the properties that have this building id + for p in input_properties: + if p.building_id == building_id: + unit_solar_panel_configuration = solar_panel_configuration[building_id].copy() + + unit_solar_panel_configuration["unit_share_of_energy"] = ( + [x for x in building_solar_config if x["property_id"] == p.id][0]["energy_consumption"] / + energy_consumption + ) + p.set_solar_panel_configuration(unit_solar_panel_configuration) + + return input_properties + + @classmethod + def unit_solar_analysis( + cls, unit_solar_config: List, input_properties: List[Property], session, body, google_solar_api_key: str + ): + + if not unit_solar_config: + return input_properties + + # Model the solar potential at the property level + for unit in tqdm(unit_solar_config): + + # We don't need to do this if we have global inclusions that don't include solar + if body.inclusions: + if "solar_pv" not in body.inclusions: + continue + + property_instance = [p for p in input_properties if p.id == unit["property_id"]][0] + # At this level, we check if the property is suitable for solar and if now, skip + # Or if we have a solar non-invasive recommendation + if ( + (not property_instance.is_solar_pv_valid()) or + [r for r in property_instance.non_invasive_recommendations if r["type"] == "solar_pv"] + ): + continue + + if unit["longitude"] is None or unit["latitude"] is None: + # At this point, we've checked that solar PV is valid, and so we provide some defaults + + property_instance.set_solar_panel_configuration( + solar_panel_configuration={ + "insights_data": None, + "panel_performance": cls.default_panel_performance(property_instance=property_instance), + "unit_share_of_energy": 1 + }, + roof_area=None + ) + continue + + solar_api_client = cls(api_key=google_solar_api_key) + solar_api_client.get( + longitude=unit["longitude"], + latitude=unit["latitude"], + energy_consumption=unit["energy_consumption"], + is_building=False, + session=session, + uprn=unit["uprn"], + property_instance=property_instance + ) + + # Store the data in the database + solar_api_client.save_to_db( + session=session, + uprns_to_location=[ + { + "uprn": property_instance.uprn, + "longitude": property_instance.spatial["longitude"], + "latitude": property_instance.spatial["latitude"] + } + ], + scenario_type="unit" + ) + + property_instance.set_solar_panel_configuration( + solar_panel_configuration={ + "insights_data": solar_api_client.insights_data, + "panel_performance": solar_api_client.panel_performance, + "unit_share_of_energy": 1 + }, + roof_area=solar_api_client.roof_area + ) + + return input_properties + @classmethod def default_panel_performance(cls, property_instance): """ diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index fb053ddb..e8543930 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -513,179 +513,24 @@ async def trigger_plan(body: PlanTriggerRequest): [p.set_features(cleaned=cleaned, kwh_client=kwh_client, kwh_predictions=kwh_preds) for p in input_properties] logger.info("Performing solar analysis") - # TODO: Tidy this up # TODO: If a property is semi-detached, we might get roof surfaces for the main building + the neighbour # TODO: If we can't get high image quality, should we use the solar API? Maybe just for semi-detached units with # extensions, since it doesn't seem to do a great job # TODO: For simple properties, we should do a comparison/check between the solar API's roof area and the # basic estimate of roof area - building_ids = [ - { - "building_id": p.building_id, - "longitude": p.spatial["longitude"], - "latitude": p.spatial["latitude"], - # Energy consumption is adjusted for the property's expected post retrofit state - # We set the target rating to EPC C, which is the typical EPC rating we would expect the - # property to achieve post retrofit of just the fabric - "energy_consumption": energy_consumption_client.estimate_new_consumption( - current_energy_efficiency=p.data["current-energy-efficiency"], - target_efficiency="69", - current_consumption=p.estimate_electrical_consumption( - assumed_ashp_efficiency=assumptions.AVERAGE_ASHP_EFFICIENCY, exclusions=body.exclusions - ) - ), - "property_id": p.id, - "uprn": p.uprn - } for p in input_properties if p.building_id is not None - ] - individual_units = [ - { - "longitude": p.spatial["longitude"], - "latitude": p.spatial["latitude"], - # Energy consumption is adjusted for the property's expected post retrofit state - # We set the target rating to EPC C, which is the typical EPC rating we would expect the - # property to achieve post retrofit of just the fabric - "energy_consumption": energy_consumption_client.estimate_new_consumption( - current_energy_efficiency=p.data["current-energy-efficiency"], - target_efficiency="69", - current_consumption=p.estimate_electrical_consumption( - assumed_ashp_efficiency=assumptions.AVERAGE_ASHP_EFFICIENCY, exclusions=body.exclusions - ), - ), - "property_id": p.id, - "uprn": p.uprn - } for p in input_properties if p.building_id is None - ] - if building_ids: - # Find the unique longitude and latitude pairs for each building id - unique_coordinates = {} - building_uprns = {} - for entry in building_ids: - building_id = entry['building_id'] - coordinate_pair = {'longitude': entry['longitude'], 'latitude': entry['latitude']} + building_solar_config, unit_solar_config = GoogleSolarApi.prepare_input_data( + input_properties=input_properties, + energy_consumption_client=energy_consumption_client, + body=body + ) - if building_id not in unique_coordinates: - unique_coordinates[building_id] = [] - - if coordinate_pair not in unique_coordinates[building_id]: - unique_coordinates[building_id].append(coordinate_pair) - - if building_id not in building_uprns: - building_uprns[building_id] = [] - - if entry['uprn'] not in building_uprns[building_id]: - building_uprns[building_id].append( - { - "uprn": entry['uprn'], "longitude": entry['longitude'], "latitude": entry['latitude'] - } - ) - - solar_panel_configuration = {} - for building_id, coordinates in unique_coordinates.items(): - if len(coordinates) > 1: - raise NotImplementedError("more than one coordinate for a building - handle me") - - coordinates = coordinates[0] - energy_consumption = sum( - [entry['energy_consumption'] for entry in building_ids if entry['building_id'] == building_id] - ) - solar_api_client = GoogleSolarApi(api_key=get_settings().GOOGLE_SOLAR_API_KEY) - solar_api_client.get( - longitude=coordinates["longitude"], - latitude=coordinates["latitude"], - energy_consumption=energy_consumption, - is_building=True, - session=session - ) - solar_panel_configuration[building_id] = { - "insights_data": solar_api_client.insights_data, - "panel_performance": solar_api_client.panel_performance, - "n_units": len([entry for entry in building_ids if entry['building_id'] == building_id]) - } - - # Store the data in the database - # TODO: Rather than just doing a straight insert, we should overwrite what's already there if it - # exists - solar_api_client.save_to_db( - session=session, uprns_to_location=building_uprns[building_id], scenario_type="building" - ) - - # Insert this into the properties that have this building id - for p in input_properties: - if p.building_id == building_id: - unit_solar_panel_configuration = solar_panel_configuration[building_id].copy() - - unit_solar_panel_configuration["unit_share_of_energy"] = ( - [x for x in building_ids if x["property_id"] == p.id][0]["energy_consumption"] / - energy_consumption - ) - p.set_solar_panel_configuration(unit_solar_panel_configuration) - if individual_units: - # Model the solar potential at the property level - for unit in tqdm(individual_units): - - # TODO: Tidy up this code - # We don't need to do this if we have global inclusions that don't include solar - if body.inclusions: - if "solar_pv" not in body.inclusions: - continue - - property_instance = [p for p in input_properties if p.id == unit["property_id"]][0] - # At this level, we check if the property is suitable for solar and if now, skip - if not property_instance.is_solar_pv_valid(): - continue - - if unit["longitude"] is None or unit["latitude"] is None: - # At this point, we've checked that solar PV is valid, and so we provide some defaults - - property_instance.set_solar_panel_configuration( - solar_panel_configuration={ - "insights_data": None, - "panel_performance": GoogleSolarApi.default_panel_performance( - property_instance=property_instance - ), - "unit_share_of_energy": 1 - }, - roof_area=None - ) - continue - - # We check if we have a solar non-invasive recommendation - if [r for r in property_instance.non_invasive_recommendations if r["type"] == "solar_pv"]: - continue - solar_api_client = GoogleSolarApi(api_key=get_settings().GOOGLE_SOLAR_API_KEY) - solar_api_client.get( - longitude=unit["longitude"], - latitude=unit["latitude"], - energy_consumption=unit["energy_consumption"], - is_building=False, - session=session, - uprn=unit["uprn"], - property_instance=property_instance - ) - - # Store the data in the database - solar_api_client.save_to_db( - session=session, - uprns_to_location=[ - { - "uprn": property_instance.uprn, - "longitude": property_instance.spatial["longitude"], - "latitude": property_instance.spatial["latitude"] - } - ], - scenario_type="unit" - ) - - property_instance.set_solar_panel_configuration( - solar_panel_configuration={ - "insights_data": solar_api_client.insights_data, - "panel_performance": solar_api_client.panel_performance, - "unit_share_of_energy": 1 - }, - roof_area=solar_api_client.roof_area - ) + input_properties = GoogleSolarApi.building_solar_analysis( + building_solar_config=building_solar_config, + input_properties=input_properties, + session=session, + google_solar_api_key=get_settings().GOOGLE_SOLAR_API_KEY + ) logger.info("Identifying property recommendations") recommendations = {} diff --git a/recommendations/SolarPvRecommendations.py b/recommendations/SolarPvRecommendations.py index dc11ce4a..bb38c73c 100644 --- a/recommendations/SolarPvRecommendations.py +++ b/recommendations/SolarPvRecommendations.py @@ -2,7 +2,7 @@ import numpy as np import pandas as pd from recommendations.Costs import Costs -from recommendations.recommendation_utils import override_costs, esimtate_pitched_roof_area +from recommendations.recommendation_utils import override_costs, estimate_pitched_roof_area class SolarPvRecommendations: @@ -174,7 +174,7 @@ class SolarPvRecommendations: if self.property.roof["is_flat"]: roof_area = self.property.insulation_floor_area else: - roof_area = esimtate_pitched_roof_area( + roof_area = estimate_pitched_roof_area( floor_area=self.property.insulation_floor_area, floor_height=self.property.data["floor-height"] ) solar_configurations = pd.DataFrame( From 33bc38fdc8bc766b855a0690dbe4d86c148772cc Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 2 Oct 2024 11:42:18 +0100 Subject: [PATCH 090/166] added call to unit solar api call --- backend/app/plan/router.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index e8543930..8ae57c92 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -516,8 +516,6 @@ async def trigger_plan(body: PlanTriggerRequest): # TODO: If a property is semi-detached, we might get roof surfaces for the main building + the neighbour # TODO: If we can't get high image quality, should we use the solar API? Maybe just for semi-detached units with # extensions, since it doesn't seem to do a great job - # TODO: For simple properties, we should do a comparison/check between the solar API's roof area and the - # basic estimate of roof area building_solar_config, unit_solar_config = GoogleSolarApi.prepare_input_data( input_properties=input_properties, @@ -532,6 +530,14 @@ async def trigger_plan(body: PlanTriggerRequest): google_solar_api_key=get_settings().GOOGLE_SOLAR_API_KEY ) + input_properties = GoogleSolarApi.unit_solar_analysis( + unit_solar_config=unit_solar_config, + input_properties=input_properties, + session=session, + body=body, + google_solar_api_key=get_settings().GOOGLE_SOLAR_API_KEY + ) + logger.info("Identifying property recommendations") recommendations = {} recommendations_scoring_data = [] From 97dfa3e57210065fdf954bbcdcebb8407dddcff8 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 2 Oct 2024 13:50:31 +0100 Subject: [PATCH 091/166] removed temp debugging code --- backend/Property.py | 11 +- backend/app/plan/router.py | 167 +------------------------ recommendations/RoofRecommendations.py | 9 +- 3 files changed, 14 insertions(+), 173 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index 418b0368..55e3a912 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -639,15 +639,16 @@ class Property: else: default_roof_area = self.insulation_floor_area + # Keep a record + self.roof_area_comparison = { + "api": roof_area, + "estimated": default_roof_area + } + # We also set the roof area if roof_area is None: self.roof_area = default_roof_area else: - # Perform a comparison between the default_roof_area and roof_area - difference = abs(default_roof_area - roof_area) - if difference / default_roof_area > 0.1: - raise Exception("Investigate difference in roof area") - self.roof_area = roof_area def set_current_energy_bill(self, kwh_client, kwh_predictions): diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 8ae57c92..9d77c1a1 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -511,12 +511,12 @@ async def trigger_plan(body: PlanTriggerRequest): input_properties = OpenUprnClient.set_spatial_data(input_properties, bucket_name=get_settings().DATA_BUCKET) [p.set_features(cleaned=cleaned, kwh_client=kwh_client, kwh_predictions=kwh_preds) for p in input_properties] - logger.info("Performing solar analysis") # TODO: If a property is semi-detached, we might get roof surfaces for the main building + the neighbour # TODO: If we can't get high image quality, should we use the solar API? Maybe just for semi-detached units with # extensions, since it doesn't seem to do a great job + logger.info("Performing solar analysis") building_solar_config, unit_solar_config = GoogleSolarApi.prepare_input_data( input_properties=input_properties, energy_consumption_client=energy_consumption_client, @@ -595,171 +595,6 @@ async def trigger_plan(body: PlanTriggerRequest): scoring_epcs.extend(property_instance.updated_simulation_epcs) recommendations[property_id] = recommendations_with_impact - # For Debugging - # recommendation_impact_df = [] - # for property_id in recommendations.keys(): - # for recs_by_type in recommendations[property_id]: - # for rec in recs_by_type: - # recommendation_impact_df.append( - # { - # "property_id": property_id, - # "uprn": [p.uprn for p in input_properties if p.id == property_id][0], - # "address": [p.address for p in input_properties if p.id == property_id][0], - # "recommendation_id": rec["recommendation_id"], - # "type": rec["type"], - # "description": rec["description"], - # "sap_points": rec["sap_points"], - # "co2_equivalent_savings": rec["co2_equivalent_savings"], - # "heat_demand": rec["heat_demand"] - # } - # ) - # recommendation_impact_df = pd.DataFrame(recommendation_impact_df) - # - # surveyed_uprns = [ - # 10024087855, 121016117, 121016124, - # 10024087902, 121016121, 121016128 - # ] - # recommendation_impact_df = recommendation_impact_df[recommendation_impact_df["uprn"].isin(surveyed_uprns)] - # # recommendation_impact_df = recommendation_impact_df[recommendation_impact_df["type"].isin( - # # ["windows_glazing", "internal_wall_insulation"]) - # # ] - # - # actual_impacts_df = pd.DataFrame( - # [ - # # 10024087855 - # {"uprn": 10024087855, "type": "internal_wall_insulation", "actual_sap_points": 5}, - # {"uprn": 10024087855, "type": "draught_proofing", "actual_sap_points": 2}, - # {"uprn": 10024087855, "type": "low_energy_lighting", "actual_sap_points": 0}, - # {"uprn": 10024087855, "type": "windows_glazing", "actual_sap_points": 4}, - # # 121016117 - # {"uprn": 121016117, "type": "internal_wall_insulation", "actual_sap_points": 6}, - # {"uprn": 121016117, "type": "draught_proofing", "actual_sap_points": 1}, - # {"uprn": 121016117, "type": "low_energy_lighting", "actual_sap_points": 1}, - # {"uprn": 121016117, "type": "windows_glazing", "actual_sap_points": 4}, - # # 121016124 - # {"uprn": 121016124, "type": "internal_wall_insulation", "actual_sap_points": 8}, - # {"uprn": 121016124, "type": "low_energy_lighting", "actual_sap_points": 2}, - # {"uprn": 121016124, "type": "windows_glazing", "actual_sap_points": 5}, - # # 10024087902 - # {"uprn": 10024087902, "type": "room_roof_insulation", "actual_sap_points": 16}, - # {"uprn": 10024087902, "type": "internal_wall_insulation", "actual_sap_points": 2}, - # {"uprn": 10024087902, "type": "low_energy_lighting", "actual_sap_points": 0}, - # # 121016121 - # {"uprn": 121016121, "type": "internal_wall_insulation", "actual_sap_points": 5}, - # {"uprn": 121016121, "type": "suspended_floor_insulation", "actual_sap_points": 2}, - # {"uprn": 121016121, "type": "draught_proofing", "actual_sap_points": 1}, - # {"uprn": 121016121, "type": "windows_glazing", "actual_sap_points": 3}, - # # 121016128 - # {"uprn": 121016128, "type": "internal_wall_insulation", "actual_sap_points": 6}, - # {"uprn": 121016128, "type": "suspended_floor_insulation", "actual_sap_points": 1}, - # {"uprn": 121016128, "type": "draught_proofing", "actual_sap_points": 1}, - # {"uprn": 121016128, "type": "low_energy_lighting", "actual_sap_points": 1}, - # {"uprn": 121016128, "type": "windows_glazing", "actual_sap_points": 3}, - # ] - # ) - # - # comparison = recommendation_impact_df.merge( - # actual_impacts_df, how="inner", on=["uprn", "type"] - # ) - # - # print(recommendation_impact_df.groupby(["uprn"])["sap_points"].sum()) - # property_recs = recommendation_impact_df[recommendation_impact_df["uprn"] == 121016128] - # property = [p for p in input_properties if p.uprn == 121016128][0] - # print(property.data["current-energy-efficiency"]) - # print(property_recs["sap_points"].sum()) - # print(property_recs["type"]) - # print(float(property.data["current-energy-efficiency"]) + property_recs["sap_points"].sum()) - # recommendations[property.id][2][0]["simulation_config"] - - # from utils.s3 import read_dataframe_from_s3_parquet - # training_data = read_dataframe_from_s3_parquet( - # bucket_name="retrofit-data-dev", - # file_key="sap_change_model/2024-08-06-11-19-49/dataset_rooms.parquet" - # ) - # import pickle - # with open("delete_me.pkl", "wb") as f: - # pickle.dump(training_data, f) - - # Read in the pickle - import pickle - with open("delete_me.pkl", "rb") as f: - training_data = pickle.load(f) - - # How do we simulate windows: - ending_cols = [col for col in training_data.columns if col.endswith("_ending")] - starting = {} - for c in ending_cols: - starting_colname = c.replace("_ending", "_starting") - if starting_colname in training_data.columns: - starting[c] = starting_colname - else: - starting[c] = c.replace("_ending", "") - - allowed_to_change = [ - # Windows - "windows_energy_eff_ending", - "glazed_type_ending", - "glazing_type_ending", - "multi_glaze_proportion_ending", - - # Other - "sap_ending", - "heat_demand_ending", - "carbon_ending", - "estimated_perimeter_ending", - "lodgement_year_ending", - "lodgement_month_ending", - "days_to_ending", - "number_habitable_rooms_ending", - "number_heated_rooms_ending", - ] - fixed = [c for c in ending_cols if c not in allowed_to_change + ["uprn"]] - training_fixed = training_data.copy() - for col in fixed: - starting_col = starting[col] - training_fixed = training_fixed[training_fixed[col] == training_fixed[starting_col]] - - training_fixed = training_fixed.reset_index(drop=True) - - # Get the recommendation config for this uprn - uprn = 121016121 - property_instance = [p for p in input_properties if p.uprn == uprn][0] - property_recs = recommendations[property_instance.id] - window_recs = [r for r in property_recs if r[0]["type"] == "windows_glazing"][0] - window_recs[0].keys() - window_recs[0]["description_simulation"]["multi-glaze-proportion"] - # TODO: - In description_simulation for windows, we update glazed-type but in the model training data there - # is a column called "glazing-type". - # - We don't update glazed-area (should be "Much More Than Typical" most likely? Or Normal??) - # TODO: I think we update eveything that we actually need to, when simulating the recommendation impact for the - # ML models - # TODO: Secondary glazing appears to go to "Good", not "Average". Investigate why - # TODO: For the two properties, force recommendations for double glazing and check impact - - z = training_data[training_data["glazed_type_ending"] == "secondary glazing"] - z = z[z["multi_glaze_proportion_ending"] == 100] - z["windows_energy_eff_ending"].value_counts() - - # Find the things that change - example = training_fixed.iloc[3] - for _, example in training_fixed.iterrows(): - things_that_change = [] - for c in ending_cols: - if example[c] != example[starting[c]]: - things_that_change.append(c) - if len(things_that_change) > 4: - print(things_that_change) - print(example["uprn"]) - # blah - - # 100051011370 (doesn't change in actual glazing) - # example["glazed_type_ending"] - # double glazing installed before 2002 - # example["glazed_type_starting"] - # double glazing, unknown install date - - # 100040925015 - # We call the API with the scoring epcs scoring_epcs = pd.DataFrame(scoring_epcs) scoring_epcs = kwh_client.transform(data=scoring_epcs, cleaned=cleaned) diff --git a/recommendations/RoofRecommendations.py b/recommendations/RoofRecommendations.py index 2a77a3a5..4d22eb2d 100644 --- a/recommendations/RoofRecommendations.py +++ b/recommendations/RoofRecommendations.py @@ -84,12 +84,17 @@ class RoofRecommendations: return (self.insulation_thickness > self.MINIMUM_LOFT_ISULATION_MM) and self.property.roof["is_pitched"] - def is_room_roof_insulated(self): + def is_room_roof_insulated_or_unsuitable(self, measures): """ Check if the room roof is already insulated """ + # If the roof is a room roof room roof is not included in the measures, we deem the recommendation unsuitable + unsuitable = "room_roof_insulation" not in measures and self.property.roof["is_roof_room"] + if unsuitable: + return True + full_insulated_room_roof = ( self.property.roof["is_roof_room"] and self.property.roof["insulation_thickness"] in ["average", "above_average"] @@ -123,7 +128,7 @@ class RoofRecommendations: if (self.insulation_thickness >= self.MINIMUM_FLAT_ROOF_ISULATION_MM) and self.property.roof["is_flat"]: return - if self.is_room_roof_insulated(): + if self.is_room_roof_insulated_or_unsuitable(measures): return # If we have a u-value already, need to implement this From 9beb8b7a24bcc273a61e11c3d40d85bf9de19062 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 2 Oct 2024 14:05:04 +0100 Subject: [PATCH 092/166] new fuel type to efficiency mapped --- backend/app/assumptions.py | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/backend/app/assumptions.py b/backend/app/assumptions.py index 5f8cb85c..80baa69f 100644 --- a/backend/app/assumptions.py +++ b/backend/app/assumptions.py @@ -1,7 +1,7 @@ # Assumes that the average efficiency of an air source heat pump is 250%, taking the median of the 200-400% range, # which is often quoted as a sensible efficiency range for air source heat pumps. PESSIMISTIC_ASHP_EFFICIENCY = 200 -AVERAGE_ASHP_EFFICIENCY = 300 +AVERAGE_ASHP_EFFICIENCY = 250 # Conservative estimate of the proportion of electricity that will be consumed, whereas the rest will # be exported @@ -11,34 +11,36 @@ DESCRIPTIONS_TO_FUEL_TYPES = { "Air source heat pump, radiators, electric": { "fuel": "Electricity", "cop": AVERAGE_ASHP_EFFICIENCY / 100 }, - "Boiler and radiators, mains gas": {"fuel": 'Natural Gas', "cop": 0.9}, + "Boiler and radiators, mains gas": {"fuel": 'Natural Gas', "cop": 0.85}, 'Electric storage heaters': {"fuel": 'Electricity', "cop": 1}, "Electric immersion, off-peak": {"fuel": 'Electricity', "cop": 1}, "Electric storage heaters, radiators": {"fuel": 'Electricity', "cop": 1}, "Room heaters, electric": {"fuel": 'Electricity', "cop": 1}, "Electric immersion, standard tariff": {"fuel": 'Electricity', "cop": 1}, "Portable electric heaters assumed for most rooms": {"fuel": 'Electricity', "cop": 1}, - "Boiler and radiators, LPG": {"fuel": 'LPG', "cop": 0.9}, + "Boiler and radiators, LPG": {"fuel": 'LPG', "cop": 0.85}, "Room heaters, dual fuel (mineral and wood)": {"fuel": 'Wood Logs', "cop": 1}, - "Room heaters, mains gas": {"fuel": 'Natural Gas', "cop": 0.9}, - "Warm air, mains gas": {"fuel": 'Natural Gas', "cop": 0.9}, - "Boiler, mains gas": {"fuel": 'Natural Gas', "cop": 0.9}, - "Gas multipoint": {"fuel": "Natural Gas", "cop": 0.9}, + "Room heaters, mains gas": {"fuel": 'Natural Gas', "cop": 0.85}, + "Warm air, mains gas": {"fuel": 'Natural Gas', "cop": 0.85}, + "Boiler, mains gas": {"fuel": 'Natural Gas', "cop": 0.85}, + "Gas multipoint": {"fuel": "Natural Gas", "cop": 0.85}, "Warm air, Electricaire": {"fuel": "Electricity", "cop": 1}, - "Gas boiler/circulator": {"fuel": "Natural Gas", "cop": 0.9}, - "Boiler and underfloor heating, mains gas": {"fuel": "Natural Gas", "cop": 0.9}, + "Gas boiler/circulator": {"fuel": "Natural Gas", "cop": 0.85}, + "Boiler and underfloor heating, mains gas": {"fuel": "Natural Gas", "cop": 0.85}, "No system present: electric heaters assumed": {"fuel": "Electricity", "cop": 1}, "Electric instantaneous at point of use": {"fuel": "Electricity", "cop": 1}, - "Boiler and radiators, oil": {"fuel": "Oil", "cop": 0.9}, + "Boiler and radiators, oil": {"fuel": "Oil", "cop": 0.85}, "Electric storage heaters, Electric storage heaters": {"fuel": "Electricity", "cop": 1}, - "Boiler and radiators, electric": {"fuel": "Electricity", "cop": 0.9}, - "Gas boiler/circulator, no cylinder thermostat": {"fuel": "Natural Gas", "cop": 0.9}, - "Boiler and radiators, dual fuel (mineral and wood)": {"fuel": "Wood Logs", "cop": 0.9}, + "Boiler and radiators, electric": {"fuel": "Electricity", "cop": 0.85}, + "Gas boiler/circulator, no cylinder thermostat": {"fuel": "Natural Gas", "cop": 0.85}, + "Boiler and radiators, dual fuel (mineral and wood)": {"fuel": "Wood Logs", "cop": 0.85}, "Electric immersion, standard tariff, plus solar": {"fuel": "Electricity + Solar Thermal", "cop": 1}, - "From main system, flue gas heat recovery": {"fuel": "Natural Gas", "cop": 0.9}, + "From main system, flue gas heat recovery": {"fuel": "Natural Gas", "cop": 0.85}, "Electric underfloor heating": {"fuel": "Electricity", "cop": 1}, "No system present: electric immersion assumed": {"fuel": "Electricity", "cop": 1}, "Air source heat pump, underfloor, electric": { "fuel": "Electricity", "cop": AVERAGE_ASHP_EFFICIENCY / 100 }, + "Gas instantaneous at point of use": {"fuel": "Natural Gas", "cop": 0.85}, + "Room heaters, wood logs": {"fuel": "Wood Logs", "cop": 1}, } From aadcc56d32547addb06af616a192b5c1446b03fd Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 2 Oct 2024 14:33:29 +0100 Subject: [PATCH 093/166] adding database queries to Outputs class --- backend/Outputs.py | 102 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 backend/Outputs.py diff --git a/backend/Outputs.py b/backend/Outputs.py new file mode 100644 index 00000000..a846965b --- /dev/null +++ b/backend/Outputs.py @@ -0,0 +1,102 @@ +from sqlalchemy.orm import sessionmaker + +from backend.app.db.connection import db_engine +from backend.app.db.models.portfolio import PropertyModel, PropertyDetailsEpcModel +from backend.app.db.models.recommendations import Recommendation, Plan, PlanRecommendations, Scenario + + +class Outputs: + FORMATS = ["mds"] + + def __init__(self, format, portfolio_id): + """ + This class handles the creation of standard outputs for the backend. For example, creation of + an excel output, to be used for the MDS data sheet, required by E.ON + + :param format: The format of the output, e.g. mds + :param portfolio_id: The id of the portfolio for which the output is being created + """ + + if format not in self.FORMATS: + raise ValueError("Invalid format, should be one of {}".format(self.FORMATS)) + + self.format = format + self.portfolio_id = portfolio_id + + # Connect to the database + self.session = sessionmaker(bind=db_engine)() + + def get_properties_from_db(self): + # Get properties and their details for a specific portfolio + self.session.begin() + properties_query = self.session.query( + PropertyModel, + PropertyDetailsEpcModel + ).join( + PropertyDetailsEpcModel, + PropertyModel.id == PropertyDetailsEpcModel.property_id + ).filter( + PropertyModel.portfolio_id == self.portfolio_id # Filter by portfolio ID + ).all() + + # Transform properties data to include all fields dynamically + properties_data = [ + {**{col.name: getattr(prop.PropertyModel, col.name) for col in PropertyModel.__table__.columns}, + **{col.name: getattr(prop.PropertyDetailsEpcModel, col.name) for col in + PropertyDetailsEpcModel.__table__.columns}} + for prop in properties_query + ] + + self.session.close() + return properties_data + + def get_plans_from_db(self): + + self.session.begin() + + plans_query = self.session.query(Plan).all() + # Transform plans data to include all fields dynamically + plans_data = [ + {col.name: getattr(plan, col.name) for col in Plan.__table__.columns} + for plan in plans_query + ] + + self.session.close() + return plans_data + + def export_mds(self): + """ + This function will export the data in the MDS format + Core data required: + - Property address + - Property postcode + - uprn + - recommended measures + - pre-EPC + - pre-SAP + - pre Heat Demand + - Property Type + - Built form + - Wall type + - Tenure + - Fuel type + - Estimated bill + - Recommended measures + - Post EPC + - Post heat demand + - Bill savings + - Kwh savings + """ + + properties_data = self.get_properties_from_db() + + plans_data = self.get_plans_from_db() + + plan_ids = [plan['id'] for plan in plans_data] + + def export(self): + """ + This function will export the data in the required format + """ + if self.format == "mds": + self.export_mds() From 6bc2ee0a6cdfc2586b7c64672debe8067f69ab4e Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 2 Oct 2024 14:39:04 +0100 Subject: [PATCH 094/166] adding rest of database query functions --- backend/Outputs.py | 39 +++++++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/backend/Outputs.py b/backend/Outputs.py index a846965b..0be2ad3b 100644 --- a/backend/Outputs.py +++ b/backend/Outputs.py @@ -2,7 +2,7 @@ from sqlalchemy.orm import sessionmaker from backend.app.db.connection import db_engine from backend.app.db.models.portfolio import PropertyModel, PropertyDetailsEpcModel -from backend.app.db.models.recommendations import Recommendation, Plan, PlanRecommendations, Scenario +from backend.app.db.models.recommendations import Recommendation, Plan, PlanRecommendations class Outputs: @@ -28,7 +28,6 @@ class Outputs: def get_properties_from_db(self): # Get properties and their details for a specific portfolio - self.session.begin() properties_query = self.session.query( PropertyModel, PropertyDetailsEpcModel @@ -47,13 +46,10 @@ class Outputs: for prop in properties_query ] - self.session.close() return properties_data def get_plans_from_db(self): - self.session.begin() - plans_query = self.session.query(Plan).all() # Transform plans data to include all fields dynamically plans_data = [ @@ -61,9 +57,36 @@ class Outputs: for plan in plans_query ] - self.session.close() return plans_data + def get_recommendations_from_db(self, plan_ids): + # Get recommendations through PlanRecommendations for those plans and that are default + recommendations_query = self.session.query( + Recommendation, + Plan.scenario_id + ).join( + PlanRecommendations, Recommendation.id == PlanRecommendations.recommendation_id + ).join( + Plan, Plan.id == PlanRecommendations.plan_id # Join with Plan to access scenario_id + ).filter( + PlanRecommendations.plan_id.in_(plan_ids), + Recommendation.default == True # Filtering for default recommendations + ).all() + + # Transform recommendations data to include all fields dynamically and include scenario_id + recommendations_data = [ + { + **{ + col.name: getattr(rec.Recommendation, col.name) if + hasattr(rec, 'Recommendation') else getattr(rec, col.name) + for col in Recommendation.__table__.columns + }, + "Scenario ID": rec.scenario_id + } for rec in recommendations_query + ] + + return recommendations_data + def export_mds(self): """ This function will export the data in the MDS format @@ -88,12 +111,16 @@ class Outputs: - Kwh savings """ + self.session.begin() properties_data = self.get_properties_from_db() plans_data = self.get_plans_from_db() plan_ids = [plan['id'] for plan in plans_data] + recommendations_data = self.get_recommendations_from_db(plan_ids) + self.session.close() + def export(self): """ This function will export the data in the required format From ad82b15c6efc59d60f3ab184392d16abcd9b3cbf Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 2 Oct 2024 17:29:04 +0100 Subject: [PATCH 095/166] added measure matrix code --- backend/Outputs.py | 153 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 151 insertions(+), 2 deletions(-) diff --git a/backend/Outputs.py b/backend/Outputs.py index 0be2ad3b..4e300e81 100644 --- a/backend/Outputs.py +++ b/backend/Outputs.py @@ -1,5 +1,7 @@ +import pandas as pd from sqlalchemy.orm import sessionmaker +from backend.app.utils import sap_to_epc from backend.app.db.connection import db_engine from backend.app.db.models.portfolio import PropertyModel, PropertyDetailsEpcModel from backend.app.db.models.recommendations import Recommendation, Plan, PlanRecommendations @@ -8,6 +10,37 @@ from backend.app.db.models.recommendations import Recommendation, Plan, PlanReco class Outputs: FORMATS = ["mds"] + MDS_MEASURE_MAPPING = { + "external_wall_insulation": "EWI (Trad Const)", + "cavity_wall_insulation": "CWI", + "loft_insulation": "LI", + "party_wall_insulation": "Party Wall Insu", + "internal_wall_insulation": "IWI (POA - Prov Sum Only)", + "suspended_floor_insulation": "U/F Insu (Manual install)", + "solid_floor_insulation": "Solid floor insl (Out of scope - Prov sum only)", + "air_source_heat_pump": "ASHP Htg", + "ground_source_heat_pump": "GSHP Htg", + "shared_ground_loops": "Shared ground loops", + "communal_heat_networks": "Communal heat networks", + "district_heating_networks": "District heating networks", + "high_heat_retention_storage_heaters": "Elec Storage Htrs (Out of scope -Prov sum only)", + "low_energy_lighting": "Low Energy Bulbs", + "cylinder_insulation": "Cyl Insulation", + "smart_controls": "Smart controls", + "zone_controls": "Zone controls", + "trvs": "Upgrade TRV's", + "solar_pv": "Solar PV", + "solar_thermal": "Solar Thermal", + "double_glazing": "Double Glazing (POA - Prov sum only)", + "draught_proofing": "Draught Proofing", + "mechanical_ventilation": "Ventilation upgrade", + "gas_boiler": "Gas Boiler Replacement", + "flat_roof_insulation": "Flat roof (Out of scope - prov sum only)", + "room_in_roof_insulation": "RIR (POA - Prov sum only)", + "ev_charging": "EV Charging", + "battery": "Battery" + } + def __init__(self, format, portfolio_id): """ This class handles the creation of standard outputs for the backend. For example, creation of @@ -50,7 +83,7 @@ class Outputs: def get_plans_from_db(self): - plans_query = self.session.query(Plan).all() + plans_query = self.session.query(Plan).filter(Plan.portfolio_id == self.portfolio_id).all() # Transform plans data to include all fields dynamically plans_data = [ {col.name: getattr(plan, col.name) for col in Plan.__table__.columns} @@ -87,6 +120,41 @@ class Outputs: return recommendations_data + def make_mds_measure_matrix(self, scenario_recommendations): + all_measures = list(self.MDS_MEASURE_MAPPING.values()) + + # Collect rows in a list + rows = [] + + # Populate the rows list + for idx, row in scenario_recommendations.iterrows(): + property_id = row["property_id"] + measure_type = row["measure_type"] + + # Get the label for the current type + measure_label = self.MDS_MEASURE_MAPPING.get(measure_type, None) + + # If the property_id already exists in the collected rows, update it + existing_row = next((item for item in rows if item["property_id"] == property_id), None) + if existing_row is None: + # Create a new row if the property_id doesn't exist + new_row = {measure: None for measure in all_measures} + new_row["property_id"] = property_id + rows.append(new_row) + else: + new_row = existing_row + + # Set the corresponding measure label in the row + new_row[measure_label] = measure_label + + # Convert the list of dictionaries to a DataFrame + matrix = pd.DataFrame(rows) + + # Reset the index for cleanliness + matrix.reset_index(drop=True, inplace=True) + + return matrix + def export_mds(self): """ This function will export the data in the MDS format @@ -115,12 +183,93 @@ class Outputs: properties_data = self.get_properties_from_db() plans_data = self.get_plans_from_db() - plan_ids = [plan['id'] for plan in plans_data] recommendations_data = self.get_recommendations_from_db(plan_ids) self.session.close() + # Convert these tables to dataframes + properties_df = pd.DataFrame(properties_data) + plans_df = pd.DataFrame(plans_data) + recommendations_df = pd.DataFrame(recommendations_data) + + scenario_ids = plans_df["scenario_id"].unique() + + # We start to create the MDS sheet + mds = properties_df[ + [ + "property_id", + "address", + "postcode", + "uprn", + "current_epc_rating", + "current_sap_points", + # TODO: Need to add current heat demand + "property_type", + "built_form", + "total_floor_area", + "walls", + "tenure", + "mainfuel", + # TODO: For estimated bill, this should probably be without the cost of appliances + ] + ].copy().rename( + columns={ + "address": "Address", + "postcode": "Postcode", + "uprn": "UPRN", + "current_epc_rating": "Pre EPC", + "current_sap_points": "EPC Source", + # TODO: Need to add current heat demand + "property_type": "Property Type", + "built_form": "Built Form", + "total_floor_area": "Floor area m2 (If known)", + "walls": "Wall Type (Mandatory field)", + "tenure": "Tenure", + "mainfuel": "Existing Fuel Type" + # TODO: For estimated bill, this should probably be without the cost of appliances + } + ) + + # TODO - format + # 1) property type + # 2) walls + # 3) tenure + # 4) mainfuel + # 5) Epc Rating + + mds_output_by_scenario = {} + for scenario_id in scenario_ids: + scenario_recommendations = recommendations_df[recommendations_df["Scenario ID"] == scenario_id] + + # For each measure, we create the measure matrix + scenario_measure_matrix = self.make_mds_measure_matrix(scenario_recommendations) + + # Calculate the predicted impact on: SAP, heat demand, bills, kwh + recommendation_impacts = scenario_recommendations.groupby("property_id")[ + ["sap_points", "heat_demand", "kwh_savings", "energy_cost_savings"] + ].sum().reset_index() + + scenario_mds = mds.merge( + recommendation_impacts, how="left", on="property_id" + ) + # If we have no recommendations, sap_points, kwh_savings, head_demand will be NaN + scenario_mds.fillna(0, inplace=True) + scenario_mds["Post SAP"] = scenario_mds["EPC Source"] + scenario_mds["sap_points"] + # Round Post SAP down to the nearest integer + scenario_mds["Post SAP"] = scenario_mds["Post SAP"].apply(lambda x: int(x)) + scenario_mds["Post EPC"] = scenario_mds["Post SAP"].apply(lambda x: sap_to_epc(x)) + + # TODO: Post heat demand + + scenario_mds = scenario_mds.rename( + columns={ + "sap_points": "Predicted SAP Points", + "kwh_savings": "Energy Saving (Kwh)", + "energy_cost_savings": "Bill Reduction (£ per yr)" + } + ) + def export(self): """ This function will export the data in the required format From 17f653340bb26a098377c8415496284ccf47d360 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 2 Oct 2024 17:31:58 +0100 Subject: [PATCH 096/166] almost completed the mds output --- backend/Outputs.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/backend/Outputs.py b/backend/Outputs.py index 4e300e81..aef47b54 100644 --- a/backend/Outputs.py +++ b/backend/Outputs.py @@ -251,9 +251,14 @@ class Outputs: ].sum().reset_index() scenario_mds = mds.merge( + scenario_measure_matrix, how="left", on="property_id" + ).merge( recommendation_impacts, how="left", on="property_id" ) # If we have no recommendations, sap_points, kwh_savings, head_demand will be NaN + to_clean = [c for c in recommendation_impacts.columns if c != "property_id"] + for col in to_clean: + scenario_mds[col].fillna(0, inplace=True) scenario_mds.fillna(0, inplace=True) scenario_mds["Post SAP"] = scenario_mds["EPC Source"] + scenario_mds["sap_points"] # Round Post SAP down to the nearest integer From 2a75a6db17c625ac9f4a78a55297a781c855beab Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 2 Oct 2024 20:19:25 +0100 Subject: [PATCH 097/166] adding heat demand to mds outputs --- backend/Outputs.py | 2 +- backend/app/db/functions/recommendations_functions.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/Outputs.py b/backend/Outputs.py index aef47b54..9cc3e38c 100644 --- a/backend/Outputs.py +++ b/backend/Outputs.py @@ -220,7 +220,7 @@ class Outputs: "uprn": "UPRN", "current_epc_rating": "Pre EPC", "current_sap_points": "EPC Source", - # TODO: Need to add current heat demand + "primary_energy_consumption": "Existing Heating Demand Kwh/m2/y", "property_type": "Property Type", "built_form": "Built Form", "total_floor_area": "Floor area m2 (If known)", diff --git a/backend/app/db/functions/recommendations_functions.py b/backend/app/db/functions/recommendations_functions.py index b03909ee..5f6791f2 100644 --- a/backend/app/db/functions/recommendations_functions.py +++ b/backend/app/db/functions/recommendations_functions.py @@ -121,6 +121,7 @@ def upload_recommendations(session: Session, recommendations_to_upload, property "energy_cost_savings": rec["energy_cost_savings"], "labour_days": rec["labour_days"], "already_installed": rec["already_installed"], + "head_demand": rec["heat_demand"] } for rec in recommendations_to_upload ] From 1f1bf9981ce2e0b3a9bf42a8d096fb9aba333f4e Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 3 Oct 2024 10:38:14 +0100 Subject: [PATCH 098/166] adding measure type for wall and roof insulation --- backend/Property.py | 8 ++++++-- backend/app/db/models/portfolio.py | 7 +++++++ backend/app/plan/router.py | 11 ++++++----- recommendations/Recommendations.py | 17 +++++++++-------- recommendations/RoofRecommendations.py | 8 +++++--- recommendations/WallRecommendations.py | 2 ++ 6 files changed, 35 insertions(+), 18 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index 55e3a912..491a886f 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -621,7 +621,7 @@ class Property: self.set_windows_count() self.set_energy_source() self.find_energy_sources() - self.set_current_energy_bill(kwh_client, kwh_predictions) + self.set_current_energy(kwh_client, kwh_predictions) def set_solar_panel_configuration( self, solar_panel_configuration, roof_area @@ -651,7 +651,7 @@ class Property: else: self.roof_area = roof_area - def set_current_energy_bill(self, kwh_client, kwh_predictions): + def set_current_energy(self, kwh_client, kwh_predictions): """ Given what we know about the property now, estimates the current energy consumption using the UCL paper https://www.sciencedirect.com/science/article/pii/S0378778823002542 @@ -808,6 +808,9 @@ class Property: def get_property_details_epc(self, portfolio_id: int, rating_lookup): + if self.current_energy_bill is None: + raise ValueError("Current energy bill has not been set") + property_details_epc = { "property_id": self.id, "portfolio_id": portfolio_id, @@ -865,6 +868,7 @@ class Property: "current_energy_demand": self.current_energy_consumption, "current_energy_demand_heating_hotwater": self.current_energy_consumption_heating_hotwater, "estimated": self.data.get("estimated", False), + **self.current_energy_bill } return property_details_epc diff --git a/backend/app/db/models/portfolio.py b/backend/app/db/models/portfolio.py index e2b258f4..5f51cf46 100644 --- a/backend/app/db/models/portfolio.py +++ b/backend/app/db/models/portfolio.py @@ -173,6 +173,13 @@ class PropertyDetailsEpcModel(Base): current_energy_demand = Column(Float) current_energy_demand_heating_hotwater = Column(Float) estimated = Column(Boolean, default=False) + # Include estimates for energy bills, across the different types of energy + heating_cost_current = Column(Float) + hot_water_cost_current = Column(Float) + lighting_cost_current = Column(Float) + appliances_cost_current = Column(Float) + gas_standing_charge = Column(Float) + electricity_standing_charge = Column(Float) class PropertyDetailsSpatial(Base): diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 9d77c1a1..6b8c5bea 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -10,7 +10,6 @@ from sqlalchemy.exc import IntegrityError, OperationalError from sqlalchemy.orm import sessionmaker from starlette.responses import Response -import backend.app.assumptions as assumptions from backend.app.config import get_settings, get_prediction_buckets from backend.app.db.connection import db_engine from backend.app.db.functions.materials_functions import get_materials @@ -614,10 +613,12 @@ async def trigger_plan(body: PlanTriggerRequest): property_recommendations = recommendations.get(property_id, []) property_instance = [p for p in input_properties if p.id == property_id][0] - property_current_energy_bill = Recommendations.calculate_recommendation_tenant_savings( - property_instance=property_instance, - kwh_simulation_predictions=kwh_simulation_predictions, - property_recommendations=property_recommendations + property_current_energy_bill = ( + Recommendations.calculate_recommendation_tenant_savings( + property_instance=property_instance, + kwh_simulation_predictions=kwh_simulation_predictions, + property_recommendations=property_recommendations + ) ) property_instance.current_energy_bill = property_current_energy_bill diff --git a/recommendations/Recommendations.py b/recommendations/Recommendations.py index 526cb2a2..78725c2d 100644 --- a/recommendations/Recommendations.py +++ b/recommendations/Recommendations.py @@ -797,13 +797,14 @@ class Recommendations: electricity_standing_charge = AnnualBillSavings.DAILY_STANDARD_CHARGE_ELECTRICITY * 365 - current_energy_bill = ( - starting_figures["heating_cost"] + - starting_figures["hotwater_cost"] + - property_instance.energy_cost_estimates["unadjusted"]["lighting"] + - property_instance.energy_cost_estimates["unadjusted"]["appliances"] + - gas_standing_charge + - electricity_standing_charge - ) + # We return a dictionary that contains the individual costs, that can be stored to the database + current_energy_bill = { + "heating_cost_current": starting_figures["heating_cost"], + "hot_water_cost_current": starting_figures["hotwater_cost"], + "lighting_cost_current": property_instance.energy_cost_estimates["unadjusted"]["lighting"], + "appliances_cost_current": property_instance.energy_cost_estimates["unadjusted"]["appliances"], + "gas_standing_charge": gas_standing_charge, + "electricity_standing_charge": electricity_standing_charge, + } return current_energy_bill diff --git a/recommendations/RoofRecommendations.py b/recommendations/RoofRecommendations.py index 4d22eb2d..3e266cee 100644 --- a/recommendations/RoofRecommendations.py +++ b/recommendations/RoofRecommendations.py @@ -250,8 +250,10 @@ class RoofRecommendations: if is_pitched: insulation_materials = self.loft_insulation_materials + measure_type = "loft_insulation" elif is_flat: insulation_materials = self.flat_roof_insulation_materials + measure_type = "flat_roof_insulation" else: raise ValueError("Roof is not pitched or flat") @@ -367,6 +369,7 @@ class RoofRecommendations: ) ], "type": material["type"], + "measure_type": measure_type, "description": self.make_roof_insulation_description(material), "starting_u_value": u_value, "new_u_value": new_u_value, @@ -490,10 +493,9 @@ class RoofRecommendations: recommendations.append( { "phase": phase, - "parts": [ - # TODO - ], + "parts": [], "type": "room_roof_insulation", + "measure_type": "room_roof_insulation", "description": "Insulate room in roof at rafters and re-decorate", "starting_u_value": u_value, "new_u_value": new_u_value, diff --git a/recommendations/WallRecommendations.py b/recommendations/WallRecommendations.py index 4902ae03..dd5e861c 100644 --- a/recommendations/WallRecommendations.py +++ b/recommendations/WallRecommendations.py @@ -374,6 +374,7 @@ class WallRecommendations(Definitions): ) ], "type": "cavity_wall_insulation", + "measure_type": "cavity_wall_insulation", "description": description, "starting_u_value": u_value, "new_u_value": new_u_value, @@ -545,6 +546,7 @@ class WallRecommendations(Definitions): ) ], "type": material["type"], + "measure_type": material["type"], # This is distinguished between EWI & IWI "description": self._make_description(material), "starting_u_value": u_value, "new_u_value": new_u_value, From 0c2463efe00ddd266f5f239dd743f1b6c2a46608 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 3 Oct 2024 10:39:59 +0100 Subject: [PATCH 099/166] added measure type for ventilation, draught proofing --- recommendations/DraughtProofingRecommendations.py | 1 + recommendations/VentilationRecommendations.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/recommendations/DraughtProofingRecommendations.py b/recommendations/DraughtProofingRecommendations.py index 197d80cc..4bd85a03 100644 --- a/recommendations/DraughtProofingRecommendations.py +++ b/recommendations/DraughtProofingRecommendations.py @@ -38,6 +38,7 @@ class DraughtProofingRecommendations: "phase": None, "parts": [], "type": "draught_proofing", + "measure_type": "draught_proofing", "description": description, "starting_u_value": None, "new_u_value": None, diff --git a/recommendations/VentilationRecommendations.py b/recommendations/VentilationRecommendations.py index 5913ab9c..9738b898 100644 --- a/recommendations/VentilationRecommendations.py +++ b/recommendations/VentilationRecommendations.py @@ -66,6 +66,7 @@ class VentilationRecommendations(Definitions): "phase": None, "parts": part, "type": part[0]["type"], + "measure_type": "mechanical_ventilation", "description": f"Install {n_units} {part[0]['description']} units", "starting_u_value": None, "new_u_value": None, @@ -106,6 +107,7 @@ class VentilationRecommendations(Definitions): "phase": None, "parts": [], "type": "trickle_vents", + "measure_type": "trickle_vents", "description": description, "starting_u_value": None, "new_u_value": None, From b3b2021a1bb96a711208a673286e1211761f0c82 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 3 Oct 2024 10:41:42 +0100 Subject: [PATCH 100/166] added floors and leds measure types --- recommendations/FloorRecommendations.py | 1 + recommendations/LightingRecommendations.py | 1 + 2 files changed, 2 insertions(+) diff --git a/recommendations/FloorRecommendations.py b/recommendations/FloorRecommendations.py index d82162da..25741e7a 100644 --- a/recommendations/FloorRecommendations.py +++ b/recommendations/FloorRecommendations.py @@ -241,6 +241,7 @@ class FloorRecommendations(Definitions): ), ], "type": material["type"], + "measure_type": material["type"], # This is distinct between suspended and solid floor "description": self._make_floor_description(material), "starting_u_value": u_value, "new_u_value": new_u_value, diff --git a/recommendations/LightingRecommendations.py b/recommendations/LightingRecommendations.py index 2b0e8724..f9a1d63a 100644 --- a/recommendations/LightingRecommendations.py +++ b/recommendations/LightingRecommendations.py @@ -152,6 +152,7 @@ class LightingRecommendations: "phase": phase, "parts": [], "type": "low_energy_lighting", + "measure_type": "low_energy_lighting", "description": description, "starting_u_value": None, "new_u_value": None, From e2d0f920b0dfa358d4019dca4fcffecd5dab87fa Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 3 Oct 2024 10:42:24 +0100 Subject: [PATCH 101/166] added windows measure_type --- recommendations/WindowsRecommendations.py | 1 + 1 file changed, 1 insertion(+) diff --git a/recommendations/WindowsRecommendations.py b/recommendations/WindowsRecommendations.py index cd1b982b..9b85b316 100644 --- a/recommendations/WindowsRecommendations.py +++ b/recommendations/WindowsRecommendations.py @@ -223,6 +223,7 @@ class WindowsRecommendations: "phase": phase, "parts": [], "type": "windows_glazing", + "measure_type": "double_glazing" if not is_secondary_glazing else "secondary_glazing", "description": description, "starting_u_value": None, "new_u_value": None, From 2b992fc9f5ddb3d394090d7ffebdc9840085d56d Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 3 Oct 2024 10:43:06 +0100 Subject: [PATCH 102/166] added measure type for minor measures --- recommendations/FireplaceRecommendations.py | 1 + recommendations/WindowsRecommendations.py | 1 + 2 files changed, 2 insertions(+) diff --git a/recommendations/FireplaceRecommendations.py b/recommendations/FireplaceRecommendations.py index 163728dd..60802bb6 100644 --- a/recommendations/FireplaceRecommendations.py +++ b/recommendations/FireplaceRecommendations.py @@ -41,6 +41,7 @@ class FireplaceRecommendations(Definitions): "phase": phase, "parts": [], "type": "sealing_open_fireplace", + "measure_type": "sealing_open_fireplace", "description": "Seal %s open fireplaces" % str(number_open_fireplaces), "starting_u_value": None, "new_u_value": None, diff --git a/recommendations/WindowsRecommendations.py b/recommendations/WindowsRecommendations.py index 9b85b316..343965e3 100644 --- a/recommendations/WindowsRecommendations.py +++ b/recommendations/WindowsRecommendations.py @@ -273,6 +273,7 @@ class WindowsRecommendations: "phase": phase, "parts": [], "type": "mixed_glazing", + "measure_type": "mixed_glazing", "description": description, "starting_u_value": None, "new_u_value": None, From 9b6f8c486458b0dfff3b838bac5e5aca713c45cf Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 3 Oct 2024 10:51:22 +0100 Subject: [PATCH 103/166] added measure type to heating recommendations --- recommendations/HeatingRecommender.py | 29 +++++++++++++++------------ 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index b54f89bb..b4ca9a49 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -220,6 +220,8 @@ class HeatingRecommender: for k in ["total", "subtotal", "vat", "labour_hours", "labour_days"]: combined_rec[k] = rec[k] + rec2[k] + combined_rec["measure_type"] = "+".join([rec["measure_type"], rec2["measure_type"]]) + combined_recommendations.append(combined_rec) self.heating_recommendations.extend(combined_recommendations) @@ -512,10 +514,9 @@ class HeatingRecommender: ashp_recommendation = { "phase": phase, - "parts": [ - # TODO - ], + "parts": [], "type": "heating", + "measure_type": "air_source_heat_pump", "description": description, "starting_u_value": None, "new_u_value": None, @@ -556,7 +557,8 @@ class HeatingRecommender: phase, heating_controls_only, system_change, - system_type + system_type, + measure_type ): """ Given a recommendation for heating controls, and a recommendation for the heating system, we combine the two @@ -572,7 +574,8 @@ class HeatingRecommender: current system. If we have a system change and we have a heat control recommendation, we only recommend both heating and controls together :param system_type: The type of heating system we are recommending - :return: + :param measure_type: The type of measure we are recommending - more granular than the "type" field, allowing us + to distinguish between different types of heating recommendations """ # We produce recommendations with & without heating controls @@ -616,10 +619,9 @@ class HeatingRecommender: recommendation = { "phase": phase, - "parts": [ - # TODO - ], + "parts": [], "type": "heating", + "measure_type": measure_type, "description": recommendation_description, "starting_u_value": None, "new_u_value": None, @@ -799,7 +801,8 @@ class HeatingRecommender: phase=phase, heating_controls_only=heating_controls_only, system_change=system_change, - system_type="high_heat_retention_storage_heater" + system_type="high_heat_retention_storage_heater", + measure_type="high_heat_retention_storage_heater" ) if _return: return recommendations @@ -977,10 +980,9 @@ class HeatingRecommender: boiler_recommendation = { "phase": recommendation_phase, - "parts": [ - # TODO - ], + "parts": [], "type": "heating", + "measure_type": "boiler_upgrade", "description": description, "starting_u_value": None, "new_u_value": None, @@ -1027,7 +1029,8 @@ class HeatingRecommender: phase=recommendation_phase, heating_controls_only=False, system_change=True, - system_type="boiler_upgrade" + system_type="boiler_upgrade", + measure_type="boiler_upgrade", ) combined_recommendations.extend(combined_recommendation) From feb60537c0ebcfb567208d369c2ceb46be4cbece Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 3 Oct 2024 10:54:46 +0100 Subject: [PATCH 104/166] added measure type to some controls recommendations --- backend/app/plan/schemas.py | 1 + recommendations/HeatingControlRecommender.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/backend/app/plan/schemas.py b/backend/app/plan/schemas.py index 6f0f6327..447e0da2 100644 --- a/backend/app/plan/schemas.py +++ b/backend/app/plan/schemas.py @@ -64,6 +64,7 @@ MEASURE_MAP = { "floor_insulation": ["suspended_floor_insulation", "solid_floor_insulation"], "heating": ["boiler_upgrade", "high_heat_retention_storage_heater", "air_source_heat_pump"], "windows": ["double_glazing", "secondary_glazing"], + "heating_controls": ["roomstat_programmer_trvs", "time_temperature_zone_control"] } diff --git a/recommendations/HeatingControlRecommender.py b/recommendations/HeatingControlRecommender.py index 62e292df..c613aa42 100644 --- a/recommendations/HeatingControlRecommender.py +++ b/recommendations/HeatingControlRecommender.py @@ -216,6 +216,7 @@ class HeatingControlRecommender: self.recommendation.append( { "type": "heating_control", + "measure_type": "roomstat_programmer_trvs", "parts": [], "description": description, **cost_result, @@ -289,6 +290,7 @@ class HeatingControlRecommender: self.recommendation.append( { "type": "heating_control", + "measure_type": "time_temperature_zone_control", "parts": [], "description": description, **cost_result, From 79746e9a3643f9e18b78dce8081208238ba758a3 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 3 Oct 2024 10:59:40 +0100 Subject: [PATCH 105/166] finished adding measure_type to recommendations and added check to ensure it's populated --- recommendations/HotwaterRecommendations.py | 6 +++--- recommendations/Recommendations.py | 5 +++++ recommendations/SecondaryHeating.py | 1 + recommendations/SolarPvRecommendations.py | 2 ++ 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/recommendations/HotwaterRecommendations.py b/recommendations/HotwaterRecommendations.py index 0d34c894..636a7be0 100644 --- a/recommendations/HotwaterRecommendations.py +++ b/recommendations/HotwaterRecommendations.py @@ -58,10 +58,9 @@ class HotwaterRecommendations: self.recommendations.append( { "phase": phase, - "parts": [ - # TODO - ], + "parts": [], "type": "hot_water_tank_insulation", + "measure_type": "hot_water_tank_insulation", "description": description, "starting_u_value": None, "new_u_value": None, @@ -107,6 +106,7 @@ class HotwaterRecommendations: "phase": phase, "parts": [], "type": "cylinder_thermostat", + "measure_type": "cylinder_thermostat", "description": description, "starting_u_value": None, "new_u_value": None, diff --git a/recommendations/Recommendations.py b/recommendations/Recommendations.py index 78725c2d..1b152238 100644 --- a/recommendations/Recommendations.py +++ b/recommendations/Recommendations.py @@ -267,6 +267,11 @@ class Recommendations: property_recommendations, ) + # Check to make sure measure_type is populated + for recs in property_recommendations: + if any(pd.isnull(rec.get("measure_type")) for rec in recs): + raise ValueError("Measure type is not populated") + return property_recommendations, property_representative_recommendations @staticmethod diff --git a/recommendations/SecondaryHeating.py b/recommendations/SecondaryHeating.py index aed48da2..7c20bcdd 100644 --- a/recommendations/SecondaryHeating.py +++ b/recommendations/SecondaryHeating.py @@ -52,6 +52,7 @@ class SecondaryHeating: "phase": phase, "parts": [], "type": "secondary_heating", + "measure_type": "secondary_heating", "description": description, "starting_u_value": None, "new_u_value": None, diff --git a/recommendations/SolarPvRecommendations.py b/recommendations/SolarPvRecommendations.py index bb38c73c..08f077d2 100644 --- a/recommendations/SolarPvRecommendations.py +++ b/recommendations/SolarPvRecommendations.py @@ -126,6 +126,7 @@ class SolarPvRecommendations: "phase": phase, "parts": [], "type": "solar_pv", + "measure_type": "solar_pv", "description": description, "starting_u_value": None, "new_u_value": None, @@ -221,6 +222,7 @@ class SolarPvRecommendations: "phase": phase, "parts": [], "type": "solar_pv", + "measure_type": "solar_pv", "description": description, "starting_u_value": None, "new_u_value": None, From 4ef2f0af28a31dae746b5c99485688826ca0db02 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 3 Oct 2024 12:10:45 +0100 Subject: [PATCH 106/166] completed mds outputs for the moment --- backend/Outputs.py | 84 ++++++++++++++++--- .../db/functions/recommendations_functions.py | 3 +- backend/app/db/models/recommendations.py | 1 + backend/app/plan/router.py | 4 +- 4 files changed, 77 insertions(+), 15 deletions(-) diff --git a/backend/Outputs.py b/backend/Outputs.py index 9cc3e38c..f9538709 100644 --- a/backend/Outputs.py +++ b/backend/Outputs.py @@ -1,6 +1,10 @@ +import msgpack import pandas as pd +import numpy as np from sqlalchemy.orm import sessionmaker +from datetime import datetime +from utils.s3 import read_from_s3, save_excel_to_s3 from backend.app.utils import sap_to_epc from backend.app.db.connection import db_engine from backend.app.db.models.portfolio import PropertyModel, PropertyDetailsEpcModel @@ -55,10 +59,19 @@ class Outputs: self.format = format self.portfolio_id = portfolio_id + self.today = datetime.now().strftime("%Y-%m-%d") # Connect to the database self.session = sessionmaker(bind=db_engine)() + # Download cleaned data + self.cleaned_epc_lookup = read_from_s3( + s3_file_name="cleaned_epc_data/cleaned.bson", + bucket_name="retrofit-data-dev" + ) + + self.cleaned_epc_lookup = msgpack.unpackb(self.cleaned_epc_lookup, raw=False) + def get_properties_from_db(self): # Get properties and their details for a specific portfolio properties_query = self.session.query( @@ -204,14 +217,19 @@ class Outputs: "uprn", "current_epc_rating", "current_sap_points", - # TODO: Need to add current heat demand + "primary_energy_consumption", "property_type", "built_form", "total_floor_area", "walls", "tenure", "mainfuel", - # TODO: For estimated bill, this should probably be without the cost of appliances + # The bills columns are split out - we include them and aggregate, without appliances + "heating_cost_current", + "hot_water_cost_current", + "lighting_cost_current", + "gas_standing_charge", + "electricity_standing_charge" ] ].copy().rename( columns={ @@ -226,17 +244,46 @@ class Outputs: "total_floor_area": "Floor area m2 (If known)", "walls": "Wall Type (Mandatory field)", "tenure": "Tenure", - "mainfuel": "Existing Fuel Type" - # TODO: For estimated bill, this should probably be without the cost of appliances } ) - # TODO - format - # 1) property type - # 2) walls - # 3) tenure - # 4) mainfuel - # 5) Epc Rating + mds["Estimated bill (£ per year)"] = ( + mds["heating_cost_current"] + + mds["hot_water_cost_current"] + + mds["lighting_cost_current"] + + mds["gas_standing_charge"] + + mds["electricity_standing_charge"] + ) + + mds = mds.drop( + columns=[ + "heating_cost_current", + "hot_water_cost_current", + "lighting_cost_current", + "gas_standing_charge", + "electricity_standing_charge" + ] + ) + + # Formatting - Pre EPC is an enum + mds["Pre EPC"] = [x.value for x in mds["Pre EPC"].values] + mds["Wall Type (Mandatory field)"] = mds["Wall Type (Mandatory field)"].str.split(",").str[0] + # Remove average thermal transmittance field + mds["Wall Type (Mandatory field)"] = np.where( + mds["Wall Type (Mandatory field)"].str.contains("Average thermal transmittance"), + "", + mds["Wall Type (Mandatory field)"] + ) + + mds = mds.merge( + pd.DataFrame(self.cleaned_epc_lookup["main-fuel"])[["clean_description", "fuel_type"]], + left_on="mainfuel", + right_on="clean_description", + how="left" + ) + mds = mds.rename(columns={"fuel_type": "Existing Fuel Type"}).drop(columns=["clean_description", "mainfuel"]) + + mds["Existing Fuel Type"].value_counts() mds_output_by_scenario = {} for scenario_id in scenario_ids: @@ -264,8 +311,9 @@ class Outputs: # Round Post SAP down to the nearest integer scenario_mds["Post SAP"] = scenario_mds["Post SAP"].apply(lambda x: int(x)) scenario_mds["Post EPC"] = scenario_mds["Post SAP"].apply(lambda x: sap_to_epc(x)) - - # TODO: Post heat demand + scenario_mds["Heating Demand Kwh/m2/y"] = ( + scenario_mds["Existing Heating Demand Kwh/m2/y"] - scenario_mds["heat_demand"] + ) scenario_mds = scenario_mds.rename( columns={ @@ -275,9 +323,21 @@ class Outputs: } ) + mds_output_by_scenario[scenario_id] = scenario_mds + + # We now save them to s3 as excels + for scenario_id, scenario_mds in mds_output_by_scenario.items(): + save_excel_to_s3( + df=scenario_mds, + file_key=f"engine_outputs/{self.format}/{self.today}_scenario_id={scenario_id}.xlsx", + bucket_name="retrofit-data-dev" + ) + def export(self): """ This function will export the data in the required format """ if self.format == "mds": self.export_mds() + + raise NotImplementedError("Export format not implemented") diff --git a/backend/app/db/functions/recommendations_functions.py b/backend/app/db/functions/recommendations_functions.py index 5f6791f2..feeced10 100644 --- a/backend/app/db/functions/recommendations_functions.py +++ b/backend/app/db/functions/recommendations_functions.py @@ -108,6 +108,7 @@ def upload_recommendations(session: Session, recommendations_to_upload, property { "property_id": property_id, "type": rec["type"], + "measure_type": rec["measure_type"], "description": rec["description"], "estimated_cost": rec["total"], "default": rec["default"], @@ -121,7 +122,7 @@ def upload_recommendations(session: Session, recommendations_to_upload, property "energy_cost_savings": rec["energy_cost_savings"], "labour_days": rec["labour_days"], "already_installed": rec["already_installed"], - "head_demand": rec["heat_demand"] + "heat_demand": rec["heat_demand"] } for rec in recommendations_to_upload ] diff --git a/backend/app/db/models/recommendations.py b/backend/app/db/models/recommendations.py index a1743436..1089dced 100644 --- a/backend/app/db/models/recommendations.py +++ b/backend/app/db/models/recommendations.py @@ -15,6 +15,7 @@ class Recommendation(Base): property_id = Column(BigInteger, ForeignKey(PropertyModel.id), nullable=False) created_at = Column(TIMESTAMP, nullable=False, server_default=func.now()) type = Column(String, nullable=False) + measure_type = Column(String) description = Column(String, nullable=False) estimated_cost = Column(Float) default = Column(Boolean, nullable=False) diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 6b8c5bea..2dc03e60 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -126,8 +126,8 @@ def extract_portfolio_aggregation_data( pre_retrofit_co2 = p.data["co2-emissions-current"] post_retrofit_co2 = pre_retrofit_co2 - carbon_savings - pre_retrofit_energy_bill = p.current_energy_bill - post_retrofit_energy_bill = p.current_energy_bill - sum( + pre_retrofit_energy_bill = sum(p.current_energy_bill.values()) + post_retrofit_energy_bill = sum(p.current_energy_bill.values()) - sum( [r["energy_cost_savings"] for r in default_recommendations] ) From bf45a5f4fa3d2b344235ef7d6dc171ddabc76dc2 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 3 Oct 2024 16:10:32 +0100 Subject: [PATCH 107/166] minor --- .../AirSourceHeatPumpEfficiency.py | 94 ++++++++++--------- etl/air_source_heat_pump/app.py | 4 +- recommendations/HeatingRecommender.py | 10 +- 3 files changed, 60 insertions(+), 48 deletions(-) diff --git a/etl/air_source_heat_pump/AirSourceHeatPumpEfficiency.py b/etl/air_source_heat_pump/AirSourceHeatPumpEfficiency.py index 044cc830..e4eeedaf 100644 --- a/etl/air_source_heat_pump/AirSourceHeatPumpEfficiency.py +++ b/etl/air_source_heat_pump/AirSourceHeatPumpEfficiency.py @@ -22,68 +22,78 @@ class AirSourceHeatPumpEfficiency: def create_dataset(self): logger.info("Creating solar photo supply dataset") - all_counts = [] + heating_data = [] for dir in tqdm(self.file_directories): filepath = dir / "certificates.csv" df = pd.read_csv(filepath, low_memory=False) - df = df[~pd.isnull(df["UPRN"])] - df["UPRN"] = df["UPRN"].astype(int).astype(str) + # df = df[~pd.isnull(df["UPRN"])] + # df["UPRN"] = df["UPRN"].astype(int).astype(str) # Take entries after SAP12 df["LODGEMENT_DATE"] = pd.to_datetime(df["LODGEMENT_DATE"]) df = df[df["LODGEMENT_DATE"] > EARLIEST_EPC_DATE] - df = df[ - ~df["TENURE"].isin( - [ - "unknown", - "Not defined - use in the case of a new dwelling for which the intended tenure in not known. " - "It is not to be used for an existing dwelling" - ] - ) - ] + # df = df[ + # ~df["TENURE"].isin( + # [ + # "unknown", + # "Not defined - use in the case of a new dwelling for which the intended tenure in not known. " + # "It is not to be used for an existing dwelling" + # ] + # ) + # ] # Take entries that contain an air source heat pump df = df[ - df["MAINHEAT_DESCRIPTION"].str.contains("air source heat pump", case=False, na=False) - ] + ( + # Air source heat pumps + (df["MAINHEAT_DESCRIPTION"] == "Air source heat pump, radiators, electric") & + (df["MAINHEATCONT_DESCRIPTION"] == "Time and temperature zone control") + ) | + ( + # High heat retention storage + df["MAINHEATCONT_DESCRIPTION"] == "Controls for high heat retention storage heaters" + ) + ] # Drop rows that have a missing PROPERTY_TYPE, BUILT_FORM, CONSTRUCTION_AGE_BAND, TOTAL_FLOOR_AREA for col in ["PROPERTY_TYPE", "BUILT_FORM", "CONSTRUCTION_AGE_BAND", "TOTAL_FLOOR_AREA"]: df = df[~pd.isnull(df[col])] - # Get the columns we're interested in - df = df[ - [ - "PROPERTY_TYPE", - "BUILT_FORM", - "MAINHEAT_DESCRIPTION", - "MAINHEAT_ENERGY_EFF", - "MAINHEATCONT_DESCRIPTION", - "MAINHEATC_ENERGY_EFF", - "MAIN_FUEL", - "HOTWATER_DESCRIPTION", - "HOT_WATER_ENERGY_EFF", - "MAINS_GAS_FLAG" - ] + + heating_data.append(df) + + # temp + # import pickle + # with open("heating_data - delete me.pkl", "wb") as f: + # pickle.dump(heating_data, f) + + heating_df = pd.concat(heating_data) + # Clean construction age band + from etl.epc.DataProcessor import EPCDataProcessor + heating_df["CONSTRUCTION_AGE_BAND_CLEAN"] = heating_df["CONSTRUCTION_AGE_BAND"].apply( + lambda x: EPCDataProcessor.clean_construction_age_band(x) + ) + + ashp_df = heating_df[ + (heating_df["MAINHEAT_DESCRIPTION"] == "Air source heat pump, radiators, electric") & + # ~heating_df["CONSTRUCTION_AGE_BAND"].str.contains("England and Wales") + (~heating_df["CONSTRUCTION_AGE_BAND"].isin(["NO DATA!", "INVALID!"])) & + (heating_df["LODGEMENT_DATE"] >= pd.to_datetime("2019-01-01")) ] - - counts = df.groupby( + ashp_efficiencies = ( + ashp_df.groupby( [ - "PROPERTY_TYPE", - "BUILT_FORM", - "MAINHEAT_DESCRIPTION", + "CONSTRUCTION_AGE_BAND_CLEAN", + # "WALLS_DESCRIPTION", + # "ROOF_DESCRIPTION", "MAINHEAT_ENERGY_EFF", - "MAINHEATCONT_DESCRIPTION", - "MAINHEATC_ENERGY_EFF", - "MAIN_FUEL", - "HOTWATER_DESCRIPTION", - "HOT_WATER_ENERGY_EFF", - "MAINS_GAS_FLAG" ] - ).size().reset_index(name="count") + )["LMK_KEY"].count().reset_index() + ) - all_counts.append(counts) + ashp_df["MAINHEAT_ENERGY_EFF"].value_counts() - all_counts = pd.concat(all_counts) + ashp_efficiencies["CONSTRUCTION_AGE_BAND_CLEAN"].value_counts() + ashp_efficiency_agg all_counts_agg = all_counts.groupby( [ diff --git a/etl/air_source_heat_pump/app.py b/etl/air_source_heat_pump/app.py index ac87b34b..ed846d23 100644 --- a/etl/air_source_heat_pump/app.py +++ b/etl/air_source_heat_pump/app.py @@ -1,8 +1,10 @@ +import inspect from pathlib import Path from backend.app.plan.utils import get_cleaned from etl.air_source_heat_pump.AirSourceHeatPumpEfficiency import AirSourceHeatPumpEfficiency -DATA_DIRECTORY = Path(__file__).parent / "local_data" / "all-domestic-certificates" +file_src = inspect.getfile(lambda: None) +DATA_DIRECTORY = Path(file_src).parent / "local_data" / "all-domestic-certificates" def app(): diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index b4ca9a49..203bab87 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -434,7 +434,7 @@ class HeatingRecommender: ashp_costs_with_controls[key] += controls_rec[key] if controls_rec is None: - description = "Install an air source heat pump." + description = "Install a Mitsubish air source heat pump." elif already_installed: description = "The property already has an air source heat pump, no further action needed." else: @@ -457,8 +457,8 @@ class HeatingRecommender: ) simulation_config = { - "mainheat_energy_eff_ending": "Good", - "hot_water_energy_eff_ending": "Good" + "mainheat_energy_eff_ending": "Very Good", + "hot_water_energy_eff_ending": "Very Good" } description_simulation = { "mainheat-description": new_heating_description, @@ -725,7 +725,7 @@ class HeatingRecommender: description_prefix = "" controls_recommender.recommend( - heating_description="Electric storage heaters, radiators", description_prefix=description_prefix + heating_description="Electric storage heaters", description_prefix=description_prefix ) has_hhr = self.is_hhr_already_installed() @@ -740,7 +740,7 @@ class HeatingRecommender: self.property.main_heating["clean_description"] ]["hhr"]["mainheating_description"] else: - new_heating_description = "Electric storage heaters, radiators" + new_heating_description = "Electric storage heaters" # Set up artefacts, suitable for the simulation and regardless of controls heating_ending_config = MainHeatAttributes(new_heating_description).process() From 955791f49953072268a57530fdd5b1f152ca481f Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Thu, 3 Oct 2024 23:12:20 +0100 Subject: [PATCH 108/166] add temp fix for None value instead of string none - should come from cleaned lookup --- etl/epc/Dataset.py | 16 +- recommendations/recommendation_utils.py | 211 +++++++++++++++--------- 2 files changed, 145 insertions(+), 82 deletions(-) diff --git a/etl/epc/Dataset.py b/etl/epc/Dataset.py index 83a85b78..3f2e810e 100644 --- a/etl/epc/Dataset.py +++ b/etl/epc/Dataset.py @@ -203,11 +203,11 @@ class TrainingDataset(BaseDataset): common_cols = [[col + "_starting", col + "_ending"] for col in common_cols] self.df = self.df.loc[ - :, - no_suffix_cols - + only_ending_cols - + [col for cols in common_cols for col in cols], - ] + :, + no_suffix_cols + + only_ending_cols + + [col for cols in common_cols for col in cols], + ] def _remove_abnormal_change_in_floor_area(self): """ @@ -511,7 +511,7 @@ class TrainingDataset(BaseDataset): expanded_df["is_sandstone_or_limestone"] == expanded_df["is_sandstone_or_limestone_ending"] ) - ] + ] elif component == "floor": expanded_df = expanded_df[ (expanded_df["is_suspended"] == expanded_df["is_suspended_ending"]) @@ -528,7 +528,7 @@ class TrainingDataset(BaseDataset): expanded_df["is_to_external_air"] == expanded_df["is_to_external_air_ending"] ) - ] + ] elif component == "roof": expanded_df = expanded_df[ (expanded_df["is_pitched"] == expanded_df["is_pitched_ending"]) @@ -541,7 +541,7 @@ class TrainingDataset(BaseDataset): expanded_df["has_dwelling_above"] == expanded_df["has_dwelling_above_ending"] ) - ] + ] return expanded_df diff --git a/recommendations/recommendation_utils.py b/recommendations/recommendation_utils.py index 883a387b..82d1b9af 100644 --- a/recommendations/recommendation_utils.py +++ b/recommendations/recommendation_utils.py @@ -7,10 +7,18 @@ import numpy as np import pandas as pd from recommendations.rdsap_tables import ( - epc_wall_description_map, wall_uvalues_df, default_wall_thickness, table_s9 as s9, table_s10 as s10, - table_s11 as s11, table_s12 as s12 + epc_wall_description_map, + wall_uvalues_df, + default_wall_thickness, + table_s9 as s9, + table_s10 as s10, + table_s11 as s11, + table_s12 as s12, +) +from recommendations.config import ( + PARTIALLY_FILLED_PERCENTAGE_ASSUMPTION, + PARTIAL_CAVITY_DESCRIPTIONS, ) -from recommendations.config import PARTIALLY_FILLED_PERCENTAGE_ASSUMPTION, PARTIAL_CAVITY_DESCRIPTIONS def r_value_per_mm_to_u_value(depth_mm: int, r_value_per_mm: float): @@ -62,7 +70,9 @@ def calculate_u_value_uplift(u_value, insulation_u_value): return u_value_uplift, new_u_value -def is_diminishing_returns(recommendations, new_u_value, lowest_selected_u_value, diminishing_returns_u_value): +def is_diminishing_returns( + recommendations, new_u_value, lowest_selected_u_value, diminishing_returns_u_value +): """ What are defines diminishing returns? 1) The new u value is lower than the lowest selected u value @@ -136,9 +146,15 @@ def apply_formula_s_5_1_1(is_granite_or_whinstone, is_sandstone_or_limestone, ag S.5.1.1 """ - stone_wall_thickness = [x for x in default_wall_thickness if x["type"] == "stone"][0] + stone_wall_thickness = [x for x in default_wall_thickness if x["type"] == "stone"][ + 0 + ] - thickness = stone_wall_thickness["J_K_L"] if age_band in ["J", "L", "L"] else stone_wall_thickness[age_band] + thickness = ( + stone_wall_thickness["J_K_L"] + if age_band in ["J", "L", "L"] + else stone_wall_thickness[age_band] + ) if is_granite_or_whinstone: return 3.3 - 0.002 * thickness @@ -146,7 +162,9 @@ def apply_formula_s_5_1_1(is_granite_or_whinstone, is_sandstone_or_limestone, ag if is_sandstone_or_limestone: return 3 - 0.002 * thickness - raise ValueError("This should only be called when is_granite_or_whinstone or is_sandstone_or_limestone is True") + raise ValueError( + "This should only be called when is_granite_or_whinstone or is_sandstone_or_limestone is True" + ) def get_wall_u_value( @@ -164,16 +182,30 @@ def get_wall_u_value( if clean_description in PARTIAL_CAVITY_DESCRIPTIONS: # If we have a partial cavity fill, we linearly interpolate the u-value. This isn't necessarily the perfect # method and how we do this should be explored, however we want to distinguish between the old - filled_uvalue = float(wall_uvalues_df[wall_uvalues_df["Wall_type"] == "Filled cavity"][age_band].values[0]) - unfilled_uvalue = float(wall_uvalues_df[wall_uvalues_df["Wall_type"] == "Cavity as built"][age_band].values[0]) + filled_uvalue = float( + wall_uvalues_df[wall_uvalues_df["Wall_type"] == "Filled cavity"][ + age_band + ].values[0] + ) + unfilled_uvalue = float( + wall_uvalues_df[wall_uvalues_df["Wall_type"] == "Cavity as built"][ + age_band + ].values[0] + ) mapped_value = str( - unfilled_uvalue - (PARTIALLY_FILLED_PERCENTAGE_ASSUMPTION * (unfilled_uvalue - filled_uvalue)) + unfilled_uvalue + - ( + PARTIALLY_FILLED_PERCENTAGE_ASSUMPTION + * (unfilled_uvalue - filled_uvalue) + ) ) else: mapped_description = epc_wall_description_map[clean_description] - mapped_value = wall_uvalues_df[wall_uvalues_df["Wall_type"] == mapped_description][age_band].values[0] + mapped_value = wall_uvalues_df[ + wall_uvalues_df["Wall_type"] == mapped_description + ][age_band].values[0] if pd.isnull(mapped_value) and "Park home" in mapped_description: # We don't know enough in this case so we default to 0 @@ -185,17 +217,19 @@ def get_wall_u_value( apply_formula_s_5_1_1( is_granite_or_whinstone=is_granite_or_whinstone, is_sandstone_or_limestone=is_sandstone_or_limestone, - age_band=age_band + age_band=age_band, ) ) if "b" in mapped_value: potential_uvalue = float(mapped_value.replace("b", "")) - formula_uvalue = float(apply_formula_s_5_1_1( - is_granite_or_whinstone=is_granite_or_whinstone, - is_sandstone_or_limestone=is_sandstone_or_limestone, - age_band=age_band - )) + formula_uvalue = float( + apply_formula_s_5_1_1( + is_granite_or_whinstone=is_granite_or_whinstone, + is_sandstone_or_limestone=is_sandstone_or_limestone, + age_band=age_band, + ) + ) return min(potential_uvalue, formula_uvalue) if mapped_value == "s1.1.2": @@ -205,11 +239,16 @@ def get_wall_u_value( return float(mapped_value) -def get_u_value_from_s9(thickness, s9, is_loft, is_roof_room, is_thatched, is_at_rafters): +def get_u_value_from_s9( + thickness, s9, is_loft, is_roof_room, is_thatched, is_at_rafters +): """Get the U-value from table S9 based on the insulation thickness.""" # If the roof as pitched & insulated at the rafters, it's a room roof if is_roof_room or is_at_rafters: + # TODO: We get None instead of a string none, this should be fixed + if thickness is None: + thickness = "none" # We re-map the thickness thickness_map = { "below average": "50", @@ -233,10 +272,14 @@ def get_u_value_from_s9(thickness, s9, is_loft, is_roof_room, is_thatched, is_at return None # Determine the column to refer based on the roof type - column = 'Thatched_roof_U_value_W_m2K' if is_thatched else 'Slates_or_tiles_U_value_W_m2K' + column = ( + "Thatched_roof_U_value_W_m2K" + if is_thatched + else "Slates_or_tiles_U_value_W_m2K" + ) # Get the correct U-value based on the insulation thickness - return s9[s9['Insulation_thickness_mm'] >= thickness][column].iloc[0] + return s9[s9["Insulation_thickness_mm"] >= thickness][column].iloc[0] def get_roof_u_value( @@ -249,7 +292,7 @@ def get_roof_u_value( is_flat, is_pitched, is_at_rafters, - **kwargs + **kwargs, ): """ Determine the U-value for a roof based on the description dictionary and age band. @@ -292,7 +335,7 @@ def get_roof_u_value( is_loft=is_loft, is_roof_room=is_roof_room, is_thatched=is_thatched, - is_at_rafters=is_at_rafters + is_at_rafters=is_at_rafters, ) if u_value is not None: @@ -302,25 +345,25 @@ def get_roof_u_value( # Define the columns to be used based on the description details if is_flat: - column = 'Flat_roof' + column = "Flat_roof" elif is_thatched: if is_roof_room: - column = 'Thatched_roof_room_in_roof' + column = "Thatched_roof_room_in_roof" else: - column = 'Thatched_roof' + column = "Thatched_roof" elif is_roof_room: - column = 'Room_in_roof_slates_or_tiles' + column = "Room_in_roof_slates_or_tiles" elif is_pitched: if is_at_rafters: - column = 'Pitched_slates_or_tiles_insulation_at_rafters' + column = "Pitched_slates_or_tiles_insulation_at_rafters" else: - column = 'Pitched_slates_or_tiles_insulation_between_joists_or_unknown' + column = "Pitched_slates_or_tiles_insulation_between_joists_or_unknown" else: # Default to pitched roof with insulation between joists or unknown - column = 'Pitched_slates_or_tiles_insulation_between_joists_or_unknown' + column = "Pitched_slates_or_tiles_insulation_between_joists_or_unknown" # Get the U-value from table S10 based on the age band and the determined column - u_value = s10.loc[s10['Age_band'].str.contains(age_band), column].values[0] + u_value = s10.loc[s10["Age_band"].str.contains(age_band), column].values[0] return float(u_value) @@ -397,10 +440,14 @@ def get_exposed_floor_uvalue(insulation_thickness_str, age_band): else: insulation_thickness = int(insulation_thickness_str.replace("mm", "")) - return s12[s12["age_band"] == age_band][f"insulation_{insulation_thickness}"].values[0] + return s12[s12["age_band"] == age_band][ + f"insulation_{insulation_thickness}" + ].values[0] -def get_floor_u_value(floor_type, area, perimeter, age_band, wall_type, insulation_thickness=None): +def get_floor_u_value( + floor_type, area, perimeter, age_band, wall_type, insulation_thickness=None +): """ Estimate the u-value of a suspended floor, based on RdSap methodology Default U-value for UNINSULATED suspended floor, based on RdSAP methodology @@ -446,14 +493,19 @@ def get_floor_u_value(floor_type, area, perimeter, age_band, wall_type, insulati Rsi = 0.17 # in m²K/W Rse = 0.04 # in m²K/W lambda_ins = 0.035 # thermal conductivity of floor insulation in W/m·K - wall_thickness = [x[age_band] for x in default_wall_thickness if x["type"] == wall_type][0] + wall_thickness = [ + x[age_band] for x in default_wall_thickness if x["type"] == wall_type + ][0] if wall_thickness is None and wall_type == "park home": # We don't know enough and likely won't make recommendations return 0 wall_thickness = wall_thickness / 1000 if insulation_thickness is None: - insulation_lookup = s11[s11["Age_band"].str.contains(age_band) & s11["Floor_construction"] == floor_type] + insulation_lookup = s11[ + s11["Age_band"].str.contains(age_band) & s11["Floor_construction"] + == floor_type + ] if insulation_lookup.empty: insulation_thickness = 0 else: @@ -465,7 +517,7 @@ def get_floor_u_value(floor_type, area, perimeter, age_band, wall_type, insulati # Calculate B B = 2 * area / perimeter - if floor_type == 'solid': + if floor_type == "solid": # Calculate dt dt = wall_thickness + lambda_g * (Rsi + Rf + Rse) @@ -475,7 +527,7 @@ def get_floor_u_value(floor_type, area, perimeter, age_band, wall_type, insulati else: U = lambda_g / (0.457 * B + dt) - elif floor_type == 'suspended': + elif floor_type == "suspended": # Define additional constants for suspended floors h = 0.3 # height above external ground level in meters v = 5 # average wind speed at 10 m height in m/s @@ -498,7 +550,9 @@ def get_floor_u_value(floor_type, area, perimeter, age_band, wall_type, insulati U = 1 / (2 * Rsi + Rf + 1 / (Ug + Ux)) else: - raise ValueError("Invalid floor type. Acceptable values are 'solid' or 'suspended'.") + raise ValueError( + "Invalid floor type. Acceptable values are 'solid' or 'suspended'." + ) return round(U, 2) # rounding U value to two decimal places @@ -509,7 +563,13 @@ def extract_insulation_thickness(insulation_thickness_str): :param insulation_thickness_str: :return: """ - if insulation_thickness_str in ["none", "average", "below average", "above average", None]: + if insulation_thickness_str in [ + "none", + "average", + "below average", + "above average", + None, + ]: return None if isinstance(insulation_thickness_str, (float, int)): @@ -527,7 +587,7 @@ def get_wall_type( is_cob, is_system_built, is_park_home, - **kwargs + **kwargs, ) -> Union[str, None]: """ Converts booleans to a string wall type, for querying the wall thickness table @@ -573,10 +633,10 @@ def estimate_external_wall_area(num_floors, floor_height, perimeter, built_form) total_wall_area = wall_area_one_floor * num_floors number_exposed_walls = { - 'End-Terrace': 3, - 'Mid-Terrace': 2, - 'Semi-Detached': 3, - 'Detached': 4, + "End-Terrace": 3, + "Mid-Terrace": 2, + "Semi-Detached": 3, + "Detached": 4, } exposed_wall_area = total_wall_area * (number_exposed_walls.get(built_form, 3) / 4) @@ -622,27 +682,12 @@ def convert_thickness_to_numeric(string_thickness, is_pitched, is_flat): return 0 if is_pitched: - lookup = { - "none": 0, - "below average": 50, - "average": 100, - "above average": 270 - } + lookup = {"none": 0, "below average": 50, "average": 100, "above average": 270} elif is_flat: # For a flat roof, if it's below average, we assume it's 0 and requires a re-roof - lookup = { - "none": 0, - "below average": 0, - "average": 100, - "above average": 150 - } + lookup = {"none": 0, "below average": 0, "average": 100, "above average": 150} else: - lookup = { - "none": 0, - "below average": 100, - "average": 270, - "above average": 270 - } + lookup = {"none": 0, "below average": 100, "average": 270, "above average": 270} mapped = lookup.get(string_thickness) @@ -697,11 +742,16 @@ def estimate_windows( # Assuming most houses will have at least one kitchen and one bathroom # Scale non-habitable windows with the number of habitable rooms non_habitable_base = 2 # Base for kitchen and bathroom - extra_non_habitable = max(0, (number_habitable_rooms - 3) // 2) # Extra for large houses + extra_non_habitable = max( + 0, (number_habitable_rooms - 3) // 2 + ) # Extra for large houses window_count += non_habitable_base + extra_non_habitable # Adjustments based on built form and property type - if property_type in ["House", "Bungalow"] and built_form in ["Semi-Detached", "Detached"]: + if property_type in ["House", "Bungalow"] and built_form in [ + "Semi-Detached", + "Detached", + ]: built_form_lookup = { "Semi-Detached": 3, "Detached": 4, @@ -728,7 +778,10 @@ def estimate_windows( window_count += 2 # Adjust for construction age band - if construction_age_band in ["England and Wales: before 1900", "England and Wales: 1900-1929"]: + if construction_age_band in [ + "England and Wales: before 1900", + "England and Wales: 1900-1929", + ]: # Older houses with smaller, more numerous windows window_count += 1 @@ -751,7 +804,11 @@ def calculate_cavity_age(newest_epc, older_epcs, cleaned): df = [] for x in all_epcs: # Get the cleaned mapping - mapped = [y for y in cleaned["walls-description"] if y["original_description"] == x["walls-description"]] + mapped = [ + y + for y in cleaned["walls-description"] + if y["original_description"] == x["walls-description"] + ] if not mapped: continue df.append( @@ -768,7 +825,9 @@ def calculate_cavity_age(newest_epc, older_epcs, cleaned): return cavity_age -def check_simulation_difference(old_config, new_config, prefix="", keys_with_prefix=None): +def check_simulation_difference( + old_config, new_config, prefix="", keys_with_prefix=None +): """ Given two dictionaries, that describe the heating control configurations, this method will compare the two and pick out the differences. These differences will be things that have been added and things that have been @@ -777,14 +836,17 @@ def check_simulation_difference(old_config, new_config, prefix="", keys_with_pre """ keys_with_prefix = ( - ["is_assumed", "thermal_transmittance", "insulation_thickness"] if keys_with_prefix is None + ["is_assumed", "thermal_transmittance", "insulation_thickness"] + if keys_with_prefix is None else keys_with_prefix ) differences = {} for key in new_config: if old_config[key] != new_config[key]: - new_key = prefix + key + "_ending" if key in keys_with_prefix else key + "_ending" + new_key = ( + prefix + key + "_ending" if key in keys_with_prefix else key + "_ending" + ) differences[new_key] = new_config[key] return differences @@ -811,17 +873,18 @@ def combine_recommendation_configs(recommendation_config1, recommendation_config """ # Efficiency values - keys which contain _energy_eff_ending eff_1 = { - k: v for k, v in recommendation_config1.items() if ("_energy_eff_ending" in k) or ("-energy-eff" in k) + k: v + for k, v in recommendation_config1.items() + if ("_energy_eff_ending" in k) or ("-energy-eff" in k) } eff_2 = { - k: v for k, v in recommendation_config2.items() if ("_energy_eff_ending" in k) or ("-energy-eff" in k) + k: v + for k, v in recommendation_config2.items() + if ("_energy_eff_ending" in k) or ("-energy-eff" in k) } # We combine the simulation configs - combined = { - **recommendation_config1, - **recommendation_config2 - } + combined = {**recommendation_config1, **recommendation_config2} # Find overlapping keys overlapping_keys = set(eff_1.keys()).intersection(set(eff_2.keys())) From aa97c777475cd036ee60004617cbb891f454f254 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 7 Oct 2024 10:38:38 +0100 Subject: [PATCH 109/166] using elmhurst roof area methodology --- backend/Property.py | 17 ++------------ backend/apis/GoogleSolarApi.py | 1 - recommendations/recommendation_utils.py | 31 ++++++------------------- 3 files changed, 9 insertions(+), 40 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index 491a886f..9108d40c 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -623,9 +623,7 @@ class Property: self.find_energy_sources() self.set_current_energy(kwh_client, kwh_predictions) - def set_solar_panel_configuration( - self, solar_panel_configuration, roof_area - ): + def set_solar_panel_configuration(self, solar_panel_configuration): """ This funtion inserts the solar panel configuration into the property object """ @@ -634,22 +632,11 @@ class Property: if not self.roof["is_flat"]: default_roof_area = estimate_pitched_roof_area( floor_area=self.insulation_floor_area, - floor_height=self.floor_height ) else: default_roof_area = self.insulation_floor_area - # Keep a record - self.roof_area_comparison = { - "api": roof_area, - "estimated": default_roof_area - } - - # We also set the roof area - if roof_area is None: - self.roof_area = default_roof_area - else: - self.roof_area = roof_area + self.roof_area = default_roof_area def set_current_energy(self, kwh_client, kwh_predictions): """ diff --git a/backend/apis/GoogleSolarApi.py b/backend/apis/GoogleSolarApi.py index bf67a786..606b6970 100644 --- a/backend/apis/GoogleSolarApi.py +++ b/backend/apis/GoogleSolarApi.py @@ -766,7 +766,6 @@ class GoogleSolarApi: "panel_performance": cls.default_panel_performance(property_instance=property_instance), "unit_share_of_energy": 1 }, - roof_area=None ) continue diff --git a/recommendations/recommendation_utils.py b/recommendations/recommendation_utils.py index 3a0412b2..dcdd9c06 100644 --- a/recommendations/recommendation_utils.py +++ b/recommendations/recommendation_utils.py @@ -205,7 +205,7 @@ def get_wall_u_value( mapped_value = wall_uvalues_df[ wall_uvalues_df["Wall_type"] == mapped_description - ][age_band].values[0] + ][age_band].values[0] if pd.isnull(mapped_value) and "Park home" in mapped_description: # We don't know enough in this case so we default to 0 @@ -505,7 +505,7 @@ def get_floor_u_value( insulation_lookup = s11[ s11["Age_band"].str.contains(age_band) & s11["Floor_construction"] == floor_type - ] + ] if insulation_lookup.empty: insulation_thickness = 0 else: @@ -700,34 +700,17 @@ def convert_thickness_to_numeric(string_thickness, is_pitched, is_flat): return int(string_thickness) -def estimate_pitched_roof_area(floor_area: float, floor_height: float) -> float: +def estimate_pitched_roof_area(floor_area: float) -> float: """ - This function will estimate the area of a pitched roof, given the floor area below the roof and the floor - height of the property. - - Given limited information about the home, this is a very rough method to estimate the roof area and we - assume the the room is a gable roof. - - We assume a roughly average pitch of 45 degrees - - Note that both floor area and height should be in the same units. E.g. if floor area is meters squared, - floor height should be in meters + This function mimics the methodology for calculating floor area in Elmhurst, so that we can simulate the outcomes + in a way that is consistent with the Elmhurst methodology. :param floor_area: area of the home's floor - :param floor_height: height of the home's floors :return: Numerical estimate of the surface area of the top of the pitched roof """ - # We estimate the length of the wall by just modelling the house as a square - wall_width = np.sqrt(floor_area) - - # We're modelling the roof as two triangles where we know two of the three sides. - # The floor height makes up one side and half of the wall width makes up the other side - slope = np.sqrt(np.square(wall_width / 2) + np.square(floor_height)) - - area = 2 * (slope * wall_width) - - return area + scalar = 1.0571283428862048 + return scalar * (floor_area / np.cos(np.radians(30))) def estimate_windows( From 5f0c0e7726b08694cad7f128895eac3667444dcc Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 7 Oct 2024 10:58:18 +0100 Subject: [PATCH 110/166] updating floor area unit tests --- .../tests/test_recommendation_utils.py | 58 ++++++------------- 1 file changed, 17 insertions(+), 41 deletions(-) diff --git a/recommendations/tests/test_recommendation_utils.py b/recommendations/tests/test_recommendation_utils.py index c42655eb..24ea6482 100644 --- a/recommendations/tests/test_recommendation_utils.py +++ b/recommendations/tests/test_recommendation_utils.py @@ -359,60 +359,36 @@ def test_park_home(): ) == 0 -def test_esimtate_pitched_roof_area(): - roof_area1 = recommendation_utils.esimtate_pitched_roof_area( - floor_area=100, floor_height=2 +def test_estimate_pitched_roof_area(): + roof_area0 = recommendation_utils.estimate_pitched_roof_area( + floor_area=80, + ) + assert np.isclose(roof_area0, 97.65333333333334) + + roof_area1 = recommendation_utils.estimate_pitched_roof_area( + floor_area=100, ) - assert np.isclose(roof_area1, 107.70329614269008) + assert np.isclose(roof_area1, 122.06666666666666) - # As the floor height gets bigger, the area should get bigger - roof_area2 = recommendation_utils.esimtate_pitched_roof_area( - floor_area=100, floor_height=3 + roof_area2 = recommendation_utils.estimate_pitched_roof_area( + floor_area=45, ) - assert np.isclose(roof_area2, 116.61903789690601) + assert np.isclose(roof_area2, 54.93) - # As the floor area gets smaller, the area should get smaller - roof_area3 = recommendation_utils.esimtate_pitched_roof_area( - floor_area=100, floor_height=1 + roof_area3 = recommendation_utils.estimate_pitched_roof_area( + floor_area=60, ) - assert np.isclose(roof_area3, 101.9803902718557) + assert np.isclose(roof_area3, 73.24) - # As the floor area decreases, area should decrease - roof_area4 = recommendation_utils.esimtate_pitched_roof_area( - floor_area=50, floor_height=2 - ) - - assert np.isclose(roof_area4, 57.44562646538029) - - # As the floor area increases, area should increase - roof_area5 = recommendation_utils.esimtate_pitched_roof_area( - floor_area=150, floor_height=2 - ) - - assert np.isclose(roof_area5, 157.797338380595) - - zero_roof_area = recommendation_utils.esimtate_pitched_roof_area( - floor_area=0, floor_height=1000 + zero_roof_area = recommendation_utils.estimate_pitched_roof_area( + floor_area=0, ) assert zero_roof_area == 0 - # If the floor height zero, we don't have a traingle, it's a flat roof - flat_roof_area = recommendation_utils.esimtate_pitched_roof_area( - floor_area=1000, floor_height=0 - ) - - assert flat_roof_area == 1000 - - zero_roof_area2 = recommendation_utils.esimtate_pitched_roof_area( - floor_area=0, floor_height=0 - ) - - assert zero_roof_area2 == 0 - def test_external_wall_area(): # Arrange: Define the test cases From 92fce7f093fbbe83ad7890e46c3de872c234f977 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 7 Oct 2024 11:03:52 +0100 Subject: [PATCH 111/166] Updating ligtiung unit test to include measure_type --- backend/Property.py | 2 +- .../tests/test_lighting_recommendations.py | 22 ++++++++++++------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index 9108d40c..be5479c5 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -112,7 +112,7 @@ class Property: self.measures = ast.literal_eval(measures) if measures else None self.uprn = epc_record.get("uprn") - self.uprn_source = self.data["uprn-source"] + self.uprn_source = self.data.get("uprn-source") self.full_sap_epc = epc_record.get("full_sap_epc") self.in_conservation_area, self.is_listed, self.is_heritage = None, None, None diff --git a/recommendations/tests/test_lighting_recommendations.py b/recommendations/tests/test_lighting_recommendations.py index 96440c01..32d607de 100644 --- a/recommendations/tests/test_lighting_recommendations.py +++ b/recommendations/tests/test_lighting_recommendations.py @@ -41,12 +41,18 @@ class TestLightingRecommendations: assert len(lr.recommendation) == 1 assert lr.recommendation == [ - {'phase': 0, 'parts': [], 'type': 'low_energy_lighting', - 'description': 'Install low energy lighting in 4 outlets', 'starting_u_value': None, 'new_u_value': None, - 'already_installed': False, 'sap_points': 0.4, 'kwh_savings': 219.0, 'co2_equivalent_savings': 0.035478, - 'description_simulation': {'lighting-energy-eff': 'Very Good', - 'lighting-description': 'Low energy lighting in all fixed outlets', - 'low-energy-lighting': 100}, 'total': 240.24, 'subtotal': 200.20000000000002, - 'vat': 40.040000000000006, 'contingency': 14.3, 'preliminaries': 14.3, 'material': 80.0, 'profit': 28.6, - 'labour_hours': 3.2, 'labour_days': 0.4, 'labour_cost': 63.0, 'survey': False} + { + 'phase': 0, 'parts': [], 'type': 'low_energy_lighting', 'measure_type': 'low_energy_lighting', + 'description': 'Install low energy lighting in 4 outlets', 'starting_u_value': None, + 'new_u_value': None, + 'already_installed': False, 'sap_points': 0.4, 'kwh_savings': 219.0, 'co2_equivalent_savings': 0.035478, + 'description_simulation': { + 'lighting-energy-eff': 'Very Good', + 'lighting-description': 'Low energy lighting in all fixed outlets', + 'low-energy-lighting': 100 + }, + 'total': 240.24, 'subtotal': 200.20000000000002, + 'vat': 40.040000000000006, 'contingency': 14.3, 'preliminaries': 14.3, 'material': 80.0, 'profit': 28.6, + 'labour_hours': 3.2, 'labour_days': 0.4, 'labour_cost': 63.0, 'survey': False + } ] From 8ff29603c326bc2b5ef2f472b79b5daa945e7ba9 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 7 Oct 2024 11:05:36 +0100 Subject: [PATCH 112/166] adding measure type to solar unit tests --- .../tests/test_solar_pv_recommendations.py | 27 +++++++------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/recommendations/tests/test_solar_pv_recommendations.py b/recommendations/tests/test_solar_pv_recommendations.py index 05349f9c..a18291e5 100644 --- a/recommendations/tests/test_solar_pv_recommendations.py +++ b/recommendations/tests/test_solar_pv_recommendations.py @@ -3,12 +3,6 @@ from recommendations.SolarPvRecommendations import SolarPvRecommendations from backend.Property import Property from etl.epc.Record import EPCRecord import pandas as pd -from datetime import datetime -from utils.s3 import read_dataframe_from_s3_parquet, read_from_s3 -from etl.solar.SolarPhotoSupply import SolarPhotoSupply -from recommendations.Recommendations import Recommendations -from backend.ml_models.api import ModelApi -import msgpack class TestSolarPvRecommendations: @@ -86,9 +80,10 @@ class TestSolarPvRecommendations: def test_valid_all_conditions(self, property_instance_valid_all): solar_pv = SolarPvRecommendations(property_instance_valid_all) solar_pv.recommend(phase=0) + assert len(solar_pv.recommendation) == 2 assert solar_pv.recommendation == [ { - 'phase': 0, 'parts': [], 'type': 'solar_pv', + 'phase': 0, 'parts': [], 'type': 'solar_pv', 'measure_type': 'solar_pv', 'description': 'Install a 4.0 kilowatt-peak (kWp) solar photovoltaic (PV) panel system on 50% the ' 'roof.', 'starting_u_value': None, 'new_u_value': None, 'sap_points': None, 'already_installed': False, @@ -97,17 +92,13 @@ class TestSolarPvRecommendations: 'description_simulation': {'photo-supply': 50.0} }, { - 'phase': 0, 'parts': [], 'type': 'solar_pv', - 'description': 'Install a 4.0 kilowatt-peak (kWp) ' - 'solar photovoltaic (PV) panel system ' - 'on 50% the roof, with a battery ' - 'storage system.', - 'starting_u_value': None, 'new_u_value': None, - 'sap_points': None, 'already_installed': False, - 'total': 7550.0, 'subtotal': 6291.666666666667, - 'vat': 1258.333333333333, 'labour_hours': 48, - 'labour_days': 2, 'photo_supply': 50.0, - 'has_battery': True, 'initial_ac_kwh_per_year': 3800, + 'phase': 0, 'parts': [], 'type': 'solar_pv', 'measure_type': 'solar_pv', + 'description': 'Install a 4.0 kilowatt-peak (kWp) solar photovoltaic (PV) panel system on 50% the ' + 'roof, ' + 'with a battery storage system.', + 'starting_u_value': None, 'new_u_value': None, 'sap_points': None, 'already_installed': False, + 'total': 7550.0, 'subtotal': 6291.666666666667, 'vat': 1258.333333333333, 'labour_hours': 48, + 'labour_days': 2, 'photo_supply': 50.0, 'has_battery': True, 'initial_ac_kwh_per_year': 3800, 'description_simulation': {'photo-supply': 50.0} } ] From 7bf897ceb4ce4e703bdb89bda58c2823dc0e1e21 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 7 Oct 2024 11:17:11 +0100 Subject: [PATCH 113/166] fixed unitt tests --- .../tests/test_window_recommendations.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/recommendations/tests/test_window_recommendations.py b/recommendations/tests/test_window_recommendations.py index 0e36d105..ae6c6377 100644 --- a/recommendations/tests/test_window_recommendations.py +++ b/recommendations/tests/test_window_recommendations.py @@ -43,7 +43,7 @@ class TestWindowRecommendations: assert recommender.recommendation == [ { - 'phase': 0, 'parts': [], 'type': 'windows_glazing', + 'phase': 0, 'parts': [], 'type': 'windows_glazing', "measure_type": "double_glazing", 'description': 'Install double glazing to all windows', 'starting_u_value': None, 'new_u_value': None, 'sap_points': None, 'already_installed': False, 'total': 7980.0, 'labour_hours': 0.0, 'labour_days': 0.0, 'is_secondary_glazing': False, @@ -92,7 +92,7 @@ class TestWindowRecommendations: assert recommender2.recommendation == [ { - 'phase': 0, 'parts': [], 'type': 'windows_glazing', + 'phase': 0, 'parts': [], 'type': 'windows_glazing', "measure_type": "double_glazing", 'description': 'Install double glazing to the remaining windows', 'starting_u_value': None, 'new_u_value': None, 'sap_points': None, 'already_installed': False, 'total': 5700.0, 'labour_hours': 0.0, @@ -193,7 +193,7 @@ class TestWindowRecommendations: assert recommender5.recommendation == [ { - 'phase': 0, 'parts': [], 'type': 'windows_glazing', + 'phase': 0, 'parts': [], 'type': 'windows_glazing', 'measure_type': 'secondary_glazing', 'description': 'Install secondary glazing to the remaining windows', 'starting_u_value': None, 'new_u_value': None, 'sap_points': None, 'already_installed': False, 'total': 4560.0, 'labour_hours': 0.0, 'labour_days': 0.0, 'is_secondary_glazing': True, @@ -240,7 +240,7 @@ class TestWindowRecommendations: assert recommender6.recommendation == [ { - 'phase': 0, 'parts': [], 'type': 'windows_glazing', + 'phase': 0, 'parts': [], 'type': 'windows_glazing', 'measure_type': 'secondary_glazing', 'description': 'Install secondary glazing to all windows. Secondary glazing recommended due to ' 'herigate building status', 'starting_u_value': None, 'new_u_value': None, 'sap_points': None, 'already_installed': False, @@ -396,7 +396,7 @@ class TestWindowRecommendations: assert recommender9.recommendation == [ { - 'phase': 0, 'parts': [], 'type': 'windows_glazing', + 'phase': 0, 'parts': [], 'type': 'windows_glazing', 'measure_type': 'double_glazing', 'description': 'Install double glazing to all windows', 'starting_u_value': None, 'new_u_value': None, 'sap_points': None, 'already_installed': False, 'total': 7980.0, 'labour_hours': 0.0, 'labour_days': 0.0, 'is_secondary_glazing': False, @@ -638,7 +638,8 @@ class TestWindowRecommendations: 'glazing_coverage_ending': 'full', 'id': '1+1' } - assert simulated_data[0] == expected_simulated_outcome + # Make sure all keys are the same, apart from days_to_ending + assert all([v == expected_simulated_outcome[k] for k, v in simulated_data[0].items() if k != "days_to_ending"]) # has_glazing_ending and glazing_coverage_ending are not in the starting record - test for this in case it # changes @@ -648,7 +649,7 @@ class TestWindowRecommendations: # Check which keys are different different = [] for k in simulated_data[0].keys(): - if k in ["id", 'has_glazing_ending', 'glazing_coverage_ending']: + if k in ["id", 'has_glazing_ending', 'glazing_coverage_ending', 'days_to_ending']: continue if simulated_data[0][k] != starting_record[k]: different.append( @@ -666,7 +667,6 @@ class TestWindowRecommendations: 'simulated': 'double glazing installed during or after 2002'}, {'variable': 'multi_glaze_proportion_ending', 'starting': 0.0, 'simulated': 100}, {'variable': 'windows_energy_eff_ending', 'starting': 'Very Poor', 'simulated': 'Average'}, - {'variable': 'days_to_ending', 'starting': 3642, 'simulated': 3713} ] assert different == expected_different From 5078368faf2e38c7cc3700bd590f2e82ae2d7280 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 7 Oct 2024 11:42:03 +0100 Subject: [PATCH 114/166] fixed generate_scenarios_data --- backend/Property.py | 19 ++++---- etl/epc/generate_scenarios_data.py | 53 ++++++++++------------- recommendations/SolarPvRecommendations.py | 4 +- 3 files changed, 33 insertions(+), 43 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index be5479c5..ab8930c5 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -566,8 +566,6 @@ class Property: if not self.data: raise ValueError("Property does not contain data") - self.set_basic_property_dimensions() - for description, attribute in cleaned.items(): if self.data[description] in self.DATA_ANOMALY_MATCHES: @@ -615,6 +613,7 @@ class Property: setattr(self, self.ATTRIBUTE_MAP[description], attributes[0]) + self.set_basic_property_dimensions() self.set_wall_type() self.set_floor_type() self.set_floor_level() @@ -629,15 +628,6 @@ class Property: """ self.solar_panel_configuration = solar_panel_configuration - if not self.roof["is_flat"]: - default_roof_area = estimate_pitched_roof_area( - floor_area=self.insulation_floor_area, - ) - else: - default_roof_area = self.insulation_floor_area - - self.roof_area = default_roof_area - def set_current_energy(self, kwh_client, kwh_predictions): """ Given what we know about the property now, estimates the current energy consumption using the UCL paper @@ -972,6 +962,13 @@ class Property: self.floor_area / self.number_of_floors ) + if not self.roof["is_flat"]: + self.roof_area = estimate_pitched_roof_area( + floor_area=self.insulation_floor_area, + ) + else: + self.roof_area = self.insulation_floor_area + def set_floor_level(self): self.floor_level = ( FLOOR_LEVEL_MAP[self.data["floor-level"]] diff --git a/etl/epc/generate_scenarios_data.py b/etl/epc/generate_scenarios_data.py index df1f9452..3497225c 100644 --- a/etl/epc/generate_scenarios_data.py +++ b/etl/epc/generate_scenarios_data.py @@ -3,25 +3,24 @@ import itertools import pandas as pd from etl.epc.Record import EPCRecord +from etl.bill_savings.KwhData import KwhData from backend.SearchEpc import SearchEpc from sqlalchemy.orm import sessionmaker -from backend.app.config import get_settings +from backend.app.config import get_settings, get_prediction_buckets from backend.app.db.connection import db_engine from backend.app.db.functions.materials_functions import get_materials +from backend.ml_models.api import ModelApi from backend.app.plan.utils import get_cleaned from backend.Property import Property -from etl.solar.SolarPhotoSupply import SolarPhotoSupply from recommendations.Recommendations import Recommendations from utils.logger import setup_logger from utils.s3 import read_dataframe_from_s3_parquet, save_dataframe_to_s3_parquet -from datetime import datetime - now = datetime.now().strftime("%d-%m-%Y-%H-%M-%S") logger = setup_logger() @@ -41,21 +40,16 @@ cleaning_data = read_dataframe_from_s3_parquet( materials = get_materials(session) cleaned = get_cleaned() -# TODO: THIS IS A TEMPORARY FIX -new_walls_description_mapping = pd.DataFrame(cleaned["walls-description"]) -new_walls_description_mapping.loc[ - ~new_walls_description_mapping["thermal_transmittance_unit"].isnull(), - "thermal_transmittance_unit", -] = "w/m-¦k" - -cleaned["walls-description"] = new_walls_description_mapping.to_dict(orient="records") - uprn_filenames = read_dataframe_from_s3_parquet( bucket_name=get_settings().DATA_BUCKET, file_key="spatial/filename_meta.parquet" ) -photo_supply_lookup, floor_area_decile_thresholds = SolarPhotoSupply.load( - bucket=get_settings().DATA_BUCKET + +kwh_client = KwhData(bucket="retrofit-data-dev", read_consumption_data=False) +kwh_client.retail_price_comparison = pd.DataFrame( + [{"Date": datetime.today().strftime("%Y-%m-%d"), + 'Average standard variable tariff (Large legacy suppliers)': 1}] ) +kwh_client.retail_price_comparison["Date"] = pd.to_datetime(kwh_client.retail_price_comparison["Date"]) scenario_properties = [ { @@ -132,7 +126,6 @@ scenario_properties = [ }, ] - recommendations_scoring_data = [] for scenario_property in scenario_properties: @@ -173,7 +166,17 @@ for scenario_property in scenario_properties: ) p.get_spatial_data(uprn_filenames) - p.get_components(cleaned, photo_supply_lookup, floor_area_decile_thresholds) + + kwh_predictions = { + "heating_kwh_predictions": pd.DataFrame([{"id": p.uprn, "predictions": 12000}]), + "hotwater_kwh_predictions": pd.DataFrame([{"id": p.uprn, "predictions": 3000}]), + } + p.set_features(cleaned, kwh_client, kwh_predictions) + p.solar_panel_configuration = { + "panel_performance": pd.DataFrame( + [{"panneled_roof_area": 34, "n_panels": 10, "array_wattage": 4000, "initial_ac_kwh_per_year": 3800}] + ) + } recommender = Recommendations(property_instance=p, materials=materials) property_recommendations = recommender.recommend() @@ -277,20 +280,12 @@ recommendations_scoring_data.insert(0, "impact", impact_col) id_col = recommendations_scoring_data.pop("id") recommendations_scoring_data.insert(0, "id", id_col) -from backend.ml_models.api import ModelApi - -model_api = ModelApi(portfolio_id="generate-scenarios-data", timestamp=created_at) - -all_predictions = model_api.predict_all( - df=recommendations_scoring_data, - bucket=get_settings().DATA_BUCKET, - prediction_buckets={ - "sap_change_predictions": get_settings().SAP_PREDICTIONS_BUCKET, - "heat_demand_predictions": get_settings().HEAT_PREDICTIONS_BUCKET, - "carbon_change_predictions": get_settings().CARBON_PREDICTIONS_BUCKET, - }, +model_api = ModelApi( + portfolio_id="generate-scenarios-data", timestamp=created_at, prediction_buckets=get_prediction_buckets() ) +all_predictions = model_api.predict_all(df=recommendations_scoring_data, bucket=get_settings().DATA_BUCKET) + save_dataframe_to_s3_parquet( recommendations_scoring_data, "retrofit-data-dev", diff --git a/recommendations/SolarPvRecommendations.py b/recommendations/SolarPvRecommendations.py index 08f077d2..1eb584ca 100644 --- a/recommendations/SolarPvRecommendations.py +++ b/recommendations/SolarPvRecommendations.py @@ -175,9 +175,7 @@ class SolarPvRecommendations: if self.property.roof["is_flat"]: roof_area = self.property.insulation_floor_area else: - roof_area = estimate_pitched_roof_area( - floor_area=self.property.insulation_floor_area, floor_height=self.property.data["floor-height"] - ) + roof_area = estimate_pitched_roof_area(floor_area=self.property.insulation_floor_area, ) solar_configurations = pd.DataFrame( [ { From f67668eb417d1f4077498afb792bc681020b2f99 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 7 Oct 2024 12:43:40 +0100 Subject: [PATCH 115/166] tweaking google solar api ranking algorithm --- backend/apis/GoogleSolarApi.py | 20 +++++++------------- backend/app/assumptions.py | 3 +++ 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/backend/apis/GoogleSolarApi.py b/backend/apis/GoogleSolarApi.py index 606b6970..f6e1b96d 100644 --- a/backend/apis/GoogleSolarApi.py +++ b/backend/apis/GoogleSolarApi.py @@ -69,7 +69,7 @@ class GoogleSolarApi: self.floor_area = None self.roof_area = None self.roof_segment_indexes = None - self.panel_area = None + self.panel_area = assumptions.RDSAP_AREA_PER_PANEL self.panel_wattage = None self.panel_performance = None @@ -166,10 +166,6 @@ class GoogleSolarApi: self.roof_area = self.insights_data["solarPotential"]["wholeRoofStats"]['areaMeters2'] self.floor_area = self.insights_data["solarPotential"]["wholeRoofStats"]['groundAreaMeters2'] - self.panel_area = ( - self.insights_data["solarPotential"]["panelHeightMeters"] * - self.insights_data["solarPotential"]["panelWidthMeters"] - ) self.panel_wattage = self.insights_data["solarPotential"]["panelCapacityWatts"] if self.panel_wattage != 400: # In the API documentation, it claims that the default output is 250W, however we've only seen 400W, so if @@ -450,8 +446,8 @@ class GoogleSolarApi: # We want max roi, minimal generation deficit, and max generation value - we create a ranking score # Assign equal weights to each metric - weights = {'roi': 0.6, 'generation_value': 0.2, 'generation_deficit': 0.2} - metrics = panel_performance[['roi', 'generation_value', 'generation_deficit']] + weights = {'roi': 0.8, 'generation_value': 0.2} + metrics = panel_performance[['roi', 'generation_value']].copy() # Normalize the columns (0 to 1 scale) scaler = MinMaxScaler() @@ -459,12 +455,11 @@ class GoogleSolarApi: # Convert normalized metrics back to a dataframe normalized_metrics_df = pd.DataFrame( - normalized_metrics, columns=['roi', 'generation_value', 'generation_deficit'] + normalized_metrics, columns=['roi', 'generation_value'] ) normalized_metrics_df['combined_score'] = ( normalized_metrics_df['roi'] * weights['roi'] + - normalized_metrics_df['generation_value'] * weights['generation_value'] + - (1 - normalized_metrics_df['generation_deficit']) * weights['generation_deficit'] + normalized_metrics_df['generation_value'] * weights['generation_value'] ) panel_performance['combined_score'] = normalized_metrics_df['combined_score'].values @@ -799,7 +794,6 @@ class GoogleSolarApi: "panel_performance": solar_api_client.panel_performance, "unit_share_of_energy": 1 }, - roof_area=solar_api_client.roof_area ) return input_properties @@ -825,7 +819,7 @@ class GoogleSolarApi: n_panels=10, has_battery=False, n_floors=property_instance.number_of_floors )["total"], 'weighted_ratio': None, - 'panneled_roof_area': 10 * 1.8, + 'panneled_roof_area': 10 * assumptions.RDSAP_AREA_PER_PANEL, 'array_wattage': 4000, 'initial_ac_kwh_per_year': 4000 * 0.95, # Assumed 95% efficient wattage -> ac 'lifetime_ac_kwh': None, @@ -845,7 +839,7 @@ class GoogleSolarApi: n_panels=6, has_battery=False, n_floors=property_instance.number_of_floors )["total"], 'weighted_ratio': None, - 'panneled_roof_area': 6 * 1.8, + 'panneled_roof_area': 6 * assumptions.RDSAP_AREA_PER_PANEL, 'array_wattage': 2400, 'initial_ac_kwh_per_year': 2400 * 0.95, # Assumed 95% efficient wattage -> ac 'lifetime_ac_kwh': None, diff --git a/backend/app/assumptions.py b/backend/app/assumptions.py index 80baa69f..371b226d 100644 --- a/backend/app/assumptions.py +++ b/backend/app/assumptions.py @@ -7,6 +7,9 @@ AVERAGE_ASHP_EFFICIENCY = 250 # be exported SOLAR_CONSUMPTION_PROPORTION = 0.5 +# Typically, each solar panel takes up around 3.4 m2 of roof space under RdSAP. This was been verified in Elmhurst +RDSAP_AREA_PER_PANEL = 3.4 + DESCRIPTIONS_TO_FUEL_TYPES = { "Air source heat pump, radiators, electric": { "fuel": "Electricity", "cop": AVERAGE_ASHP_EFFICIENCY / 100 From 4b1dab4abe8478ff0c3da94c4beb9d0501239041 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 7 Oct 2024 12:56:26 +0100 Subject: [PATCH 116/166] constrain solar api to allow max 70% roof coverage --- recommendations/SolarPvRecommendations.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/recommendations/SolarPvRecommendations.py b/recommendations/SolarPvRecommendations.py index 08f077d2..2c93689d 100644 --- a/recommendations/SolarPvRecommendations.py +++ b/recommendations/SolarPvRecommendations.py @@ -17,6 +17,8 @@ class SolarPvRecommendations: MAX_SYSTEM_WATTAGE = 6000 MIN_SYSTEM_WATTAGE = 1000 + MAX_ROOF_AREA_PERCENTAGE = 0.7 + def __init__(self, property_instance): """ :param property_instance: Instance of the Property class, for the home associated to property_id @@ -189,15 +191,20 @@ class SolarPvRecommendations: ) else: # TODO: There may be some instances where we don't want to use the solar API so we should cover for them - panel_performance = self.property.solar_panel_configuration["panel_performance"] + panel_performance = self.property.solar_panel_configuration["panel_performance"].copy() + # We don't allow for more than 70% of the roof to be covered + panel_performance = panel_performance[ + panel_performance["panneled_roof_area"] / self.property.roof_area <= self.MAX_ROOF_AREA_PERCENTAGE + ] + roof_area = self.property.roof_area solar_configurations = panel_performance.head(3).reset_index(drop=True) # We combine each of these configurations with estimates with and without a battery for rank, recommendation_config in solar_configurations.iterrows(): roof_coverage_percent = round(recommendation_config["panneled_roof_area"] / roof_area * 100) - # We round up to the nearest 10 - roof_coverage_percent = np.ceil(roof_coverage_percent / 10) * 10 + # We round up to the nearest 5 + roof_coverage_percent = np.ceil(roof_coverage_percent / 5) * 5 for has_battery in [False, True]: cost_result = self.costs.solar_pv( has_battery=has_battery, From 022bf6742cc87e4a6c9ccfbac3b155fd47a1935c Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 7 Oct 2024 15:59:44 +0100 Subject: [PATCH 117/166] adding new cosing for ashp --- recommendations/Costs.py | 15 ++++-- recommendations/HeatingRecommender.py | 73 ++++++++++++++++++++++++++- 2 files changed, 82 insertions(+), 6 deletions(-) diff --git a/recommendations/Costs.py b/recommendations/Costs.py index 08b05a8a..afcd48d9 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -1143,7 +1143,7 @@ class Costs: "labour_days": labour_days, } - def air_source_heat_pump(self): + def air_source_heat_pump(self, ashp_size): """ Based on the region and type of property, this function will produce a cost estimation for an air source heat pump. This cost will include the boiler upgrade scheme grant @@ -1151,14 +1151,19 @@ class Costs: """ # This is the average cost of a project, we'll add some additional contingency - regional_cost = MCS_AIR_SOURCE_HEAT_PUMP_COST_DATA[self.region] - total_cost = regional_cost * (1 + self.CONTINGENCY) - BOILER_UPGRADE_SCHEME_ASHP_VALUE + if ashp_size is None: + cost = [x for x in INSTALLER_ASHP_COSTS if x["capacity_kw"] is None][0]["cost"] + else: + cost = [x for x in INSTALLER_ASHP_COSTS if x][0]["cost"] + + # We add some contingency since there are additional costs such as resizing radiators, that could be required + total_cost = cost * (1 + self.CONTINGENCY) subtotal_before_vat = total_cost / (1 + self.VAT_RATE) vat = total_cost - subtotal_before_vat - # We assume 3 days installation - labour_days = 3 + # We assume 5 days installation + labour_days = 5 labour_hours = labour_days * 8 return { diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index 203bab87..bdc73e3b 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -1,3 +1,4 @@ +import re from recommendations.Costs import Costs, BOILER_UPGRADE_SCHEME_ASHP_VALUE from recommendations.recommendation_utils import ( check_simulation_difference, override_costs, combine_recommendation_configs @@ -368,6 +369,75 @@ class HeatingRecommender: description = ("Replace the existing boiler and cylinder without a thermostat with a new electric combi " "boiler") + def size_heat_pump(self): + """ + Given the methodology by installers (SCIS) this function will perform a basic heat loss calculation and + produce a recommendation for the size of the heat pump + :return: + """ + + floor_area = self.property.floor_area + + # We use the default heat loss W/m2 values are specified by the insaller, depending on the property type + + def remap_to_heat_loss(construction_age_band): + if "before 1900" in construction_age_band: + return "Pre 1900 (solid stone)" + elif "1900-1929" in construction_age_band: + return "Early 1900s (solid brick)" + elif re.search(r'1930|1949|1950|1966|1967|1975', construction_age_band): + return "1950-1980 (cavity void)" + elif re.search(r'1976|1982|1983|1990', construction_age_band): + return "Post 1980 (cavity wall construction)" + elif re.search(r'1991|1995|1996|2002|2003|2011', construction_age_band): + return "2000-2018" + elif "2012 onwards" in construction_age_band: + return "New build (2018+)" + else: + return None + + def select_heatpump_size(heat_loss_calculation): + """ + This function calculates the size of the heat pump based on the heat loss calculation, mapping + the heat loss calculation to the size of the heat pump in KW + :param heat_loss_calculation: This is calcualted as the floor area multipled by the heat loss constant, + divided by 1000 + """ + if heat_loss_calculation < 5: + return 5 + elif 5 <= heat_loss_calculation < 6: + return 6 + elif 6 <= heat_loss_calculation < 8.5: + return 8.5 + elif 8.5 <= heat_loss_calculation < 11.2: + return 11.2 + elif 11.2 <= heat_loss_calculation < 14: + return 14 + elif 14 <= heat_loss_calculation < 17: + return 17 + elif 17 <= heat_loss_calculation < 20: + return 20 + else: + return None + + heat_loss_constants = { + "New build (2018+)": 35, + "2000-2018": 50, + "Post 1980 (cavity wall construction)": 60, + "1950-1980 (cavity void)": 70, + "Early 1900s (solid brick)": 80, + "Pre 1900 (solid stone)": 90 + } + + heat_loss_group = remap_to_heat_loss(self.property.construction_age_band) + heat_loss_constant = heat_loss_constants[heat_loss_group] + + heat_loss_calculation = floor_area * heat_loss_constant / 1000 + + heat_pump_size = select_heatpump_size(heat_loss_calculation) + + return heat_pump_size + def recommend_air_source_heat_pump(self, phase, has_cavity_or_loft_recommendations, _return=False): """ This method will implement the recommendation for an air source heat pump @@ -383,8 +453,9 @@ class HeatingRecommender: controls_recommender = HeatingControlRecommender(self.property) controls_recommender.recommend(heating_description="Air source heat pump, radiators, electric") + ashp_size = self.size_heat_pump() - ashp_costs = self.costs.air_source_heat_pump() + ashp_costs = self.costs.air_source_heat_pump(ashp_size) if non_intrusive_recommendation: # Update with non-intrusive recommendation if non_intrusive_recommendation.get("cost"): From 614f5f0c980c826a31f34a87e0e73841c270ea0d Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 8 Oct 2024 10:29:14 +0100 Subject: [PATCH 118/166] language updates to heat pump recommendations --- backend/app/assumptions.py | 2 ++ recommendations/HeatingRecommender.py | 22 ++++++++++++---------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/backend/app/assumptions.py b/backend/app/assumptions.py index 371b226d..5571e13b 100644 --- a/backend/app/assumptions.py +++ b/backend/app/assumptions.py @@ -10,6 +10,8 @@ SOLAR_CONSUMPTION_PROPORTION = 0.5 # Typically, each solar panel takes up around 3.4 m2 of roof space under RdSAP. This was been verified in Elmhurst RDSAP_AREA_PER_PANEL = 3.4 +SOCIAL_TENURES = ["Rented (social)", "rental (social)"] + DESCRIPTIONS_TO_FUEL_TYPES = { "Air source heat pump, radiators, electric": { "fuel": "Electricity", "cop": AVERAGE_ASHP_EFFICIENCY / 100 diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index bdc73e3b..48063a8e 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -1,4 +1,5 @@ import re +import backend.app.assumptions as assumptions from recommendations.Costs import Costs, BOILER_UPGRADE_SCHEME_ASHP_VALUE from recommendations.recommendation_utils import ( check_simulation_difference, override_costs, combine_recommendation_configs @@ -484,11 +485,13 @@ class HeatingRecommender: # This is a map from the heating controls description to the description of the air source heat pump set up ashp_descriptions = { "Time and temperature zone control": ( - "Install an air source heat pump, and upgrade heating controls to Smart Thermostats, " - "room sensors and smart radiator valves (time & temperature zone control)." + f"Install a {ashp_size}KW air source heat pump, and upgrade heating controls to Smart Thermostats, " + "room sensors and smart radiator valves (time & temperature zone control). Ensure you have an 18 or " + "24 hour tariff" ), "Programmer, TRVs and bypass": ( - "Install an air source heat pump, with programmer, TRVs and a Bypass valve." + f"Install a {ashp_size}KW air source heat pump, with programmer, TRVs and a Bypass valve. Ensure you " + "have an 18 or 24 hour tariff" ), } @@ -505,7 +508,7 @@ class HeatingRecommender: ashp_costs_with_controls[key] += controls_rec[key] if controls_rec is None: - description = "Install a Mitsubish air source heat pump." + description = f"Install a {ashp_size}KW Air source heat pump. Ensure you have an 18 or 24 hour tariff" elif already_installed: description = "The property already has an air source heat pump, no further action needed." else: @@ -514,17 +517,16 @@ class HeatingRecommender: # If the property does not have existing cavity and loft insulation, we include a note that the cost # includes the boiler upgrade scheme and that the cavity and loft need to be treated, to ensure access # to the funding - if not non_intrusive_recommendation: + if not non_intrusive_recommendation and self.property.data["tenure"] not in assumptions.SOCIAL_TENURES: if has_cavity_or_loft_recommendations: description = description + ( - f" The cost includes the £" - f"{BOILER_UPGRADE_SCHEME_ASHP_VALUE} boiler upgrade scheme grant. " - f"You must ensure that the property has an insulated cavity and " - f"270mm+ loft insulation to qualify for the grant" + f" You must ensure that the property has an insulated cavity and " + f"270mm+ loft insulation to qualify for the grant, to claim £" + f"{BOILER_UPGRADE_SCHEME_ASHP_VALUE} of funding from the boiler upgrade scheme grant. " ) else: description = description + ( - f" The cost includes the £{BOILER_UPGRADE_SCHEME_ASHP_VALUE} boiler upgrade scheme grant" + f" £{BOILER_UPGRADE_SCHEME_ASHP_VALUE} of funding can be claimed from the boiler upgrade scheme" ) simulation_config = { From 98a91a5a35944ab9bb1a29e2528dde80d33665f1 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 8 Oct 2024 11:14:13 +0100 Subject: [PATCH 119/166] adding simulation with default u-values to see impact on generate_scenarios_data --- etl/epc/generate_scenarios_data.py | 11 +++++++- recommendations/Recommendations.py | 5 +++- recommendations/WallRecommendations.py | 35 ++++++++++++++++++++------ 3 files changed, 41 insertions(+), 10 deletions(-) diff --git a/etl/epc/generate_scenarios_data.py b/etl/epc/generate_scenarios_data.py index 3497225c..15661d99 100644 --- a/etl/epc/generate_scenarios_data.py +++ b/etl/epc/generate_scenarios_data.py @@ -178,7 +178,7 @@ for scenario_property in scenario_properties: ) } - recommender = Recommendations(property_instance=p, materials=materials) + recommender = Recommendations(property_instance=p, materials=materials, default_u_values=True) property_recommendations = recommender.recommend() wall_recommendations = recommender.wall_recomender.recommendations @@ -286,6 +286,15 @@ model_api = ModelApi( all_predictions = model_api.predict_all(df=recommendations_scoring_data, bucket=get_settings().DATA_BUCKET) +sap_impact = pd.concat( + [ + all_predictions["sap_change_predictions"], + recommendations_scoring_data[["uprn", "sap_starting"]], + ], + axis=1 +) +sap_impact["predicted_impact"] = sap_impact["predictions"] - sap_impact["sap_starting"] + save_dataframe_to_s3_parquet( recommendations_scoring_data, "retrofit-data-dev", diff --git a/recommendations/Recommendations.py b/recommendations/Recommendations.py index 1b152238..ac7635c5 100644 --- a/recommendations/Recommendations.py +++ b/recommendations/Recommendations.py @@ -34,6 +34,7 @@ class Recommendations: materials: List, exclusions: List[str] = None, inclusions: List[str] = None, + default_u_values: bool = False, ): """ :param property_instance: Instance of the Property class, for the home associated to property_id @@ -42,12 +43,14 @@ class Recommendations: None, meaning no exclusions to be applied :param inclusions: List of specific measures of measure types to include. Defaulted to None, meaning all measures are included + :param default_u_values: Boolean, if True, the recommendations will use the default u-values for the property """ self.property_instance = property_instance self.materials = materials self.exclusions = exclusions if exclusions else [] self.inclusions = inclusions if inclusions else [] + self.default_u_values = default_u_values self.all_specific_measures = SPECIFIC_MEASURES self.all_non_invase_measures = NON_INVASIVE_SPECIFIC_MEASURES @@ -120,7 +123,7 @@ class Recommendations: non_invasive_recommendation_types = [r["type"] for r in self.property_instance.non_invasive_recommendations] # Building Fabric - self.wall_recomender.recommend(phase=phase, measures=measures) + self.wall_recomender.recommend(phase=phase, measures=measures, default_u_values=self.default_u_values) if self.wall_recomender.recommendations: property_recommendations.append(self.wall_recomender.recommendations) phase += 1 diff --git a/recommendations/WallRecommendations.py b/recommendations/WallRecommendations.py index dd5e861c..28e35584 100644 --- a/recommendations/WallRecommendations.py +++ b/recommendations/WallRecommendations.py @@ -179,7 +179,7 @@ class WallRecommendations(Definitions): return ewi_recommendations - def recommend(self, phase=0, measures=None): + def recommend(self, phase=0, measures=None, default_u_values=False): # if building built after 1990 + we're able to identify U-value + # U-value less than 0.18 and if in or close to a conversation area, # recommend internal wall insulation as a possible measure @@ -255,19 +255,19 @@ class WallRecommendations(Definitions): if (is_cavity_wall and "cavity_wall_insulation" in measures) or "cavity_extract_and_refill" in measures: if u_value >= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE: # Test filling cavity - self.find_cavity_insulation(u_value, insulation_thickness, phase, measures) + self.find_cavity_insulation(u_value, insulation_thickness, phase, measures, default_u_values) return # Remaining wall types are treated with IWI or EWI if (u_value >= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE) and self.is_suitable_for_solid_insulation(): - self.find_insulation(u_value, phase, measures=measures) + self.find_insulation(u_value, phase, measures=measures, default_u_values=default_u_values) return # If the u-value is within regulations, we don't do anything return - def find_cavity_insulation(self, u_value, insulation_thickness, phase, measures): + def find_cavity_insulation(self, u_value, insulation_thickness, phase, measures, default_u_values): """ This method tests different materials to fill the cavity wall, determining which material will give us the best U-value. @@ -289,6 +289,7 @@ class WallRecommendations(Definitions): filled cavity wall :param phase: The phase of the recommendation :param measures: The measures we're considering + :param default_u_values: If we should use default u values """ insulation_materials = pd.DataFrame(self.cavity_wall_insulation_materials) @@ -344,7 +345,15 @@ class WallRecommendations(Definitions): description = self._make_description(material) # updated the new u-value with the best possible our installers have - new_u_value = max(0.31, new_u_value) + if default_u_values: + new_u_value = get_wall_u_value( + clean_description="Cavity wall, filled cavity", + age_band="G", + is_granite_or_whinstone=self.property.walls["is_granite_or_whinstone"], + is_sandstone_or_limestone=self.property.walls["is_sandstone_or_limestone"], + ) + else: + new_u_value = max(0.31, new_u_value) wall_ending_config = WallAttributes("Cavity wall, filled cavity").process() @@ -359,7 +368,7 @@ class WallRecommendations(Definitions): simulation_config = { **simulation_config, **walls_simulation_config, - "walls_thermal_transmittance_ending": new_u_value, + "walls_thermal_transmittance_ending": new_u_value if not default_u_values else 0.7, } recommendations.append( @@ -439,7 +448,7 @@ class WallRecommendations(Definitions): return simulation_config - def _find_insulation(self, u_value, insulation_materials, phase): + def _find_insulation(self, u_value, insulation_materials, phase, default_u_values): lowest_selected_u_value = None recommendations = [] @@ -534,6 +543,14 @@ class WallRecommendations(Definitions): "walls_thermal_transmittance_ending": new_u_value } + if default_u_values: + new_u_value = get_wall_u_value( + clean_description=new_description, + age_band=self.property.age_band, + is_granite_or_whinstone=self.property.walls["is_granite_or_whinstone"], + is_sandstone_or_limestone=self.property.walls["is_sandstone_or_limestone"], + ) + recommendations.append( { "phase": phase, @@ -564,7 +581,7 @@ class WallRecommendations(Definitions): return recommendations - def find_insulation(self, u_value, phase, measures): + def find_insulation(self, u_value, phase, measures, default_u_values): """ This function contains the logic for finding potential insulation measures for a property, depending on the parts available and whether the property can have external wall insulation installed @@ -584,6 +601,7 @@ class WallRecommendations(Definitions): self.external_wall_insulation_materials ), phase=phase, + default_u_values=default_u_values ) iwi_recommendations = [] @@ -592,6 +610,7 @@ class WallRecommendations(Definitions): u_value=u_value, insulation_materials=pd.DataFrame(self.internal_wall_insulation_materials), phase=phase, + default_u_values=default_u_values ) self.recommendations += ewi_recommendations + iwi_recommendations From 35911c0db7b511969fd3691eb4fc44da54eef20e Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 8 Oct 2024 14:17:24 +0100 Subject: [PATCH 120/166] added test cases for roof u-values --- recommendations/Recommendations.py | 2 +- recommendations/RoofRecommendations.py | 40 +++++++++-- recommendations/WallRecommendations.py | 1 + recommendations/rdsap_tables.py | 9 +++ recommendations/recommendation_utils.py | 67 ++++++++++++++++--- .../tests/test_recommendation_utils.py | 21 ++++++ 6 files changed, 124 insertions(+), 16 deletions(-) diff --git a/recommendations/Recommendations.py b/recommendations/Recommendations.py index ac7635c5..684b0915 100644 --- a/recommendations/Recommendations.py +++ b/recommendations/Recommendations.py @@ -128,7 +128,7 @@ class Recommendations: property_recommendations.append(self.wall_recomender.recommendations) phase += 1 - self.roof_recommender.recommend(phase=phase, measures=measures) + self.roof_recommender.recommend(phase=phase, measures=measures, default_u_values=self.default_u_values) if self.roof_recommender.recommendations: property_recommendations.append(self.roof_recommender.recommendations) phase += 1 diff --git a/recommendations/RoofRecommendations.py b/recommendations/RoofRecommendations.py index 3e266cee..e8a43db0 100644 --- a/recommendations/RoofRecommendations.py +++ b/recommendations/RoofRecommendations.py @@ -108,7 +108,7 @@ class RoofRecommendations: return full_insulated_room_roof or room_roof_insulated_at_rafters - def recommend(self, phase, measures=None): + def recommend(self, phase, measures=None, default_u_values=False): if self.property.roof["has_dwelling_above"]: return @@ -171,7 +171,8 @@ class RoofRecommendations: insulation_thickness=self.insulation_thickness, phase=phase, is_flat=False, - is_pitched=True + is_pitched=True, + default_u_values=default_u_values ) return @@ -184,7 +185,8 @@ class RoofRecommendations: insulation_thickness=0, phase=phase, is_flat=True, - is_pitched=False + is_pitched=False, + default_u_values=default_u_values ) return @@ -193,7 +195,7 @@ class RoofRecommendations: if self.property.roof["is_roof_room"] and ("room_roof_insulation" in measures) or ( "room_roof_insulation" in [x["type"] for x in non_invasive_recommendations] ): - self.recommend_room_roof_insulation(u_value, phase) + self.recommend_room_roof_insulation(u_value, phase, default_u_values) return raise NotImplementedError("Implement me") @@ -215,7 +217,7 @@ class RoofRecommendations: raise ValueError("Invalid material type") def recommend_roof_insulation( - self, u_value, insulation_thickness, phase, is_pitched, is_flat + self, u_value, insulation_thickness, phase, is_pitched, is_flat, default_u_values ): """ @@ -241,6 +243,7 @@ class RoofRecommendations: :param phase: Phase of the recommendation :param is_pitched: Is the roof pitched :param is_flat: Is the roof flat + :param default_u_values: Use default u-values :return: """ @@ -266,7 +269,6 @@ class RoofRecommendations: recommendations = [] for _, insulation_material_group in insulation_materials.groupby("description"): for _, material in insulation_material_group.iterrows(): - # We make sure we hit a depth of 270mm. We should factor in any existing insulation if the # loft is already partially insulated. # Note: This requirement is only for loft insulation @@ -340,9 +342,35 @@ class RoofRecommendations: new_description = f"Pitched, {int(proposed_depth)}mm loft insulation" + if default_u_values: + # We update the u-value with the default if we're using default u-values + new_u_value = get_roof_u_value( + insulation_thickness=str(int(new_thickness)), + has_dwelling_above=self.property.roof["has_dwelling_above"], + is_loft=self.property.roof["is_loft"], + is_roof_room=self.property.roof["is_roof_room"], + is_thatched=self.property.roof["is_thatched"], + age_band=self.property.age_band, + is_flat=self.property.roof["is_flat"], + is_pitched=self.property.roof["is_pitched"], + is_at_rafters=self.property.roof["is_at_rafters"], + ) + elif material["type"] == "flat_roof_insulation": new_description = "Flat, insulated" new_efficiency = "Good" + if default_u_values: + new_u_value = get_roof_u_value( + insulation_thickness="100", + has_dwelling_above=self.property.roof["has_dwelling_above"], + is_loft=self.property.roof["is_loft"], + is_roof_room=self.property.roof["is_roof_room"], + is_thatched=self.property.roof["is_thatched"], + age_band=self.property.age_band, + is_flat=self.property.roof["is_flat"], + is_pitched=self.property.roof["is_pitched"], + is_at_rafters=self.property.roof["is_at_rafters"], + ) else: raise ValueError("Invalid material type") diff --git a/recommendations/WallRecommendations.py b/recommendations/WallRecommendations.py index 28e35584..358547dc 100644 --- a/recommendations/WallRecommendations.py +++ b/recommendations/WallRecommendations.py @@ -544,6 +544,7 @@ class WallRecommendations(Definitions): } if default_u_values: + # If we're using default U-values, we overwrite new_u_value new_u_value = get_wall_u_value( clean_description=new_description, age_band=self.property.age_band, diff --git a/recommendations/rdsap_tables.py b/recommendations/rdsap_tables.py index 5110764b..16c7d26e 100644 --- a/recommendations/rdsap_tables.py +++ b/recommendations/rdsap_tables.py @@ -340,6 +340,7 @@ s9_list = [ s10_list = [ { "Age_band": "A, B, C, D", + "Insulation_Thickness": "none", "Pitched_slates_or_tiles_insulation_between_joists_or_unknown": 2.3, "Pitched_slates_or_tiles_insulation_at_rafters": 2.3, "Flat_roof": 2.3, @@ -350,6 +351,7 @@ s10_list = [ }, { "Age_band": "E", + "Insulation_Thickness": 12, "Pitched_slates_or_tiles_insulation_between_joists_or_unknown": 1.5, "Pitched_slates_or_tiles_insulation_at_rafters": 1.5, "Flat_roof": 1.5, @@ -360,6 +362,7 @@ s10_list = [ }, { "Age_band": "F", + "Insulation_Thickness": 50, "Pitched_slates_or_tiles_insulation_between_joists_or_unknown": 0.68, "Pitched_slates_or_tiles_insulation_at_rafters": 0.68, "Flat_roof": 0.68, @@ -370,6 +373,7 @@ s10_list = [ }, { "Age_band": "G", + "Insulation_Thickness": 100, "Pitched_slates_or_tiles_insulation_between_joists_or_unknown": 0.40, "Pitched_slates_or_tiles_insulation_at_rafters": 0.40, "Flat_roof": 0.40, @@ -380,6 +384,7 @@ s10_list = [ }, { "Age_band": "H", + "Insulation_Thickness": 150, "Pitched_slates_or_tiles_insulation_between_joists_or_unknown": 0.30, "Pitched_slates_or_tiles_insulation_at_rafters": 0.35, "Flat_roof": 0.35, @@ -390,6 +395,7 @@ s10_list = [ }, { "Age_band": "I", + "Insulation_Thickness": 150, "Pitched_slates_or_tiles_insulation_between_joists_or_unknown": 0.26, "Pitched_slates_or_tiles_insulation_at_rafters": 0.35, "Flat_roof": 0.35, @@ -400,6 +406,7 @@ s10_list = [ }, { "Age_band": "J", + "Insulation_Thickness": 270, "Pitched_slates_or_tiles_insulation_between_joists_or_unknown": 0.16, "Pitched_slates_or_tiles_insulation_at_rafters": 0.20, "Flat_roof": 0.25, @@ -410,6 +417,7 @@ s10_list = [ }, { "Age_band": "K", + "Insulation_Thickness": 270, "Pitched_slates_or_tiles_insulation_between_joists_or_unknown": 0.16, "Pitched_slates_or_tiles_insulation_at_rafters": 0.20, "Flat_roof": 0.25, @@ -420,6 +428,7 @@ s10_list = [ }, { "Age_band": "L", + "Insulation_Thickness": 270, "Pitched_slates_or_tiles_insulation_between_joists_or_unknown": 0.16, "Pitched_slates_or_tiles_insulation_at_rafters": 0.18, "Flat_roof": 0.18, diff --git a/recommendations/recommendation_utils.py b/recommendations/recommendation_utils.py index dcdd9c06..8ddfe1ef 100644 --- a/recommendations/recommendation_utils.py +++ b/recommendations/recommendation_utils.py @@ -239,12 +239,7 @@ def get_wall_u_value( return float(mapped_value) -def get_u_value_from_s9( - thickness, s9, is_loft, is_roof_room, is_thatched, is_at_rafters -): - """Get the U-value from table S9 based on the insulation thickness.""" - - # If the roof as pitched & insulated at the rafters, it's a room roof +def extract_thickness(thickness, is_roof_room, is_at_rafters, is_loft, is_flat): if is_roof_room or is_at_rafters: # TODO: We get None instead of a string none, this should be fixed if thickness is None: @@ -258,19 +253,40 @@ def get_u_value_from_s9( } thickness = thickness_map[thickness] + if is_flat: + try: + thickness = int(thickness) + return thickness + except ValueError: + # If thickness is not a valid number (could be a string or None), return None + return None + if thickness in ["below average", "average", "above average", "none", None] or ( not is_loft and not is_roof_room and not is_at_rafters ): return None elif thickness.endswith("+"): thickness = int(thickness[:-1]) + return thickness else: try: thickness = int(thickness) + return thickness except ValueError: # If thickness is not a valid number (could be a string or None), return None return None + +def get_u_value_from_s9( + thickness, s9, is_loft, is_roof_room, is_thatched, is_at_rafters +): + """Get the U-value from table S9 based on the insulation thickness.""" + + if thickness in ["below average", "average", "above average", "none", None, "0", 0] or ( + not is_loft and not is_roof_room and not is_at_rafters + ): + return None + # Determine the column to refer based on the roof type column = ( "Thatched_roof_U_value_W_m2K" @@ -323,6 +339,14 @@ def get_roof_u_value( if has_dwelling_above: return 0.0 + thickness = extract_thickness( + thickness=insulation_thickness, + is_roof_room=is_roof_room, + is_at_rafters=is_at_rafters, + is_loft=is_loft, + is_flat=is_flat + ) + # Step 1: Try to get the U-value from table S9 based on the insulation thickness # The conditions for using table S9 are: # - The insulation thickness is known @@ -330,7 +354,7 @@ def get_roof_u_value( # The criteria for using this table is predominately defined by insulation around joists which is predominately # a feature of lofts and roof rooms u_value = get_u_value_from_s9( - thickness=insulation_thickness, + thickness=thickness, s9=s9, is_loft=is_loft, is_roof_room=is_roof_room, @@ -363,9 +387,34 @@ def get_roof_u_value( column = "Pitched_slates_or_tiles_insulation_between_joists_or_unknown" # Get the U-value from table S10 based on the age band and the determined column - u_value = s10.loc[s10["Age_band"].str.contains(age_band), column].values[0] + if is_flat and thickness is not None: + u_value = s10.loc[ + (s10["Insulation_Thickness"] == thickness) | + s10["Age_band"].str.contains(age_band), + column + ].values.min() + else: + u_value = s10.loc[s10["Age_band"].str.contains(age_band), column].values[0] - return float(u_value) + u_value = float(u_value) + + # As per the documentation here: https://bregroup.com/documents/d/bre-group/rdsap_2012_9-94-20-09-2019 + # Table s.10 + # "The value from the table applies for unknown and as built. If the roof is known to have more insulation than + # would normally be expected for the age band, either observed or on the basis of documentary evidence, use the + # lower of the value in the table and: + # 50 mm insulation 0.68 + # 100 mm insulation: 0.40 + # 150 mm or more insulation: 0.30" + if thickness is not None: + if thickness == 50: + u_value = min(u_value, 0.68) + if thickness == 100: + u_value = min(u_value, 0.40) + if thickness >= 150: + u_value = min(u_value, 0.30) + + return u_value def estimate_number_of_floors(property_type): diff --git a/recommendations/tests/test_recommendation_utils.py b/recommendations/tests/test_recommendation_utils.py index 24ea6482..b445a798 100644 --- a/recommendations/tests/test_recommendation_utils.py +++ b/recommendations/tests/test_recommendation_utils.py @@ -6,6 +6,7 @@ from recommendations import recommendation_utils from datatypes.enums import QuantityUnits from recommendations.tests.test_data.wall_uvalue_test_cases import wall_uvalue_test_cases from recommendations.tests.test_data.floor_uvalue_test_cases import floor_uvalue_test_cases +from recommendations.tests.test_data.roof_uvalue_test_cases import roof_uvalue_test_cases class TestRecommendationUtils: @@ -222,6 +223,26 @@ class TestRecommendationUtils: u_value = recommendation_utils.get_roof_u_value(**inputs) assert u_value == 0.0, f"Expected 0.0, but got {u_value}" + @pytest.mark.parametrize( + "test_case", + roof_uvalue_test_cases + ) + def test_roof_uvalues(self, test_case): + expected_uvalue = test_case["uvalue"] + inputs = test_case.copy() + del inputs["uvalue"] + # insulation_thickness = inputs["insulation_thickness"] + # has_dwelling_above = inputs["has_dwelling_above"] + # is_loft = inputs["is_loft"] + # is_roof_room = inputs["is_roof_room"] + # is_thatched = inputs["is_thatched"] + # age_band = inputs["age_band"] + # is_flat = inputs["is_flat"] + # is_pitched = inputs["is_pitched"] + # is_at_rafters = inputs["is_at_rafters"] + uvalue = recommendation_utils.get_roof_u_value(**inputs) + assert expected_uvalue == uvalue, f"Expected u value {expected_uvalue}, recieved {uvalue}" + @pytest.mark.parametrize( "test_case", wall_uvalue_test_cases From e1593445ac8065cb82e1fbb9daa2deb18807ef98 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 8 Oct 2024 14:18:45 +0100 Subject: [PATCH 121/166] removing temp code --- recommendations/tests/test_recommendation_utils.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/recommendations/tests/test_recommendation_utils.py b/recommendations/tests/test_recommendation_utils.py index b445a798..38322c41 100644 --- a/recommendations/tests/test_recommendation_utils.py +++ b/recommendations/tests/test_recommendation_utils.py @@ -231,15 +231,6 @@ class TestRecommendationUtils: expected_uvalue = test_case["uvalue"] inputs = test_case.copy() del inputs["uvalue"] - # insulation_thickness = inputs["insulation_thickness"] - # has_dwelling_above = inputs["has_dwelling_above"] - # is_loft = inputs["is_loft"] - # is_roof_room = inputs["is_roof_room"] - # is_thatched = inputs["is_thatched"] - # age_band = inputs["age_band"] - # is_flat = inputs["is_flat"] - # is_pitched = inputs["is_pitched"] - # is_at_rafters = inputs["is_at_rafters"] uvalue = recommendation_utils.get_roof_u_value(**inputs) assert expected_uvalue == uvalue, f"Expected u value {expected_uvalue}, recieved {uvalue}" From ab05082829bd9e92b852d30752840a71c23ab1e2 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 8 Oct 2024 15:07:04 +0100 Subject: [PATCH 122/166] updated scenario data --- etl/epc/generate_scenarios_data.py | 18 +++++++++--------- recommendations/RoofRecommendations.py | 6 ++++-- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/etl/epc/generate_scenarios_data.py b/etl/epc/generate_scenarios_data.py index 15661d99..d4833baa 100644 --- a/etl/epc/generate_scenarios_data.py +++ b/etl/epc/generate_scenarios_data.py @@ -79,8 +79,8 @@ scenario_properties = [ "measures": [ [ ["cavity_wall_insulation", "loft_insulation"], - "15", - {"walls_insulation_thickness_ending": "average"}, + "11", + {}, [0, 1], ], ], @@ -92,8 +92,8 @@ scenario_properties = [ "measures": [ [ ["cavity_wall_insulation", "loft_insulation"], - "15", - {"walls_insulation_thickness_ending": "average"}, + "10", + {}, [0, 1], ], ], @@ -105,8 +105,8 @@ scenario_properties = [ "measures": [ [ ["cavity_wall_insulation", "loft_insulation"], - "15", - {"walls_insulation_thickness_ending": "average"}, + "11", + {}, [0, 1], ], ], @@ -118,8 +118,8 @@ scenario_properties = [ "measures": [ [ ["cavity_wall_insulation", "loft_insulation"], - "15", - {"walls_insulation_thickness_ending": "average"}, + "10", + {}, [0, 1], ], ], @@ -289,7 +289,7 @@ all_predictions = model_api.predict_all(df=recommendations_scoring_data, bucket= sap_impact = pd.concat( [ all_predictions["sap_change_predictions"], - recommendations_scoring_data[["uprn", "sap_starting"]], + recommendations_scoring_data[["uprn", "sap_starting", "impact"]], ], axis=1 ) diff --git a/recommendations/RoofRecommendations.py b/recommendations/RoofRecommendations.py index e8a43db0..c86c5b30 100644 --- a/recommendations/RoofRecommendations.py +++ b/recommendations/RoofRecommendations.py @@ -307,7 +307,9 @@ class RoofRecommendations: cost_result = override_costs(cost_result) if material["type"] == "loft_insulation": - new_thickness = insulation_thickness + material["depth"] + # We take the new thickness as just the thickness of the insulation, to be conservative + # and assume that any existing insulation will be replaced + new_thickness = material["depth"] # This is based on the values we have in the training data valid_numeric_values = [ @@ -332,7 +334,7 @@ class RoofRecommendations: valid_numeric_values, key=lambda x: abs(x - proposed_depth) ) - if proposed_depth >= 270: + if proposed_depth >= 300: new_efficiency = "Very Good" else: if self.property.data["walls-energy-eff"] not in ["Good", "Very Good"]: From 87879d54336316b7876017c16e1a7476e255a5b8 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 8 Oct 2024 15:26:15 +0100 Subject: [PATCH 123/166] minor --- etl/epc/generate_scenarios_data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etl/epc/generate_scenarios_data.py b/etl/epc/generate_scenarios_data.py index d4833baa..b61618b1 100644 --- a/etl/epc/generate_scenarios_data.py +++ b/etl/epc/generate_scenarios_data.py @@ -60,13 +60,13 @@ scenario_properties = [ [ ["internal_wall_insulation"], "11", - {"walls_insulation_thickness_ending": "average"}, + {}, [0], ], [ ["external_wall_insulation"], "10", - {"walls_insulation_thickness_ending": "average"}, + {}, [0], ], [["solar", "windows"], "15", {"photo_supply_ending": 50}, [0, 1]], From d9a82984f8f76968d8498429c1aba980a2a88bab Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 8 Oct 2024 15:55:28 +0100 Subject: [PATCH 124/166] added default u-value option to room roofs --- recommendations/RoofRecommendations.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/recommendations/RoofRecommendations.py b/recommendations/RoofRecommendations.py index c86c5b30..03136547 100644 --- a/recommendations/RoofRecommendations.py +++ b/recommendations/RoofRecommendations.py @@ -416,7 +416,7 @@ class RoofRecommendations: self.recommendations = recommendations - def recommend_room_roof_insulation(self, u_value, phase): + def recommend_room_roof_insulation(self, u_value, phase, default_u_values): """ This method recommends room in roof insulation for properties that have been identified to possess a room in roof. @@ -455,6 +455,8 @@ class RoofRecommendations: - Flat ceilings can be insulated like a standard loft. :param u_value: Current u-value of the roof + :param phase: Phase of the recommendation + :param default_u_values: Use default u-values :return: """ @@ -495,7 +497,7 @@ class RoofRecommendations: sap_points = rir_non_invasive_recommendation.get("sap_points", None) # Could also be Roof room(s), ceiling insulated - new_descriptin = "Pitched, insulated at rafters" + new_descriptin = "Roof room(s), insulated" roof_ending_config = RoofAttributes(new_descriptin).process() roof_simulation_config = check_simulation_difference( new_config=roof_ending_config, old_config=self.property.roof, prefix="roof_" @@ -505,6 +507,19 @@ class RoofRecommendations: else: new_efficiency = self.property.data["roof-energy-eff"] + if default_u_values: + new_u_value = get_roof_u_value( + insulation_thickness="average", + has_dwelling_above=self.property.roof["has_dwelling_above"], + is_loft=self.property.roof["is_loft"], + is_roof_room=self.property.roof["is_roof_room"], + is_thatched=self.property.roof["is_thatched"], + age_band=self.property.age_band, + is_flat=self.property.roof["is_flat"], + is_pitched=self.property.roof["is_pitched"], + is_at_rafters=self.property.roof["is_at_rafters"], + ) + simulation_config = { **roof_simulation_config, "roof_thermal_transmittance_ending": new_u_value, From 93a21e5436665a5321730978f563174135b226e7 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 8 Oct 2024 16:08:50 +0100 Subject: [PATCH 125/166] updating recommendation utils tests --- recommendations/recommendation_utils.py | 12 +++++++++--- recommendations/tests/test_recommendation_utils.py | 9 +++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/recommendations/recommendation_utils.py b/recommendations/recommendation_utils.py index 8ddfe1ef..3364f0fa 100644 --- a/recommendations/recommendation_utils.py +++ b/recommendations/recommendation_utils.py @@ -282,11 +282,14 @@ def get_u_value_from_s9( ): """Get the U-value from table S9 based on the insulation thickness.""" - if thickness in ["below average", "average", "above average", "none", None, "0", 0] or ( + if thickness in ["below average", "average", "above average", "none", None] or ( not is_loft and not is_roof_room and not is_at_rafters ): return None + if thickness in [0, "0"] and is_loft: + return None + # Determine the column to refer based on the roof type column = ( "Thatched_roof_U_value_W_m2K" @@ -294,8 +297,11 @@ def get_u_value_from_s9( else "Slates_or_tiles_U_value_W_m2K" ) - # Get the correct U-value based on the insulation thickness - return s9[s9["Insulation_thickness_mm"] >= thickness][column].iloc[0] + if thickness in [0, "0"] and is_roof_room: + return s9[pd.isnull(s9["Insulation_thickness_mm"])][column].iloc[0] + else: + # Get the correct U-value based on the insulation thickness + return s9[s9["Insulation_thickness_mm"] >= thickness][column].iloc[0] def get_roof_u_value( diff --git a/recommendations/tests/test_recommendation_utils.py b/recommendations/tests/test_recommendation_utils.py index 38322c41..b445a798 100644 --- a/recommendations/tests/test_recommendation_utils.py +++ b/recommendations/tests/test_recommendation_utils.py @@ -231,6 +231,15 @@ class TestRecommendationUtils: expected_uvalue = test_case["uvalue"] inputs = test_case.copy() del inputs["uvalue"] + # insulation_thickness = inputs["insulation_thickness"] + # has_dwelling_above = inputs["has_dwelling_above"] + # is_loft = inputs["is_loft"] + # is_roof_room = inputs["is_roof_room"] + # is_thatched = inputs["is_thatched"] + # age_band = inputs["age_band"] + # is_flat = inputs["is_flat"] + # is_pitched = inputs["is_pitched"] + # is_at_rafters = inputs["is_at_rafters"] uvalue = recommendation_utils.get_roof_u_value(**inputs) assert expected_uvalue == uvalue, f"Expected u value {expected_uvalue}, recieved {uvalue}" From 08b8124d0c3135e8d3db0833e55a1dcaec8dd9a6 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 8 Oct 2024 16:40:02 +0100 Subject: [PATCH 126/166] fixed broken tests --- recommendations/RoofRecommendations.py | 2 +- .../test_data/heating_recommendations_data.py | 318 +++++++----------- .../tests/test_heating_recommendations.py | 14 +- .../tests/test_roof_recommendations.py | 12 +- 4 files changed, 129 insertions(+), 217 deletions(-) diff --git a/recommendations/RoofRecommendations.py b/recommendations/RoofRecommendations.py index 03136547..52313121 100644 --- a/recommendations/RoofRecommendations.py +++ b/recommendations/RoofRecommendations.py @@ -337,7 +337,7 @@ class RoofRecommendations: if proposed_depth >= 300: new_efficiency = "Very Good" else: - if self.property.data["walls-energy-eff"] not in ["Good", "Very Good"]: + if self.property.data["roof-energy-eff"] not in ["Good", "Very Good"]: new_efficiency = "Good" else: new_efficiency = "Very Good" diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index 8697e095..51bf0378 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -39,15 +39,8 @@ testing_examples = [ 'fixed-lighting-outlets-count': 10.0, 'low-energy-fixed-light-count': 7.0, 'uprn': 100110195416.0, 'uprn-source': 'Address Matched' }, - "heating_recommendation_descriptions": [ - "Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and " - "smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade " - "scheme grant", - ], - "heating_controls_recommendation_descriptions": [ - "Upgrade heating controls to Smart Thermostats, room sensors and smart radiator valves (time & " - "temperature zone control)" - ], + "heating_measure_types": ["air_source_heat_pump"], + "heating_controls_measure_types": ["time_temperature_zone_control"], "notes": "This property has a boiler, radiators & mains gas with good efficiency so the only recommendation" "we expect here is for an air source heat pump. The heating controls are a programmer, room thermostat" "and TRVs and so we should expect a TTZC recommendation" @@ -93,11 +86,10 @@ testing_examples = [ 'fixed-lighting-outlets-count': None, 'low-energy-fixed-light-count': None, 'uprn': 10012342725.0, 'uprn-source': 'Address Matched', }, - "heating_recommendation_descriptions": [ - "Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention " - "Storage Heater Controls" + "heating_measure_types": [ + "high_heat_retention_storage_heater", ], - "heating_controls_recommendation_descriptions": [], + "heating_controls_measure_types": [], "notes": "This property has electric room heaters and is off gas so a boiler recommendation is not appropriate." "We would expect a high heat retention storage recommendation. The property is a flat and therefore" "we don't expect an air source heat pump recommendation. We also wouldn't expect a specific heating" @@ -144,16 +136,8 @@ testing_examples = [ 'tenure': 'owner-occupied', 'fixed-lighting-outlets-count': 6.0, 'low-energy-fixed-light-count': 4.0, 'uprn': 100090311351.0, 'uprn-source': 'Address Matched', 'property-type_y': None, 'built-form_y': None, }, - "heating_recommendation_descriptions": [ - 'Install high heat retention electric storage heaters. The current electric heaters may be retrofit with ' - 'high heat retention storage controls however this is dependent on the existing system and may not be ' - 'possible. Upgrade heating controls to High Heat Retention Storage Heater Controls', - 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' - 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' - 'scheme grant' - - ], - "heating_controls_recommendation_descriptions": [], + "heating_measure_types": ['high_heat_retention_storage_heater', 'air_source_heat_pump'], + "heating_controls_measure_types": [], "notes": "This test has electric storage heaters with automatic charge control - we recommend hhr storage" "heaters in this case, but because there are already electic storage heaters in place, we " "note, in the description of the recommendation, that this upgrade may be possible by retrofitting" @@ -197,13 +181,10 @@ testing_examples = [ 'tenure': 'owner-occupied', 'fixed-lighting-outlets-count': None, 'low-energy-fixed-light-count': None, 'uprn': 100021560521.0, 'uprn-source': 'Address Matched', }, - "heating_recommendation_descriptions": [ - 'Upgrade to a new condensing boiler.' - ], - "heating_controls_recommendation_descriptions": [ - 'Upgrade heating controls to Room thermostat, programmer and TRVs', - 'Upgrade heating controls to Smart Thermostats, room sensors and smart radiator valves (time & ' - 'temperature zone control)' + "heating_measure_types": ['boiler_upgrade'], + "heating_controls_measure_types": [ + 'roomstat_programmer_trvs', + 'time_temperature_zone_control', ], "notes": "Because of this property is a maisonette, which already has a boiler (but an inefficient one due to " "the current water heating efficiency) the only recommendation we expect is for " @@ -250,12 +231,10 @@ testing_examples = [ 'tenure': 'owner-occupied', 'fixed-lighting-outlets-count': 9.0, 'low-energy-fixed-light-count': 5.0, 'uprn': 100021936225.0, 'uprn-source': 'Address Matched', }, - "heating_recommendation_descriptions": [ - ], - "heating_controls_recommendation_descriptions": [ - 'Upgrade heating controls to Room thermostat, programmer and TRVs', - 'Upgrade heating controls to Smart Thermostats, room sensors and smart radiator valves (time & ' - 'temperature zone control)' + "heating_measure_types": [], + "heating_controls_measure_types": [ + 'roomstat_programmer_trvs', + 'time_temperature_zone_control', ], "notes": "Because this property already has a boiler, we don't recommend HHR. We don't recommend an ashp " "because the home is mid-terraced. Because the heating controls are " @@ -302,11 +281,10 @@ testing_examples = [ 'tenure': 'rental (private)', 'fixed-lighting-outlets-count': 7.0, 'low-energy-fixed-light-count': 6.0, 'uprn': 43088770.0, 'uprn-source': 'Address Matched', }, - "heating_recommendation_descriptions": [ - 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' - 'Storage Heater Controls' + "heating_measure_types": [ + 'high_heat_retention_storage_heater', ], - "heating_controls_recommendation_descriptions": [], + "heating_controls_measure_types": [], "notes": "This property is a flat so we don't have an ASHP recommendation. It also doesn't have access to the " "mains and so it can't have a gas boiler. We don't expect any controls recommendations" }, @@ -348,16 +326,10 @@ testing_examples = [ 'tenure': 'owner-occupied', 'fixed-lighting-outlets-count': None, 'low-energy-fixed-light-count': None, 'uprn': 100080513604.0, 'uprn-source': 'Address Matched' }, - "heating_recommendation_descriptions": [ - 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' - 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' - 'scheme grant' - ], - "heating_controls_recommendation_descriptions": [ - 'Upgrade heating controls to Room thermostat, programmer and TRVs', - 'Upgrade heating controls to Smart Thermostats, room sensors and smart radiator valves (time & ' - 'temperature zone control)' - + "heating_measure_types": ['air_source_heat_pump'], + "heating_controls_measure_types": [ + 'roomstat_programmer_trvs', + 'time_temperature_zone_control', ], "notes": "This has a very efficient boiler and is a detached bungalow, but only has " "Programmer and room thermostat for heating controls so we'd expect an ASHP heating recommendation" @@ -404,14 +376,12 @@ testing_examples = [ 'fixed-lighting-outlets-count': 10.0, 'low-energy-fixed-light-count': 9.0, 'uprn': 100070358594, 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None }, - "heating_recommendation_descriptions": [ - 'Upgrade to a new condensing boiler. Upgrade heating controls to Room thermostat, programmer and TRVs', - 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' - 'Storage Heater Controls', - 'Upgrade to a new condensing boiler. Upgrade heating controls to Smart Thermostats, room sensors and smart ' - 'radiator valves (time & temperature zone control)' + "heating_measure_types": [ + 'boiler_upgrade', + 'high_heat_retention_storage_heater', + 'boiler_upgrade' ], - "heating_controls_recommendation_descriptions": [], + "heating_controls_measure_types": [], "notes": "This property has assumed electric heating and is mid-terrace house. It has a mains gas connection." "We can recommend a boiler upgrade and high heat retention storage heaters" }, @@ -453,15 +423,11 @@ testing_examples = [ 'fixed-lighting-outlets-count': 42.0, 'low-energy-fixed-light-count': 13.0, 'uprn': 100070985545, 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None }, - "heating_recommendation_descriptions": [ - 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' - 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' - 'scheme grant', - 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' - 'Storage Heater Controls' - + "heating_measure_types": [ + 'air_source_heat_pump', + 'high_heat_retention_storage_heater', ], - "heating_controls_recommendation_descriptions": [], + "heating_controls_measure_types": [], "notes": "This property has an oil boiler and doesn't have a mains gas connection so we can only recommend" "an air source heat pump and HHR (since if the home has a non-gas boiler, we recommend HHR)" }, @@ -505,17 +471,13 @@ testing_examples = [ 'fixed-lighting-outlets-count': None, 'low-energy-fixed-light-count': None, 'uprn': 2465031849, 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None }, - "heating_recommendation_descriptions": [ - 'Upgrade to a new condensing boiler. Upgrade heating controls to Room thermostat, programmer and TRVs', - 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' - 'Storage Heater Controls', - 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' - 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' - 'scheme grant', - 'Upgrade to a new condensing boiler. Upgrade heating controls to Smart Thermostats, room sensors and ' - 'smart radiator valves (time & temperature zone control)' + "heating_measure_types": [ + 'boiler_upgrade', + 'high_heat_retention_storage_heater', + 'air_source_heat_pump', + 'boiler_upgrade' # TTZs ], - "heating_controls_recommendation_descriptions": [], + "heating_controls_measure_types": [], "notes": "This property has room heaters, from the mains gas supply. We recommend a boiler upgrade as" "well as an air source heat pump and HHR (since the home has a room heater set up)" }, @@ -558,14 +520,12 @@ testing_examples = [ 'uprn-source': 'Energy Assessor', 'sheating-energy-eff': None, 'sheating-env-eff': None }, - "heating_recommendation_descriptions": [ - 'Upgrade to a new condensing boiler. Upgrade heating controls to Room thermostat, programmer and TRVs', - 'Upgrade to a new condensing boiler. Upgrade heating controls to Smart Thermostats, room sensors and smart ' - 'radiator valves (time & temperature zone control)', - 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' - 'Storage Heater Controls', + "heating_measure_types": [ + 'boiler_upgrade', + 'boiler_upgrade', + 'high_heat_retention_storage_heater', ], - "heating_controls_recommendation_descriptions": [], + "heating_controls_measure_types": [], "notes": "This property has assumed electric heaters. Boiler upgrade, HHR are recommended. We don't recommend" "an ASHP off of the bat because it's mid-terrace." }, @@ -607,14 +567,12 @@ testing_examples = [ 'fixed-lighting-outlets-count': 10.0, 'low-energy-fixed-light-count': None, 'uprn': 100071089116, 'uprn-source': 'Energy Assessor', 'sheating-energy-eff': None, 'sheating-env-eff': None }, - "heating_recommendation_descriptions": [ - 'Upgrade to a new condensing boiler. Upgrade heating controls to Room thermostat, programmer and TRVs', - 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' - 'Storage Heater Controls', - 'Upgrade to a new condensing boiler. Upgrade heating controls to Smart Thermostats, room sensors and ' - 'smart radiator valves (time & temperature zone control)' + "heating_measure_types": [ + 'boiler_upgrade', + 'high_heat_retention_storage_heater', + 'boiler_upgrade' ], - "heating_controls_recommendation_descriptions": [], + "heating_controls_measure_types": [], "notes": "This has a form of assumed electric heating and has a mains connection so we recommend HHR, boiler" "upgrade and ASHP" }, @@ -657,15 +615,12 @@ testing_examples = [ 'fixed-lighting-outlets-count': None, 'low-energy-fixed-light-count': None, 'uprn': 100030352255, 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None }, - "heating_recommendation_descriptions": [ - 'Upgrade to a new condensing boiler. Upgrade heating controls to Room thermostat, programmer and TRVs', - 'Upgrade to a new condensing boiler. Upgrade heating controls to Smart Thermostats, room sensors and smart ' - 'radiator valves (time & temperature zone control)', - 'Install high heat retention electric storage heaters. The current electric heaters may be retrofit with ' - 'high heat retention storage controls however this is dependent on the existing system and may not be ' - 'possible. Upgrade heating controls to High Heat Retention Storage Heater Controls' + "heating_measure_types": [ + 'boiler_upgrade', + 'boiler_upgrade', + 'high_heat_retention_storage_heater', ], - "heating_controls_recommendation_descriptions": [], + "heating_controls_measure_types": [], "notes": "This property already has storage heaters with manual charge control. The home is mid terrace so" "the ashp is not suitable" }, @@ -709,14 +664,11 @@ testing_examples = [ 'fixed-lighting-outlets-count': None, 'low-energy-fixed-light-count': None, 'uprn': 10009573249, 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None }, - "heating_recommendation_descriptions": [ - 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' - 'Storage Heater Controls', - 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' - 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' - 'scheme grant' + "heating_measure_types": [ + 'high_heat_retention_storage_heater', + 'air_source_heat_pump', ], - "heating_controls_recommendation_descriptions": [], + "heating_controls_measure_types": [], "notes": "This property has an LFG boiler but it doesn't have a mains gas connection so we can only recommend" "an air source heat pump and hhr storage" }, @@ -758,14 +710,11 @@ testing_examples = [ 'fixed-lighting-outlets-count': 8.0, 'low-energy-fixed-light-count': None, 'uprn': 10013181470, 'uprn-source': 'Energy Assessor', 'sheating-energy-eff': None, 'sheating-env-eff': None }, - "heating_recommendation_descriptions": [ - 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' - 'Storage Heater Controls', - 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' - 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' - 'scheme grant' + "heating_measure_types": [ + 'high_heat_retention_storage_heater', + 'air_source_heat_pump', ], - "heating_controls_recommendation_descriptions": [], + "heating_controls_measure_types": [], "notes": "This property has electric boilers in place, but does not have a mains connection so we don't " "recommend a boiler upgrade. We recommend HHR and ASHP" }, @@ -809,14 +758,11 @@ testing_examples = [ 'fixed-lighting-outlets-count': 11.0, 'low-energy-fixed-light-count': 7.0, 'uprn': 452047507, 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None }, - "heating_recommendation_descriptions": [ - 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' - 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' - 'scheme grant', - 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' - 'Storage Heater Controls' + "heating_measure_types": [ + 'air_source_heat_pump', + 'high_heat_retention_storage_heater' ], - "heating_controls_recommendation_descriptions": [], + "heating_controls_measure_types": [], "notes": "This property has a dual fuel boiler and no mains gas connection. We recommend ASHP and HHR, but" "no gas condensing boiler" }, @@ -857,14 +803,11 @@ testing_examples = [ 'fixed-lighting-outlets-count': 16.0, 'low-energy-fixed-light-count': 4.0, 'uprn': 100030309413, 'uprn-source': 'Energy Assessor', 'sheating-energy-eff': None, 'sheating-env-eff': None }, - "heating_recommendation_descriptions": [ - 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' - 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' - 'scheme grant', - 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' - 'Storage Heater Controls' + "heating_measure_types": [ + 'air_source_heat_pump', + 'high_heat_retention_storage_heater', ], - "heating_controls_recommendation_descriptions": [], + "heating_controls_measure_types": [], "notes": "This property has a coal boiler and no mains gas connection. We recommend ASHP and HHR, but" "no gas condensing boiler" }, @@ -908,15 +851,11 @@ testing_examples = [ 'fixed-lighting-outlets-count': None, 'low-energy-fixed-light-count': None, 'uprn': 10007366417, 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None }, - "heating_recommendation_descriptions": [ - 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' - 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' - 'scheme grant', - 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' - 'Storage Heater Controls' - + "heating_measure_types": [ + 'air_source_heat_pump', + 'high_heat_retention_storage_heater', ], - "heating_controls_recommendation_descriptions": [], + "heating_controls_measure_types": [], "notes": "This property has a smokeless fuel boiler and no mains gas connection. We recommend ASHP and HHR, but" "no gas condensing boiler" }, @@ -958,15 +897,11 @@ testing_examples = [ 'fixed-lighting-outlets-count': None, 'low-energy-fixed-light-count': None, 'uprn': 100030256931, 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None }, - "heating_recommendation_descriptions": [ - 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' - 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' - 'scheme grant', - 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' - 'Storage Heater Controls' - + "heating_measure_types": [ + 'air_source_heat_pump', + 'high_heat_retention_storage_heater', ], - "heating_controls_recommendation_descriptions": [], + "heating_controls_measure_types": [], "notes": "This property has a wood pellets boiler and no mains gas connection. We recommend ASHP and HHR, but" "no gas condensing boiler" }, @@ -1009,11 +944,10 @@ testing_examples = [ 'fixed-lighting-outlets-count': None, 'low-energy-fixed-light-count': None, 'uprn': 10000460605, 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None }, - "heating_recommendation_descriptions": [ - 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' - 'Storage Heater Controls' + "heating_measure_types": [ + 'high_heat_retention_storage_heater', ], - "heating_controls_recommendation_descriptions": [], + "heating_controls_measure_types": [], "notes": "This is an end-terrace house, without mains gas connection, so all we recommend is HHR" }, { @@ -1054,8 +988,8 @@ testing_examples = [ 'fixed-lighting-outlets-count': None, 'low-energy-fixed-light-count': None, 'uprn': 100031045596, 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None }, - "heating_recommendation_descriptions": [], - "heating_controls_recommendation_descriptions": [], + "heating_measure_types": [], + "heating_controls_measure_types": [], "notes": "This property already has an ashp. We don't recommend any heating upgrades" }, { @@ -1095,17 +1029,12 @@ testing_examples = [ 'tenure': 'Rented (social)', 'fixed-lighting-outlets-count': 11.0, 'low-energy-fixed-light-count': None, 'uprn': 90041166, 'uprn-source': 'Energy Assessor', 'sheating-energy-eff': None, 'sheating-env-eff': None }, - "heating_recommendation_descriptions": [ - 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' - 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' - 'scheme grant', - 'Install high heat retention electric storage heaters alongside the boiler. The current electric heaters ' - 'may be retrofit with high heat retention storage controls however this is dependent on the existing ' - 'system and may not be possible. Upgrade heating controls to High Heat Retention Storage Heater Controls' + "heating_measure_types": [ + 'air_source_heat_pump', + 'high_heat_retention_storage_heater' ], - "heating_controls_recommendation_descriptions": [ - 'Upgrade heating controls to Smart Thermostats, room sensors and smart radiator valves (time & ' - 'temperature zone control)' + "heating_controls_measure_types": [ + 'time_temperature_zone_control', ], "notes": "This property has dual heating. A boiler and electric storage heaters. The heating is efficient so" "we recommend ASHP and HHR. We also recommend upgrading the heating controls for the boiler" @@ -1147,20 +1076,14 @@ testing_examples = [ 'tenure': 'Rented (social)', 'fixed-lighting-outlets-count': 11.0, 'low-energy-fixed-light-count': None, 'uprn': 90041166, 'uprn-source': 'Energy Assessor', 'sheating-energy-eff': None, 'sheating-env-eff': None }, - "heating_recommendation_descriptions": [ - 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' - 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' - 'scheme grant', - 'Upgrade the existing boiler to a new, more efficient condensing boiler. ', - 'Upgrade both the existing boiler to a new condensing boiler and upgrade storage heaters to high heat ' - 'retention storage heaters.', - 'Install high heat retention electric storage heaters alongside the boiler. The current electric heaters ' - 'may be retrofit with high heat retention storage controls however this is dependent on the existing ' - 'system and may not be possible. Upgrade heating controls to High Heat Retention Storage Heater Controls' + "heating_measure_types": [ + 'air_source_heat_pump', + 'boiler_upgrade', + 'boiler_upgrade+high_heat_retention_storage_heater', + 'high_heat_retention_storage_heater' ], - "heating_controls_recommendation_descriptions": [ - 'Upgrade heating controls to Smart Thermostats, room sensors and smart radiator valves (time & ' - 'temperature zone control)' + "heating_controls_measure_types": [ + 'time_temperature_zone_control' ], "notes": "This property is a modified version of the previous dual heating property, where we lower the" "starting heating efficiency so that we a combined heating upgrade to both the boiler and the electric" @@ -1204,14 +1127,11 @@ testing_examples = [ 'tenure': 'rental (social)', 'fixed-lighting-outlets-count': None, 'low-energy-fixed-light-count': None, 'uprn': 10009574286, 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None }, - "heating_recommendation_descriptions": [ - 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' - 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' - 'scheme grant', - 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' - 'Storage Heater Controls' + "heating_measure_types": [ + 'air_source_heat_pump', + 'high_heat_retention_storage_heater' ], - "heating_controls_recommendation_descriptions": [], + "heating_controls_measure_types": [], "notes": "This property has anthracite heating without mains. " "We recommend ASHP and HHR, but no gas condensing boiler" }, @@ -1254,14 +1174,12 @@ testing_examples = [ 'fixed-lighting-outlets-count': 8.0, 'low-energy-fixed-light-count': None, 'uprn': 100031556691, 'uprn-source': 'Energy Assessor', 'sheating-energy-eff': None, 'sheating-env-eff': None }, - "heating_recommendation_descriptions": [ - 'Upgrade to a new condensing boiler. Upgrade heating controls to Room thermostat, programmer and TRVs', - 'Upgrade to a new condensing boiler. Upgrade heating controls to Smart Thermostats, room sensors and smart ' - 'radiator valves (time & temperature zone control)', - 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' - 'Storage Heater Controls' + "heating_measure_types": [ + 'boiler_upgrade', + 'boiler_upgrade', + 'high_heat_retention_storage_heater' ], - "heating_controls_recommendation_descriptions": [], + "heating_controls_measure_types": [], "notes": "This property has room heaters with two different fuel sources, so we recommend HHR, ASHP, and a " "boiler upgrade" }, @@ -1302,11 +1220,10 @@ testing_examples = [ 'uprn': 100070685908, 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None }, - "heating_recommendation_descriptions": [ - 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' - 'Storage Heater Controls' + "heating_measure_types": [ + 'high_heat_retention_storage_heater' ], - "heating_controls_recommendation_descriptions": [], + "heating_controls_measure_types": [], "notes": "This property is a flag, without mains gas connection. Currently has underfloor electric heating" "so we recommend HHR" }, @@ -1348,17 +1265,13 @@ testing_examples = [ 'uprn': 100071209105, 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None }, - "heating_recommendation_descriptions": [ - 'Install an air source heat pump, and upgrade heating controls to Smart Thermostats, room sensors and ' - 'smart radiator valves (time & temperature zone control). The cost includes the £7500 boiler upgrade ' - 'scheme grant', - 'Upgrade to a new condensing boiler. Upgrade heating controls to Room thermostat, programmer and TRVs', - 'Upgrade to a new condensing boiler. Upgrade heating controls to Smart Thermostats, room sensors and smart ' - 'radiator valves (time & temperature zone control)', - 'Install high heat retention electric storage heaters. Upgrade heating controls to High Heat Retention ' - 'Storage Heater Controls' + "heating_measure_types": [ + 'air_source_heat_pump', + 'boiler_upgrade', + 'boiler_upgrade', + 'high_heat_retention_storage_heater' ], - "heating_controls_recommendation_descriptions": [], + "heating_controls_measure_types": [], "notes": "The property has warm air electricaire heating, so we recommend ASHP and HHR. It also has a mains" "connection so we recommend a gas condensing boiler" }, @@ -1402,12 +1315,11 @@ testing_examples = [ 'uprn': 100070955137, 'uprn-source': 'Address Matched', 'sheating-energy-eff': None, 'sheating-env-eff': None }, - "heating_recommendation_descriptions": [ - 'Upgrade to a new condensing boiler. Upgrade heating controls to Room thermostat, programmer and TRVs', - 'Upgrade to a new condensing boiler. Upgrade heating controls to Smart Thermostats, room sensors and smart ' - 'radiator valves (time & temperature zone control)' + "heating_measure_types": [ + 'boiler_upgrade', + 'boiler_upgrade', ], - "heating_controls_recommendation_descriptions": [], + "heating_controls_measure_types": [], "notes": "This property has warm air mains gas heating, so we recommend a gas condensing boiler" } ] diff --git a/recommendations/tests/test_heating_recommendations.py b/recommendations/tests/test_heating_recommendations.py index 35373729..ed2e037d 100644 --- a/recommendations/tests/test_heating_recommendations.py +++ b/recommendations/tests/test_heating_recommendations.py @@ -96,19 +96,19 @@ class TestHeatingRecommendations: recommender.recommend(has_cavity_or_loft_recommendations=False) - assert len(recommender.heating_recommendations) == len(test_case["heating_recommendation_descriptions"]) + assert len(recommender.heating_recommendations) == len(test_case["heating_measure_types"]) assert ( len(recommender.heating_control_recommendations) == - len(test_case["heating_controls_recommendation_descriptions"]) + len(test_case["heating_controls_measure_types"]) ) - # Check the exact descriptions + # Check the exact measure types assert ( - {x["description"] for x in recommender.heating_recommendations} == - set(test_case["heating_recommendation_descriptions"]) + {x["measure_type"] for x in recommender.heating_recommendations} == + set(test_case["heating_measure_types"]) ) assert ( - {x["description"] for x in recommender.heating_control_recommendations} == - set(test_case["heating_controls_recommendation_descriptions"]) + {x["measure_type"] for x in recommender.heating_control_recommendations} == + set(test_case["heating_controls_measure_types"]) ) diff --git a/recommendations/tests/test_roof_recommendations.py b/recommendations/tests/test_roof_recommendations.py index 139975bd..7df861f7 100644 --- a/recommendations/tests/test_roof_recommendations.py +++ b/recommendations/tests/test_roof_recommendations.py @@ -35,7 +35,7 @@ class TestRoofRecommendations: def test_loft_insulation_recommendation_50mm_insulation(self): epc_record = EPCRecord() - epc_record.prepared_epc = {"county": "Kent"} + epc_record.prepared_epc = {"county": "Kent", "roof-energy-eff": "Very Poor"} property_instance2 = Property(id=0, address="fake", postcode="fake", epc_record=epc_record) property_instance2.age_band = "F" property_instance2.insulation_floor_area = 100 @@ -63,7 +63,7 @@ class TestRoofRecommendations: assert roof_recommender2.recommendations[0]["parts"][0]["depth"] == 270 epc_record = EPCRecord() - epc_record.prepared_epc = {"county": "Greater London Authority"} + epc_record.prepared_epc = {"county": "Greater London Authority", "roof-energy-eff": "Very Poor"} property_instance3 = Property(id=0, address="fake", postcode="fake", epc_record=epc_record) property_instance3.age_band = "F" property_instance3.insulation_floor_area = 100 @@ -89,7 +89,7 @@ class TestRoofRecommendations: def test_loft_insulation_recommendation_150mm_insulation(self): epc_record = EPCRecord() - epc_record.prepared_epc = {"county": "North East Lincolnshire"} + epc_record.prepared_epc = {"county": "North East Lincolnshire", "roof-energy-eff": "Good"} property_instance4 = Property(id=0, address="fake", postcode="fake", epc_record=epc_record) property_instance4.age_band = "F" property_instance4.insulation_floor_area = 100 @@ -117,7 +117,7 @@ class TestRoofRecommendations: assert roof_recommender4.recommendations[0]["parts"][0]["depth"] == 200 epc_record = EPCRecord() - epc_record.prepared_epc = {"county": "Somerset"} + epc_record.prepared_epc = {"county": "Somerset", "roof-energy-eff": "Good"} property_instance5 = Property(id=0, address="fake", postcode="fake", epc_record=epc_record) property_instance5.age_band = "F" property_instance5.insulation_floor_area = 100 @@ -189,8 +189,8 @@ class TestRoofRecommendations: roof_recommender7.recommend(phase=0) assert len(roof_recommender7.recommendations) == 1 - assert roof_recommender7.recommendations[0]["new_u_value"] == 0.23 - assert roof_recommender7.recommendations[0]["starting_u_value"] == 1.5 + assert roof_recommender7.recommendations[0]["new_u_value"] == 0.24 + assert roof_recommender7.recommendations[0]["starting_u_value"] == 2.3 assert roof_recommender7.recommendations[0]["description"] == "Insulate room in roof at rafters and re-decorate" def test_ceiling_insulated_room_in_roof(self): From 83d56d9bc6a07ac96ec11f8006ce9aa63d873e8f Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 8 Oct 2024 18:20:44 +0100 Subject: [PATCH 127/166] Updated HHR storage recommendation for dual storage --- backend/app/plan/schemas.py | 3 +++ recommendations/HeatingRecommender.py | 21 ++++++++++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/backend/app/plan/schemas.py b/backend/app/plan/schemas.py index 447e0da2..29a7d5e3 100644 --- a/backend/app/plan/schemas.py +++ b/backend/app/plan/schemas.py @@ -86,6 +86,9 @@ class PlanTriggerRequest(BaseModel): # exists in the portfolio, it will be ignored multi_plan: Optional[bool] = False + # if False, allows optimisation to be switched off + optimise: Optional[bool] = False + _allowed_goals = {"Increasing EPC"} _allowed_housing_types = {"Social", "Private"} diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index 48063a8e..7c26fcb6 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -812,20 +812,37 @@ class HeatingRecommender: new_heating_description = self.DUAL_HEATING_DESCRIPTIONS[ self.property.main_heating["clean_description"] ]["hhr"]["mainheating_description"] + new_hot_water_description = self.property.hotwater["clean_description"] # We keep the hot water system else: new_heating_description = "Electric storage heaters" + new_hot_water_description = "Electric immersion, off-peak" # Set up artefacts, suitable for the simulation and regardless of controls heating_ending_config = MainHeatAttributes(new_heating_description).process() heating_simulation_config = check_simulation_difference( new_config=heating_ending_config, old_config=self.property.main_heating ) + + hot_water_end_config = HotWaterAttributes(new_hot_water_description).process() + hot_water_simulation_config = check_simulation_difference( + new_config=hot_water_end_config, old_config=self.property.hotwater + ) + + heating_simulation_config = { + **heating_simulation_config, + **hot_water_simulation_config + } # This upgrade will only take the heating system to average energy efficiency - if self.property.data["mainheat-energy-eff"] in ["Very Poor", "Poor"]: + if self.property.data["mainheat-energy-eff"] in ["Very Poor", "Poor"] and not self.dual_heating: heating_simulation_config["mainheat_energy_eff_ending"] = "Average" else: heating_simulation_config["mainheat_energy_eff_ending"] = self.property.data["mainheat-energy-eff"] + if self.property.data["hot-water-energy-eff"] in ["Very Poor", "Poor"]: + heating_simulation_config["hot_water_energy_eff_ending"] = "Average" + else: + heating_simulation_config["hot_water_energy_eff_ending"] = self.property.data["hot-water-energy-eff"] + # If the property is off-gas and has no heating system in place, the number of heated rooms will actually # be 0, so we use the number of rooms as the figure number_heated_rooms = ( @@ -863,6 +880,8 @@ class HeatingRecommender: heating_description_simulation = { "mainheat-description": new_heating_description, "mainheat-energy-eff": heating_simulation_config["mainheat_energy_eff_ending"], + "hotwater-description": new_hot_water_description, + "hot-water-energy-eff": heating_simulation_config["hot_water_energy_eff_ending"] } recommendations = self.combine_heating_and_controls( From 703cdcf1cbeb4ff884439f335ae6f46179790690 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 8 Oct 2024 18:25:45 +0100 Subject: [PATCH 128/166] added cost for dual immersion heater --- recommendations/Costs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/recommendations/Costs.py b/recommendations/Costs.py index afcd48d9..35fd237f 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -852,7 +852,8 @@ class Costs: :param number_heated_rooms: int, number of rooms to be heated """ - total_cost = 1500 * number_heated_rooms + # 500 is the cost of a dual immersion heater - a rough estimate + total_cost = 1200 * number_heated_rooms + 500 subtotal_before_vat = total_cost / (1 + self.VAT_RATE) vat = total_cost - subtotal_before_vat From c47d1db2ba23a218c8197851b3de0a1771eda092 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 8 Oct 2024 18:34:42 +0100 Subject: [PATCH 129/166] basic estimate for cost of cylinder --- recommendations/Costs.py | 10 +++++++--- recommendations/HeatingRecommender.py | 5 +++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/recommendations/Costs.py b/recommendations/Costs.py index 35fd237f..5554245f 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -841,7 +841,7 @@ class Costs: "labour_days": labour_days, } - def high_heat_electric_storage_heaters(self, number_heated_rooms): + def high_heat_electric_storage_heaters(self, number_heated_rooms, needs_cylinder): """ We base the estimates for the cost of electric storage heaters on the cost per room as estimated by the @@ -852,8 +852,12 @@ class Costs: :param number_heated_rooms: int, number of rooms to be heated """ - # 500 is the cost of a dual immersion heater - a rough estimate - total_cost = 1200 * number_heated_rooms + 500 + if needs_cylinder: + # 1000 is the cost of a new hot water cylinder + total_cost = 1200 * number_heated_rooms + 1000 + else: + # 500 is the cost of a dual immersion heater - a rough estimate + total_cost = 1200 * number_heated_rooms + 500 subtotal_before_vat = total_cost / (1 + self.VAT_RATE) vat = total_cost - subtotal_before_vat diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index 7c26fcb6..7dc4f8b2 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -858,7 +858,8 @@ class HeatingRecommender: # Upgrade to electric storage heaters costs = self.costs.high_heat_electric_storage_heaters( - number_heated_rooms=number_heated_rooms + number_heated_rooms=number_heated_rooms, + needs_cylinder=self.property.hotwater["system_type"] == "from main system" ) if self.dual_heating: description = self.DUAL_HEATING_DESCRIPTIONS[ @@ -866,7 +867,7 @@ class HeatingRecommender: ]["hhr"]["recommendation_description"] else: - description = "Install high heat retention electric storage heaters." + description = "Install high heat retention electric storage heaters with an appropriate off-peak tariff." # We check the existing heating system and controls if ( From b0118655179e4c26c3ae523dd7264e0341f326a9 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 8 Oct 2024 18:43:24 +0100 Subject: [PATCH 130/166] adding the ability to switch off optimisation --- backend/app/plan/router.py | 37 ++++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 2dc03e60..f4924c71 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -631,27 +631,38 @@ async def trigger_plan(body: PlanTriggerRequest): for p in input_properties: if not recommendations.get(p.id): continue + input_measures = prepare_input_measures(recommendations[p.id], body.goal) current_sap_points = int(p.data["current-energy-efficiency"]) target_sap_points = epc_to_sap_lower_bound(body.goal_value) sap_gain = CostOptimiser.calculate_sap_gain_with_slack(target_sap_points - current_sap_points) - if body.budget: - optimiser = GainOptimiser( - input_measures, max_cost=body.budget, max_gain=sap_gain if sap_gain > 0 else 0 - ) + if not body.optimise: + if body.goal != "Increasing EPC": + raise NotImplementedError("Only EPC optimisation is currently supported") + solution = [] + for sub_list in input_measures: + # Select the entry with the highest gain, and if tied, choose the one with the lowest cost + best_measure = max(sub_list, key=lambda x: (x['gain'], -x['cost'])) + solution.append(best_measure) else: - # The minimum gain is the minimum number of SAP points required to get to the target SAP band - # If the gain is negative, the optimiser will return an empty solution - optimiser = CostOptimiser( - input_measures, - min_gain=sap_gain - ) - optimiser.setup() - optimiser.solve() - solution = optimiser.solution + if body.budget: + optimiser = GainOptimiser( + input_measures, max_cost=body.budget, max_gain=sap_gain if sap_gain > 0 else 0 + ) + else: + # The minimum gain is the minimum number of SAP points required to get to the target SAP band + # If the gain is negative, the optimiser will return an empty solution + optimiser = CostOptimiser( + input_measures, + min_gain=sap_gain + ) + + optimiser.setup() + optimiser.solve() + solution = optimiser.solution selected_recommendations = {r["id"] for r in solution} From 73324e540cefdfd9593490a90cb4919effc7b02a Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 8 Oct 2024 18:46:44 +0100 Subject: [PATCH 131/166] default optimisation on --- backend/app/plan/schemas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/app/plan/schemas.py b/backend/app/plan/schemas.py index 29a7d5e3..0d58c7e9 100644 --- a/backend/app/plan/schemas.py +++ b/backend/app/plan/schemas.py @@ -87,7 +87,7 @@ class PlanTriggerRequest(BaseModel): multi_plan: Optional[bool] = False # if False, allows optimisation to be switched off - optimise: Optional[bool] = False + optimise: Optional[bool] = True _allowed_goals = {"Increasing EPC"} From bbefbea8f0ef2239fcac59e8245dcbe6fa5affcf Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 8 Oct 2024 18:51:16 +0100 Subject: [PATCH 132/166] added missing file --- .../tests/test_data/roof_uvalue_test_cases.py | 355 ++++++++++++++++++ 1 file changed, 355 insertions(+) create mode 100644 recommendations/tests/test_data/roof_uvalue_test_cases.py diff --git a/recommendations/tests/test_data/roof_uvalue_test_cases.py b/recommendations/tests/test_data/roof_uvalue_test_cases.py new file mode 100644 index 00000000..1f230cc8 --- /dev/null +++ b/recommendations/tests/test_data/roof_uvalue_test_cases.py @@ -0,0 +1,355 @@ +roof_uvalue_test_cases = [ + # Pitched roof + { + 'insulation_thickness': '0', + 'is_loft': True, + 'is_roof_room': False, + 'is_thatched': False, + 'has_dwelling_above': False, + 'is_flat': False, + 'is_pitched': True, + 'is_at_rafters': False, + 'age_band': 'A', + 'uvalue': 2.3, + }, + { + 'insulation_thickness': '12', + 'is_loft': True, + 'is_roof_room': False, + 'is_thatched': False, + 'has_dwelling_above': False, + 'is_flat': False, + 'is_pitched': True, + 'is_at_rafters': False, + 'age_band': 'A', + 'uvalue': 1.5 + }, + { + 'insulation_thickness': '25', + 'is_loft': True, + 'is_roof_room': False, + 'is_thatched': False, + 'has_dwelling_above': False, + 'is_flat': False, + 'is_pitched': True, + 'is_at_rafters': False, + 'age_band': 'A', + 'uvalue': 1 + }, + { + 'insulation_thickness': '50', + 'is_loft': True, + 'is_roof_room': False, + 'is_thatched': False, + 'has_dwelling_above': False, + 'is_flat': False, + 'is_pitched': True, + 'is_at_rafters': False, + 'age_band': 'A', + 'uvalue': 0.68 + }, + { + 'insulation_thickness': '75', + 'is_loft': True, + 'is_roof_room': False, + 'is_thatched': False, + 'has_dwelling_above': False, + 'is_flat': False, + 'is_pitched': True, + 'is_at_rafters': False, + 'age_band': 'A', + 'uvalue': 0.5 + }, + { + 'insulation_thickness': '100', + 'is_loft': True, + 'is_roof_room': False, + 'is_thatched': False, + 'has_dwelling_above': False, + 'is_flat': False, + 'is_pitched': True, + 'is_at_rafters': False, + 'age_band': 'A', + 'uvalue': 0.4 + }, + { + 'insulation_thickness': '150', + 'is_loft': True, + 'is_roof_room': False, + 'is_thatched': False, + 'has_dwelling_above': False, + 'is_flat': False, + 'is_pitched': True, + 'is_at_rafters': False, + 'age_band': 'A', + 'uvalue': 0.3 + }, + { + 'insulation_thickness': '200', + 'is_loft': True, + 'is_roof_room': False, + 'is_thatched': False, + 'has_dwelling_above': False, + 'is_flat': False, + 'is_pitched': True, + 'is_at_rafters': False, + 'age_band': 'A', + 'uvalue': 0.21 + }, + { + 'insulation_thickness': '250', + 'is_loft': True, + 'is_roof_room': False, + 'is_thatched': False, + 'has_dwelling_above': False, + 'is_flat': False, + 'is_pitched': True, + 'is_at_rafters': False, + 'age_band': 'A', + 'uvalue': 0.17 + }, + { + 'insulation_thickness': '270', + 'is_loft': True, + 'is_roof_room': False, + 'is_thatched': False, + 'has_dwelling_above': False, + 'is_flat': False, + 'is_pitched': True, + 'is_at_rafters': False, + 'age_band': 'A', + 'uvalue': 0.16 + }, + { + 'insulation_thickness': '300', + 'is_loft': True, + 'is_roof_room': False, + 'is_thatched': False, + 'has_dwelling_above': False, + 'is_flat': False, + 'is_pitched': True, + 'is_at_rafters': False, + 'age_band': 'A', + 'uvalue': 0.14 + }, + { + 'insulation_thickness': '350', + 'is_loft': True, + 'is_roof_room': False, + 'is_thatched': False, + 'has_dwelling_above': False, + 'is_flat': False, + 'is_pitched': True, + 'is_at_rafters': False, + 'age_band': 'A', + 'uvalue': 0.12 + }, + { + 'insulation_thickness': '400+', + 'is_loft': True, + 'is_roof_room': False, + 'is_thatched': False, + 'has_dwelling_above': False, + 'is_flat': False, + 'is_pitched': True, + 'is_at_rafters': False, + 'age_band': 'A', + 'uvalue': 0.11 + }, + # Flat roofs - no insulation + { + 'insulation_thickness': 'none', + 'is_loft': False, + 'is_roof_room': False, + 'is_thatched': False, + 'has_dwelling_above': False, + 'is_flat': True, + 'is_pitched': False, + 'is_at_rafters': False, + 'age_band': 'B', + 'uvalue': 2.3 + }, + { + 'insulation_thickness': 'none', + 'is_loft': False, + 'is_roof_room': False, + 'is_thatched': False, + 'has_dwelling_above': False, + 'is_flat': True, + 'is_pitched': False, + 'is_at_rafters': False, + 'age_band': 'D', + 'uvalue': 2.3 + }, + { + 'insulation_thickness': 'none', + 'is_loft': False, + 'is_roof_room': False, + 'is_thatched': False, + 'has_dwelling_above': False, + 'is_flat': True, + 'is_pitched': False, + 'is_at_rafters': False, + 'age_band': 'E', + 'uvalue': 1.5 + }, + { + 'insulation_thickness': 'none', + 'is_loft': False, + 'is_roof_room': False, + 'is_thatched': False, + 'has_dwelling_above': False, + 'is_flat': True, + 'is_pitched': False, + 'is_at_rafters': False, + 'age_band': 'F', + 'uvalue': 0.68 + }, + { + 'insulation_thickness': 'none', + 'is_loft': False, + 'is_roof_room': False, + 'is_thatched': False, + 'has_dwelling_above': False, + 'is_flat': True, + 'is_pitched': False, + 'is_at_rafters': False, + 'age_band': 'G', + 'uvalue': 0.4 + }, + { + 'insulation_thickness': 'none', + 'is_loft': False, + 'is_roof_room': False, + 'is_thatched': False, + 'has_dwelling_above': False, + 'is_flat': True, + 'is_pitched': False, + 'is_at_rafters': False, + 'age_band': 'H', + 'uvalue': 0.35 + }, + # Flat roofs - 50mm insulation + { + 'insulation_thickness': '50', + 'is_loft': False, + 'is_roof_room': False, + 'is_thatched': False, + 'has_dwelling_above': False, + 'is_flat': True, + 'is_pitched': False, + 'is_at_rafters': False, + 'age_band': 'B', + 'uvalue': 0.68 + }, + { + 'insulation_thickness': '50', + 'is_loft': False, + 'is_roof_room': False, + 'is_thatched': False, + 'has_dwelling_above': False, + 'is_flat': True, + 'is_pitched': False, + 'is_at_rafters': False, + 'age_band': 'E', + 'uvalue': 0.68 + }, + { + 'insulation_thickness': '50', + 'is_loft': False, + 'is_roof_room': False, + 'is_thatched': False, + 'has_dwelling_above': False, + 'is_flat': True, + 'is_pitched': False, + 'is_at_rafters': False, + 'age_band': 'G', + 'uvalue': 0.4 + }, + { + 'insulation_thickness': '50', + 'is_loft': False, + 'is_roof_room': False, + 'is_thatched': False, + 'has_dwelling_above': False, + 'is_flat': True, + 'is_pitched': False, + 'is_at_rafters': False, + 'age_band': 'I', + 'uvalue': 0.35 + }, + # Flat roofs - 100mm insulation + { + 'insulation_thickness': '100', + 'is_loft': False, + 'is_roof_room': False, + 'is_thatched': False, + 'has_dwelling_above': False, + 'is_flat': True, + 'is_pitched': False, + 'is_at_rafters': False, + 'age_band': 'B', + 'uvalue': 0.4 + }, + { + 'insulation_thickness': '100', + 'is_loft': False, + 'is_roof_room': False, + 'is_thatched': False, + 'has_dwelling_above': False, + 'is_flat': True, + 'is_pitched': False, + 'is_at_rafters': False, + 'age_band': 'F', + 'uvalue': 0.4 + }, + { + 'insulation_thickness': '100', + 'is_loft': False, + 'is_roof_room': False, + 'is_thatched': False, + 'has_dwelling_above': False, + 'is_flat': True, + 'is_pitched': False, + 'is_at_rafters': False, + 'age_band': 'J', + 'uvalue': 0.25 + }, + # Flat roofs - 150mm insulation + { + 'insulation_thickness': '150', + 'is_loft': False, + 'is_roof_room': False, + 'is_thatched': False, + 'has_dwelling_above': False, + 'is_flat': True, + 'is_pitched': False, + 'is_at_rafters': False, + 'age_band': 'B', + 'uvalue': 0.3 + }, + { + 'insulation_thickness': '150', + 'is_loft': False, + 'is_roof_room': False, + 'is_thatched': False, + 'has_dwelling_above': False, + 'is_flat': True, + 'is_pitched': False, + 'is_at_rafters': False, + 'age_band': 'J', + 'uvalue': 0.25 + }, + { + 'insulation_thickness': '150', + 'is_loft': False, + 'is_roof_room': False, + 'is_thatched': False, + 'has_dwelling_above': False, + 'is_flat': True, + 'is_pitched': False, + 'is_at_rafters': False, + 'age_band': 'L', + 'uvalue': 0.18 + }, +] From a2694de7e62997a23ab096a6febf57e900ea177c Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 8 Oct 2024 19:44:02 +0100 Subject: [PATCH 133/166] added additional room roof unit tests --- recommendations/recommendation_utils.py | 4 +- .../tests/test_data/roof_uvalue_test_cases.py | 111 ++++++++++++++++++ .../tests/test_recommendation_utils.py | 2 +- 3 files changed, 114 insertions(+), 3 deletions(-) diff --git a/recommendations/recommendation_utils.py b/recommendations/recommendation_utils.py index 3364f0fa..d5788544 100644 --- a/recommendations/recommendation_utils.py +++ b/recommendations/recommendation_utils.py @@ -248,7 +248,7 @@ def extract_thickness(thickness, is_roof_room, is_at_rafters, is_loft, is_flat): thickness_map = { "below average": "50", "average": "100", - "above average": "270", + "above average": "150", "none": "0", } thickness = thickness_map[thickness] @@ -287,7 +287,7 @@ def get_u_value_from_s9( ): return None - if thickness in [0, "0"] and is_loft: + if thickness in [0, "0"] and (is_loft or is_roof_room): return None # Determine the column to refer based on the roof type diff --git a/recommendations/tests/test_data/roof_uvalue_test_cases.py b/recommendations/tests/test_data/roof_uvalue_test_cases.py index 1f230cc8..8bf8b35f 100644 --- a/recommendations/tests/test_data/roof_uvalue_test_cases.py +++ b/recommendations/tests/test_data/roof_uvalue_test_cases.py @@ -352,4 +352,115 @@ roof_uvalue_test_cases = [ 'age_band': 'L', 'uvalue': 0.18 }, + # Room roof - age band A + { + 'insulation_thickness': 'none', + 'is_loft': False, + 'is_roof_room': True, + 'is_thatched': False, + 'has_dwelling_above': False, + 'is_flat': False, + 'is_pitched': False, + 'is_at_rafters': False, + 'age_band': 'A', + 'uvalue': 2.3 + }, + { + 'insulation_thickness': 'below average', + 'is_loft': False, + 'is_roof_room': True, + 'is_thatched': False, + 'has_dwelling_above': False, + 'is_flat': False, + 'is_pitched': False, + 'is_at_rafters': False, + 'age_band': 'A', + 'uvalue': 0.68 + }, + { + 'insulation_thickness': 'average', + 'is_loft': False, + 'is_roof_room': True, + 'is_thatched': False, + 'has_dwelling_above': False, + 'is_flat': False, + 'is_pitched': False, + 'is_at_rafters': False, + 'age_band': 'A', + 'uvalue': 0.4 + }, + { + 'insulation_thickness': 'above average', + 'is_loft': False, + 'is_roof_room': True, + 'is_thatched': False, + 'has_dwelling_above': False, + 'is_flat': False, + 'is_pitched': False, + 'is_at_rafters': False, + 'age_band': 'A', + 'uvalue': 0.3 + }, + # Room roof - age band E + { + 'insulation_thickness': 'none', + 'is_loft': False, + 'is_roof_room': True, + 'is_thatched': False, + 'has_dwelling_above': False, + 'is_flat': False, + 'is_pitched': False, + 'is_at_rafters': False, + 'age_band': 'E', + 'uvalue': 1.5 + }, + { + 'insulation_thickness': 'average', + 'is_loft': False, + 'is_roof_room': True, + 'is_thatched': False, + 'has_dwelling_above': False, + 'is_flat': False, + 'is_pitched': False, + 'is_at_rafters': False, + 'age_band': 'E', + 'uvalue': 0.4 + }, + # Room roof - age band H + { + 'insulation_thickness': 'none', + 'is_loft': False, + 'is_roof_room': True, + 'is_thatched': False, + 'has_dwelling_above': False, + 'is_flat': False, + 'is_pitched': False, + 'is_at_rafters': False, + 'age_band': 'H', + 'uvalue': 0.35 + }, + { + 'insulation_thickness': 'below average', + 'is_loft': False, + 'is_roof_room': True, + 'is_thatched': False, + 'has_dwelling_above': False, + 'is_flat': False, + 'is_pitched': False, + 'is_at_rafters': False, + 'age_band': 'H', + 'uvalue': 0.68 + }, + { + 'insulation_thickness': 'average', + 'is_loft': False, + 'is_roof_room': True, + 'is_thatched': False, + 'has_dwelling_above': False, + 'is_flat': False, + 'is_pitched': False, + 'is_at_rafters': False, + 'age_band': 'H', + 'uvalue': 0.4 + }, ] diff --git a/recommendations/tests/test_recommendation_utils.py b/recommendations/tests/test_recommendation_utils.py index b445a798..fa707b4b 100644 --- a/recommendations/tests/test_recommendation_utils.py +++ b/recommendations/tests/test_recommendation_utils.py @@ -198,7 +198,7 @@ class TestRecommendationUtils: } u_value = recommendation_utils.get_roof_u_value(**inputs) - assert u_value == 0.16, f"Expected 0.16, but got {u_value}" + assert u_value == 0.3, f"Expected 0.3, but got {u_value}" def test_get_roof_u_value_case_8(self): # Test case where there is a dwelling above the roof, U-value should be 0 From 75c490c0894e7b48c51d52263fcd8cee2488e0c0 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 8 Oct 2024 22:13:08 +0100 Subject: [PATCH 134/166] fix a bug with None not being caught by ValueError --- recommendations/recommendation_utils.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/recommendations/recommendation_utils.py b/recommendations/recommendation_utils.py index d5788544..00da6107 100644 --- a/recommendations/recommendation_utils.py +++ b/recommendations/recommendation_utils.py @@ -205,7 +205,7 @@ def get_wall_u_value( mapped_value = wall_uvalues_df[ wall_uvalues_df["Wall_type"] == mapped_description - ][age_band].values[0] + ][age_band].values[0] if pd.isnull(mapped_value) and "Park home" in mapped_description: # We don't know enough in this case so we default to 0 @@ -257,7 +257,7 @@ def extract_thickness(thickness, is_roof_room, is_at_rafters, is_loft, is_flat): try: thickness = int(thickness) return thickness - except ValueError: + except (TypeError, ValueError): # If thickness is not a valid number (could be a string or None), return None return None @@ -350,7 +350,7 @@ def get_roof_u_value( is_roof_room=is_roof_room, is_at_rafters=is_at_rafters, is_loft=is_loft, - is_flat=is_flat + is_flat=is_flat, ) # Step 1: Try to get the U-value from table S9 based on the insulation thickness @@ -395,9 +395,9 @@ def get_roof_u_value( # Get the U-value from table S10 based on the age band and the determined column if is_flat and thickness is not None: u_value = s10.loc[ - (s10["Insulation_Thickness"] == thickness) | - s10["Age_band"].str.contains(age_band), - column + (s10["Insulation_Thickness"] == thickness) + | s10["Age_band"].str.contains(age_band), + column, ].values.min() else: u_value = s10.loc[s10["Age_band"].str.contains(age_band), column].values[0] @@ -560,7 +560,7 @@ def get_floor_u_value( insulation_lookup = s11[ s11["Age_band"].str.contains(age_band) & s11["Floor_construction"] == floor_type - ] + ] if insulation_lookup.empty: insulation_thickness = 0 else: From 0560a6ad73a468811e45aa3db69d41be88f519e3 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 9 Oct 2024 11:22:19 +0100 Subject: [PATCH 135/166] fixed room roof tests --- recommendations/tests/test_roof_recommendations.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/recommendations/tests/test_roof_recommendations.py b/recommendations/tests/test_roof_recommendations.py index 7df861f7..6dbc3b72 100644 --- a/recommendations/tests/test_roof_recommendations.py +++ b/recommendations/tests/test_roof_recommendations.py @@ -189,8 +189,8 @@ class TestRoofRecommendations: roof_recommender7.recommend(phase=0) assert len(roof_recommender7.recommendations) == 1 - assert roof_recommender7.recommendations[0]["new_u_value"] == 0.24 - assert roof_recommender7.recommendations[0]["starting_u_value"] == 2.3 + assert roof_recommender7.recommendations[0]["new_u_value"] == 0.2 + assert roof_recommender7.recommendations[0]["starting_u_value"] == 0.8 assert roof_recommender7.recommendations[0]["description"] == "Insulate room in roof at rafters and re-decorate" def test_ceiling_insulated_room_in_roof(self): From 5acb0c33e6f8e58abcf17911795b81ecfd6fc7b7 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 9 Oct 2024 18:22:14 +0100 Subject: [PATCH 136/166] updating windows scenarios to set efficiency to good --- etl/epc/generate_scenarios_data.py | 2 +- recommendations/WindowsRecommendations.py | 13 +++++-------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/etl/epc/generate_scenarios_data.py b/etl/epc/generate_scenarios_data.py index b61618b1..f1406759 100644 --- a/etl/epc/generate_scenarios_data.py +++ b/etl/epc/generate_scenarios_data.py @@ -65,7 +65,7 @@ scenario_properties = [ ], [ ["external_wall_insulation"], - "10", + "11", {}, [0], ], diff --git a/recommendations/WindowsRecommendations.py b/recommendations/WindowsRecommendations.py index 343965e3..1f755369 100644 --- a/recommendations/WindowsRecommendations.py +++ b/recommendations/WindowsRecommendations.py @@ -127,12 +127,12 @@ class WindowsRecommendations: ) # Set up the simulation config + windows_energy_eff = "Good" if self.property.windows["glazing_type"] == "multiple": glazing_type_ending = "multiple" glazed_type_ending = ( "secondary glazing" if is_secondary_glazing else "double glazing installed during or after 2002" ) - windows_energy_eff = "Good" new_windows_description = "Multiple glazing throughout" elif self.property.windows["glazing_type"] == "single": @@ -145,10 +145,8 @@ class WindowsRecommendations: ) if is_secondary_glazing: - windows_energy_eff = "Good" new_windows_description = "Full secondary glazing" else: - windows_energy_eff = "Average" new_windows_description = "Fully double glazed" elif self.property.windows["glazing_type"] == "double": @@ -164,7 +162,6 @@ class WindowsRecommendations: if not is_secondary_glazing: glazed_type_ending = "double glazing installed during or after 2002" new_windows_description = "Fully double glazed" - windows_energy_eff = "Average" else: if self.property.data["multi-glaze-proportion"] < 50: glazed_type_ending = "secondary glazing" @@ -172,13 +169,11 @@ class WindowsRecommendations: glazed_type_ending = "double glazing installed during or after 2002" new_windows_description = "Multiple glazing throughout" - windows_energy_eff = "Good" elif self.property.windows["glazing_type"] == "secondary": glazing_type_ending = ( "secondary" if is_secondary_glazing else "multiple" ) - windows_energy_eff = "Good" # This is the opposite. If there is secondary glazing in place, and we're recommending double # we set glazed_type_ending, depending on the proportion of glazing in place if is_secondary_glazing: @@ -194,8 +189,10 @@ class WindowsRecommendations: else: raise ValueError("Invalid glazing type - implement me") - if (self.property.data["windows-energy-eff"] in ["Good", "Very Good"]) and (windows_energy_eff == "Average"): - windows_energy_eff = self.property.data["windows-energy-eff"] + if self.property.data["windows-energy-eff"] == "Very Good": + raise ValueError("Very Good energy efficiency is not supported") + + # For post 2002 windows, the energy efficiency is "Good" and so for the simulation, we simulate with "Good" windows_ending_config = WindowAttributes(new_windows_description).process() From 2d7e9a3cc9bb19bb558d9fbdefc495e4b2826e26 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 10 Oct 2024 18:12:29 +0100 Subject: [PATCH 137/166] setting up code for gla proposal --- etl/customers/gla/__init__.py | 0 etl/customers/gla/proposal_investigation.py | 76 +++++++++++++++++++++ etl/ownership/Ownership.py | 19 ++++-- 3 files changed, 88 insertions(+), 7 deletions(-) create mode 100644 etl/customers/gla/__init__.py create mode 100644 etl/customers/gla/proposal_investigation.py diff --git a/etl/customers/gla/__init__.py b/etl/customers/gla/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/etl/customers/gla/proposal_investigation.py b/etl/customers/gla/proposal_investigation.py new file mode 100644 index 00000000..e36d82b8 --- /dev/null +++ b/etl/customers/gla/proposal_investigation.py @@ -0,0 +1,76 @@ +""" +This script performs some basic analysis to identify EPC data for postcodes specified in the Warmer Homes Local Grant +""" +from nis import match + +import pandas as pd +from etl.ownership.Ownership import Ownership + +postcodes = pd.read_excel( + "/Users/khalimconn-kowlessar/Downloads/WHLG-eligible-postcodes.xlsx", sheet_name='Eligible postcodes' +) +# Take just the first two columns +postcodes = postcodes[ + ['List of eligible postcodes via the IMD Income Decile 1-2 pathway', 'Unnamed: 1'] +] + +postcodes.columns = ['postcode', 'Local Authority'] +# Drop the first row +postcodes = postcodes.drop([0, 1]) +# Since there are a large number of potcodes (425k), let's just take a few examples +# Take postcodes that begin with "BN15" +postcodes = postcodes[postcodes["postcode"].str.startswith("BN15")] + +# The Local Authority is Adur, so let's get the EPC data for this area +# epc_data = pd.read_csv( +# "/Users/khalimconn-kowlessar/Documents/hestia/Model/local_data/all-domestic-certificates/domestic-E07000223-Adur" +# "/certificates.csv", low_memory=False +# ) +# # Filter on these postcodes +# epc_data = epc_data[epc_data["POSTCODE"].str.lower().isin(postcodes["postcode"].str.lower())] +# epc_data = epc_data[~pd.isnull(epc_data["UPRN"])] +# # Take the newest EPC for each UPRN, based on LODGEMENT_DATE +# epc_data["LODGEMENT_DATE"] = pd.to_datetime(epc_data["LODGEMENT_DATE"]) +# epc_data = epc_data.sort_values("LODGEMENT_DATE", ascending=False).drop_duplicates("UPRN") +# +# # Let's look at the breakdown of EPC ratings. We want the count and the % of the total +# ratings_distribution = epc_data.groupby("CURRENT_ENERGY_RATING").size().reset_index() +# ratings_distribution.columns = ["Rating", "Count"] +# ratings_distribution["Percentage"] = ratings_distribution["Count"] / ratings_distribution["Count"].sum() * 100 + +# Can we identify the owners of these units so we can contact them? +ownership = Ownership( + epc_paths=[ + "/Users/khalimconn-kowlessar/Documents/hestia/Model/local_data/all-domestic-certificates/domestic-E07000223" + "-Adur/certificates.csv" + ], + domestic_ownership_path="/Users/khalimconn-kowlessar/Downloads/CCOD_FULL_2024_07.csv", + overseas_ownership_path="/Users/khalimconn-kowlessar/Downloads/OCOD_FULL_2024_07.csv", + land_registry_path="/Users/khalimconn-kowlessar/Downloads/pp-complete.csv", + project_name="gla-proposal", + bucket="retrofit-data-dev", + average_property_value=0, + portfolio_value=0, + excluded_owners=[], + excluded_uprns=[], + save=False +) + +# Data will be found at ownership/gla-proposal +ownership.source_epc_properties(column_filters={}) + +# Step 2: Get company ownership data +ownership.load_company_ownership() + +# Step 3: Prepare data for matching +ownership.prepare_for_matching() + +# Step 4: Match EPC data to ownership data +ownership.match() + +# We have the matches, which we now need to match to the postcodes +matches = ownership.matched_addresses.copy() +# filter matches on the postcodes we're interested in +matches = matches[matches["epc_postcode"].str.lower().isin(postcodes["postcode"].str.lower())] +# Remove any social transactions +matches = matches[~matches["TENURE"].isin(["Rented (social)", "rental (social)"])] diff --git a/etl/ownership/Ownership.py b/etl/ownership/Ownership.py index 3bc4b60d..2079391c 100644 --- a/etl/ownership/Ownership.py +++ b/etl/ownership/Ownership.py @@ -61,6 +61,7 @@ class Ownership: portfolio_value: float, excluded_owners: List[str] = None, excluded_uprns: List[int] = None, + save=True ): """ @@ -115,6 +116,8 @@ class Ownership: f"ownership/{self.project_name}/{self.run_timestamp}/portfolio_epc_data.xlsx" ) + self.save = save + # Data self.epc_data = None self.ownership_data = None @@ -210,12 +213,13 @@ class Ownership: if self.excluded_uprns: self.epc_data = self.epc_data[~self.epc_data["UPRN"].astype(float).isin(self.excluded_uprns)] - # We now store the data in s3 - save_excel_to_s3( - df=self.epc_data, - bucket_name=self.bucket, - file_key=self.epc_data_filepath - ) + if self.save: + # We now store the data in s3 + save_excel_to_s3( + df=self.epc_data, + bucket_name=self.bucket, + file_key=self.epc_data_filepath + ) def load_company_ownership(self): """ @@ -590,7 +594,8 @@ class Ownership: "CURRENT_ENERGY_RATING", "POSTCODE", "LODGEMENT_DATE", - "TRANSACTION_TYPE" + "TRANSACTION_TYPE", + "TENURE", ] ].rename( columns={ From a953a1f0ee215f30cb5a17953f0d8f4b167caa18 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 10 Oct 2024 18:38:08 +0100 Subject: [PATCH 138/166] Improving ownership matching algorithm --- etl/customers/gla/proposal_investigation.py | 29 +++++++++++++++++++++ etl/ownership/Ownership.py | 10 +++---- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/etl/customers/gla/proposal_investigation.py b/etl/customers/gla/proposal_investigation.py index e36d82b8..57df0554 100644 --- a/etl/customers/gla/proposal_investigation.py +++ b/etl/customers/gla/proposal_investigation.py @@ -74,3 +74,32 @@ matches = ownership.matched_addresses.copy() matches = matches[matches["epc_postcode"].str.lower().isin(postcodes["postcode"].str.lower())] # Remove any social transactions matches = matches[~matches["TENURE"].isin(["Rented (social)", "rental (social)"])] + +matches.head() +owners_count = matches.groupby(['Proprietor Name (1)', 'Company Registration No. (1)']).size().reset_index() +owners_count.columns = ['Owner', 'Owner Registration #', 'Count'] +owners_count = owners_count.sort_values('Count', ascending=False) +owners_count["Percentage"] = owners_count["Count"] / owners_count["Count"].sum() * 100 + +companies_house_api_key = "1d9c2877-3271-4642-80ed-a6170971653f" + +import requests +import json + +company_number = "13197205" +url = f'https://api.company-information.service.gov.uk/company/{company_number}' + +# Make the API request +response = requests.get(url, auth=(companies_house_api_key, '')) + +# Check if the request was successful +if response.status_code == 200: + company_data = response.json() + # Pretty-print the fetched data + print(json.dumps(company_data, indent=4)) +else: + print(f"Failed to fetch data. Status code: {response.status_code}") + +psc_url = f'https://api.company-information.service.gov.uk/company/{company_number}/persons-with-significant-control' +psc_response = requests.get(psc_url, auth=(companies_house_api_key, '')) +psc_data = psc_response.json() diff --git a/etl/ownership/Ownership.py b/etl/ownership/Ownership.py index 2079391c..52181452 100644 --- a/etl/ownership/Ownership.py +++ b/etl/ownership/Ownership.py @@ -488,11 +488,11 @@ class Ownership: house_no = house_no.replace(",", "") if house_no is None: - # It's hard for us to get a reliable match - # filtered = filtered[filtered["Property Address"].str.contains(address["ADDRESS1"])] - # if filtered.shape[0] > 1: - # raise Exception("No valid - maybe we should do levenstein?") - continue + # If the house number is missing, it means that we usually have a named property so we look for an + # exact match on that name + filtered = filtered[filtered["Property Address"].str.lower().str.contains(address["ADDRESS"].lower())] + if filtered.shape[0] != 1: + continue else: From f53ce8b4302482ce54785e9da807c6b6ad9296b3 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 10 Oct 2024 18:51:12 +0100 Subject: [PATCH 139/166] allow postcode filtering --- etl/customers/gla/proposal_investigation.py | 25 ++++++++++++--------- etl/ownership/Ownership.py | 8 ++++++- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/etl/customers/gla/proposal_investigation.py b/etl/customers/gla/proposal_investigation.py index 57df0554..776bbc59 100644 --- a/etl/customers/gla/proposal_investigation.py +++ b/etl/customers/gla/proposal_investigation.py @@ -1,9 +1,12 @@ """ This script performs some basic analysis to identify EPC data for postcodes specified in the Warmer Homes Local Grant """ -from nis import match +import inspect +import requests +import json import pandas as pd +from pathlib import Path from etl.ownership.Ownership import Ownership postcodes = pd.read_excel( @@ -19,7 +22,7 @@ postcodes.columns = ['postcode', 'Local Authority'] postcodes = postcodes.drop([0, 1]) # Since there are a large number of potcodes (425k), let's just take a few examples # Take postcodes that begin with "BN15" -postcodes = postcodes[postcodes["postcode"].str.startswith("BN15")] +# postcodes = postcodes[postcodes["postcode"].str.startswith("BN15")] # The Local Authority is Adur, so let's get the EPC data for this area # epc_data = pd.read_csv( @@ -39,11 +42,14 @@ postcodes = postcodes[postcodes["postcode"].str.startswith("BN15")] # ratings_distribution["Percentage"] = ratings_distribution["Count"] / ratings_distribution["Count"].sum() * 100 # Can we identify the owners of these units so we can contact them? + +file_src = inspect.getfile(lambda x: None) +DATA_DIRECTORY = Path(file_src).parent / "local_data" / "all-domestic-certificates" +epc_paths = [entry for entry in DATA_DIRECTORY.iterdir() if entry.is_dir()] +epc_paths = [str(entry / "certificates.csv") for entry in epc_paths] + ownership = Ownership( - epc_paths=[ - "/Users/khalimconn-kowlessar/Documents/hestia/Model/local_data/all-domestic-certificates/domestic-E07000223" - "-Adur/certificates.csv" - ], + epc_paths=epc_paths, domestic_ownership_path="/Users/khalimconn-kowlessar/Downloads/CCOD_FULL_2024_07.csv", overseas_ownership_path="/Users/khalimconn-kowlessar/Downloads/OCOD_FULL_2024_07.csv", land_registry_path="/Users/khalimconn-kowlessar/Downloads/pp-complete.csv", @@ -53,11 +59,11 @@ ownership = Ownership( portfolio_value=0, excluded_owners=[], excluded_uprns=[], - save=False + save=True ) # Data will be found at ownership/gla-proposal -ownership.source_epc_properties(column_filters={}) +ownership.source_epc_properties(column_filters={}, postcodes=postcodes["postcode"].str.lower().tolist()) # Step 2: Get company ownership data ownership.load_company_ownership() @@ -83,9 +89,6 @@ owners_count["Percentage"] = owners_count["Count"] / owners_count["Count"].sum() companies_house_api_key = "1d9c2877-3271-4642-80ed-a6170971653f" -import requests -import json - company_number = "13197205" url = f'https://api.company-information.service.gov.uk/company/{company_number}' diff --git a/etl/ownership/Ownership.py b/etl/ownership/Ownership.py index 52181452..2c04ac8a 100644 --- a/etl/ownership/Ownership.py +++ b/etl/ownership/Ownership.py @@ -175,7 +175,7 @@ class Ownership: # Prepare the final outputs: self.create_final_matches() - def source_epc_properties(self, column_filters=None): + def source_epc_properties(self, column_filters=None, postcodes=None): """ This function will filter the epc data as specified by column filters, searching across all of the EPC tables :param column_filters: Dictionary with column names as keys and list of acceptable values as values. This @@ -183,6 +183,7 @@ class Ownership: {"column_name": ["value1", "value2", ...]}, where column_name is the name of the column in the EPC data and ["value1", "value2", ...] is a list of acceptable values for that column. If a column is not found in the EPC data, an exception is raised. + :param postcodes: A list of postcodes to filter the data on """ column_filters = {} if column_filters is None else column_filters @@ -206,6 +207,11 @@ class Ownership: else: raise Exception(f"Column {column} not found in data. column_filters is malformed") + if postcodes is not None: + epc_data = epc_data[epc_data["POSTCODE"].str.lower().isin(postcodes)] + if epc_data.empty: + continue + data.append(epc_data) self.epc_data = pd.concat(data, ignore_index=True) From 722a3dba55271454e8482c42494baa66572dec29 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 11 Oct 2024 10:16:48 +0100 Subject: [PATCH 140/166] working on gla proposal --- etl/customers/gla/proposal_investigation.py | 50 +++++++++++++++--- etl/ownership/Ownership.py | 56 +++++++++++---------- 2 files changed, 72 insertions(+), 34 deletions(-) diff --git a/etl/customers/gla/proposal_investigation.py b/etl/customers/gla/proposal_investigation.py index 776bbc59..05df6be7 100644 --- a/etl/customers/gla/proposal_investigation.py +++ b/etl/customers/gla/proposal_investigation.py @@ -10,16 +10,18 @@ from pathlib import Path from etl.ownership.Ownership import Ownership postcodes = pd.read_excel( - "/Users/khalimconn-kowlessar/Downloads/WHLG-eligible-postcodes.xlsx", sheet_name='Eligible postcodes' + "/Users/khalimconn-kowlessar/Downloads/WHLG-eligible-postcodes_RP edit.xlsx", sheet_name='Eligible postcodes' ) -# Take just the first two columns +# Take just the first three columns postcodes = postcodes[ - ['List of eligible postcodes via the IMD Income Decile 1-2 pathway', 'Unnamed: 1'] + ['List of eligible postcodes via the IMD Income Decile 1-2 pathway', 'Unnamed: 1', 'Unnamed: 2'] ] -postcodes.columns = ['postcode', 'Local Authority'] +postcodes.columns = ['postcode', 'Local Authority', 'London Borough?'] # Drop the first row postcodes = postcodes.drop([0, 1]) +# Take just the London Boroughs +postcodes = postcodes[postcodes["London Borough?"] == "Yes"] # Since there are a large number of potcodes (425k), let's just take a few examples # Take postcodes that begin with "BN15" # postcodes = postcodes[postcodes["postcode"].str.startswith("BN15")] @@ -74,22 +76,46 @@ ownership.prepare_for_matching() # Step 4: Match EPC data to ownership data ownership.match() +from utils.s3 import save_excel_to_s3 + +# Save the data to S3 +save_excel_to_s3( + df=ownership.matched_addresses, + bucket_name=ownership.bucket, + file_key=ownership.matched_addresses_pre_filter_filepath +) + # We have the matches, which we now need to match to the postcodes matches = ownership.matched_addresses.copy() # filter matches on the postcodes we're interested in matches = matches[matches["epc_postcode"].str.lower().isin(postcodes["postcode"].str.lower())] # Remove any social transactions -matches = matches[~matches["TENURE"].isin(["Rented (social)", "rental (social)"])] +matches = matches[~matches["TENURE"].isin( + ["Rented (social)", "rental (social)", + "Not defined - use in the case of a new dwelling for which the intended tenure in not known. It is not to be " + "used for an existing dwelling", "NO DATA!"]) +] +# Look at the EPC ratings +epc_ratings = matches.groupby(["CURRENT_ENERGY_RATING"]).size().reset_index() +epc_ratings.columns = ["EPC Rating", "Count"] +epc_ratings["Percentage"] = epc_ratings["Count"] / epc_ratings["Count"].sum() * 100 + +# Take properties that are below an EPC C rating, as defined by the guidance and remove any new builds +matches = matches[matches["CURRENT_ENERGY_RATING"].isin(["D", "E", "F", "G"])] +# 11,694 properties -matches.head() owners_count = matches.groupby(['Proprietor Name (1)', 'Company Registration No. (1)']).size().reset_index() owners_count.columns = ['Owner', 'Owner Registration #', 'Count'] owners_count = owners_count.sort_values('Count', ascending=False) owners_count["Percentage"] = owners_count["Count"] / owners_count["Count"].sum() * 100 +# Take an example postal region +matches = matches.sort_values("epc_postcode", ascending=True) +example = matches[matches["epc_postcode"].str.startswith("BR1 ")].copy() + companies_house_api_key = "1d9c2877-3271-4642-80ed-a6170971653f" -company_number = "13197205" +company_number = example.head(1)["Company Registration No. (1)"].values[0] url = f'https://api.company-information.service.gov.uk/company/{company_number}' # Make the API request @@ -102,7 +128,17 @@ if response.status_code == 200: print(json.dumps(company_data, indent=4)) else: print(f"Failed to fetch data. Status code: {response.status_code}") + # Try appending a zero the beginning of the company number + company_number = f"0{company_number}" + url = f'https://api.company-information.service.gov.uk/company/{company_number}' + response = requests.get(url, auth=(companies_house_api_key, '')) + company_data = response.json() + +from pprint import pprint + +pprint(company_data) psc_url = f'https://api.company-information.service.gov.uk/company/{company_number}/persons-with-significant-control' psc_response = requests.get(psc_url, auth=(companies_house_api_key, '')) psc_data = psc_response.json() +pprint(psc_data) diff --git a/etl/ownership/Ownership.py b/etl/ownership/Ownership.py index 2c04ac8a..68dee9ed 100644 --- a/etl/ownership/Ownership.py +++ b/etl/ownership/Ownership.py @@ -161,16 +161,17 @@ class Ownership: # Step 5: Match land registry data to existing matches self.match_with_land_registry() # We store this data in s3 before we perform any filtering - save_excel_to_s3( - df=self.matched_addresses, - bucket_name=self.bucket, - file_key=self.matched_addresses_pre_filter_filepath - ) - save_excel_to_s3( - df=self.combined_matching_lookup, - bucket_name=self.bucket, - file_key=self.combined_matching_lookup_pre_filter_filepath - ) + if self.save: + save_excel_to_s3( + df=self.matched_addresses, + bucket_name=self.bucket, + file_key=self.matched_addresses_pre_filter_filepath + ) + save_excel_to_s3( + df=self.combined_matching_lookup, + bucket_name=self.bucket, + file_key=self.combined_matching_lookup_pre_filter_filepath + ) # Prepare the final outputs: self.create_final_matches() @@ -1013,25 +1014,26 @@ class Ownership: if self.portfolio_properties["UPRN"].nunique() != self.portfolio_epc_data["UPRN"].nunique(): raise ValueError("Portfolio properties and epc data don't match") - logger.info("Storing final outpus") - # Store data - save_excel_to_s3( - df=self.portfolio_owners, - bucket_name=self.bucket, - file_key=self.portfolio_owners_filepath, - ) + if self.save: + logger.info("Storing final outpus") + # Store data + save_excel_to_s3( + df=self.portfolio_owners, + bucket_name=self.bucket, + file_key=self.portfolio_owners_filepath, + ) - save_excel_to_s3( - df=self.portfolio_properties, - bucket_name=self.bucket, - file_key=self.portfolio_properties_filepath, - ) + save_excel_to_s3( + df=self.portfolio_properties, + bucket_name=self.bucket, + file_key=self.portfolio_properties_filepath, + ) - save_excel_to_s3( - df=self.portfolio_epc_data, - bucket_name=self.bucket, - file_key=self.portfolio_epc_data_filepath, - ) + save_excel_to_s3( + df=self.portfolio_epc_data, + bucket_name=self.bucket, + file_key=self.portfolio_epc_data_filepath, + ) def get_asset_list(self): """ From dadbb0ef61d4cb402029158ffb0acad3cec2ad22 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 16 Oct 2024 10:19:35 +0100 Subject: [PATCH 141/166] Finished GLA proposal --- backend/Property.py | 2 +- backend/app/plan/router.py | 6 +++- backend/app/plan/schemas.py | 3 ++ backend/ml_models/Valuation.py | 2 ++ etl/customers/gla/example_model_outputs.py | 38 ++++++++++++++++++++ etl/customers/gla/proposal_investigation.py | 39 ++++++++++++++++++--- 6 files changed, 83 insertions(+), 7 deletions(-) create mode 100644 etl/customers/gla/example_model_outputs.py diff --git a/backend/Property.py b/backend/Property.py index ab8930c5..79108dc1 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -1204,7 +1204,7 @@ class Property: return False suitable_house = self.data["property-type"] == "House" and self.data["built-form"] in [ - "Detached", "Semi-Detached", + "Detached", "Semi-Detached", "End-Terrace", ] suitable_bungalow = self.data["property-type"] == "Bungalow" and self.data["built-form"] in [ diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index f4924c71..3b91a461 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -543,7 +543,11 @@ async def trigger_plan(body: PlanTriggerRequest): representative_recommendations = {} for p in tqdm(input_properties): recommender = Recommendations( - property_instance=p, materials=materials, exclusions=body.exclusions, inclusions=body.inclusions + property_instance=p, + materials=materials, + exclusions=body.exclusions, + inclusions=body.inclusions, + default_u_values=body.default_u_values ) property_recommendations, property_representative_recommendations = recommender.recommend() diff --git a/backend/app/plan/schemas.py b/backend/app/plan/schemas.py index 0d58c7e9..4b43db80 100644 --- a/backend/app/plan/schemas.py +++ b/backend/app/plan/schemas.py @@ -89,6 +89,9 @@ class PlanTriggerRequest(BaseModel): # if False, allows optimisation to be switched off optimise: Optional[bool] = True + # If True, uses default u-values for models + default_u_values: Optional[bool] = True + _allowed_goals = {"Increasing EPC"} _allowed_housing_types = {"Social", "Private"} diff --git a/backend/ml_models/Valuation.py b/backend/ml_models/Valuation.py index cbcebb9f..68432577 100644 --- a/backend/ml_models/Valuation.py +++ b/backend/ml_models/Valuation.py @@ -103,6 +103,8 @@ class PropertyValuation: # Vander Elliot Intrusive surveys 12103116: 1_537_000, 12103117: 1_404_000, + # GLA Proposal + 100020606627: 409_000 } # We base our valuation uplifts on a number of sources diff --git a/etl/customers/gla/example_model_outputs.py b/etl/customers/gla/example_model_outputs.py new file mode 100644 index 00000000..e239c43d --- /dev/null +++ b/etl/customers/gla/example_model_outputs.py @@ -0,0 +1,38 @@ +import pandas as pd +from utils.s3 import save_csv_to_s3 + +asset_list = [ + { + "address": "4, King Henrys Drive", + "postcode": "CR0 0PA" + }, +] +portfolio_id = 110 +user_id = 8 + +asset_list = pd.DataFrame(asset_list) + +filename = f"{user_id}/{portfolio_id}/asset_list.csv" +save_csv_to_s3( + dataframe=asset_list, + bucket_name="retrofit-plan-inputs-dev", + file_name=filename +) + +body1 = { + "portfolio_id": str(portfolio_id), + "housing_type": "Private", + "goal": "Increasing EPC", + "goal_value": "A", + "trigger_file_path": filename, + "already_installed_file_path": "", + "patches_file_path": "", + "non_invasive_recommendations_file_path": "", + "inclusions": [ + "cavity_wall_insulation", "loft_insulation", "air_source_heat_pump", "solar_pv" + ], + "budget": None, + "scenario_name": "Whole House", + "multi_plan": False, +} +print(body1) diff --git a/etl/customers/gla/proposal_investigation.py b/etl/customers/gla/proposal_investigation.py index 05df6be7..f6a87af1 100644 --- a/etl/customers/gla/proposal_investigation.py +++ b/etl/customers/gla/proposal_investigation.py @@ -76,13 +76,20 @@ ownership.prepare_for_matching() # Step 4: Match EPC data to ownership data ownership.match() -from utils.s3 import save_excel_to_s3 +from utils.s3 import save_excel_to_s3, read_excel_from_s3 # Save the data to S3 -save_excel_to_s3( - df=ownership.matched_addresses, +# save_excel_to_s3( +# df=ownership.matched_addresses, +# bucket_name=ownership.bucket, +# file_key=ownership.matched_addresses_pre_filter_filepath +# ) + +# Read in matches +matches = read_excel_from_s3( bucket_name=ownership.bucket, - file_key=ownership.matched_addresses_pre_filter_filepath + file_key="ownership/gla-proposal/2024-10-10 19:02:34.131365/matched_addresses_pre_filter.xlsx", + header_row=0 ) # We have the matches, which we now need to match to the postcodes @@ -95,6 +102,7 @@ matches = matches[~matches["TENURE"].isin( "Not defined - use in the case of a new dwelling for which the intended tenure in not known. It is not to be " "used for an existing dwelling", "NO DATA!"]) ] +matches["is_prs"] = matches["TENURE"].isin(["rental (private)", "Rented (private)"]) # Look at the EPC ratings epc_ratings = matches.groupby(["CURRENT_ENERGY_RATING"]).size().reset_index() epc_ratings.columns = ["EPC Rating", "Count"] @@ -103,6 +111,8 @@ epc_ratings["Percentage"] = epc_ratings["Count"] / epc_ratings["Count"].sum() * # Take properties that are below an EPC C rating, as defined by the guidance and remove any new builds matches = matches[matches["CURRENT_ENERGY_RATING"].isin(["D", "E", "F", "G"])] # 11,694 properties +matches["epc_postcode"].nunique() +# 6899 owners_count = matches.groupby(['Proprietor Name (1)', 'Company Registration No. (1)']).size().reset_index() owners_count.columns = ['Owner', 'Owner Registration #', 'Count'] @@ -111,7 +121,26 @@ owners_count["Percentage"] = owners_count["Count"] / owners_count["Count"].sum() # Take an example postal region matches = matches.sort_values("epc_postcode", ascending=True) -example = matches[matches["epc_postcode"].str.startswith("BR1 ")].copy() +# BR1, BR5 +example = matches[matches["epc_postcode"].str.startswith("CR0 ")].copy() +example = example[example["TENURE"].isin(["rental (private)", "Rented (private)"])] + +pd.set_option('display.max_rows', 500) +pd.set_option('display.max_columns', 500) +pd.set_option('display.width', 1000) +example[ + ["epc_address", "epc_postcode", "CURRENT_ENERGY_RATING", "CURRENT_ENERGY_EFFICIENCY", "Proprietor Name (1)", + "Company Registration No. (1)"] +].head(4) + +ownership.epc_data["UPRN"] = ownership.epc_data["UPRN"].astype(int) +example = example.merge( + ownership.epc_data[["UPRN", "BUILT_FORM", "PROPERTY_TYPE", "WALLS_DESCRIPTION", "ROOF_DESCRIPTION"]], + on="UPRN", + how="left" +) +z = example[example["CURRENT_ENERGY_RATING"] == "E"] +z = z[z["TENURE"].isin(["rental (private)", "Rented (private)"])] companies_house_api_key = "1d9c2877-3271-4642-80ed-a6170971653f" From 72627b5db95b88892815717d3f5ec249b6dc1713 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 16 Oct 2024 15:14:27 +0100 Subject: [PATCH 142/166] set up vectis scnearios in generate_scenarios_data --- backend/app/plan/router.py | 5 +- etl/epc/generate_scenarios_data.py | 1039 +++++++++++++++++++++++++++- 2 files changed, 1042 insertions(+), 2 deletions(-) diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 3b91a461..30834657 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -412,9 +412,12 @@ async def trigger_plan(body: PlanTriggerRequest): # We check for an energy assessment we have performed on this property: energy_assessment = get_latest_assessment_by_uprn(session, uprn if uprn is not None else epc_searcher.uprn) + if not energy_assessment["epc"]: + continue # Create a record in db property_id, is_new = create_property( - session, body.portfolio_id, epc_searcher.address_clean, epc_searcher.postcode_clean, epc_searcher.uprn, + session, body.portfolio_id, epc_searcher.address_clean, epc_searcher.postcode_clean, + epc_searcher.uprn, energy_assessment ) if not is_new and not body.multi_plan: diff --git a/etl/epc/generate_scenarios_data.py b/etl/epc/generate_scenarios_data.py index f1406759..1714fe39 100644 --- a/etl/epc/generate_scenarios_data.py +++ b/etl/epc/generate_scenarios_data.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import datetime, timezone, date import itertools import pandas as pd @@ -124,6 +124,1043 @@ scenario_properties = [ ], ], }, + # Vectis properties + { + "address": "Flat 2, Grove Mansions", + "postcode": "SW4 9SL", + "lmk-key": None, + "epc": { + 'low-energy-fixed-light-count': '18', 'address': 'Flat 2 Grove Mansions, 111, Clapham Common North Side', + 'uprn-source': 'Address Matched', 'floor-height': 3.3, 'heating-cost-potential': '696', + 'unheated-corridor-length': '8.45', 'hot-water-cost-potential': '195', + 'construction-age-band': 'England and Wales: before 1900', 'potential-energy-rating': 'C', + 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Very Poor', 'lighting-energy-eff': 'Very Good', + 'environment-impact-potential': '72', 'glazed-type': 'Single glazing', 'heating-cost-current': '1219', + 'address3': None, 'mainheatcont-description': 'Programmer, room thermostat and TRVs', + 'sheating-energy-eff': 'N/A', 'property-type': 'Flat', 'local-authority-label': 'Wandsworth', + 'fixed-lighting-outlets-count': '22', 'energy-tariff': 'Dual', 'mechanical-ventilation': 'natural', + 'hot-water-cost-current': '193', 'county': '', 'postcode': 'SW4 9SL', 'solar-water-heating-flag': 'N', + 'constituency': 'E14000549', 'co2-emissions-potential': '2.5', 'number-heated-rooms': '4', + 'floor-description': 'Suspended, no insulation (assumed)', 'energy-consumption-potential': '134', + 'local-authority': 'E09000032', 'built-form': 'Mid-Terrace', 'number-open-fireplaces': '4', + 'windows-description': 'Single glazed', 'glazed-area': 'Much More Than Typical', + 'inspection-date': datetime(2024, 8, 13, 0, 0, tzinfo=timezone.utc), + 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '38', 'address1': 'Flat 2 Grove Mansions, 111', + 'heat-loss-corridor': 'unheated corridor', 'flat-storey-count': None, 'constituency-label': 'Battersea', + 'roof-energy-eff': 'N/A', 'total-floor-area': '105', 'building-reference-number': '', + 'environment-impact-current': '56', 'co2-emissions-current': '4.0', + 'roof-description': '(another dwelling above)', 'floor-energy-eff': 'N/A', 'number-habitable-rooms': '4', + 'address2': 'Clapham Common North Side', 'hot-water-env-eff': 'Good', 'posttown': 'LONDON', + 'mainheatc-energy-eff': 'Good', 'main-fuel': 'mains gas (not community)', 'lighting-env-eff': 'Very Good', + 'windows-energy-eff': 'Very Poor', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A', + 'lighting-description': 'Low energy lighting in 82% of fixed outlets', 'roof-env-eff': 'N/A', + 'walls-energy-eff': 'Poor', 'photo-supply': '0', 'lighting-cost-potential': '164', + 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '0', + 'main-heating-controls': 'Programmer, room thermostat and TRVs', + 'lodgement-datetime': datetime(2024, 8, 13, 0, 0), 'flat-top-storey': 'N', + 'current-energy-rating': 'D', 'secondheat-description': 'None', 'walls-env-eff': 'Poor', + 'transaction-type': 'ECO assessment', 'uprn': 121016121, 'current-energy-efficiency': '63', + 'energy-consumption-current': '216', 'mainheat-description': 'Boiler and radiators, mains gas', + 'lighting-cost-current': '162', 'lodgement-date': date(2024, 8, 13), 'extension-count': '1', + 'mainheatc-env-eff': 'Good', 'lmk-key': '', 'wind-turbine-count': '0', 'tenure': 'Rented (social)', + 'floor-level': '1', 'potential-energy-efficiency': '74', 'hot-water-energy-eff': 'Good', + 'low-energy-lighting': '82', 'walls-description': 'Solid brick, as built, no insulation (assumed)', + 'hotwater-description': 'From main system' + }, + "old_epcs": [ + {'low-energy-fixed-light-count': '11', 'address': 'Flat 2 Grove Mansions, Clapham Common North Side', + 'uprn-source': 'Address Matched', 'floor-height': '', 'heating-cost-potential': '495', + 'unheated-corridor-length': '10.16', 'hot-water-cost-potential': '91', + 'construction-age-band': 'England and Wales: 1900-1929', 'potential-energy-rating': 'C', + 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Very Poor', 'lighting-energy-eff': 'Very Good', + 'environment-impact-potential': '75', 'glazed-type': 'not defined', 'heating-cost-current': '932', + 'address3': '', 'mainheatcont-description': 'Programmer, room thermostat and TRVs', + 'sheating-energy-eff': 'N/A', 'property-type': 'Flat', 'local-authority-label': 'Wandsworth', + 'fixed-lighting-outlets-count': '11', 'energy-tariff': 'Unknown', 'mechanical-ventilation': 'natural', + 'hot-water-cost-current': '90', 'county': 'Greater London Authority', 'postcode': 'SW4 9SL', + 'solar-water-heating-flag': '', 'constituency': 'E14000549', 'co2-emissions-potential': '2.9', + 'number-heated-rooms': '5', 'floor-description': 'Suspended, no insulation (assumed)', + 'energy-consumption-potential': '127', 'local-authority': 'E09000032', 'built-form': 'Mid-Terrace', + 'number-open-fireplaces': '1', 'windows-description': 'Single glazed', 'glazed-area': 'Normal', + 'inspection-date': '2012-11-28', 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '45', + 'address1': 'Flat 2 Grove Mansions', 'heat-loss-corridor': 'unheated corridor', 'flat-storey-count': '', + 'constituency-label': 'Battersea', 'roof-energy-eff': 'N/A', 'total-floor-area': '121.0', + 'building-reference-number': '9347419868', 'environment-impact-current': '53', + 'co2-emissions-current': '5.5', 'roof-description': '(another dwelling above)', 'floor-energy-eff': 'N/A', + 'number-habitable-rooms': '5', 'address2': 'Clapham Common North Side', 'hot-water-env-eff': 'Good', + 'posttown': 'LONDON', 'mainheatc-energy-eff': 'Good', 'main-fuel': 'mains gas (not community)', + 'lighting-env-eff': 'Very Good', 'windows-energy-eff': 'Very Poor', 'floor-env-eff': 'N/A', + 'sheating-env-eff': 'N/A', 'lighting-description': 'Low energy lighting in all fixed outlets', + 'roof-env-eff': 'N/A', 'walls-energy-eff': 'Very Poor', 'photo-supply': '0.0', + 'lighting-cost-potential': '64', 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '0', + 'main-heating-controls': '2106', 'lodgement-datetime': '2012-11-29 00:17:32', 'flat-top-storey': 'N', + 'current-energy-rating': 'D', 'secondheat-description': 'None', 'walls-env-eff': 'Very Poor', + 'transaction-type': 'rental (private)', 'uprn': '121016121', 'current-energy-efficiency': '59', + 'energy-consumption-current': '234', 'mainheat-description': 'Boiler and radiators, mains gas', + 'lighting-cost-current': '62', 'lodgement-date': '2012-11-29', 'extension-count': '0', + 'mainheatc-env-eff': 'Good', 'lmk-key': '664404619342012112900173280922988', 'wind-turbine-count': '0', + 'tenure': 'rental (private)', 'floor-level': 'Basement', 'potential-energy-efficiency': '76', + 'hot-water-energy-eff': 'Good', 'low-energy-lighting': '100', + 'walls-description': 'Solid brick, as built, no insulation (assumed)', + 'hotwater-description': 'From main system'}, + {'low-energy-fixed-light-count': '5', 'address': 'Flat 2 Grove Mansions, Clapham Common North Side', + 'uprn-source': 'Address Matched', 'floor-height': '3.04', 'heating-cost-potential': '778', + 'unheated-corridor-length': '6.83', 'hot-water-cost-potential': '89', + 'construction-age-band': 'England and Wales: before 1900', 'potential-energy-rating': 'D', + 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Very Poor', 'lighting-energy-eff': 'Average', + 'environment-impact-potential': '61', 'glazed-type': 'not defined', 'heating-cost-current': '773', + 'address3': '', 'mainheatcont-description': 'Programmer, room thermostat and TRVs', + 'sheating-energy-eff': 'N/A', 'property-type': 'Flat', 'local-authority-label': 'Wandsworth', + 'fixed-lighting-outlets-count': '12', 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', + 'hot-water-cost-current': '89', 'county': 'Greater London Authority', 'postcode': 'SW4 9SL', + 'solar-water-heating-flag': '', 'constituency': 'E14000549', 'co2-emissions-potential': '4.6', + 'number-heated-rooms': '5', 'floor-description': 'Solid, no insulation (assumed)', + 'energy-consumption-potential': '210', 'local-authority': 'E09000032', 'built-form': 'Semi-Detached', + 'number-open-fireplaces': '0', 'windows-description': 'Single glazed', 'glazed-area': 'Normal', + 'inspection-date': '2011-08-09', 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '41', + 'address1': 'Flat 2 Grove Mansions', 'heat-loss-corridor': 'unheated corridor', 'flat-storey-count': '', + 'constituency-label': 'Battersea', 'roof-energy-eff': 'N/A', 'total-floor-area': '113.52', + 'building-reference-number': '9347419868', 'environment-impact-current': '60', + 'co2-emissions-current': '4.7', 'roof-description': '(another dwelling above)', 'floor-energy-eff': 'N/A', + 'number-habitable-rooms': '5', 'address2': 'Clapham Common North Side', 'hot-water-env-eff': 'Good', + 'posttown': 'LONDON', 'mainheatc-energy-eff': 'Good', 'main-fuel': 'mains gas (not community)', + 'lighting-env-eff': 'Average', 'windows-energy-eff': 'Very Poor', 'floor-env-eff': 'N/A', + 'sheating-env-eff': 'N/A', 'lighting-description': 'Low energy lighting in 42% of fixed outlets', + 'roof-env-eff': 'N/A', 'walls-energy-eff': 'Very Poor', 'photo-supply': '0.0', + 'lighting-cost-potential': '58', 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '0', + 'main-heating-controls': '2106', 'lodgement-datetime': '2011-08-09 14:08:07', 'flat-top-storey': 'N', + 'current-energy-rating': 'D', 'secondheat-description': 'None', 'walls-env-eff': 'Very Poor', + 'transaction-type': 'rental (private)', 'uprn': '121016121', 'current-energy-efficiency': '64', + 'energy-consumption-current': '216', 'mainheat-description': 'Boiler and radiators, mains gas', + 'lighting-cost-current': '91', 'lodgement-date': '2011-08-09', 'extension-count': '0', + 'mainheatc-env-eff': 'Good', 'lmk-key': '664404649022011080914080739118499', 'wind-turbine-count': '0', + 'tenure': 'rental (private)', 'floor-level': 'Ground', 'potential-energy-efficiency': '65', + 'hot-water-energy-eff': 'Good', 'low-energy-lighting': '42', + 'walls-description': 'Solid brick, as built, no insulation (assumed)', + 'hotwater-description': 'From main system'}, + {'low-energy-fixed-light-count': '', 'address': 'Flat 2, Grove Mansions, 111 Clapham Common North Side', + 'uprn-source': 'Energy Assessor', 'floor-height': '2.24', 'heating-cost-potential': '433', + 'unheated-corridor-length': '13.31', 'hot-water-cost-potential': '99', + 'construction-age-band': 'England and Wales: before 1900', 'potential-energy-rating': 'C', + 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Very Poor', 'lighting-energy-eff': 'Very Good', + 'environment-impact-potential': '73', 'glazed-type': 'not defined', 'heating-cost-current': '1013', + 'address3': '111 Clapham Common North Side', + 'mainheatcont-description': 'Programmer, room thermostat and TRVs', 'sheating-energy-eff': 'N/A', + 'property-type': 'Flat', 'local-authority-label': 'Wandsworth', 'fixed-lighting-outlets-count': '9', + 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '98', + 'county': '', 'postcode': 'SW4 9SL', 'solar-water-heating-flag': 'N', 'constituency': 'E14000549', + 'co2-emissions-potential': '2.7', 'number-heated-rooms': '5', + 'floor-description': 'Solid, no insulation (assumed)', 'energy-consumption-potential': '141', + 'local-authority': 'E09000032', 'built-form': 'Semi-Detached', 'number-open-fireplaces': '0', + 'windows-description': 'Single glazed', 'glazed-area': 'Normal', 'inspection-date': '2022-10-25', + 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '56', 'address1': 'Flat 2', + 'heat-loss-corridor': 'unheated corridor', 'flat-storey-count': '', 'constituency-label': 'Battersea', + 'roof-energy-eff': 'N/A', 'total-floor-area': '106.0', 'building-reference-number': '10003506322', + 'environment-impact-current': '41', 'co2-emissions-current': '6.0', + 'roof-description': '(another dwelling above)', 'floor-energy-eff': 'N/A', 'number-habitable-rooms': '5', + 'address2': 'Grove Mansions', 'hot-water-env-eff': 'Good', 'posttown': 'LONDON', + 'mainheatc-energy-eff': 'Good', 'main-fuel': 'mains gas (not community)', 'lighting-env-eff': 'Very Good', + 'windows-energy-eff': 'Very Poor', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A', + 'lighting-description': 'Low energy lighting in all fixed outlets', 'roof-env-eff': 'N/A', + 'walls-energy-eff': 'Very Poor', 'photo-supply': '0.0', 'lighting-cost-potential': '93', + 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '0', 'main-heating-controls': '', + 'lodgement-datetime': '2022-10-26 10:18:37', 'flat-top-storey': 'N', 'current-energy-rating': 'E', + 'secondheat-description': 'None', 'walls-env-eff': 'Very Poor', 'transaction-type': 'marketed sale', + 'uprn': '121016121', 'current-energy-efficiency': '50', 'energy-consumption-current': '318', + 'mainheat-description': 'Boiler and radiators, mains gas', 'lighting-cost-current': '91', + 'lodgement-date': '2022-10-26', 'extension-count': '0', 'mainheatc-env-eff': 'Good', + 'lmk-key': '33092029f56530df7c284290fd45a8a3e985f42e3bfb6f6f85785c8accc42445', 'wind-turbine-count': '0', + 'tenure': 'Rented (private)', 'floor-level': '00', 'potential-energy-efficiency': '75', + 'hot-water-energy-eff': 'Good', 'low-energy-lighting': '100', + 'walls-description': 'Solid brick, as built, no insulation (assumed)', + 'hotwater-description': 'From main system'} + ], + "measures": [ + [["secondary_glazing"], "2", {}, [0]], + [ + ["secondary_glazing", "low_energy_lighting", "internal_wall_insulation", "suspended_floor_insulation"], + "8", + {}, + [0, 1, 2, 3] + ], + [ + ["double_glazing", "internal_wall_insulation", "suspended_floor_insulation"], + "11", + {}, + [0, 1, 2] + ], + ], + }, + + { + "address": "Flat 8 Grove Mansions", + "postcode": "SW4 9SL", + "lmk-key": None, + "epc": { + 'low-energy-fixed-light-count': '10', + 'address': 'Flat 8 Grove Mansions, 111, Clapham Common North Side', 'uprn-source': 'Address Matched', + 'floor-height': 2.75, 'heating-cost-potential': '319', 'unheated-corridor-length': '3.9', + 'hot-water-cost-potential': '180', 'construction-age-band': 'England and Wales: before 1900', + 'potential-energy-rating': 'B', 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Very Poor', + 'lighting-energy-eff': 'Very Good', 'environment-impact-potential': '84', + 'glazed-type': 'Single glazing', 'heating-cost-current': '666', 'address3': None, + 'mainheatcont-description': 'Programmer and room thermostat', 'sheating-energy-eff': 'N/A', + 'property-type': 'Flat', 'local-authority-label': 'Wandsworth', 'fixed-lighting-outlets-count': '14', + 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '176', + 'county': '', 'postcode': 'SW4 9SL', 'solar-water-heating-flag': 'N', 'constituency': 'E14000549', + 'co2-emissions-potential': '1.2', 'number-heated-rooms': '3', + 'floor-description': '(another dwelling below)', 'energy-consumption-potential': '96', + 'local-authority': 'E09000032', 'built-form': 'Mid-Terrace', 'number-open-fireplaces': '1', + 'windows-description': 'Single glazed', 'glazed-area': 'Much More Than Typical', + 'inspection-date': datetime(2024, 8, 13, 0, 0, tzinfo=timezone.utc), + 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '30', 'address1': 'Flat 8 Grove Mansions, 111', + 'heat-loss-corridor': 'unheated corridor', 'flat-storey-count': None, 'constituency-label': 'Battersea', + 'roof-energy-eff': 'N/A', 'total-floor-area': '72', 'building-reference-number': '', + 'environment-impact-current': '70', 'co2-emissions-current': '2.2', + 'roof-description': '(another dwelling above)', 'floor-energy-eff': 'N/A', + 'number-habitable-rooms': '3', 'address2': 'Clapham Common North Side', 'hot-water-env-eff': 'Good', + 'posttown': 'LONDON', 'mainheatc-energy-eff': 'Average', 'main-fuel': 'mains gas (not community)', + 'lighting-env-eff': 'Very Good', 'windows-energy-eff': 'Very Poor', 'floor-env-eff': 'N/A', + 'sheating-env-eff': 'N/A', 'lighting-description': 'Low energy lighting in 71% of fixed outlets', + 'roof-env-eff': 'N/A', 'walls-energy-eff': 'Very Poor', 'photo-supply': '0', + 'lighting-cost-potential': '93', 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '0', + 'main-heating-controls': 'Programmer and room thermostat', + 'lodgement-datetime': datetime(2024, 8, 13, 0, 0), 'flat-top-storey': 'N', + 'current-energy-rating': 'C', 'secondheat-description': 'Room heaters, electric', + 'walls-env-eff': 'Very Poor', 'transaction-type': 'ECO assessment', 'uprn': 10024087855, + 'current-energy-efficiency': '70', 'energy-consumption-current': '170', + 'mainheat-description': 'Boiler and radiators, mains gas', 'lighting-cost-current': '118', + 'lodgement-date': date(2024, 8, 13), 'extension-count': '0', 'mainheatc-env-eff': 'Average', + 'lmk-key': '', 'wind-turbine-count': '0', 'tenure': 'Rented (social)', 'floor-level': '2', + 'potential-energy-efficiency': '81', 'hot-water-energy-eff': 'Good', 'low-energy-lighting': '71', + 'walls-description': 'Solid brick, as built, no insulation (assumed)', + 'hotwater-description': 'From main system' + }, + "old_epcs": [ + {'low-energy-fixed-light-count': '', 'address': 'Flat 8 Grove Mansions, Clapham Common North Side', + 'uprn-source': 'Address Matched', 'floor-height': '3.07', 'heating-cost-potential': '385', + 'unheated-corridor-length': '4.5', 'hot-water-cost-potential': '100', + 'construction-age-band': 'England and Wales: before 1900', 'potential-energy-rating': 'C', + 'mainheat-energy-eff': 'Very Good', 'windows-env-eff': 'Very Poor', 'lighting-energy-eff': 'Very Good', + 'environment-impact-potential': '75', 'glazed-type': 'not defined', 'heating-cost-current': '405', + 'address3': '', 'mainheatcont-description': 'Programmer, TRVs and bypass', 'sheating-energy-eff': 'N/A', + 'property-type': 'Flat', 'local-authority-label': 'Wandsworth', 'fixed-lighting-outlets-count': '', + 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '106', + 'county': 'Greater London Authority', 'postcode': 'SW4 9SL', 'solar-water-heating-flag': 'N', + 'constituency': 'E14000549', 'co2-emissions-potential': '2.4', 'number-heated-rooms': '3', + 'floor-description': '(other premises below)', 'energy-consumption-potential': '169', + 'local-authority': 'E09000032', 'built-form': 'Mid-Terrace', 'number-open-fireplaces': '0', + 'windows-description': 'Single glazed', 'glazed-area': 'Normal', 'inspection-date': '2010-12-07', + 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '30', 'address1': 'Flat 8 Grove Mansions', + 'heat-loss-corridor': 'unheated corridor', 'flat-storey-count': '5.0', 'constituency-label': 'Battersea', + 'roof-energy-eff': 'N/A', 'total-floor-area': '85.53', 'building-reference-number': '5346736568', + 'environment-impact-current': '74', 'co2-emissions-current': '2.6', + 'roof-description': '(another dwelling above)', 'floor-energy-eff': 'N/A', 'number-habitable-rooms': '3', + 'address2': 'Clapham Common North Side', 'hot-water-env-eff': 'Very Good', 'posttown': 'LONDON', + 'mainheatc-energy-eff': 'Average', + 'main-fuel': 'mains gas - this is for backwards compatibility only and should not be used', + 'lighting-env-eff': 'Very Good', 'windows-energy-eff': 'Very Poor', 'floor-env-eff': 'N/A', + 'sheating-env-eff': 'N/A', 'lighting-description': 'Low energy lighting in all fixed outlets', + 'roof-env-eff': 'N/A', 'walls-energy-eff': 'Very Poor', 'photo-supply': '0.0', + 'lighting-cost-potential': '47', 'mainheat-env-eff': 'Very Good', 'multi-glaze-proportion': '0', + 'main-heating-controls': '2107', 'lodgement-datetime': '2010-12-13 09:01:15', 'flat-top-storey': 'N', + 'current-energy-rating': 'C', 'secondheat-description': 'None', 'walls-env-eff': 'Very Poor', + 'transaction-type': 'rental (private)', 'uprn': '10024087855', 'current-energy-efficiency': '77', + 'energy-consumption-current': '179', 'mainheat-description': 'Boiler and radiators, mains gas', + 'lighting-cost-current': '47', 'lodgement-date': '2010-12-13', 'extension-count': '0', + 'mainheatc-env-eff': 'Average', 'lmk-key': '213678952032010121309011556068793', 'wind-turbine-count': '0', + 'tenure': 'rental (private)', 'floor-level': '3rd', 'potential-energy-efficiency': '78', + 'hot-water-energy-eff': 'Very Good', 'low-energy-lighting': '100', + 'walls-description': 'Solid brick, as built, no insulation (assumed)', + 'hotwater-description': 'From main system'}, + {'low-energy-fixed-light-count': '', 'address': 'Flat 8 Grove Mansions, Clapham Common North Side', + 'uprn-source': 'Address Matched', 'floor-height': '2.95', 'heating-cost-potential': '385', + 'unheated-corridor-length': '', 'hot-water-cost-potential': '85', 'construction-age-band': 'NO DATA!', + 'potential-energy-rating': 'C', 'mainheat-energy-eff': 'Very Good', 'windows-env-eff': 'Very Poor', + 'lighting-energy-eff': 'Poor', 'environment-impact-potential': '71', 'glazed-type': 'NO DATA!', + 'heating-cost-current': '380', 'address3': '', + 'mainheatcont-description': 'Programmer, room thermostat and TRVs', 'sheating-energy-eff': 'N/A', + 'property-type': 'Flat', 'local-authority-label': 'Wandsworth', 'fixed-lighting-outlets-count': '', + 'energy-tariff': 'standard tariff', 'mechanical-ventilation': 'NO DATA!', 'hot-water-cost-current': '85', + 'county': 'Greater London Authority', 'postcode': 'SW4 9SL', 'solar-water-heating-flag': '', + 'constituency': 'E14000549', 'co2-emissions-potential': '2.4', 'number-heated-rooms': '', + 'floor-description': '(other premises below)', 'energy-consumption-potential': '211', + 'local-authority': 'E09000032', 'built-form': 'Mid-Terrace', 'number-open-fireplaces': '', + 'windows-description': 'Single glazed', 'glazed-area': 'NO DATA!', 'inspection-date': '2009-10-16', + 'mains-gas-flag': '', 'co2-emiss-curr-per-floor-area': '36', 'address1': 'Flat 8 Grove Mansions', + 'heat-loss-corridor': 'NO DATA!', 'flat-storey-count': '', 'constituency-label': 'Battersea', + 'roof-energy-eff': 'Very Good', 'total-floor-area': '69.03', 'building-reference-number': '5346736568', + 'environment-impact-current': '71', 'co2-emissions-current': '2.5', + 'roof-description': 'Average thermal transmittance 0.06 W/m?K', 'floor-energy-eff': 'N/A', + 'number-habitable-rooms': '', 'address2': 'Clapham Common North Side', 'hot-water-env-eff': 'Very Good', + 'posttown': 'LONDON', 'mainheatc-energy-eff': 'Average', + 'main-fuel': 'mains gas - this is for backwards compatibility only and should not be used', + 'lighting-env-eff': 'Poor', 'windows-energy-eff': 'Very Poor', 'floor-env-eff': 'N/A', + 'sheating-env-eff': 'N/A', 'lighting-description': 'Low energy lighting in 17% of fixed outlets', + 'roof-env-eff': 'Very Good', 'walls-energy-eff': 'Very Poor', 'photo-supply': '', + 'lighting-cost-potential': '36', 'mainheat-env-eff': 'Very Good', 'multi-glaze-proportion': '', + 'main-heating-controls': '', 'lodgement-datetime': '2009-10-16 13:30:30', 'flat-top-storey': '', + 'current-energy-rating': 'C', 'secondheat-description': 'None', 'walls-env-eff': 'Very Poor', + 'transaction-type': 'new dwelling', 'uprn': '10024087855', 'current-energy-efficiency': '73', + 'energy-consumption-current': '219', 'mainheat-description': 'Boiler and radiators, mains gas', + 'lighting-cost-current': '66', 'lodgement-date': '2009-10-16', 'extension-count': '', + 'mainheatc-env-eff': 'Average', 'lmk-key': '213678970922009101613303036468251', 'wind-turbine-count': '0', + 'tenure': '', 'floor-level': 'mid floor', 'potential-energy-efficiency': '75', + 'hot-water-energy-eff': 'Very Good', 'low-energy-lighting': '', + 'walls-description': 'Average thermal transmittance 1.84 W/m?K', + 'hotwater-description': 'From main system'}, + {'low-energy-fixed-light-count': '', 'address': 'Flat 8 Grove Mansions, Clapham Common North Side', + 'uprn-source': 'Address Matched', 'floor-height': '2.95', 'heating-cost-potential': '251', + 'unheated-corridor-length': '', 'hot-water-cost-potential': '85', 'construction-age-band': 'NO DATA!', + 'potential-energy-rating': 'B', 'mainheat-energy-eff': 'Very Good', 'windows-env-eff': 'Good', + 'lighting-energy-eff': 'Poor', 'environment-impact-potential': '82', 'glazed-type': 'NO DATA!', + 'heating-cost-current': '247', 'address3': '', + 'mainheatcont-description': 'Programmer, room thermostat and TRVs', 'sheating-energy-eff': 'N/A', + 'property-type': 'Flat', 'local-authority-label': 'Wandsworth', 'fixed-lighting-outlets-count': '', + 'energy-tariff': 'standard tariff', 'mechanical-ventilation': 'NO DATA!', 'hot-water-cost-current': '85', + 'county': 'Greater London Authority', 'postcode': 'SW4 9SL', 'solar-water-heating-flag': '', + 'constituency': 'E14000549', 'co2-emissions-potential': '1.6', 'number-heated-rooms': '', + 'floor-description': '(other premises below)', 'energy-consumption-potential': '137', + 'local-authority': 'E09000032', 'built-form': 'Mid-Terrace', 'number-open-fireplaces': '', + 'windows-description': 'Fully double glazed', 'glazed-area': 'NO DATA!', 'inspection-date': '2009-10-12', + 'mains-gas-flag': '', 'co2-emiss-curr-per-floor-area': '24', 'address1': 'Flat 8 Grove Mansions', + 'heat-loss-corridor': 'NO DATA!', 'flat-storey-count': '', 'constituency-label': 'Battersea', + 'roof-energy-eff': 'Very Good', 'total-floor-area': '69.03', 'building-reference-number': '5346736568', + 'environment-impact-current': '81', 'co2-emissions-current': '1.7', + 'roof-description': 'Average thermal transmittance 0.06 W/m?K', 'floor-energy-eff': 'N/A', + 'number-habitable-rooms': '', 'address2': 'Clapham Common North Side', 'hot-water-env-eff': 'Very Good', + 'posttown': 'LONDON', 'mainheatc-energy-eff': 'Average', + 'main-fuel': 'mains gas - this is for backwards compatibility only and should not be used', + 'lighting-env-eff': 'Poor', 'windows-energy-eff': 'Good', 'floor-env-eff': 'N/A', + 'sheating-env-eff': 'N/A', 'lighting-description': 'Low energy lighting in 17% of fixed outlets', + 'roof-env-eff': 'Very Good', 'walls-energy-eff': 'Good', 'photo-supply': '', + 'lighting-cost-potential': '37', 'mainheat-env-eff': 'Very Good', 'multi-glaze-proportion': '', + 'main-heating-controls': '', 'lodgement-datetime': '2009-10-12 10:59:43', 'flat-top-storey': '', + 'current-energy-rating': 'B', 'secondheat-description': 'None', 'walls-env-eff': 'Good', + 'transaction-type': 'new dwelling', 'uprn': '10024087855', 'current-energy-efficiency': '81', + 'energy-consumption-current': '145', 'mainheat-description': 'Boiler and radiators, mains gas', + 'lighting-cost-current': '68', 'lodgement-date': '2009-10-12', 'extension-count': '', + 'mainheatc-env-eff': 'Average', 'lmk-key': '213678930302009101210594355619328', 'wind-turbine-count': '0', + 'tenure': '', 'floor-level': 'mid floor', 'potential-energy-efficiency': '83', + 'hot-water-energy-eff': 'Very Good', 'low-energy-lighting': '', + 'walls-description': 'Average thermal transmittance 0.41 W/m?K', + 'hotwater-description': 'From main system'}, + {'low-energy-fixed-light-count': '', 'address': 'Flat 8 Grove Mansions, Clapham Common North Side', + 'uprn-source': 'Address Matched', 'floor-height': '2.75', 'heating-cost-potential': '936', + 'unheated-corridor-length': '15.15', 'hot-water-cost-potential': '97', + 'construction-age-band': 'England and Wales: before 1900', 'potential-energy-rating': 'E', + 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Very Poor', 'lighting-energy-eff': 'Average', + 'environment-impact-potential': '46', 'glazed-type': 'single glazing', 'heating-cost-current': '927', + 'address3': '', 'mainheatcont-description': 'Programmer and room thermostat', 'sheating-energy-eff': 'N/A', + 'property-type': 'Flat', 'local-authority-label': 'Wandsworth', 'fixed-lighting-outlets-count': '', + 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '97', + 'county': 'Greater London Authority', 'postcode': 'SW4 9SL', 'solar-water-heating-flag': 'N', + 'constituency': 'E14000549', 'co2-emissions-potential': '5.9', 'number-heated-rooms': '5', + 'floor-description': '(other premises below)', 'energy-consumption-potential': '374', + 'local-authority': 'E09000032', 'built-form': 'Mid-Terrace', 'number-open-fireplaces': '0', + 'windows-description': 'Single glazed', 'glazed-area': 'More Than Typical', + 'inspection-date': '2009-01-12', 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '63', + 'address1': 'Flat 8 Grove Mansions', 'heat-loss-corridor': 'unheated corridor', 'flat-storey-count': '4.0', + 'constituency-label': 'Battersea', 'roof-energy-eff': 'Very Poor', 'total-floor-area': '96.03', + 'building-reference-number': '5346736568', 'environment-impact-current': '45', + 'co2-emissions-current': '6.0', 'roof-description': 'Pitched, no insulation (assumed)', + 'floor-energy-eff': 'N/A', 'number-habitable-rooms': '5', 'address2': 'Clapham Common North Side', + 'hot-water-env-eff': 'Very Good', 'posttown': 'LONDON', 'mainheatc-energy-eff': 'Poor', + 'main-fuel': 'mains gas - this is for backwards compatibility only and should not be used', + 'lighting-env-eff': 'Average', 'windows-energy-eff': 'Very Poor', 'floor-env-eff': 'N/A', + 'sheating-env-eff': 'N/A', 'lighting-description': 'Low energy lighting in 25% of fixed outlets', + 'roof-env-eff': 'Very Poor', 'walls-energy-eff': 'Very Poor', 'photo-supply': '0.0', + 'lighting-cost-potential': '46', 'mainheat-env-eff': 'Very Good', 'multi-glaze-proportion': '0', + 'main-heating-controls': '2104', 'lodgement-datetime': '2009-01-13 11:24:12', 'flat-top-storey': 'Y', + 'current-energy-rating': 'E', 'secondheat-description': 'Room heaters, electric', + 'walls-env-eff': 'Very Poor', 'transaction-type': 'rental (private)', 'uprn': '10024087855', + 'current-energy-efficiency': '45', 'energy-consumption-current': '380', + 'mainheat-description': 'Boiler and radiators, mains gas', 'lighting-cost-current': '80', + 'lodgement-date': '2009-01-13', 'extension-count': '0', 'mainheatc-env-eff': 'Poor', + 'lmk-key': '213678970922009011311241236228851', 'wind-turbine-count': '0', 'tenure': 'rental (private)', + 'floor-level': '3rd', 'potential-energy-efficiency': '47', 'hot-water-energy-eff': 'Good', + 'low-energy-lighting': '25', 'walls-description': 'Solid brick, as built, no insulation (assumed)', + 'hotwater-description': 'From main system'}, + {'low-energy-fixed-light-count': '', 'address': 'Flat 8 Grove Mansions, Clapham Common North Side', + 'uprn-source': 'Address Matched', 'floor-height': '2.71', 'heating-cost-potential': '220', + 'unheated-corridor-length': '5.17', 'hot-water-cost-potential': '90', + 'construction-age-band': 'England and Wales: before 1900', 'potential-energy-rating': 'B', + 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Very Poor', 'lighting-energy-eff': 'Very Good', + 'environment-impact-potential': '83', 'glazed-type': 'not defined', 'heating-cost-current': '418', + 'address3': '', 'mainheatcont-description': 'Programmer, room thermostat and TRVs', + 'sheating-energy-eff': 'N/A', 'property-type': 'Flat', 'local-authority-label': 'Wandsworth', + 'fixed-lighting-outlets-count': '6', 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', + 'hot-water-cost-current': '90', 'county': '', 'postcode': 'SW4 9SL', 'solar-water-heating-flag': 'N', + 'constituency': 'E14000549', 'co2-emissions-potential': '1.3', 'number-heated-rooms': '3', + 'floor-description': '(another dwelling below)', 'energy-consumption-potential': '101', + 'local-authority': 'E09000032', 'built-form': 'Mid-Terrace', 'number-open-fireplaces': '1', + 'windows-description': 'Single glazed', 'glazed-area': 'Normal', 'inspection-date': '2021-04-20', + 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '33', 'address1': 'Flat 8 Grove Mansions', + 'heat-loss-corridor': 'unheated corridor', 'flat-storey-count': '', 'constituency-label': 'Battersea', + 'roof-energy-eff': 'N/A', 'total-floor-area': '74.0', 'building-reference-number': '5346736568', + 'environment-impact-current': '68', 'co2-emissions-current': '2.4', + 'roof-description': '(another dwelling above)', 'floor-energy-eff': 'N/A', 'number-habitable-rooms': '3', + 'address2': 'Clapham Common North Side', 'hot-water-env-eff': 'Good', 'posttown': 'LONDON', + 'mainheatc-energy-eff': 'Good', 'main-fuel': 'mains gas (not community)', 'lighting-env-eff': 'Very Good', + 'windows-energy-eff': 'Very Poor', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A', + 'lighting-description': 'Low energy lighting in all fixed outlets', 'roof-env-eff': 'N/A', + 'walls-energy-eff': 'Very Poor', 'photo-supply': '0.0', 'lighting-cost-potential': '64', + 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '0', 'main-heating-controls': '', + 'lodgement-datetime': '2021-04-20 17:04:30', 'flat-top-storey': 'N', 'current-energy-rating': 'C', + 'secondheat-description': 'None', 'walls-env-eff': 'Very Poor', 'transaction-type': 'rental', + 'uprn': '10024087855', 'current-energy-efficiency': '70', 'energy-consumption-current': '185', + 'mainheat-description': 'Boiler and radiators, mains gas', 'lighting-cost-current': '63', + 'lodgement-date': '2021-04-20', 'extension-count': '0', 'mainheatc-env-eff': 'Good', + 'lmk-key': '9ad43b42840825a143c7dad8ca36a199f15a72c4df9dd2f07696cc087f04e1e3', 'wind-turbine-count': '0', + 'tenure': 'Rented (private)', 'floor-level': '03', 'potential-energy-efficiency': '81', + 'hot-water-energy-eff': 'Good', 'low-energy-lighting': '100', + 'walls-description': 'Solid brick, as built, no insulation (assumed)', + 'hotwater-description': 'From main system'} + ], + "measures": [ + [["secondary_glazing", "low_energy_lighting"], "3", {}, [0, 1]], + [ + ["secondary_glazing", "low_energy_lighting", "internal_wall_insulation"], + "8", + {}, + [0, 1, 2] + ], + [ + ["double_glazing", "internal_wall_insulation", "low_energy_lighting"], + "9", + {}, + [0, 1, 2] + ], + ], + }, + + { + "address": "Flat 9 Grove Mansions", + "postcode": "SW4 9SL", + "lmk-key": None, + "epc": { + 'low-energy-fixed-light-count': '10', 'address': 'Flat 9 Grove Mansions, 111, Clapham Common North Side', + 'uprn-source': 'Address Matched', 'floor-height': 3.25, 'heating-cost-potential': '692', + 'unheated-corridor-length': '8.45', 'hot-water-cost-potential': '195', + 'construction-age-band': 'England and Wales: before 1900', 'potential-energy-rating': 'C', + 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Very Poor', 'lighting-energy-eff': 'Good', + 'environment-impact-potential': '73', 'glazed-type': 'Single glazing', 'heating-cost-current': '1206', + 'address3': None, 'mainheatcont-description': 'Programmer, room thermostat and TRVs', + 'sheating-energy-eff': 'N/A', 'property-type': 'Flat', 'local-authority-label': 'Wandsworth', + 'fixed-lighting-outlets-count': '18', 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', + 'hot-water-cost-current': '193', 'county': '', 'postcode': 'SW4 9SL', 'solar-water-heating-flag': 'N', + 'constituency': 'E14000549', 'co2-emissions-potential': '2.5', 'number-heated-rooms': '4', + 'floor-description': 'Suspended, no insulation (assumed)', 'energy-consumption-potential': '131', + 'local-authority': 'E09000032', 'built-form': 'Mid-Terrace', 'number-open-fireplaces': '4', + 'windows-description': 'Single glazed', 'glazed-area': 'Much More Than Typical', + 'inspection-date': datetime(2024, 8, 13, 0, 0, tzinfo=timezone.utc), + 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '38', 'address1': 'Flat 9 Grove Mansions, 111', + 'heat-loss-corridor': 'unheated corridor', 'flat-storey-count': None, 'constituency-label': 'Battersea', + 'roof-energy-eff': 'N/A', 'total-floor-area': '106', 'building-reference-number': '', + 'environment-impact-current': '56', 'co2-emissions-current': '4.0', + 'roof-description': '(another dwelling above)', 'floor-energy-eff': 'N/A', 'number-habitable-rooms': '4', + 'address2': 'Clapham Common North Side', 'hot-water-env-eff': 'Good', 'posttown': 'LONDON', + 'mainheatc-energy-eff': 'Good', 'main-fuel': 'mains gas (not community)', 'lighting-env-eff': 'Good', + 'windows-energy-eff': 'Very Poor', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A', + 'lighting-description': 'Low energy lighting in 56% of fixed outlets', 'roof-env-eff': 'N/A', + 'walls-energy-eff': 'Poor', 'photo-supply': '0', 'lighting-cost-potential': '122', + 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '0', + 'main-heating-controls': 'Programmer, room thermostat and TRVs', + 'lodgement-datetime': datetime(2024, 8, 13, 0, 0), 'flat-top-storey': 'N', + 'current-energy-rating': 'D', 'secondheat-description': 'None', 'walls-env-eff': 'Poor', + 'transaction-type': 'ECO assessment', 'uprn': 121016128, 'current-energy-efficiency': '63', + 'energy-consumption-current': '216', 'mainheat-description': 'Boiler and radiators, mains gas', + 'lighting-cost-current': '174', 'lodgement-date': date(2024, 8, 13), 'extension-count': '1', + 'mainheatc-env-eff': 'Good', 'lmk-key': '', 'wind-turbine-count': '0', 'tenure': 'Rented (social)', + 'floor-level': '1', 'potential-energy-efficiency': '75', 'hot-water-energy-eff': 'Good', + 'low-energy-lighting': '56', 'walls-description': 'Solid brick, as built, no insulation (assumed)', + 'hotwater-description': 'From main system' + }, + "old_epcs": [ + {'low-energy-fixed-light-count': '10', 'address': 'Flat 9 Grove Mansions, Clapham Common North Side', + 'uprn-source': 'Address Matched', 'floor-height': '', 'heating-cost-potential': '419', + 'unheated-corridor-length': '10.99', 'hot-water-cost-potential': '91', + 'construction-age-band': 'England and Wales: 1900-1929', 'potential-energy-rating': 'C', + 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Very Poor', 'lighting-energy-eff': 'Very Good', + 'environment-impact-potential': '78', 'glazed-type': 'not defined', 'heating-cost-current': '775', + 'address3': '', 'mainheatcont-description': 'Programmer, room thermostat and TRVs', + 'sheating-energy-eff': 'N/A', 'property-type': 'Flat', 'local-authority-label': 'Wandsworth', + 'fixed-lighting-outlets-count': '10', 'energy-tariff': 'Unknown', 'mechanical-ventilation': 'natural', + 'hot-water-cost-current': '90', 'county': 'Greater London Authority', 'postcode': 'SW4 9SL', + 'solar-water-heating-flag': '', 'constituency': 'E14000549', 'co2-emissions-potential': '2.5', + 'number-heated-rooms': '5', 'floor-description': 'Suspended, no insulation (assumed)', + 'energy-consumption-potential': '113', 'local-authority': 'E09000032', 'built-form': 'Mid-Terrace', + 'number-open-fireplaces': '1', 'windows-description': 'Single glazed', 'glazed-area': 'Normal', + 'inspection-date': '2012-10-23', 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '40', + 'address1': 'Flat 9 Grove Mansions', 'heat-loss-corridor': 'unheated corridor', 'flat-storey-count': '', + 'constituency-label': 'Battersea', 'roof-energy-eff': 'N/A', 'total-floor-area': '115.0', + 'building-reference-number': '3676662078', 'environment-impact-current': '60', + 'co2-emissions-current': '4.5', 'roof-description': '(another dwelling above)', 'floor-energy-eff': 'N/A', + 'number-habitable-rooms': '5', 'address2': 'Clapham Common North Side', 'hot-water-env-eff': 'Good', + 'posttown': 'LONDON', 'mainheatc-energy-eff': 'Good', 'main-fuel': 'mains gas (not community)', + 'lighting-env-eff': 'Very Good', 'windows-energy-eff': 'Very Poor', 'floor-env-eff': 'N/A', + 'sheating-env-eff': 'N/A', 'lighting-description': 'Low energy lighting in all fixed outlets', + 'roof-env-eff': 'N/A', 'walls-energy-eff': 'Very Poor', 'photo-supply': '0.0', + 'lighting-cost-potential': '62', 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '0', + 'main-heating-controls': '2106', 'lodgement-datetime': '2012-10-23 22:45:57', 'flat-top-storey': 'N', + 'current-energy-rating': 'D', 'secondheat-description': 'None', 'walls-env-eff': 'Very Poor', + 'transaction-type': 'rental (private)', 'uprn': '121016128', 'current-energy-efficiency': '64', + 'energy-consumption-current': '205', 'mainheat-description': 'Boiler and radiators, mains gas', + 'lighting-cost-current': '60', 'lodgement-date': '2012-10-23', 'extension-count': '1', + 'mainheatc-env-eff': 'Good', 'lmk-key': '849060769222012102322455762878332', 'wind-turbine-count': '0', + 'tenure': 'rental (private)', 'floor-level': 'Ground', 'potential-energy-efficiency': '78', + 'hot-water-energy-eff': 'Good', 'low-energy-lighting': '100', + 'walls-description': 'Solid brick, as built, no insulation (assumed)', + 'hotwater-description': 'From main system'}, + {'low-energy-fixed-light-count': '', 'address': 'Flat 9, Grove Mansions, 111 Clapham Common North Side', + 'uprn-source': 'Energy Assessor', 'floor-height': '2.9', 'heating-cost-potential': '269', + 'unheated-corridor-length': '8.05', 'hot-water-cost-potential': '99', + 'construction-age-band': 'England and Wales: before 1900', 'potential-energy-rating': 'B', + 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Very Poor', 'lighting-energy-eff': 'Very Good', + 'environment-impact-potential': '82', 'glazed-type': 'not defined', 'heating-cost-current': '490', + 'address3': '111 Clapham Common North Side', + 'mainheatcont-description': 'Programmer, room thermostat and TRVs', 'sheating-energy-eff': 'N/A', + 'property-type': 'Flat', 'local-authority-label': 'Wandsworth', 'fixed-lighting-outlets-count': '10', + 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '98', + 'county': '', 'postcode': 'SW4 9SL', 'solar-water-heating-flag': 'N', 'constituency': 'E14000549', + 'co2-emissions-potential': '1.7', 'number-heated-rooms': '5', + 'floor-description': '(another dwelling below)', 'energy-consumption-potential': '96', + 'local-authority': 'E09000032', 'built-form': 'Semi-Detached', 'number-open-fireplaces': '0', + 'windows-description': 'Single glazed', 'glazed-area': 'Normal', 'inspection-date': '2022-10-25', + 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '29', 'address1': 'Flat 9', + 'heat-loss-corridor': 'unheated corridor', 'flat-storey-count': '', 'constituency-label': 'Battersea', + 'roof-energy-eff': 'N/A', 'total-floor-area': '101.0', 'building-reference-number': '10003516043', + 'environment-impact-current': '68', 'co2-emissions-current': '3.0', + 'roof-description': '(another dwelling above)', 'floor-energy-eff': 'N/A', 'number-habitable-rooms': '5', + 'address2': 'Grove Mansions', 'hot-water-env-eff': 'Good', 'posttown': 'LONDON', + 'mainheatc-energy-eff': 'Good', 'main-fuel': 'mains gas (not community)', 'lighting-env-eff': 'Very Good', + 'windows-energy-eff': 'Very Poor', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A', + 'lighting-description': 'Low energy lighting in all fixed outlets', 'roof-env-eff': 'N/A', + 'walls-energy-eff': 'Very Poor', 'photo-supply': '0.0', 'lighting-cost-potential': '89', + 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '0', 'main-heating-controls': '', + 'lodgement-datetime': '2022-10-26 12:39:24', 'flat-top-storey': 'N', 'current-energy-rating': 'C', + 'secondheat-description': 'None', 'walls-env-eff': 'Very Poor', 'transaction-type': 'rental', + 'uprn': '121016128', 'current-energy-efficiency': '71', 'energy-consumption-current': '167', + 'mainheat-description': 'Boiler and radiators, mains gas', 'lighting-cost-current': '87', + 'lodgement-date': '2022-10-26', 'extension-count': '0', 'mainheatc-env-eff': 'Good', + 'lmk-key': '23b377c4b89f9c38549193042ad6d991972db2a43404ba2826ccb9a2bf64540d', 'wind-turbine-count': '0', + 'tenure': 'Rented (private)', 'floor-level': '01', 'potential-energy-efficiency': '81', + 'hot-water-energy-eff': 'Good', 'low-energy-lighting': '100', + 'walls-description': 'Solid brick, as built, no insulation (assumed)', + 'hotwater-description': 'From main system'}], + "measures": [ + [["secondary_glazing", "low_energy_lighting"], "2", {}, [0, 1]], + [ + ["secondary_glazing", "low_energy_lighting", "internal_wall_insulation", "suspending_floor_insulation"], + "9", + {}, + [0, 1, 2, 3] + ], + [ + ["double_glazing", "low_energy_lighting", "internal_wall_insulation", "suspending_floor_insulation"], + "11", + {}, + [0, 1, 2, 3] + ], + ], + }, + + { + "address": "Flat 5 Grove Mansions", + "postcode": "SW4 9SL", + "lmk-key": None, + "epc": { + 'low-energy-fixed-light-count': '1', 'address': 'Flat 5 Grove Mansions, 111, Clapham Common North Side', + 'uprn-source': 'Address Matched', 'floor-height': 2.95, 'heating-cost-potential': '390', + 'unheated-corridor-length': '3.85', 'hot-water-cost-potential': '192', + 'construction-age-band': 'England and Wales: 1900-1929', 'potential-energy-rating': 'B', + 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Very Poor', 'lighting-energy-eff': 'Poor', + 'environment-impact-potential': '83', 'glazed-type': 'Single glazing', 'heating-cost-current': '945', + 'address3': None, 'mainheatcont-description': 'Programmer and room thermostat', + 'sheating-energy-eff': 'N/A', 'property-type': 'Flat', 'local-authority-label': 'Wandsworth', + 'fixed-lighting-outlets-count': '8', 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', + 'hot-water-cost-current': '188', 'county': '', 'postcode': 'SW4 9SL', 'solar-water-heating-flag': 'N', + 'constituency': 'E14000549', 'co2-emissions-potential': '1.5', 'number-heated-rooms': '4', + 'floor-description': '(another dwelling below)', 'energy-consumption-potential': '97', + 'local-authority': 'E09000032', 'built-form': 'End-Terrace', 'number-open-fireplaces': '0', + 'windows-description': 'Single glazed', 'glazed-area': 'Much More Than Typical', + 'inspection-date': datetime(2024, 8, 27, 0, 0, tzinfo=timezone.utc), + 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '37', 'address1': 'Flat 5 Grove Mansions, 111', + 'heat-loss-corridor': 'unheated corridor', 'flat-storey-count': None, 'constituency-label': 'Battersea', + 'roof-energy-eff': 'N/A', 'total-floor-area': '90', 'building-reference-number': '', + 'environment-impact-current': '62', 'co2-emissions-current': '3.3', + 'roof-description': '(another dwelling above)', 'floor-energy-eff': 'N/A', + 'number-habitable-rooms': '4', 'address2': 'Clapham Common North Side', 'hot-water-env-eff': 'Good', + 'posttown': 'LONDON', 'mainheatc-energy-eff': 'Average', 'main-fuel': 'mains gas (not community)', + 'lighting-env-eff': 'Poor', 'windows-energy-eff': 'Very Poor', 'floor-env-eff': 'N/A', + 'sheating-env-eff': 'N/A', 'lighting-description': 'Low energy lighting in 13% of fixed outlets', + 'roof-env-eff': 'N/A', 'walls-energy-eff': 'Very Poor', 'photo-supply': '0', + 'lighting-cost-potential': '108', 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '0', + 'main-heating-controls': 'Programmer and room thermostat', + 'lodgement-datetime': datetime(2024, 8, 27, 0, 0), 'flat-top-storey': 'N', + 'current-energy-rating': 'D', 'secondheat-description': 'None', 'walls-env-eff': 'Very Poor', + 'transaction-type': 'ECO assessment', 'uprn': 121016124, 'current-energy-efficiency': '66', + 'energy-consumption-current': '209', 'mainheat-description': 'Boiler and radiators, mains gas', + 'lighting-cost-current': '202', 'lodgement-date': date(2024, 8, 27), 'extension-count': '1', + 'mainheatc-env-eff': 'Average', 'lmk-key': '', 'wind-turbine-count': '0', 'tenure': 'Rented (private)', + 'floor-level': '2', 'potential-energy-efficiency': '81', 'hot-water-energy-eff': 'Good', + 'low-energy-lighting': '13', 'walls-description': 'Solid brick, as built, no insulation (assumed)', + 'hotwater-description': 'From main system' + }, + "old_epcs": [ + {'low-energy-fixed-light-count': '10', 'address': 'Flat 9 Grove Mansions, Clapham Common North Side', + 'uprn-source': 'Address Matched', 'floor-height': '', 'heating-cost-potential': '419', + 'unheated-corridor-length': '10.99', 'hot-water-cost-potential': '91', + 'construction-age-band': 'England and Wales: 1900-1929', 'potential-energy-rating': 'C', + 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Very Poor', 'lighting-energy-eff': 'Very Good', + 'environment-impact-potential': '78', 'glazed-type': 'not defined', 'heating-cost-current': '775', + 'address3': '', 'mainheatcont-description': 'Programmer, room thermostat and TRVs', + 'sheating-energy-eff': 'N/A', 'property-type': 'Flat', 'local-authority-label': 'Wandsworth', + 'fixed-lighting-outlets-count': '10', 'energy-tariff': 'Unknown', 'mechanical-ventilation': 'natural', + 'hot-water-cost-current': '90', 'county': 'Greater London Authority', 'postcode': 'SW4 9SL', + 'solar-water-heating-flag': '', 'constituency': 'E14000549', 'co2-emissions-potential': '2.5', + 'number-heated-rooms': '5', 'floor-description': 'Suspended, no insulation (assumed)', + 'energy-consumption-potential': '113', 'local-authority': 'E09000032', 'built-form': 'Mid-Terrace', + 'number-open-fireplaces': '1', 'windows-description': 'Single glazed', 'glazed-area': 'Normal', + 'inspection-date': '2012-10-23', 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '40', + 'address1': 'Flat 9 Grove Mansions', 'heat-loss-corridor': 'unheated corridor', 'flat-storey-count': '', + 'constituency-label': 'Battersea', 'roof-energy-eff': 'N/A', 'total-floor-area': '115.0', + 'building-reference-number': '3676662078', 'environment-impact-current': '60', + 'co2-emissions-current': '4.5', 'roof-description': '(another dwelling above)', 'floor-energy-eff': 'N/A', + 'number-habitable-rooms': '5', 'address2': 'Clapham Common North Side', 'hot-water-env-eff': 'Good', + 'posttown': 'LONDON', 'mainheatc-energy-eff': 'Good', 'main-fuel': 'mains gas (not community)', + 'lighting-env-eff': 'Very Good', 'windows-energy-eff': 'Very Poor', 'floor-env-eff': 'N/A', + 'sheating-env-eff': 'N/A', 'lighting-description': 'Low energy lighting in all fixed outlets', + 'roof-env-eff': 'N/A', 'walls-energy-eff': 'Very Poor', 'photo-supply': '0.0', + 'lighting-cost-potential': '62', 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '0', + 'main-heating-controls': '2106', 'lodgement-datetime': '2012-10-23 22:45:57', 'flat-top-storey': 'N', + 'current-energy-rating': 'D', 'secondheat-description': 'None', 'walls-env-eff': 'Very Poor', + 'transaction-type': 'rental (private)', 'uprn': '121016128', 'current-energy-efficiency': '64', + 'energy-consumption-current': '205', 'mainheat-description': 'Boiler and radiators, mains gas', + 'lighting-cost-current': '60', 'lodgement-date': '2012-10-23', 'extension-count': '1', + 'mainheatc-env-eff': 'Good', 'lmk-key': '849060769222012102322455762878332', 'wind-turbine-count': '0', + 'tenure': 'rental (private)', 'floor-level': 'Ground', 'potential-energy-efficiency': '78', + 'hot-water-energy-eff': 'Good', 'low-energy-lighting': '100', + 'walls-description': 'Solid brick, as built, no insulation (assumed)', + 'hotwater-description': 'From main system'}, + {'low-energy-fixed-light-count': '', 'address': 'Flat 9, Grove Mansions, 111 Clapham Common North Side', + 'uprn-source': 'Energy Assessor', 'floor-height': '2.9', 'heating-cost-potential': '269', + 'unheated-corridor-length': '8.05', 'hot-water-cost-potential': '99', + 'construction-age-band': 'England and Wales: before 1900', 'potential-energy-rating': 'B', + 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Very Poor', 'lighting-energy-eff': 'Very Good', + 'environment-impact-potential': '82', 'glazed-type': 'not defined', 'heating-cost-current': '490', + 'address3': '111 Clapham Common North Side', + 'mainheatcont-description': 'Programmer, room thermostat and TRVs', 'sheating-energy-eff': 'N/A', + 'property-type': 'Flat', 'local-authority-label': 'Wandsworth', 'fixed-lighting-outlets-count': '10', + 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '98', + 'county': '', 'postcode': 'SW4 9SL', 'solar-water-heating-flag': 'N', 'constituency': 'E14000549', + 'co2-emissions-potential': '1.7', 'number-heated-rooms': '5', + 'floor-description': '(another dwelling below)', 'energy-consumption-potential': '96', + 'local-authority': 'E09000032', 'built-form': 'Semi-Detached', 'number-open-fireplaces': '0', + 'windows-description': 'Single glazed', 'glazed-area': 'Normal', 'inspection-date': '2022-10-25', + 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '29', 'address1': 'Flat 9', + 'heat-loss-corridor': 'unheated corridor', 'flat-storey-count': '', 'constituency-label': 'Battersea', + 'roof-energy-eff': 'N/A', 'total-floor-area': '101.0', 'building-reference-number': '10003516043', + 'environment-impact-current': '68', 'co2-emissions-current': '3.0', + 'roof-description': '(another dwelling above)', 'floor-energy-eff': 'N/A', 'number-habitable-rooms': '5', + 'address2': 'Grove Mansions', 'hot-water-env-eff': 'Good', 'posttown': 'LONDON', + 'mainheatc-energy-eff': 'Good', 'main-fuel': 'mains gas (not community)', 'lighting-env-eff': 'Very Good', + 'windows-energy-eff': 'Very Poor', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A', + 'lighting-description': 'Low energy lighting in all fixed outlets', 'roof-env-eff': 'N/A', + 'walls-energy-eff': 'Very Poor', 'photo-supply': '0.0', 'lighting-cost-potential': '89', + 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '0', 'main-heating-controls': '', + 'lodgement-datetime': '2022-10-26 12:39:24', 'flat-top-storey': 'N', 'current-energy-rating': 'C', + 'secondheat-description': 'None', 'walls-env-eff': 'Very Poor', 'transaction-type': 'rental', + 'uprn': '121016128', 'current-energy-efficiency': '71', 'energy-consumption-current': '167', + 'mainheat-description': 'Boiler and radiators, mains gas', 'lighting-cost-current': '87', + 'lodgement-date': '2022-10-26', 'extension-count': '0', 'mainheatc-env-eff': 'Good', + 'lmk-key': '23b377c4b89f9c38549193042ad6d991972db2a43404ba2826ccb9a2bf64540d', 'wind-turbine-count': '0', + 'tenure': 'Rented (private)', 'floor-level': '01', 'potential-energy-efficiency': '81', + 'hot-water-energy-eff': 'Good', 'low-energy-lighting': '100', + 'walls-description': 'Solid brick, as built, no insulation (assumed)', + 'hotwater-description': 'From main system'}], + "measures": [ + [["secondary_glazing", "low_energy_lighting"], "5", {}, [0, 1]], + [ + ["secondary_glazing", "low_energy_lighting", "internal_wall_insulation"], + "13", + {}, + [0, 1, 2] + ], + [ + ["double_glazing", "low_energy_lighting", "internal_wall_insulation"], + "15", + {}, + [0, 1, 2] + ], + ], + }, + + { + "address": "Flat 14 Grove Mansions", + "postcode": "SW4 9SL", + "lmk-key": None, + "epc": { + 'low-energy-fixed-light-count': '9', 'address': 'Flat 14 Grove Mansions, 111, Clapham Common North Side', + 'uprn-source': 'Address Matched', 'floor-height': 3.0, 'heating-cost-potential': '437', + 'unheated-corridor-length': '3.9', 'hot-water-cost-potential': '187', + 'construction-age-band': 'England and Wales: before 1900', 'potential-energy-rating': 'C', + 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Very Poor', 'lighting-energy-eff': 'Good', + 'environment-impact-potential': '79', 'glazed-type': 'Single glazing', 'heating-cost-current': '841', + 'address3': None, 'mainheatcont-description': 'Programmer, room thermostat and TRVs', + 'sheating-energy-eff': 'N/A', 'property-type': 'Flat', 'local-authority-label': 'Wandsworth', + 'fixed-lighting-outlets-count': '19', 'energy-tariff': 'Dual', 'mechanical-ventilation': 'natural', + 'hot-water-cost-current': '184', 'county': '', 'postcode': 'SW4 9SL', 'solar-water-heating-flag': 'N', + 'constituency': 'E14000549', 'co2-emissions-potential': '1.6', 'number-heated-rooms': '4', + 'floor-description': '(another dwelling below)', 'energy-consumption-potential': '111', + 'local-authority': 'E09000032', 'built-form': 'End-Terrace', 'number-open-fireplaces': '3', + 'windows-description': 'Single glazed', 'glazed-area': 'Much More Than Typical', + 'inspection-date': datetime(2024, 8, 13, 0, 0, tzinfo=timezone.utc), + 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '35', 'address1': 'Flat 14 Grove Mansions, 111', + 'heat-loss-corridor': 'unheated corridor', 'flat-storey-count': None, 'constituency-label': 'Battersea', + 'roof-energy-eff': 'N/A', 'total-floor-area': '83', 'building-reference-number': '', + 'environment-impact-current': '64', 'co2-emissions-current': '2.9', + 'roof-description': '(another dwelling above)', 'floor-energy-eff': 'N/A', 'number-habitable-rooms': '4', + 'address2': 'Clapham Common North Side', 'hot-water-env-eff': 'Good', 'posttown': 'LONDON', + 'mainheatc-energy-eff': 'Good', 'main-fuel': 'mains gas (not community)', 'lighting-env-eff': 'Good', + 'windows-energy-eff': 'Very Poor', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A', + 'lighting-description': 'Low energy lighting in 47% of fixed outlets', 'roof-env-eff': 'N/A', + 'walls-energy-eff': 'Poor', 'photo-supply': '0', 'lighting-cost-potential': '117', + 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '0', + 'main-heating-controls': 'Programmer, room thermostat and TRVs', + 'lodgement-datetime': datetime(2024, 8, 13, 0, 0), 'flat-top-storey': 'N', + 'current-energy-rating': 'D', 'secondheat-description': 'None', 'walls-env-eff': 'Poor', + 'transaction-type': 'ECO assessment', 'uprn': 121016117, 'current-energy-efficiency': '67', + 'energy-consumption-current': '198', 'mainheat-description': 'Boiler and radiators, mains gas', + 'lighting-cost-current': '179', 'lodgement-date': date(2024, 8, 13), 'extension-count': '1', + 'mainheatc-env-eff': 'Good', 'lmk-key': '', 'wind-turbine-count': '0', 'tenure': 'Rented (social)', + 'floor-level': '2', 'potential-energy-efficiency': '79', 'hot-water-energy-eff': 'Good', + 'low-energy-lighting': '47', 'walls-description': 'Solid brick, as built, no insulation (assumed)', + 'hotwater-description': 'From main system' + }, + "old_epcs": [ + {'low-energy-fixed-light-count': '12', 'address': 'Flat 14 Grove Mansions, Clapham Common North Side', + 'uprn-source': 'Address Matched', 'floor-height': '', 'heating-cost-potential': '305', + 'unheated-corridor-length': '8.15', 'hot-water-cost-potential': '92', + 'construction-age-band': 'England and Wales: before 1900', 'potential-energy-rating': 'B', + 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Very Poor', 'lighting-energy-eff': 'Very Good', + 'environment-impact-potential': '84', 'glazed-type': 'not defined', 'heating-cost-current': '639', + 'address3': '', 'mainheatcont-description': 'Programmer, room thermostat and TRVs', + 'sheating-energy-eff': 'N/A', 'property-type': 'Maisonette', 'local-authority-label': 'Wandsworth', + 'fixed-lighting-outlets-count': '12', 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', + 'hot-water-cost-current': '90', 'county': 'Greater London Authority', 'postcode': 'SW4 9SL', + 'solar-water-heating-flag': '', 'constituency': 'E14000549', 'co2-emissions-potential': '1.8', + 'number-heated-rooms': '4', 'floor-description': '(other premises below)', + 'energy-consumption-potential': '83', 'local-authority': 'E09000032', 'built-form': 'End-Terrace', + 'number-open-fireplaces': '0', 'windows-description': 'Single glazed', 'glazed-area': 'Normal', + 'inspection-date': '2012-07-09', 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '33', + 'address1': 'Flat 14 Grove Mansions', 'heat-loss-corridor': 'unheated corridor', 'flat-storey-count': '', + 'constituency-label': 'Battersea', 'roof-energy-eff': 'N/A', 'total-floor-area': '115.0', + 'building-reference-number': '8277395668', 'environment-impact-current': '67', + 'co2-emissions-current': '3.7', 'roof-description': '(another dwelling above)', 'floor-energy-eff': 'N/A', + 'number-habitable-rooms': '4', 'address2': 'Clapham Common North Side', 'hot-water-env-eff': 'Good', + 'posttown': 'LONDON', 'mainheatc-energy-eff': 'Good', 'main-fuel': 'mains gas (not community)', + 'lighting-env-eff': 'Very Good', 'windows-energy-eff': 'Very Poor', 'floor-env-eff': 'N/A', + 'sheating-env-eff': 'N/A', 'lighting-description': 'Low energy lighting in all fixed outlets', + 'roof-env-eff': 'N/A', 'walls-energy-eff': 'Very Poor', 'photo-supply': '0.0', + 'lighting-cost-potential': '62', 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '0', + 'main-heating-controls': '2106', 'lodgement-datetime': '2012-07-25 10:03:53', 'flat-top-storey': 'N', + 'current-energy-rating': 'C', 'secondheat-description': 'None', 'walls-env-eff': 'Very Poor', + 'transaction-type': 'rental (private)', 'uprn': '121016117', 'current-energy-efficiency': '69', + 'energy-consumption-current': '169', 'mainheat-description': 'Boiler and radiators, mains gas', + 'lighting-cost-current': '61', 'lodgement-date': '2012-07-25', 'extension-count': '0', + 'mainheatc-env-eff': 'Good', 'lmk-key': '342255683232012072510035337068302', 'wind-turbine-count': '0', + 'tenure': 'rental (private)', 'floor-level': '1st', 'potential-energy-efficiency': '82', + 'hot-water-energy-eff': 'Good', 'low-energy-lighting': '100', + 'walls-description': 'Solid brick, as built, no insulation (assumed)', + 'hotwater-description': 'From main system'}, + {'low-energy-fixed-light-count': '', 'address': 'Flat 14 Grove Mansions, Clapham Common North Side', + 'uprn-source': 'Address Matched', 'floor-height': '3.0', 'heating-cost-potential': '833', + 'unheated-corridor-length': '16.07', 'hot-water-cost-potential': '101', + 'construction-age-band': 'England and Wales: before 1900', 'potential-energy-rating': 'D', + 'mainheat-energy-eff': 'Very Good', 'windows-env-eff': 'Very Poor', 'lighting-energy-eff': 'Good', + 'environment-impact-potential': '53', 'glazed-type': 'single glazing', 'heating-cost-current': '853', + 'address3': '', 'mainheatcont-description': 'Programmer and room thermostat', 'sheating-energy-eff': 'N/A', + 'property-type': 'Flat', 'local-authority-label': 'Wandsworth', 'fixed-lighting-outlets-count': '', + 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '101', + 'county': 'Greater London Authority', 'postcode': 'SW4 9SL', 'solar-water-heating-flag': 'N', + 'constituency': 'E14000549', 'co2-emissions-potential': '5.0', 'number-heated-rooms': '4', + 'floor-description': '(other premises below)', 'energy-consumption-potential': '312', + 'local-authority': 'E09000032', 'built-form': 'Semi-Detached', 'number-open-fireplaces': '0', + 'windows-description': 'Single glazed', 'glazed-area': 'Normal', 'inspection-date': '2009-07-31', + 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '54', 'address1': 'Flat 14 Grove Mansions', + 'heat-loss-corridor': 'unheated corridor', 'flat-storey-count': '4.0', 'constituency-label': 'Battersea', + 'roof-energy-eff': 'N/A', 'total-floor-area': '96.6', 'building-reference-number': '8277395668', + 'environment-impact-current': '52', 'co2-emissions-current': '5.2', + 'roof-description': '(another dwelling above)', 'floor-energy-eff': 'N/A', 'number-habitable-rooms': '5', + 'address2': 'Clapham Common North Side', 'hot-water-env-eff': 'Very Good', 'posttown': 'LONDON', + 'mainheatc-energy-eff': 'Poor', + 'main-fuel': 'mains gas - this is for backwards compatibility only and should not be used', + 'lighting-env-eff': 'Good', 'windows-energy-eff': 'Very Poor', 'floor-env-eff': 'N/A', + 'sheating-env-eff': 'N/A', 'lighting-description': 'Low energy lighting in 50% of fixed outlets', + 'roof-env-eff': 'N/A', 'walls-energy-eff': 'Very Poor', 'photo-supply': '0.0', + 'lighting-cost-potential': '50', 'mainheat-env-eff': 'Very Good', 'multi-glaze-proportion': '0', + 'main-heating-controls': '2104', 'lodgement-datetime': '2009-08-10 19:02:03', 'flat-top-storey': 'N', + 'current-energy-rating': 'E', 'secondheat-description': 'Portable electric heaters', + 'walls-env-eff': 'Very Poor', 'transaction-type': 'rental (private)', 'uprn': '121016117', + 'current-energy-efficiency': '53', 'energy-consumption-current': '326', + 'mainheat-description': 'Boiler and radiators, mains gas', 'lighting-cost-current': '76', + 'lodgement-date': '2009-08-10', 'extension-count': '0', 'mainheatc-env-eff': 'Poor', + 'lmk-key': '342255690202009081019020365517398', 'wind-turbine-count': '0', 'tenure': 'rental (private)', + 'floor-level': '1st', 'potential-energy-efficiency': '55', 'hot-water-energy-eff': 'Very Good', + 'low-energy-lighting': '50', 'walls-description': 'Solid brick, as built, no insulation (assumed)', + 'hotwater-description': 'From main system'}, + {'low-energy-fixed-light-count': '', 'address': 'Flat 14, Grove Mansions, 111 Clapham Common North Side', + 'uprn-source': 'Energy Assessor', 'floor-height': '2.76', 'heating-cost-potential': '266', + 'unheated-corridor-length': '14.16', 'hot-water-cost-potential': '95', + 'construction-age-band': 'England and Wales: before 1900', 'potential-energy-rating': 'B', + 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Very Poor', 'lighting-energy-eff': 'Good', + 'environment-impact-potential': '81', 'glazed-type': 'not defined', 'heating-cost-current': '535', + 'address3': '111 Clapham Common North Side', + 'mainheatcont-description': 'Programmer, room thermostat and TRVs', 'sheating-energy-eff': 'N/A', + 'property-type': 'Flat', 'local-authority-label': 'Wandsworth', 'fixed-lighting-outlets-count': '10', + 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '94', + 'county': '', 'postcode': 'SW4 9SL', 'solar-water-heating-flag': 'N', 'constituency': 'E14000549', + 'co2-emissions-potential': '1.7', 'number-heated-rooms': '5', + 'floor-description': '(another dwelling below)', 'energy-consumption-potential': '101', + 'local-authority': 'E09000032', 'built-form': 'Enclosed End-Terrace', 'number-open-fireplaces': '0', + 'windows-description': 'Single glazed', 'glazed-area': 'Normal', 'inspection-date': '2022-07-05', + 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '35', 'address1': 'Flat 14', + 'heat-loss-corridor': 'unheated corridor', 'flat-storey-count': '', 'constituency-label': 'Battersea', + 'roof-energy-eff': 'N/A', 'total-floor-area': '96.0', 'building-reference-number': '10003195613', + 'environment-impact-current': '62', 'co2-emissions-current': '3.4', + 'roof-description': '(another dwelling above)', 'floor-energy-eff': 'N/A', 'number-habitable-rooms': '5', + 'address2': 'Grove Mansions', 'hot-water-env-eff': 'Good', 'posttown': 'LONDON', + 'mainheatc-energy-eff': 'Good', 'main-fuel': 'mains gas (not community)', 'lighting-env-eff': 'Good', + 'windows-energy-eff': 'Very Poor', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A', + 'lighting-description': 'Low energy lighting in 50% of fixed outlets', 'roof-env-eff': 'N/A', + 'walls-energy-eff': 'Very Poor', 'photo-supply': '0.0', 'lighting-cost-potential': '82', + 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '0', 'main-heating-controls': '', + 'lodgement-datetime': '2022-07-07 12:44:41', 'flat-top-storey': 'N', 'current-energy-rating': 'D', + 'secondheat-description': 'None', 'walls-env-eff': 'Very Poor', 'transaction-type': 'rental', + 'uprn': '121016117', 'current-energy-efficiency': '67', 'energy-consumption-current': '201', + 'mainheat-description': 'Boiler and radiators, mains gas', 'lighting-cost-current': '120', + 'lodgement-date': '2022-07-07', 'extension-count': '0', 'mainheatc-env-eff': 'Good', + 'lmk-key': '17466c218fec8705c20da5b78f1ba528e17d3796d10aa6c4a8c6a8534d098a73', 'wind-turbine-count': '0', + 'tenure': 'Rented (private)', 'floor-level': '02', 'potential-energy-efficiency': '81', + 'hot-water-energy-eff': 'Good', 'low-energy-lighting': '50', + 'walls-description': 'Solid brick, as built, no insulation (assumed)', + 'hotwater-description': 'From main system'} + ], + "measures": [ + [["secondary_glazing", "low_energy_lighting"], "3", {}, [0, 1]], + [ + ["secondary_glazing", "low_energy_lighting", "internal_wall_insulation"], + "9", + {}, + [0, 1, 2] + ], + [ + ["double_glazing", "low_energy_lighting", "internal_wall_insulation"], + "11", + {}, + [0, 1, 2] + ], + ], + }, + + { + "address": "Flat 19 Grove Mansions", + "postcode": "SW4 9SL", + "lmk-key": None, + "epc": { + 'low-energy-fixed-light-count': '4', 'address': 'Flat 19 Grove Mansions, 111, Clapham Common North Side', + 'uprn-source': 'Address Matched', 'floor-height': 2.46, 'heating-cost-potential': '1010', + 'unheated-corridor-length': '2.4', 'hot-water-cost-potential': '165', + 'construction-age-band': 'England and Wales: before 1900', 'potential-energy-rating': 'D', + 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Average', 'lighting-energy-eff': 'Good', + 'environment-impact-potential': '58', 'glazed-type': 'double glazing, unknown install date', + 'heating-cost-current': '1598', 'address3': None, + 'mainheatcont-description': 'Programmer, room thermostat and TRVs', 'sheating-energy-eff': 'N/A', + 'property-type': 'Flat', 'local-authority-label': 'Wandsworth', 'fixed-lighting-outlets-count': '7', + 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '165', + 'county': '', 'postcode': 'SW4 9SL', 'solar-water-heating-flag': 'N', 'constituency': 'E14000549', + 'co2-emissions-potential': '3.0', 'number-heated-rooms': '3', + 'floor-description': '(another dwelling below)', 'energy-consumption-potential': '259', + 'local-authority': 'E09000032', 'built-form': 'Mid-Terrace', 'number-open-fireplaces': '0', + 'windows-description': 'Fully double glazed', 'glazed-area': 'Much More Than Typical', + 'inspection-date': datetime(2024, 8, 13, 0, 0, tzinfo=timezone.utc), + 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '70', 'address1': 'Flat 19 Grove Mansions, 111', + 'heat-loss-corridor': 'unheated corridor', 'flat-storey-count': None, 'constituency-label': 'Battersea', + 'roof-energy-eff': 'Very Poor', 'total-floor-area': '66', 'building-reference-number': '', + 'environment-impact-current': '40', 'co2-emissions-current': '4.6', + 'roof-description': 'Flat, no insulation (assumed)', 'floor-energy-eff': 'N/A', + 'number-habitable-rooms': '3', 'address2': 'Clapham Common North Side', 'hot-water-env-eff': 'Good', + 'posttown': 'LONDON', 'mainheatc-energy-eff': 'Good', 'main-fuel': 'mains gas (not community)', + 'lighting-env-eff': 'Good', 'windows-energy-eff': 'Average', 'floor-env-eff': 'N/A', + 'sheating-env-eff': 'N/A', 'lighting-description': 'Low energy lighting in 57% of fixed outlets', + 'roof-env-eff': 'Very Poor', 'walls-energy-eff': 'Poor', 'photo-supply': '0', + 'lighting-cost-potential': '91', 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '100', + 'main-heating-controls': 'Programmer, room thermostat and TRVs', + 'lodgement-datetime': datetime(2024, 8, 13, 0, 0), 'flat-top-storey': 'Y', + 'current-energy-rating': 'E', 'secondheat-description': 'Room heaters, electric', 'walls-env-eff': 'Poor', + 'transaction-type': 'ECO assessment', 'uprn': 10024087902, 'current-energy-efficiency': '42', + 'energy-consumption-current': '399', 'mainheat-description': 'Boiler and radiators, mains gas', + 'lighting-cost-current': '130', 'lodgement-date': date(2024, 8, 13), 'extension-count': '1', + 'mainheatc-env-eff': 'Good', 'lmk-key': '', 'wind-turbine-count': '0', 'tenure': 'Rented (social)', + 'floor-level': '3', 'potential-energy-efficiency': '60', 'hot-water-energy-eff': 'Good', + 'low-energy-lighting': '57', 'walls-description': 'Solid brick, as built, no insulation (assumed)', + 'hotwater-description': 'From main system' + }, + "old_epcs": [ + {'low-energy-fixed-light-count': '', 'address': 'Flat 19, Grove Mansions, 111 Clapham Common North Side', + 'uprn-source': 'Address Matched', 'floor-height': '2.46', 'heating-cost-potential': '279', + 'unheated-corridor-length': '7.1', 'hot-water-cost-potential': '100', + 'construction-age-band': 'England and Wales: 2007 onwards', 'potential-energy-rating': 'B', + 'mainheat-energy-eff': 'Very Good', 'windows-env-eff': 'Good', 'lighting-energy-eff': 'Very Good', + 'environment-impact-potential': '82', 'glazed-type': 'double glazing installed during or after 2002', + 'heating-cost-current': '293', 'address3': '111 Clapham Common North Side', + 'mainheatcont-description': 'Programmer, TRVs and bypass', 'sheating-energy-eff': 'N/A', + 'property-type': 'Maisonette', 'local-authority-label': 'Wandsworth', 'fixed-lighting-outlets-count': '', + 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '106', + 'county': 'Greater London Authority', 'postcode': 'SW4 9SL', 'solar-water-heating-flag': 'N', + 'constituency': 'E14000549', 'co2-emissions-potential': '1.7', 'number-heated-rooms': '3', + 'floor-description': '(other premises below)', 'energy-consumption-potential': '130', + 'local-authority': 'E09000032', 'built-form': 'Mid-Terrace', 'number-open-fireplaces': '0', + 'windows-description': 'Fully double glazed', 'glazed-area': 'Normal', 'inspection-date': '2011-01-28', + 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '23', 'address1': 'Flat 19', + 'heat-loss-corridor': 'unheated corridor', 'flat-storey-count': '5.0', 'constituency-label': 'Battersea', + 'roof-energy-eff': 'Good', 'total-floor-area': '17.99', 'building-reference-number': '0862458668', + 'environment-impact-current': '81', 'co2-emissions-current': '1.8', + 'roof-description': 'Roof room(s), insulated', 'floor-energy-eff': 'N/A', 'number-habitable-rooms': '3', + 'address2': 'Grove Mansions', 'hot-water-env-eff': 'Very Good', 'posttown': 'LONDON', + 'mainheatc-energy-eff': 'Average', + 'main-fuel': 'mains gas - this is for backwards compatibility only and should not be used', + 'lighting-env-eff': 'Very Good', 'windows-energy-eff': 'Good', 'floor-env-eff': 'N/A', + 'sheating-env-eff': 'N/A', 'lighting-description': 'Low energy lighting in all fixed outlets', + 'roof-env-eff': 'Good', 'walls-energy-eff': 'Good', 'photo-supply': '0.0', 'lighting-cost-potential': '46', + 'mainheat-env-eff': 'Very Good', 'multi-glaze-proportion': '100', 'main-heating-controls': '2107', + 'lodgement-datetime': '2011-02-04 10:10:19', 'flat-top-storey': 'Y', 'current-energy-rating': 'B', + 'secondheat-description': 'None', 'walls-env-eff': 'Good', 'transaction-type': 'rental (private)', + 'uprn': '10024087902', 'current-energy-efficiency': '82', 'energy-consumption-current': '138', + 'mainheat-description': 'Boiler and radiators, mains gas', 'lighting-cost-current': '46', + 'lodgement-date': '2011-02-04', 'extension-count': '0', 'mainheatc-env-eff': 'Average', + 'lmk-key': '378427544752011020410101995290068', 'wind-turbine-count': '0', 'tenure': 'rental (private)', + 'floor-level': '4th', 'potential-energy-efficiency': '83', 'hot-water-energy-eff': 'Very Good', + 'low-energy-lighting': '100', 'walls-description': 'Cavity wall, with internal insulation', + 'hotwater-description': 'From main system'}, + {'low-energy-fixed-light-count': '', 'address': 'Flat 19, Grove Mansions, 111 Clapham Common North Side', + 'uprn-source': 'Address Matched', 'floor-height': '2.5', 'heating-cost-potential': '285', + 'unheated-corridor-length': '', 'hot-water-cost-potential': '84', 'construction-age-band': 'NO DATA!', + 'potential-energy-rating': 'B', 'mainheat-energy-eff': 'Very Good', 'windows-env-eff': 'Poor', + 'lighting-energy-eff': 'Average', 'environment-impact-potential': '79', 'glazed-type': 'NO DATA!', + 'heating-cost-current': '282', 'address3': '111 Clapham Common North Side', + 'mainheatcont-description': 'Programmer, room thermostat and TRVs', 'sheating-energy-eff': 'N/A', + 'property-type': 'Flat', 'local-authority-label': 'Wandsworth', 'fixed-lighting-outlets-count': '', + 'energy-tariff': 'standard tariff', 'mechanical-ventilation': 'NO DATA!', 'hot-water-cost-current': '84', + 'county': 'Greater London Authority', 'postcode': 'SW4 9SL', 'solar-water-heating-flag': '', + 'constituency': 'E14000549', 'co2-emissions-potential': '1.8', 'number-heated-rooms': '', + 'floor-description': '(other premises below)', 'energy-consumption-potential': '157', + 'local-authority': 'E09000032', 'built-form': 'End-Terrace', 'number-open-fireplaces': '', + 'windows-description': 'Partial double glazing', 'glazed-area': 'NO DATA!', + 'inspection-date': '2009-10-16', 'mains-gas-flag': '', 'co2-emiss-curr-per-floor-area': '27', + 'address1': 'Flat 19', 'heat-loss-corridor': 'NO DATA!', 'flat-storey-count': '', + 'constituency-label': 'Battersea', 'roof-energy-eff': 'Good', 'total-floor-area': '67.78', + 'building-reference-number': '0862458668', 'environment-impact-current': '78', + 'co2-emissions-current': '1.8', 'roof-description': 'Average thermal transmittance 0.22 W/m?K', + 'floor-energy-eff': 'N/A', 'number-habitable-rooms': '', 'address2': 'Grove Mansions', + 'hot-water-env-eff': 'Very Good', 'posttown': 'LONDON', 'mainheatc-energy-eff': 'Average', + 'main-fuel': 'mains gas - this is for backwards compatibility only and should not be used', + 'lighting-env-eff': 'Average', 'windows-energy-eff': 'Poor', 'floor-env-eff': 'N/A', + 'sheating-env-eff': 'N/A', 'lighting-description': 'Low energy lighting in 43% of fixed outlets', + 'roof-env-eff': 'Good', 'walls-energy-eff': 'Good', 'photo-supply': '', 'lighting-cost-potential': '34', + 'mainheat-env-eff': 'Very Good', 'multi-glaze-proportion': '', 'main-heating-controls': '', + 'lodgement-datetime': '2009-10-16 13:50:54', 'flat-top-storey': '', 'current-energy-rating': 'C', + 'secondheat-description': 'None', 'walls-env-eff': 'Good', 'transaction-type': 'new dwelling', + 'uprn': '10024087902', 'current-energy-efficiency': '80', 'energy-consumption-current': '162', + 'mainheat-description': 'Boiler and radiators, mains gas', 'lighting-cost-current': '54', + 'lodgement-date': '2009-10-16', 'extension-count': '', 'mainheatc-env-eff': 'Average', + 'lmk-key': '378427540962009101613505488368001', 'wind-turbine-count': '0', 'tenure': '', + 'floor-level': 'top floor', 'potential-energy-efficiency': '81', 'hot-water-energy-eff': 'Very Good', + 'low-energy-lighting': '', 'walls-description': 'Average thermal transmittance 0.32 W/m?K', + 'hotwater-description': 'From main system'}, + {'low-energy-fixed-light-count': '', 'address': 'Flat 19, Grove Mansions, 111 Clapham Common North Side', + 'uprn-source': 'Address Matched', 'floor-height': '2.5', 'heating-cost-potential': '272', + 'unheated-corridor-length': '', 'hot-water-cost-potential': '84', 'construction-age-band': 'NO DATA!', + 'potential-energy-rating': 'B', 'mainheat-energy-eff': 'Very Good', 'windows-env-eff': 'Good', + 'lighting-energy-eff': 'Average', 'environment-impact-potential': '80', 'glazed-type': 'NO DATA!', + 'heating-cost-current': '270', 'address3': '111 Clapham Common North Side', + 'mainheatcont-description': 'Programmer, room thermostat and TRVs', 'sheating-energy-eff': 'N/A', + 'property-type': 'Flat', 'local-authority-label': 'Wandsworth', 'fixed-lighting-outlets-count': '', + 'energy-tariff': 'standard tariff', 'mechanical-ventilation': 'NO DATA!', 'hot-water-cost-current': '84', + 'county': 'Greater London Authority', 'postcode': 'SW4 9SL', 'solar-water-heating-flag': '', + 'constituency': 'E14000549', 'co2-emissions-potential': '1.7', 'number-heated-rooms': '', + 'floor-description': '(other premises below)', 'energy-consumption-potential': '150', + 'local-authority': 'E09000032', 'built-form': 'End-Terrace', 'number-open-fireplaces': '', + 'windows-description': 'Fully double glazed', 'glazed-area': 'NO DATA!', 'inspection-date': '2009-10-12', + 'mains-gas-flag': '', 'co2-emiss-curr-per-floor-area': '26', 'address1': 'Flat 19', + 'heat-loss-corridor': 'NO DATA!', 'flat-storey-count': '', 'constituency-label': 'Battersea', + 'roof-energy-eff': 'Good', 'total-floor-area': '67.78', 'building-reference-number': '0862458668', + 'environment-impact-current': '79', 'co2-emissions-current': '1.7', + 'roof-description': 'Average thermal transmittance 0.22 W/m?K', 'floor-energy-eff': 'N/A', + 'number-habitable-rooms': '', 'address2': 'Grove Mansions', 'hot-water-env-eff': 'Very Good', + 'posttown': 'LONDON', 'mainheatc-energy-eff': 'Average', + 'main-fuel': 'mains gas - this is for backwards compatibility only and should not be used', + 'lighting-env-eff': 'Average', 'windows-energy-eff': 'Good', 'floor-env-eff': 'N/A', + 'sheating-env-eff': 'N/A', 'lighting-description': 'Low energy lighting in 43% of fixed outlets', + 'roof-env-eff': 'Good', 'walls-energy-eff': 'Good', 'photo-supply': '', 'lighting-cost-potential': '34', + 'mainheat-env-eff': 'Very Good', 'multi-glaze-proportion': '', 'main-heating-controls': '', + 'lodgement-datetime': '2009-10-12 16:21:11', 'flat-top-storey': '', 'current-energy-rating': 'B', + 'secondheat-description': 'None', 'walls-env-eff': 'Good', 'transaction-type': 'new dwelling', + 'uprn': '10024087902', 'current-energy-efficiency': '81', 'energy-consumption-current': '155', + 'mainheat-description': 'Boiler and radiators, mains gas', 'lighting-cost-current': '54', + 'lodgement-date': '2009-10-12', 'extension-count': '', 'mainheatc-env-eff': 'Average', + 'lmk-key': '378427550802009101216211162819628', 'wind-turbine-count': '0', 'tenure': '', + 'floor-level': 'top floor', 'potential-energy-efficiency': '82', 'hot-water-energy-eff': 'Very Good', + 'low-energy-lighting': '', 'walls-description': 'Average thermal transmittance 0.32 W/m?K', + 'hotwater-description': 'From main system'}, + {'low-energy-fixed-light-count': '', 'address': '19 GROVE MANSIONS, 111 CLAPHAM COMMON NORTH SIDE, LONDON', + 'uprn-source': 'Energy Assessor', 'floor-height': '2.48', 'heating-cost-potential': '258', + 'unheated-corridor-length': '2.46', 'hot-water-cost-potential': '87', + 'construction-age-band': 'England and Wales: 2007-2011', 'potential-energy-rating': 'C', + 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Average', 'lighting-energy-eff': 'Very Good', + 'environment-impact-potential': '79', 'glazed-type': 'double glazing, unknown install date', + 'heating-cost-current': '264', 'address3': 'LONDON', + 'mainheatcont-description': 'Programmer, room thermostat and TRVs', 'sheating-energy-eff': 'N/A', + 'property-type': 'Flat', 'local-authority-label': 'Wandsworth', 'fixed-lighting-outlets-count': '7', + 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '87', + 'county': '', 'postcode': 'SW4 9SL', 'solar-water-heating-flag': 'N', 'constituency': 'E14000549', + 'co2-emissions-potential': '1.5', 'number-heated-rooms': '3', + 'floor-description': '(another dwelling below)', 'energy-consumption-potential': '128', + 'local-authority': 'E09000032', 'built-form': 'End-Terrace', 'number-open-fireplaces': '0', + 'windows-description': 'Fully double glazed', 'glazed-area': 'Normal', 'inspection-date': '2021-03-10', + 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '23', 'address1': '19 GROVE MANSIONS', + 'heat-loss-corridor': 'unheated corridor', 'flat-storey-count': '', 'constituency-label': 'Battersea', + 'roof-energy-eff': 'Good', 'total-floor-area': '66.0', 'building-reference-number': '10000974677', + 'environment-impact-current': '78', 'co2-emissions-current': '1.5', + 'roof-description': 'Roof room(s), insulated (assumed)', 'floor-energy-eff': 'N/A', + 'number-habitable-rooms': '3', 'address2': '111 CLAPHAM COMMON NORTH SIDE', 'hot-water-env-eff': 'Good', + 'posttown': 'LONDON', 'mainheatc-energy-eff': 'Good', 'main-fuel': 'mains gas (not community)', + 'lighting-env-eff': 'Very Good', 'windows-energy-eff': 'Average', 'floor-env-eff': 'N/A', + 'sheating-env-eff': 'N/A', 'lighting-description': 'Low energy lighting in all fixed outlets', + 'roof-env-eff': 'Good', 'walls-energy-eff': 'Good', 'photo-supply': '0.0', 'lighting-cost-potential': '62', + 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '100', 'main-heating-controls': '', + 'lodgement-datetime': '2021-05-16 15:24:17', 'flat-top-storey': 'Y', 'current-energy-rating': 'C', + 'secondheat-description': 'None', 'walls-env-eff': 'Good', 'transaction-type': 'rental', + 'uprn': '10024087902', 'current-energy-efficiency': '77', 'energy-consumption-current': '131', + 'mainheat-description': 'Boiler and radiators, mains gas', 'lighting-cost-current': '62', + 'lodgement-date': '2021-05-16', 'extension-count': '0', 'mainheatc-env-eff': 'Good', + 'lmk-key': '706a5b2d4a92a2f27b7179d6d713dacf81134ff249cad0481f885b13b1b3ffc0', 'wind-turbine-count': '0', + 'tenure': 'Rented (private)', 'floor-level': '03', 'potential-energy-efficiency': '78', + 'hot-water-energy-eff': 'Good', 'low-energy-lighting': '100', + 'walls-description': 'Solid brick, as built, insulated (assumed)', + 'hotwater-description': 'From main system'} + ], + "measures": [ + [["low_energy_lighting"], "1", {}, [0, 1]], + [ + ["low_energy_lighting", "internal_wall_insulation"], + "13", + {}, + [0, 1, 2] + ], + ], + }, ] recommendations_scoring_data = [] From 15bcd46e24aac991dbc9bdd31c8c40970649f0f0 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 16 Oct 2024 17:30:15 +0100 Subject: [PATCH 143/166] extended vectis cases and added metrics --- etl/epc/Record.py | 2 + etl/epc/generate_scenarios_data.py | 84 +++++++++++++++++++++++------- 2 files changed, 67 insertions(+), 19 deletions(-) diff --git a/etl/epc/Record.py b/etl/epc/Record.py index cc70d42b..4c1a912b 100644 --- a/etl/epc/Record.py +++ b/etl/epc/Record.py @@ -575,6 +575,8 @@ class EPCRecord: mains_gas_map = { "Y": True, "N": False, + True: True, + False: False } self.prepared_epc["mains-gas-flag"] = ( diff --git a/etl/epc/generate_scenarios_data.py b/etl/epc/generate_scenarios_data.py index 1714fe39..fe27cc91 100644 --- a/etl/epc/generate_scenarios_data.py +++ b/etl/epc/generate_scenarios_data.py @@ -1,5 +1,6 @@ from datetime import datetime, timezone, date import itertools +from tqdm import tqdm import pandas as pd from etl.epc.Record import EPCRecord @@ -20,6 +21,7 @@ from backend.Property import Property from recommendations.Recommendations import Recommendations from utils.logger import setup_logger from utils.s3 import read_dataframe_from_s3_parquet, save_dataframe_to_s3_parquet +from sklearn.metrics import mean_absolute_error, mean_absolute_percentage_error now = datetime.now().strftime("%d-%m-%Y-%H-%M-%S") @@ -1165,7 +1167,7 @@ scenario_properties = [ recommendations_scoring_data = [] -for scenario_property in scenario_properties: +for scenario_property in tqdm(scenario_properties): # We validate each record in the file. If the record is NOT valid, we need to handle this accordingly epc_searcher = SearchEpc( @@ -1174,22 +1176,35 @@ for scenario_property in scenario_properties: auth_token=get_settings().EPC_AUTH_TOKEN, os_api_key=get_settings().ORDNANCE_SURVEY_API_KEY, ) - epc_searcher.find_property() - # Find the epc with the same LMK key - all_epcs = epc_searcher.older_epcs.copy() - all_epcs.extend([epc_searcher.newest_epc, epc_searcher.full_sap_epc]) - original_epc = [ - epc - for epc in all_epcs - if epc.get("lmk-key", None) == scenario_property.get("lmk-key") - ][0] + if scenario_property["lmk-key"] is None: + epc_records = { + "original_epc": scenario_property["epc"], + "full_sap_epc": {}, + "old_data": scenario_property["old_epcs"], + } + address = scenario_property["address"] + postcode = scenario_property["postcode"] + else: + epc_searcher.find_property() - epc_records = { - "original_epc": original_epc, - "full_sap_epc": {}, - "old_data": [], - } + # Find the epc with the same LMK key + all_epcs = epc_searcher.older_epcs.copy() + all_epcs.extend([epc_searcher.newest_epc, epc_searcher.full_sap_epc]) + original_epc = [ + epc + for epc in all_epcs + if epc.get("lmk-key", None) == scenario_property.get("lmk-key") + ][0] + + epc_records = { + "original_epc": original_epc, + "full_sap_epc": {}, + "old_data": [], + } + + address = epc_searcher.address_clean + postcode = epc_searcher.postcode_clean prepared_epc = EPCRecord( epc_records=epc_records, run_mode="newdata", cleaning_data=cleaning_data @@ -1197,8 +1212,8 @@ for scenario_property in scenario_properties: p = Property( id=prepared_epc.uprn, - address=epc_searcher.address_clean, - postcode=epc_searcher.postcode_clean, + address=address, + postcode=postcode, epc_record=prepared_epc, ) @@ -1220,8 +1235,10 @@ for scenario_property in scenario_properties: wall_recommendations = recommender.wall_recomender.recommendations loft_recommendations = recommender.roof_recommender.recommendations + floor_recommendations = recommender.floor_recommender.recommendations solar_recommendations = recommender.solar_recommender.recommendation windows_recommendations = recommender.windows_recommender.recommendation + led_recommendations = recommender.lighting_recommender.recommendation p.create_base_difference_epc_record(cleaned_lookup=cleaned) @@ -1236,8 +1253,10 @@ for scenario_property in scenario_properties: wall_recs = [] loft_recs = [] + floor_recs = [] solar_recs = [] windows_recs = [] + lighting_recs = [] if "internal_wall_insulation" in measure: for rec in wall_recommendations: @@ -1265,12 +1284,22 @@ for scenario_property in scenario_properties: if rec["type"] == "solar_pv": solar_recs.append(rec) - if "windows" in measure: + if "windows" in measure or "secondary_glazing" in measure or "double_glazing" in measure: for rec in windows_recommendations: if rec["type"] == "windows_glazing": windows_recs.append(rec) - combi_list = [wall_recs, loft_recs, solar_recs, windows_recs] + if "low_energy_lighting" in measure: + for rec in led_recommendations: + if rec["type"] == "led_lighting": + lighting_recs.append(rec) + + if "suspended_floor_insulation" in measure: + for rec in floor_recommendations: + if rec["type"] == "suspended_floor_insulation": + floor_recs.append(rec) + + combi_list = [wall_recs, loft_recs, solar_recs, windows_recs, lighting_recs, floor_recs] combi_list = [element for element in combi_list if len(element) != 0] all_combi_recommendations = list(itertools.product(*combi_list)) @@ -1331,6 +1360,23 @@ sap_impact = pd.concat( axis=1 ) sap_impact["predicted_impact"] = sap_impact["predictions"] - sap_impact["sap_starting"] +sap_impact["actual_post_sap"] = sap_impact["impact"] + sap_impact["sap_starting"] +sap_impact = sap_impact[ + [ + 'id', 'property_id', 'recommendation_id', 'phase', 'uprn', 'sap_starting', 'predictions', 'actual_post_sap', + 'impact', 'predicted_impact' + ] +].rename( + columns={"predictions": "predicted_post_sap", "impact": "actual_impact"} +) + +# Get some metrics - MAPE for local testing +mae = mean_absolute_error(sap_impact["actual_post_sap"], sap_impact["predicted_post_sap"]) +# 2.2958333333333347 +mape = mean_absolute_percentage_error(sap_impact["actual_post_sap"], sap_impact["predicted_post_sap"]) +# 0.034359867214274246 +mape_impact = mean_absolute_percentage_error(sap_impact["actual_impact"], sap_impact["predicted_impact"]) +# 0.4853675375550377 save_dataframe_to_s3_parquet( recommendations_scoring_data, From 135831febb322d7b9e28bab0e8f1cf2fef19e8f5 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 16 Oct 2024 19:55:15 +0100 Subject: [PATCH 144/166] increasing the number of tests in generate_scenarios_data --- etl/epc/generate_scenarios_data.py | 145 ++++++++++++++++++++++++----- 1 file changed, 123 insertions(+), 22 deletions(-) diff --git a/etl/epc/generate_scenarios_data.py b/etl/epc/generate_scenarios_data.py index fe27cc91..fb3177a2 100644 --- a/etl/epc/generate_scenarios_data.py +++ b/etl/epc/generate_scenarios_data.py @@ -71,6 +71,18 @@ scenario_properties = [ {}, [0], ], + [ + ["internal_wall_insulation", "suspended_floor_insulation"], + "13", + {}, + [0, 1], + ], + [ + ["external_wall_insulation", "suspended_floor_insulation"], + "13", + {}, + [0, 1], + ], [["solar", "windows"], "15", {"photo_supply_ending": 50}, [0, 1]], ], }, @@ -85,6 +97,24 @@ scenario_properties = [ {}, [0, 1], ], + [ + ["loft_insulation"], + "2", + {}, + [0], + ], + [ + ["cavity_wall_insulation", "loft_insulation", "solid_floor_insulation"], + "12", + {}, + [0], + ], + [ + ["cavity_wall_insulation", "loft_insulation", "solid_floor_insulation", "low_energy_lighting"], + "13", + {}, + [0], + ], ], }, { @@ -287,7 +317,19 @@ scenario_properties = [ ], [ ["double_glazing", "internal_wall_insulation", "suspended_floor_insulation"], - "11", + "10", + {}, + [0, 1, 2] + ], + [ + ["internal_wall_insulation"], + "5", + {}, + [0] + ], + [ + ["internal_wall_insulation", "suspended_floor_insulation"], + "7", {}, [0, 1, 2] ], @@ -533,6 +575,18 @@ scenario_properties = [ {}, [0, 1, 2] ], + [ + ["internal_wall_insulation"], + "5", + {}, + [0] + ], + [ + ["internal_wall_insulation", "low_energy_lighting"], + "5", + {}, + [0, 1] + ], ], }, @@ -653,17 +707,35 @@ scenario_properties = [ "measures": [ [["secondary_glazing", "low_energy_lighting"], "2", {}, [0, 1]], [ - ["secondary_glazing", "low_energy_lighting", "internal_wall_insulation", "suspending_floor_insulation"], + ["secondary_glazing", "low_energy_lighting", "internal_wall_insulation", "suspended_floor_insulation"], "9", {}, [0, 1, 2, 3] ], [ - ["double_glazing", "low_energy_lighting", "internal_wall_insulation", "suspending_floor_insulation"], + ["double_glazing", "low_energy_lighting", "internal_wall_insulation", "suspended_floor_insulation"], "11", {}, [0, 1, 2, 3] ], + [ + ["internal_wall_insulation"], + "6", + {}, + [0] + ], + [ + ["internal_wall_insulation", "suspended_floor_insulation"], + "7", + {}, + [0, 1] + ], + [ + ["internal_wall_insulation", "suspended_floor_insulation", "low_energy_lighting"], + "8", + {}, + [0, 1, 2] + ], ], }, @@ -795,6 +867,18 @@ scenario_properties = [ {}, [0, 1, 2] ], + [ + ["internal_wall_insulation"], + "8", + {}, + [0] + ], + [ + ["low_energy_lighting", "internal_wall_insulation"], + "10", + {}, + [0, 1, 2] + ], ], }, @@ -963,6 +1047,18 @@ scenario_properties = [ {}, [0, 1, 2] ], + [ + ["internal_wall_insulation"], + "6", + {}, + [0, 1, 2] + ], + [ + ["internal_wall_insulation", "low_energy_lighting"], + "7", + {}, + [0, 1, 2] + ], ], }, @@ -1164,7 +1260,6 @@ scenario_properties = [ ], }, ] - recommendations_scoring_data = [] for scenario_property in tqdm(scenario_properties): @@ -1230,20 +1325,7 @@ for scenario_property in tqdm(scenario_properties): ) } - recommender = Recommendations(property_instance=p, materials=materials, default_u_values=True) - property_recommendations = recommender.recommend() - - wall_recommendations = recommender.wall_recomender.recommendations - loft_recommendations = recommender.roof_recommender.recommendations - floor_recommendations = recommender.floor_recommender.recommendations - solar_recommendations = recommender.solar_recommender.recommendation - windows_recommendations = recommender.windows_recommender.recommendation - led_recommendations = recommender.lighting_recommender.recommendation - - p.create_base_difference_epc_record(cleaned_lookup=cleaned) - scoring_list = [] - # Create the record for each of the different measures for measure_impact_override in scenario_property["measures"]: @@ -1251,6 +1333,20 @@ for scenario_property in tqdm(scenario_properties): impact = measure_impact_override[1] override = measure_impact_override[2] + recommender = Recommendations( + property_instance=p, materials=materials, default_u_values=True, inclusions=measure + ) + property_recommendations = recommender.recommend() + + wall_recommendations = recommender.wall_recomender.recommendations.copy() + loft_recommendations = recommender.roof_recommender.recommendations.copy() + floor_recommendations = recommender.floor_recommender.recommendations.copy() + solar_recommendations = recommender.solar_recommender.recommendation.copy() + windows_recommendations = recommender.windows_recommender.recommendation.copy() + led_recommendations = recommender.lighting_recommender.recommendation.copy() + + p.create_base_difference_epc_record(cleaned_lookup=cleaned) + wall_recs = [] loft_recs = [] floor_recs = [] @@ -1299,7 +1395,12 @@ for scenario_property in tqdm(scenario_properties): if rec["type"] == "suspended_floor_insulation": floor_recs.append(rec) - combi_list = [wall_recs, loft_recs, solar_recs, windows_recs, lighting_recs, floor_recs] + if "solid_floor_insulation" in measure: + for rec in floor_recommendations: + if rec["type"] == "solid_floor_insulation": + floor_recs.append(rec) + + combi_list = [wall_recs, loft_recs, floor_recs, solar_recs, windows_recs, lighting_recs] combi_list = [element for element in combi_list if len(element) != 0] all_combi_recommendations = list(itertools.product(*combi_list)) @@ -1312,7 +1413,7 @@ for scenario_property in tqdm(scenario_properties): property_id=i, primary_recommendation_id=i, recommendation_record=recommendation_record, - recommendations=combi, + recommendations=list(combi), ) if override is not None: @@ -1372,11 +1473,11 @@ sap_impact = sap_impact[ # Get some metrics - MAPE for local testing mae = mean_absolute_error(sap_impact["actual_post_sap"], sap_impact["predicted_post_sap"]) -# 2.2958333333333347 +# 1.6828571428571433 mape = mean_absolute_percentage_error(sap_impact["actual_post_sap"], sap_impact["predicted_post_sap"]) -# 0.034359867214274246 +# 0.02510877585853886 mape_impact = mean_absolute_percentage_error(sap_impact["actual_impact"], sap_impact["predicted_impact"]) -# 0.4853675375550377 +# 0.35805350998208146 save_dataframe_to_s3_parquet( recommendations_scoring_data, From 7ef08b1778143337830f082db9338cad177c5858 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 16 Oct 2024 20:50:20 +0100 Subject: [PATCH 145/166] fixed bug in creation of simulation data --- etl/epc/generate_scenarios_data.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/etl/epc/generate_scenarios_data.py b/etl/epc/generate_scenarios_data.py index fb3177a2..06d0e377 100644 --- a/etl/epc/generate_scenarios_data.py +++ b/etl/epc/generate_scenarios_data.py @@ -3,6 +3,7 @@ import itertools from tqdm import tqdm import pandas as pd + from etl.epc.Record import EPCRecord from etl.bill_savings.KwhData import KwhData from backend.SearchEpc import SearchEpc @@ -1257,6 +1258,12 @@ scenario_properties = [ {}, [0, 1, 2] ], + [ + ["internal_wall_insulation"], + "12", + {}, + [0, 1, 2] + ], ], }, ] @@ -1387,7 +1394,7 @@ for scenario_property in tqdm(scenario_properties): if "low_energy_lighting" in measure: for rec in led_recommendations: - if rec["type"] == "led_lighting": + if rec["type"] == "low_energy_lighting": lighting_recs.append(rec) if "suspended_floor_insulation" in measure: @@ -1473,11 +1480,11 @@ sap_impact = sap_impact[ # Get some metrics - MAPE for local testing mae = mean_absolute_error(sap_impact["actual_post_sap"], sap_impact["predicted_post_sap"]) -# 1.6828571428571433 +# 1.6511627906976745 mape = mean_absolute_percentage_error(sap_impact["actual_post_sap"], sap_impact["predicted_post_sap"]) -# 0.02510877585853886 +# 0.02493675525723527 mape_impact = mean_absolute_percentage_error(sap_impact["actual_impact"], sap_impact["predicted_impact"]) -# 0.35805350998208146 +# 0.34215659663334086 save_dataframe_to_s3_parquet( recommendations_scoring_data, From 3699c9cd641cf48a459f6907d1b0416b0ecef928 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 17 Oct 2024 12:05:34 +0100 Subject: [PATCH 146/166] added additional scenarios to automated model tests --- etl/epc/generate_scenarios_data.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/etl/epc/generate_scenarios_data.py b/etl/epc/generate_scenarios_data.py index 06d0e377..94f6b3ee 100644 --- a/etl/epc/generate_scenarios_data.py +++ b/etl/epc/generate_scenarios_data.py @@ -1480,11 +1480,11 @@ sap_impact = sap_impact[ # Get some metrics - MAPE for local testing mae = mean_absolute_error(sap_impact["actual_post_sap"], sap_impact["predicted_post_sap"]) -# 1.6511627906976745 +# 1.4325581395348832 mape = mean_absolute_percentage_error(sap_impact["actual_post_sap"], sap_impact["predicted_post_sap"]) -# 0.02493675525723527 +# 0.02260368763204902 mape_impact = mean_absolute_percentage_error(sap_impact["actual_impact"], sap_impact["predicted_impact"]) -# 0.34215659663334086 +# 0.38072532764393224 save_dataframe_to_s3_parquet( recommendations_scoring_data, From 34101234915c966d42ea36b848bacc674ae93440 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 17 Oct 2024 13:48:55 +0100 Subject: [PATCH 147/166] remocing temp code which created a bug --- backend/app/plan/router.py | 4 +- backend/app/plan/schemas.py | 4 ++ etl/customers/remote_assessments/app.py | 73 +++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 3 deletions(-) create mode 100644 etl/customers/remote_assessments/app.py diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 30834657..356aca84 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -411,9 +411,7 @@ async def trigger_plan(body: PlanTriggerRequest): # We check for an energy assessment we have performed on this property: energy_assessment = get_latest_assessment_by_uprn(session, uprn if uprn is not None else epc_searcher.uprn) - - if not energy_assessment["epc"]: - continue + # Create a record in db property_id, is_new = create_property( session, body.portfolio_id, epc_searcher.address_clean, epc_searcher.postcode_clean, diff --git a/backend/app/plan/schemas.py b/backend/app/plan/schemas.py index 4b43db80..e0c5f35d 100644 --- a/backend/app/plan/schemas.py +++ b/backend/app/plan/schemas.py @@ -52,6 +52,9 @@ NON_INVASIVE_SPECIFIC_MEASURES = [ "draught_proofing", "mixed_glazing", # This covers partial double glazing and secondary glazing "cavity_extract_and_refill", + # Indicates that there is one (need to handle the case where there are multiple) + # extension that requires cavity wall insulation + "extension_cavity_wall_insulation", ] # This allows us to extend high level categories for measures such as "wall_insulation" to the specific measures @@ -78,6 +81,7 @@ class PlanTriggerRequest(BaseModel): already_installed_file_path: Optional[str] = None patches_file_path: Optional[str] = None non_invasive_recommendations_file_path: Optional[str] = None + valuation_file_path: Optional[str] = None exclusions: Optional[conlist(str, min_items=1)] = None inclusions: Optional[conlist(str, min_items=1)] = None diff --git a/etl/customers/remote_assessments/app.py b/etl/customers/remote_assessments/app.py new file mode 100644 index 00000000..ba3563cd --- /dev/null +++ b/etl/customers/remote_assessments/app.py @@ -0,0 +1,73 @@ +import pandas as pd +from utils.s3 import save_csv_to_s3 + +PORTFOLIO_ID = 111 +USER_ID = 8 + + +def app(): + """ + This application is used to initialise and run remote assessments + :return: + """ + + asset_list = [ + { + "uprn": 100050770761, + "address": "12 Sheardown Street", + "postcode": "DN4 0BH" + } + ] + asset_list = pd.DataFrame(asset_list) + + # Store the asset list in s3 + filename = f"{USER_ID}/{PORTFOLIO_ID}/asset_list.csv" + save_csv_to_s3( + dataframe=asset_list, + bucket_name="retrofit-plan-inputs-dev", + file_name=filename + ) + + non_invasive_recommendations = [ + { + "uprn": 100050770761, + "recommendations": [ + { + "type": "extension_cavity_wall_insulation", + "sap_points": 2, + } + ] + } + ] + # Store non-invasive recommendations in S3 + non_invasive_recommendations_filename = f"{USER_ID}/{PORTFOLIO_ID}/non_invasive_recommendations.csv" + save_csv_to_s3( + dataframe=pd.DataFrame(non_invasive_recommendations), + bucket_name="retrofit-plan-inputs-dev", + file_name=non_invasive_recommendations_filename + ) + + valuation_data = [{100050770761: 67_000}] + # Store valuation data to s3 + valuation_filename = f"{USER_ID}/{PORTFOLIO_ID}/valuation.csv" + save_csv_to_s3( + dataframe=pd.DataFrame(valuation_data), + bucket_name="retrofit-plan-inputs-dev", + file_name=valuation_filename + ) + + body = { + "portfolio_id": str(PORTFOLIO_ID), + "housing_type": "Private", + "goal": "Increasing EPC", + "goal_value": "C", + "trigger_file_path": filename, + "already_installed_file_path": "", + "patches_file_path": "", + "non_invasive_recommendations_file_path": non_invasive_recommendations_filename, + "valuation_file_path": valuation_filename, + "scenario_name": "Full package remote assessment", + "multi_plan": True, + "budget": None, + } + print(body) From 6600ad516c247ade7a54b26466536ea21fa6e861 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 17 Oct 2024 14:23:02 +0100 Subject: [PATCH 148/166] added extension cavity wall insulation as a recommendation --- recommendations/Recommendations.py | 6 ++ recommendations/WallRecommendations.py | 80 ++++++++++++++++++++++++++ 2 files changed, 86 insertions(+) diff --git a/recommendations/Recommendations.py b/recommendations/Recommendations.py index 684b0915..be713e91 100644 --- a/recommendations/Recommendations.py +++ b/recommendations/Recommendations.py @@ -128,6 +128,12 @@ class Recommendations: property_recommendations.append(self.wall_recomender.recommendations) phase += 1 + # We handle recommendations covering specific non-invasive measures + self.wall_recomender.recommend_extended(measures=measures) + if self.wall_recomender.extended_recommendations: + property_recommendations.append(self.wall_recomender.extended_recommendations) + # We don't have any phasing here + self.roof_recommender.recommend(phase=phase, measures=measures, default_u_values=self.default_u_values) if self.roof_recommender.recommendations: property_recommendations.append(self.roof_recommender.recommendations) diff --git a/recommendations/WallRecommendations.py b/recommendations/WallRecommendations.py index 358547dc..aca87edf 100644 --- a/recommendations/WallRecommendations.py +++ b/recommendations/WallRecommendations.py @@ -1,6 +1,7 @@ import math from typing import List +import numpy as np import pandas as pd from datatypes.enums import QuantityUnits @@ -99,6 +100,8 @@ class WallRecommendations(Definitions): # Will contains a list of recommended measures self.recommendations = [] + # Contains a list of extended recommendation measures, such as extension insulation + self.extended_recommendations = [] self.cavity_wall_insulation_materials = [ part for part in materials if part["type"] == "cavity_wall_insulation" @@ -267,6 +270,83 @@ class WallRecommendations(Definitions): # If the u-value is within regulations, we don't do anything return + def recommend_extended(self, measures): + """ + Where we have extended measures, such as extension insulation, which cannot typically be picked up + from the EPC api, we handle the recommendation of these here + :param measures: + :return: + """ + + # These are the measures that are covered by this function + extended_measures = ["extension_cavity_wall_insulation"] + + measures_to_recommend = [measure for measure in measures if measure in extended_measures] + if not measures_to_recommend: + return + + # We reset this to be empty + self.extended_recommendations = [] + + for measure in measures_to_recommend: + if measure == "extension_cavity_wall_insulation": + recommendation = self.recommend_extension_cavity_wall_insulation() + else: + raise NotImplementedError(f"Measure {measure} is not implemented") + + self.extended_recommendations.append(recommendation) + + return + + def recommend_extension_cavity_wall_insulation(self): + """ + This function produces the recommendation for extension cavity wall insulation + :return: + """ + + # TODO: We aren't provided with carbon, heat or bill savings figures for this measure + + extension_cavity_insulation_recommendation = [ + r for r in self.property.non_invasive_recommendations if r["type"] == "extension_cavity_wall_insulation" + ][0] + + # https://surreybuildingprojects.co.uk/how-much-does-a-24m2-extension-cost + average_extension_floor_area = 24 + # https://assets.publishing.service.gov.uk/media/5f047a01d3bf7f2be8350262 + # /Size_of_English_Homes_Fact_Sheet_EHS_2018.pdf + # This is rough + average_house_floor_area = 94 + + proposed_extension_floor_area = self.property.floor_area * ( + average_extension_floor_area / average_house_floor_area + ) + # assume 3 walls are external + proposed_extension_insulation_wall_area = ( + np.sqrt(proposed_extension_floor_area) * self.property.floor_height * 3 + ) + + cost_result = self.costs.cavity_wall_insulation( + wall_area=proposed_extension_insulation_wall_area, + material=self.cavity_wall_insulation_materials[0], + ) + + recommendation = { + "phase": None, + "parts": [], + "type": "extension_cavity_wall_insulation", + "measure_type": "extension_cavity_wall_insulation", + "description": "Insulate the cavity walls of the extension", + "starting_u_value": None, + "new_u_value": None, + "sap_points": extension_cavity_insulation_recommendation["sap_points"], + "already_installed": False, + "simulation_config": {}, + "description_simulation": {}, + **cost_result + } + + return recommendation + def find_cavity_insulation(self, u_value, insulation_thickness, phase, measures, default_u_values): """ This method tests different materials to fill the cavity wall, determining which From cb3f9c0252e67b817e7ab66f4e63f9c887f4f804 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 17 Oct 2024 14:29:17 +0100 Subject: [PATCH 149/166] hanling extension cwi --- recommendations/Recommendations.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/recommendations/Recommendations.py b/recommendations/Recommendations.py index be713e91..b9862585 100644 --- a/recommendations/Recommendations.py +++ b/recommendations/Recommendations.py @@ -476,7 +476,9 @@ class Recommendations: impact_summary = [] for recommendations_by_type in property_recommendations: for rec in recommendations_by_type: - if rec["type"] in ["mechanical_ventilation", "trickle_vents", "draught_proofing"]: + if rec["type"] in [ + "mechanical_ventilation", "trickle_vents", "draught_proofing", "extension_cavity_wall_insulation" + ]: # We don't have a percieved sap impact of mechanical ventilation or trickle vents, and we don't # have the capacity to score draught proofing continue @@ -757,7 +759,9 @@ class Recommendations: # We now deduce if any of the recommendations result in a change of fuel type for recs in property_recommendations: for rec in recs: - if rec["type"] in ["mechanical_ventilation", "trickle_vents", "draught_proofing"]: + if rec["type"] in [ + "mechanical_ventilation", "trickle_vents", "draught_proofing", "extension_cavity_wall_insulation" + ]: # We cannot score the impact on draught proofing continue From de7e98ac036176ff23f120e1c4bf92bb303a8aa1 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 17 Oct 2024 14:46:38 +0100 Subject: [PATCH 150/166] added missing fields to extension cwi recommendation --- recommendations/WallRecommendations.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/recommendations/WallRecommendations.py b/recommendations/WallRecommendations.py index aca87edf..5dea13c8 100644 --- a/recommendations/WallRecommendations.py +++ b/recommendations/WallRecommendations.py @@ -339,10 +339,16 @@ class WallRecommendations(Definitions): "starting_u_value": None, "new_u_value": None, "sap_points": extension_cavity_insulation_recommendation["sap_points"], + "heat_demand": None, + "kwh_savings": None, + "energy_savings": None, + "energy_cost_savings": None, + "co2_equivalent_savings": None, "already_installed": False, "simulation_config": {}, "description_simulation": {}, - **cost_result + **cost_result, + "default": True, } return recommendation From 800b673150a2cb37c90bdff36b497a6afa30ae60 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 17 Oct 2024 15:02:03 +0100 Subject: [PATCH 151/166] Making it possible to pass valutations as a separate data input and injecting it into the property" --- backend/Property.py | 3 +++ backend/app/plan/router.py | 37 +++++++++++++++++++------- backend/ml_models/Valuation.py | 6 ++++- recommendations/WallRecommendations.py | 10 +++---- 4 files changed, 41 insertions(+), 15 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index 79108dc1..2f5341bb 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -75,6 +75,7 @@ class Property: postcode, address, epc_record, + property_valuation=None, already_installed=None, non_invasive_recommendations=None, measures=None, @@ -111,6 +112,8 @@ class Property: else: self.measures = ast.literal_eval(measures) if measures else None + self.valuation = property_valuation + self.uprn = epc_record.get("uprn") self.uprn_source = self.data.get("uprn-source") diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 356aca84..d376b01e 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -289,7 +289,7 @@ def create_epc_records(epc_searcher: SearchEpc, energy_assessment: dict): }, energy_assessment_is_newer -def get_on_site_data(body: PlanTriggerRequest): +def get_request_property_data(body: PlanTriggerRequest): """ This function will read in the on-site data from the S3 bucket :param body: The request body @@ -311,10 +311,18 @@ def get_on_site_data(body: PlanTriggerRequest): bucket_name=get_settings().PLAN_TRIGGER_BUCKET, filepath=body.non_invasive_recommendations_file_path ) - return patches, already_installed, non_invasive_recommendations + valuation_data = [] + if body.valuation_file_path: + valuation_data = read_csv_from_s3( + bucket_name=get_settings().PLAN_TRIGGER_BUCKET, filepath=body.valuation_file_path + ) + + return patches, already_installed, non_invasive_recommendations, valuation_data -def extract_property_on_site_recommendations(config, patches, already_installed, non_invasive_recommendations, uprn): +def extract_property_request_data( + config, patches, already_installed, non_invasive_recommendations, valuation_data, uprn +): patch_has_uprn = "uprn" in patches[0] if patches else True if patch_has_uprn: patch = next(( @@ -360,7 +368,12 @@ def extract_property_on_site_recommendations(config, patches, already_installed, property_non_invasive_recommendations["recommendations"] = str(transformed) - return patch, property_already_installed, property_non_invasive_recommendations + property_valution = next(( + x for x in valuation_data if + (str(x["uprn"]) == str(uprn)) + ), {}) + + return patch, property_already_installed, property_non_invasive_recommendations, property_valution router = APIRouter( @@ -384,7 +397,7 @@ async def trigger_plan(body: PlanTriggerRequest): logger.info("Getting the inputs") plan_input = read_csv_from_s3(bucket_name=get_settings().PLAN_TRIGGER_BUCKET, filepath=body.trigger_file_path) # If we have patches or overrides, we should read them in here - patches, already_installed, non_invasive_recommendations = get_on_site_data(body) + patches, already_installed, non_invasive_recommendations, valuation_data = get_request_property_data(body) cleaning_data = read_dataframe_from_s3_parquet( bucket_name=get_settings().DATA_BUCKET, file_key="sap_change_model/cleaning_dataset.parquet", @@ -411,7 +424,7 @@ async def trigger_plan(body: PlanTriggerRequest): # We check for an energy assessment we have performed on this property: energy_assessment = get_latest_assessment_by_uprn(session, uprn if uprn is not None else epc_searcher.uprn) - + # Create a record in db property_id, is_new = create_property( session, body.portfolio_id, epc_searcher.address_clean, epc_searcher.postcode_clean, @@ -438,9 +451,14 @@ async def trigger_plan(body: PlanTriggerRequest): epc_searcher, energy_assessment ) - patch, property_already_installed, property_non_invasive_recommendations = ( - extract_property_on_site_recommendations( - config, patches, already_installed, non_invasive_recommendations, uprn + patch, property_already_installed, property_non_invasive_recommendations, property_valuation = ( + extract_property_request_data( + config=config, + patches=patches, + already_installed=already_installed, + non_invasive_recommendations=non_invasive_recommendations, + valuation_data=valuation_data, + uprn=epc_searcher.uprn, ) ) @@ -460,6 +478,7 @@ async def trigger_plan(body: PlanTriggerRequest): postcode=epc_searcher.postcode_clean, epc_record=prepared_epc, already_installed=property_already_installed, + property_valuation=property_valuation, non_invasive_recommendations=property_non_invasive_recommendations, energy_assessment=energy_assessment, **Property.extract_kwargs(config), # TODO: Depraecate this diff --git a/backend/ml_models/Valuation.py b/backend/ml_models/Valuation.py index 68432577..c0e49ffc 100644 --- a/backend/ml_models/Valuation.py +++ b/backend/ml_models/Valuation.py @@ -1,4 +1,5 @@ import numpy as np +from scipy.constants import value class PropertyValuation: @@ -203,7 +204,10 @@ class PropertyValuation: @classmethod def estimate(cls, property_instance, target_epc): - value = cls.UPRN_VALUE_LOOKUP.get(property_instance.uprn) + value = ( + property_instance.valuation if property_instance.valuation else + cls.UPRN_VALUE_LOOKUP.get(property_instance.uprn) + ) if not value: return { diff --git a/recommendations/WallRecommendations.py b/recommendations/WallRecommendations.py index 5dea13c8..88242709 100644 --- a/recommendations/WallRecommendations.py +++ b/recommendations/WallRecommendations.py @@ -339,11 +339,11 @@ class WallRecommendations(Definitions): "starting_u_value": None, "new_u_value": None, "sap_points": extension_cavity_insulation_recommendation["sap_points"], - "heat_demand": None, - "kwh_savings": None, - "energy_savings": None, - "energy_cost_savings": None, - "co2_equivalent_savings": None, + "heat_demand": 0, + "kwh_savings": 0, + "energy_savings": 0, + "energy_cost_savings": 0, + "co2_equivalent_savings": 0, "already_installed": False, "simulation_config": {}, "description_simulation": {}, From d7ed4dd9a4468d3a54c56ad3992c3145e324bdfa Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 17 Oct 2024 18:31:37 +0100 Subject: [PATCH 152/166] tweaking solar recommendations --- backend/Property.py | 3 +- backend/apis/GoogleSolarApi.py | 27 +++++++-------- backend/app/assumptions.py | 4 ++- backend/app/plan/router.py | 4 +-- etl/customers/remote_assessments/app.py | 7 +++- recommendations/Recommendations.py | 41 +++++++++++++++++++++-- recommendations/RoofRecommendations.py | 20 ++++++++++- recommendations/SolarPvRecommendations.py | 2 +- recommendations/WallRecommendations.py | 14 ++++---- 9 files changed, 92 insertions(+), 30 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index 2f5341bb..31f207ab 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -538,7 +538,8 @@ class Property: "loft_insulation", "room_roof_insulation", "flat_roof_insulation", "solid_floor_insulation", "suspended_floor_insulation", "windows_glazing", "solar_pv", "heating", "hot_water_tank_insulation", - "heating_control", "secondary_heating", "cylinder_thermostat", "mixed_glazing" + "heating_control", "secondary_heating", "cylinder_thermostat", "mixed_glazing", + "extension_cavity_wall_insulation", ]: raise NotImplementedError( "Implement me, given type %s" % recommendation["type"] diff --git a/backend/apis/GoogleSolarApi.py b/backend/apis/GoogleSolarApi.py index f6e1b96d..ed7b7422 100644 --- a/backend/apis/GoogleSolarApi.py +++ b/backend/apis/GoogleSolarApi.py @@ -272,23 +272,10 @@ class GoogleSolarApi: roi_summary = [] for segment in roof_segment_summaries: - - if segment["panelsCount"] < min_panels: - continue - wattage = segment["panelsCount"] * self.insights_data["solarPotential"]["panelCapacityWatts"] generated_dc_energy = segment["yearlyEnergyDcKwh"] ratio = generated_dc_energy / wattage - if cost_instance is None: - cost = MCS_SOLAR_PV_COST_DATA["average_cost_per_kwh"] * (wattage / 1000) - else: - cost = cost_instance.solar_pv( - n_panels=segment["panelsCount"], - has_battery=False, - n_floors=property_instance.number_of_floors, - )["total"] - roi_summary.append( { "segmentIndex": segment["segmentIndex"], @@ -296,7 +283,6 @@ class GoogleSolarApi: "generated_dc_energy": generated_dc_energy, "ratio": ratio, "n_panels": segment["panelsCount"], - "cost": cost, "panneled_roof_area": self.panel_area * int(segment["panelsCount"]) } ) @@ -305,10 +291,21 @@ class GoogleSolarApi: if roi_summary.empty: continue + if roi_summary["n_panels"].sum() < min_panels: + continue + + if cost_instance is None: + total_cost = MCS_SOLAR_PV_COST_DATA["average_cost_per_kwh"] * (wattage / 1000) + else: + total_cost = cost_instance.solar_pv( + n_panels=roi_summary["n_panels"].sum(), + has_battery=False, + n_floors=property_instance.number_of_floors, + )["total"] + weighted_ratio = np.average( roi_summary["ratio"].values, weights=roi_summary["generated_dc_energy"].values ) - total_cost = roi_summary["cost"].sum() yearly_dc_energy = roi_summary["generated_dc_energy"].sum() panel_performance.append( diff --git a/backend/app/assumptions.py b/backend/app/assumptions.py index 5571e13b..79f2a087 100644 --- a/backend/app/assumptions.py +++ b/backend/app/assumptions.py @@ -4,8 +4,10 @@ PESSIMISTIC_ASHP_EFFICIENCY = 200 AVERAGE_ASHP_EFFICIENCY = 250 # Conservative estimate of the proportion of electricity that will be consumed, whereas the rest will -# be exported +# be exported. These are averages based on Google research. E.g +# https://www.nea.org.uk/who-we-are/innovation-technical-evaluation/solarpv/solarpv-batteries SOLAR_CONSUMPTION_PROPORTION = 0.5 +SOLAR_CONSUMPTION_WITH_BATTERY_PROPORTION = 0.7 # Typically, each solar panel takes up around 3.4 m2 of roof space under RdSAP. This was been verified in Elmhurst RDSAP_AREA_PER_PANEL = 3.4 diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index d376b01e..925fb05b 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -369,9 +369,9 @@ def extract_property_request_data( property_non_invasive_recommendations["recommendations"] = str(transformed) property_valution = next(( - x for x in valuation_data if + float(x["value"]) for x in valuation_data if (str(x["uprn"]) == str(uprn)) - ), {}) + ), None) return patch, property_already_installed, property_non_invasive_recommendations, property_valution diff --git a/etl/customers/remote_assessments/app.py b/etl/customers/remote_assessments/app.py index ba3563cd..a0d01f7d 100644 --- a/etl/customers/remote_assessments/app.py +++ b/etl/customers/remote_assessments/app.py @@ -47,7 +47,12 @@ def app(): file_name=non_invasive_recommendations_filename ) - valuation_data = [{100050770761: 67_000}] + valuation_data = [ + { + "uprn": 100050770761, + "value": 67_000 + } + ] # Store valuation data to s3 valuation_filename = f"{USER_ID}/{PORTFOLIO_ID}/valuation.csv" save_csv_to_s3( diff --git a/recommendations/Recommendations.py b/recommendations/Recommendations.py index b9862585..aa7e041e 100644 --- a/recommendations/Recommendations.py +++ b/recommendations/Recommendations.py @@ -129,10 +129,11 @@ class Recommendations: phase += 1 # We handle recommendations covering specific non-invasive measures - self.wall_recomender.recommend_extended(measures=measures) + new_phase = self.wall_recomender.recommend_extended(phase=phase, measures=measures) if self.wall_recomender.extended_recommendations: property_recommendations.append(self.wall_recomender.extended_recommendations) # We don't have any phasing here + phase = new_phase self.roof_recommender.recommend(phase=phase, measures=measures, default_u_values=self.default_u_values) if self.roof_recommender.recommendations: @@ -481,6 +482,27 @@ class Recommendations: ]: # We don't have a percieved sap impact of mechanical ventilation or trickle vents, and we don't # have the capacity to score draught proofing + if rec["type"] == "extension_cavity_wall_insulation": + + previous_phase = [x for x in impact_summary if x["phase"] == (rec["phase"] - 1)] + if previous_phase: + sap = previous_phase[0]["sap"] + carbon = previous_phase[0]["carbon"] + heat_demand = previous_phase[0]["heat_demand"] + else: + sap = float(property_instance.data["current-energy-efficiency"]) + carbon = float(property_instance.data["co2-emissions-current"]) + heat_demand = float(property_instance.data["energy-consumption-current"]) + + impact_summary.append( + { + "phase": rec["phase"], + "recommendation_id": rec["recommendation_id"], + "sap": sap + rec["sap_points"], + "carbon": carbon - rec["co2_equivalent_savings"], + "heat_demand": heat_demand - rec["heat_demand"], + } + ) continue phase_energy_efficiency_metrics = { @@ -571,6 +593,17 @@ class Recommendations: property_phase_impact["carbon"], rec["co2_equivalent_savings"] ) + if rec["type"] == "loft_insulation": + # When we have a loft insulation recommendation, where there is an extension and the existing + # amount of loft insulation is already good, we limit the SAP points + # By limiting here, we don't change the value in current_phase_values. This means that the + # future recommendations won't have an impact that is too large + li_sap_limit = RoofRecommendations.get_loft_insulation_sap_limit( + property_instance.data["roof-energy-eff"], property_instance.data["extension-count"] + ) + if li_sap_limit is not None: + property_phase_impact["sap"] = min(property_phase_impact["sap"], li_sap_limit) + # Insert this information into the recommendation. if not rec.get("survey", False): rec["sap_points"] = property_phase_impact["sap"] @@ -672,7 +705,11 @@ class Recommendations: { "phase": r["phase"], "recommendation_id": r["recommendation_id"], - "solar_kwh_savings": r["initial_ac_kwh_per_year"] * assumptions.SOLAR_CONSUMPTION_PROPORTION, + "solar_kwh_savings": ( + r["initial_ac_kwh_per_year"] * assumptions.SOLAR_CONSUMPTION_PROPORTION + ) if not r["has_battery"] else ( + r["initial_ac_kwh_per_year"] * assumptions.SOLAR_CONSUMPTION_WITH_BATTERY_PROPORTION + ), } for recs in property_recommendations for r in recs if r["type"] == "solar_pv" ], columns=["phase", "recommendation_id", "solar_kwh_savings"]) diff --git a/recommendations/RoofRecommendations.py b/recommendations/RoofRecommendations.py index 52313121..c0fa4eb2 100644 --- a/recommendations/RoofRecommendations.py +++ b/recommendations/RoofRecommendations.py @@ -59,6 +59,23 @@ class RoofRecommendations: self.property.roof["is_flat"] ) + @classmethod + def get_loft_insulation_sap_limit(cls, roof_energy_eff, extension_count): + """ + Get the SAP limit for loft insulation + :param roof_energy_eff: + :return: + """ + + if extension_count == 0: + # No limit + return None + + if roof_energy_eff in ["Good", "Very Good"]: + return 1 + + return None + def mds_loft_insulation(self, phase): """ For usages within the mds report @@ -273,7 +290,7 @@ class RoofRecommendations: # loft is already partially insulated. # Note: This requirement is only for loft insulation if ( - (material["depth"] + insulation_thickness) < self.MINIMUM_RECOMMENDED_LOFT_INSULATION + material["depth"] < self.MINIMUM_RECOMMENDED_LOFT_INSULATION ) and is_pitched: continue @@ -295,6 +312,7 @@ class RoofRecommendations: # We allow a small tolerance for error so we don't discount the recommendation entirely if new_u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE: + lowest_selected_u_value = update_lowest_selected_u_value(lowest_selected_u_value, new_u_value) cost_result = self.costs.loft_and_flat_insulation( diff --git a/recommendations/SolarPvRecommendations.py b/recommendations/SolarPvRecommendations.py index eb4616ea..66c1d0c3 100644 --- a/recommendations/SolarPvRecommendations.py +++ b/recommendations/SolarPvRecommendations.py @@ -196,7 +196,7 @@ class SolarPvRecommendations: ] roof_area = self.property.roof_area - solar_configurations = panel_performance.head(3).reset_index(drop=True) + solar_configurations = panel_performance.head(6).reset_index(drop=True) # We combine each of these configurations with estimates with and without a battery for rank, recommendation_config in solar_configurations.iterrows(): diff --git a/recommendations/WallRecommendations.py b/recommendations/WallRecommendations.py index 88242709..c7917911 100644 --- a/recommendations/WallRecommendations.py +++ b/recommendations/WallRecommendations.py @@ -270,7 +270,7 @@ class WallRecommendations(Definitions): # If the u-value is within regulations, we don't do anything return - def recommend_extended(self, measures): + def recommend_extended(self, phase, measures): """ Where we have extended measures, such as extension insulation, which cannot typically be picked up from the EPC api, we handle the recommendation of these here @@ -283,22 +283,24 @@ class WallRecommendations(Definitions): measures_to_recommend = [measure for measure in measures if measure in extended_measures] if not measures_to_recommend: - return + return phase # We reset this to be empty self.extended_recommendations = [] + recommendation_phase = phase for measure in measures_to_recommend: if measure == "extension_cavity_wall_insulation": - recommendation = self.recommend_extension_cavity_wall_insulation() + recommendation = self.recommend_extension_cavity_wall_insulation(phase=recommendation_phase) else: raise NotImplementedError(f"Measure {measure} is not implemented") + recommendation_phase += 1 self.extended_recommendations.append(recommendation) - return + return recommendation_phase - def recommend_extension_cavity_wall_insulation(self): + def recommend_extension_cavity_wall_insulation(self, phase): """ This function produces the recommendation for extension cavity wall insulation :return: @@ -331,7 +333,7 @@ class WallRecommendations(Definitions): ) recommendation = { - "phase": None, + "phase": phase, "parts": [], "type": "extension_cavity_wall_insulation", "measure_type": "extension_cavity_wall_insulation", From 681a449187f0e93ffc2b655b25021189ce74628e Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 18 Oct 2024 15:52:49 +0100 Subject: [PATCH 153/166] preparing the data for lewes council --- etl/customers/newhaven/slides.py | 249 ++++++++++++++++++++++++------- 1 file changed, 192 insertions(+), 57 deletions(-) diff --git a/etl/customers/newhaven/slides.py b/etl/customers/newhaven/slides.py index 61ed89cc..3c62de53 100644 --- a/etl/customers/newhaven/slides.py +++ b/etl/customers/newhaven/slides.py @@ -417,9 +417,14 @@ def slides(): # Show more characters in a column pd.set_option('display.max_colwidth', None) - # preparing of this data for the following 2 needs: - # 1) dataset to share with Nextgen heating - # 2) Breakdown of results by property type + +def app(): + """ + preparing of this data for the following 2 needs: + 1) dataset to share with Nextgen heating + 2) Breakdown of results by property type + :return: + """ # get the asset list asset_list = read_csv_from_s3(bucket_name="retrofit-plan-inputs-dev", filepath="8/90/pilot.csv") @@ -431,6 +436,14 @@ def slides(): ) non_intrusive_recommendations = pd.DataFrame(non_intrusive_recommendations) + # Right now this is the second version of the nehaven portfolio + portfolio_id = 90 + # Look at one scenario at a time, otherwise this is agony + scenario_ids = [47, 48, 49, 50, 51] + properties_data, plans_data, recommendations_data = get_data(portfolio_id, scenario_ids) + properties_df = pd.DataFrame(properties_data) + recommendations_df = pd.DataFrame(recommendations_data) + # Unnest this import ast survey_recs = [] @@ -502,27 +515,74 @@ def slides(): # We now pull out the recommendations impact by property type and sub type + # Exclude sealing open fireplaces + recommendations_df = recommendations_df[recommendations_df["type"] != "sealing_open_fireplace"] + + # We update the type column so that if type == heating, and the description contains "air source heat pump", + # the type is "air_source_heat_pump", else if the description contains "high heat retention storage heaters", else + # if the description contains "condensing boiler, the type is updated to "boiler_upgrade" + recommendations_df["type"] = np.where( + recommendations_df["type"] == "heating", + np.where( + recommendations_df["description"].str.contains("air source heat pump"), + "air_source_heat_pump", + np.where( + recommendations_df["description"].str.contains("high heat retention"), + "high_heat_retention_storage_heaters", + np.where( + recommendations_df["description"].str.contains("condensing boiler"), + "boiler_upgrade", + recommendations_df["type"] + ) + ) + ), + recommendations_df["type"] + ) + + recommendation_types = recommendations_df["type"].unique().tolist() + rename_dict = { + 'hot_water_tank_insulation': 'Hot Water Tank Insulation', + 'windows_glazing': 'Windows Glazing', + 'secondary_heating': 'Secondary Heating', + 'cavity_wall_insulation': 'Cavity Wall Insulation', + 'flat_roof_insulation': 'Flat Roof Insulation', + 'mechanical_ventilation': 'Mechanical Ventilation', + 'loft_insulation': 'Loft Insulation', + 'cylinder_thermostat': 'Cylinder Thermostat', + 'room_roof_insulation': 'Room Roof Insulation', + 'low_energy_lighting': 'Low Energy Lighting', + 'external_wall_insulation': 'External Wall Insulation', + 'heating': 'Heating', + 'solar_pv': 'Solar PV', + 'heating_control': 'Heating Control', + 'solid_floor_insulation': 'Solid Floor Insulation', + 'suspended_floor_insulation': 'Suspended Floor Insulation', + 'internal_wall_insulation': 'Internal Wall Insulation' + } + property_scenario_impact = [] - for scenario_id in scenario_ids: + for scenario_id in tqdm(scenario_ids): # Get the recommendations for the scenario, default scenario_recommendations = recommendations_df[ (recommendations_df["Scenario ID"] == scenario_id) & (recommendations_df["default"] == True) ].copy() - scenario_recommendations['ligting_kwh'] = scenario_recommendations.apply( + scenario_recommendations['Estimated Lighting kWh Savings'] = scenario_recommendations.apply( lambda x: x['kwh_savings'] if x['type'] == 'low_energy_lighting' else 0, axis=1) - scenario_recommendations['solar_kwh'] = scenario_recommendations.apply( + scenario_recommendations['Estimated Solar kWh Savings'] = scenario_recommendations.apply( lambda x: x['kwh_savings'] if x['type'] == 'solar_pv' else 0, axis=1) # Set 'Estimated Kwh Savings' to zero where specific kwh columns are used - scenario_recommendations['Estimated Kwh Savings'] = scenario_recommendations.apply( + scenario_recommendations['Estimated Heating Demand kWh Savings'] = scenario_recommendations.apply( lambda x: 0 if x['type'] in ['low_energy_lighting', 'solar_pv'] else x[ 'kwh_savings'], axis=1) scenario_grouped_data = scenario_recommendations.groupby(['property_id']).agg({ - 'Estimated Kwh Savings': 'sum', + 'Estimated Heating Demand kWh Savings': 'sum', + 'Estimated Lighting kWh Savings': 'sum', + 'Estimated Solar kWh Savings': 'sum', "estimated_cost": "sum" }).reset_index() @@ -531,18 +591,52 @@ def slides(): ].merge( scenario_grouped_data, on=["property_id"], how="left" ) - comparison["Estimated Kwh Savings"] = comparison["Estimated Kwh Savings"].fillna(0) + comparison["Estimated Heating Demand kWh Savings"] = ( + comparison["Estimated Heating Demand kWh Savings"].fillna(0) + ) + comparison["Estimated Lighting kWh Savings"] = ( + comparison["Estimated Lighting kWh Savings"].fillna(0) + ) + comparison["Estimated Solar kWh Savings"] = ( + comparison["Estimated Solar kWh Savings"].fillna(0) + ) comparison["estimated_cost"] = comparison["estimated_cost"].fillna(0) comparison["post_scenario_heating_hotwater_kwh"] = ( - comparison["current_energy_demand_heating_hotwater"] - comparison["Estimated Kwh Savings"] + comparison["current_energy_demand_heating_hotwater"] - comparison["Estimated Heating Demand kWh Savings"] + ) + + # For each scenario, we create a measure matrix + measure_matrix = scenario_recommendations.pivot_table( + index='property_id', + columns='type', + values='id', # Using 'id' just as a placeholder for the pivot + aggfunc=lambda x: True, # If an ID exists for a given type, mark as True + fill_value=False # Fill other entries as False + ).reset_index() + + non_zero_heat_demand_impact = comparison[ + (comparison["Estimated Heating Demand kWh Savings"] > 0) | + (comparison["Estimated Lighting kWh Savings"] > 0) | + (comparison["Estimated Solar kWh Savings"] > 0) + ] + measure_matrix = measure_matrix[ + measure_matrix["property_id"].isin(non_zero_heat_demand_impact["property_id"].values) + ] + measure_matrix = measure_matrix.rename(columns=rename_dict) + + comparison = comparison.merge( + measure_matrix, on="property_id", how="left" ) comparison["scenario_id"] = scenario_id property_scenario_impact.append(comparison) property_scenario_impact = pd.concat(property_scenario_impact) - property_scenario_impact = property_scenario_impact.drop(columns=["property_id", "Estimated Kwh Savings"]) + # property_scenario_impact = property_scenario_impact.drop(columns=["property_id", "Estimated Kwh Savings"]) + for v in rename_dict.values(): + # Fill NaNs with False + property_scenario_impact[v] = property_scenario_impact[v].fillna(False) # Scale property_scenario_impact["post_scenario_heating_hotwater_kwh_scaled"] = ( @@ -600,57 +694,98 @@ def slides(): "post_scenario_heating_hotwater_kwh_scaled"]].empty: raise Exception("someting went wrong") - # Reorder the columns - grouped_data = grouped_data[ - [ - 'property_type', - 'property_sub_type', - 'scenario', - 'estimated_heating_hotwater_kwh', - 'post_scenario_heating_hotwater_kwh', - 'estimated_heating_hotwater_kwh_scaled', - 'post_scenario_heating_hotwater_kwh_scaled', - 'estimated_cost', - ] + # Reorder the columns + grouped_data = grouped_data[ + [ + 'property_type', + 'property_sub_type', + 'scenario', + 'estimated_heating_hotwater_kwh', + 'post_scenario_heating_hotwater_kwh', + 'estimated_heating_hotwater_kwh_scaled', + 'post_scenario_heating_hotwater_kwh_scaled', + 'estimated_cost', + ] + ] + + grouped_data = grouped_data.rename( + columns={ + "property_type": "Property Type", + "property_sub_type": "Property Sub Type", + "scenario": "Scenario", + "estimated_heating_hotwater_kwh": "Estimated Heating & Hot Water kwh", + "post_scenario_heating_hotwater_kwh": "Post Scenario Heating & Hot Water kwh", + "estimated_heating_hotwater_kwh_scaled": "Estimated Heating & Hot Water kwh (scaled)", + "post_scenario_heating_hotwater_kwh_scaled": "Post Scenario Heating & Hot Water kwh (scaled)", + "estimated_cost": "Estimated Cost or Retrofit", + } + ) + + # grouped_data.to_excel( + # "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Newhaven/outputs/Scenario kWh Impact by Property " + # "Type.xlsx", + # index=False + # ) + + property_scenario_impact = property_scenario_impact.merge( + scenario_names, how="left", on="scenario_id" + ) + + property_scenario_impact = property_scenario_impact.sort_values( + ["postcode", "uprn", "scenario_id"], ascending=True + ) + + lewes_data = next_gen_dataset.merge( + property_scenario_impact, how="left", on="uprn" + ) + + # Rearrange, rename columns and drop what we don't need + # TODO - remap the heating type + lewes_data = lewes_data[ + [ + 'uprn', 'address', 'postcode', 'property_type', 'built_form', 'estimated_heating_hotwater_kwh', + 'primary_fuel_type', 'gross_floor_area', 'floor_height', 'number_of_floors', 'ashp_suitable', + 'ashp_size_kw', + 'ashp_cost', 'solar_suitable', 'solar_size_kwp', 'solar_cost', 'estimated_heating_hotwater_kwh_scaled', + # 'property_id', - dropped + 'current_energy_demand_heating_hotwater', 'Estimated Heating Demand kWh Savings', + 'Estimated Lighting kWh Savings', 'Estimated Solar kWh Savings', 'estimated_cost', + 'post_scenario_heating_hotwater_kwh', 'Cavity Wall Insulation', 'Cylinder Thermostat', + 'Flat Roof Insulation', + 'Hot Water Tank Insulation', 'Loft Insulation', 'Mechanical Ventilation', 'Room Roof Insulation', + # 'scenario_id', - dropped + 'Low Energy Lighting', 'Secondary Heating', 'Windows Glazing', 'External Wall Insulation', + 'Heating', + 'Heating Control', + 'Solar PV', + 'Internal Wall Insulation', + 'Solid Floor Insulation', + 'Suspended Floor Insulation', + 'post_scenario_heating_hotwater_kwh_scaled', + 'scenario' ] - grouped_data = grouped_data.rename( - columns={ - "property_type": "Property Type", - "property_sub_type": "Property Sub Type", - "scenario": "Scenario", - "estimated_heating_hotwater_kwh": "Estimated Heating & Hot Water kwh", - "post_scenario_heating_hotwater_kwh": "Post Scenario Heating & Hot Water kwh", - "estimated_heating_hotwater_kwh_scaled": "Estimated Heating & Hot Water kwh (scaled)", - "post_scenario_heating_hotwater_kwh_scaled": "Post Scenario Heating & Hot Water kwh (scaled)", - "estimated_cost": "Estimated Cost or Retrofit", - } - ) + ] - grouped_data.to_excel( - "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Newhaven/outputs/Scenario kWh Impact by Property " - "Type.xlsx", - index=False - ) + # We save this dataset, which will be shared with Lewes Council + lewes_data.to_csv( + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Newhaven/outputs/property data.csv", index=False + ) - property_scenario_impact = property_scenario_impact.merge( - scenario_names, how="left", on="scenario_id" - ) + df_pivot = property_scenario_impact.pivot_table(index='uprn', columns='scenario', + values=['post_scenario_heating_hotwater_kwh', + 'post_scenario_heating_hotwater_kwh_scaled']) - df_pivot = property_scenario_impact.pivot_table(index='uprn', columns='scenario', - values=['post_scenario_heating_hotwater_kwh', - 'post_scenario_heating_hotwater_kwh_scaled']) + # Flattening multi-index columns + df_pivot.columns = [f'{col[0]}_{col[1]}' for col in df_pivot.columns] - # Flattening multi-index columns - df_pivot.columns = [f'{col[0]}_{col[1]}' for col in df_pivot.columns] + # Reset the index to have a clean dataframe + df_pivot.reset_index(inplace=True) - # Reset the index to have a clean dataframe - df_pivot.reset_index(inplace=True) + next_gen_dataset = next_gen_dataset.merge( + df_pivot, how="left", on="uprn" + ) - next_gen_dataset = next_gen_dataset.merge( - df_pivot, how="left", on="uprn" - ) - - next_gen_dataset.to_csv( - "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Newhaven/outputs/next_gen_dataset.csv", index=False - ) + next_gen_dataset.to_csv( + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Newhaven/outputs/next_gen_dataset.csv", index=False + ) From 6a83fe7d9bc509cc8466d2697362875c50472d9b Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 21 Oct 2024 12:28:25 +0100 Subject: [PATCH 154/166] exporting data for newhaven --- backend/ml_models/Valuation.py | 15 ++++---- etl/customers/newhaven/slides.py | 60 +++++++++++++++++++++----------- 2 files changed, 48 insertions(+), 27 deletions(-) diff --git a/backend/ml_models/Valuation.py b/backend/ml_models/Valuation.py index c0e49ffc..c6c1582b 100644 --- a/backend/ml_models/Valuation.py +++ b/backend/ml_models/Valuation.py @@ -204,12 +204,12 @@ class PropertyValuation: @classmethod def estimate(cls, property_instance, target_epc): - value = ( + current_value = ( property_instance.valuation if property_instance.valuation else cls.UPRN_VALUE_LOOKUP.get(property_instance.uprn) ) - if not value: + if not current_value: return { "current_value": 0, "lower_bound_increased_value": 0, @@ -239,12 +239,13 @@ class PropertyValuation: max_increase = max(all_increases) min_increase = min(all_increases) + avg_increase = np.mean(all_increases) return { - "current_value": value, - "lower_bound_increased_value": value * (1 + min_increase), - "upper_bound_increased_value": value * (1 + max_increase), - "average_increased_value": value * (1 + avg_increase), - "average_increase": value * (1 + avg_increase) - value + "current_value": current_value, + "lower_bound_increased_value": current_value * (1 + min_increase), + "upper_bound_increased_value": current_value * (1 + max_increase), + "average_increased_value": current_value * (1 + avg_increase), + "average_increase": current_value * (1 + avg_increase) - current_value } diff --git a/etl/customers/newhaven/slides.py b/etl/customers/newhaven/slides.py index 3c62de53..45108fec 100644 --- a/etl/customers/newhaven/slides.py +++ b/etl/customers/newhaven/slides.py @@ -418,7 +418,7 @@ def slides(): pd.set_option('display.max_colwidth', None) -def app(): +def lewes_outputs(): """ preparing of this data for the following 2 needs: 1) dataset to share with Nextgen heating @@ -525,13 +525,13 @@ def app(): recommendations_df["type"] == "heating", np.where( recommendations_df["description"].str.contains("air source heat pump"), - "air_source_heat_pump", + "Air Source Heat Pump", np.where( recommendations_df["description"].str.contains("high heat retention"), - "high_heat_retention_storage_heaters", + "High Heat Retention Storage", np.where( recommendations_df["description"].str.contains("condensing boiler"), - "boiler_upgrade", + "Boiler Upgrade", recommendations_df["type"] ) ) @@ -552,7 +552,6 @@ def app(): 'room_roof_insulation': 'Room Roof Insulation', 'low_energy_lighting': 'Low Energy Lighting', 'external_wall_insulation': 'External Wall Insulation', - 'heating': 'Heating', 'solar_pv': 'Solar PV', 'heating_control': 'Heating Control', 'solid_floor_insulation': 'Solid Floor Insulation', @@ -634,7 +633,7 @@ def app(): property_scenario_impact = pd.concat(property_scenario_impact) # property_scenario_impact = property_scenario_impact.drop(columns=["property_id", "Estimated Kwh Savings"]) - for v in rename_dict.values(): + for v in list(rename_dict.values()) + ["Air Source Heat Pump", "High Heat Retention Storage", "Boiler Upgrade"]: # Fill NaNs with False property_scenario_impact[v] = property_scenario_impact[v].fillna(False) @@ -731,45 +730,66 @@ def app(): scenario_names, how="left", on="scenario_id" ) - property_scenario_impact = property_scenario_impact.sort_values( - ["postcode", "uprn", "scenario_id"], ascending=True - ) - lewes_data = next_gen_dataset.merge( property_scenario_impact, how="left", on="uprn" ) + lewes_data = lewes_data.sort_values( + ["postcode", "uprn", "scenario_id"], ascending=True + ) + # Rearrange, rename columns and drop what we don't need # TODO - remap the heating type lewes_data = lewes_data[ [ - 'uprn', 'address', 'postcode', 'property_type', 'built_form', 'estimated_heating_hotwater_kwh', + 'uprn', 'address', 'postcode', 'property_type', 'built_form', + # 'estimated_heating_hotwater_kwh', 'primary_fuel_type', 'gross_floor_area', 'floor_height', 'number_of_floors', 'ashp_suitable', 'ashp_size_kw', - 'ashp_cost', 'solar_suitable', 'solar_size_kwp', 'solar_cost', 'estimated_heating_hotwater_kwh_scaled', + 'ashp_cost', 'solar_suitable', 'solar_size_kwp', 'solar_cost', + 'scenario', + 'estimated_heating_hotwater_kwh_scaled', + 'post_scenario_heating_hotwater_kwh_scaled', # 'property_id', - dropped - 'current_energy_demand_heating_hotwater', 'Estimated Heating Demand kWh Savings', - 'Estimated Lighting kWh Savings', 'Estimated Solar kWh Savings', 'estimated_cost', + # 'current_energy_demand_heating_hotwater', + 'Estimated Heating Demand kWh Savings', + 'Estimated Lighting kWh Savings', + 'Estimated Solar kWh Savings', + 'estimated_cost', 'post_scenario_heating_hotwater_kwh', 'Cavity Wall Insulation', 'Cylinder Thermostat', 'Flat Roof Insulation', 'Hot Water Tank Insulation', 'Loft Insulation', 'Mechanical Ventilation', 'Room Roof Insulation', # 'scenario_id', - dropped 'Low Energy Lighting', 'Secondary Heating', 'Windows Glazing', 'External Wall Insulation', - 'Heating', 'Heating Control', 'Solar PV', + 'Air Source Heat Pump', 'Boiler Upgrade', 'High Heat Retention Storage', 'Internal Wall Insulation', 'Solid Floor Insulation', 'Suspended Floor Insulation', - 'post_scenario_heating_hotwater_kwh_scaled', - 'scenario' ] - - ] + ].rename( + columns={ + "primary_fuel_type": "Primary Fuel Type", + "gross_floor_area": "Gross Floor Area", + "floor_height": "Floor Height", + "number_of_floors": "Number of Floors", + "ashp_suitable": "Is an ASHP Suitable?", + "ashp_size_kw": "ASHP Size (kW)", + "ashp_cost": "ASHP Cost", + "solar_suitable": "Is Solar PV Suitable?", + "solar_size_kwp": "Solar PV Size (kWp)", + "solar_cost": "Solar PV Cost", + # "estimated_heating_hotwater_kwh": "Estimated Heating & Hot Water kwh", + "estimated_heating_hotwater_kwh_scaled": "Estimated Heating & Hot Water kwh", + "post_scenario_heating_hotwater_kwh_scaled": "Post Scenario Heating & Hot Water kwh", + "estimated_cost": "Estimated Cost of Scenario" + } + ) # We save this dataset, which will be shared with Lewes Council lewes_data.to_csv( - "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Newhaven/outputs/property data.csv", index=False + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Newhaven/outputs/Lewes property data.csv", index=False ) df_pivot = property_scenario_impact.pivot_table(index='uprn', columns='scenario', From 888357d797837d47f547af759305c2ec64dad296 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 21 Oct 2024 12:36:01 +0100 Subject: [PATCH 155/166] removed old mds app and updating dockerfile --- backend/app/plan/router.py | 588 +------------------------------ backend/docker/lambda.Dockerfile | 15 +- 2 files changed, 6 insertions(+), 597 deletions(-) diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 925fb05b..51b03d01 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -24,7 +24,7 @@ from backend.app.db.functions.recommendations_functions import ( from backend.app.db.functions.energy_assessment_functions import get_latest_assessment_by_uprn from backend.app.db.models.portfolio import rating_lookup from backend.app.dependencies import validate_token -from backend.app.plan.schemas import PlanTriggerRequest, MdsRequest +from backend.app.plan.schemas import PlanTriggerRequest from backend.app.plan.utils import get_cleaned from backend.app.utils import epc_to_sap_lower_bound, sap_to_epc @@ -36,7 +36,6 @@ from recommendations.optimiser.CostOptimiser import CostOptimiser from recommendations.optimiser.GainOptimiser import GainOptimiser from recommendations.optimiser.optimiser_functions import prepare_input_measures from recommendations.Recommendations import Recommendations -from recommendations.Mds import Mds from utils.logger import setup_logger from utils.s3 import read_dataframe_from_s3_parquet, read_csv_from_s3 from backend.ml_models.Valuation import PropertyValuation @@ -44,7 +43,6 @@ from backend.ml_models.Valuation import PropertyValuation from etl.bill_savings.EnergyConsumptionModel import EnergyConsumptionModel from etl.bill_savings.KwhData import KwhData from etl.spatial.OpenUprnClient import OpenUprnClient -from etl.solar.SolarPhotoSupply import SolarPhotoSupply logger = setup_logger() @@ -868,587 +866,3 @@ async def trigger_plan(body: PlanTriggerRequest): session.close() return Response(status_code=200) - - -@router.post("/mds") -async def build_mds(body: MdsRequest): - # TODO: This is a placeholder location for the MDS endpoint, which this is being assembled - - logger.info("Connecting to db") - session = sessionmaker(bind=db_engine)() - created_at = datetime.now().isoformat() - - try: - session.begin() - logger.info("Getting the inputs") - plan_input = read_csv_from_s3(bucket_name=get_settings().PLAN_TRIGGER_BUCKET, filepath=body.trigger_file_path) - measure_set = body.measures - optimise_measures = measure_set is not None - - cleaning_data = read_dataframe_from_s3_parquet( - bucket_name=get_settings().DATA_BUCKET, file_key="sap_change_model/cleaning_dataset.parquet", - ) - - input_properties = [] - for property_id, config in tqdm(enumerate(plan_input), total=len(plan_input)): - # We validate each record in the file. If the record is NOT valid, we need to handle this accordingly - uprn = config.get("uprn", None) - uprn = None if uprn == "" else uprn - if uprn: - uprn = int(float(uprn)) - - epc_searcher = SearchEpc( - address1=config["address"], - postcode=config["postcode"], - uprn=uprn, - auth_token=get_settings().EPC_AUTH_TOKEN, - os_api_key=get_settings().ORDNANCE_SURVEY_API_KEY, - ) - epc_searcher.ordnance_survey_client.built_form = config.get("built_form", None) - epc_searcher.ordnance_survey_client.property_type = config.get("property_type", None) - # For the moment, our OS API access is unavailable, so we skip and interpolate - epc_searcher.find_property(skip_os=True) - - if config["address"] == "35b High Street": - print("Performing temporary patch on 35b High Street") - epc_searcher.newest_epc["uprn"] = 10002911892 - epc_searcher.full_sap_epc["uprn"] = 10002911892 - - if config["address"] == "Cobnut Barn": - print("Performing temporary patch on Cobnut Barn") - epc_searcher.newest_epc["uprn"] = 10013924689 - - # Create a record in db - # TODO: If we productionise the creation of this mds report, we will need to store this in the db - # property_id, is_new = create_property( - # session, body.portfolio_id, epc_searcher.address_clean, epc_searcher.postcode_clean, epc_searcher.uprn - # ) - # if not is_new: - # continue - # - # create_property_targets( - # session, - # property_id=property_id, - # portfolio_id=body.portfolio_id, - # epc_target=body.goal_value, - # heat_demand_target=None - # ) - - epc_records = { - 'original_epc': epc_searcher.newest_epc.copy(), - 'full_sap_epc': epc_searcher.full_sap_epc.copy(), - 'old_data': epc_searcher.older_epcs.copy(), - } - - # patch = next(( - # x for x in patches if (x["address"] == config["address"]) and (x["postcode"] == config["postcode"]) - # ), {}) - # epc_records = patch_epc(patch, epc_records) - - prepared_epc = EPCRecord( - epc_records=epc_records, - run_mode="newdata", - cleaning_data=cleaning_data - ) - - # property_already_installed = next(( - # x for x in already_installed if - # (x["address"] == config["address"]) and (x["postcode"] == config["postcode"]) - # ), {}) - # - # property_non_invasive_recommendations = next(( - # x for x in non_invasive_recommendations if - # (x["address"] == config["address"]) and (x["postcode"] == config["postcode"]) - # ), {}) - - if measure_set is None: - measures = config["measures"] if "measures" in config else None - else: - measures = measure_set - - input_properties.append( - Property( - id=property_id, - address=epc_searcher.address_clean, - postcode=epc_searcher.postcode_clean, - epc_record=prepared_epc, - # already_installed=property_already_installed, - # non_invasive_recommendations=property_non_invasive_recommendations, - measures=measures, - is_new=is_new, - **Property.extract_kwargs(config) - ) - ) - - logger.info("Reading in materials and cleaned datasets") - materials = get_materials(session) - cleaned = get_cleaned() - - uprn_filenames = read_dataframe_from_s3_parquet( - bucket_name=get_settings().DATA_BUCKET, file_key="spatial/filename_meta.parquet" - ) - photo_supply_lookup, floor_area_decile_thresholds = SolarPhotoSupply.load(bucket=get_settings().DATA_BUCKET) - - logger.info("Getting spatial data") - for p in tqdm(input_properties): - p.get_spatial_data(uprn_filenames) - - logger.info("Getting components and epc recommendations") - recommendations_scoring_data = [] - representative_recommendations = {} - recommendations = {} - - for p in tqdm(input_properties): - p.set_features(cleaned, photo_supply_lookup, floor_area_decile_thresholds) - - mds = Mds(property_instance=p, materials=materials, optimise_measures=optimise_measures) - mds_recommendations, property_representative_recommendations, errors = mds.build() - - if isinstance(errors, list): - if errors: - raise Exception("Errors occurred during MDS build") - else: - if any([len(x) for x in errors.values()]): - raise Exception("Errors occurred during MDS build") - - recommendations[p.id] = mds_recommendations - representative_recommendations[p.id] = property_representative_recommendations - - # Build the scoring data - p.create_base_difference_epc_record(cleaned_lookup=cleaned) - if optimise_measures: - for _id, mds_recs in mds_recommendations.items(): - representative_ids = [r["recommendation_id"] for r in property_representative_recommendations[_id]] - simulation_mds_recs = [] - for recs in mds_recs: - simulation_mds_recs.append( - [r for r in recs if r["recommendation_id"] in representative_ids] - ) - - p.adjust_difference_record_with_recommendations( - simulation_mds_recs, property_representative_recommendations[_id] - ) - - data = p.recommendations_scoring_data.copy() - for d in data: - d["id"] = d["id"] + "*" + _id - - recommendations_scoring_data.extend(data) - - else: - recommendations_scoring_data.append( - p.simulate_all_representative_recommendations(property_representative_recommendations) - ) - - logger.info("Preparing data for scoring in sap change api") - recommendations_scoring_data = pd.DataFrame(recommendations_scoring_data) - - recommendations_scoring_data = recommendations_scoring_data.drop( - columns=["rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending", - "carbon_ending"] - ) - - model_api = ModelApi( - portfolio_id=body.portfolio_id, timestamp=created_at, prediction_buckets=get_prediction_buckets() - ) - - all_predictions = { - "sap_change_predictions": pd.DataFrame(), - "heat_demand_predictions": pd.DataFrame(), - "carbon_change_predictions": pd.DataFrame() - } - to_loop_over = range(0, recommendations_scoring_data.shape[0], SCORING_BATCH_SIZE) - for chunk in tqdm(to_loop_over, total=len(to_loop_over)): - predictions_dict = model_api.predict_all( - df=recommendations_scoring_data.iloc[chunk:chunk + SCORING_BATCH_SIZE], - ) - - # Append the predictions to the predictions dictionary - for key, scored in predictions_dict.items(): - all_predictions[key] = pd.concat([all_predictions[key], scored]) - - # TODO: 1) walls_insulation_thickness_ending is not being set in the recommendations_scoring_data, - # insulation_thickness_ending is being set instead - # 2) - - # TODO: TEMP - for p in plan_input: - if p["uprn"]: - p["uprn"] = str(int(float(p["uprn"]))) - - import re - from backend.ml_models.AnnualBillSavings import AnnualBillSavings - - if optimise_measures: - results = [] - for p in input_properties: - - sap_before = int(p.data["current-energy-efficiency"]) - epc_before = p.data["current-energy-rating"] - heat_demand_before = p.data["energy-consumption-current"] - carbon_before = p.data["co2-emissions-current"] - current_adjusted_energy = AnnualBillSavings.adjust_energy_to_metered( - epc_energy_consumption=heat_demand_before * p.floor_area, - current_epc_rating=epc_before, - ) - current_energy_bill = AnnualBillSavings.calculate_annual_bill(current_adjusted_energy) - - package_comparison = [] - for _id in recommendations[p.id].keys(): - - sap_prediction = all_predictions["sap_change_predictions"][ - (all_predictions["sap_change_predictions"]["property_id"] == str(p.id)) & - (all_predictions["sap_change_predictions"]["recommendation_id"].str.contains(re.escape(_id))) - ].copy().reset_index(drop=True) - sap_prediction["row_id"] = sap_prediction.index - - heat_demand_prediction = all_predictions["heat_demand_predictions"][ - (all_predictions["heat_demand_predictions"]["property_id"] == str(p.id)) & - (all_predictions["heat_demand_predictions"]["recommendation_id"].str.contains(re.escape(_id))) - ].copy().reset_index(drop=True) - heat_demand_prediction["row_id"] = heat_demand_prediction.index - - carbon_prediction = all_predictions["carbon_change_predictions"][ - (all_predictions["carbon_change_predictions"]["property_id"] == str(p.id)) & - (all_predictions["carbon_change_predictions"]["recommendation_id"].str.contains(re.escape(_id))) - ].copy().reset_index(drop=True) - carbon_prediction["row_id"] = carbon_prediction.index - - epc_target = body.goal_value - if epc_before == epc_target: - continue - - sap_target = epc_to_sap_lower_bound(epc_target) - # Define the measures - sap_threshold_barrier = sap_prediction[sap_prediction["predictions"] >= sap_target] - meets_threshold = True - if sap_threshold_barrier.empty: - sap_threshold_barrier = sap_prediction.tail(1) - meets_threshold = False - sap_threshold_barrier = sap_threshold_barrier.head(1) - - sap_prediction = sap_prediction[ - sap_prediction["row_id"] <= sap_threshold_barrier["row_id"].values[0] - ] - heat_demand_prediction = heat_demand_prediction[ - heat_demand_prediction["row_id"] <= sap_threshold_barrier["row_id"].values[0] - ] - carbon_prediction = carbon_prediction[ - carbon_prediction["row_id"] <= sap_threshold_barrier["row_id"].values[0] - ] - - reverse_map = {v: k for k, v in Mds.format_map.items()} - - selected_measures = [ - reverse_map[x.split("-")[0]] for x in sap_prediction["recommendation_id"].values - ] - selected_measure_ids = [x.split("*")[0] for x in sap_prediction["recommendation_id"].values] - - costs = [ - r["total"] for r in representative_recommendations[p.id][_id] if - r["recommendation_id"] in selected_measure_ids - ] - costs = sum(costs) - - sap_after = sap_prediction["predictions"].values[-1] - epc_after = sap_to_epc(sap_after) - heat_demand_after = heat_demand_prediction["predictions"].values[-1] - carbon_after = carbon_prediction["predictions"].values[-1] - - expected_adjusted_energy = AnnualBillSavings.adjust_energy_to_metered( - epc_energy_consumption=heat_demand_after * p.floor_area, - current_epc_rating=epc_before, - ) - - expected_energy_bill = AnnualBillSavings.calculate_annual_bill(expected_adjusted_energy) - - bill_savings = current_energy_bill - expected_energy_bill - energy_savings = current_adjusted_energy - expected_adjusted_energy - - package_comparison.append( - { - "id": _id, - "cost": costs, - "measures": selected_measures, - "sap_before": sap_before, - "sap_after": sap_after, - "epc_before": epc_before, - "epc_after": epc_after, - "heat_demand_before": heat_demand_before, - "heat_demand_after": heat_demand_after, - "carbon_before": carbon_before, - "carbon_after": carbon_after, - "bill_savings": bill_savings, - "energy_savings": energy_savings, - "current_energy_bill": current_energy_bill, - "meets_threshold": meets_threshold - } - ) - - package_comparison = pd.DataFrame(package_comparison) - # Find the smallest cost package - if not package_comparison.empty: - - # We check if any of the packages meet the threshold - # If none of them do, take the one that gets closest to the target - if package_comparison["meets_threshold"].any(): - package_comparison = package_comparison[package_comparison["meets_threshold"]] - package_comparison = package_comparison.sort_values("cost") - else: - package_comparison = package_comparison.sort_values("sap_after", ascending=False) - - package_comparison = package_comparison.head(1).to_dict("records")[0] - else: - package_comparison = { - "measures": [], - "sap_before": sap_before, - "sap_after": sap_before, - "epc_before": epc_before, - "epc_after": epc_before, - "heat_demand_before": heat_demand_before, - "heat_demand_after": heat_demand_before, - "carbon_before": carbon_before, - "carbon_after": carbon_before, - "bill_savings": 0, - "energy_savings": 0, - "current_energy_bill": current_energy_bill, - "meets_threshold": False - } - - config = [c for c in plan_input if c["uprn"] == str(p.uprn)] - if not config: - config = {"address": None, "postcode": None} - else: - config = config[0] - - results.append({ - "config_address": config["address"], - "config_postcode": config["postcode"], - "uprn": p.uprn, - "address": p.address, - "postcode": p.postcode, - "measures": package_comparison["measures"], - "year_of_epc": p.data['lodgement-date'], - "sap_before": package_comparison["sap_before"], - "sap_after": package_comparison["sap_after"], - "epc_before": package_comparison["epc_before"], - "epc_after": package_comparison["epc_after"], - "heat_demand_before": package_comparison["heat_demand_before"], - "heat_demand_after": package_comparison["heat_demand_after"], - "carbon_before": package_comparison["carbon_before"], - "carbon_after": package_comparison["carbon_after"], - "bill_savings": round(package_comparison["bill_savings"], 2), - "energy_savings": round(package_comparison["energy_savings"], 2), - "current_energy_bill": round(package_comparison["current_energy_bill"], 2), - "EWI": "EWI" if "external_wall_insulation" in package_comparison["measures"] else None, - "CWI": "CWI" if "cavity_wall_insulation" in package_comparison["measures"] else None, - "LI": "LI" if "loft_insulation" in package_comparison["measures"] else None, - "ASHP Htg": "ASHP Htg" if "air_source_heat_pump" in package_comparison["measures"] else None, - "Elec Storage": ( - "Elec Storage Htrs (Out of scope -Prov sum only)" if "high_heat_retention_storage_heaters" in - package_comparison["measures"] else None - ), - "Solar PV": "Solar PV" if "solar_pv" in package_comparison["measures"] else None, - }) - - results = pd.DataFrame(results) - - # For the different measures, we check the impact with a few debugging functions - - walls_check, hhr_check = check_mds(results, input_properties, recommendations, optimise_measures) - - results.to_excel("optimised mds_results 5th June.xlsx") - - results = [] - for p in input_properties: - measures = p.measures - property_recommendations = [r['type'] for r in representative_recommendations[p.id]] - - # TODO: Check high heat retention storage heaters - looks like it's excluded controls! - - sap_prediction = all_predictions["sap_change_predictions"][ - all_predictions["sap_change_predictions"]["property_id"] == str(p.id) - ] - - heat_demand_prediction = all_predictions["heat_demand_predictions"][ - all_predictions["heat_demand_predictions"]["property_id"] == str(p.id) - ] - - carbon_prediction = all_predictions["carbon_change_predictions"][ - all_predictions["carbon_change_predictions"]["property_id"] == str(p.id) - ] - - # Get a before and after for SAP, heat demand, CO2 and also calculate energy bill and energy savings - sap_before = int(p.data["current-energy-efficiency"]) - sap_after = sap_prediction["predictions"].values[0] if measures else sap_before - - epc_before = p.data["current-energy-rating"] - epc_after = sap_to_epc(sap_after) if measures else epc_before - - heat_demand_before = p.data["energy-consumption-current"] - heat_demand_after = heat_demand_prediction["predictions"].values[0] if measures else heat_demand_before - - carbon_before = p.data["co2-emissions-current"] - carbon_after = carbon_prediction["predictions"].values[0] if measures else carbon_before - - # Estimate bill savings - - from backend.ml_models.AnnualBillSavings import AnnualBillSavings - current_adjusted_energy = AnnualBillSavings.adjust_energy_to_metered( - epc_energy_consumption=heat_demand_before * p.floor_area, - current_epc_rating=epc_before, - ) - - # TODO: This isn't quite right as this is based on EVERY possible measure, not just the ones that are - # actually implemented - expected_adjusted_energy = AnnualBillSavings.adjust_energy_to_metered( - epc_energy_consumption=heat_demand_after * p.floor_area, - current_epc_rating=epc_before, - ) - - # TODO: We should determine if the home is gas & electricity or just electricity - - # Determine if the heating and hotwater was previously electric only or both - - current_energy_bill = AnnualBillSavings.calculate_annual_bill( - kwh=current_adjusted_energy, - ) - expected_energy_bill = AnnualBillSavings.calculate_annual_bill( - kwh=expected_adjusted_energy, - ) - - bill_savings = current_energy_bill - expected_energy_bill - energy_savings = current_adjusted_energy - expected_adjusted_energy - - config = [c for c in plan_input if c["uprn"] == str(p.uprn)] - if not config: - config = {"address": None, "postcode": None} - else: - config = config[0] - - to_append = { - "config_address": config["address"], - "config_postcode": config["postcode"], - "uprn": p.uprn, - "address": p.address, - "postcode": p.postcode, - "measures": measures, - "property_recommendations": property_recommendations, - "year_of_epc": p.data['lodgement-date'], - "sap_before": sap_before, - "sap_after": sap_after, - "epc_before": epc_before, - "epc_after": epc_after, - "heat_demand_before": heat_demand_before, - "heat_demand_after": heat_demand_after, - "carbon_before": carbon_before, - "carbon_after": carbon_after, - "bill_savings": round(bill_savings, 2), - "energy_savings": round(energy_savings, 2), - "current_energy_bill": round(current_energy_bill, 2), - "fuel_type": p.main_fuel["fuel_type"], - } - results.append(to_append) - - results = pd.DataFrame(results) - results["sap_uplift"] = results["sap_after"] - results["sap_before"] - - # results.to_excel("mds_results 5th June.xlsx") - - walls_check, hhr_check = check_mds(results, input_properties, recommendations, optimise_measures) - - except IntegrityError: - logger.error("Database integrity error occurred", exc_info=True) - session.rollback() - return Response(status_code=500, content="Database integrity error.") - except OperationalError: - logger.error("Database operational error occurred", exc_info=True) - session.rollback() - return Response(status_code=500, content="Database operational error.") - except ValueError: - logger.error("Value error - possibly due to malformed data", exc_info=True) - session.rollback() - return Response(status_code=400, content="Bad request: malformed data.") - except Exception as e: # General exception handling - logger.error(f"An error occurred: {e}") - session.rollback() - return Response(status_code=500, content="An unexpected error occurred.") - finally: - session.close() - - -def check_mds(results, input_properties, recommendations, optimise_measures): - import ast - walls_check = [] - hhr_check = [] - for p in input_properties: - res = results[results["uprn"] == p.uprn] - wall = p.walls - heating = p.main_heating - heating_controls = p.main_heating_controls - - if optimise_measures: - measures = res["measures"].values[0] - else: - measures = [list(z.keys())[0] for z in res["measures"].values[0]] - - wall_recommendation = [ - x for x in measures if - x in ["internal_wall_insulation", "external_wall_insulation", "cavity_wall_insulation"] - ] - - hhr_recommendation = [ - x for x in measures if - x in ["high_heat_retention_storage_heaters"] - ] - - if optimise_measures: - possible_measures = [ast.literal_eval(x) for x in list(recommendations[p.id].keys())] - # Unlist them - possible_measures = [x for sublist in possible_measures for x in sublist] - possible_measures = list(set(possible_measures)) - else: - possible_measures = p.measures - - if wall_recommendation: - if len(wall_recommendation) > 1: - raise Exception("something went wrong") - wall_recommendation = wall_recommendation[0] - else: - wall_recommendation = None - - hhr_recommendation = hhr_recommendation[0] if hhr_recommendation else None - - walls_check.append( - { - "uprn": p.uprn, - "address": p.address, - "postcode": p.postcode, - "property_type": p.data['property-type'], - "conservation_status": p.spatial["conservation_status"], - "is_listed_building": p.spatial["is_listed_building"], - "is_heritage_building": p.spatial["is_heritage_building"], - "wall": wall["clean_description"], - "recommendation": wall_recommendation, - "possible_measures": possible_measures, - "selected_measures": res["measures"].values[0], - } - ) - - hhr_check.append( - { - "uprn": p.uprn, - "address": p.address, - "postcode": p.postcode, - "heating": heating["clean_description"], - "heating_controls": heating_controls["clean_description"], - "recommendation": hhr_recommendation, - "possible_measures": possible_measures, - "selected_measures": res["measures"].values[0], - } - ) - - walls_check = pd.DataFrame(walls_check) - hhr_check = pd.DataFrame(hhr_check) - - return walls_check, hhr_check diff --git a/backend/docker/lambda.Dockerfile b/backend/docker/lambda.Dockerfile index 13c30b88..50773fd5 100644 --- a/backend/docker/lambda.Dockerfile +++ b/backend/docker/lambda.Dockerfile @@ -35,17 +35,12 @@ COPY --from=build-image /usr/local/lib/python3.10/site-packages/ /usr/local/lib/ # Copy project files COPY ./backend/ ./backend COPY ./recommendations/ ./recommendations -COPY ./model_data/BaseUtility.py ./model_data/BaseUtility.py -COPY ./model_data/config.py ./model_data/config.py -COPY ./model_data/optimiser/ ./model_data/optimiser/ -COPY ./model_data/__init__.py ./model_data/__init__.py -COPY ./model_data/EpcClean.py ./model_data/EpcClean.py -COPY ./model_data/utils.py ./model_data/utils.py -COPY ./model_data/epc_attributes/ ./model_data/epc_attributes/ -COPY ./model_data/simulation_system/core/DataProcessor.py ./model_data/simulation_system/core/DataProcessor.py -COPY ./model_data/simulation_system/core/Settings.py ./model_data/simulation_system/core/Settings.py -COPY ./datatypes/ ./datatypes/ COPY ./utils/ ./utils/ +COPY ./etl/epc/ ./etl/epc/ +COPY ./etl/epc_clean/ ./etl/epc_clean/ +COPY ./etl/bill_savings/ ./etl/bill_savings/ +COPY ./etl/spatial/ ./etl/spatial/ + # Set the ENTRYPOINT to the AWS Lambda RIC and CMD to your function handler ENTRYPOINT [ "/usr/local/bin/python", "-m", "awslambdaric" ] From 638b88c5f2c3a93913c2e6293d7882e7f87f86aa Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 21 Oct 2024 12:48:36 +0100 Subject: [PATCH 156/166] added docker ignore --- backend/docker/.dockerignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 backend/docker/.dockerignore diff --git a/backend/docker/.dockerignore b/backend/docker/.dockerignore new file mode 100644 index 00000000..601cc053 --- /dev/null +++ b/backend/docker/.dockerignore @@ -0,0 +1,2 @@ +# Ignore all test directories +**/tests \ No newline at end of file From 7e8802a3449572987383f72672658d37e21faad2 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 21 Oct 2024 12:59:41 +0100 Subject: [PATCH 157/166] updating docker ignore --- .dockerignore | 8 ++++++++ backend/docker/.dockerignore | 2 -- 2 files changed, 8 insertions(+), 2 deletions(-) delete mode 100644 backend/docker/.dockerignore diff --git a/.dockerignore b/.dockerignore index 083ae2c7..246f8354 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,9 @@ +# Ignore all test directories model_data/local_data/* backend/tests/* backend/node_modules/* +backend/.idea/* +backend/.env recommendations/tests/* model_data/tests/* infrastructure/* @@ -12,3 +15,8 @@ land_registry/* pytest.ini */README.md utils/tests/* +etl/epc/tests/* +etl/epc_clean/tests/* +etl/spatial/tests/* + + diff --git a/backend/docker/.dockerignore b/backend/docker/.dockerignore deleted file mode 100644 index 601cc053..00000000 --- a/backend/docker/.dockerignore +++ /dev/null @@ -1,2 +0,0 @@ -# Ignore all test directories -**/tests \ No newline at end of file From ed8cbac431dd3b3234d892a7ebbbb9b21d44a206 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 21 Oct 2024 15:53:49 +0100 Subject: [PATCH 158/166] creating new requirements file --- .idea/Model.iml | 2 +- .idea/misc.xml | 2 +- backend/docker/Dockerfile | 17 ++++++----------- backend/docker/lambda.Dockerfile | 8 ++++++-- backend/requirements/base.txt | 3 ++- backend/requirements/requirements.txt | 10 ++++++++++ 6 files changed, 26 insertions(+), 16 deletions(-) create mode 100644 backend/requirements/requirements.txt diff --git a/.idea/Model.iml b/.idea/Model.iml index b0f9c00d..cfc6ba61 100644 --- a/.idea/Model.iml +++ b/.idea/Model.iml @@ -7,7 +7,7 @@ - + diff --git a/.idea/misc.xml b/.idea/misc.xml index 78660f34..4caca8d5 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -3,7 +3,7 @@ - + diff --git a/backend/docker/Dockerfile b/backend/docker/Dockerfile index fd498cdb..122f07ff 100644 --- a/backend/docker/Dockerfile +++ b/backend/docker/Dockerfile @@ -12,10 +12,10 @@ WORKDIR var/task/Model RUN #apt-get update && apt-get install -y netcat-openbsd # Install python dependencies -COPY ./backend/requirements/base.txt ./backend/requirements/base.txt +COPY ./backend/requirements/requirements.txt ./backend/requirements/requirements.txt RUN pip install --upgrade pip # Install and clean up temp caches -RUN pip install -r backend/requirements/base.txt && rm -rf /root/.cache +RUN pip install -r backend/requirements/requirements.txt && rm -rf /root/.cache # Since we are not using a base AWS image, there is some additional setup required. We need to set up the runtime # interface client @@ -35,16 +35,11 @@ COPY --from=build-image /usr/local/lib/python3.10/site-packages/ /usr/local/lib/ # Copy project files COPY ./backend/ ./backend COPY ./recommendations/ ./recommendations -COPY ./model_data/BaseUtility.py ./model_data/BaseUtility.py -COPY ./model_data/config.py ./model_data/config.py -COPY ./model_data/optimiser/ ./model_data/optimiser/ -COPY ./model_data/__init__.py ./model_data/__init__.py -COPY ./model_data/EpcClean.py ./model_data/EpcClean.py -COPT ./model_data/simulation_system/core/ ./model_data/simulation_system/core/ -COPY ./model_data/utils.py ./model_data/utils.py -COPY ./model_data/epc_attributes/ ./model_data/epc_attributes/ -COPY ./datatypes/ ./datatypes/ COPY ./utils/ ./utils/ +COPY ./etl/epc/ ./etl/epc/ +COPY ./etl/epc_clean/ ./etl/epc_clean/ +COPY ./etl/bill_savings/ ./etl/bill_savings/ +COPY ./etl/spatial/ ./etl/spatial/ # Set the ENTRYPOINT to the AWS Lambda RIC and CMD to your function handler ENTRYPOINT [ "/usr/local/bin/python", "-m", "awslambdaric" ] diff --git a/backend/docker/lambda.Dockerfile b/backend/docker/lambda.Dockerfile index 50773fd5..e6f5ea63 100644 --- a/backend/docker/lambda.Dockerfile +++ b/backend/docker/lambda.Dockerfile @@ -12,10 +12,10 @@ WORKDIR var/task/Model #RUN apt-get update && apt-get install -y netcat-openbsd # Install python dependencies -COPY ./backend/requirements/base.txt ./backend/requirements/base.txt +COPY ./backend/requirements/requirements.txt ./backend/requirements/requirements.txt # Install and clean up temp caches RUN pip install --upgrade pip \ - && pip install -r backend/requirements/base.txt && rm -rf /root/.cache + && pip install -r backend/requirements/requirements.txt && rm -rf /root/.cache # Since we are not using a base AWS image, there is some additional setup required. We need to set up the runtime # interface client @@ -26,6 +26,9 @@ RUN pip install awslambdaric # Second stage: "runtime-image" FROM python:3.10.12-slim-buster +# Create the extensions directory to avoid warnings with RIE +RUN mkdir -p /opt/extensions + # Set work directory to the root of your project WORKDIR /var/task/Model @@ -40,6 +43,7 @@ COPY ./etl/epc/ ./etl/epc/ COPY ./etl/epc_clean/ ./etl/epc_clean/ COPY ./etl/bill_savings/ ./etl/bill_savings/ COPY ./etl/spatial/ ./etl/spatial/ +COPY ./BaseUtility.py ./BaseUtility.py # Set the ENTRYPOINT to the AWS Lambda RIC and CMD to your function handler diff --git a/backend/requirements/base.txt b/backend/requirements/base.txt index c4e7367c..c2b4ab71 100644 --- a/backend/requirements/base.txt +++ b/backend/requirements/base.txt @@ -33,7 +33,8 @@ psycopg2-binary pytz==2023.3 mip==1.15.0 boto3==1.28.3 -pandas==1.5.3 +pandas==2.2.3 +numpy==1.24.4 pyarrow==12.0.1 textblob usaddress==0.5.10 diff --git a/backend/requirements/requirements.txt b/backend/requirements/requirements.txt new file mode 100644 index 00000000..f1fa45cb --- /dev/null +++ b/backend/requirements/requirements.txt @@ -0,0 +1,10 @@ +# Pandas and numpy +numpy==2.1.2 +pandas==2.2.3 +pytz==2024.2 +six==1.16.0 +# tqdm +tqdm==4.66.5 +# fastapi +fastapi==0.115.2 +sqlalchemy==2.0.36 From 3d28af415349d44a84bb58a740f0890cfc3c6a5b Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 21 Oct 2024 17:04:37 +0100 Subject: [PATCH 159/166] updating plan trigger for new pydantic --- .idea/Model.iml | 2 +- .idea/misc.xml | 2 +- backend/app/config.py | 2 +- backend/app/plan/schemas.py | 140 +++++++-------------- backend/requirements/requirements.txt | 16 +++ etl/bill_savings/EnergyConsumptionModel.py | 56 ++++----- etl/spatial/OpenUprnClient.py | 12 -- etl/spatial/app.py | 13 +- 8 files changed, 103 insertions(+), 140 deletions(-) diff --git a/.idea/Model.iml b/.idea/Model.iml index cfc6ba61..df6c4faa 100644 --- a/.idea/Model.iml +++ b/.idea/Model.iml @@ -7,7 +7,7 @@ - + diff --git a/.idea/misc.xml b/.idea/misc.xml index 4caca8d5..50cad4ca 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -3,7 +3,7 @@ - + diff --git a/backend/app/config.py b/backend/app/config.py index 9aaa0a52..21e8f21c 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -1,5 +1,5 @@ from functools import lru_cache -from pydantic import BaseSettings +from pydantic_settings import BaseSettings class Settings(BaseSettings): diff --git a/backend/app/plan/schemas.py b/backend/app/plan/schemas.py index e0c5f35d..c1a51769 100644 --- a/backend/app/plan/schemas.py +++ b/backend/app/plan/schemas.py @@ -1,60 +1,25 @@ -from pydantic import BaseModel, conlist, validator -from typing import Optional +from pydantic import BaseModel, Field, BeforeValidator, field_validator +from typing import Annotated, List, Optional +# Example constants for validation TYPICAL_MEASURE_TYPES = [ - "wall_insulation", - "roof_insulation", - "ventilation", - "floor_insulation", - "windows", - "fireplace", - "heating", - "hot_water", - "low_energy_lighting", - "secondary_heating", - "solar_pv" + "wall_insulation", "roof_insulation", "ventilation", "floor_insulation", + "windows", "fireplace", "heating", "hot_water", "low_energy_lighting", + "secondary_heating", "solar_pv" ] SPECIFIC_MEASURES = [ - # Specific measures - # Walls - "internal_wall_insulation", - "external_wall_insulation", - "cavity_wall_insulation", - # Roof - "loft_insulation", - "flat_roof_insulation", - "room_roof_insulation", - # Floor - "suspended_floor_insulation", - "solid_floor_insulation", - # Heating - "boiler_upgrade", - "high_heat_retention_storage_heater", - "air_source_heat_pump", - "secondary_heating", - # Solar - "solar_pv", - # Windows Glazing - "double_glazing", - "secondary_glazing", - # Mechanical ventilation - "ventilation", - # Other - "low_energy_lighting", - "fireplace", - "hot_water", + "internal_wall_insulation", "external_wall_insulation", "cavity_wall_insulation", + "loft_insulation", "flat_roof_insulation", "room_roof_insulation", + "suspended_floor_insulation", "solid_floor_insulation", + "boiler_upgrade", "high_heat_retention_storage_heater", "air_source_heat_pump", + "secondary_heating", "solar_pv", "double_glazing", "secondary_glazing", + "ventilation", "low_energy_lighting", "fireplace", "hot_water" ] NON_INVASIVE_SPECIFIC_MEASURES = [ - # Specific measures that will typically come from an energy assessment - "trickle_vents", - "draught_proofing", - "mixed_glazing", # This covers partial double glazing and secondary glazing - "cavity_extract_and_refill", - # Indicates that there is one (need to handle the case where there are multiple) - # extension that requires cavity wall insulation - "extension_cavity_wall_insulation", + "trickle_vents", "draught_proofing", "mixed_glazing", "cavity_extract_and_refill", + "extension_cavity_wall_insulation" ] # This allows us to extend high level categories for measures such as "wall_insulation" to the specific measures @@ -70,11 +35,37 @@ MEASURE_MAP = { "heating_controls": ["roomstat_programmer_trvs", "time_temperature_zone_control"] } +VALID_GOALS = ["Increasing EPC"] +VALID_HOUSING_TYPES = ["Social", "Private"] + + +# Define the validation function for inclusions/exclusions +def check_inclusion_or_exclusion(value: str) -> str: + if value not in TYPICAL_MEASURE_TYPES + SPECIFIC_MEASURES + NON_INVASIVE_SPECIFIC_MEASURES: + raise ValueError(f"{value} is not an allowed inclusion") + return value + + +def check_goals(value: str) -> str: + assert value in VALID_GOALS, f"{value} is not a valid goal" + return value + + +def check_housing_type(value: str) -> str: + assert value in VALID_HOUSING_TYPES, f"{value} is not a valid housing type" + return value + + +# Use Annotated with BeforeValidator for each list item validation +InclusionOrExclusionItem = Annotated[str, BeforeValidator(check_inclusion_or_exclusion)] +Goal = Annotated[str, BeforeValidator(check_goals)] +HousingType = Annotated[str, BeforeValidator(check_housing_type)] + class PlanTriggerRequest(BaseModel): budget: Optional[float] = None - goal: str - housing_type: str + goal: Goal + housing_type: HousingType goal_value: str portfolio_id: int trigger_file_path: str @@ -82,53 +73,10 @@ class PlanTriggerRequest(BaseModel): patches_file_path: Optional[str] = None non_invasive_recommendations_file_path: Optional[str] = None valuation_file_path: Optional[str] = None - exclusions: Optional[conlist(str, min_items=1)] = None - inclusions: Optional[conlist(str, min_items=1)] = None + exclusions: Optional[List[InclusionOrExclusionItem]] = Field(default=None, min_length=1) + inclusions: Optional[List[InclusionOrExclusionItem]] = Field(default=None, min_length=1) scenario_name: Optional[str] = "" - # If true, will allow us to create multiple plans for the same portfolio, whereas if this is false, if this property - # exists in the portfolio, it will be ignored multi_plan: Optional[bool] = False - - # if False, allows optimisation to be switched off optimise: Optional[bool] = True - - # If True, uses default u-values for models default_u_values: Optional[bool] = True - - _allowed_goals = {"Increasing EPC"} - - _allowed_housing_types = {"Social", "Private"} - - # Validator to ensure exclusions are within the pre-defined possibilities - @validator('exclusions', each_item=True) - def check_exclusions(cls, v): - if v not in TYPICAL_MEASURE_TYPES + SPECIFIC_MEASURES + NON_INVASIVE_SPECIFIC_MEASURES: - raise ValueError(f"{v} is not an allowed exclusion") - return v - - @validator('inclusions', each_item=True) - def check_inclusions(cls, v): - if v not in TYPICAL_MEASURE_TYPES + SPECIFIC_MEASURES + NON_INVASIVE_SPECIFIC_MEASURES: - raise ValueError(f"{v} is not an allowed inclusion") - return v - - # Validator to ensure that the goal is within the pre-defined possibilities - @validator('goal') - def check_goal(cls, v): - if v not in cls._allowed_goals: - raise ValueError(f"{v} is not a valid goal") - return v - - # Validator to ensure that the housing type is within the pre-defined possibilities - @validator('housing_type') - def check_housing_type(cls, v): - if v not in cls._allowed_housing_types: - raise ValueError(f"{v} is not a valid housing type") - return v - - -class MdsRequest(PlanTriggerRequest): - # When creating the mds report, we allow an optional list of measures to select from. If this is passed, it will - # cause the service to select the optimal package from the list of measures - measures: Optional[conlist(str, min_items=1)] = None diff --git a/backend/requirements/requirements.txt b/backend/requirements/requirements.txt index f1fa45cb..82e44fcf 100644 --- a/backend/requirements/requirements.txt +++ b/backend/requirements/requirements.txt @@ -8,3 +8,19 @@ tqdm==4.66.5 # fastapi fastapi==0.115.2 sqlalchemy==2.0.36 +pydantic-settings==2.6.0 +psycopg2-binary==2.9.10 +python-jose==3.3.0 +cryptography==43.0.3 +# AWS +boto3==1.35.44 +# ML, Data, Data Science +usaddress==0.5.11 +epc-api-python==1.0.2 +fuzzywuzzy==0.18.0 +python-Levenshtein==0.26.0 +textblob==0.18.0.post0 +msgpack==1.1.0 +scikit-learn==1.5.2 +cffi==1.15.1 +mip==1.15.0 diff --git a/etl/bill_savings/EnergyConsumptionModel.py b/etl/bill_savings/EnergyConsumptionModel.py index 4daf2b31..153f4ee2 100644 --- a/etl/bill_savings/EnergyConsumptionModel.py +++ b/etl/bill_savings/EnergyConsumptionModel.py @@ -1,6 +1,6 @@ import pandas as pd import numpy as np -from xgboost import XGBRegressor +# from xgboost import XGBRegressor from sklearn.model_selection import train_test_split from sklearn.metrics import mean_squared_error, r2_score, mean_absolute_percentage_error from sklearn.feature_selection import RFECV @@ -278,33 +278,33 @@ class EnergyConsumptionModel: logger.info(f"Feature selection completed for target {target}") - def init_model(self, feature_selection=False): - - if feature_selection: - # Set up a smaller model to work it - return XGBRegressor( - objective='reg:squarederror', - n_estimators=50, - learning_rate=0.05, - max_depth=6, - subsample=0.8, - colsample_bytree=0.8, - reg_alpha=0.1, - reg_lambda=0.1 - ) - - return XGBRegressor( - objective='reg:squarederror', - n_estimators=1000, - learning_rate=0.05, - max_depth=6, - min_child_weight=3, - subsample=0.8, - colsample_bytree=0.8, - reg_alpha=0.1, - reg_lambda=0.1 - # n_jobs=self.n_jobs - ) + # def init_model(self, feature_selection=False): + # + # if feature_selection: + # # Set up a smaller model to work it + # return XGBRegressor( + # objective='reg:squarederror', + # n_estimators=50, + # learning_rate=0.05, + # max_depth=6, + # subsample=0.8, + # colsample_bytree=0.8, + # reg_alpha=0.1, + # reg_lambda=0.1 + # ) + # + # return XGBRegressor( + # objective='reg:squarederror', + # n_estimators=1000, + # learning_rate=0.05, + # max_depth=6, + # min_child_weight=3, + # subsample=0.8, + # colsample_bytree=0.8, + # reg_alpha=0.1, + # reg_lambda=0.1 + # # n_jobs=self.n_jobs + # ) def fit_model(self, target): """Fits the model to the training data and removes zero-importance features.""" diff --git a/etl/spatial/OpenUprnClient.py b/etl/spatial/OpenUprnClient.py index 5c43347a..c0cd3992 100644 --- a/etl/spatial/OpenUprnClient.py +++ b/etl/spatial/OpenUprnClient.py @@ -1,7 +1,6 @@ import os from tqdm import tqdm import pandas as pd -import geopandas as gpd from utils.logger import setup_logger from utils.s3 import read_io_from_s3, save_dataframe_to_s3_parquet, read_dataframe_from_s3_parquet from backend.Property import Property @@ -86,17 +85,6 @@ class OpenUprnClient: return filename return None - @staticmethod - def convert_bng_data_to_gpd(df): - - gpd_data = gpd.GeoDataFrame( - df, - geometry=gpd.points_from_xy(df.X_COORDINATE, df.Y_COORDINATE), - crs="EPSG:27700" # British National Grid - ) - - return gpd_data - def save_filenames_to_s3(self, bucket_name): """ Save the filenames to s3 diff --git a/etl/spatial/app.py b/etl/spatial/app.py index d58509dd..e8055432 100644 --- a/etl/spatial/app.py +++ b/etl/spatial/app.py @@ -6,6 +6,7 @@ our database for querying from other services import os from tqdm import tqdm import pandas as pd +import geopandas as gpd from etl.spatial.ConservationAreaClient import ConservationAreaClient from etl.spatial.OpenUprnClient import OpenUprnClient from etl.spatial.SpecialBuildingsClient import SpecialBuildingsClient @@ -25,6 +26,16 @@ HISTORIC_ENGLAND_HERITAGE_BUILDINGS_PATHNAME = \ logger = setup_logger() +def convert_bng_data_to_gpd(df): + gpd_data = gpd.GeoDataFrame( + df, + geometry=gpd.points_from_xy(df.X_COORDINATE, df.Y_COORDINATE), + crs="EPSG:27700" # British National Grid + ) + + return gpd_data + + def app(): """ This application uses the conservation area datasets to determine if a UPRN is @@ -85,7 +96,7 @@ def app(): to_loop_over = open_uprn_client.data.groupby("filename") for filename, uprn_df in tqdm(open_uprn_client.data.groupby("filename"), total=len(to_loop_over)): - uprn_gdf = OpenUprnClient.convert_bng_data_to_gpd(uprn_df) + uprn_gdf = convert_bng_data_to_gpd(uprn_df) uprn_gdf = conservation_area_client.is_in_conservation_area_vectorised(uprn_gdf=uprn_gdf) uprn_gdf = special_buildings_client.is_listed_building_vectorised(uprn_gdf=uprn_gdf) From a2c61395de78fb175c8207ebc698fcca2f4eef6b Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 21 Oct 2024 18:34:18 +0100 Subject: [PATCH 160/166] new environment working --- .../db/functions/recommendations_functions.py | 22 +++++++++---------- backend/app/plan/router.py | 1 + backend/ml_models/Valuation.py | 8 +++---- backend/requirements/requirements.txt | 3 +++ etl/bill_savings/EnergyConsumptionModel.py | 14 ++++++------ etl/epc/DataProcessor.py | 16 +++++++------- etl/epc/settings.py | 7 ++++-- recommendations/Recommendations.py | 16 +++++++------- utils/s3.py | 2 +- 9 files changed, 48 insertions(+), 41 deletions(-) diff --git a/backend/app/db/functions/recommendations_functions.py b/backend/app/db/functions/recommendations_functions.py index feeced10..d6e41c61 100644 --- a/backend/app/db/functions/recommendations_functions.py +++ b/backend/app/db/functions/recommendations_functions.py @@ -110,19 +110,19 @@ def upload_recommendations(session: Session, recommendations_to_upload, property "type": rec["type"], "measure_type": rec["measure_type"], "description": rec["description"], - "estimated_cost": rec["total"], + "estimated_cost": float(rec["total"]), "default": rec["default"], - "starting_u_value": rec.get("starting_u_value"), - "new_u_value": rec.get("new_u_value"), - "sap_points": rec["sap_points"], - "energy_savings": rec["heat_demand"], - "kwh_savings": rec["kwh_savings"], - "co2_equivalent_savings": rec["co2_equivalent_savings"], - "total_work_hours": rec["labour_hours"], - "energy_cost_savings": rec["energy_cost_savings"], - "labour_days": rec["labour_days"], + "starting_u_value": float(rec.get("starting_u_value")) if rec.get("starting_u_value") else None, + "new_u_value": float(rec.get("new_u_value")) if rec.get("new_u_value") else None, + "sap_points": float(rec["sap_points"]), + "energy_savings": float(rec["heat_demand"]), + "kwh_savings": float(rec["kwh_savings"]), + "co2_equivalent_savings": float(rec["co2_equivalent_savings"]), + "total_work_hours": float(rec["labour_hours"]), + "energy_cost_savings": float(rec["energy_cost_savings"]), + "labour_days": float(rec["labour_days"]), "already_installed": rec["already_installed"], - "heat_demand": rec["heat_demand"] + "heat_demand": float(rec["heat_demand"]) } for rec in recommendations_to_upload ] diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 51b03d01..074b5b75 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -774,6 +774,7 @@ async def trigger_plan(body: PlanTriggerRequest): update_or_create_property_spatial_details(session, p.uprn, p.spatial) property_data = p.get_full_property_data(current_valuation=valuations["current_value"]) + update_property_data( session, property_id=p.id, portfolio_id=body.portfolio_id, property_data=property_data ) diff --git a/backend/ml_models/Valuation.py b/backend/ml_models/Valuation.py index c6c1582b..92c55641 100644 --- a/backend/ml_models/Valuation.py +++ b/backend/ml_models/Valuation.py @@ -244,8 +244,8 @@ class PropertyValuation: return { "current_value": current_value, - "lower_bound_increased_value": current_value * (1 + min_increase), - "upper_bound_increased_value": current_value * (1 + max_increase), - "average_increased_value": current_value * (1 + avg_increase), - "average_increase": current_value * (1 + avg_increase) - current_value + "lower_bound_increased_value": float(current_value * (1 + min_increase)), + "upper_bound_increased_value": float(current_value * (1 + max_increase)), + "average_increased_value": float(current_value * (1 + avg_increase)), + "average_increase": float(current_value * (1 + avg_increase) - current_value) } diff --git a/backend/requirements/requirements.txt b/backend/requirements/requirements.txt index 82e44fcf..11f29183 100644 --- a/backend/requirements/requirements.txt +++ b/backend/requirements/requirements.txt @@ -24,3 +24,6 @@ msgpack==1.1.0 scikit-learn==1.5.2 cffi==1.15.1 mip==1.15.0 +pyarrow==17.0.0 +fastparquet==2024.5.0 + diff --git a/etl/bill_savings/EnergyConsumptionModel.py b/etl/bill_savings/EnergyConsumptionModel.py index 153f4ee2..1ccfee60 100644 --- a/etl/bill_savings/EnergyConsumptionModel.py +++ b/etl/bill_savings/EnergyConsumptionModel.py @@ -79,13 +79,13 @@ class EnergyConsumptionModel: if x not in self.CATEGORICAL_COLUMNS }) - if model_paths: - for target, path in model_paths.items(): - # Read model - self.models[target] = read_pickle_from_s3( - bucket_name=f"retrofit-model-directory-{environment}", s3_file_name=path - ) - # Read dummy schema + # if model_paths: + # for target, path in model_paths.items(): + # # Read model + # self.models[target] = read_pickle_from_s3( + # bucket_name=f"retrofit-model-directory-{environment}", s3_file_name=path + # ) + # Read dummy schema if dummy_schema_path: self.dummy_schema = read_pickle_from_s3( diff --git a/etl/epc/DataProcessor.py b/etl/epc/DataProcessor.py index 4ad854c1..9655cf77 100644 --- a/etl/epc/DataProcessor.py +++ b/etl/epc/DataProcessor.py @@ -263,7 +263,7 @@ class EPCDataProcessor: # Use replace function to map data (if exists in key), to corresponding value - i.e. Remove invalid values data = self.data.replace(data_anomaly_map) - data = data.replace(np.NAN, None) + data = data.replace(np.nan, None) self.data = data @@ -384,7 +384,7 @@ class EPCDataProcessor: has_missings = pd.isnull(self.data[col]).sum() while has_missings: self.data = apply_clean( - data=self.data, matching_columns=matching_columns[0 : to_index + 1] + data=self.data, matching_columns=matching_columns[0: to_index + 1] ) has_missings = pd.isnull(self.data[col]).sum() @@ -487,7 +487,7 @@ class EPCDataProcessor: filled_data = ( self.data.groupby("UPRN", group_keys=True)[columns_to_fill] - .apply(lambda group: group.fillna(method="bfill").fillna(method="ffill")) + .apply(lambda group: group.bfill().ffill().infer_objects(copy=False)) .reset_index() .set_index("level_1") .sort_index() @@ -791,7 +791,7 @@ class EPCDataProcessor: We fill photo supply with zeros where it's missing """ - self.data["PHOTO_SUPPLY"] = self.data["PHOTO_SUPPLY"].fillna(0) + self.data["PHOTO_SUPPLY"] = self.data["PHOTO_SUPPLY"].astype("Int64").fillna(0) @staticmethod def apply_averages_cleaning( @@ -858,12 +858,12 @@ class EPCDataProcessor: # Fill NaN values with averages for col in cols_to_clean: - data_to_clean[col].fillna(data_to_clean[f"{col}_AVERAGE"], inplace=True) - data_to_clean.drop(columns=[f"{col}_AVERAGE"], inplace=True) + data_to_clean[col] = data_to_clean[col].fillna(data_to_clean[f"{col}_AVERAGE"]) + data_to_clean = data_to_clean.drop(columns=[f"{col}_AVERAGE"]) # If we still have missings - data_to_clean[col].fillna(data_to_clean[col].mean(), inplace=True) + data_to_clean[col] = data_to_clean[col].fillna(data_to_clean[col].mean()) # Final step if we still have missings - use global mean - data_to_clean[col].fillna(global_averages[col], inplace=True) + data_to_clean[col] = data_to_clean[col].fillna(global_averages[col]) return data_to_clean diff --git a/etl/epc/settings.py b/etl/epc/settings.py index a814750f..2a9b1746 100644 --- a/etl/epc/settings.py +++ b/etl/epc/settings.py @@ -182,7 +182,6 @@ EFFICIENCY_FEATURES = [ ROOM_FEATURES = ["number_habitable_rooms", "number_heated_rooms"] - COMPONENT_FEATURES = CORE_COMPONENT_FEATURES + [ "TRANSACTION_TYPE", "ENERGY_TARIFF", # Not sure if this is relevant @@ -241,7 +240,11 @@ BUILT_FORM_REMAP = { DATA_PROCESSOR_SETTINGS = { "low_memory": False, "epc_minimum_count": 1, - "column_mappings": {"UPRN": [int, str]}, + "column_mappings": { + "UPRN": [int, str], + "NUMBER_HEATED_ROOMS": [float], + "NUMBER_HABITABLE_ROOMS": [float], + }, } # This has a manual mapping of the column types required diff --git a/recommendations/Recommendations.py b/recommendations/Recommendations.py index aa7e041e..dd51b47d 100644 --- a/recommendations/Recommendations.py +++ b/recommendations/Recommendations.py @@ -730,8 +730,8 @@ class Recommendations: "id": STARTING_DUMMY_ID_VALUE, "phase": STARTING_DUMMY_ID_VALUE, "recommendation_id": STARTING_DUMMY_ID_VALUE, - "predictions_heating": property_kwh["heating"], - "predictions_hotwater": property_kwh["hot_water"], + "predictions_heating": float(property_kwh["heating"]), + "predictions_hotwater": float(property_kwh["hot_water"]), } ] ), @@ -854,12 +854,12 @@ class Recommendations: # We return a dictionary that contains the individual costs, that can be stored to the database current_energy_bill = { - "heating_cost_current": starting_figures["heating_cost"], - "hot_water_cost_current": starting_figures["hotwater_cost"], - "lighting_cost_current": property_instance.energy_cost_estimates["unadjusted"]["lighting"], - "appliances_cost_current": property_instance.energy_cost_estimates["unadjusted"]["appliances"], - "gas_standing_charge": gas_standing_charge, - "electricity_standing_charge": electricity_standing_charge, + "heating_cost_current": float(starting_figures["heating_cost"]), + "hot_water_cost_current": float(starting_figures["hotwater_cost"]), + "lighting_cost_current": float(property_instance.energy_cost_estimates["unadjusted"]["lighting"]), + "appliances_cost_current": float(property_instance.energy_cost_estimates["unadjusted"]["appliances"]), + "gas_standing_charge": float(gas_standing_charge), + "electricity_standing_charge": float(electricity_standing_charge), } return current_energy_bill diff --git a/utils/s3.py b/utils/s3.py index ca0cbfac..1a686b55 100644 --- a/utils/s3.py +++ b/utils/s3.py @@ -192,7 +192,7 @@ def read_pickle_from_s3(bucket_name, s3_file_name): try: data = pickle.loads(serialized_data) except Exception as e: - logger.errpr(f'Failed to deserialize data: {str(e)}') + logger.error(f'Failed to deserialize data: {str(e)}') return None return data From d80f0b8fd2cb255026fc5eddd094e9c13a08a518 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 21 Oct 2024 18:35:08 +0100 Subject: [PATCH 161/166] deleted old requirements files --- backend/requirements/base.txt | 43 ---------------------------------- backend/requirements/local.txt | 28 ---------------------- 2 files changed, 71 deletions(-) delete mode 100644 backend/requirements/base.txt delete mode 100644 backend/requirements/local.txt diff --git a/backend/requirements/base.txt b/backend/requirements/base.txt deleted file mode 100644 index c2b4ab71..00000000 --- a/backend/requirements/base.txt +++ /dev/null @@ -1,43 +0,0 @@ -msgpack==1.0.5 -anyio==3.7.1 -cffi==1.15.1 -click==8.1.3 -cryptography==37.0.4 -ecdsa==0.18.0 -epc-api-python==1.0.2 -exceptiongroup==1.1.2 -fastapi==0.99.1 -h11==0.14.0 -httptools==0.5.0 -idna==3.4 -mangum==0.17.0 -pyasn1==0.5.0 -pycparser==2.21 -pydantic==1.10.11 -PyJWT==2.7.0 -python-dotenv==1.0.0 -python-jose==3.3.0 -PyYAML==6.0 -rsa==4.9 -six==1.16.0 -sniffio==1.3.0 -starlette==0.27.0 -typing_extensions==4.7.1 -uvicorn==0.22.0 -uvloop==0.17.0 -urllib3<2 -watchfiles==0.19.0 -websockets==11.0.3 -sqlalchemy==2.0.19 -psycopg2-binary -pytz==2023.3 -mip==1.15.0 -boto3==1.28.3 -pandas==2.2.3 -numpy==1.24.4 -pyarrow==12.0.1 -textblob -usaddress==0.5.10 - -# Requirements we may not need -xgboost==1.7.6 \ No newline at end of file diff --git a/backend/requirements/local.txt b/backend/requirements/local.txt deleted file mode 100644 index 5a1693c4..00000000 --- a/backend/requirements/local.txt +++ /dev/null @@ -1,28 +0,0 @@ -anyio==3.7.1 -cffi==1.15.1 -click==8.1.3 -cryptography==37.0.4 -ecdsa==0.18.0 -exceptiongroup==1.1.2 -fastapi==0.99.1 -h11==0.14.0 -httptools==0.5.0 -idna==3.4 -mangum==0.17.0 -pyasn1==0.5.0 -pycparser==2.21 -pydantic==1.10.11 -PyJWT==2.7.0 -python-dotenv==1.0.0 -python-jose==3.3.0 -PyYAML==6.0 -rsa==4.9 -six==1.16.0 -sniffio==1.3.0 -starlette==0.27.0 -typing_extensions==4.7.1 -uvicorn==0.22.0 -uvloop==0.17.0 -watchfiles==0.19.0 -websockets==11.0.3 -boto3 \ No newline at end of file From 47c9f26260040e7760e4c7573812b1682519af77 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 22 Oct 2024 10:13:53 +0100 Subject: [PATCH 162/166] fixing the docker to use python 3.11 --- backend/docker/Dockerfile | 2 +- backend/docker/lambda.Dockerfile | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/docker/Dockerfile b/backend/docker/Dockerfile index 122f07ff..5302476d 100644 --- a/backend/docker/Dockerfile +++ b/backend/docker/Dockerfile @@ -1,5 +1,5 @@ # Pull base image -FROM python:3.10.12-slim-buster as build-image +FROM python:3.11.10-slim-bullseye as build-image # Set environment variables ENV PYTHONDONTWRITEBYTECODE 1 diff --git a/backend/docker/lambda.Dockerfile b/backend/docker/lambda.Dockerfile index e6f5ea63..763a24b2 100644 --- a/backend/docker/lambda.Dockerfile +++ b/backend/docker/lambda.Dockerfile @@ -1,5 +1,5 @@ # Pull base image -FROM python:3.10.12-slim-buster as build-image +FROM python:3.11.10-slim-bullseye as build-image # Set environment variables ENV PYTHONDONTWRITEBYTECODE 1 @@ -24,7 +24,7 @@ RUN pip install --upgrade pip \ RUN pip install awslambdaric # Second stage: "runtime-image" -FROM python:3.10.12-slim-buster +FROM python:3.11.10-slim-bullseye # Create the extensions directory to avoid warnings with RIE RUN mkdir -p /opt/extensions @@ -33,7 +33,7 @@ RUN mkdir -p /opt/extensions WORKDIR /var/task/Model # Copy the python dependencies from the build-image -COPY --from=build-image /usr/local/lib/python3.10/site-packages/ /usr/local/lib/python3.10/site-packages/ +COPY --from=build-image /usr/local/lib/python3.11/site-packages/ /usr/local/lib/python3.11/site-packages/ # Copy project files COPY ./backend/ ./backend From 5e5609a7caf3320166a4414c6da5c25e11e0a3f8 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 22 Oct 2024 11:08:48 +0100 Subject: [PATCH 163/166] making model api async --- backend/apis/GoogleSolarApi.py | 59 +++++++++++-- backend/app/plan/router.py | 49 ++++++----- backend/docker/Dockerfile | 1 + backend/docker/lambda.Dockerfile | 1 + backend/ml_models/api.py | 114 ++++++++++++++++++++++++++ backend/requirements/requirements.txt | 6 +- 6 files changed, 197 insertions(+), 33 deletions(-) diff --git a/backend/apis/GoogleSolarApi.py b/backend/apis/GoogleSolarApi.py index ed7b7422..75f28ceb 100644 --- a/backend/apis/GoogleSolarApi.py +++ b/backend/apis/GoogleSolarApi.py @@ -589,14 +589,61 @@ class GoogleSolarApi: self.double_property = True @staticmethod + def calculate_percentage_decrease(start_efficiency, end_efficiency, consumption_averages): + """ + Calculate the percentage decrease in consumption between two energy efficiency ratings. + :param start_efficiency: The starting energy efficiency rating. + :param end_efficiency: The ending energy efficiency rating. + :param consumption_averages: The DataFrame containing the consumption averages. + :return: + """ + + start_consumption = consumption_averages.loc[ + consumption_averages["current-energy-efficiency"].astype(str) == str(start_efficiency), "total_consumption" + ].values[0] + + end_consumption = consumption_averages.loc[ + consumption_averages["current-energy-efficiency"].astype(str) == str(end_efficiency), "total_consumption" + ].values[0] + + percentage_decrease = ((start_consumption - end_consumption) / start_consumption) * 100 + # percentage_decrease cannot be nehative + if percentage_decrease < 0: + percentage_decrease = 0 + return percentage_decrease + + @classmethod + def estimate_new_consumption( + cls, current_energy_efficiency, target_efficiency, current_consumption, ofgem_consumption_averages + ): + """ + Given then consumption_averages dataset, which is produced as a result of the training_data.py script, + for the energy kwh models, this function will estimate the new consumption based on the current consumption, + based on the expected reduction in consumption from the current rating to the target rating. + :param current_energy_efficiency: The current energy efficiency rating + :param target_efficiency: The target energy efficiency rating + :param current_consumption: The current consumption of the property + :param ofgem_consumption_averages: DataFrame of the Ofgem consumption averages + :return: + """ + percentage_decrease = cls.calculate_percentage_decrease( + start_efficiency=current_energy_efficiency, + end_efficiency=target_efficiency, + consumption_averages=ofgem_consumption_averages + ) + new_consumption = current_consumption * (1 - percentage_decrease / 100) + return new_consumption + + @classmethod def prepare_input_data( + cls, input_properties: List[Property], - energy_consumption_client: EnergyConsumptionModel, + ofgem_consumption_averages: pd.DataFrame, body: PlanTriggerRequest ): """ :param input_properties: List of properties - :param energy_consumption_client: EnergyConsumptionModel instance + :param ofgem_consumption_averages: DataFrame of the Ofgem consumption averages :param body: PlanTriggerRequest instance This sets up the data required to make the solar api request :return: @@ -610,12 +657,13 @@ class GoogleSolarApi: # Energy consumption is adjusted for the property's expected post retrofit state # We set the target rating to EPC C, which is the typical EPC rating we would expect the # property to achieve post retrofit of just the fabric - "energy_consumption": energy_consumption_client.estimate_new_consumption( + "energy_consumption": cls.estimate_new_consumption( current_energy_efficiency=p.data["current-energy-efficiency"], target_efficiency="69", current_consumption=p.estimate_electrical_consumption( assumed_ashp_efficiency=assumptions.AVERAGE_ASHP_EFFICIENCY, exclusions=body.exclusions - ) + ), + ofgem_consumption_averages=ofgem_consumption_averages ), "property_id": p.id, "uprn": p.uprn @@ -628,12 +676,13 @@ class GoogleSolarApi: # Energy consumption is adjusted for the property's expected post retrofit state # We set the target rating to EPC C, which is the typical EPC rating we would expect the # property to achieve post retrofit of just the fabric - "energy_consumption": energy_consumption_client.estimate_new_consumption( + "energy_consumption": cls.estimate_new_consumption( current_energy_efficiency=p.data["current-energy-efficiency"], target_efficiency="69", current_consumption=p.estimate_electrical_consumption( assumed_ashp_efficiency=assumptions.AVERAGE_ASHP_EFFICIENCY, exclusions=body.exclusions ), + ofgem_consumption_averages=ofgem_consumption_averages ), "property_id": p.id, "uprn": p.uprn diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 074b5b75..26c84c81 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -40,7 +40,6 @@ from utils.logger import setup_logger from utils.s3 import read_dataframe_from_s3_parquet, read_csv_from_s3 from backend.ml_models.Valuation import PropertyValuation -from etl.bill_savings.EnergyConsumptionModel import EnergyConsumptionModel from etl.bill_savings.KwhData import KwhData from etl.spatial.OpenUprnClient import OpenUprnClient @@ -486,6 +485,16 @@ async def trigger_plan(body: PlanTriggerRequest): if not input_properties: return Response(status_code=204) + # Set up model api and warm up the lambdas + model_api = ModelApi( + portfolio_id=body.portfolio_id, + timestamp=created_at, + prediction_buckets=get_prediction_buckets() + ) + await model_api.async_warm_up_lambdas( + model_prefies=model_api.KWH_MODEL_PREFIXES + model_api.MODEL_PREFIXES + ) + # The materials data could be cached or local so we don't need to make # consistent requests to the backend for # the same data @@ -493,32 +502,14 @@ async def trigger_plan(body: PlanTriggerRequest): materials = get_materials(session) cleaned = get_cleaned() - dataset_version = "2024-07-08" - energy_consumption_client = EnergyConsumptionModel( - model_paths={ - "heating_kwh": f"model_directory/energy_consumption_model/heating_kwh_{dataset_version}.pkl", - "hot_water_kwh": f"model_directory/energy_consumption_model/hot_water_kwh_{dataset_version}.pkl" - }, - dummy_schema_path=f"model_directory/energy_consumption_model/{dataset_version}_dummy_schema.pkl", - consumption_average_path=f"energy_consumption/{dataset_version}/consumption_averages.parquet", - cleaned=cleaned, - environment=get_settings().ENVIRONMENT - ) - kwh_client = KwhData(bucket=get_settings().DATA_BUCKET, read_consumption_data=True) - model_api = ModelApi( - portfolio_id=body.portfolio_id, - timestamp=created_at, - prediction_buckets=get_prediction_buckets() - ) - epcs_for_scoring = kwh_client.transform(data=kwh_client.prepare_epc(input_properties), cleaned=cleaned) - kwh_preds = model_api.paginated_predictions( + kwh_preds = await model_api.async_paginated_predictions( data=epcs_for_scoring, bucket=get_settings().DATA_BUCKET, - model_prefixes=["heating_kwh_predictions", "hotwater_kwh_predictions"], + model_prefixes=model_api.KWH_MODEL_PREFIXES, extract_ids=False, batch_size=SCORING_BATCH_SIZE ) @@ -534,9 +525,15 @@ async def trigger_plan(body: PlanTriggerRequest): # extensions, since it doesn't seem to do a great job logger.info("Performing solar analysis") + + ofgem_consumption_averages = read_dataframe_from_s3_parquet( + bucket_name=get_settings().DATA_BUCKET, + file_key=f"energy_consumption/2024-07-08/consumption_averages.parquet" + ) + building_solar_config, unit_solar_config = GoogleSolarApi.prepare_input_data( input_properties=input_properties, - energy_consumption_client=energy_consumption_client, + ofgem_consumption_averages=ofgem_consumption_averages, body=body ) @@ -591,7 +588,7 @@ async def trigger_plan(body: PlanTriggerRequest): "carbon_ending"] ) - all_predictions = model_api.paginated_predictions( + all_predictions = await model_api.async_paginated_predictions( data=recommendations_scoring_data, bucket=get_settings().DATA_BUCKET, batch_size=SCORING_BATCH_SIZE @@ -620,10 +617,10 @@ async def trigger_plan(body: PlanTriggerRequest): scoring_epcs = pd.DataFrame(scoring_epcs) scoring_epcs = kwh_client.transform(data=scoring_epcs, cleaned=cleaned) - kwh_simulation_predictions = model_api.paginated_predictions( + kwh_simulation_predictions = await model_api.async_paginated_predictions( data=scoring_epcs, bucket=get_settings().DATA_BUCKET, - model_prefixes=["heating_kwh_predictions", "hotwater_kwh_predictions"], + model_prefixes=model_api.KWH_MODEL_PREFIXES, batch_size=SCORING_BATCH_SIZE ) @@ -774,7 +771,7 @@ async def trigger_plan(body: PlanTriggerRequest): update_or_create_property_spatial_details(session, p.uprn, p.spatial) property_data = p.get_full_property_data(current_valuation=valuations["current_value"]) - + update_property_data( session, property_id=p.id, portfolio_id=body.portfolio_id, property_data=property_data ) diff --git a/backend/docker/Dockerfile b/backend/docker/Dockerfile index 5302476d..006f088a 100644 --- a/backend/docker/Dockerfile +++ b/backend/docker/Dockerfile @@ -40,6 +40,7 @@ COPY ./etl/epc/ ./etl/epc/ COPY ./etl/epc_clean/ ./etl/epc_clean/ COPY ./etl/bill_savings/ ./etl/bill_savings/ COPY ./etl/spatial/ ./etl/spatial/ +COPY ./datatypes/ ./datatypes/ # Set the ENTRYPOINT to the AWS Lambda RIC and CMD to your function handler ENTRYPOINT [ "/usr/local/bin/python", "-m", "awslambdaric" ] diff --git a/backend/docker/lambda.Dockerfile b/backend/docker/lambda.Dockerfile index 763a24b2..1c079981 100644 --- a/backend/docker/lambda.Dockerfile +++ b/backend/docker/lambda.Dockerfile @@ -44,6 +44,7 @@ COPY ./etl/epc_clean/ ./etl/epc_clean/ COPY ./etl/bill_savings/ ./etl/bill_savings/ COPY ./etl/spatial/ ./etl/spatial/ COPY ./BaseUtility.py ./BaseUtility.py +COPY ./datatypes/ ./datatypes/ # Set the ENTRYPOINT to the AWS Lambda RIC and CMD to your function handler diff --git a/backend/ml_models/api.py b/backend/ml_models/api.py index e922d7fc..6e0a4162 100644 --- a/backend/ml_models/api.py +++ b/backend/ml_models/api.py @@ -1,3 +1,5 @@ +import aiohttp +import asyncio import pandas as pd from tqdm import tqdm import requests @@ -18,6 +20,8 @@ class ModelApi: # "hot_water_cost_predictions", ] + KWH_MODEL_PREFIXES = ["heating_kwh_predictions", "hotwater_kwh_predictions"] + MODEL_URLS = { "sap_change_predictions": "sapmodel", "heat_demand_predictions": "heatmodel", @@ -120,6 +124,28 @@ class ModelApi: # depending on how you want to handle errors in your application return None + async def predict_async(self, file_location, model_prefix: str): + """Makes an asynchronous POST request to the Model API with the provided parameters.""" + logger.info(f"Making request to {model_prefix} change api") + url = f"{self.base_url}/{self.MODEL_URLS[model_prefix]}/predict" + payload = { + "file_location": file_location, + "property_id": "", # This should get removed + "portfolio_id": self.portfolio_id, + "created_at": self.timestamp + } + + async with aiohttp.ClientSession() as session: + try: + async with session.post( + url, json=payload, headers={"Content-Type": "application/json"}, timeout=120 + ) as response: + response.raise_for_status() + return await response.json() + except aiohttp.ClientError as e: + logger.error(f"An error occurred: {e}") + return None + @staticmethod def extract_phase(recommendation_id): if 'phase=' in recommendation_id: @@ -180,6 +206,43 @@ class ModelApi: return predictions + async def predict_all_async(self, df, bucket, model_prefixes=None, extract_ids=True) -> dict: + """Uploads data and makes asynchronous requests to the model APIs for predictions.""" + model_prefixes = self.MODEL_PREFIXES if model_prefixes is None else model_prefixes + + predictions = {} + tasks = [] + async with aiohttp.ClientSession() as session: + for model_prefix in model_prefixes: + logger.info(f"Scoring for model prefix: {model_prefix}") + file_location = self.upload_scoring_data(df, bucket, model_prefix) + # Schedule the prediction request as a coroutine + tasks.append( + self.predict_async(f"s3://{bucket}/" + file_location, model_prefix) + ) + + # Gather all asynchronous tasks (execute them concurrently) + responses = await asyncio.gather(*tasks, return_exceptions=True) + + for model_prefix, response in zip(model_prefixes, responses): + if response: + predictions_bucket = self.prediction_buckets[model_prefix] + predictions_df = pd.DataFrame( + read_dataframe_from_s3_parquet( + bucket_name=predictions_bucket, + file_key=response["storage_filepath"].split(predictions_bucket + "/")[1] + ) + ) + predictions_df['predictions'] = predictions_df["predictions"].astype(float).round(1) + if extract_ids: + predictions_df[['property_id', 'recommendation_id']] = predictions_df['id'].str.split('+', + expand=True) + predictions_df['phase'] = predictions_df['recommendation_id'].apply(self.extract_phase) + + predictions[model_prefix] = predictions_df + + return predictions + def paginated_predictions(self, data, bucket, batch_size, model_prefixes=None, extract_ids=True): all_predictions = self.predictions_template() to_loop_over = range(0, data.shape[0], batch_size) @@ -196,3 +259,54 @@ class ModelApi: all_predictions[key] = pd.concat([all_predictions[key], scored]) return all_predictions + + async def async_warm_up_lambdas(self, model_prefies=None): + """Send asynchronous pre-flight requests to each model endpoint to wake up the cold Lambdas without waiting + for responses.""" + logger.info("Asynchronously warming up Lambda functions...") + + model_prefixes = self.MODEL_PREFIXES if model_prefies is None else model_prefies + + tasks = [] + async with aiohttp.ClientSession() as session: + for model_prefix in model_prefixes: + url = f"{self.base_url}/{self.MODEL_URLS[model_prefix]}/predict" + # Create a coroutine for each warm-up request and add it to the tasks list + tasks.append(self._send_warm_up_request(session, url, model_prefix)) + + # Run all tasks concurrently but don't wait for the responses to finish + await asyncio.gather(*tasks, return_exceptions=True) + + @staticmethod + async def _send_warm_up_request(session, url, model_prefix): + """Helper method to send a pre-flight request to a given model URL.""" + try: + async with session.post(url, json={}, timeout=2) as response: + # Log success for monitoring but do not block on the response + logger.info(f"Warmed up {model_prefix} with status code: {response.status}") + except aiohttp.ClientError as e: + logger.warning(f"Failed to warm up {model_prefix}: {e}") + + logger.info("Lambda functions are warmed up and ready to go!") + + def async_paginated_predictions(self, data, bucket, batch_size, model_prefixes=None, extract_ids=True): + + all_predictions = self.predictions_template() + to_loop_over = range(0, data.shape[0], batch_size) + + async def run_batches(): + for chunk in tqdm(to_loop_over, total=len(to_loop_over)): + predictions_dict = await self.predict_all_async( + df=data.iloc[chunk:chunk + batch_size], + bucket=bucket, + model_prefixes=model_prefixes, + extract_ids=extract_ids + ) + + for key, scored in predictions_dict.items(): + all_predictions[key] = pd.concat([all_predictions[key], scored]) + + # Run the async function + asyncio.run(run_batches()) + + return all_predictions diff --git a/backend/requirements/requirements.txt b/backend/requirements/requirements.txt index 11f29183..dd5c34ca 100644 --- a/backend/requirements/requirements.txt +++ b/backend/requirements/requirements.txt @@ -12,9 +12,10 @@ pydantic-settings==2.6.0 psycopg2-binary==2.9.10 python-jose==3.3.0 cryptography==43.0.3 +mangum==0.19.0 # AWS boto3==1.35.44 -# ML, Data, Data Science +# ML, Data Science usaddress==0.5.11 epc-api-python==1.0.2 fuzzywuzzy==0.18.0 @@ -24,6 +25,7 @@ msgpack==1.1.0 scikit-learn==1.5.2 cffi==1.15.1 mip==1.15.0 +# Data pyarrow==17.0.0 fastparquet==2024.5.0 - +aiohttp==3.10.10 From ff44212f9844d0de405401e28be2fe97934800e1 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 22 Oct 2024 11:17:57 +0100 Subject: [PATCH 164/166] making model api async so we can run from within fastapi endpoint and locally --- backend/ml_models/api.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/backend/ml_models/api.py b/backend/ml_models/api.py index 6e0a4162..c2f2dcd9 100644 --- a/backend/ml_models/api.py +++ b/backend/ml_models/api.py @@ -289,8 +289,7 @@ class ModelApi: logger.info("Lambda functions are warmed up and ready to go!") - def async_paginated_predictions(self, data, bucket, batch_size, model_prefixes=None, extract_ids=True): - + async def async_paginated_predictions(self, data, bucket, batch_size, model_prefixes=None, extract_ids=True): all_predictions = self.predictions_template() to_loop_over = range(0, data.shape[0], batch_size) @@ -306,7 +305,13 @@ class ModelApi: for key, scored in predictions_dict.items(): all_predictions[key] = pd.concat([all_predictions[key], scored]) - # Run the async function - asyncio.run(run_batches()) + # Check if there is an existing event loop + try: + # If there is an existing event loop, await the coroutine directly + loop = asyncio.get_running_loop() + await run_batches() + except RuntimeError: # No running event loop + # If no event loop is running, use asyncio.run() + asyncio.run(run_batches()) return all_predictions From 10db0498f5d2dc0108cd05a76bc50313ecd18f1a Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 22 Oct 2024 12:14:19 +0100 Subject: [PATCH 165/166] fixed unit tests --- .idea/Model.iml | 2 +- .idea/misc.xml | 2 +- backend/app/plan/schemas.py | 2 +- model_data/requirements/requirements.txt | 13 ++++++++----- .../test_data/heating_recommendations_data.py | 3 ++- .../tests/test_roof_recommendations.py | 18 +++++++++--------- .../tests/test_window_recommendations.py | 12 ++++++------ 7 files changed, 28 insertions(+), 24 deletions(-) diff --git a/.idea/Model.iml b/.idea/Model.iml index df6c4faa..850c0cda 100644 --- a/.idea/Model.iml +++ b/.idea/Model.iml @@ -7,7 +7,7 @@ - + diff --git a/.idea/misc.xml b/.idea/misc.xml index 50cad4ca..e4070118 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -3,7 +3,7 @@ - + diff --git a/backend/app/plan/schemas.py b/backend/app/plan/schemas.py index c1a51769..f84912fe 100644 --- a/backend/app/plan/schemas.py +++ b/backend/app/plan/schemas.py @@ -1,4 +1,4 @@ -from pydantic import BaseModel, Field, BeforeValidator, field_validator +from pydantic import BaseModel, Field, BeforeValidator from typing import Annotated, List, Optional # Example constants for validation diff --git a/model_data/requirements/requirements.txt b/model_data/requirements/requirements.txt index 1d84fc3d..845166d9 100644 --- a/model_data/requirements/requirements.txt +++ b/model_data/requirements/requirements.txt @@ -1,8 +1,9 @@ -pydantic==1.10.11 +pydantic==2.9.2 +pydantic-settings==2.6.0 epc-api-python==1.0.2 -pandas==2.0.3 -numpy==1.25.1 -pytz==2023.3 +numpy==2.1.2 +pandas==2.2.3 +pytz==2024.2 tzdata==2023.3 tqdm mypy @@ -20,4 +21,6 @@ pyspellchecker textblob boto3 pyarrow -msgpack==1.0.5 +msgpack==1.1.0 + + diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index 51bf0378..26263826 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -946,9 +946,10 @@ testing_examples = [ }, "heating_measure_types": [ 'high_heat_retention_storage_heater', + 'air_source_heat_pump', ], "heating_controls_measure_types": [], - "notes": "This is an end-terrace house, without mains gas connection, so all we recommend is HHR" + "notes": "This is an end-terrace house, without mains gas connection, so we recommend is HHR & ASHP" }, { "epc": { diff --git a/recommendations/tests/test_roof_recommendations.py b/recommendations/tests/test_roof_recommendations.py index 6dbc3b72..214ea6c0 100644 --- a/recommendations/tests/test_roof_recommendations.py +++ b/recommendations/tests/test_roof_recommendations.py @@ -57,10 +57,10 @@ class TestRoofRecommendations: assert len(roof_recommender2.recommendations) == 1 - assert roof_recommender2.recommendations[0]["total"] == 1610.0000000000002 - assert roof_recommender2.recommendations[0]["new_u_value"] == 0.14 + assert roof_recommender2.recommendations[0]["total"] == 1653 + assert roof_recommender2.recommendations[0]["new_u_value"] == 0.13 assert roof_recommender2.recommendations[0]["starting_u_value"] == 0.68 - assert roof_recommender2.recommendations[0]["parts"][0]["depth"] == 270 + assert roof_recommender2.recommendations[0]["parts"][0]["depth"] == 300 epc_record = EPCRecord() epc_record.prepared_epc = {"county": "Greater London Authority", "roof-energy-eff": "Very Poor"} @@ -85,7 +85,7 @@ class TestRoofRecommendations: assert roof_recommender3.recommendations assert len(roof_recommender3.recommendations) == 1 - assert roof_recommender3.recommendations[0]["parts"][0]["depth"] == 270 + assert roof_recommender3.recommendations[0]["parts"][0]["depth"] == 300.0 def test_loft_insulation_recommendation_150mm_insulation(self): epc_record = EPCRecord() @@ -107,14 +107,14 @@ class TestRoofRecommendations: assert not roof_recommender4.recommendations - roof_recommender4.recommend(phase=0) + roof_recommender4.recommend(phase=0, default_u_values=True) assert len(roof_recommender4.recommendations) == 1 - assert roof_recommender4.recommendations[0]["total"] == 1552.5 - assert roof_recommender4.recommendations[0]["new_u_value"] == 0.13 + assert roof_recommender4.recommendations[0]["total"] == 1653.0 + assert roof_recommender4.recommendations[0]["new_u_value"] == 0.14 assert roof_recommender4.recommendations[0]["starting_u_value"] == 0.3 - assert roof_recommender4.recommendations[0]["parts"][0]["depth"] == 200 + assert roof_recommender4.recommendations[0]["parts"][0]["depth"] == 300 epc_record = EPCRecord() epc_record.prepared_epc = {"county": "Somerset", "roof-energy-eff": "Good"} @@ -139,7 +139,7 @@ class TestRoofRecommendations: assert roof_recommender5.recommendations assert len(roof_recommender5.recommendations) == 1 - assert roof_recommender5.recommendations[0]["parts"][0]["depth"] == 200 + assert roof_recommender5.recommendations[0]["parts"][0]["depth"] == 300 def test_loft_insulation_recommendation_270mm_insulation(self): # We shouldn't recommend anything in this case diff --git a/recommendations/tests/test_window_recommendations.py b/recommendations/tests/test_window_recommendations.py index ae6c6377..baef3574 100644 --- a/recommendations/tests/test_window_recommendations.py +++ b/recommendations/tests/test_window_recommendations.py @@ -48,13 +48,13 @@ class TestWindowRecommendations: 'starting_u_value': None, 'new_u_value': None, 'sap_points': None, 'already_installed': False, 'total': 7980.0, 'labour_hours': 0.0, 'labour_days': 0.0, 'is_secondary_glazing': False, 'description_simulation': { - 'multi-glaze-proportion': 100, 'windows-energy-eff': 'Average', + 'multi-glaze-proportion': 100, 'windows-energy-eff': 'Good', 'windows-description': 'Fully double glazed', 'glazed-type': 'double glazing installed during or after 2002' }, 'simulation_config': { 'has_glazing_ending': True, 'glazing_type_ending': 'double', - 'multi_glaze_proportion_ending': 100, 'windows_energy_eff_ending': 'Average', + 'multi_glaze_proportion_ending': 100, 'windows_energy_eff_ending': 'Good', 'glazed_type_ending': 'double glazing installed during or after 2002' } } @@ -401,14 +401,14 @@ class TestWindowRecommendations: 'sap_points': None, 'already_installed': False, 'total': 7980.0, 'labour_hours': 0.0, 'labour_days': 0.0, 'is_secondary_glazing': False, 'description_simulation': { - 'multi-glaze-proportion': 100, 'windows-energy-eff': 'Average', + 'multi-glaze-proportion': 100, 'windows-energy-eff': 'Good', 'windows-description': 'Fully double glazed', 'glazed-type': 'double glazing installed during or after 2002' }, 'simulation_config': { 'has_glazing_ending': True, 'glazing_coverage_ending': 'full', 'glazing_type_ending': 'double', 'multi_glaze_proportion_ending': 100, - 'windows_energy_eff_ending': 'Average', + 'windows_energy_eff_ending': 'Good', 'glazed_type_ending': 'double glazing installed during or after 2002' } } @@ -624,7 +624,7 @@ class TestWindowRecommendations: 'total_floor_area_ending': 61.0, 'floor_height_starting': 2.37, 'floor_height_ending': 2.37, 'hot_water_energy_eff_starting': 'Good', 'hot_water_energy_eff_ending': 'Good', 'floor_energy_eff_starting': 'NO_RATING', 'floor_energy_eff_ending': 'NO_RATING', - 'windows_energy_eff_starting': 'Very Poor', 'windows_energy_eff_ending': 'Average', + 'windows_energy_eff_starting': 'Very Poor', 'windows_energy_eff_ending': 'Good', 'walls_energy_eff_starting': 'Very Poor', 'walls_energy_eff_ending': 'Very Poor', 'sheating_energy_eff_starting': 'NO_RATING', 'sheating_energy_eff_ending': 'NO_RATING', 'roof_energy_eff_starting': 'Very Poor', 'roof_energy_eff_ending': 'Very Poor', @@ -666,7 +666,7 @@ class TestWindowRecommendations: {'variable': 'glazed_type_ending', 'starting': 'not defined', 'simulated': 'double glazing installed during or after 2002'}, {'variable': 'multi_glaze_proportion_ending', 'starting': 0.0, 'simulated': 100}, - {'variable': 'windows_energy_eff_ending', 'starting': 'Very Poor', 'simulated': 'Average'}, + {'variable': 'windows_energy_eff_ending', 'starting': 'Very Poor', 'simulated': 'Good'}, ] assert different == expected_different From 19c3c429d29971dd8ab32f37addc3b9c2a3ee8fe Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 22 Oct 2024 12:19:46 +0100 Subject: [PATCH 166/166] removed redundant requirements file from github actions unit test --- .github/workflows/unit_tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 339a3429..39d285f2 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -35,7 +35,6 @@ jobs: - name: Run tests with pytest run: | pip install -r model_data/requirements/dev.txt - pip install -r backend/requirements/base.txt pytest # - name: Upload coverage to Codecov # uses: codecov/codecov-action@v2