From 9f42344e044b68bc69a2dbdaac085aba18c16b1b Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 18 Dec 2023 15:57:53 +0000 Subject: [PATCH 01/16] recommendation class template --- recommendations/WindowsRecommendations.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 recommendations/WindowsRecommendations.py diff --git a/recommendations/WindowsRecommendations.py b/recommendations/WindowsRecommendations.py new file mode 100644 index 00000000..ed2ea7a5 --- /dev/null +++ b/recommendations/WindowsRecommendations.py @@ -0,0 +1,7 @@ +class WindowsRecommendations: + + def __init__(self): + pass + + def recommend(self): + pass From c6c8de6dcf096349de0a75b4f168bfea122536ab Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 18 Dec 2023 16:03:36 +0000 Subject: [PATCH 02/16] Added init and started adding the recommendation function --- recommendations/WindowsRecommendations.py | 32 ++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/recommendations/WindowsRecommendations.py b/recommendations/WindowsRecommendations.py index ed2ea7a5..e5a11f8a 100644 --- a/recommendations/WindowsRecommendations.py +++ b/recommendations/WindowsRecommendations.py @@ -1,7 +1,33 @@ +from typing import List +from backend.Property import Property +from recommendations.Costs import Costs + + class WindowsRecommendations: - def __init__(self): - pass + def __init__(self, property_instance: Property, materials: List): + self.property = property_instance + self.costs = Costs(self.property) + + self.recommendations = [] + + self.glazing_materials = [ + material for material in materials if material["type"] == "window_glazing" + ] + + # TODO: This will be populated with the works associated to upgrading glazing. This will involve removal of + # previous glazing and installation of new glazing. This will not include the cost of the windows material + # themselfs + self.glazing_works_materials = [] def recommend(self): - pass + """ + This method will recommend the best possible glazing options for a property. + + In order to do this, we need to estimate the number of windows that the home has. This information will be + stored in the property object, under property.number_of_windows + :return: + """ + + if not self.property.number_of_windows: + raise ValueError("Number of windows not specified") From 57f92e60602abdf232be22f8ed2d1cc97a42cc67 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 19 Dec 2023 19:27:33 +0000 Subject: [PATCH 03/16] completed estimate windows function, though it needs more testing --- .idea/Model.iml | 2 +- .idea/misc.xml | 2 +- recommendations/recommendation_utils.py | 61 +++++++++++++ .../tests/test_recommendation_utils.py | 91 +++++++++++++++++++ 4 files changed, 154 insertions(+), 2 deletions(-) diff --git a/.idea/Model.iml b/.idea/Model.iml index 4413bb06..b0f9c00d 100644 --- a/.idea/Model.iml +++ b/.idea/Model.iml @@ -7,7 +7,7 @@ - + diff --git a/.idea/misc.xml b/.idea/misc.xml index 6f308057..1122b380 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -3,7 +3,7 @@ - + diff --git a/recommendations/recommendation_utils.py b/recommendations/recommendation_utils.py index 100ecb15..7cfe023e 100644 --- a/recommendations/recommendation_utils.py +++ b/recommendations/recommendation_utils.py @@ -652,3 +652,64 @@ def esimtate_pitched_roof_area(floor_area: float, floor_height: float) -> float: area = 2 * (slope * wall_width) return area + + +def estimate_windows( + property_type, built_form, construction_age_band, floor_area, number_habitable_rooms, extension_count +): + # Base window count based on habitable rooms + window_count = number_habitable_rooms + + # Additional windows for non-habitable rooms (e.g., kitchen, bathroom) + # Assuming most houses will have at least one kitchen and one bathroom + # Scale non-habitable windows with the number of habitable rooms + non_habitable_base = 2 # Base for kitchen and bathroom + extra_non_habitable = max(0, (number_habitable_rooms - 3) // 2) # Extra for large houses + window_count += non_habitable_base + extra_non_habitable + + # Adjustments based on built form and property type + if property_type in ["House", "Bungalow"] and built_form in ["Semi-Detached", "Detached"]: + built_form_lookup = { + "Semi-Detached": 3, + "Detached": 4, + } + else: + # For Flats and Maisonettes, adjustments might be less + built_form_lookup = { + "Mid-Terrace": 0, + "End-Terrace": 1, + "Semi-Detached": 1, + "Detached": 2, + } + window_count += built_form_lookup.get(built_form, 0) + + # Adjust for floor area (larger floor area might indicate more rooms/windows) + if floor_area < 85: # Small to medium properties + # Standard window count likely sufficient + pass + elif 85 <= floor_area <= 120: # Medium to large properties + # More rooms or larger rooms likely, potentially more windows + window_count += 1 + elif floor_area > 120: # Very large properties + # Likely to have significantly more or larger rooms + window_count += 2 + + # Adjust for construction age band + if construction_age_band in ["England and Wales: before 1900", "England and Wales: 1900-1929"]: + # Older houses with smaller, more numerous windows + window_count += 1 + + # Adjust for extensions (each extension might add windows) + window_count += extension_count + + # Adjustments for specific property types + if property_type in ["Flat", "Maisontte"]: + # Flats might have fewer windows due to shared walls + # Maisonettes might follow a similar pattern to flats or small houses + window_count -= 1 + + # Ensure window count is not negative + if window_count < 0: + raise ValueError("Window count cannot be negative.") + + return window_count diff --git a/recommendations/tests/test_recommendation_utils.py b/recommendations/tests/test_recommendation_utils.py index aefc70b0..7c29bbb0 100644 --- a/recommendations/tests/test_recommendation_utils.py +++ b/recommendations/tests/test_recommendation_utils.py @@ -427,3 +427,94 @@ def test_external_wall_area(): for num_floors, floor_height, perimeter, built_form, expected in test_cases: result = recommendation_utils.estimate_external_wall_area(num_floors, floor_height, perimeter, built_form) assert result == expected, f"Test failed for {built_form}: Expected {expected}, got {result}" + + +def test_estimate_windows(): + # Based on data from an EPR that has 4 windows + windows_case_1 = recommendation_utils.estimate_windows( + property_type="Flat", + built_form="Semi-Detached", + construction_age_band="England and Wales: 1976-1982", + floor_area=37, + number_habitable_rooms=2, + extension_count=0, + ) + + assert windows_case_1 == 4, f"Expected 4 windows, got {windows_case_1}" + + # Based on data from an EPR that has 7 winows, however two of the windows were very small, having areas of + # 0.21m^2 and 0.3m^2 respectively. We see 6 as a reasonable estimate for the number of windows + windows_case_2 = recommendation_utils.estimate_windows( + property_type="House", + built_form="Mid-Terrace", + construction_age_band="England and Wales: 1950-1966", + floor_area=69, + number_habitable_rooms=4, + extension_count=0, + ) + + assert windows_case_2 == 6, f"Expected 6 windows, got {windows_case_2}" + + # Based on data from an EPR on a bungalow, that has 6 windows. Two of the windows are small, both have a 0.4m^2 area + # and so 5 windows is an acceptable estimate + windows_case_3 = recommendation_utils.estimate_windows( + property_type="Bungalow", + built_form="Mid-Terrace", + construction_age_band="England and Wales: 1967-1975", + floor_area=56, + number_habitable_rooms=3, + extension_count=0, + ) + + assert windows_case_3 == 5, f"Expected 5 windows, got {windows_case_3}" + + # Based on data from an EPR on a end terrace house that has 8 windows. One of the windows is very small, with an + # area of 0.25 m^2 and so 7 windows is an acceptable estimate + windows_case_4 = recommendation_utils.estimate_windows( + property_type="House", + built_form="End-Terrace", + construction_age_band="England and Wales: 1967-1975", + floor_area=77.28, + number_habitable_rooms=4, + extension_count=0, + ) + + assert windows_case_4 == 7, f"Expected 7 windows, got {windows_case_4}" + + # Based on data from an EPR on a Semi-detatched house that has 11 windows based on the associated condition report + # Right now, we estimate 12 windows for this property + windows_case_5 = recommendation_utils.estimate_windows( + property_type="House", + built_form="Semi-Detached", + construction_age_band="England and Wales: 1950-1966", + floor_area=88.4, + number_habitable_rooms=5, + extension_count=0, + ) + + assert windows_case_5 == 12, f"Expected 12 windows, got {windows_case_5}" + + # Based on Khalim's flat which has 3 windows. There is no construction age band on the EPC. The windows are large + # so an estimate of 5 windows is a reasonable estimate + windows_case_6 = recommendation_utils.estimate_windows( + property_type="Flat", + built_form="", + construction_age_band="", + floor_area=100, + number_habitable_rooms=3, + extension_count=0, + ) + + assert windows_case_6 == 5, f"Expected 5 windows, got {windows_case_6}" + + # Based on an EPR semi detatched house though we don't have the exact number of windows. We estimate 10 + windows_case_7 = recommendation_utils.estimate_windows( + property_type="House", + built_form="Semi-Detached", + construction_age_band="England and Wales: 1967-1975", + floor_area=85, + number_habitable_rooms=4, + extension_count=0, + ) + + assert windows_case_7 == 10, f"Expected 10 windows, got {windows_case_7}" From 1b10f5c9b1b77936e5e275e6ff125866a37f889c Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 20 Dec 2023 10:34:08 +0000 Subject: [PATCH 04/16] unit tests for estimate windows --- recommendations/tests/test_recommendation_utils.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/recommendations/tests/test_recommendation_utils.py b/recommendations/tests/test_recommendation_utils.py index 7c29bbb0..559a51b2 100644 --- a/recommendations/tests/test_recommendation_utils.py +++ b/recommendations/tests/test_recommendation_utils.py @@ -518,3 +518,15 @@ def test_estimate_windows(): ) assert windows_case_7 == 10, f"Expected 10 windows, got {windows_case_7}" + + # Base on Khalim's parents flat + windows_case_8 = recommendation_utils.estimate_windows( + property_type="Flat", + built_form="End-Terrace", + construction_age_band="", + floor_area=50, + number_habitable_rooms=3, + extension_count=0, + ) + + assert windows_case_8 == 5, f"Expected 5 windows, got {windows_case_8}" From 6388d45638809d098f1b8af818dc3494e7fb7482 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 20 Dec 2023 10:37:52 +0000 Subject: [PATCH 05/16] added set method for windows count into property class --- backend/Property.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/backend/Property.py b/backend/Property.py index bb545248..41696c37 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -14,7 +14,7 @@ from epc_api.client import EpcClient from BaseUtility import Definitions from recommendations.rdsap_tables import england_wales_age_band_lookup, FLOOR_LEVEL_MAP from recommendations.recommendation_utils import ( - estimate_perimeter, get_wall_type, estimate_external_wall_area, esimtate_pitched_roof_area + estimate_perimeter, get_wall_type, estimate_external_wall_area, esimtate_pitched_roof_area, estimate_windows ) ENVIRONMENT = os.environ.get('ENVIRONMENT', 'dev') @@ -87,6 +87,7 @@ class Property(Definitions): self.insulation_floor_area = None self.number_lighting_outlets = None self.floor_level = None + self.number_of_windows = None self.current_adjusted_energy = None self.expected_adjusted_energy = None @@ -332,6 +333,7 @@ class Property(Definitions): self.set_wall_type() self.set_floor_type() self.set_floor_level() + self.set_windows_count() def set_age_band(self): """ @@ -850,3 +852,18 @@ class Property(Definitions): """ self.current_adjusted_energy = current_adjusted_energy self.expected_adjusted_energy = expected_adjusted_energy + + def set_windows_count(self): + """ + Using the estimate_windows function, this method will set the number of windows in the property + :return: + """ + + self.number_of_windows = estimate_windows( + property_type=self.data["property-type"], + built_form=self.data["built-form"], + construction_age_band=self.construction_age_band, + floor_area=self.floor_area, + number_habitable_rooms=self.number_of_rooms, + extension_count=float(self.data["extension-count"]), + ) From 3426629418872f820577275d2736e0d30145805a Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 20 Dec 2023 14:38:31 +0000 Subject: [PATCH 06/16] Added windows costs to costs etl process --- .idea/Model.iml | 2 +- .idea/misc.xml | 2 +- etl/costs/app.py | 1 + .../epc_attributes/WindowAttributes.py | 6 +- .../test_data/test_window_attributes_cases.py | 6 +- recommendations/Costs.py | 68 +++++++++++++++++++ recommendations/WindowsRecommendations.py | 27 +++++++- .../tests/test_window_recommendations.py | 30 ++++++++ 8 files changed, 135 insertions(+), 7 deletions(-) create mode 100644 recommendations/tests/test_window_recommendations.py diff --git a/.idea/Model.iml b/.idea/Model.iml index b0f9c00d..4413bb06 100644 --- a/.idea/Model.iml +++ b/.idea/Model.iml @@ -7,7 +7,7 @@ - + diff --git a/.idea/misc.xml b/.idea/misc.xml index 1122b380..6f308057 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -3,7 +3,7 @@ - + diff --git a/etl/costs/app.py b/etl/costs/app.py index 4d53ce28..30eff735 100644 --- a/etl/costs/app.py +++ b/etl/costs/app.py @@ -75,6 +75,7 @@ def app(): ewi_costs = pd.read_excel(DATA_DIRECTORY, sheet_name="external_wall_insulation", header=0) lel_costs = pd.read_excel(DATA_DIRECTORY, sheet_name="low_energy_lighting", header=0) flat_roof_costs = pd.read_excel(DATA_DIRECTORY, sheet_name="flat_roof_insulation", header=0) + window_costs = pd.read_excel(DATA_DIRECTORY, sheet_name="window_glazing", header=0) # Form a single table to be uploaded costs = pd.concat( diff --git a/etl/epc_clean/epc_attributes/WindowAttributes.py b/etl/epc_clean/epc_attributes/WindowAttributes.py index e962cd31..cd767359 100644 --- a/etl/epc_clean/epc_attributes/WindowAttributes.py +++ b/etl/epc_clean/epc_attributes/WindowAttributes.py @@ -52,7 +52,7 @@ class WindowAttributes(Definitions): raise ValueError('Invalid description') def process(self) -> Dict[str, Union[str, bool]]: - result: Dict[str, Union[str, bool]] = { + result: Dict[str, Union[str, bool, None]] = { "has_glazing": False, "glazing_coverage": None, "glazing_type": None, @@ -83,4 +83,8 @@ class WindowAttributes(Definitions): if not result["glazing_coverage"]: result["glazing_coverage"] = "full" + # We reset some values if the glazing is single + if result["glazing_type"] == "single": + result["has_glazing"] = False + return result diff --git a/etl/epc_clean/tests/test_data/test_window_attributes_cases.py b/etl/epc_clean/tests/test_data/test_window_attributes_cases.py index 1eeeee21..f01ccba9 100644 --- a/etl/epc_clean/tests/test_data/test_window_attributes_cases.py +++ b/etl/epc_clean/tests/test_data/test_window_attributes_cases.py @@ -30,7 +30,8 @@ windows_cases = [ 'glazing_type': 'triple', 'no_data': False}, {'original_description': 'Gwydrau triphlyg rhannol', 'has_glazing': True, 'glazing_coverage': 'partial', 'glazing_type': 'triple', 'no_data': False}, - {'original_description': 'Single glazed', 'has_glazing': True, 'glazing_coverage': 'full', 'glazing_type': 'single', + {'original_description': 'Single glazed', 'has_glazing': False, 'glazing_coverage': None, + 'glazing_type': 'single', 'no_data': False}, {'original_description': 'Some double glazing', 'has_glazing': True, 'glazing_coverage': 'partial', 'glazing_type': 'double', 'no_data': False}, @@ -46,7 +47,8 @@ windows_cases = [ 'glazing_type': 'double', 'no_data': False}, {'original_description': 'Gwydrau dwbl gan mwyaf', 'has_glazing': True, 'glazing_coverage': 'most', 'glazing_type': 'double', 'no_data': False}, - {'original_description': 'Gwydrau sengl', 'has_glazing': True, 'glazing_coverage': 'full', 'glazing_type': 'single', + {'original_description': 'Gwydrau sengl', 'has_glazing': False, 'glazing_coverage': None, + 'glazing_type': 'single', 'no_data': False}, {'original_description': 'Ffenestri perfformiad uchel', 'has_glazing': True, 'glazing_coverage': 'full', 'glazing_type': 'high performance', 'no_data': False}, diff --git a/recommendations/Costs.py b/recommendations/Costs.py index 0d9031b2..84a7468c 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -64,6 +64,12 @@ class Costs: VAT_RATE = 0.2 PROFIT_MARGIN = 0.2 + # Based on this greenmatch article, on average, a Sash window is around 50% more expensive than a casement window. + # Therefore, for a conservative cost estimate, and allowance for a more premium window type, we inflate the material + # cost of the windows to allow for a sash window type + # https://www.greenmatch.co.uk/windows/double-glazing/cost + SASH_WINDOW_INFLATION_FACTOR = 1.5 + def __init__(self, property_instance): """ Initializes the Costs class with a property instance. @@ -719,3 +725,65 @@ class Costs: "labour_days": labour_days, "labour_cost": labour_costs } + + def window_glazing(self, number_of_windows, material): + """ + We characterise the jobs to be done for window glazing as the following: + 1) Initial Assessment and Measurements: Before removing the existing window, it's essential to assess the + condition of the window frame and opening. Precise measurements are taken to ensure the new double glazed + windows fit perfectly. + + 2) Remove the Existing Window: This involves carefully dismantling and removing the old single glazed window. It + requires skill to avoid damaging the surrounding wall and the window frame (if it's to be reused). + + 3) Dispose of the Existing Window: The old window, especially if it's a single glazed unit, needs to be + disposed of responsibly. Glass and other materials should be recycled where possible. + + 4) Surface Preparation: The window opening might need some preparation, especially if there's damage or if + adjustments are needed to accommodate the new window. This can include repairing or replacing parts of the + window frame, sealing gaps, and ensuring the opening is level and square. + + 5) Install the Window Frame (if new frames are used): In many cases, double glazed windows come with their + frames. These need to be installed securely into the window opening. This process involves aligning, leveling, + and fixing the frame in place. + + 6) Install the Window Sill: If a new window sill is required, it is installed at this stage. It needs to be + correctly aligned with the frame and securely attached. + + 7) Install the Double Glazed Glass Units: The glass units are carefully inserted into the frame. This step + requires precision to ensure a snug fit without causing stress on the glass, which could lead to cracking or + breaking. + + 8) Sealing and Weatherproofing: After the glass units are in place, it's crucial to seal around the frame and + between the glass and frame to ensure there are no drafts and that the installation is weather-tight. This + typically involves applying silicone sealant or other appropriate sealing materials. + + 9) Finishing Touches: This includes any cosmetic work, such as trimming, painting, or staining the frame and + sill to match the rest of the property. It might also involve cleaning up any mess created during the + installation. + + 10) Inspection and Testing: Finally, the new windows should be inspected to ensure they open, close, and lock + correctly. This is also a good time to check for any gaps or issues with the sealing. + + For this cost estimation process, we factor in initial assement into the preliminaries + + :param number_of_windows: + :return: + """ + + number_of_windows = 7 + windows_cost = 345 + + material_cost = windows_cost * number_of_windows * self.SASH_WINDOW_INFLATION_FACTOR + + subtotal = material_cost + + contingency_cost = subtotal * 0.2 + preliminaries_cost = subtotal * 0.2 + profit_cost = subtotal * 0.2 + + subtotal_before_vat = subtotal + contingency_cost + preliminaries_cost + profit_cost + + vat_cost = subtotal_before_vat * 0.2 + + total_cost = subtotal_before_vat + vat_cost diff --git a/recommendations/WindowsRecommendations.py b/recommendations/WindowsRecommendations.py index e5a11f8a..d3424d1b 100644 --- a/recommendations/WindowsRecommendations.py +++ b/recommendations/WindowsRecommendations.py @@ -9,7 +9,7 @@ class WindowsRecommendations: self.property = property_instance self.costs = Costs(self.property) - self.recommendations = [] + self.recommendation = [] self.glazing_materials = [ material for material in materials if material["type"] == "window_glazing" @@ -29,5 +29,28 @@ class WindowsRecommendations: :return: """ - if not self.property.number_of_windows: + number_of_windows = self.property.number_of_windows + + if not number_of_windows: raise ValueError("Number of windows not specified") + + if self.property.windows["has_glazing"] & self.property.windows["glazing_coverage"] == "full": + return + + # We then price the job based on the number of windows that there are + + cost_result = {} + + description = None + + self.recommendation = [ + { + "parts": [], + "type": "window_glazing", + "description": description, + "starting_u_value": None, + "new_u_value": None, + "sap_points": None, + **cost_result + } + ] diff --git a/recommendations/tests/test_window_recommendations.py b/recommendations/tests/test_window_recommendations.py new file mode 100644 index 00000000..da50d25c --- /dev/null +++ b/recommendations/tests/test_window_recommendations.py @@ -0,0 +1,30 @@ +from recommendations.WindowsRecommendations import WindowsRecommendations +from backend.Property import Property +from unittest.mock import Mock + + +class TestWindowRecommendations: + + def test_fully_single_glazed(self): + """ + For this property, we expect all windows to be single glazed and should recommend full double glazing + :return: + """ + + property_1 = Property( + id=1, + postcode='1', + address1='1', + epc_client=Mock(), + data={ + "county": "Wychavon" + } + ) + property_1.windows = { + 'original_description': 'Single glazed', 'has_glazing': False, 'glazing_coverage': 'full', + 'glazing_type': 'single', + 'no_data': False + } + property_1.number_of_windows = 5 + + recommender = WindowsRecommendations(property_instance=property_1, materials=[]) From 0b52dc80974155ed7661de15a1a6d772ad011ad1 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 20 Dec 2023 14:55:04 +0000 Subject: [PATCH 07/16] Implemented cost framework --- recommendations/Costs.py | 37 ++++++++++++++----- recommendations/WindowsRecommendations.py | 19 +++++----- recommendations/tests/test_data/materials.py | 20 +++++++++- .../tests/test_window_recommendations.py | 9 ++++- 4 files changed, 63 insertions(+), 22 deletions(-) diff --git a/recommendations/Costs.py b/recommendations/Costs.py index 84a7468c..239dadd1 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -771,19 +771,38 @@ class Costs: :return: """ - number_of_windows = 7 - windows_cost = 345 + material_cost = material["material_cost"] * number_of_windows * self.SASH_WINDOW_INFLATION_FACTOR + labour_cost = ( + material["labour_cost"] * number_of_windows * self.SASH_WINDOW_INFLATION_FACTOR * + self.labour_adjustment_factor + ) - material_cost = windows_cost * number_of_windows * self.SASH_WINDOW_INFLATION_FACTOR + subtotal = material_cost + labour_cost - subtotal = material_cost - - contingency_cost = subtotal * 0.2 - preliminaries_cost = subtotal * 0.2 - profit_cost = subtotal * 0.2 + contingency_cost = subtotal * self.CONTINGENCY + preliminaries_cost = subtotal * self.PRELIMINARIES + profit_cost = subtotal * self.PROFIT_MARGIN subtotal_before_vat = subtotal + contingency_cost + preliminaries_cost + profit_cost - vat_cost = subtotal_before_vat * 0.2 + vat_cost = subtotal_before_vat * self.VAT_RATE total_cost = subtotal_before_vat + vat_cost + + labour_hours = material["labour_hours_per_unit"] * number_of_windows + + # Assume a team of 2 + labour_days = (labour_hours / 8) / 2 + + return { + "total": total_cost, + "subtotal": subtotal_before_vat, + "vat": vat_cost, + "contingency": contingency_cost, + "preliminaries": preliminaries_cost, + "material": material_cost, + "profit": profit_cost, + "labour_hours": labour_hours, + "labour_cost": labour_cost, + "labour_days": labour_days + } diff --git a/recommendations/WindowsRecommendations.py b/recommendations/WindowsRecommendations.py index d3424d1b..474071d9 100644 --- a/recommendations/WindowsRecommendations.py +++ b/recommendations/WindowsRecommendations.py @@ -11,14 +11,13 @@ class WindowsRecommendations: self.recommendation = [] - self.glazing_materials = [ - material for material in materials if material["type"] == "window_glazing" + self.glazing_material = [ + material for material in materials if material["type"] == "windows_glazing" ] - # TODO: This will be populated with the works associated to upgrading glazing. This will involve removal of - # previous glazing and installation of new glazing. This will not include the cost of the windows material - # themselfs - self.glazing_works_materials = [] + if len(self.glazing_material) != 1: + raise ValueError("There should only be one window glazing material") + self.glazing_material = self.glazing_material[0] def recommend(self): """ @@ -34,12 +33,14 @@ class WindowsRecommendations: if not number_of_windows: raise ValueError("Number of windows not specified") - if self.property.windows["has_glazing"] & self.property.windows["glazing_coverage"] == "full": + if self.property.windows["has_glazing"] & (self.property.windows["glazing_coverage"] == "full"): return # We then price the job based on the number of windows that there are - - cost_result = {} + cost_result = self.costs.window_glazing( + number_of_windows=number_of_windows, + material=self.glazing_material, + ) description = None diff --git a/recommendations/tests/test_data/materials.py b/recommendations/tests/test_data/materials.py index d7241be5..187d1401 100644 --- a/recommendations/tests/test_data/materials.py +++ b/recommendations/tests/test_data/materials.py @@ -942,8 +942,24 @@ materials = [ 'https://www.hamuch.com/cost/led-spot-light#:~:text=It%20costs%20an%20average%20of,' 'will%20drive%20up%20the%20cost.', 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True, 'prime_material_cost': None, - 'material_cost': 20.0, 'labour_cost': 46.0, 'labour_hours_per_unit': 0.8, 'plant_cost': 0.0, 'total_cost': 66.0, + 'material_cost': 20.0, 'labour_cost': 15.0, 'labour_hours_per_unit': 0.8, 'plant_cost': 0.0, 'total_cost': 66.0, 'notes': 'We estimate the unit economics from the checkatrade article. We assume that the average job consists ' 'of installing 6 lights based on the hamuch article. We use the median value of 400 for a job of 6 ' - 'lights'} + 'lights'}, + {'id': 1235, 'type': 'windows_glazing', + 'description': 'uPVC windows; Profile 22 or other equal and approved; reinforced where appropriate with ' + 'aluminium alloy; in refurbishment work, including standard ironmongery; sills and factory glazed ' + 'with low-e 24 mm double glazing; removing existing windows and fixing new in position; including ' + 'lugs plugged and screwed to brickwork or blockwork; Casement/fixed light; including vents; ' + 'e.p.d.m. glazing gaskets and weather seals; 1770 mm × 1200 mm; ref P312WW', + 'depth': 0.0, 'depth_unit': None, 'cost': None, 'cost_unit': 'gbp_per_unit', 'r_value_per_mm': None, + 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None, 'thermal_conductivity_unit': None, + 'link': 'SPONs', + 'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), + 'is_active': True, 'prime_material_cost': 176.55, + 'material_cost': 182.25, 'labour_cost': 163.36, 'labour_hours_per_unit': 6.5, 'plant_cost': 0.0, + 'total_cost': 345.61, + 'notes': 'This is the cost of removal of existing windows and installation of new windows. This is a casement ' + 'style window, which is the most common but also the cheapest style. In the cost estimation framework, ' + 'we can inflate prices for different finishes, to be conservative on price.'} ] diff --git a/recommendations/tests/test_window_recommendations.py b/recommendations/tests/test_window_recommendations.py index da50d25c..1a146b41 100644 --- a/recommendations/tests/test_window_recommendations.py +++ b/recommendations/tests/test_window_recommendations.py @@ -1,6 +1,7 @@ from recommendations.WindowsRecommendations import WindowsRecommendations from backend.Property import Property from unittest.mock import Mock +from recommendations.tests.test_data.materials import materials class TestWindowRecommendations: @@ -25,6 +26,10 @@ class TestWindowRecommendations: 'glazing_type': 'single', 'no_data': False } - property_1.number_of_windows = 5 + property_1.number_of_windows = 7 - recommender = WindowsRecommendations(property_instance=property_1, materials=[]) + recommender = WindowsRecommendations(property_instance=property_1, materials=materials) + + assert not recommender.recommendation + + recommender.recommend() From ebad3d7dd69a8e8cdf81f9bba8f99a8d9b8b7073 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 20 Dec 2023 15:55:34 +0000 Subject: [PATCH 08/16] Implementing recommendation logic and tests --- recommendations/Costs.py | 23 ++-- recommendations/WindowsRecommendations.py | 41 ++++++- .../tests/test_window_recommendations.py | 104 +++++++++++++++++- 3 files changed, 157 insertions(+), 11 deletions(-) diff --git a/recommendations/Costs.py b/recommendations/Costs.py index 239dadd1..24ea0584 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -70,6 +70,10 @@ class Costs: # https://www.greenmatch.co.uk/windows/double-glazing/cost SASH_WINDOW_INFLATION_FACTOR = 1.5 + # Typically, secondary glazing can be installed for 25% of the cost of double glazed windows - to be conservative, + # we scale the cost by half + SECONDARY_GLAZING_SCALING_FACTOR = 0.5 + def __init__(self, property_instance): """ Initializes the Costs class with a property instance. @@ -726,7 +730,7 @@ class Costs: "labour_cost": labour_costs } - def window_glazing(self, number_of_windows, material): + def window_glazing(self, number_of_windows, material, is_secondary_glazing=False): """ We characterise the jobs to be done for window glazing as the following: 1) Initial Assessment and Measurements: Before removing the existing window, it's essential to assess the @@ -767,17 +771,17 @@ class Costs: For this cost estimation process, we factor in initial assement into the preliminaries - :param number_of_windows: - :return: """ - material_cost = material["material_cost"] * number_of_windows * self.SASH_WINDOW_INFLATION_FACTOR - labour_cost = ( - material["labour_cost"] * number_of_windows * self.SASH_WINDOW_INFLATION_FACTOR * - self.labour_adjustment_factor - ) + material_cost = material["material_cost"] * number_of_windows - subtotal = material_cost + labour_cost + labour_cost = ( + material["labour_cost"] * number_of_windows * self.labour_adjustment_factor + ) + multiplier = self.SECONDARY_GLAZING_SCALING_FACTOR if is_secondary_glazing else ( + self.SASH_WINDOW_INFLATION_FACTOR) + + subtotal = (material_cost + labour_cost) * multiplier contingency_cost = subtotal * self.CONTINGENCY preliminaries_cost = subtotal * self.PRELIMINARIES @@ -790,6 +794,7 @@ class Costs: total_cost = subtotal_before_vat + vat_cost labour_hours = material["labour_hours_per_unit"] * number_of_windows + labour_hours = labour_hours * self.SECONDARY_GLAZING_SCALING_FACTOR if is_secondary_glazing else labour_hours # Assume a team of 2 labour_days = (labour_hours / 8) / 2 diff --git a/recommendations/WindowsRecommendations.py b/recommendations/WindowsRecommendations.py index 474071d9..0d3fb0af 100644 --- a/recommendations/WindowsRecommendations.py +++ b/recommendations/WindowsRecommendations.py @@ -1,9 +1,20 @@ from typing import List + +import numpy as np + from backend.Property import Property from recommendations.Costs import Costs class WindowsRecommendations: + # If the property has existing glazing, we scale down the number of windows that need to be glazed + COVERAGE_MAP = { + # If most of the windows have already been glazed, we assume that 2/3 are glazed and 1/2 are remaining to be + # glazed + "most": 0.33, + # If glazing is partial, we assume 50/50 split between glazed and unglazed + "partial": 0.5 + } def __init__(self, property_instance: Property, materials: List): self.property = property_instance @@ -28,7 +39,14 @@ class WindowsRecommendations: :return: """ + # If the property is in a conservation area or is a listed building, it becomes more difficult to install + # double glazing. Therefore, we don't recommend it. It is still possible but is not practical as it + # requires planning permission and might require a more expensive window type, such as timber. + number_of_windows = self.property.number_of_windows + is_secondary_glazing = self.property.restricted_measures or ( + self.property.windows["glazing_type"] == "secondary" + ) if not number_of_windows: raise ValueError("Number of windows not specified") @@ -36,13 +54,34 @@ class WindowsRecommendations: if self.property.windows["has_glazing"] & (self.property.windows["glazing_coverage"] == "full"): return + # We scale the number of windows based on the proportion of existing glazing + if self.property.data["multi-glaze-proportion"] != "": + n_windows_scalar = 1 - (self.property.data["multi-glaze-proportion"] / 100) + else: + n_windows_scalar = self.COVERAGE_MAP.get(self.property.windows["glazing_coverage"], 1) + + number_of_windows *= n_windows_scalar + number_of_windows = np.ceil(number_of_windows) + # We then price the job based on the number of windows that there are cost_result = self.costs.window_glazing( number_of_windows=number_of_windows, material=self.glazing_material, + is_secondary_glazing=is_secondary_glazing ) - description = None + glazing_type = "secondary glazing" if is_secondary_glazing else "double glazing" + if self.property.windows["glazing_coverage"] in ["partial", "most"]: + description = f"Install {glazing_type} to the remaining windows" + else: + description = f"Install {glazing_type} to all windows" + + if self.property.is_listed: + description += ". Secondary glazing recommended due to listed building status" + elif self.property.is_heritage: + description += ". Secondary glazing recommended due to conservation area status" + elif self.property.in_conservation_area: + description += ". Secondary glazing recommended due to conservation area status" self.recommendation = [ { diff --git a/recommendations/tests/test_window_recommendations.py b/recommendations/tests/test_window_recommendations.py index 1a146b41..fbb67243 100644 --- a/recommendations/tests/test_window_recommendations.py +++ b/recommendations/tests/test_window_recommendations.py @@ -18,7 +18,8 @@ class TestWindowRecommendations: address1='1', epc_client=Mock(), data={ - "county": "Wychavon" + "county": "Wychavon", + "multi-glaze-proportion": 0 } ) property_1.windows = { @@ -33,3 +34,104 @@ class TestWindowRecommendations: assert not recommender.recommendation recommender.recommend() + + assert recommender.recommendation == [ + {'parts': [], 'type': 'window_glazing', 'description': 'Install double glazing to all windows', + 'starting_u_value': None, 'new_u_value': None, 'sap_points': None, 'total': 5721.943248, + 'subtotal': 4768.28604, 'vat': 953.6572080000001, 'contingency': 340.59186, 'preliminaries': 340.59186, + 'material': 1275.75, 'profit': 681.18372, 'labour_hours': 45.5, 'labour_cost': 994.8624, + 'labour_days': 2.84375}] + + def test_partial_double_glazed(self): + """ + For this property, the double glazing is describes as partial, therefore we recommend completion of + double glazing + :return: + """ + + property_2 = Property( + id=1, + postcode='1', + address1='1', + epc_client=Mock(), + data={ + "county": "Wychavon", + "multi-glaze-proportion": 33 + } + ) + property_2.windows = {'original_description': 'Mostly double glazing', 'has_glazing': True, + 'glazing_coverage': 'most', + 'glazing_type': 'double', 'no_data': False} + property_2.number_of_windows = 7 + + recommender2 = WindowsRecommendations(property_instance=property_2, materials=materials) + + assert not recommender2.recommendation + + recommender2.recommend() + + assert recommender2.recommendation == [ + {'parts': [], 'type': 'window_glazing', 'description': 'Install double glazing to the remaining windows', + 'starting_u_value': None, 'new_u_value': None, 'sap_points': None, 'total': 4087.10232, + 'subtotal': 3405.9186, 'vat': 681.18372, 'contingency': 243.2799, 'preliminaries': 243.2799, + 'material': 911.25, 'profit': 486.5598, 'labour_hours': 32.5, 'labour_cost': 710.6160000000001, + 'labour_days': 2.03125}] + + def test_fully_double_glazed(self): + """ + This property has full double glazing so we shouldn't recommend anything + :return: + """ + + property_3 = Property( + id=1, + postcode='1', + address1='1', + epc_client=Mock(), + data={ + "county": "Wychavon", + "multi-glaze-proportion": 80 + } + ) + property_3.windows = {'original_description': 'Fully double glazed', 'has_glazing': True, + 'glazing_coverage': 'full', + 'glazing_type': 'double', 'no_data': False} + property_3.number_of_windows = 7 + + recommender3 = WindowsRecommendations(property_instance=property_3, materials=materials) + + assert not recommender3.recommendation + + recommender3.recommend() + + assert not recommender3.recommendation + + def test_fully_secondary_glazed(self): + property_4 = Property( + id=1, + postcode='1', + address1='1', + epc_client=Mock(), + data={ + "county": "Wychavon", + "multi-glaze-proportion": 100 + } + ) + property_4.windows = {'original_description': 'Full secondary glazing', 'has_glazing': True, + 'glazing_coverage': 'full', + 'glazing_type': 'secondary', 'no_data': False} + property_4.number_of_windows = 7 + + recommender4 = WindowsRecommendations(property_instance=property_4, materials=materials) + + assert not recommender4.recommendation + + recommender4.recommend() + + assert not recommender4.recommendation + + def test_partial_secondary_glazing(self): + pass + + def test_single_glazed_restricted_measures(self): + pass From 654a0715fa0b0bda1139f1d1a4c2d35b13b984a0 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 20 Dec 2023 16:55:00 +0000 Subject: [PATCH 09/16] Added remaining windows unit tests --- recommendations/WindowsRecommendations.py | 2 +- .../tests/test_window_recommendations.py | 119 +++++++++++++++++- 2 files changed, 118 insertions(+), 3 deletions(-) diff --git a/recommendations/WindowsRecommendations.py b/recommendations/WindowsRecommendations.py index 0d3fb0af..572b27fa 100644 --- a/recommendations/WindowsRecommendations.py +++ b/recommendations/WindowsRecommendations.py @@ -79,7 +79,7 @@ class WindowsRecommendations: if self.property.is_listed: description += ". Secondary glazing recommended due to listed building status" elif self.property.is_heritage: - description += ". Secondary glazing recommended due to conservation area status" + description += ". Secondary glazing recommended due to herigate building status" elif self.property.in_conservation_area: description += ". Secondary glazing recommended due to conservation area status" diff --git a/recommendations/tests/test_window_recommendations.py b/recommendations/tests/test_window_recommendations.py index fbb67243..f7884d77 100644 --- a/recommendations/tests/test_window_recommendations.py +++ b/recommendations/tests/test_window_recommendations.py @@ -131,7 +131,122 @@ class TestWindowRecommendations: assert not recommender4.recommendation def test_partial_secondary_glazing(self): - pass + property_5 = Property( + id=1, + postcode='1', + address1='1', + epc_client=Mock(), + data={ + "county": "Wychavon", + "multi-glaze-proportion": 50 + } + ) + property_5.windows = {'original_description': 'Partial secondary glazing', 'has_glazing': True, + 'glazing_coverage': 'partial', + 'glazing_type': 'secondary', 'no_data': False} + property_5.number_of_windows = 7 + + recommender5 = WindowsRecommendations(property_instance=property_5, materials=materials) + + assert not recommender5.recommendation + + recommender5.recommend() + + assert recommender5.recommendation == [ + {'parts': [], 'type': 'window_glazing', 'description': 'Install secondary glazing to the remaining windows', + 'starting_u_value': None, 'new_u_value': None, 'sap_points': None, 'total': 1089.893952, + 'subtotal': 908.24496, 'vat': 181.64899200000002, 'contingency': 64.87464, 'preliminaries': 64.87464, + 'material': 729.0, 'profit': 129.74928, 'labour_hours': 13.0, 'labour_cost': 568.4928, + 'labour_days': 0.8125}] def test_single_glazed_restricted_measures(self): - pass + property_6 = Property( + id=1, + postcode='1', + address1='1', + epc_client=Mock(), + data={ + "county": "Wychavon", + "multi-glaze-proportion": 0 + } + ) + property_6.windows = {'original_description': 'Single glazed', 'has_glazing': False, 'glazing_coverage': None, + 'glazing_type': 'single', + 'no_data': False} + property_6.number_of_windows = 7 + property_6.restricted_measures = True + property_6.is_heritage = True + + recommender6 = WindowsRecommendations(property_instance=property_6, materials=materials) + + assert not recommender6.recommendation + + recommender6.recommend() + + assert recommender6.recommendation == [ + {'parts': [], 'type': 'window_glazing', + 'description': 'Install secondary glazing to all windows. Secondary ' + 'glazing recommended due to herigate building status', + 'starting_u_value': None, 'new_u_value': None, 'sap_points': None, + 'total': 1907.314416, 'subtotal': 1589.42868, 'vat': 317.885736, + 'contingency': 113.53062, 'preliminaries': 113.53062, + 'material': 1275.75, 'profit': 227.06124, 'labour_hours': 22.75, + 'labour_cost': 994.8624, 'labour_days': 1.421875} + ] + + def test_full_triple_glazed(self): + property_7 = Property( + id=1, + postcode='1', + address1='1', + epc_client=Mock(), + data={ + "county": "Wychavon", + "multi-glaze-proportion": 100 + } + ) + property_7.windows = {'original_description': 'Fully triple glazed', 'has_glazing': True, + 'glazing_coverage': 'full', + 'glazing_type': 'triple', 'no_data': False} + property_7.number_of_windows = 7 + + recommender7 = WindowsRecommendations(property_instance=property_7, materials=materials) + + assert not recommender7.recommendation + + recommender7.recommend() + + assert not recommender7.recommendation + + def test_partial_triple_glazed(self): + """ + We should just recommend double glazing to the remaining windows, since it's a cheaper option + """ + + property_8 = Property( + id=1, + postcode='1', + address1='1', + epc_client=Mock(), + data={ + "county": "Wychavon", + "multi-glaze-proportion": 80 + } + ) + property_8.windows = {'original_description': 'Mostly triple glazing', 'has_glazing': True, + 'glazing_coverage': 'most', + 'glazing_type': 'triple', 'no_data': False} + property_8.number_of_windows = 7 + + recommender8 = WindowsRecommendations(property_instance=property_8, materials=materials) + + assert not recommender8.recommendation + + recommender8.recommend() + + assert recommender8.recommendation == [ + {'parts': [], 'type': 'window_glazing', 'description': 'Install double glazing to the remaining windows', + 'starting_u_value': None, 'new_u_value': None, 'sap_points': None, 'total': 1634.840928, + 'subtotal': 1362.36744, 'vat': 272.47348800000003, 'contingency': 97.31196, 'preliminaries': 97.31196, + 'material': 364.5, 'profit': 194.62392, 'labour_hours': 13.0, 'labour_cost': 284.2464, + 'labour_days': 0.8125}] From c5af2c23bb1aa0a8a2ee34b65de0033ec0e70e18 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 20 Dec 2023 17:06:37 +0000 Subject: [PATCH 10/16] created basic script to create windows portfolio --- etl/testing_data/windows_portfolio.py | 43 +++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 etl/testing_data/windows_portfolio.py diff --git a/etl/testing_data/windows_portfolio.py b/etl/testing_data/windows_portfolio.py new file mode 100644 index 00000000..16b28c8f --- /dev/null +++ b/etl/testing_data/windows_portfolio.py @@ -0,0 +1,43 @@ +""" +This script will create an input csv for the recommendation engine and upload it to S3, which can be used for +testing +""" +import pandas as pd +from utils.s3 import save_csv_to_s3 + +USER_ID = 8 +PORTFOLIO_ID = 56 + + +def app(): + """ + This portfolio is for testing windows recommendations + :return: + """ + + test_file = pd.DataFrame( + [ + {"address": "3 Church Terrace", "postcode": "LE13 0PW", "Notes": None}, + {"address": "3, Main Street, Redmile", "postcode": "NG13 0GA", "Notes": None}, + {"address": "The Hollies Farm House, Church Walk, Little Dalby", "postcode": "LE14 2UQ", "Notes": None}, + {"address": "13 Main Street", "postcode": "LE14 2JU", "Notes": None}, + {"address": "8 The Crescent, Coston Road, Buckminster", "postcode": "NG33 5SF", "Notes": None}, + ] + ) + + # Store the data in s3 + filename = f"{USER_ID}/{PORTFOLIO_ID}/windows_portfolio_inputs.csv" + save_csv_to_s3( + dataframe=test_file, + bucket_name="retrofit-plan-inputs-dev", + file_name=filename + ) + + body = { + "portfolio_id": str(PORTFOLIO_ID), + "housing_type": "Social", + "goal": "Increase EPC", + "goal_value": "A", + "trigger_file_path": filename + } + print(body) From fceb6a1d9626e75aed7c699b518107ce227bac1c Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 20 Dec 2023 17:25:28 +0000 Subject: [PATCH 11/16] debugging windows recommenddations integration --- backend/app/db/models/materials.py | 1 + backend/app/plan/router.py | 5 +++++ backend/app/plan/utils.py | 11 ++++++++++- recommendations/Recommendations.py | 7 +++++++ recommendations/WindowsRecommendations.py | 4 ++-- 5 files changed, 25 insertions(+), 3 deletions(-) diff --git a/backend/app/db/models/materials.py b/backend/app/db/models/materials.py index f887fc25..97085d7a 100644 --- a/backend/app/db/models/materials.py +++ b/backend/app/db/models/materials.py @@ -18,6 +18,7 @@ class MaterialType(enum.Enum): exposed_floor_insulation = "exposed_floor_insulation" flat_roof_insulation = "flat_roof_insulation" room_roof_insulation = "room_roof_insulation" + windows_glazing = "windows_glazing" iwi_wall_demolition = "iwi_wall_demolition" iwi_vapour_barrier = "iwi_vapour_barrier" diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 7a6df2da..f89395cc 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -61,6 +61,7 @@ async def trigger_plan(body: PlanTriggerRequest): logger.info("Getting the inputs") epc_client = EpcClient(auth_token=get_settings().EPC_AUTH_TOKEN) plan_input = read_csv_from_s3(bucket_name=get_settings().PLAN_TRIGGER_BUCKET, filepath=body.trigger_file_path) + uprn_filenames = read_dataframe_from_s3_parquet( bucket_name=get_settings().DATA_BUCKET, file_key="spatial/filename_meta.parquet" ) @@ -126,6 +127,10 @@ async def trigger_plan(body: PlanTriggerRequest): for p in input_properties: + # TODO: TEMP + if p.address1 == "The Hollies Farm House, Church Walk, Little Dalby": + continue + # Property recommendations p.get_components(cleaned) diff --git a/backend/app/plan/utils.py b/backend/app/plan/utils.py index 7aba99c9..4185c30e 100644 --- a/backend/app/plan/utils.py +++ b/backend/app/plan/utils.py @@ -175,11 +175,20 @@ def create_recommendation_scoring_data( scoring_dict["LOW_ENERGY_LIGHTING_ENDING"] = 100 scoring_dict["LIGHTING_ENERGY_EFF_STARTING"] = "Very Good" + if recommendation["type"] == "windows_glazing": + scoring_dict["MULTI_GLAZE_PROPORTION_ENDING"] = 100 + scoring_dict["WINDOWS_ENERGY_EFF_ENDING"] = "Average" + if scoring_dict["glazing_type_ENDING"] == "multiple": + pass + else: + raise NotImplementedError("Implement me") + if recommendation["type"] not in [ "mechanical_ventilation", "sealing_open_fireplace", "low_energy_lighting", "internal_wall_insulation", "external_wall_insulation", "cavity_wall_insulation", "loft_insulation", "room_roof_insulation", "flat_roof_insulation", - "solid_floor_insulation", "suspended_floor_insulation", "exposed_floor_insulation" + "solid_floor_insulation", "suspended_floor_insulation", "exposed_floor_insulation", + "windows_glazing" ]: raise NotImplementedError("Implement me") diff --git a/recommendations/Recommendations.py b/recommendations/Recommendations.py index 60cdb696..4b54cb52 100644 --- a/recommendations/Recommendations.py +++ b/recommendations/Recommendations.py @@ -6,6 +6,7 @@ from recommendations.RoofRecommendations import RoofRecommendations from recommendations.VentilationRecommendations import VentilationRecommendations from recommendations.FireplaceRecommendations import FireplaceRecommendations from recommendations.LightingRecommendations import LightingRecommendations +from recommendations.WindowsRecommendations import WindowsRecommendations from backend.ml_models.AnnualBillSavings import AnnualBillSavings @@ -35,6 +36,7 @@ class Recommendations: ) self.fireplace_recommender = FireplaceRecommendations(property_instance=property_instance) self.lighting_recommender = LightingRecommendations(property_instance=property_instance, materials=materials) + self.windows_recommender = WindowsRecommendations(property_instance=property_instance, materials=materials) def recommend(self): @@ -77,6 +79,11 @@ class Recommendations: if self.lighting_recommender.recommendation: property_recommendations.append(self.lighting_recommender.recommendation) + # Windows recommendations + self.windows_recommender.recommend() + if self.windows_recommender.recommendation: + property_recommendations.append(self.windows_recommender.recommendation) + # We insert temporary ids into the recommendations which is important for the optimiser later property_recommendations = self.insert_temp_recommendation_id(property_recommendations) diff --git a/recommendations/WindowsRecommendations.py b/recommendations/WindowsRecommendations.py index 572b27fa..6246fd95 100644 --- a/recommendations/WindowsRecommendations.py +++ b/recommendations/WindowsRecommendations.py @@ -56,7 +56,7 @@ class WindowsRecommendations: # We scale the number of windows based on the proportion of existing glazing if self.property.data["multi-glaze-proportion"] != "": - n_windows_scalar = 1 - (self.property.data["multi-glaze-proportion"] / 100) + n_windows_scalar = 1 - (int(self.property.data["multi-glaze-proportion"]) / 100) else: n_windows_scalar = self.COVERAGE_MAP.get(self.property.windows["glazing_coverage"], 1) @@ -86,7 +86,7 @@ class WindowsRecommendations: self.recommendation = [ { "parts": [], - "type": "window_glazing", + "type": "windows_glazing", "description": description, "starting_u_value": None, "new_u_value": None, From e65d190e1938eddfb28d549ee1a760a591833d0f Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 21 Dec 2023 11:23:30 +0000 Subject: [PATCH 12/16] removed todo --- .idea/Model.iml | 2 +- .idea/misc.xml | 2 +- backend/app/plan/router.py | 7 ------- backend/app/plan/utils.py | 2 +- etl/testing_data/windows_portfolio.py | 2 +- 5 files changed, 4 insertions(+), 11 deletions(-) diff --git a/.idea/Model.iml b/.idea/Model.iml index 4413bb06..b0f9c00d 100644 --- a/.idea/Model.iml +++ b/.idea/Model.iml @@ -7,7 +7,7 @@ - + diff --git a/.idea/misc.xml b/.idea/misc.xml index 6f308057..1122b380 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -3,7 +3,7 @@ - + diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index f89395cc..952d4982 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -118,19 +118,12 @@ async def trigger_plan(body: PlanTriggerRequest): logger.info("Getting components and epc recommendations") - # TODO: Move this to a class. We probably want a Recommender class which takes the injects the optimisers - # in as a dependency and then the optimisers can take the input measures in as part of the setup() method - recommendations = {} recommendations_scoring_data = [] property_scoring_data = {} for p in input_properties: - # TODO: TEMP - if p.address1 == "The Hollies Farm House, Church Walk, Little Dalby": - continue - # Property recommendations p.get_components(cleaned) diff --git a/backend/app/plan/utils.py b/backend/app/plan/utils.py index 4185c30e..90042fa2 100644 --- a/backend/app/plan/utils.py +++ b/backend/app/plan/utils.py @@ -178,7 +178,7 @@ def create_recommendation_scoring_data( if recommendation["type"] == "windows_glazing": scoring_dict["MULTI_GLAZE_PROPORTION_ENDING"] = 100 scoring_dict["WINDOWS_ENERGY_EFF_ENDING"] = "Average" - if scoring_dict["glazing_type_ENDING"] == "multiple": + if scoring_dict["glazing_type_ENDING"] in ["multiple", "double"]: pass else: raise NotImplementedError("Implement me") diff --git a/etl/testing_data/windows_portfolio.py b/etl/testing_data/windows_portfolio.py index 16b28c8f..356d107e 100644 --- a/etl/testing_data/windows_portfolio.py +++ b/etl/testing_data/windows_portfolio.py @@ -19,7 +19,7 @@ def app(): [ {"address": "3 Church Terrace", "postcode": "LE13 0PW", "Notes": None}, {"address": "3, Main Street, Redmile", "postcode": "NG13 0GA", "Notes": None}, - {"address": "The Hollies Farm House, Church Walk, Little Dalby", "postcode": "LE14 2UQ", "Notes": None}, + {"address": "Manor House, Kennel Lane, Reepham", "postcode": "LN3 4DZ", "Notes": None}, {"address": "13 Main Street", "postcode": "LE14 2JU", "Notes": None}, {"address": "8 The Crescent, Coston Road, Buckminster", "postcode": "NG33 5SF", "Notes": None}, ] From 5de6b3f02968eab9dc04c46024253bb4ce20a4aa Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 21 Dec 2023 11:27:22 +0000 Subject: [PATCH 13/16] fixing unit tests --- .../epc_attributes/WindowAttributes.py | 2 +- etl/epc_clean/tests/test_roof_attributes.py | 18 ++++++++++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/etl/epc_clean/epc_attributes/WindowAttributes.py b/etl/epc_clean/epc_attributes/WindowAttributes.py index cd767359..ce0b156a 100644 --- a/etl/epc_clean/epc_attributes/WindowAttributes.py +++ b/etl/epc_clean/epc_attributes/WindowAttributes.py @@ -80,7 +80,7 @@ class WindowAttributes(Definitions): break # If we didn't find any coverage or type, we assume full coverage - if not result["glazing_coverage"]: + if (not result["glazing_coverage"]) & (result["glazing_type"] != "single"): result["glazing_coverage"] = "full" # We reset some values if the glazing is single diff --git a/etl/epc_clean/tests/test_roof_attributes.py b/etl/epc_clean/tests/test_roof_attributes.py index b0663a3e..481beedc 100644 --- a/etl/epc_clean/tests/test_roof_attributes.py +++ b/etl/epc_clean/tests/test_roof_attributes.py @@ -3,12 +3,13 @@ from pathlib import Path from etl.epc_clean.tests.test_data.test_roof_attributes_cases import clean_roof_test_cases from etl.epc_clean.epc_attributes.RoofAttributes import RoofAttributes + # For local testing -if __file__ == "": - input_data_path = Path("./model_data/tests/test_data/EpcClean_inputs.obj") -else: - current_file_path = Path(__file__) - input_data_path = current_file_path.parent / 'test_data' / 'EpcClean_inputs.obj' +# if __file__ == "": +# input_data_path = Path("./model_data/tests/test_data/EpcClean_inputs.obj") +# else: +# current_file_path = Path(__file__) +# input_data_path = current_file_path.parent / 'test_data' / 'EpcClean_inputs.obj' class TestRoofAttributes: @@ -88,7 +89,12 @@ class TestRoofAttributes: def test_clean_roof_no_description(self): roof = RoofAttributes('').process() - assert roof == {} + assert roof == { + 'thermal_transmittance': False, 'thermal_transmittance_unit': False, 'is_pitched': False, + 'is_roof_room': False, 'is_loft': False, 'is_flat': False, 'is_thatched': False, + 'is_at_rafters': False, 'is_assumed': False, 'has_dwelling_above': False, 'is_valid': False, + 'insulation_thickness': False + } def test_clean_roof_edge_cases(self): # Insulation thickness edge case From 10c9a6f472b6b3ebcbd99842529d24fb3548a10e Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 21 Dec 2023 14:09:32 +0000 Subject: [PATCH 14/16] fixed all recommendations apart from windows --- backend/tests/test_property.py | 15 ++++++++++----- etl/epc/property_change_app.py | 13 +++++++++++++ .../tests/test_floor_recommendations.py | 4 ++++ .../tests/test_lighting_recommendations.py | 7 +++---- 4 files changed, 30 insertions(+), 9 deletions(-) diff --git a/backend/tests/test_property.py b/backend/tests/test_property.py index 871c9291..0113d690 100644 --- a/backend/tests/test_property.py +++ b/backend/tests/test_property.py @@ -37,7 +37,8 @@ mock_epc_response = { "floor-height": 2.5, "total-floor-area": 100, "construction-age-band": "England and Wales: 1967-1975", - "floor-description": "Floor Description" + "floor-description": "Floor Description", + "floor-level": "Ground" }, { "lmk-key": 2, @@ -68,7 +69,8 @@ mock_epc_response = { "floor-height": 2.5, "total-floor-area": 100, "construction-age-band": "England and Wales: 1967-1975", - "floor-description": "Floor Description" + "floor-description": "Floor Description", + "floor-level": "Ground" } ] } @@ -100,7 +102,8 @@ mock_epc_response_dupe = { "floor-height": 2.5, "total-floor-area": 100, "construction-age-band": "England and Wales: 1967-1975", - "floor-description": "Floor Description" + "floor-description": "Floor Description", + "floor-level": "Ground" }, { "lmk-key": 2, @@ -128,7 +131,8 @@ mock_epc_response_dupe = { "floor-height": 2.5, "total-floor-area": 100, "construction-age-band": "England and Wales: 1967-1975", - "floor-description": "Floor Description" + "floor-description": "Floor Description", + "floor-level": "Ground" }, { "lmk-key": 3, @@ -156,7 +160,8 @@ mock_epc_response_dupe = { "floor-height": 2.5, "total-floor-area": 100, "construction-age-band": "England and Wales: 1967-1975", - "floor-description": "Floor Description" + "floor-description": "Floor Description", + "floor-level": "Ground" } ] } diff --git a/etl/epc/property_change_app.py b/etl/epc/property_change_app.py index 4f49f6da..f8c293a1 100644 --- a/etl/epc/property_change_app.py +++ b/etl/epc/property_change_app.py @@ -631,6 +631,19 @@ def app(): file_key="sap_change_model/all_equal_rows.parquet", ) + from utils.s3 import read_dataframe_from_s3_parquet + dataset = read_dataframe_from_s3_parquet( + bucket_name="retrofit-data-dev", + file_key="sap_change_model/dataset_test.parquet", + ) + + z = dataset[dataset["CONSTITUENCY"].isin(["E14000707", "E14000909"])] + z["CONSTITUENCY"].value_counts() + + z[z["CONSTITUENCY"] == "E14000909"]["UPRN"].sample(1) + + self.data[self.data["UPRN"] == "100030549358"] + if __name__ == "__main__": app() diff --git a/recommendations/tests/test_floor_recommendations.py b/recommendations/tests/test_floor_recommendations.py index 43e98d60..700d33d3 100644 --- a/recommendations/tests/test_floor_recommendations.py +++ b/recommendations/tests/test_floor_recommendations.py @@ -68,6 +68,7 @@ class TestFloorRecommendations: input_properties[2].wall_type = "solid brick" input_properties[2].floor_type = "suspended" input_properties[2].number_of_floors = 1 + input_properties[2].floor_level = 0 recommender = FloorRecommendations(property_instance=input_properties[2], materials=materials) assert recommender.estimated_u_value is None @@ -93,6 +94,8 @@ class TestFloorRecommendations: input_properties[3].insulation_floor_area = 100 input_properties[3].insulation_wall_area = 100 input_properties[3].number_of_floors = 1 + input_properties[3].floor_level = 0 + recommender = FloorRecommendations(property_instance=input_properties[3], materials=materials) assert recommender.estimated_u_value is None recommender.recommend() @@ -114,6 +117,7 @@ class TestFloorRecommendations: input_properties[4].wall_type = "solid brick" input_properties[4].floor_type = "solid" input_properties[4].number_of_floors = 1 + input_properties[4].floor_level = 0 # In this case, we have no county, so in this case, it should yse the local-authority-label if possible input_properties[4].data["county"] = "" diff --git a/recommendations/tests/test_lighting_recommendations.py b/recommendations/tests/test_lighting_recommendations.py index 06d1163f..5a4545eb 100644 --- a/recommendations/tests/test_lighting_recommendations.py +++ b/recommendations/tests/test_lighting_recommendations.py @@ -40,8 +40,7 @@ class TestLightingRecommendations: assert lr.recommendation == [ {'parts': [], 'type': 'low_energy_lighting', 'description': 'Install low energy lighting in 4 outlets', - 'starting_u_value': None, 'new_u_value': None, 'sap_points': 0.4, 'total': 458.976, 'subtotal': 382.48, - 'vat': 76.49600000000001, 'contingency': 27.320000000000007, 'preliminaries': 27.320000000000007, - 'material': 80.0, 'profit': 54.640000000000015, 'labour_hours': 3.2, 'labour_days': 0.4, - 'labour_cost': 193.20000000000002} + 'starting_u_value': None, 'new_u_value': None, 'sap_points': 0.4, 'total': 240.24, + 'subtotal': 200.20000000000002, 'vat': 40.040000000000006, 'contingency': 14.3, 'preliminaries': 14.3, + 'material': 80.0, 'profit': 28.6, 'labour_hours': 3.2, 'labour_days': 0.4, 'labour_cost': 63.0} ] From 7282800240d31732cf4c200fe7ea60af9b0647a6 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 21 Dec 2023 14:11:47 +0000 Subject: [PATCH 15/16] Fixed unit tests --- recommendations/tests/test_window_recommendations.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/recommendations/tests/test_window_recommendations.py b/recommendations/tests/test_window_recommendations.py index f7884d77..359dd962 100644 --- a/recommendations/tests/test_window_recommendations.py +++ b/recommendations/tests/test_window_recommendations.py @@ -36,7 +36,7 @@ class TestWindowRecommendations: recommender.recommend() assert recommender.recommendation == [ - {'parts': [], 'type': 'window_glazing', 'description': 'Install double glazing to all windows', + {'parts': [], 'type': 'windows_glazing', 'description': 'Install double glazing to all windows', 'starting_u_value': None, 'new_u_value': None, 'sap_points': None, 'total': 5721.943248, 'subtotal': 4768.28604, 'vat': 953.6572080000001, 'contingency': 340.59186, 'preliminaries': 340.59186, 'material': 1275.75, 'profit': 681.18372, 'labour_hours': 45.5, 'labour_cost': 994.8624, @@ -71,7 +71,7 @@ class TestWindowRecommendations: recommender2.recommend() assert recommender2.recommendation == [ - {'parts': [], 'type': 'window_glazing', 'description': 'Install double glazing to the remaining windows', + {'parts': [], 'type': 'windows_glazing', 'description': 'Install double glazing to the remaining windows', 'starting_u_value': None, 'new_u_value': None, 'sap_points': None, 'total': 4087.10232, 'subtotal': 3405.9186, 'vat': 681.18372, 'contingency': 243.2799, 'preliminaries': 243.2799, 'material': 911.25, 'profit': 486.5598, 'labour_hours': 32.5, 'labour_cost': 710.6160000000001, @@ -153,7 +153,8 @@ class TestWindowRecommendations: recommender5.recommend() assert recommender5.recommendation == [ - {'parts': [], 'type': 'window_glazing', 'description': 'Install secondary glazing to the remaining windows', + {'parts': [], 'type': 'windows_glazing', + 'description': 'Install secondary glazing to the remaining windows', 'starting_u_value': None, 'new_u_value': None, 'sap_points': None, 'total': 1089.893952, 'subtotal': 908.24496, 'vat': 181.64899200000002, 'contingency': 64.87464, 'preliminaries': 64.87464, 'material': 729.0, 'profit': 129.74928, 'labour_hours': 13.0, 'labour_cost': 568.4928, @@ -184,7 +185,7 @@ class TestWindowRecommendations: recommender6.recommend() assert recommender6.recommendation == [ - {'parts': [], 'type': 'window_glazing', + {'parts': [], 'type': 'windows_glazing', 'description': 'Install secondary glazing to all windows. Secondary ' 'glazing recommended due to herigate building status', 'starting_u_value': None, 'new_u_value': None, 'sap_points': None, @@ -245,7 +246,7 @@ class TestWindowRecommendations: recommender8.recommend() assert recommender8.recommendation == [ - {'parts': [], 'type': 'window_glazing', 'description': 'Install double glazing to the remaining windows', + {'parts': [], 'type': 'windows_glazing', 'description': 'Install double glazing to the remaining windows', 'starting_u_value': None, 'new_u_value': None, 'sap_points': None, 'total': 1634.840928, 'subtotal': 1362.36744, 'vat': 272.47348800000003, 'contingency': 97.31196, 'preliminaries': 97.31196, 'material': 364.5, 'profit': 194.62392, 'labour_hours': 13.0, 'labour_cost': 284.2464, From c6247f331a11a3ebb5150d413661a3822d9222c3 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 21 Dec 2023 15:33:17 +0000 Subject: [PATCH 16/16] integrating windows recommendations into engine api --- .idea/Model.iml | 2 +- .idea/misc.xml | 2 +- backend/app/plan/router.py | 4 +--- backend/app/plan/utils.py | 15 +++++++++++++-- backend/ml_models/Valuation.py | 9 ++++++++- recommendations/WindowsRecommendations.py | 3 ++- 6 files changed, 26 insertions(+), 9 deletions(-) diff --git a/.idea/Model.iml b/.idea/Model.iml index b0f9c00d..4413bb06 100644 --- a/.idea/Model.iml +++ b/.idea/Model.iml @@ -7,7 +7,7 @@ - + diff --git a/.idea/misc.xml b/.idea/misc.xml index 1122b380..6f308057 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -3,7 +3,7 @@ - + diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 952d4982..a284e50b 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -544,9 +544,7 @@ async def trigger_plan(body: PlanTriggerRequest): valuations = PropertyValuation.estimate(property_instance=p, target_epc=new_epc) - property_valuation_increases.append( - valuations["average_increased_value"] - valuations["current_value"] - ) + property_valuation_increases.append(valuations["average_increase"]) # Commit the session after each batch session.commit() diff --git a/backend/app/plan/utils.py b/backend/app/plan/utils.py index 90042fa2..b82be297 100644 --- a/backend/app/plan/utils.py +++ b/backend/app/plan/utils.py @@ -178,10 +178,21 @@ def create_recommendation_scoring_data( if recommendation["type"] == "windows_glazing": scoring_dict["MULTI_GLAZE_PROPORTION_ENDING"] = 100 scoring_dict["WINDOWS_ENERGY_EFF_ENDING"] = "Average" - if scoring_dict["glazing_type_ENDING"] in ["multiple", "double"]: + + is_secondary_glazing = recommendation["is_secondary_glazing"] + + if scoring_dict["glazing_type_ENDING"] == "multiple": pass + elif scoring_dict["glazing_type_ENDING"] == "single": + scoring_dict["glazing_type_ENDING"] = "secondary" if is_secondary_glazing else "double" + elif scoring_dict["glazing_type_ENDING"] == "double": + scoring_dict["glazing_type_ENDING"] = "multiple" if is_secondary_glazing else "double" + elif scoring_dict["glazing_type_ENDING"] == "secondary": + scoring_dict["glazing_type_ENDING"] = "secondary" if is_secondary_glazing else "multiple" + elif scoring_dict["glazing_type_ENDING"] in ["triple", "high performance"]: + scoring_dict["glazing_type_ENDING"] = "multiple" else: - raise NotImplementedError("Implement me") + raise ValueError("Invalid glazing type - implement me") if recommendation["type"] not in [ "mechanical_ventilation", "sealing_open_fireplace", "low_energy_lighting", diff --git a/backend/ml_models/Valuation.py b/backend/ml_models/Valuation.py index f5a7e2bb..cdbbe698 100644 --- a/backend/ml_models/Valuation.py +++ b/backend/ml_models/Valuation.py @@ -93,7 +93,13 @@ class PropertyValuation: value = cls.UPRN_VALUE_LOOKUP.get(property_instance.uprn) if not value: - raise ValueError("Have not implemented valuation for this property") + return { + "current_value": None, + "lower_bound_increased_value": None, + "upper_bound_increased_value": None, + "average_increased_value": None, + "average_increase": None + } current_epc = property_instance.data["current-energy-rating"] # We get the spectrum of ratings between the current and target EPC @@ -119,4 +125,5 @@ class PropertyValuation: "lower_bound_increased_value": value * (1 + min_increase), "upper_bound_increased_value": value * (1 + max_increase), "average_increased_value": value * (1 + avg_increase), + "average_increase": value * (1 + avg_increase) - value } diff --git a/recommendations/WindowsRecommendations.py b/recommendations/WindowsRecommendations.py index 6246fd95..b6ecd099 100644 --- a/recommendations/WindowsRecommendations.py +++ b/recommendations/WindowsRecommendations.py @@ -91,6 +91,7 @@ class WindowsRecommendations: "starting_u_value": None, "new_u_value": None, "sap_points": None, - **cost_result + **cost_result, + "is_secondary_glazing": is_secondary_glazing } ]