From e2633dfa5b7246338a083b0676af55c3fd61a71a Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 6 Oct 2023 11:39:41 +0100 Subject: [PATCH] restructuing property class to move more property level attributes to the Property class --- .idea/Model.iml | 2 +- .idea/misc.xml | 2 +- backend/Property.py | 55 +++++++++++++++-- backend/app/plan/router.py | 1 - etl/epc/DataProcessor.py | 2 +- etl/epc/settings.py | 22 ------- recommendations/FloorRecommendations.py | 60 +++---------------- recommendations/rdsap_tables.py | 27 +++++++++ recommendations/recommendation_utils.py | 19 ++++++ utils/tests/test_uvalue_estimates.py | 80 ------------------------- utils/uvalue_estimates.py | 39 ------------ 11 files changed, 107 insertions(+), 202 deletions(-) delete mode 100644 utils/tests/test_uvalue_estimates.py delete mode 100644 utils/uvalue_estimates.py diff --git a/.idea/Model.iml b/.idea/Model.iml index b0f9c00d..4413bb06 100644 --- a/.idea/Model.iml +++ b/.idea/Model.iml @@ -7,7 +7,7 @@ - + diff --git a/.idea/misc.xml b/.idea/misc.xml index ca0e1cd9..3b05c6ac 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -1,6 +1,6 @@ - + diff --git a/backend/Property.py b/backend/Property.py index 036ab87e..efc48531 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -9,6 +9,7 @@ from utils.s3 import read_dataframe_from_s3_parquet from epc_api.client import EpcClient from BaseUtility import Definitions from recommendations.rdsap_tables import england_wales_age_band_lookup +from recommendations.recommendation_utils import estimate_floors, estimate_perimeter, get_wall_type ENVIRONMENT = os.environ.get('ENVIRONMENT', 'dev') EPC_AUTH_TOKEN = os.environ.get('EPC_AUTH_TOKEN') @@ -41,7 +42,6 @@ class Property(Definitions): lighting = None coordinates = None - age_band = None def __init__(self, id, postcode, address1, epc_client=None, data=None): self.id = id @@ -54,6 +54,10 @@ class Property(Definitions): self.restricted_measures = False self.year_built = None self.number_of_rooms = None + self.age_band = None + self.number_of_floors = None + self.perimeter = None + self.wall_type = None self.energy = None self.ventilation = None @@ -263,6 +267,10 @@ class Property(Definitions): self.set_floor_area() self.set_age_band() + self.set_number_floors() + self.set_perimeter() + self.set_wall_type() + for description, attribute in cleaned.items(): if self.data[description] in self.DATA_ANOMALY_MATCHES: @@ -467,11 +475,9 @@ class Property(Definitions): """ This method is placeholder It implements our floor area model to produce an estimate of the property's insulatable wall area + While we do not have the """ - import random - self.insulation_wall_area = random.uniform(60, 100) - def set_floor_area(self): """ Sets the floor area based on the EPC data @@ -508,3 +514,44 @@ class Property(Definitions): # Pull out spatial features self.set_spatial(spatial) + + def set_number_floors(self): + """ + This method sets the number of floors of the property, using a simple approach based on an estimate for + average room size, number of rooms and total floor area + :return: + """ + + total_floor_area = float(self.data["total-floor-area"]) + number_of_rooms = float(self.data["number-habitable-rooms"]) + + if self.data["property-type"] == "House": + self.number_of_floors = estimate_floors(total_floor_area, number_of_rooms) + elif self.data["property-type"] == "Flat": + self.number_of_floors = 1 + else: + raise NotImplementedError("Implement me") + + def set_perimeter(self): + """ + This method sets the perimeter of the property, using a simple approach based on average room + size, number of rooms and total floor area + :return: + """ + + if not self.number_of_floors: + raise ValueError("Number of floors not set, run set_number_floors") + + total_floor_area = float(self.data["total-floor-area"]) + number_of_rooms = float(self.data["number-habitable-rooms"]) + + self.perimeter = estimate_perimeter( + total_floor_area / self.number_of_floors, number_of_rooms / self.number_of_floors + ) + + def set_wall_type(self): + """ + This method sets the wall type of the property, using a simple approach based on the wall description + :return: + """ + self.wall_type = get_wall_type(**self.walls) diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 0749803b..cbac55c3 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -151,7 +151,6 @@ async def trigger_plan(body: PlanTriggerRequest): if not property_recommendations: continue - fewf recommendations[p.id] = property_recommendations diff --git a/etl/epc/DataProcessor.py b/etl/epc/DataProcessor.py index 357faa08..27bac020 100644 --- a/etl/epc/DataProcessor.py +++ b/etl/epc/DataProcessor.py @@ -7,7 +7,6 @@ from etl.epc.settings import ( EARLIEST_EPC_DATE, FULLY_GLAZED_DESCRIPTIONS, AVERAGE_FIXED_FEATURES, - FLOOR_LEVEL_MAP, BUILT_FORM_REMAP, COLUMNS_TO_MERGE_ON, COMPONENT_FEATURES, @@ -17,6 +16,7 @@ from etl.epc.settings import ( MAX_SAP_SCORE, fill_na_map, ) +from recommendations.rdsap_tables import FLOOR_LEVEL_MAP from typing import List diff --git a/etl/epc/settings.py b/etl/epc/settings.py index e64ba210..9ebb0806 100644 --- a/etl/epc/settings.py +++ b/etl/epc/settings.py @@ -133,28 +133,6 @@ RDSAP_RESPONSE = "CURRENT_ENERGY_EFFICIENCY" HEAT_DEMAND_RESPONSE = "ENERGY_CONSUMPTION_CURRENT" CARBON_RESPONSE = "CO2_EMISSIONS_CURRENT" - -def ordinal(n): - if 10 <= n % 100 <= 20: - suffix = "th" - else: - suffix = {1: "st", 2: "nd", 3: "rd"}.get(n % 10, "th") - - return str(n) + suffix - - -FLOOR_LEVEL_MAP = { - "Basement": -1, - "Ground": 0, - "ground floor": 0, - "20+": 20, - "21st or above": 21, - **{str(i).zfill(2): i for i in range(0, 21)}, - **{ordinal(i): i for i in range(-1, 21)}, - **{str(i): i for i in range(-1, 21)}, - **{i: i for i in range(-1, 21)}, -} - BUILT_FORM_REMAP = { "Enclosed End-Terrace": "End-Terrace", "Enclosed Mid-Terrace": "Mid-Terrace", diff --git a/recommendations/FloorRecommendations.py b/recommendations/FloorRecommendations.py index ef8c1a68..980f6ec4 100644 --- a/recommendations/FloorRecommendations.py +++ b/recommendations/FloorRecommendations.py @@ -5,9 +5,9 @@ from datatypes.enums import QuantityUnits from backend.Property import Property from recommendations.recommendation_utils import ( r_value_per_mm_to_u_value, calculate_u_value_uplift, is_diminishing_returns, update_lowest_selected_u_value, - get_recommended_part, estimate_perimeter, get_wall_type, - get_floor_u_value + get_recommended_part, get_floor_u_value ) +from recommendations.rdsap_tables import FLOOR_LEVEL_MAP class FloorRecommendations(Definitions): @@ -24,18 +24,6 @@ class FloorRecommendations(Definitions): PART_L_YEAR_CUTOFF = 2002 - # TODO: This is a placeholder methodology which isn't particularly scalable as more - # unusual floor descriptions are introduced - FLOOR_LEVELS = { - "Ground": 0, - # We don't know what floor level, we just make sure it's not 0 - "mid floor": 1, - "4th": 4, - # We set - "00": 0, - "3rd": 3 - } - def __init__( self, property_instance: Property, @@ -60,10 +48,9 @@ class FloorRecommendations(Definitions): def recommend(self): u_value = self.property.floor["thermal_transmittance"] is_suspended = self.property.floor["is_suspended"] - insulation_thickness = self.property.floor["insulation_thickness"] is_solid = self.property.floor["is_solid"] floor_level = ( - self.FLOOR_LEVELS[self.property.data["floor-level"]] if + FLOOR_LEVEL_MAP[self.property.data["floor-level"]] if self.property.data["floor-level"] not in self.DATA_ANOMALY_MATCHES else None ) property_type = self.property.data["property-type"] @@ -91,27 +78,13 @@ class FloorRecommendations(Definitions): # The floor is already compliant return - total_floor_area = float(self.property.data["total-floor-area"]) - number_of_rooms = float(self.property.data["number-habitable-rooms"]) - - if self.property.data["property-type"] == "House": - num_floors = self._estimate_floors(total_floor_area, number_of_rooms) - elif self.property.data["property-type"] == "Flat": - num_floors = 1 - else: - raise NotImplementedError("Implement me") - - estimated_perimeter = estimate_perimeter(total_floor_area / num_floors, number_of_rooms / num_floors) - - wall_type = get_wall_type(**self.property.walls) - u_value = get_floor_u_value( floor_type="suspended" if is_suspended else "solid", - area=total_floor_area, - perimeter=estimated_perimeter, + area=float(self.property.data["total-floor-area"]), + perimeter=self.property.perimeter, age_band=self.property.age_band, - insulation_thickness=insulation_thickness, - wall_type=wall_type + insulation_thickness=self.property.floor["insulation_thickness"], + wall_type=self.property.wall_type ) self.estimated_u_value = u_value @@ -170,22 +143,3 @@ class FloorRecommendations(Definitions): "cost": estimated_cost, } ) - - @staticmethod - def _estimate_floors(floor_area, num_rooms): - """ - Simple utility funciton, which assuming a 15m squared room, estimates the number of floors in a property - :param floor_area: Gross floor area of a property - :param num_rooms: Number of rooms in a property - :return: Number of floors in a property - """ - # Estimate total room area - total_room_area = num_rooms * 15 - - # Estimate the number of floors - floors = floor_area / total_room_area - - # Round up to the nearest whole number - floors = round(floors) - - return floors diff --git a/recommendations/rdsap_tables.py b/recommendations/rdsap_tables.py index 71c52354..589d3bb4 100644 --- a/recommendations/rdsap_tables.py +++ b/recommendations/rdsap_tables.py @@ -462,3 +462,30 @@ s11_list = [ ] table_s11 = pd.DataFrame(s11_list) + + +######################################################################################################################## +# +# this map is used to clean the floor value field we see in EPCs +# +######################################################################################################################## +def ordinal(n): + if 10 <= n % 100 <= 20: + suffix = "th" + else: + suffix = {1: "st", 2: "nd", 3: "rd"}.get(n % 10, "th") + + return str(n) + suffix + + +FLOOR_LEVEL_MAP = { + "Basement": -1, + "Ground": 0, + "ground floor": 0, + "20+": 20, + "21st or above": 21, + **{str(i).zfill(2): i for i in range(0, 21)}, + **{ordinal(i): i for i in range(-1, 21)}, + **{str(i): i for i in range(-1, 21)}, + **{i: i for i in range(-1, 21)}, +} diff --git a/recommendations/recommendation_utils.py b/recommendations/recommendation_utils.py index 8a113050..35665a92 100644 --- a/recommendations/recommendation_utils.py +++ b/recommendations/recommendation_utils.py @@ -476,3 +476,22 @@ def get_wall_type( return "park home" return None + + +def estimate_floors(floor_area, num_rooms): + """ + Simple utility funciton, which assuming a 15m squared room, estimates the number of floors in a property + :param floor_area: Gross floor area of a property + :param num_rooms: Number of rooms in a property + :return: Number of floors in a property + """ + # Estimate total room area + total_room_area = num_rooms * 15 + + # Estimate the number of floors + floors = floor_area / total_room_area + + # Round up to the nearest whole number + floors = round(floors) + + return floors diff --git a/utils/tests/test_uvalue_estimates.py b/utils/tests/test_uvalue_estimates.py deleted file mode 100644 index 1b29994c..00000000 --- a/utils/tests/test_uvalue_estimates.py +++ /dev/null @@ -1,80 +0,0 @@ -from utils.uvalue_estimates import classify_decile_newvalues - - -def test_classify_decile_newvalues_edge_cases(): - decile_labels = [f"Decile {i + 1}" for i in range(10)] - decile_boundaries = list(range(11)) - - # Test with values at the exact boundaries - assert classify_decile_newvalues(decile_boundaries, decile_labels, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) == ['Decile 1', - 'Decile 2', - 'Decile 3', - 'Decile 4', - 'Decile 5', - 'Decile 6', - 'Decile 7', - 'Decile 8', - 'Decile 9', - 'Decile 10'] - - # Test with values at the exact boundaries, but in reverse order - assert classify_decile_newvalues(decile_boundaries, decile_labels, [10, 9, 8, 7, 6, 5, 4, 3, 2, 1]) == ['Decile 10', - 'Decile 9', - 'Decile 8', - 'Decile 7', - 'Decile 6', - 'Decile 5', - 'Decile 4', - 'Decile 3', - 'Decile 2', - 'Decile 1'] - - # Test with values just below the boundaries - assert classify_decile_newvalues(decile_boundaries, decile_labels, [x - 0.5 for x in range(2, 12)]) == ['Decile 1', - 'Decile 2', - 'Decile 3', - 'Decile 4', - 'Decile 5', - 'Decile 6', - 'Decile 7', - 'Decile 8', - 'Decile 9', - 'Decile 10'] - - # Test with values just above the boundaries - assert classify_decile_newvalues(decile_boundaries, decile_labels, [x + 0.5 for x in range(1, 11)]) == ['Decile 2', - 'Decile 3', - 'Decile 4', - 'Decile 5', - 'Decile 6', - 'Decile 7', - 'Decile 8', - 'Decile 9', - 'Decile 10', - None] - - # Test with empty list - assert classify_decile_newvalues(decile_boundaries, decile_labels, []) == [] - - # Test with a single value - assert classify_decile_newvalues(decile_boundaries, decile_labels, [5.5]) == ['Decile 6'] - - # Test with all values the same - assert classify_decile_newvalues(decile_boundaries, decile_labels, [5, 5, 5, 5, 5]) == ['Decile 5', 'Decile 5', - 'Decile 5', 'Decile 5', - 'Decile 5'] - - # Test with values out of order - assert classify_decile_newvalues(decile_boundaries, decile_labels, [10, 5, 1, 7, 3]) == ['Decile 10', 'Decile 5', - 'Decile 1', 'Decile 7', - 'Decile 3'] - - # Test with negative decile boundaries - decile_boundaries = list(range(-10, 1)) - assert classify_decile_newvalues(decile_boundaries, decile_labels, [-9, -5, -1]) == ['Decile 2', 'Decile 6', - 'Decile 10'] - - # Test with floating point decile boundaries - decile_boundaries = [x / 10 for x in range(11)] - assert classify_decile_newvalues(decile_boundaries, decile_labels, [0.35, 0.55, 0.75]) == ['Decile 4', 'Decile 6', - 'Decile 8'] diff --git a/utils/uvalue_estimates.py b/utils/uvalue_estimates.py deleted file mode 100644 index 8e2dbaa6..00000000 --- a/utils/uvalue_estimates.py +++ /dev/null @@ -1,39 +0,0 @@ -from typing import List - -from bisect import bisect_left - - -def classify_decile_newvalues( - decile_boundaries: List[float], decile_labels: List[str], new_values: List[float] -) -> List[str]: - """ - Classify a list of new values into pre-established deciles. - - This function is an alternative to UvalueEstimations.classify_decile_newvalues that does not depend on pandas, - making it suitable for use in environments where pandas may not be available (such as AWS Lambda). - - :param decile_boundaries: A list of decile boundaries. These define the ranges of the deciles. - :param decile_labels: A list of labels for the deciles. These are the classifications to be assigned to the values. - :param new_values: A list of new values to be classified into the deciles. - - :return: A list of classifications for the new values. Each classification corresponds to the decile in which - the respective new value falls. If a value falls outside the range of the deciles, its classification is - None. - """ - classifications = [] - - # For each new value... - for value in new_values: - # If the value is outside the range of the deciles, classify it as None - if value < decile_boundaries[0] or value > decile_boundaries[-1]: - classifications.append(None) - else: - # Use bisect_left to find the decile in which the value falls - i = bisect_left(decile_boundaries, value) - # If the value falls exactly on a decile boundary, classify it in the lower decile - if i: - i -= 1 - # Append the classification to the list of classifications - classifications.append(decile_labels[i]) - return classifications -