From a44912defac44ceb64c4305fb65e8ae473fab4d9 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 5 Dec 2023 09:29:06 +0000 Subject: [PATCH 01/10] set up test file data --- etl/testing_data/birmingham_pilot.py | 137 +++++++++++++++++++++------ 1 file changed, 109 insertions(+), 28 deletions(-) diff --git a/etl/testing_data/birmingham_pilot.py b/etl/testing_data/birmingham_pilot.py index ab39df7e..56f9cc37 100644 --- a/etl/testing_data/birmingham_pilot.py +++ b/etl/testing_data/birmingham_pilot.py @@ -22,49 +22,130 @@ def app(): # Birmingham has a Local Authority Code of E08000025 + # ~~~~~~~~~~~~~~~~~~~~ + # First example + # ~~~~~~~~~~~~~~~~~~~~ # Let's take an EPC D property example_1_reponse = epc_client.domestic.search( params={ "local-authority": "E08000025", "property-type": "house", - } + }, + size=1000 ) + example_1_reponse = example_1_reponse["rows"] + # Get a property with a cavity wall + example_1_reponse_filtered = [ + x for x in example_1_reponse if + "cavity wall, as built, no insulation (assumed)" in x["walls-description"].lower() + ] + example_1_reponse_filtered = [ + x for x in example_1_reponse_filtered if "pitched, no insulation (assumed)" in x["roof-description"].lower() + ] + print(example_1_reponse_filtered[0]["postcode"]) + # 21 Penshaw Grove + print(example_1_reponse_filtered[0]["address1"]) + # B13 9NL + print(example_1_reponse_filtered[0]["built-form"]) + # Mid-Terrace + print(example_1_reponse_filtered[0]["current-energy-rating"]) + # 'D' - g_data = epc_client.domestic.search(params={"energy-band": "g"}, size=n_g) - f_data = epc_client.domestic.search(params={"energy-band": "f"}, size=n_f) - e_data = epc_client.domestic.search(params={"energy-band": "e"}, size=n_e) - d_data = epc_client.domestic.search(params={"energy-band": "d"}, size=n_d) - c_data = epc_client.domestic.search(params={"energy-band": "c"}, size=n_c) - b_data = epc_client.domestic.search(params={"energy-band": "b"}, size=n_b) - a_data = epc_client.domestic.search(params={"energy-band": "a"}, size=n_a) + # ~~~~~~~~~~~~~~~~~~~~ + # Second example + # ~~~~~~~~~~~~~~~~~~~~ - # Combine the final data - final_data = ( - g_data["rows"] + f_data["rows"] + e_data["rows"] + d_data["rows"] + c_data["rows"] + b_data["rows"] - + a_data["rows"] + # Let's take an EPC E property + example_2_reponse = epc_client.domestic.search( + params={ + "local-authority": "E08000025", + "property-type": "house", + "energy-band": "e" + }, + size=1000 ) - - # TODO: We also take homes with just a specific type of wall - - final_data = [ - x for x in final_data if ("cavity wall" in x["walls-description"].lower()) or ( - "solid brick" in x["walls-description"].lower() - ) or ("average thermal transmittance" in x["walls-description"].lower()) + example_2_reponse = example_2_reponse["rows"] + # Get a solid wall example + example_2_reponse_filtered = [ + x for x in example_2_reponse if + "solid brick, as built, no insulation (assumed)" in x["walls-description"].lower() + ] + # With some existing loft insulation + example_2_reponse_filtered = [ + x for x in example_2_reponse_filtered if "pitched, 100 mm loft insulation" in x["roof-description"].lower() ] - # TODO: For the moment, don't use park homes - final_csv_data = pd.DataFrame( - [{"address": x["address"], "postcode": x["postcode"], "Notes": None} for x - in final_data if - x["property-type"] not in ["Park home"]] - ) + print(example_2_reponse_filtered[0]["postcode"]) + # B13 9BY + print(example_2_reponse_filtered[0]["address1"]) + # 2 Bloomfield Road + print(example_2_reponse_filtered[0]["built-form"]) + # Semi-Detached + print(example_2_reponse_filtered[0]["current-energy-rating"]) + # E - final_csv_data = pd.concat([starting_csv, final_csv_data]).reset_index(drop=True) + # ~~~~~~~~~~~~~~~~~~~~ + # Third example + # ~~~~~~~~~~~~~~~~~~~~ + example_3_reponse = epc_client.domestic.search( + params={ + "local-authority": "E08000025", + "property-type": "house", + "energy-band": "f" + }, + size=1000 + ) + example_3_reponse = example_3_reponse["rows"] + print(example_3_reponse[2]["walls-description"]) + print(example_3_reponse[2]["floor-description"]) + print(example_3_reponse[3]["roof-description"]) + print(example_3_reponse[3]["postcode"]) + # B32 3DG + print(example_3_reponse[3]["address1"]) + # 18 Parkside + print(example_3_reponse[3]["built-form"]) + # End-Terrace + + # ~~~~~~~~~~~~~~~~~~~~ + # Final example + # ~~~~~~~~~~~~~~~~~~~~ + # Let's take a flat that is a D + example_4_reponse = epc_client.domestic.search( + params={ + "local-authority": "E08000025", + "property-type": "flat", + "energy-band": "d" + }, + size=1000 + ) + example_4_reponse = example_4_reponse["rows"] + + example_4_reponse_filtered = [ + x for x in example_4_reponse if + "cavity wall, as built, no insulation (assumed)" in x["walls-description"].lower() + ] + print(example_4_reponse_filtered[0]["postcode"]) + # B14 6PU + print(example_4_reponse_filtered[0]["address1"]) + # Flat 3 + + print(example_4_reponse_filtered[0]["floor-description"]) + print(example_4_reponse_filtered[0]["property-type"]) + # Flat + + test_file = pd.DataFrame( + [ + {"address": "21 Penshaw Grove", "postcode": "B13 9NL", "Notes": None}, + {"address": "2 Bloomfield Road", "postcode": "B13 9BY", "Notes": None}, + {"address": "18 Parkside", "postcode": "B32 3DG", "Notes": None}, + {"address": "Flat 3", "postcode": "B14 6PU", "Notes": None}, + ] + ) # Store the data in s3 filename = f"{USER_ID}/{PORTFOLIO_ID}/test_inputs.csv" save_csv_to_s3( - dataframe=final_csv_data, + dataframe=test_file, bucket_name="retrofit-plan-inputs-dev", file_name=filename ) @@ -73,7 +154,7 @@ def app(): "portfolio_id": str(PORTFOLIO_ID), "housing_type": "Social", "goal": "Increase EPC", - "goal_value": "B", + "goal_value": "C", "trigger_file_path": filename } print(body) From 84bcb99ad994d8e3ac5c9e7b034af8f0a3b9b70a Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 5 Dec 2023 11:02:12 +0000 Subject: [PATCH 02/10] Added further placeholder valuation figures and added mvp valuation increase methodology --- .idea/Model.iml | 2 +- .idea/misc.xml | 2 +- backend/app/db/models/materials.py | 2 +- backend/ml_models/Valuation.py | 109 +++++++++++++++++++++++++++-- 4 files changed, 105 insertions(+), 10 deletions(-) diff --git a/.idea/Model.iml b/.idea/Model.iml index b0f9c00d..4413bb06 100644 --- a/.idea/Model.iml +++ b/.idea/Model.iml @@ -7,7 +7,7 @@ - + diff --git a/.idea/misc.xml b/.idea/misc.xml index 1122b380..6f308057 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -3,7 +3,7 @@ - + diff --git a/backend/app/db/models/materials.py b/backend/app/db/models/materials.py index 1a41f14f..f887fc25 100644 --- a/backend/app/db/models/materials.py +++ b/backend/app/db/models/materials.py @@ -35,7 +35,7 @@ class MaterialType(enum.Enum): low_energy_lighting_installation = "low_energy_lighting_installation" flat_roof_preparation = "flat_roof_preparation" flat_roof_vapour_barrier = "flat_roof_vapour_barrier" - flat_roof_waterpoofing = "flat_roof_waterpoofing" + flat_roof_waterproofing = "flat_roof_waterproofing" class DepthUnit(enum.Enum): diff --git a/backend/ml_models/Valuation.py b/backend/ml_models/Valuation.py index ad296409..6888e45e 100644 --- a/backend/ml_models/Valuation.py +++ b/backend/ml_models/Valuation.py @@ -1,22 +1,117 @@ +import numpy as np + + class PropertyValuation: """ This is a placeholder class for the property valuation model """ UPRN_VALUE_LOOKUP = { - 15038202: {"current_value": 202000, "increase_percentage": 0.05725}, - 37024763: {"current_value": 213000, "increase_percentage": 0.025}, + 15038202: 202000, + 37024763: 213000, + 100070478545: 212000, + 100070297696: 235000, + 100070476394: 222000, # Based on Zoopla's estimation of next door, 20 Parkside + 100071264896: 128000, + # Based on next door neighbour: https://themovemarket.com/tools/propertyprices/flat-2-queens-wood-house-219 + # -brandwood-road-birmingham-b14-6pu } + # We base our valuation uplifts on a number of sources + # https://www.moneysupermarket.com/gas-and-electricity/value-of-efficiency/ + MSM_MAPPING = [ + {"start": "G", "end": "F", "increase_percentage": 0.06}, + {"start": "F", "end": "E", "increase_percentage": 0.01}, + {"start": "E", "end": "D", "increase_percentage": 0.01}, + {"start": "D", "end": "C", "increase_percentage": 0.02}, + {"start": "C", "end": "B", "increase_percentage": 0.04}, + {"start": "B", "end": "A", "increase_percentage": 0.0}, + ] + + # https://www.lloydsbankinggroup.com/media/press-releases/2021/halifax/homebuyers-pay-a-green-premium-of-40000 + # -for-the-most-energy-efficient-properties.html + LLOYDS_MAPPING = [ + {"start": "G", "end": "F", "increase_percentage": 0.038}, + {"start": "F", "end": "E", "increase_percentage": 0.029}, + {"start": "E", "end": "D", "increase_percentage": 0.024}, + {"start": "D", "end": "C", "increase_percentage": 0.02}, + {"start": "C", "end": "B", "increase_percentage": 0.02}, + {"start": "B", "end": "A", "increase_percentage": 0.018}, + ] + + KNIGHT_FRANK_MAPPING = [ + {"start": "D", "end": "C", "increase_percentage": 0.03}, + {"start": "D", "end": "B", "increase_percentage": 0.088}, + ] + + NATIONWIDE_MAPPING = [ + {"start": "G", "end": "D", "increase_percentage": 0.035}, + {"start": "F", "end": "D", "increase_percentage": 0.035}, + {"start": "D", "end": "B", "increase_percentage": 0.017}, + {"start": "D", "end": "A", "increase_percentage": 0.017}, + ] + + EPC_BANDS = ["G", "F", "E", "D", "C", "B", "A"] + + @classmethod + def get_increase(cls, epc_band_range): + + increases = [] + for i in range(len(epc_band_range)): + + if i == len(epc_band_range) - 1: + break + + current = epc_band_range[i] + next = epc_band_range[i + 1] + + msm_increase = [x for x in cls.MSM_MAPPING if x["start"] == current and x["end"] == next][0] + lloyds_increase = [x for x in cls.LLOYDS_MAPPING if x["start"] == current and x["end"] == next][0] + + increases.append( + { + "start": current, + "end": next, + "msm_increase": msm_increase["increase_percentage"], + "lloyds_increase": lloyds_increase["increase_percentage"], + } + ) + + # We now aggregate the increases. The should be compound increases so we multiply them together + msm_increase = np.prod([1 + x["msm_increase"] for x in increases]) - 1 + lloyds_increase = np.prod([1 + x["lloyds_increase"] for x in increases]) - 1 + + return msm_increase, lloyds_increase + @classmethod def estimate(cls, property_instance, target_epc): - data = cls.UPRN_VALUE_LOOKUP.get(property_instance.uprn) + value = cls.UPRN_VALUE_LOOKUP.get(property_instance.uprn) - if not data: + if not value: raise ValueError("Have not implemented valuation for this property") - new_valuation = (1 + data["increase_percentage"]) * data["current_value"] + current_epc = property_instance.data["current-energy-rating"] + # We get the spectrum of ratings between the current and target EPC + epc_band_range = cls.EPC_BANDS[cls.EPC_BANDS.index(current_epc): cls.EPC_BANDS.index(target_epc) + 1] - increase = round(new_valuation - data["current_value"], 2) + msm_increase, lloyds_increase = cls.get_increase(epc_band_range) - return increase + # We now use the knight frank and nationwide data to get further valuation evidence, if we have it + kf_increase = [x for x in cls.KNIGHT_FRANK_MAPPING if x["start"] == current_epc and x["end"] == target_epc] + nw_increase = [x for x in cls.NATIONWIDE_MAPPING if x["start"] == current_epc and x["end"] == target_epc] + + kf_increase = kf_increase[0]["increase_percentage"] if kf_increase else None + nw_increase = nw_increase[0]["increase_percentage"] if nw_increase else None + + all_increases = [x for x in [msm_increase, lloyds_increase, kf_increase, nw_increase] if x is not None] + + max_increase = max(all_increases) + min_increase = min(all_increases) + avg_increase = np.mean(all_increases) + + return { + "current_value": value, + "lower_bound_increased_value": value * (1 + min_increase), + "upper_bound_increased_value": value * (1 + max_increase), + "average_increased_value": value * (1 + avg_increase), + } From 195578a1aa0d95b02ec7de428b572305da087718 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 5 Dec 2023 11:05:26 +0000 Subject: [PATCH 03/10] integrated new valuation into backend router --- backend/app/plan/router.py | 66 ++++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 32 deletions(-) diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 42014bb3..a35f36ec 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -492,38 +492,38 @@ async def trigger_plan(body: PlanTriggerRequest): for p in batch_properties: # Your existing operations - property_details_epc = p.get_property_details_epc( - portfolio_id=body.portfolio_id, rating_lookup=rating_lookup - ) - create_property_details_epc(session, property_details_epc) - - update_or_create_property_spatial_details(session, p.uprn, p.spatial) - - # TODO: TEMP - if p.data["uprn"] == "": - print("Get rid of me!") - p.data["uprn"] = 0 - - property_data = p.get_full_property_data() - update_property_data( - session, property_id=p.id, portfolio_id=body.portfolio_id, property_data=property_data - ) - + # property_details_epc = p.get_property_details_epc( + # portfolio_id=body.portfolio_id, rating_lookup=rating_lookup + # ) + # create_property_details_epc(session, property_details_epc) + # + # update_or_create_property_spatial_details(session, p.uprn, p.spatial) + # + # # TODO: TEMP + # if p.data["uprn"] == "": + # print("Get rid of me!") + # p.data["uprn"] = 0 + # + # property_data = p.get_full_property_data() + # update_property_data( + # session, property_id=p.id, portfolio_id=body.portfolio_id, property_data=property_data + # ) + # recommendations_to_upload = recommendations.get(p.id, []) - if not recommendations_to_upload: - continue - - new_plan_id = create_plan(session, { - "portfolio_id": body.portfolio_id, - "property_id": p.id, - "is_default": True - }) - - uploaded_recommendation_ids = upload_recommendations(session, recommendations_to_upload, p.id) - - create_plan_recommendations( - session, plan_id=new_plan_id, recommendation_ids=uploaded_recommendation_ids - ) + # if not recommendations_to_upload: + # continue + # + # new_plan_id = create_plan(session, { + # "portfolio_id": body.portfolio_id, + # "property_id": p.id, + # "is_default": True + # }) + # + # uploaded_recommendation_ids = upload_recommendations(session, recommendations_to_upload, p.id) + # + # create_plan_recommendations( + # session, plan_id=new_plan_id, recommendation_ids=uploaded_recommendation_ids + # ) # Get defaults default_recommendations = [r for r in recommendations_to_upload if r["default"]] @@ -531,8 +531,10 @@ async def trigger_plan(body: PlanTriggerRequest): new_sap_points = float(p.data["current-energy-efficiency"]) + total_sap_points new_epc = sap_to_epc(new_sap_points) + valuations = PropertyValuation.estimate(property_instance=p, target_epc=new_epc) + property_valuation_increases.append( - PropertyValuation.estimate(property_instance=p, target_epc=new_epc) + valuations["average_increased_value"] - valuations["current_value"] ) # Commit the session after each batch From a45164e29eca0434f3dbc00f84eae7196e448c8e Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 5 Dec 2023 12:12:12 +0000 Subject: [PATCH 04/10] Changed properties to social housing homes for birmingham pilot --- backend/app/plan/router.py | 62 ++++++++++++++-------------- backend/ml_models/Valuation.py | 2 +- etl/testing_data/birmingham_pilot.py | 57 ++++++++++++++++--------- 3 files changed, 70 insertions(+), 51 deletions(-) diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index a35f36ec..adc9eac1 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -492,38 +492,38 @@ async def trigger_plan(body: PlanTriggerRequest): for p in batch_properties: # Your existing operations - # property_details_epc = p.get_property_details_epc( - # portfolio_id=body.portfolio_id, rating_lookup=rating_lookup - # ) - # create_property_details_epc(session, property_details_epc) - # - # update_or_create_property_spatial_details(session, p.uprn, p.spatial) - # - # # TODO: TEMP - # if p.data["uprn"] == "": - # print("Get rid of me!") - # p.data["uprn"] = 0 - # - # property_data = p.get_full_property_data() - # update_property_data( - # session, property_id=p.id, portfolio_id=body.portfolio_id, property_data=property_data - # ) - # + property_details_epc = p.get_property_details_epc( + portfolio_id=body.portfolio_id, rating_lookup=rating_lookup + ) + create_property_details_epc(session, property_details_epc) + + update_or_create_property_spatial_details(session, p.uprn, p.spatial) + + # TODO: TEMP + if p.data["uprn"] == "": + print("Get rid of me!") + p.data["uprn"] = 0 + + property_data = p.get_full_property_data() + update_property_data( + session, property_id=p.id, portfolio_id=body.portfolio_id, property_data=property_data + ) + recommendations_to_upload = recommendations.get(p.id, []) - # if not recommendations_to_upload: - # continue - # - # new_plan_id = create_plan(session, { - # "portfolio_id": body.portfolio_id, - # "property_id": p.id, - # "is_default": True - # }) - # - # uploaded_recommendation_ids = upload_recommendations(session, recommendations_to_upload, p.id) - # - # create_plan_recommendations( - # session, plan_id=new_plan_id, recommendation_ids=uploaded_recommendation_ids - # ) + if not recommendations_to_upload: + continue + + new_plan_id = create_plan(session, { + "portfolio_id": body.portfolio_id, + "property_id": p.id, + "is_default": True + }) + + uploaded_recommendation_ids = upload_recommendations(session, recommendations_to_upload, p.id) + + create_plan_recommendations( + session, plan_id=new_plan_id, recommendation_ids=uploaded_recommendation_ids + ) # Get defaults default_recommendations = [r for r in recommendations_to_upload if r["default"]] diff --git a/backend/ml_models/Valuation.py b/backend/ml_models/Valuation.py index 6888e45e..51925b22 100644 --- a/backend/ml_models/Valuation.py +++ b/backend/ml_models/Valuation.py @@ -10,7 +10,7 @@ class PropertyValuation: 15038202: 202000, 37024763: 213000, 100070478545: 212000, - 100070297696: 235000, + 100070297696: 662000, # Based on Zoopla's estimation of nearby house, 8 bloomfield road 100070476394: 222000, # Based on Zoopla's estimation of next door, 20 Parkside 100071264896: 128000, # Based on next door neighbour: https://themovemarket.com/tools/propertyprices/flat-2-queens-wood-house-219 diff --git a/etl/testing_data/birmingham_pilot.py b/etl/testing_data/birmingham_pilot.py index 56f9cc37..a049e35e 100644 --- a/etl/testing_data/birmingham_pilot.py +++ b/etl/testing_data/birmingham_pilot.py @@ -42,10 +42,15 @@ def app(): example_1_reponse_filtered = [ x for x in example_1_reponse_filtered if "pitched, no insulation (assumed)" in x["roof-description"].lower() ] + # Get a social housing property + example_1_reponse_filtered = [ + x for x in example_1_reponse_filtered if x["tenure"] == "Rented (social)" + ] + print(example_1_reponse_filtered[0]["postcode"]) - # 21 Penshaw Grove + # B13 9LT print(example_1_reponse_filtered[0]["address1"]) - # B13 9NL + # 113 Tenby Road print(example_1_reponse_filtered[0]["built-form"]) # Mid-Terrace print(example_1_reponse_filtered[0]["current-energy-rating"]) @@ -74,11 +79,15 @@ def app(): example_2_reponse_filtered = [ x for x in example_2_reponse_filtered if "pitched, 100 mm loft insulation" in x["roof-description"].lower() ] + # Get a social housing property + example_2_reponse_filtered = [ + x for x in example_2_reponse_filtered if x["tenure"] == "Rented (social)" + ] print(example_2_reponse_filtered[0]["postcode"]) - # B13 9BY + # B28 8JF print(example_2_reponse_filtered[0]["address1"]) - # 2 Bloomfield Road + # 139 School Road print(example_2_reponse_filtered[0]["built-form"]) # Semi-Detached print(example_2_reponse_filtered[0]["current-energy-rating"]) @@ -96,15 +105,20 @@ def app(): size=1000 ) example_3_reponse = example_3_reponse["rows"] - print(example_3_reponse[2]["walls-description"]) - print(example_3_reponse[2]["floor-description"]) - print(example_3_reponse[3]["roof-description"]) - print(example_3_reponse[3]["postcode"]) - # B32 3DG - print(example_3_reponse[3]["address1"]) - # 18 Parkside - print(example_3_reponse[3]["built-form"]) - # End-Terrace + # Get a social housing property] + example_3_reponse_filtered = [ + x for x in example_3_reponse if x["tenure"] == "Rented (social)" + ] + + print(example_3_reponse_filtered[4]["walls-description"]) + print(example_3_reponse_filtered[4]["floor-description"]) + print(example_3_reponse_filtered[4]["roof-description"]) + print(example_3_reponse_filtered[4]["postcode"]) + # B32 1SL + print(example_3_reponse_filtered[4]["address1"]) + # 77 Simmons Drive + print(example_3_reponse_filtered[4]["built-form"]) + # Semi-Detached # ~~~~~~~~~~~~~~~~~~~~ # Final example @@ -124,10 +138,14 @@ def app(): x for x in example_4_reponse if "cavity wall, as built, no insulation (assumed)" in x["walls-description"].lower() ] + # Get a social housing property + example_4_reponse_filtered = [ + x for x in example_4_reponse_filtered if x["tenure"] == "Rented (social)" + ] print(example_4_reponse_filtered[0]["postcode"]) - # B14 6PU + # B32 1LS print(example_4_reponse_filtered[0]["address1"]) - # Flat 3 + # Flat 2 print(example_4_reponse_filtered[0]["floor-description"]) print(example_4_reponse_filtered[0]["property-type"]) @@ -135,10 +153,11 @@ def app(): test_file = pd.DataFrame( [ - {"address": "21 Penshaw Grove", "postcode": "B13 9NL", "Notes": None}, - {"address": "2 Bloomfield Road", "postcode": "B13 9BY", "Notes": None}, - {"address": "18 Parkside", "postcode": "B32 3DG", "Notes": None}, - {"address": "Flat 3", "postcode": "B14 6PU", "Notes": None}, + # New properties + {"address": "113 Tenby Road", "postcode": "B13 9LT", "Notes": None}, + {"address": "139 School Road", "postcode": "B28 8JF", "Notes": None}, + {"address": "77 Simmons Drive", "postcode": "B32 1SL", "Notes": None}, + {"address": "Flat 2, 54 Wedgewood Road", "postcode": "B32 1LS", "Notes": None}, ] ) From db7eb69dd10621843f8ae3b693005882432c33d3 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 5 Dec 2023 13:30:45 +0000 Subject: [PATCH 05/10] handle bug where neigther ewi or iwi are selected as options via optimisation --- backend/app/plan/router.py | 11 +++++++++++ backend/ml_models/Valuation.py | 4 ++++ 2 files changed, 15 insertions(+) diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index adc9eac1..6954d89f 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -298,6 +298,17 @@ async def trigger_plan(body: PlanTriggerRequest): t for t in missing_types if t not in ["internal_wall_insulation", "external_wall_insulation"] ] + # We check if NO wall insulation was selected but iwi and ewi are available + # This condition will check + # 1) iwi and ewi are both in missing_types + # 2) iwi and ewi are not in default_types + # If both of these are true, it means that no wall insulation was selected via the optimisation routine + # but both are possible, so we need to select a default. We default to iwi because it's usually cheaper + if (("internal_wall_insulation" in missing_types) and ("external_wall_insulation" in missing_types)) and ( + ("internal_wall_insulation" not in default_types) and ("external_wall_insulation" not in default_types) + ): + missing_types = [t for t in missing_types if t != "external_wall_insulation"] + if missing_types: for missed_type in missing_types: missed = [r for r in property_recommendations if r["type"] == missed_type] diff --git a/backend/ml_models/Valuation.py b/backend/ml_models/Valuation.py index 51925b22..9e409b9f 100644 --- a/backend/ml_models/Valuation.py +++ b/backend/ml_models/Valuation.py @@ -15,6 +15,10 @@ class PropertyValuation: 100071264896: 128000, # Based on next door neighbour: https://themovemarket.com/tools/propertyprices/flat-2-queens-wood-house-219 # -brandwood-road-birmingham-b14-6pu + 100070533688: 218000, # Based on Zoopla's estimation of 95 Tenby Road, which is also end terrace + 100070505235: 344000, # Based on Zoopla's estimation of 131 School road, which is also semi-detached + 100070513306: 182000, # Based on Zoopla's estimation of 61 Simmons Drive + 100071306896: 77000, # Based on Flat 2 of 44 Wedgewood Road on Zoopla } # We base our valuation uplifts on a number of sources From bf6020026a6d2231232b7f4e2e633939bdfcd184 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 5 Dec 2023 14:51:20 +0000 Subject: [PATCH 06/10] modified energy estimate adjustment to use current epc rating rather than expected --- backend/app/plan/router.py | 4 +--- backend/ml_models/AnnualBillSavings.py | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 6954d89f..b5acb3c0 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -415,12 +415,10 @@ async def trigger_plan(body: PlanTriggerRequest): # We sum up the SAP points of the default recommendations and calculate a new EPC category. This # category is then used to produce adjusted energy figures - total_sap_points = sum([x["sap_points"] for x in representative_recs[property_id]]) - expected_epc = sap_to_epc(float(property_instance.data["current-energy-efficiency"]) + total_sap_points) expected_adjusted_energy = AnnualBillSavings.adjust_energy_to_metered( epc_energy_consumption=expected_heat_demand, - current_epc_rating=expected_epc, + current_epc_rating=property_instance.data["current-energy-rating"], ) heat_demand_change = ( diff --git a/backend/ml_models/AnnualBillSavings.py b/backend/ml_models/AnnualBillSavings.py index 1519a866..f3e5074a 100644 --- a/backend/ml_models/AnnualBillSavings.py +++ b/backend/ml_models/AnnualBillSavings.py @@ -18,6 +18,8 @@ class AnnualBillSavings: # This is a weighted mean of the price caps, using the consumption figures above as weights PRICE_FACTOR = 0.11183098591549295 + EPC_BANDS = ["G", "F", "E", "D", "C", "B", "A"] + @classmethod def estimate(cls, kwh: float): """ @@ -70,3 +72,22 @@ class AnnualBillSavings: adjusted_consumption = (epc_energy_consumption + consumption_difference) return adjusted_consumption + + @classmethod + def adjust_expected_band(cls, expected_epc_rating, current_epc_rating): + """ + Because of the differing intercepts and intercepts when adjusting, it's possible for + expected_adjusted_energy to be bigger than current_adjusted_energy. In this case, we'll + adjust, against at most 1 EPC band above the curent. This function performs the EPC adjustment + :param expected_epc_rating: The expected EPC rating + :param current_epc_rating: The current EPC rating + """ + + # Find index of expected EPC rating + expected_index = cls.EPC_BANDS.index(expected_epc_rating) + current_index = cls.EPC_BANDS.index(current_epc_rating) + + if expected_index - 1 < current_index: + return current_epc_rating + + return cls.EPC_BANDS[expected_index - 1] From afeac56f5e944249cc16435795b12e274de4cd74 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 5 Dec 2023 15:04:08 +0000 Subject: [PATCH 07/10] fixed bug with conservation area and added test --- backend/Property.py | 3 ++- backend/tests/test_property.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/backend/Property.py b/backend/Property.py index c04a3ed9..79bd1659 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -369,7 +369,8 @@ class Property(Definitions): self.is_listed = spatial["is_listed_building"].values[0] self.is_heritage = spatial["is_heritage_building"].values[0] - if self.in_conservation_area is True | self.is_listed is True | self.is_heritage is True: + # We do an equals True, in the case of one of these variables being True + if (self.in_conservation_area == True) | (self.is_listed == True) | (self.is_heritage == True): self.restricted_measures = True spatial_dict = spatial.to_dict("records")[0] diff --git a/backend/tests/test_property.py b/backend/tests/test_property.py index d8519b6b..39a7e86e 100644 --- a/backend/tests/test_property.py +++ b/backend/tests/test_property.py @@ -1,3 +1,4 @@ +import pandas as pd import pytest from unittest.mock import Mock from epc_api.client import EpcClient @@ -345,3 +346,32 @@ class TestProperty: # Verify that ValueError is raised when multiple attributes are found with pytest.raises(ValueError, match="Either No attributes or multiple found for roof-description"): property_instance.get_components(cleaned) + + def test_set_spatial(self, mock_epc_client): + prop = Property(1, "AB12CD", "Test Address", mock_epc_client) + + spatial1 = pd.DataFrame([{ + 'X_COORDINATE': 411143.0, 'Y_COORDINATE': 281701.0, 'LATITUDE': 52.4331896, 'LONGITUDE': -1.8375238, + 'conservation_status': True, 'is_listed_building': False, 'is_heritage_building': True + }]) + + prop.set_spatial(spatial1) + + assert prop.in_conservation_area + assert not prop.is_listed + assert prop.is_heritage + assert prop.restricted_measures + + prop2 = Property(1, "AB12CD", "Test Address", mock_epc_client) + + spatial2 = pd.DataFrame([{ + 'X_COORDINATE': 411143.0, 'Y_COORDINATE': 281701.0, 'LATITUDE': 52.4331896, 'LONGITUDE': -1.8375238, + 'conservation_status': None, 'is_listed_building': False, 'is_heritage_building': False + }]) + + prop2.set_spatial(spatial2) + + assert prop2.in_conservation_area is None + assert not prop2.is_listed + assert not prop2.is_heritage + assert not prop2.restricted_measures From 32e0d78cd806437a5b9d44da56e897935e6c96e8 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 5 Dec 2023 15:19:58 +0000 Subject: [PATCH 08/10] updated unit tests for slight increase in contingency costs for suspended floor insulation --- .idea/Model.iml | 2 +- .idea/misc.xml | 2 +- recommendations/Costs.py | 4 +++- recommendations/tests/test_costs.py | 2 +- recommendations/tests/test_floor_recommendations.py | 2 +- 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.idea/Model.iml b/.idea/Model.iml index 4413bb06..b0f9c00d 100644 --- a/.idea/Model.iml +++ b/.idea/Model.iml @@ -7,7 +7,7 @@ - + diff --git a/.idea/misc.xml b/.idea/misc.xml index 6f308057..1122b380 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -3,7 +3,7 @@ - + diff --git a/recommendations/Costs.py b/recommendations/Costs.py index a57092f9..0d9031b2 100644 --- a/recommendations/Costs.py +++ b/recommendations/Costs.py @@ -315,7 +315,9 @@ class Costs: subtotal_before_profit = labour_costs + materials_costs - contingency_cost = subtotal_before_profit * self.CONTINGENCY + # Because of the possiblity of damage to the existing floor, or difficulties associated to moving fittings, + # we use a higher contingency rate + contingency_cost = subtotal_before_profit * self.HIGH_RISK_CONTINGENCY preliminaries_cost = subtotal_before_profit * self.PRELIMINARIES profit_cost = subtotal_before_profit * self.PROFIT_MARGIN diff --git a/recommendations/tests/test_costs.py b/recommendations/tests/test_costs.py index 1d66ff47..1d519b91 100644 --- a/recommendations/tests/test_costs.py +++ b/recommendations/tests/test_costs.py @@ -240,7 +240,7 @@ class TestCosts: ) assert sus_floor_results == { - 'total': 3114.6027360000003, 'subtotal': 2595.50228, 'vat': 519.100456, 'contingency': 185.39302, + 'total': 3337.07436, 'subtotal': 2780.8953, 'vat': 556.17906, 'contingency': 370.78604, 'preliminaries': 185.39302, 'material': 483.405, 'profit': 370.78604, 'labour_hours': 54.940000000000005, 'labour_days': 2.289166666666667, 'labour_cost': 1370.5252 } diff --git a/recommendations/tests/test_floor_recommendations.py b/recommendations/tests/test_floor_recommendations.py index 01bd308e..43e98d60 100644 --- a/recommendations/tests/test_floor_recommendations.py +++ b/recommendations/tests/test_floor_recommendations.py @@ -81,7 +81,7 @@ class TestFloorRecommendations: assert types == {"suspended_floor_insulation"} assert len(recommender.recommendations) == 6 - assert recommender.recommendations[0]["total"] == 4596.858 + assert recommender.recommendations[0]["total"] == 4925.205 assert recommender.recommendations[0]["new_u_value"] == 0.21 def test_uvalue_0_12(self, input_properties): From 2a601a3003f7b2d7fa676066ba4f5d986c1033dc Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 5 Dec 2023 16:04:52 +0000 Subject: [PATCH 09/10] Created set_floor_level function and added tests for it --- .idea/Model.iml | 2 +- .idea/misc.xml | 2 +- backend/Property.py | 25 ++++++++++++- backend/tests/test_property.py | 48 +++++++++++++++++++++++++ recommendations/FloorRecommendations.py | 9 ++--- 5 files changed, 77 insertions(+), 9 deletions(-) diff --git a/.idea/Model.iml b/.idea/Model.iml index b0f9c00d..4413bb06 100644 --- a/.idea/Model.iml +++ b/.idea/Model.iml @@ -7,7 +7,7 @@ - + diff --git a/.idea/misc.xml b/.idea/misc.xml index 1122b380..6f308057 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -3,7 +3,7 @@ - + diff --git a/backend/Property.py b/backend/Property.py index 79bd1659..4c8bd5bc 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -10,7 +10,7 @@ from utils.logger import setup_logger from utils.s3 import read_dataframe_from_s3_parquet from epc_api.client import EpcClient from BaseUtility import Definitions -from recommendations.rdsap_tables import england_wales_age_band_lookup +from recommendations.rdsap_tables import england_wales_age_band_lookup, FLOOR_LEVEL_MAP from recommendations.recommendation_utils import ( estimate_perimeter, get_wall_type, estimate_external_wall_area, esimtate_pitched_roof_area ) @@ -84,6 +84,7 @@ class Property(Definitions): self.pitched_roof_area = None self.insulation_floor_area = None self.number_lighting_outlets = None + self.floor_level = None self.current_adjusted_energy = None self.expected_adjusted_energy = None @@ -324,6 +325,7 @@ class Property(Definitions): self.set_wall_type() self.set_floor_type() + self.set_floor_level() def set_age_band(self): """ @@ -642,6 +644,27 @@ class Property(Definitions): floor_area=self.insulation_floor_area, floor_height=self.floor_height ) + def set_floor_level(self): + self.floor_level = ( + FLOOR_LEVEL_MAP[self.data["floor-level"]] if + self.data["floor-level"] not in self.DATA_ANOMALY_MATCHES else None + ) + + # We perform some extra checks, if the property is not on the ground floor, as we have found cases + # where a property is marked as being on the first floor + if self.floor_level > 0: + + # We check if there is another property below + if not self.floor["another_property_below"]: + self.floor_level = 0 + return + + if self.floor_level == 0: + # Check if another property below + if self.floor["another_property_below"]: + self.floor_level = 1 + return + def set_wall_type(self): """ This method sets the wall type of the property, using a simple approach based on the wall description diff --git a/backend/tests/test_property.py b/backend/tests/test_property.py index 39a7e86e..9188f545 100644 --- a/backend/tests/test_property.py +++ b/backend/tests/test_property.py @@ -375,3 +375,51 @@ class TestProperty: assert not prop2.is_listed assert not prop2.is_heritage assert not prop2.restricted_measures + + def test_set_floor_level(self, mock_epc_client): + # In this case, we have a flat which looks looks it's on the first floor, but it's actually on the ground + # floor, so we should set floor_level to 0 + prop = Property(1, "AB12CD", "Test Address", mock_epc_client) + prop.data = {'floor-level': '01', 'property-type': 'Flat'} + prop.floor = { + 'original_description': 'Solid, no insulation (assumed)', 'clean_description': 'Solid, no insulation', + 'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_assumed': True, + 'is_to_unheated_space': False, 'is_to_external_air': False, 'is_suspended': False, 'is_solid': True, + 'another_property_below': False, 'insulation_thickness': 'none', 'floor_thermal_transmittance': None, + 'floor_insulation_thickness': 'none' + } + + prop.set_floor_level() + + assert prop.floor_level == 0 + + # This property is labelled as being on the ground floor but actually has another property below + # so we set floor level to 1 + prop2 = Property(1, "AB12CD", "Test Address", mock_epc_client) + prop2.data = {'floor-level': 'Ground', 'property-type': 'Flat'} + prop2.floor = { + 'original_description': '(Another dwelling below)', 'clean_description': 'Solid, no insulation', + 'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_assumed': False, + 'is_to_unheated_space': False, 'is_to_external_air': False, 'is_suspended': False, 'is_solid': False, + 'another_property_below': True, 'insulation_thickness': 'none', 'floor_thermal_transmittance': None, + 'floor_insulation_thickness': 'none' + } + + prop2.set_floor_level() + + assert prop2.floor_level == 1 + + # this property is correctly labelled as being on the 2nd floor + prop3 = Property(1, "AB12CD", "Test Address", mock_epc_client) + prop3.data = {'floor-level': '02', 'property-type': 'Flat'} + prop3.floor = { + 'original_description': '(Another dwelling below)', 'clean_description': 'Solid, no insulation', + 'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_assumed': False, + 'is_to_unheated_space': False, 'is_to_external_air': False, 'is_suspended': False, 'is_solid': False, + 'another_property_below': True, 'insulation_thickness': 'none', 'floor_thermal_transmittance': None, + 'floor_insulation_thickness': 'none' + } + + prop3.set_floor_level() + + assert prop3.floor_level == 2 diff --git a/recommendations/FloorRecommendations.py b/recommendations/FloorRecommendations.py index 48245554..a246c8cb 100644 --- a/recommendations/FloorRecommendations.py +++ b/recommendations/FloorRecommendations.py @@ -10,7 +10,6 @@ from recommendations.recommendation_utils import ( r_value_per_mm_to_u_value, calculate_u_value_uplift, is_diminishing_returns, update_lowest_selected_u_value, get_recommended_part, get_floor_u_value ) -from recommendations.rdsap_tables import FLOOR_LEVEL_MAP from recommendations.Costs import Costs @@ -73,10 +72,6 @@ class FloorRecommendations(Definitions): def recommend(self): u_value = self.property.floor["thermal_transmittance"] - floor_level = ( - FLOOR_LEVEL_MAP[self.property.data["floor-level"]] if - self.property.data["floor-level"] not in self.DATA_ANOMALY_MATCHES else None - ) property_type = self.property.data["property-type"] floor_area = self.property.insulation_floor_area @@ -90,7 +85,9 @@ class FloorRecommendations(Definitions): return # If the property is a flat that isn't at ground level, it's likely impractical to recommend a floor upgrade - if (floor_level != 0) and (property_type == "Flat"): + if (self.property.floor_level != 0) and (property_type == "Flat") and ( + self.property.floor["another_property_below"] + ): return if u_value: From e190b9858684003a4b5ebec955a02db9bb2bb232 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 5 Dec 2023 16:09:56 +0000 Subject: [PATCH 10/10] Added test covering house for set_floor_level --- backend/Property.py | 11 +++++++++++ backend/tests/test_property.py | 15 +++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/backend/Property.py b/backend/Property.py index 4c8bd5bc..d400d439 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -650,6 +650,17 @@ class Property(Definitions): self.data["floor-level"] not in self.DATA_ANOMALY_MATCHES else None ) + if self.floor_level is None: + + if self.data["property-type"] != "Flat": + return + + if self.floor["another_property_below"]: + self.floor_level = 1 + else: + self.floor_level = 0 + return + # We perform some extra checks, if the property is not on the ground floor, as we have found cases # where a property is marked as being on the first floor if self.floor_level > 0: diff --git a/backend/tests/test_property.py b/backend/tests/test_property.py index 9188f545..871c9291 100644 --- a/backend/tests/test_property.py +++ b/backend/tests/test_property.py @@ -423,3 +423,18 @@ class TestProperty: prop3.set_floor_level() assert prop3.floor_level == 2 + + # Example of a house + prop4 = Property(1, "AB12CD", "Test Address", mock_epc_client) + prop4.data = {'floor-level': '', 'property-type': 'House'} + prop4.floor = { + 'original_description': '(Another dwelling below)', 'clean_description': 'Solid, no insulation', + 'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_assumed': False, + 'is_to_unheated_space': False, 'is_to_external_air': False, 'is_suspended': False, 'is_solid': False, + 'another_property_below': False, 'insulation_thickness': 'none', 'floor_thermal_transmittance': None, + 'floor_insulation_thickness': 'none' + } + + prop4.set_floor_level() + + assert prop4.floor_level is None