From ddfbf33494f6741b974217fffc5bb4ba784560a0 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 26 Feb 2025 11:00:12 +0000 Subject: [PATCH] westward complete --- asset_list/AssetList.py | 95 ++++++++++++++----------- asset_list/app.py | 42 +++++++---- asset_list/mappings/walls.py | 2 +- etl/customers/remote_assessments/app.py | 14 ++-- recommendations/HeatingRecommender.py | 2 +- 5 files changed, 94 insertions(+), 61 deletions(-) diff --git a/asset_list/AssetList.py b/asset_list/AssetList.py index 76f2b145..31b11c66 100644 --- a/asset_list/AssetList.py +++ b/asset_list/AssetList.py @@ -286,7 +286,7 @@ class AssetList: # This SAP threshold is a key search criteria for properties that may be eligible for extraction FILLED_CAVITY_SAP_THRESHOLD = 75 # This SAP the - EMPTY_CAVITY_SAP_THRESHOLD = 71 + EMPTY_CAVITY_SAP_THRESHOLD = 75 # Any EPC deemed to have been conducted prior to this year is deemed to be unreliable EPC_YEAR_THRESHOLD = pd.Timestamp.now().year - 5 @@ -956,13 +956,28 @@ class AssetList: (~self.standardised_asset_list[self.STANDARD_PROPERTY_TYPE].isin(["bedsit"])) & (self.standardised_asset_list['non-intrusives: Construction'] == "CAVITY") & self.standardised_asset_list['non-intrusives: Insulated'].isin(["EMPTY", "PARTIAL"]) & - (self.standardised_asset_list[self.STANDARD_YEAR_BUILT] <= 2000) & + (self.standardised_asset_list[self.STANDARD_YEAR_BUILT] <= 2002) & ( self.standardised_asset_list[ self.EPC_API_DATA_NAMES["current-energy-efficiency"] ] <= self.EMPTY_CAVITY_SAP_THRESHOLD ) ) + # Let's also flag work that looks eligible without the SAP filter + self.standardised_asset_list["non_intrusive_indicates_empty_cavity_no_sap_filter"] = ( + (~self.standardised_asset_list[self.STANDARD_PROPERTY_TYPE].isin(["bedsit"])) & + (self.standardised_asset_list['non-intrusives: Construction'] == "CAVITY") & + self.standardised_asset_list['non-intrusives: Insulated'].isin(["EMPTY", "PARTIAL"]) & + (self.standardised_asset_list[self.STANDARD_YEAR_BUILT] <= 2002) + ) + + # If non_intrusive_indicates_empty_cavity is True, + # set non_intrusive_indicates_empty_cavity_no_sap_filter to False + self.standardised_asset_list["non_intrusive_indicates_empty_cavity_no_sap_filter"] = np.where( + self.standardised_asset_list["non_intrusive_indicates_empty_cavity"], + False, + self.standardised_asset_list["non_intrusive_indicates_empty_cavity_no_sap_filter"] + ) self.standardised_asset_list["epc_indicates_empty_cavity"] = ( self.standardised_asset_list[self.EPC_API_DATA_NAMES["walls-description"]].str.lower().isin( @@ -977,17 +992,16 @@ class AssetList: ) ) - z0 = self.standardised_asset_list[ - self.standardised_asset_list["epc_indicates_empty_cavity"] & ( - ~self.standardised_asset_list["non_intrusive_indicates_empty_cavity"] - ) - ] - z0['non-intrusives: Construction'].value_counts() - z0['non-intrusives: Insulated'].value_counts() - z00 = z0[z0['non-intrusives: Insulated'] == "EWI"] - - # If the EPC is estimated, perhaps we should defer to the non-intrusives? - z00[""] + # If the EPC is esimtated, we defer to the non-intrusives + self.standardised_asset_list["epc_indicates_empty_cavity"] = np.where( + ( + self.standardised_asset_list["epc_indicates_empty_cavity"] & + ~self.standardised_asset_list["non_intrusive_indicates_empty_cavity"] & + self.standardised_asset_list["estimated"] + ), + False, + self.standardised_asset_list["epc_indicates_empty_cavity"] + ) ###################################################### # Extraction @@ -997,33 +1011,14 @@ class AssetList: self.standardised_asset_list["non_intrusive_indicates_cavity_extraction"] = ( (self.standardised_asset_list["non-intrusives: Construction"] == "CAVITY") & (self.standardised_asset_list["non-intrusives: Insulated"].isin(["RETRO DRILLED", "FILLED AT BUILD"])) & - (~self.standardised_asset_list['non-intrusives: Material'].isin(["GREY LOOSE BEAD", "COMPACTED BEAD"]) + (~self.standardised_asset_list['non-intrusives: Material'].isin( + ["GREY LOOSE BEAD", "COMPACTED BEAD", "FIBRE BATT NO CAVITY", "EMPTY NARROW BELOW 30mm"] + ) ) & ( self.standardised_asset_list[self.ATTRIBUTE_SAP_THRESHOLD_AND_BELOW] ) ) - # z3 = self.standardised_asset_list[ - # self.standardised_asset_list["non_intrusive_indicates_cavity_extraction"] - # ] - # z3['non-intrusives: Material'].value_counts() - # self.standardised_asset_list['non-intrusives: Material'].value_counts() - # - # z = self.standardised_asset_list[ - # self.standardised_asset_list["non-intrusives: CIGA Check Required"] == "YES" - # ] - # z["non-intrusives: Insulated"].value_counts() - # z["non-intrusives: Material"].value_counts() - # z[self.ATTRIBUTE_SAP_THRESHOLD_AND_BELOW].value_counts() - # z[self.EPC_API_DATA_NAMES["current-energy-efficiency"]].max() - # z[self.EPC_API_DATA_NAMES["current-energy-efficiency"]].min() - # z[self.STANDARD_YEAR_BUILT].describe() - # - # zz = z[z[self.EPC_API_DATA_NAMES["current-energy-efficiency"]] == 105] - # z2 = self.standardised_asset_list[ - # self.standardised_asset_list["non-intrusives: CIGA Check Required"] == "NO" - # ] - ###################################################### # Solar ###################################################### @@ -1114,7 +1109,7 @@ class AssetList: ) | ( self.standardised_asset_list[ "walls_u_value"].apply( - lambda x: x <= 0.3 if not pd.isnull( + lambda x: x <= 0.7 if not pd.isnull( x) else False ) ) @@ -1141,7 +1136,7 @@ class AssetList: "|".join(self.EPC_INSULATED_ROOF_SUBSTRINGS), regex=False ) | ( self.standardised_asset_list[self.ATTRIBUTE_EPC_ROOF_INSULATION_THICKNESS].apply( - lambda x: int(x) >= 270 if str(x).isdigit() else False + lambda x: int(x) >= 200 if str(x).isdigit() else False ) ) | ( self.standardised_asset_list["roof_u_value"].apply( @@ -1152,7 +1147,7 @@ class AssetList: self.standardised_asset_list["solar_epc_loft_needs_topup"] = self.standardised_asset_list[ self.ATTRIBUTE_EPC_ROOF_INSULATION_THICKNESS].apply( - lambda x: int(x) < 270 if str(x).isdigit() else False + lambda x: int(x) < 200 if str(x).isdigit() else False ) # TODO: Fill with False - should be temp! @@ -1187,7 +1182,7 @@ class AssetList: ) & ( # We do not utilise estimated EPCs for this method because we will always find that # "epc_has_floor_recommendation" is False - ~self.standardised_asset_list["estimated"] + (self.standardised_asset_list["estimated"] == False) ) ) | ( ( @@ -1212,7 +1207,7 @@ class AssetList: ) & ( # We do not utilise estimated EPCs for this method because we will always find that # "epc_has_floor_recommendation" is False - ~self.standardised_asset_list["estimated"] + self.standardised_asset_list["estimated"] == False ) ) | ( ( @@ -1274,6 +1269,7 @@ class AssetList: ) # Other floor type, fully insulated + self.standardised_asset_list["solar_eligible_other_floor"] = ( # Landlord data or EPC data indicates the heating system is appropriate ( @@ -1332,6 +1328,9 @@ class AssetList: "Empty Cavity (non-intrusives)": ( self.standardised_asset_list["non_intrusive_indicates_empty_cavity"].sum() ), + "Empty Cavity (non-intrusives, no SAP filter)": ( + self.standardised_asset_list["non_intrusive_indicates_empty_cavity_no_sap_filter"].sum() + ), "Empty Cavity (EPC)": ( ( self.standardised_asset_list["epc_indicates_empty_cavity"] & @@ -1359,6 +1358,17 @@ class AssetList: ) } + # We produce a breakdown of the property types, for cavity fills + cavity_fills = self.standardised_asset_list[ + self.standardised_asset_list["non_intrusive_indicates_empty_cavity"] | ( + self.standardised_asset_list["epc_indicates_empty_cavity"] + ) + ] + + self.work_type_breakdowns = { + "empty_cavity": cavity_fills[self.STANDARD_PROPERTY_TYPE].value_counts() + } + # Finally, we note why each property has been flagged self.standardised_asset_list["cavity_reason"] = None self.standardised_asset_list["cavity_reason"] = np.where( @@ -1366,6 +1376,11 @@ class AssetList: "Non-Intrusive Data Showed Empty Cavity", self.standardised_asset_list["cavity_reason"] ) + self.standardised_asset_list["cavity_reason"] = np.where( + self.standardised_asset_list["non_intrusive_indicates_empty_cavity_no_sap_filter"], + "Non-Intrusive Data Showed Empty Cavity but all SAP scores allowed", + self.standardised_asset_list["cavity_reason"] + ) self.standardised_asset_list["cavity_reason"] = np.where( ( self.standardised_asset_list["epc_indicates_empty_cavity"] & diff --git a/asset_list/app.py b/asset_list/app.py index a24c4043..09ccac02 100644 --- a/asset_list/app.py +++ b/asset_list/app.py @@ -246,22 +246,40 @@ def app(): # - We want: fully insulated property (all wall types), EPC D or below (floors should be solid) # - Or the insulation required is loft/cavity (floors should be solid) - data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Colchester" - data_filename = "Warmfront data- Colchester Borough Homes (Complete).xlsx" + # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Colchester" + # data_filename = "Warmfront data- Colchester Borough Homes (Complete).xlsx" + # sheet_name = "Sheet1" + # postcode_column = 'Full Address.1' + # fulladdress_column = "Full Address" + # address1_column = None + # address1_method = "first_word" + # address_cols_to_concat = [] + # missing_postcodes_method = None + # landlord_year_built = "Build Date" + # landlord_os_uprn = None + # landlord_property_type = "Property Type" + # landlord_wall_construction = "Wallinsul" + # landlord_heating_system = "HeatSorc" + # landlord_existing_pv = None + # landlord_property_id = "Property Reference" + + # For Westward + data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Westward" + data_filename = "WESTWARD - completed list..xlsx" sheet_name = "Sheet1" - postcode_column = 'Full Address.1' - fulladdress_column = "Full Address" + postcode_column = "WFT EDIT Postcode" + fulladdress_column = "Address" address1_column = None - address1_method = "first_word" + address1_method = "house_number_extraction" address_cols_to_concat = [] missing_postcodes_method = None - landlord_year_built = "Build Date" - landlord_os_uprn = None - landlord_property_type = "Property Type" - landlord_wall_construction = "Wallinsul" - landlord_heating_system = "HeatSorc" - landlord_existing_pv = None - landlord_property_id = "Property Reference" + landlord_year_built = "Build date" + landlord_os_uprn = "UPRN" + landlord_property_type = "Location type" + landlord_wall_construction = "Wall Construction (EPC)" + landlord_heating_system = "Heat Source" + landlord_existing_pv = "PV (Y/N)" + landlord_property_id = "Place ref" # Maps addresses to uprn in problematic cases MANUAL_UPRN_MAP = {} diff --git a/asset_list/mappings/walls.py b/asset_list/mappings/walls.py index 82b31d01..78d64988 100644 --- a/asset_list/mappings/walls.py +++ b/asset_list/mappings/walls.py @@ -84,7 +84,7 @@ WALL_CONSTRUCTION_MAPPINGS = { 'Timber frame, as built, no insulation (assumed)': 'timber frame', 'Timber frame, as built, partial insulation (assumed)': 'timber frame', 'Timber frame, with additional insulation': 'timber frame', - 'CAVITY': 'partial unknown cavity', + 'CAVITY': 'cavity unknown insulation', 'COMB': 'unknown', 'NONE': 'unknown', 'NOTKNOWN': 'unknown', diff --git a/etl/customers/remote_assessments/app.py b/etl/customers/remote_assessments/app.py index 15f59c5e..aac0a1a6 100644 --- a/etl/customers/remote_assessments/app.py +++ b/etl/customers/remote_assessments/app.py @@ -4,7 +4,7 @@ from dotenv import load_dotenv from utils.s3 import save_csv_to_s3 from etl.find_my_epc.AssetListEpcData import AssetListEpcData -PORTFOLIO_ID = 133 +PORTFOLIO_ID = 137 USER_ID = 8 load_dotenv(dotenv_path="backend/.env") @@ -19,10 +19,10 @@ def app(): asset_list = [ { - "address": "40", - "postcode": "PE4 5BB", - "uprn": 100090220519, - } + "address": "41 Gainsborough Way", + "postcode": "BA21 5XU", + "uprn": 30016708, + }, ] asset_list = pd.DataFrame(asset_list) @@ -52,8 +52,8 @@ def app(): valuation_data = [ { - "uprn": 100090220519, - "valuation": 135_000 + "uprn": 30016708, + "valuation": 189000 } ] # Store valuation data to s3 diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index c5c07f89..dd81680a 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -993,7 +993,7 @@ class HeatingRecommender: # We check if there's a mains connection and the hot water is inefficient, as this will improve with a boiler has_inefficient_water = ( self.property.data["mains-gas-flag"] and - self.property.data["hot-water-energy-eff"] in ["Very Poor", "Poor", "Average"] + self.property.data["hot-water-energy-eff"] in ["Very Poor", "Poor"] ) non_invasive_recommendation = next((