diff --git a/.idea/Model.iml b/.idea/Model.iml index 96ad7a95..df6c4faa 100644 --- a/.idea/Model.iml +++ b/.idea/Model.iml @@ -7,7 +7,7 @@ - + diff --git a/asset_list/AssetList.py b/asset_list/AssetList.py index 3b5535d5..18e202b6 100644 --- a/asset_list/AssetList.py +++ b/asset_list/AssetList.py @@ -444,6 +444,25 @@ class AssetList: self.standardised_asset_list[self.address1_colname].copy() ) + # Handle the case where the property type column and built form are missing + if self.landlord_property_type is None and self.landlord_built_form is None: + if "Archetype" in self.raw_asset_list.columns: + # We use the non-intrusives as our property type and built form + self.landlord_property_type = self.STANDARD_PROPERTY_TYPE + self.landlord_built_form = self.STANDARD_BUILT_FORM + self.standardised_asset_list[self.landlord_property_type] = ( + self.standardised_asset_list["Archetype"].copy() + ) + self.standardised_asset_list[self.landlord_built_form] = ( + self.standardised_asset_list["Archetype"].copy() + ) + else: + # We use the EPC data as our property type and built form + self.landlord_property_type = self.STANDARD_PROPERTY_TYPE + self.landlord_built_form = self.STANDARD_BUILT_FORM + self.standardised_asset_list[self.landlord_property_type] = None + self.standardised_asset_list[self.landlord_built_form] = None + # Handle the case where the property type column is the same as the built type if self.landlord_property_type == self.landlord_built_form: self.landlord_built_form = self.STANDARD_BUILT_FORM @@ -1342,7 +1361,7 @@ class AssetList: ) self.standardised_asset_list["solar_landlord_data_indicates_needs_heating_upgrade"] = ( self.standardised_asset_list[self.STANDARD_HEATING_SYSTEM].isin( - ["electric storage heaters", "room heaters", "electric radiators", "no heating"] + ["electric storage heaters", "room heaters", "electric radiators", "no heating", "electric fuel"] ) ) @@ -1651,7 +1670,7 @@ class AssetList: "SAP Category"], self.standardised_asset_list["cavity_reason"] ) - else: + elif self.non_intrusives_present: self.standardised_asset_list["cavity_reason"] = np.where( ( self.standardised_asset_list["epc_indicates_empty_cavity"] & @@ -1675,6 +1694,16 @@ class AssetList: "SAP Category"], self.standardised_asset_list["cavity_reason"] ) + else: + self.standardised_asset_list["cavity_reason"] = np.where( + ( + self.standardised_asset_list["epc_indicates_empty_cavity"] & + ~self.standardised_asset_list["non_intrusive_indicates_empty_cavity"] & + pd.isnull(self.standardised_asset_list["cavity_reason"]) + ), + "EPC Shows Empty Cavity: " + self.standardised_asset_list["SAP Category"], + self.standardised_asset_list["cavity_reason"] + ) self.standardised_asset_list["cavity_reason"] = np.where( ( @@ -1716,17 +1745,18 @@ class AssetList: self.standardised_asset_list["solar_reason"] = None # Map of variables and fill values for the solar_reason variable + # ordering of this map is important, where we flag our prioritised work types first solar_reason_map = { "solar_eligible": "Solar Eligible: ", + "solar_eligible_solid_wall_uninsulated": "Solar Eligible, Solid Wall Uninsulated, EPC E or Below: ", "solar_eligible_needs_heating_upgrade": ( "Solar Eligible, Needs Heating Upgrade: " - ), - "solar_eligible_solid_wall_uninsulated": "Solar Eligible, Solid Wall Uninsulated, EPC E or Below: ", + ) } for variable, reason in solar_reason_map.items(): self.standardised_asset_list["solar_reason"] = np.where( - self.standardised_asset_list[variable], + self.standardised_asset_list[variable] & pd.isnull(self.standardised_asset_list["solar_reason"]), reason + self.standardised_asset_list["SAP Category"], self.standardised_asset_list["solar_reason"] ) @@ -2401,6 +2431,7 @@ class AssetList: master_data = pd.read_csv(filepath) # Strip columns master_data.columns = [c.strip() for c in master_data.columns] + master_data.columns = [re.sub(r'\s+', ' ', c) for c in master_data.columns] if not id_map.empty: master_data = master_data.merge( @@ -2537,8 +2568,8 @@ class AssetList: ] scheme_col = ( - "AFFORDABLE WARMTH OR EPC FOR HOUSING ASSOCIATION" if - "AFFORDABLE WARMTH OR EPC FOR HOUSING ASSOCIATION" in master_data.columns else "AFFORDABLE WARMTH" + "AFFORDABLE WARMTH OR EPC FOR HOUSING ASSOCIATION" if + "AFFORDABLE WARMTH OR EPC FOR HOUSING ASSOCIATION" in master_data.columns else "AFFORDABLE WARMTH" ) # The columns are massively different - we take just a few unmatched_df = unmatched_df[ diff --git a/asset_list/app.py b/asset_list/app.py index e8388100..37e687fc 100644 --- a/asset_list/app.py +++ b/asset_list/app.py @@ -89,43 +89,6 @@ def app(): # - We want: fully insulated property (all wall types), EPC D or below (floors should be solid) # - Or the insulation required is loft/cavity (floors should be solid) - # Bromford - data_folder = ("/Users/khalimconn-kowlessar/Documents/hestia/Customers/Bromford/Apr 2025 Programme " - "Rebuild/Prepared data/") - data_filename = "asset_list.xlsx" - sheet_name = "Sheet1" - postcode_column = 'PostCode' - fulladdress_column = "FullAddress" - address1_column = None - address1_method = "house_number_extraction" - address_cols_to_concat = [] - missing_postcodes_method = None - landlord_year_built = "ConYear" - landlord_os_uprn = None - landlord_property_type = "AssetTypeDesc" - landlord_built_form = "PropTypeDesc" - landlord_wall_construction = "Construction type" - landlord_roof_construction = None - landlord_heating_system = "Heating Type" - landlord_existing_pv = None - landlord_property_id = "Asset" - landlord_sap = None - outcomes_filename = "outcomes.xlsx" - outcomes_sheetname = "Sheet1" - outcomes_postcode = "Postcode" - outcomes_houseno = "No" - outcomes_id = None - outcomes_address = "Address" - master_filepaths = [ - "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Bromford/Apr 2025 Programme Rebuild/Prepared data/ECO " - "3 submissions.csv", - "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Bromford/Apr 2025 Programme Rebuild/Prepared data/ECO " - "4 submissions.csv", - ] - master_to_asset_list_filepath = None - phase = False - ecosurv_landlords = "paul butler|bromford" - # Torus data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Torus/Phase 1" data_filename = "Torus Property Asset List - Phase 1.xlsx" @@ -156,33 +119,6 @@ def app(): master_to_asset_list_filepath = None phase = True - # Ealing - houses - data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Ealing" - data_filename = "Ealing_rechecked_cleaned_05042025.csv" - sheet_name = None - postcode_column = 'Postcode' - fulladdress_column = "Address" - address1_column = None - address1_method = "house_number_extraction" - address_cols_to_concat = [] - missing_postcodes_method = None - landlord_year_built = "Year Built" - landlord_os_uprn = None - landlord_property_type = "Property Type Code" - landlord_built_form = None - landlord_wall_construction = None - landlord_heating_system = None - landlord_existing_pv = None - landlord_property_id = "Property ref" - outcomes_filename = None - outcomes_sheetname = None - outcomes_postcode = None - outcomes_houseno = None - outcomes_id = None - outcomes_address = None - master_filepaths = [] - master_to_asset_list_filepath = None - # Southern Midlands data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Southern/Midlands Properties - Apr 2025" data_filename = "Southern Housing Midlands Property List - combined.xlsx" @@ -210,67 +146,6 @@ def app(): master_filepaths = [] master_to_asset_list_filepath = None - # Live West (2018 Asset list) - data_folder = ( - "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Livewest/Programme Update - March 2025/2018 Asset List" - ) - data_filename = "LIVEWEST STOCK - 23rd October 2018.xlsx" - sheet_name = "Assets" - postcode_column = 'Postcode' - fulladdress_column = "Address" - address1_column = None - address1_method = "house_number_extraction" - address_cols_to_concat = [] - missing_postcodes_method = None - landlord_year_built = "Build Year" - landlord_os_uprn = None - landlord_property_type = "Property Archetype" - landlord_built_form = None - landlord_wall_construction = None - landlord_heating_system = "Heating Fuel Type" - landlord_existing_pv = None - landlord_property_id = "Uprn - DO NOT DELETE" - outcomes_filename = "RT - LiveWest.xlsx" - outcomes_sheetname = "Feedback" - outcomes_postcode = "Poscode" - outcomes_houseno = "No." - outcomes_id = "UPRN" - master_filepaths = [ - "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Livewest/Programme Update - March 2025/Rolling Master " - "- redacted for analysis/CAVITY-Table 1.csv" - ] - master_to_asset_list_filepath = None - - # Live West (South West asset list) - data_folder = ("/Users/khalimconn-kowlessar/Documents/hestia/Customers/Livewest/Programme Update - March " - "2025/Livewest Asset List (Original) - csv") - data_filename = "Report-Table 1.csv" - sheet_name = None - postcode_column = 'Postcode' - fulladdress_column = "T1_Address" - address1_column = None - address1_method = "house_number_extraction" - address_cols_to_concat = [] - missing_postcodes_method = None - landlord_year_built = "Build Yr" - landlord_os_uprn = None - landlord_property_type = "T1_AssetType" - landlord_built_form = "T1_AssetType" - landlord_wall_construction = "Wall Type Cavity" - landlord_heating_system = "Heating Fuel" - landlord_existing_pv = None - landlord_property_id = "T1_UPRN" - outcomes_filename = "RT - LiveWest.xlsx" - outcomes_sheetname = "Feedback" - outcomes_postcode = "Poscode" - outcomes_houseno = "No." - outcomes_id = "UPRN" - master_filepaths = [ - "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Livewest/Programme Update - March 2025/Rolling Master " - "- redacted for analysis/CAVITY-Table 1.csv" - ] - master_to_asset_list_filepath = None - # PFP London data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Places For People/London" data_filename = "PFP AREAS SURROUNDING LONDON - JAY, RUTH & LANE.xlsx" @@ -459,6 +334,8 @@ def app(): landlord_heating_system = "Heat Source" landlord_existing_pv = "PV (Y/N)" landlord_property_id = "Place ref" + landlord_roof_construction = None + landlord_sap = None outcomes_filename = None outcomes_sheetname = None outcomes_postcode = None @@ -466,6 +343,9 @@ def app(): master_filepaths = [] master_to_asset_list_filepath = None outcomes_id = None + outcomes_address = None + phase = False + ecosurv_landlords = None # For ACIS - programme re-build # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/ACIS/ACIS Full Programme Review March 2025" @@ -483,17 +363,23 @@ def app(): # landlord_property_type = "Property type" # landlord_built_form = None # landlord_wall_construction = "Wall Constuction" + # landlord_roof_construction = None + # landlord_sap = None # landlord_heating_system = "Heating" # landlord_existing_pv = None # outcomes_filename = "ACIS Group - 25.11.2024 - outcomes.xlsx" # outcomes_sheetname = "Feedback" # outcomes_postcode = "Postcode" + # outcomes_address = "Address" # outcomes_houseno = "No" + # outcomes_id = None # master_filepaths = [ # os.path.join(data_folder, "ECO 3 -Table 1.csv"), # os.path.join(data_folder, "ECO 4 -Table 1.csv"), # ] # master_to_asset_list_filepath = None + # phase = False + # ecosurv_landlords = None # For plus dane data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Plus Dane" @@ -697,6 +583,9 @@ def app(): epc_data.append(csv_data) epc_df = pd.concat(epc_data) + if "estimated" not in epc_df.columns: + epc_df["estimated"] = False + epc_df["estimated"] = epc_df["estimated"].fillna(False) # We expand out the recommendations @@ -804,98 +693,6 @@ def app(): asset_list.flat_analysis() - ################################################################ - # WESTWARD - comparison between Kieran's method & automated - ################################################################ - - # Check 1) - cavity_fills = pd.read_excel( - os.path.join(data_folder, "WESTWARD - Route March Prep.xlsx"), - sheet_name="Straight Fill" - ) - cavity_fills = cavity_fills.merge( - asset_list.standardised_asset_list[ - [asset_list.STANDARD_LANDLORD_PROPERTY_ID, "cavity_reason"] - ], - how="left", - left_on=asset_list.landlord_property_id, - right_on=asset_list.STANDARD_LANDLORD_PROPERTY_ID - ) - cavity_fills["cavity_reason"] = cavity_fills["cavity_reason"].fillna("Not identified") - print(cavity_fills["cavity_reason"].value_counts()) - # Didn't identify 3 properties because they're bedsits - # 4 properties were identified, not based on the non-intrusives but instead because - # Westward said they were built in 2003/2007. Have adjusted this to use the age from the - # epc as well, as EPC says 1975 and they look like 1975 properties - # 37 properties flagged as already having solar - these are all because the landlord said they have solar - # e.g. - # https://earth.google.com/web/search/11+Winsland+Avenue+TOTNES+TQ9+5FT/@50.43354465,-3.71318276,46.57468503a, - # 59.14004365d,35y,0h,0t, - # 0r/data=CpABGmISXAolMHg0ODZkMWQxOGE4NWRiZjdkOjB4YjBhM2E5M2Q3YWVlMWEwYhlZYgp7fzdJQCHFfC9027QNwCohMTEgV2luc2xhbmQgQXZlbnVlIFRPVE5FUyBUUTkgNUZUGAIgASImCiQJbxsQEoo3SUARXQcp_HE3SUAZBmiZGJ6yDcAhCA0fqq63DcBCAggBOgMKATBCAggASg0I____________ARAA - # https://earth.google.com/web/search/15+St+Anne%27s+Ct,+Newton+Abbot+TQ12+1TL/@50.53068337,-3.61611128, - # 11.74908956a,135.73212429d,35y,0h,0t, - # 0r/data=CpUBGmcSYQolMHg0ODZkMDVkMjFhODhjZjgxOjB4MjBmMzE2Zjc3MGI2NGMwYxlCxHLw8UNJQCFZqyzALe4MwComMTUgU3QgQW5uZSdzIEN0LCBOZXd0b24gQWJib3QgVFExMiAxVEwYAiABIiYKJAm-r6U2iDdJQBHS5ICRdDdJQBmYGVpmiLINwCG8wcrtqbYNwEICCAE6AwoBMEICCABKDQj___________8BEAA - - # Check 2) - cavity_fills_with_solar = pd.read_excel( - os.path.join(data_folder, "WESTWARD - Route March Prep.xlsx"), - sheet_name="Solar PV - Straight Fill" - ) - cavity_fills_with_solar = cavity_fills_with_solar.merge( - asset_list.standardised_asset_list[ - [asset_list.STANDARD_LANDLORD_PROPERTY_ID, "cavity_reason"] - ], - how="left", - left_on=asset_list.landlord_property_id, - right_on=asset_list.STANDARD_LANDLORD_PROPERTY_ID - ) - cavity_fills_with_solar["cavity_reason"] = cavity_fills_with_solar["cavity_reason"].fillna("Not identified") - print(cavity_fills_with_solar["cavity_reason"].value_counts()) - # 203 properties total - # 140 properties were flagged up based on non-intrusives (Non-Intrusive Data Showed Empty Cavity) - # 63 property already has solar - - # Check 3) RDF - rdf = pd.read_excel( - os.path.join(data_folder, "WESTWARD - Route March Prep.xlsx"), - sheet_name="RDF CIGA checks" - ) - rdf = rdf.merge( - asset_list.standardised_asset_list[ - [asset_list.STANDARD_LANDLORD_PROPERTY_ID, "cavity_reason", "solar_reason"] - ], - how="left", - left_on=asset_list.landlord_property_id, - right_on=asset_list.STANDARD_LANDLORD_PROPERTY_ID - ) - rdf["cavity_reason"] = rdf["cavity_reason"].fillna("Not identified") - print(rdf["cavity_reason"].value_counts()) - # 264 properties are not identified, 261 of which are due to the fact they contain materials - # The other 3 were determined to be eligible for solar instead - # Many of these units that were identified for rdf works could be solar jobs - - rdf_with_solar = pd.read_excel( - os.path.join(data_folder, "WESTWARD - Route March Prep.xlsx"), - sheet_name="Solar PV - RDF CIGA Checks" - ) - rdf_with_solar = rdf_with_solar.merge( - asset_list.standardised_asset_list[ - [asset_list.STANDARD_LANDLORD_PROPERTY_ID, "cavity_reason", "solar_reason"] - ], - how="left", - left_on=asset_list.landlord_property_id, - right_on=asset_list.STANDARD_LANDLORD_PROPERTY_ID - ) - rdf_with_solar["cavity_reason"] = rdf_with_solar["cavity_reason"].fillna("Not identified") - rdf_with_solar["cavity_reason"].value_counts() - - # All others identified - some flagged as empties due to EPC or landlord data suggesting as much - # 5 not identified due to containing COMPACTED BEAD - - asset_list.standardised_asset_list = asset_list.standardised_asset_list[ - asset_list.standardised_asset_list[asset_list.landlord_property_id] - ] - asset_list.load_contact_details( local_filepath=os.path.join(data_folder, "Full property list wth D&V report V look up 12.2.25.xlsx"), sheet_name="Report 1", diff --git a/asset_list/mappings/built_form.py b/asset_list/mappings/built_form.py index e103f794..1d0aecf5 100644 --- a/asset_list/mappings/built_form.py +++ b/asset_list/mappings/built_form.py @@ -143,6 +143,74 @@ BUILT_FORM_MAPPINGS = { 'Sixth Floor': 'top-floor', 'Sheltered Bung': 'semi-detached', 'Guest': 'unknown', - 'Fifth Floor': 'mid-floor' - + 'Fifth Floor': 'mid-floor', + 'Flat Within Block': 'mid-floor', + 'Coach House with Garage': 'detached', + 'Over Garage House': 'top-floor', + 'Apartment': 'mid-floor', + 'Flat Over Shop': 'top-floor', + 'Flat Over Garage': 'top-floor', + 'Bridge Flat': 'mid-floor', + 'House Mid Terrace': 'mid-terrace', + 'Semi-detached house': 'semi-detached', + 'House Semi Detached': 'semi-detached', + 'House Detached': 'detached', + 'Detached house': 'detached', + 'House End Terrace': 'end-terrace', + 'Flat Ground Floor Mr': 'ground floor', + 'Mais Flat 1St Fl Mr': 'mid-floor', + 'Top-floor maisonette': 'top-floor', + 'Flat 1St Warden Lr': 'mid-floor', + 'Cranwell': 'unknown', + 'No Fines': 'unknown', + 'Flat 1St Elderly Mr': 'mid-floor', + 'Stent Mod': 'unknown', + 'Mais Flat Grd Fl Mr': 'ground floor', + 'Flat 1St Floor Mr': 'mid-floor', + 'Mid-terrace house': 'mid-terrace', + 'Stent Unmod': 'unknown', + 'Flat 2Nd Floor Mr': 'mid-floor', + 'Studio Grd Warden Lr': 'ground floor', + 'Flat Grd Elderly Mr': 'ground floor', + 'Studio Fl Grd Eld Lr': 'ground floor', + 'Scottwood': 'unknown', + 'Airey': 'unknown', + 'Studio Flat 1Stfl Lr': 'mid-floor', + 'Studio Flat 1Stfl Mr': 'mid-floor', + 'Flat Grd Elderly Lr': 'ground floor', + 'Trusteel MKII': 'unknown', + 'No-Fines Concrete': 'unknown', + 'Crosswall': 'unknown', + 'Fidler': 'unknown', + 'Ground-floor maisonette': 'ground floor', + 'Studio Flat Grdfl Mr': 'ground floor', + 'Studio Flat Grd Lr': 'ground floor', + 'Studio Fl Grd Eld Mr': 'ground floor', + 'Bungalow Eld Person': 'unknown', + 'Cornish': 'unknown', + 'B.I.S.F.': 'unknown', + 'Flat 1St Floor Lr': 'mid-floor', + 'Mid-floor flat': 'mid-floor', + 'Bsit Bung Warden Sch': 'unknown', + 'Hawksley': 'unknown', + 'Orlit': 'unknown', + 'Mid-floor maisonette': 'mid-floor', + 'Ground-floor flat': 'ground floor', + 'Flat Grd Floor Lr': 'ground floor', + 'Studio 1St Warden Lr': 'mid-floor', + 'Flat Grd Warden Lr': 'ground floor', + 'end-terrace house': 'end-terrace', + 'Top-floor flat': 'top-floor', + 'End-terrace house': 'end-terrace', + 'Mais Flat 2Nd Fl Mr': 'mid-floor', + 'Flat 1St Elderly Lr': 'mid-floor', + 'Bfly Bung Bed Sitter': 'unknown', + 'Swedish': 'unknown', + 'Bungalow Semi Detach': 'semi-detached', + '4 Ext. Wall Flat': 'unknown', + '6 Ext. Wall Flat': 'unknown', + '5 Ext. Wall Flat': 'unknown', + 'Unknown': 'unknown', + 'Enclosed mid-terrace': 'mid-terrace', + 'Enclosed end-terrace': 'end-terrace' } diff --git a/asset_list/mappings/heating_systems.py b/asset_list/mappings/heating_systems.py index 7f2f81f2..b5cf500f 100644 --- a/asset_list/mappings/heating_systems.py +++ b/asset_list/mappings/heating_systems.py @@ -2,8 +2,10 @@ import numpy as np STANDARD_HEATING_SYSTEMS = { "gas combi boiler", + "gas boiler, radiators", "electric storage heaters", "district heating", + "communal heating" "gas condensing boiler", "oil boiler", "gas condensing combi", @@ -24,7 +26,8 @@ STANDARD_HEATING_SYSTEMS = { 'unknown', "electric ceiling", "electric underfloor", - "no heating" + "no heating", + "non-electric underfloor" } HEATING_MAPPINGS = { @@ -202,5 +205,20 @@ HEATING_MAPPINGS = { 'Wet - Underfloor Solar': 'other', 'No Heating Required Gas': 'unknown', 'Electric - Storage/Panel Heaters Gas': 'electric storage heaters', - 'Electric - Storage/Panel Heaters Solid': 'electric storage heaters' + 'Electric - Storage/Panel Heaters Solid': 'electric storage heaters', + 'District Heat Network': 'district heating', + 'Not Applicable': 'no heating', + 'Not Responsible': 'unknown', + 'Communal Oil': 'communal heating', + 'Communal Electric': 'communal heating', + 'Renewables (Air / Ground Source Pumps)': 'air source heat pump', + 'Communal Renewable': 'air source heat pump', + 'Room heaters/ Electric': 'room heaters', + 'Room heaters/ Gas': 'room heaters', + 'Radiator system': "gas boiler, radiators", + 'Drilled and filled': 'unknown', + 'Boiler/ underfloor': 'electric underfloor', + 'Storage system': "non-electric underfloor", + 'BOILER': 'gas combi boiler', + 'SPACE_HEATER': 'room heaters' } diff --git a/asset_list/mappings/property_type.py b/asset_list/mappings/property_type.py index dc8dbf21..f01ab5eb 100644 --- a/asset_list/mappings/property_type.py +++ b/asset_list/mappings/property_type.py @@ -178,5 +178,21 @@ PROPERTY_MAPPING = { 'Parking Space': 'other', 'Community Centre': 'other', 'Communal Facility': 'other', - 'Semi': 'house' + 'Semi': 'house', + 'House with Compulsory Garage': 'house', + 'Flat with Compulsory Garage': 'flat', + 'Other': 'other', + 'Maisonette with Compulsory Garage': 'maisonette', + 'Room in Shared Property': 'other', + 'Bungalow Mid Terrace': 'bungalow', + '4 Ext. Wall Flat': 'flat', + '3 Ext. Wall Flat': 'flat', + 'Bungalow End Terrace': 'bungalow', + '6 Ext. Wall Flat': 'flat', + 'Bungalow Detached': 'bungalow', + 'Maisonette 3 Ext. Wall': 'maisonette', + 'Maisonette 2 Ext. Wall': 'maisonette', + '5 Ext. Wall Flat': 'flat', + 'Bungalow Semi Detached': 'bungalow', + 'COMINT': 'unknown' } diff --git a/asset_list/mappings/walls.py b/asset_list/mappings/walls.py index c327338a..1fb8cb79 100644 --- a/asset_list/mappings/walls.py +++ b/asset_list/mappings/walls.py @@ -158,7 +158,6 @@ WALL_CONSTRUCTION_MAPPINGS = { '2017 onwards': 'new build - average thermal transmittance', 'ND (inferred)': 'unknown', 'Flat / maisonette': 'other', - 'Other': 'other', 'Timber Frame': 'timber frame unknown insulation', 'Cavity Wall': 'cavity unknown insulation', @@ -166,5 +165,59 @@ WALL_CONSTRUCTION_MAPPINGS = { 'PRC': 'system built', 'Cross Wall': 'system built', 'Solid Wall': 'solid brick unknown insulation', - 'Traditional': 'other' + 'Traditional': 'unknown', + + 'Solid': 'solid brick unknown insulation', + 'Wates no fines': 'system built', + 'Concrete Frame': 'system built', + 'PRCWATES': 'system built', + 'Refurbished Cornish': 'system built', + 'Bailey Stratton': 'other', + 'Refurbished Reema': 'system built', + 'PRCREEMA': 'system built', + 'Trustsell Type': 'system built', + 'Petra Nissan': 'unknown', + 'Reinstated Airey': 'system built', + 'Refurbished Airey': 'system built', + # From Abri- slightly unclear on types but not a large portion of the data + 'No Fines Type': 'system built', + 'Refurbished Unity': 'system built', + 'Timber Framed': 'timber frame unknown insulation', + 'Refurbished Woolaway': 'system built', + 'Modern Methods of Construction': 'other', + 'BISF - Brit Iron & Steel Federation': 'system built', + 'Steel Framed': 'system built', + 'Timber Framed with confirmed Fire Stopping': 'timber frame unknown insulation', + 'Sipporex': 'system built', + + 'Wates': 'system built', + 'Bryants': 'system built', + 'Gregory (Crosswall)': 'system built', + 'Rsmit': 'system built', + 'Dorman Long': 'system built', + 'Tarmac': 'system built', + 'RBIS': 'system built', + 'Five Oaks': 'system built', + 'Not known': 'unknown', + 'Smiths': 'system built', + 'Kendrick': 'system built', + 'IDC': 'system built', + 'Wimpey (Part Brick)': 'system built', + 'Whitehall': 'system built', + 'Wimpey': 'system built', + 'Bison': 'system built', + 'Zinns': 'system built', + 'Bisf': 'system built', + 'Integer': 'system built', + 'Cornish': 'system built', + 'Rwate': 'system built', + 'Hill Presweld Steel': 'system built', + + 'Cavity Filled Cavity': 'filled cavity', + 'Cavity Unknown': 'cavity unknown insulation', + 'Cavity Filled Cavity (internal)': 'filled cavity', + '': 'unknown', + 'Cavity Internal Insulation': 'filled cavity', + 'Cavity As Built': "uninsulated cavity" + } diff --git a/backend/Property.py b/backend/Property.py index 52e8c213..91c1265a 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -1261,6 +1261,11 @@ class Property: # may be installed such that they are not visible from the street return False + if (self.data["property-type"] in ["House", "Bungalow"]) and ( + not pd.isnull(self.roof["thermal_transmittance"]) + ): + return True + is_valid_property_type = self.data["property-type"] in ["House", "Bungalow", "Maisonette"] is_valid_roof_type = ( self.roof["is_flat"] or self.roof["is_pitched"] or self.roof["is_roof_room"] diff --git a/backend/engine/engine.py b/backend/engine/engine.py index 7c99360d..faa1ed94 100644 --- a/backend/engine/engine.py +++ b/backend/engine/engine.py @@ -411,7 +411,7 @@ def get_funding_data(): async def model_engine(body: PlanTriggerRequest): - logger.info("Model Engine triggered with body: %s", body.model_dump_json()) + logger.info("Model Engine triggered with body: %s", json.loads(body.model_dump_json())) logger.info("Connecting to db") session = sessionmaker(bind=db_engine)() diff --git a/etl/customers/l_and_g/risk_matrix.py b/etl/customers/l_and_g/risk_matrix.py new file mode 100644 index 00000000..bc1bc952 --- /dev/null +++ b/etl/customers/l_and_g/risk_matrix.py @@ -0,0 +1,323 @@ +from itertools import product +from recommendations.recommendation_utils import estimate_external_wall_area, estimate_windows + +import numpy as np + +import pandas as pd + + +def app(): + # Given a combination of variables, this code attempts to break down the costs of works to achieve upgrade + # targets + + upgrade_path = [ + "wall_insulation", "roof_insulation", "ventilation", "windows", "low_energy_lighting", + "heating", "solar" + ] + + pricing_matrix = { + "Cavity wall insulation": 14.5, + "ventilation": 350, + "Room Roof Insulation": 210, + "Loft insulation": 15, + "Internal wall insulation": 215, + "External wall insulation": 298.35, + "Solid wall insulation": 215, + "LEDs": 35, # per light + "Flat Roof Insulation": 195, + "Double Glazing": 1140, + "secondary_glazing": 970, + "5kw ASHP feeding heating & Hot water (dual tariff)": 14738, + "11.2kw ASHP feeding heating & Hot water (dual tariff)": 16541, + "3 kWp Solar PV": 4552.32, + "4 kWp Solar PV": 4892.8, + "4.3 kWp Solar PV": 4961.44, + "4.8 kWp Solar PV": 5414, + '5 kWp Solar PV': 5509.71, + '5.5 kWp Solar PV': 5631.92, + "HHRSH (dual tariff)": 1000, # per heater + "Suspended floor insulation": 75 + } + dwelling_types = [ + "Semi Detached House", + "Detached house", + "Mid Terrace house", + "Mid Floor Flat", + "Top Floor Flat", + "Ground Floor Flat" + ] + num_floors_map = { + "Semi-detached house": 2, + "Detached house": 2, + "Mid Terrace house": 2, + "Mid Floor Flat": 1, + "Top Floor Flat": 1, + "Ground Floor Flat": 1 + } + built_form_map = { + "Semi-detached house": "Semi-Detached", + "Detached house": "Detached", + "Mid Terrace house": "Mid Terrace", + "Mid Floor Flat": "Semi-Detached", + "Top Floor Flat": "Semi-Detached", + "Ground Floor Flat": "Semi-Detached" + } + lighting_count = { + "Semi-detached house": 15, + "Detached house": 19, + "Mid Terrace house": 12, + "Mid Floor Flat": 10, + "Top Floor Flat": 10, + "Ground Floor Flat": 10 + } + + # If we have a flat, we won't use the 199m2 floor area + floor_areas = [73, 97, 199] + # We remove age bracket, as we ended up with 360 combinations + # age_brackets = ["1945-1970", "1971-2002", "Post 2002"] + wall_type = ["cavity", "non-cavity"] + roof_type = ["pitched", "other"] + planning_constraints = [True, False] + + # This is the list of all combinations of the above variables + combinations_untrimmed = product( + *[ + dwelling_types, floor_areas, wall_type, roof_type, planning_constraints + ] + ) + + # TODO: Possibly need to add an additional cost for immersion hot water + combinations = [] + for comb in combinations_untrimmed: + if "Flat" in comb[0] and comb[1] == 199: + continue + + # If we have a flat, not too much difference if it's in a conservation area or not + if "Flat" in comb[0] and comb[4] is True: + continue + combinations.append(comb) + + risk_matrix = [] + for combination in combinations: + n_floors = num_floors_map[combination[0]] + bf = built_form_map[combination[0]] + pt = "House" if "Flat" not in combination[0] else "Flat" + # Model the home as a box + ground_floor_area = combination[1] / n_floors + perimeter = np.sqrt(ground_floor_area) * 4 + + # This is the amount of insulation required + external_wall_area = estimate_external_wall_area( + num_floors=n_floors, + floor_height=2.5, + perimeter=perimeter, + built_form=bf + ) + + n_rooms = np.floor(combination[1] / 15) + + n_windows = estimate_windows( + property_type=pt, + built_form=bf, + construction_age_band="", + floor_area=combination[1], + number_habitable_rooms=n_rooms + ) + + # We determine the exact upgrade pathway for this combination, guided by the generic upgrade pathway + combination_upgrade_pathway = [] + for upgrade in upgrade_path: + if upgrade == "wall_insulation": + if combination[2] == "cavity": + combination_upgrade_pathway.append("cavity_wall_insulation") + else: + combination_upgrade_pathway.append("solid_wall_insulation") + continue + + if upgrade == "roof_insulation": + if combination[3] == "pitched": + combination_upgrade_pathway.append("loft_insulation") + else: + combination_upgrade_pathway.append("non_pitched_roof_insualtion") + continue + + if upgrade == "ventilation": + combination_upgrade_pathway.append("ventilation") + continue + + if upgrade == "low_energy_lighting": + combination_upgrade_pathway.append("low_energy_lighting") + continue + + if upgrade == "windows": + if not combination[4]: + combination_upgrade_pathway.append("double_glazing") + else: + combination_upgrade_pathway.append("secondary_glazing") + continue + + if upgrade == "heating": + if combination[0] in ["Semi Detached House", "Detached House"]: + combination_upgrade_pathway.append("high_heat_retention_storage") + else: + combination_upgrade_pathway.append("air_source_heat_pump") + continue + + if upgrade == "solar": + if combination[0] in ["Semi Detached House", "Detached House", "Mid Terrace House"]: + combination_upgrade_pathway.append("solar_pv") + continue + + combination_costs = [] + for measure in combination_upgrade_pathway: + unit_cost = pricing_matrix[measure] + # Wall insulation + if measure in ["cavity_wall_insulation", "internal_wall_insulation", "external_wall_insulation"]: + cost = unit_cost * external_wall_area + elif measure in ["loft_insulation"]: + cost = unit_cost * ground_floor_area + elif measure == "ventilation": + if combination[1] == 73: + cost = unit_cost * 2 + elif combination[1] == 97: + cost = unit_cost * 3 + else: + cost = unit_cost * 4 + elif measure == "low_energy_lighting": + n_lights = lighting_count[combination[0]] + if combination[1] == 73: + inflation = 1 + elif combination[1] == 97: + inflation = 1.2 + else: + inflation = 1.5 + cost = unit_cost * n_lights * inflation + elif measure in ["double_glazing", "secondary_glazing"]: + cost = unit_cost * n_windows + elif measure == "high_heat_retention_storage": + cost = unit_cost * n_rooms + elif measure in ["air_source_heat_pump", "solar_pv"]: + cost = unit_cost + else: + raise NotImplementedError("Implement: %s" % measure) + + combination_costs.append( + { + "measure": measure, + "cost": cost + } + ) + + combination_costs = pd.DataFrame(combination_costs) + + contingency = 0.26 + + epr_data = pd.read_excel( + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/L&G/Risk Matrix/EPR Data.xlsx", header=1 + ) + epr_data["Measure added"].value_counts() + epr_data["row_id"] = epr_data.index + # We need to calculate the costs + cost_data = [] + for _, row in epr_data.iterrows(): + epc = row["EPC"][0] + sap = int(row["EPC"][1:]) + + n_floors = num_floors_map[row["Property Type"]] + bf = built_form_map[row["Property Type"]] + pt = "House" if "flat" not in row["Property Type"].lower() else "Flat" + # Model the home as a box + ground_floor_area = row["area"] / n_floors + perimeter = np.sqrt(ground_floor_area) * 4 + + # This is the amount of insulation required + external_wall_area = estimate_external_wall_area( + num_floors=n_floors, + floor_height=2.5, + perimeter=perimeter, + built_form=bf + ) + + n_rooms = np.floor(row["area"] / 15) + + n_windows = estimate_windows( + property_type=pt, + built_form=bf, + construction_age_band="", + floor_area=row["area"], + number_habitable_rooms=n_rooms + ) + cost_upper_bound = None + if pd.isnull(row["Measure added"]): + unit_cost = None + else: + measure = row["Measure added"] + unit_cost = pricing_matrix[measure] + + if pd.isnull(row["Measure added"]): + cost = None + elif row["Measure added"] == "Loft insulation": + cost = unit_cost * ground_floor_area + elif row["Measure added"] in ["Cavity wall insulation", "Internal wall insulation"]: + cost = unit_cost * external_wall_area + pricing_matrix["ventilation"] * 3 + elif row["Measure added"] == "Solid wall insulation": + cost = unit_cost * external_wall_area + pricing_matrix["ventilation"] * 3 + cost_upper_bound = pricing_matrix["External wall insulation"] * external_wall_area + pricing_matrix[ + "ventilation"] * 3 + elif row["Measure added"] == "Double Glazing": + cost = unit_cost * n_windows + elif row["Measure added"] == "LEDs": + cost = unit_cost * lighting_count[row["Property Type"]] + elif row["Measure added"] in [ + '5kw ASHP feeding heating & Hot water (dual tariff)', + '11.2kw ASHP feeding heating & Hot water (dual tariff)', + "3 kWp Solar PV", + '4 kWp Solar PV', + "4.3 kWp Solar PV", + '4.8 kWp Solar PV', + '5 kWp Solar PV', + '5.5 kWp Solar PV' + ]: + cost = unit_cost + elif row["Measure added"] == "HHRSH (dual tariff)": + cost = unit_cost * (n_rooms + 1) + elif row["Measure added"] == "Suspended floor insulation": + cost = unit_cost * ground_floor_area + else: + raise Exception() + + cost_data.append( + { + "row_id": row["row_id"], + "epc": epc, + "sap": sap, + "cost": cost, + "cost upper bound": cost_upper_bound + } + ) + + cost_data = pd.DataFrame(cost_data) + + risk_matrix = pd.merge( + epr_data, + cost_data, + on="row_id", + ) + + risk_matrix["contingency"] = risk_matrix["cost"] * contingency + risk_matrix["upper bound coningency"] = risk_matrix["cost upper bound"] * contingency + + pricing_df = pd.DataFrame( + [ + { + "Measure": k, + "Unit Cost": v + } + for k, v in pricing_matrix.items() + ] + ) + + with pd.ExcelWriter( + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/L&G/Risk Matrix/risk_matrix.xlsx") as writer: + risk_matrix.to_excel(writer, sheet_name="Risk Matrix", index=False) + pricing_df.to_excel(writer, sheet_name="Pricing Assumptions", index=False) diff --git a/etl/epc/Record.py b/etl/epc/Record.py index 9ff1de0a..1ed33567 100644 --- a/etl/epc/Record.py +++ b/etl/epc/Record.py @@ -355,6 +355,7 @@ class EPCRecord: self._clean_floor_level() self._clean_floor_height() self._clean_constituency() + self._clean_new_build_descriptions() # self._clean_potential_energy_efficiency() # self._clean_environment_impact_potential() @@ -397,6 +398,10 @@ class EPCRecord: if self.prepared_epc["floor-height"] <= 1.665: self.prepared_epc["floor-height"] = average + def _clean_new_build_descriptions(self): + for col in ['roof-description', 'walls-description', 'floor-description']: + self.prepared_epc[col] = self.prepared_epc[col].replace("W/m²K", "W/m-¦K") + def _clean_constituency(self): """ We handle the single case of finding a missing constituency by using the local authority diff --git a/recommendations/RoofRecommendations.py b/recommendations/RoofRecommendations.py index cd7f82c4..fa8b831c 100644 --- a/recommendations/RoofRecommendations.py +++ b/recommendations/RoofRecommendations.py @@ -163,15 +163,12 @@ class RoofRecommendations: if self.property.roof["is_thatched"]: return - # If we have a u-value already, need to implement this - if u_value: - if u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE: - # The Roof is already compliant - return - - if self.property.data["transaction-type"] in ["new dwelling", "not sale or rental"]: - return - raise NotImplementedError("Implement me") + # If we have a u-value and we don't have a non-invasive recommendation, we can't recommend anything + if u_value and not any( + x in MEASURE_MAP["roof_insulation"] for x in [r["type"] for r in self.property.non_invasive_recommendations] + ): + # We don't have enough information to provide a recommendation + return u_value = get_roof_u_value( insulation_thickness=self.property.roof["insulation_thickness"], diff --git a/recommendations/WallRecommendations.py b/recommendations/WallRecommendations.py index 92147fb8..dbb7d674 100644 --- a/recommendations/WallRecommendations.py +++ b/recommendations/WallRecommendations.py @@ -227,24 +227,14 @@ class WallRecommendations(Definitions): # external wall insulation if ( (not is_cavity_wall) - and (self.property.year_built >= self.YEAR_WALLS_BUILT_WITH_INSULATION) and (u_value >= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE) ): # Recommend insulation - self.find_insulation(u_value, phase, measures) + self.find_insulation(u_value, phase, measures=measures, default_u_values=default_u_values) return - # We can't detect it's a cavity wall, but it was built after 1990 so likely built with insulation already - # + it already has a U-value better than the building regulations, so we don't need to recommend anything - if ( - (not is_cavity_wall) - and ((self.property.year_built >= self.YEAR_WALLS_BUILT_WITH_INSULATION) - or (u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE)) - ): - # Recommend nothing - return - - raise NotImplementedError("Not implemented yet") + # We have a sufficiently low U-value + return u_value = get_wall_u_value( clean_description=self.property.walls["clean_description"], @@ -626,7 +616,7 @@ class WallRecommendations(Definitions): "walls_thermal_transmittance_ending": new_u_value } - if default_u_values: + if default_u_values and "Average thermal transmittance" not in new_description: # If we're using default U-values, we overwrite new_u_value new_u_value = get_wall_u_value( clean_description=new_description,