From 3283347efe166ba25275cac714fc244b6513e016 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 29 Apr 2025 23:27:16 +0100 Subject: [PATCH] implementing costing for risk matric --- asset_list/AssetList.py | 18 ++++++- asset_list/app.py | 69 ++++++-------------------- asset_list/mappings/built_form.py | 61 ++++++++++++++++++++++- asset_list/mappings/heating_systems.py | 14 ++++-- asset_list/mappings/property_type.py | 13 ++++- asset_list/mappings/walls.py | 27 +++++++++- etl/customers/l_and_g/risk_matrix.py | 64 ++++++++++++++++++++++-- 7 files changed, 199 insertions(+), 67 deletions(-) diff --git a/asset_list/AssetList.py b/asset_list/AssetList.py index 6632e8de..5637cd42 100644 --- a/asset_list/AssetList.py +++ b/asset_list/AssetList.py @@ -444,6 +444,19 @@ class AssetList: self.standardised_asset_list[self.address1_colname].copy() ) + # Handle the case where the property type column and built form are missing + if self.landlord_property_type is None and self.landlord_built_form is None: + if "Archetype" in self.raw_asset_list.columns: + # We use the non-intrusives as our property type and built form + self.landlord_property_type = self.STANDARD_PROPERTY_TYPE + self.landlord_built_form = self.STANDARD_BUILT_FORM + self.standardised_asset_list[self.landlord_property_type] = ( + self.standardised_asset_list["Archetype"].copy() + ) + self.standardised_asset_list[self.landlord_built_form] = ( + self.standardised_asset_list["Archetype"].copy() + ) + # Handle the case where the property type column is the same as the built type if self.landlord_property_type == self.landlord_built_form: self.landlord_built_form = self.STANDARD_BUILT_FORM @@ -2412,6 +2425,7 @@ class AssetList: master_data = pd.read_csv(filepath) # Strip columns master_data.columns = [c.strip() for c in master_data.columns] + master_data.columns = [re.sub(r'\s+', ' ', c) for c in master_data.columns] if not id_map.empty: master_data = master_data.merge( @@ -2548,8 +2562,8 @@ class AssetList: ] scheme_col = ( - "AFFORDABLE WARMTH OR EPC FOR HOUSING ASSOCIATION" if - "AFFORDABLE WARMTH OR EPC FOR HOUSING ASSOCIATION" in master_data.columns else "AFFORDABLE WARMTH" + "AFFORDABLE WARMTH OR EPC FOR HOUSING ASSOCIATION" if + "AFFORDABLE WARMTH OR EPC FOR HOUSING ASSOCIATION" in master_data.columns else "AFFORDABLE WARMTH" ) # The columns are massively different - we take just a few unmatched_df = unmatched_df[ diff --git a/asset_list/app.py b/asset_list/app.py index 77d550f0..abce8d53 100644 --- a/asset_list/app.py +++ b/asset_list/app.py @@ -89,25 +89,25 @@ def app(): # - We want: fully insulated property (all wall types), EPC D or below (floors should be solid) # - Or the insulation required is loft/cavity (floors should be solid) - # Abri - data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Abri" - data_filename = "data for domna.xlsx" + # Sandwell + data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Sandwell" + data_filename = "Sandwell BC - Full Asset List MAIN.xlsx" sheet_name = "Sheet1" - postcode_column = 'post_code' - fulladdress_column = None - address1_column = "address##1" - address1_method = None - address_cols_to_concat = ["address##1", "address##2", "address##3"] + postcode_column = 'Post-Code' + fulladdress_column = "Address" + address1_column = None + address1_method = "house_number_extraction" + address_cols_to_concat = [] missing_postcodes_method = None - landlord_year_built = "build_date" + landlord_year_built = "Build-Date" landlord_os_uprn = None - landlord_property_type = "PropertyType" - landlord_built_form = "BuildForm" - landlord_wall_construction = "Wall Construction" + landlord_property_type = None + landlord_built_form = None + landlord_wall_construction = "ConstructionTypeName" landlord_roof_construction = None - landlord_heating_system = "HeatingType" + landlord_heating_system = "Heat Type" landlord_existing_pv = None - landlord_property_id = "place_ref" + landlord_property_id = "Place-Ref" landlord_sap = None outcomes_filename = None outcomes_sheetname = None @@ -117,46 +117,9 @@ def app(): outcomes_address = None master_filepaths = [] master_to_asset_list_filepath = None - phase = False + phase = True ecosurv_landlords = None - # Bromford - data_folder = ("/Users/khalimconn-kowlessar/Documents/hestia/Customers/Bromford/Apr 2025 Programme " - "Rebuild/Prepared data/") - data_filename = "asset_list.xlsx" - sheet_name = "Sheet1" - postcode_column = 'PostCode' - fulladdress_column = "FullAddress" - address1_column = None - address1_method = "house_number_extraction" - address_cols_to_concat = [] - missing_postcodes_method = None - landlord_year_built = "ConYear" - landlord_os_uprn = None - landlord_property_type = "AssetTypeDesc" - landlord_built_form = "PropTypeDesc" - landlord_wall_construction = "Construction type" - landlord_roof_construction = None - landlord_heating_system = "Heating Type" - landlord_existing_pv = None - landlord_property_id = "Asset" - landlord_sap = None - outcomes_filename = "outcomes.xlsx" - outcomes_sheetname = "Sheet1" - outcomes_postcode = "Postcode" - outcomes_houseno = "No" - outcomes_id = None - outcomes_address = "Address" - master_filepaths = [ - "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Bromford/Apr 2025 Programme Rebuild/Prepared data/ECO " - "3 submissions.csv", - "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Bromford/Apr 2025 Programme Rebuild/Prepared data/ECO " - "4 submissions.csv", - ] - master_to_asset_list_filepath = None - phase = False - ecosurv_landlords = "paul butler|bromford" - # Torus data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Torus/Phase 1" data_filename = "Torus Property Asset List - Phase 1.xlsx" @@ -660,7 +623,7 @@ def app(): epc_api_only = False force_retrieve_data = False skip = None # Used to skip already completed chunks - chunk_size = 5000 + chunk_size = 1000 filename = "Chunk {i}.csv" download_folder = os.path.join(data_folder, "Chunks") if not os.path.exists(download_folder): diff --git a/asset_list/mappings/built_form.py b/asset_list/mappings/built_form.py index 0e5cb3ec..f162e49e 100644 --- a/asset_list/mappings/built_form.py +++ b/asset_list/mappings/built_form.py @@ -150,5 +150,64 @@ BUILT_FORM_MAPPINGS = { 'Apartment': 'mid-floor', 'Flat Over Shop': 'top-floor', 'Flat Over Garage': 'top-floor', - 'Bridge Flat': 'mid-floor' + 'Bridge Flat': 'mid-floor', + 'House Mid Terrace': 'mid-terrace', + 'Semi-detached house': 'semi-detached', + 'House Semi Detached': 'semi-detached', + 'House Detached': 'detached', + 'Detached house': 'detached', + 'House End Terrace': 'end-terrace', + 'Flat Ground Floor Mr': 'ground floor', + 'Mais Flat 1St Fl Mr': 'mid-floor', + 'Top-floor maisonette': 'top-floor', + 'Flat 1St Warden Lr': 'mid-floor', + 'Cranwell': 'unknown', + 'No Fines': 'unknown', + 'Flat 1St Elderly Mr': 'mid-floor', + 'Stent Mod': 'unknown', + 'Mais Flat Grd Fl Mr': 'ground floor', + 'Flat 1St Floor Mr': 'mid-floor', + 'Mid-terrace house': 'mid-terrace', + 'Stent Unmod': 'unknown', + 'Flat 2Nd Floor Mr': 'mid-floor', + 'Studio Grd Warden Lr': 'ground floor', + 'Flat Grd Elderly Mr': 'ground floor', + 'Studio Fl Grd Eld Lr': 'ground floor', + 'Scottwood': 'unknown', + 'Airey': 'unknown', + 'Studio Flat 1Stfl Lr': 'mid-floor', + 'Studio Flat 1Stfl Mr': 'mid-floor', + 'Flat Grd Elderly Lr': 'ground floor', + 'Trusteel MKII': 'unknown', + 'No-Fines Concrete': 'unknown', + 'Crosswall': 'unknown', + 'Fidler': 'unknown', + 'Ground-floor maisonette': 'ground floor', + 'Studio Flat Grdfl Mr': 'ground floor', + 'Studio Flat Grd Lr': 'ground floor', + 'Studio Fl Grd Eld Mr': 'ground floor', + 'Bungalow Eld Person': 'unknown', + 'Cornish': 'unknown', + 'B.I.S.F.': 'unknown', + 'Flat 1St Floor Lr': 'mid-floor', + 'Mid-floor flat': 'mid-floor', + 'Bsit Bung Warden Sch': 'unknown', + 'Hawksley': 'unknown', + 'Orlit': 'unknown', + 'Mid-floor maisonette': 'mid-floor', + 'Ground-floor flat': 'ground floor', + 'Flat Grd Floor Lr': 'ground floor', + 'Studio 1St Warden Lr': 'mid-floor', + 'Flat Grd Warden Lr': 'ground floor', + 'end-terrace house': 'end-terrace', + 'Top-floor flat': 'top-floor', + 'End-terrace house': 'end-terrace', + 'Mais Flat 2Nd Fl Mr': 'mid-floor', + 'Flat 1St Elderly Lr': 'mid-floor', + 'Bfly Bung Bed Sitter': 'unknown', + 'Swedish': 'unknown', + 'Bungalow Semi Detach': 'semi-detached', + '4 Ext. Wall Flat': 'unknown', + '6 Ext. Wall Flat': 'unknown', + '5 Ext. Wall Flat': 'unknown' } diff --git a/asset_list/mappings/heating_systems.py b/asset_list/mappings/heating_systems.py index a3b3d451..b5cf500f 100644 --- a/asset_list/mappings/heating_systems.py +++ b/asset_list/mappings/heating_systems.py @@ -2,6 +2,7 @@ import numpy as np STANDARD_HEATING_SYSTEMS = { "gas combi boiler", + "gas boiler, radiators", "electric storage heaters", "district heating", "communal heating" @@ -25,7 +26,8 @@ STANDARD_HEATING_SYSTEMS = { 'unknown', "electric ceiling", "electric underfloor", - "no heating" + "no heating", + "non-electric underfloor" } HEATING_MAPPINGS = { @@ -204,7 +206,6 @@ HEATING_MAPPINGS = { 'No Heating Required Gas': 'unknown', 'Electric - Storage/Panel Heaters Gas': 'electric storage heaters', 'Electric - Storage/Panel Heaters Solid': 'electric storage heaters', - 'District Heat Network': 'district heating', 'Not Applicable': 'no heating', 'Not Responsible': 'unknown', @@ -212,5 +213,12 @@ HEATING_MAPPINGS = { 'Communal Electric': 'communal heating', 'Renewables (Air / Ground Source Pumps)': 'air source heat pump', 'Communal Renewable': 'air source heat pump', - + 'Room heaters/ Electric': 'room heaters', + 'Room heaters/ Gas': 'room heaters', + 'Radiator system': "gas boiler, radiators", + 'Drilled and filled': 'unknown', + 'Boiler/ underfloor': 'electric underfloor', + 'Storage system': "non-electric underfloor", + 'BOILER': 'gas combi boiler', + 'SPACE_HEATER': 'room heaters' } diff --git a/asset_list/mappings/property_type.py b/asset_list/mappings/property_type.py index fdeb457c..082bc443 100644 --- a/asset_list/mappings/property_type.py +++ b/asset_list/mappings/property_type.py @@ -183,6 +183,15 @@ PROPERTY_MAPPING = { 'Flat with Compulsory Garage': 'flat', 'Other': 'other', 'Maisonette with Compulsory Garage': 'maisonette', - 'Room in Shared Property': 'other' - + 'Room in Shared Property': 'other', + 'Bungalow Mid Terrace': 'bungalow', + '4 Ext. Wall Flat': 'flat', + '3 Ext. Wall Flat': 'flat', + 'Bungalow End Terrace': 'bungalow', + '6 Ext. Wall Flat': 'flat', + 'Bungalow Detached': 'bungalow', + 'Maisonette 3 Ext. Wall': 'maisonette', + 'Maisonette 2 Ext. Wall': 'maisonette', + '5 Ext. Wall Flat': 'flat', + 'Bungalow Semi Detached': 'bungalow' } diff --git a/asset_list/mappings/walls.py b/asset_list/mappings/walls.py index 96363fc0..ad09d067 100644 --- a/asset_list/mappings/walls.py +++ b/asset_list/mappings/walls.py @@ -165,7 +165,7 @@ WALL_CONSTRUCTION_MAPPINGS = { 'PRC': 'system built', 'Cross Wall': 'system built', 'Solid Wall': 'solid brick unknown insulation', - 'Traditional': 'other', + 'Traditional': 'unknown', 'Solid': 'solid brick unknown insulation', 'Wates no fines': 'system built', @@ -188,6 +188,29 @@ WALL_CONSTRUCTION_MAPPINGS = { 'BISF - Brit Iron & Steel Federation': 'system built', 'Steel Framed': 'system built', 'Timber Framed with confirmed Fire Stopping': 'timber frame unknown insulation', - 'Sipporex': 'system built' + 'Sipporex': 'system built', + + 'Wates': 'system built', + 'Bryants': 'system built', + 'Gregory (Crosswall)': 'system built', + 'Rsmit': 'system built', + 'Dorman Long': 'system built', + 'Tarmac': 'system built', + 'RBIS': 'system built', + 'Five Oaks': 'system built', + 'Not known': 'unknown', + 'Smiths': 'system built', + 'Kendrick': 'system built', + 'IDC': 'system built', + 'Wimpey (Part Brick)': 'system built', + 'Whitehall': 'system built', + 'Wimpey': 'system built', + 'Bison': 'system built', + 'Zinns': 'system built', + 'Bisf': 'system built', + 'Integer': 'system built', + 'Cornish': 'system built', + 'Rwate': 'system built', + 'Hill Presweld Steel': 'system built' } diff --git a/etl/customers/l_and_g/risk_matrix.py b/etl/customers/l_and_g/risk_matrix.py index b742a459..0b250039 100644 --- a/etl/customers/l_and_g/risk_matrix.py +++ b/etl/customers/l_and_g/risk_matrix.py @@ -19,7 +19,7 @@ def app(): "cavity_wall_insulation": 14.5, "ventilation": 350, "room_roof_insulation": 210, - "loft_insulation": 15, + "Loft insulation": 15, "internal_wall_insulation": 215, "external_wall_insulation": 298.35, "low_energy_lighting": 35, # per light @@ -39,7 +39,7 @@ def app(): "Ground Floor Flat" ] num_floors_map = { - "Semi Detached House": 2, + "Semi-detached house": 2, "Detached House": 2, "Mid Terrace House": 2, "Mid Floor Flat": 1, @@ -47,7 +47,7 @@ def app(): "Ground Floor Flat": 1 } built_form_map = { - "Semi Detached House": "Semi-Detached", + "Semi-detached house": "Semi-Detached", "Detached House": "Detached", "Mid Terrace House": "Mid Terrace", "Mid Floor Flat": "Semi-Detached", @@ -55,7 +55,7 @@ def app(): "Ground Floor Flat": "Semi-Detached" } lighting_count = { - "Semi Detached House": 15, + "Semi-detached house": 15, "Detached House": 19, "Mid Terrace House": 12, "Mid Floor Flat": 10, @@ -201,3 +201,59 @@ def app(): ) combination_costs = pd.DataFrame(combination_costs) + + contingency = 0.26 + + epr_data = pd.read_excel( + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/L&G/Risk Matrix/EPR Data.xlsx", header=1 + ) + epr_data["Measure added"].value_counts() + epr_data["row_id"] = epr_data.index + # We need to calculate the costs + cost_data = [] + for _, row in epr_data.iterrows(): + epc = row["EPC"][0] + sap = int(row["EPC"][1:]) + + n_floors = num_floors_map[row["Property Type"]] + bf = built_form_map[row["Property Type"]] + pt = "House" if "flat" not in row["Property Type"].lower() else "Flat" + # Model the home as a box + ground_floor_area = row["area"] / n_floors + perimeter = np.sqrt(ground_floor_area) * 4 + + # This is the amount of insulation required + external_wall_area = estimate_external_wall_area( + num_floors=n_floors, + floor_height=2.5, + perimeter=perimeter, + built_form=bf + ) + + n_rooms = np.floor(row["area"] / 15) + + n_windows = estimate_windows( + property_type=pt, + built_form=bf, + construction_age_band="", + floor_area=row["area"], + number_habitable_rooms=n_rooms + ) + measure = row["Measure added"] + unit_cost = pricing_matrix[measure] + + if pd.isnull(row["Measure added"]): + cost = None + elif row["Measure added"] == "Loft insulation": + cost = unit_cost * ground_floor_area + else: + raise Exception() + + cost_data.append( + { + "row_id": row["row_id"], + "epc": epc, + "sap": sap, + "cost": cost + } + )