diff --git a/.idea/Model.iml b/.idea/Model.iml
index 96ad7a95..df6c4faa 100644
--- a/.idea/Model.iml
+++ b/.idea/Model.iml
@@ -7,7 +7,7 @@
-
+
diff --git a/asset_list/AssetList.py b/asset_list/AssetList.py
index 3b5535d5..18e202b6 100644
--- a/asset_list/AssetList.py
+++ b/asset_list/AssetList.py
@@ -444,6 +444,25 @@ class AssetList:
self.standardised_asset_list[self.address1_colname].copy()
)
+ # Handle the case where the property type column and built form are missing
+ if self.landlord_property_type is None and self.landlord_built_form is None:
+ if "Archetype" in self.raw_asset_list.columns:
+ # We use the non-intrusives as our property type and built form
+ self.landlord_property_type = self.STANDARD_PROPERTY_TYPE
+ self.landlord_built_form = self.STANDARD_BUILT_FORM
+ self.standardised_asset_list[self.landlord_property_type] = (
+ self.standardised_asset_list["Archetype"].copy()
+ )
+ self.standardised_asset_list[self.landlord_built_form] = (
+ self.standardised_asset_list["Archetype"].copy()
+ )
+ else:
+ # We use the EPC data as our property type and built form
+ self.landlord_property_type = self.STANDARD_PROPERTY_TYPE
+ self.landlord_built_form = self.STANDARD_BUILT_FORM
+ self.standardised_asset_list[self.landlord_property_type] = None
+ self.standardised_asset_list[self.landlord_built_form] = None
+
# Handle the case where the property type column is the same as the built type
if self.landlord_property_type == self.landlord_built_form:
self.landlord_built_form = self.STANDARD_BUILT_FORM
@@ -1342,7 +1361,7 @@ class AssetList:
)
self.standardised_asset_list["solar_landlord_data_indicates_needs_heating_upgrade"] = (
self.standardised_asset_list[self.STANDARD_HEATING_SYSTEM].isin(
- ["electric storage heaters", "room heaters", "electric radiators", "no heating"]
+ ["electric storage heaters", "room heaters", "electric radiators", "no heating", "electric fuel"]
)
)
@@ -1651,7 +1670,7 @@ class AssetList:
"SAP Category"],
self.standardised_asset_list["cavity_reason"]
)
- else:
+ elif self.non_intrusives_present:
self.standardised_asset_list["cavity_reason"] = np.where(
(
self.standardised_asset_list["epc_indicates_empty_cavity"] &
@@ -1675,6 +1694,16 @@ class AssetList:
"SAP Category"],
self.standardised_asset_list["cavity_reason"]
)
+ else:
+ self.standardised_asset_list["cavity_reason"] = np.where(
+ (
+ self.standardised_asset_list["epc_indicates_empty_cavity"] &
+ ~self.standardised_asset_list["non_intrusive_indicates_empty_cavity"] &
+ pd.isnull(self.standardised_asset_list["cavity_reason"])
+ ),
+ "EPC Shows Empty Cavity: " + self.standardised_asset_list["SAP Category"],
+ self.standardised_asset_list["cavity_reason"]
+ )
self.standardised_asset_list["cavity_reason"] = np.where(
(
@@ -1716,17 +1745,18 @@ class AssetList:
self.standardised_asset_list["solar_reason"] = None
# Map of variables and fill values for the solar_reason variable
+ # ordering of this map is important, where we flag our prioritised work types first
solar_reason_map = {
"solar_eligible": "Solar Eligible: ",
+ "solar_eligible_solid_wall_uninsulated": "Solar Eligible, Solid Wall Uninsulated, EPC E or Below: ",
"solar_eligible_needs_heating_upgrade": (
"Solar Eligible, Needs Heating Upgrade: "
- ),
- "solar_eligible_solid_wall_uninsulated": "Solar Eligible, Solid Wall Uninsulated, EPC E or Below: ",
+ )
}
for variable, reason in solar_reason_map.items():
self.standardised_asset_list["solar_reason"] = np.where(
- self.standardised_asset_list[variable],
+ self.standardised_asset_list[variable] & pd.isnull(self.standardised_asset_list["solar_reason"]),
reason + self.standardised_asset_list["SAP Category"],
self.standardised_asset_list["solar_reason"]
)
@@ -2401,6 +2431,7 @@ class AssetList:
master_data = pd.read_csv(filepath)
# Strip columns
master_data.columns = [c.strip() for c in master_data.columns]
+ master_data.columns = [re.sub(r'\s+', ' ', c) for c in master_data.columns]
if not id_map.empty:
master_data = master_data.merge(
@@ -2537,8 +2568,8 @@ class AssetList:
]
scheme_col = (
- "AFFORDABLE WARMTH OR EPC FOR HOUSING ASSOCIATION" if
- "AFFORDABLE WARMTH OR EPC FOR HOUSING ASSOCIATION" in master_data.columns else "AFFORDABLE WARMTH"
+ "AFFORDABLE WARMTH OR EPC FOR HOUSING ASSOCIATION" if
+ "AFFORDABLE WARMTH OR EPC FOR HOUSING ASSOCIATION" in master_data.columns else "AFFORDABLE WARMTH"
)
# The columns are massively different - we take just a few
unmatched_df = unmatched_df[
diff --git a/asset_list/app.py b/asset_list/app.py
index e8388100..37e687fc 100644
--- a/asset_list/app.py
+++ b/asset_list/app.py
@@ -89,43 +89,6 @@ def app():
# - We want: fully insulated property (all wall types), EPC D or below (floors should be solid)
# - Or the insulation required is loft/cavity (floors should be solid)
- # Bromford
- data_folder = ("/Users/khalimconn-kowlessar/Documents/hestia/Customers/Bromford/Apr 2025 Programme "
- "Rebuild/Prepared data/")
- data_filename = "asset_list.xlsx"
- sheet_name = "Sheet1"
- postcode_column = 'PostCode'
- fulladdress_column = "FullAddress"
- address1_column = None
- address1_method = "house_number_extraction"
- address_cols_to_concat = []
- missing_postcodes_method = None
- landlord_year_built = "ConYear"
- landlord_os_uprn = None
- landlord_property_type = "AssetTypeDesc"
- landlord_built_form = "PropTypeDesc"
- landlord_wall_construction = "Construction type"
- landlord_roof_construction = None
- landlord_heating_system = "Heating Type"
- landlord_existing_pv = None
- landlord_property_id = "Asset"
- landlord_sap = None
- outcomes_filename = "outcomes.xlsx"
- outcomes_sheetname = "Sheet1"
- outcomes_postcode = "Postcode"
- outcomes_houseno = "No"
- outcomes_id = None
- outcomes_address = "Address"
- master_filepaths = [
- "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Bromford/Apr 2025 Programme Rebuild/Prepared data/ECO "
- "3 submissions.csv",
- "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Bromford/Apr 2025 Programme Rebuild/Prepared data/ECO "
- "4 submissions.csv",
- ]
- master_to_asset_list_filepath = None
- phase = False
- ecosurv_landlords = "paul butler|bromford"
-
# Torus
data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Torus/Phase 1"
data_filename = "Torus Property Asset List - Phase 1.xlsx"
@@ -156,33 +119,6 @@ def app():
master_to_asset_list_filepath = None
phase = True
- # Ealing - houses
- data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Ealing"
- data_filename = "Ealing_rechecked_cleaned_05042025.csv"
- sheet_name = None
- postcode_column = 'Postcode'
- fulladdress_column = "Address"
- address1_column = None
- address1_method = "house_number_extraction"
- address_cols_to_concat = []
- missing_postcodes_method = None
- landlord_year_built = "Year Built"
- landlord_os_uprn = None
- landlord_property_type = "Property Type Code"
- landlord_built_form = None
- landlord_wall_construction = None
- landlord_heating_system = None
- landlord_existing_pv = None
- landlord_property_id = "Property ref"
- outcomes_filename = None
- outcomes_sheetname = None
- outcomes_postcode = None
- outcomes_houseno = None
- outcomes_id = None
- outcomes_address = None
- master_filepaths = []
- master_to_asset_list_filepath = None
-
# Southern Midlands
data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Southern/Midlands Properties - Apr 2025"
data_filename = "Southern Housing Midlands Property List - combined.xlsx"
@@ -210,67 +146,6 @@ def app():
master_filepaths = []
master_to_asset_list_filepath = None
- # Live West (2018 Asset list)
- data_folder = (
- "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Livewest/Programme Update - March 2025/2018 Asset List"
- )
- data_filename = "LIVEWEST STOCK - 23rd October 2018.xlsx"
- sheet_name = "Assets"
- postcode_column = 'Postcode'
- fulladdress_column = "Address"
- address1_column = None
- address1_method = "house_number_extraction"
- address_cols_to_concat = []
- missing_postcodes_method = None
- landlord_year_built = "Build Year"
- landlord_os_uprn = None
- landlord_property_type = "Property Archetype"
- landlord_built_form = None
- landlord_wall_construction = None
- landlord_heating_system = "Heating Fuel Type"
- landlord_existing_pv = None
- landlord_property_id = "Uprn - DO NOT DELETE"
- outcomes_filename = "RT - LiveWest.xlsx"
- outcomes_sheetname = "Feedback"
- outcomes_postcode = "Poscode"
- outcomes_houseno = "No."
- outcomes_id = "UPRN"
- master_filepaths = [
- "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Livewest/Programme Update - March 2025/Rolling Master "
- "- redacted for analysis/CAVITY-Table 1.csv"
- ]
- master_to_asset_list_filepath = None
-
- # Live West (South West asset list)
- data_folder = ("/Users/khalimconn-kowlessar/Documents/hestia/Customers/Livewest/Programme Update - March "
- "2025/Livewest Asset List (Original) - csv")
- data_filename = "Report-Table 1.csv"
- sheet_name = None
- postcode_column = 'Postcode'
- fulladdress_column = "T1_Address"
- address1_column = None
- address1_method = "house_number_extraction"
- address_cols_to_concat = []
- missing_postcodes_method = None
- landlord_year_built = "Build Yr"
- landlord_os_uprn = None
- landlord_property_type = "T1_AssetType"
- landlord_built_form = "T1_AssetType"
- landlord_wall_construction = "Wall Type Cavity"
- landlord_heating_system = "Heating Fuel"
- landlord_existing_pv = None
- landlord_property_id = "T1_UPRN"
- outcomes_filename = "RT - LiveWest.xlsx"
- outcomes_sheetname = "Feedback"
- outcomes_postcode = "Poscode"
- outcomes_houseno = "No."
- outcomes_id = "UPRN"
- master_filepaths = [
- "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Livewest/Programme Update - March 2025/Rolling Master "
- "- redacted for analysis/CAVITY-Table 1.csv"
- ]
- master_to_asset_list_filepath = None
-
# PFP London
data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Places For People/London"
data_filename = "PFP AREAS SURROUNDING LONDON - JAY, RUTH & LANE.xlsx"
@@ -459,6 +334,8 @@ def app():
landlord_heating_system = "Heat Source"
landlord_existing_pv = "PV (Y/N)"
landlord_property_id = "Place ref"
+ landlord_roof_construction = None
+ landlord_sap = None
outcomes_filename = None
outcomes_sheetname = None
outcomes_postcode = None
@@ -466,6 +343,9 @@ def app():
master_filepaths = []
master_to_asset_list_filepath = None
outcomes_id = None
+ outcomes_address = None
+ phase = False
+ ecosurv_landlords = None
# For ACIS - programme re-build
# data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/ACIS/ACIS Full Programme Review March 2025"
@@ -483,17 +363,23 @@ def app():
# landlord_property_type = "Property type"
# landlord_built_form = None
# landlord_wall_construction = "Wall Constuction"
+ # landlord_roof_construction = None
+ # landlord_sap = None
# landlord_heating_system = "Heating"
# landlord_existing_pv = None
# outcomes_filename = "ACIS Group - 25.11.2024 - outcomes.xlsx"
# outcomes_sheetname = "Feedback"
# outcomes_postcode = "Postcode"
+ # outcomes_address = "Address"
# outcomes_houseno = "No"
+ # outcomes_id = None
# master_filepaths = [
# os.path.join(data_folder, "ECO 3 -Table 1.csv"),
# os.path.join(data_folder, "ECO 4 -Table 1.csv"),
# ]
# master_to_asset_list_filepath = None
+ # phase = False
+ # ecosurv_landlords = None
# For plus dane
data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Plus Dane"
@@ -697,6 +583,9 @@ def app():
epc_data.append(csv_data)
epc_df = pd.concat(epc_data)
+ if "estimated" not in epc_df.columns:
+ epc_df["estimated"] = False
+
epc_df["estimated"] = epc_df["estimated"].fillna(False)
# We expand out the recommendations
@@ -804,98 +693,6 @@ def app():
asset_list.flat_analysis()
- ################################################################
- # WESTWARD - comparison between Kieran's method & automated
- ################################################################
-
- # Check 1)
- cavity_fills = pd.read_excel(
- os.path.join(data_folder, "WESTWARD - Route March Prep.xlsx"),
- sheet_name="Straight Fill"
- )
- cavity_fills = cavity_fills.merge(
- asset_list.standardised_asset_list[
- [asset_list.STANDARD_LANDLORD_PROPERTY_ID, "cavity_reason"]
- ],
- how="left",
- left_on=asset_list.landlord_property_id,
- right_on=asset_list.STANDARD_LANDLORD_PROPERTY_ID
- )
- cavity_fills["cavity_reason"] = cavity_fills["cavity_reason"].fillna("Not identified")
- print(cavity_fills["cavity_reason"].value_counts())
- # Didn't identify 3 properties because they're bedsits
- # 4 properties were identified, not based on the non-intrusives but instead because
- # Westward said they were built in 2003/2007. Have adjusted this to use the age from the
- # epc as well, as EPC says 1975 and they look like 1975 properties
- # 37 properties flagged as already having solar - these are all because the landlord said they have solar
- # e.g.
- # https://earth.google.com/web/search/11+Winsland+Avenue+TOTNES+TQ9+5FT/@50.43354465,-3.71318276,46.57468503a,
- # 59.14004365d,35y,0h,0t,
- # 0r/data=CpABGmISXAolMHg0ODZkMWQxOGE4NWRiZjdkOjB4YjBhM2E5M2Q3YWVlMWEwYhlZYgp7fzdJQCHFfC9027QNwCohMTEgV2luc2xhbmQgQXZlbnVlIFRPVE5FUyBUUTkgNUZUGAIgASImCiQJbxsQEoo3SUARXQcp_HE3SUAZBmiZGJ6yDcAhCA0fqq63DcBCAggBOgMKATBCAggASg0I____________ARAA
- # https://earth.google.com/web/search/15+St+Anne%27s+Ct,+Newton+Abbot+TQ12+1TL/@50.53068337,-3.61611128,
- # 11.74908956a,135.73212429d,35y,0h,0t,
- # 0r/data=CpUBGmcSYQolMHg0ODZkMDVkMjFhODhjZjgxOjB4MjBmMzE2Zjc3MGI2NGMwYxlCxHLw8UNJQCFZqyzALe4MwComMTUgU3QgQW5uZSdzIEN0LCBOZXd0b24gQWJib3QgVFExMiAxVEwYAiABIiYKJAm-r6U2iDdJQBHS5ICRdDdJQBmYGVpmiLINwCG8wcrtqbYNwEICCAE6AwoBMEICCABKDQj___________8BEAA
-
- # Check 2)
- cavity_fills_with_solar = pd.read_excel(
- os.path.join(data_folder, "WESTWARD - Route March Prep.xlsx"),
- sheet_name="Solar PV - Straight Fill"
- )
- cavity_fills_with_solar = cavity_fills_with_solar.merge(
- asset_list.standardised_asset_list[
- [asset_list.STANDARD_LANDLORD_PROPERTY_ID, "cavity_reason"]
- ],
- how="left",
- left_on=asset_list.landlord_property_id,
- right_on=asset_list.STANDARD_LANDLORD_PROPERTY_ID
- )
- cavity_fills_with_solar["cavity_reason"] = cavity_fills_with_solar["cavity_reason"].fillna("Not identified")
- print(cavity_fills_with_solar["cavity_reason"].value_counts())
- # 203 properties total
- # 140 properties were flagged up based on non-intrusives (Non-Intrusive Data Showed Empty Cavity)
- # 63 property already has solar
-
- # Check 3) RDF
- rdf = pd.read_excel(
- os.path.join(data_folder, "WESTWARD - Route March Prep.xlsx"),
- sheet_name="RDF CIGA checks"
- )
- rdf = rdf.merge(
- asset_list.standardised_asset_list[
- [asset_list.STANDARD_LANDLORD_PROPERTY_ID, "cavity_reason", "solar_reason"]
- ],
- how="left",
- left_on=asset_list.landlord_property_id,
- right_on=asset_list.STANDARD_LANDLORD_PROPERTY_ID
- )
- rdf["cavity_reason"] = rdf["cavity_reason"].fillna("Not identified")
- print(rdf["cavity_reason"].value_counts())
- # 264 properties are not identified, 261 of which are due to the fact they contain materials
- # The other 3 were determined to be eligible for solar instead
- # Many of these units that were identified for rdf works could be solar jobs
-
- rdf_with_solar = pd.read_excel(
- os.path.join(data_folder, "WESTWARD - Route March Prep.xlsx"),
- sheet_name="Solar PV - RDF CIGA Checks"
- )
- rdf_with_solar = rdf_with_solar.merge(
- asset_list.standardised_asset_list[
- [asset_list.STANDARD_LANDLORD_PROPERTY_ID, "cavity_reason", "solar_reason"]
- ],
- how="left",
- left_on=asset_list.landlord_property_id,
- right_on=asset_list.STANDARD_LANDLORD_PROPERTY_ID
- )
- rdf_with_solar["cavity_reason"] = rdf_with_solar["cavity_reason"].fillna("Not identified")
- rdf_with_solar["cavity_reason"].value_counts()
-
- # All others identified - some flagged as empties due to EPC or landlord data suggesting as much
- # 5 not identified due to containing COMPACTED BEAD
-
- asset_list.standardised_asset_list = asset_list.standardised_asset_list[
- asset_list.standardised_asset_list[asset_list.landlord_property_id]
- ]
-
asset_list.load_contact_details(
local_filepath=os.path.join(data_folder, "Full property list wth D&V report V look up 12.2.25.xlsx"),
sheet_name="Report 1",
diff --git a/asset_list/mappings/built_form.py b/asset_list/mappings/built_form.py
index e103f794..1d0aecf5 100644
--- a/asset_list/mappings/built_form.py
+++ b/asset_list/mappings/built_form.py
@@ -143,6 +143,74 @@ BUILT_FORM_MAPPINGS = {
'Sixth Floor': 'top-floor',
'Sheltered Bung': 'semi-detached',
'Guest': 'unknown',
- 'Fifth Floor': 'mid-floor'
-
+ 'Fifth Floor': 'mid-floor',
+ 'Flat Within Block': 'mid-floor',
+ 'Coach House with Garage': 'detached',
+ 'Over Garage House': 'top-floor',
+ 'Apartment': 'mid-floor',
+ 'Flat Over Shop': 'top-floor',
+ 'Flat Over Garage': 'top-floor',
+ 'Bridge Flat': 'mid-floor',
+ 'House Mid Terrace': 'mid-terrace',
+ 'Semi-detached house': 'semi-detached',
+ 'House Semi Detached': 'semi-detached',
+ 'House Detached': 'detached',
+ 'Detached house': 'detached',
+ 'House End Terrace': 'end-terrace',
+ 'Flat Ground Floor Mr': 'ground floor',
+ 'Mais Flat 1St Fl Mr': 'mid-floor',
+ 'Top-floor maisonette': 'top-floor',
+ 'Flat 1St Warden Lr': 'mid-floor',
+ 'Cranwell': 'unknown',
+ 'No Fines': 'unknown',
+ 'Flat 1St Elderly Mr': 'mid-floor',
+ 'Stent Mod': 'unknown',
+ 'Mais Flat Grd Fl Mr': 'ground floor',
+ 'Flat 1St Floor Mr': 'mid-floor',
+ 'Mid-terrace house': 'mid-terrace',
+ 'Stent Unmod': 'unknown',
+ 'Flat 2Nd Floor Mr': 'mid-floor',
+ 'Studio Grd Warden Lr': 'ground floor',
+ 'Flat Grd Elderly Mr': 'ground floor',
+ 'Studio Fl Grd Eld Lr': 'ground floor',
+ 'Scottwood': 'unknown',
+ 'Airey': 'unknown',
+ 'Studio Flat 1Stfl Lr': 'mid-floor',
+ 'Studio Flat 1Stfl Mr': 'mid-floor',
+ 'Flat Grd Elderly Lr': 'ground floor',
+ 'Trusteel MKII': 'unknown',
+ 'No-Fines Concrete': 'unknown',
+ 'Crosswall': 'unknown',
+ 'Fidler': 'unknown',
+ 'Ground-floor maisonette': 'ground floor',
+ 'Studio Flat Grdfl Mr': 'ground floor',
+ 'Studio Flat Grd Lr': 'ground floor',
+ 'Studio Fl Grd Eld Mr': 'ground floor',
+ 'Bungalow Eld Person': 'unknown',
+ 'Cornish': 'unknown',
+ 'B.I.S.F.': 'unknown',
+ 'Flat 1St Floor Lr': 'mid-floor',
+ 'Mid-floor flat': 'mid-floor',
+ 'Bsit Bung Warden Sch': 'unknown',
+ 'Hawksley': 'unknown',
+ 'Orlit': 'unknown',
+ 'Mid-floor maisonette': 'mid-floor',
+ 'Ground-floor flat': 'ground floor',
+ 'Flat Grd Floor Lr': 'ground floor',
+ 'Studio 1St Warden Lr': 'mid-floor',
+ 'Flat Grd Warden Lr': 'ground floor',
+ 'end-terrace house': 'end-terrace',
+ 'Top-floor flat': 'top-floor',
+ 'End-terrace house': 'end-terrace',
+ 'Mais Flat 2Nd Fl Mr': 'mid-floor',
+ 'Flat 1St Elderly Lr': 'mid-floor',
+ 'Bfly Bung Bed Sitter': 'unknown',
+ 'Swedish': 'unknown',
+ 'Bungalow Semi Detach': 'semi-detached',
+ '4 Ext. Wall Flat': 'unknown',
+ '6 Ext. Wall Flat': 'unknown',
+ '5 Ext. Wall Flat': 'unknown',
+ 'Unknown': 'unknown',
+ 'Enclosed mid-terrace': 'mid-terrace',
+ 'Enclosed end-terrace': 'end-terrace'
}
diff --git a/asset_list/mappings/heating_systems.py b/asset_list/mappings/heating_systems.py
index 7f2f81f2..b5cf500f 100644
--- a/asset_list/mappings/heating_systems.py
+++ b/asset_list/mappings/heating_systems.py
@@ -2,8 +2,10 @@ import numpy as np
STANDARD_HEATING_SYSTEMS = {
"gas combi boiler",
+ "gas boiler, radiators",
"electric storage heaters",
"district heating",
+ "communal heating"
"gas condensing boiler",
"oil boiler",
"gas condensing combi",
@@ -24,7 +26,8 @@ STANDARD_HEATING_SYSTEMS = {
'unknown',
"electric ceiling",
"electric underfloor",
- "no heating"
+ "no heating",
+ "non-electric underfloor"
}
HEATING_MAPPINGS = {
@@ -202,5 +205,20 @@ HEATING_MAPPINGS = {
'Wet - Underfloor Solar': 'other',
'No Heating Required Gas': 'unknown',
'Electric - Storage/Panel Heaters Gas': 'electric storage heaters',
- 'Electric - Storage/Panel Heaters Solid': 'electric storage heaters'
+ 'Electric - Storage/Panel Heaters Solid': 'electric storage heaters',
+ 'District Heat Network': 'district heating',
+ 'Not Applicable': 'no heating',
+ 'Not Responsible': 'unknown',
+ 'Communal Oil': 'communal heating',
+ 'Communal Electric': 'communal heating',
+ 'Renewables (Air / Ground Source Pumps)': 'air source heat pump',
+ 'Communal Renewable': 'air source heat pump',
+ 'Room heaters/ Electric': 'room heaters',
+ 'Room heaters/ Gas': 'room heaters',
+ 'Radiator system': "gas boiler, radiators",
+ 'Drilled and filled': 'unknown',
+ 'Boiler/ underfloor': 'electric underfloor',
+ 'Storage system': "non-electric underfloor",
+ 'BOILER': 'gas combi boiler',
+ 'SPACE_HEATER': 'room heaters'
}
diff --git a/asset_list/mappings/property_type.py b/asset_list/mappings/property_type.py
index dc8dbf21..f01ab5eb 100644
--- a/asset_list/mappings/property_type.py
+++ b/asset_list/mappings/property_type.py
@@ -178,5 +178,21 @@ PROPERTY_MAPPING = {
'Parking Space': 'other',
'Community Centre': 'other',
'Communal Facility': 'other',
- 'Semi': 'house'
+ 'Semi': 'house',
+ 'House with Compulsory Garage': 'house',
+ 'Flat with Compulsory Garage': 'flat',
+ 'Other': 'other',
+ 'Maisonette with Compulsory Garage': 'maisonette',
+ 'Room in Shared Property': 'other',
+ 'Bungalow Mid Terrace': 'bungalow',
+ '4 Ext. Wall Flat': 'flat',
+ '3 Ext. Wall Flat': 'flat',
+ 'Bungalow End Terrace': 'bungalow',
+ '6 Ext. Wall Flat': 'flat',
+ 'Bungalow Detached': 'bungalow',
+ 'Maisonette 3 Ext. Wall': 'maisonette',
+ 'Maisonette 2 Ext. Wall': 'maisonette',
+ '5 Ext. Wall Flat': 'flat',
+ 'Bungalow Semi Detached': 'bungalow',
+ 'COMINT': 'unknown'
}
diff --git a/asset_list/mappings/walls.py b/asset_list/mappings/walls.py
index c327338a..1fb8cb79 100644
--- a/asset_list/mappings/walls.py
+++ b/asset_list/mappings/walls.py
@@ -158,7 +158,6 @@ WALL_CONSTRUCTION_MAPPINGS = {
'2017 onwards': 'new build - average thermal transmittance',
'ND (inferred)': 'unknown',
'Flat / maisonette': 'other',
-
'Other': 'other',
'Timber Frame': 'timber frame unknown insulation',
'Cavity Wall': 'cavity unknown insulation',
@@ -166,5 +165,59 @@ WALL_CONSTRUCTION_MAPPINGS = {
'PRC': 'system built',
'Cross Wall': 'system built',
'Solid Wall': 'solid brick unknown insulation',
- 'Traditional': 'other'
+ 'Traditional': 'unknown',
+
+ 'Solid': 'solid brick unknown insulation',
+ 'Wates no fines': 'system built',
+ 'Concrete Frame': 'system built',
+ 'PRCWATES': 'system built',
+ 'Refurbished Cornish': 'system built',
+ 'Bailey Stratton': 'other',
+ 'Refurbished Reema': 'system built',
+ 'PRCREEMA': 'system built',
+ 'Trustsell Type': 'system built',
+ 'Petra Nissan': 'unknown',
+ 'Reinstated Airey': 'system built',
+ 'Refurbished Airey': 'system built',
+ # From Abri- slightly unclear on types but not a large portion of the data
+ 'No Fines Type': 'system built',
+ 'Refurbished Unity': 'system built',
+ 'Timber Framed': 'timber frame unknown insulation',
+ 'Refurbished Woolaway': 'system built',
+ 'Modern Methods of Construction': 'other',
+ 'BISF - Brit Iron & Steel Federation': 'system built',
+ 'Steel Framed': 'system built',
+ 'Timber Framed with confirmed Fire Stopping': 'timber frame unknown insulation',
+ 'Sipporex': 'system built',
+
+ 'Wates': 'system built',
+ 'Bryants': 'system built',
+ 'Gregory (Crosswall)': 'system built',
+ 'Rsmit': 'system built',
+ 'Dorman Long': 'system built',
+ 'Tarmac': 'system built',
+ 'RBIS': 'system built',
+ 'Five Oaks': 'system built',
+ 'Not known': 'unknown',
+ 'Smiths': 'system built',
+ 'Kendrick': 'system built',
+ 'IDC': 'system built',
+ 'Wimpey (Part Brick)': 'system built',
+ 'Whitehall': 'system built',
+ 'Wimpey': 'system built',
+ 'Bison': 'system built',
+ 'Zinns': 'system built',
+ 'Bisf': 'system built',
+ 'Integer': 'system built',
+ 'Cornish': 'system built',
+ 'Rwate': 'system built',
+ 'Hill Presweld Steel': 'system built',
+
+ 'Cavity Filled Cavity': 'filled cavity',
+ 'Cavity Unknown': 'cavity unknown insulation',
+ 'Cavity Filled Cavity (internal)': 'filled cavity',
+ '': 'unknown',
+ 'Cavity Internal Insulation': 'filled cavity',
+ 'Cavity As Built': "uninsulated cavity"
+
}
diff --git a/backend/Property.py b/backend/Property.py
index 52e8c213..91c1265a 100644
--- a/backend/Property.py
+++ b/backend/Property.py
@@ -1261,6 +1261,11 @@ class Property:
# may be installed such that they are not visible from the street
return False
+ if (self.data["property-type"] in ["House", "Bungalow"]) and (
+ not pd.isnull(self.roof["thermal_transmittance"])
+ ):
+ return True
+
is_valid_property_type = self.data["property-type"] in ["House", "Bungalow", "Maisonette"]
is_valid_roof_type = (
self.roof["is_flat"] or self.roof["is_pitched"] or self.roof["is_roof_room"]
diff --git a/backend/engine/engine.py b/backend/engine/engine.py
index 7c99360d..faa1ed94 100644
--- a/backend/engine/engine.py
+++ b/backend/engine/engine.py
@@ -411,7 +411,7 @@ def get_funding_data():
async def model_engine(body: PlanTriggerRequest):
- logger.info("Model Engine triggered with body: %s", body.model_dump_json())
+ logger.info("Model Engine triggered with body: %s", json.loads(body.model_dump_json()))
logger.info("Connecting to db")
session = sessionmaker(bind=db_engine)()
diff --git a/etl/customers/l_and_g/risk_matrix.py b/etl/customers/l_and_g/risk_matrix.py
new file mode 100644
index 00000000..bc1bc952
--- /dev/null
+++ b/etl/customers/l_and_g/risk_matrix.py
@@ -0,0 +1,323 @@
+from itertools import product
+from recommendations.recommendation_utils import estimate_external_wall_area, estimate_windows
+
+import numpy as np
+
+import pandas as pd
+
+
+def app():
+ # Given a combination of variables, this code attempts to break down the costs of works to achieve upgrade
+ # targets
+
+ upgrade_path = [
+ "wall_insulation", "roof_insulation", "ventilation", "windows", "low_energy_lighting",
+ "heating", "solar"
+ ]
+
+ pricing_matrix = {
+ "Cavity wall insulation": 14.5,
+ "ventilation": 350,
+ "Room Roof Insulation": 210,
+ "Loft insulation": 15,
+ "Internal wall insulation": 215,
+ "External wall insulation": 298.35,
+ "Solid wall insulation": 215,
+ "LEDs": 35, # per light
+ "Flat Roof Insulation": 195,
+ "Double Glazing": 1140,
+ "secondary_glazing": 970,
+ "5kw ASHP feeding heating & Hot water (dual tariff)": 14738,
+ "11.2kw ASHP feeding heating & Hot water (dual tariff)": 16541,
+ "3 kWp Solar PV": 4552.32,
+ "4 kWp Solar PV": 4892.8,
+ "4.3 kWp Solar PV": 4961.44,
+ "4.8 kWp Solar PV": 5414,
+ '5 kWp Solar PV': 5509.71,
+ '5.5 kWp Solar PV': 5631.92,
+ "HHRSH (dual tariff)": 1000, # per heater
+ "Suspended floor insulation": 75
+ }
+ dwelling_types = [
+ "Semi Detached House",
+ "Detached house",
+ "Mid Terrace house",
+ "Mid Floor Flat",
+ "Top Floor Flat",
+ "Ground Floor Flat"
+ ]
+ num_floors_map = {
+ "Semi-detached house": 2,
+ "Detached house": 2,
+ "Mid Terrace house": 2,
+ "Mid Floor Flat": 1,
+ "Top Floor Flat": 1,
+ "Ground Floor Flat": 1
+ }
+ built_form_map = {
+ "Semi-detached house": "Semi-Detached",
+ "Detached house": "Detached",
+ "Mid Terrace house": "Mid Terrace",
+ "Mid Floor Flat": "Semi-Detached",
+ "Top Floor Flat": "Semi-Detached",
+ "Ground Floor Flat": "Semi-Detached"
+ }
+ lighting_count = {
+ "Semi-detached house": 15,
+ "Detached house": 19,
+ "Mid Terrace house": 12,
+ "Mid Floor Flat": 10,
+ "Top Floor Flat": 10,
+ "Ground Floor Flat": 10
+ }
+
+ # If we have a flat, we won't use the 199m2 floor area
+ floor_areas = [73, 97, 199]
+ # We remove age bracket, as we ended up with 360 combinations
+ # age_brackets = ["1945-1970", "1971-2002", "Post 2002"]
+ wall_type = ["cavity", "non-cavity"]
+ roof_type = ["pitched", "other"]
+ planning_constraints = [True, False]
+
+ # This is the list of all combinations of the above variables
+ combinations_untrimmed = product(
+ *[
+ dwelling_types, floor_areas, wall_type, roof_type, planning_constraints
+ ]
+ )
+
+ # TODO: Possibly need to add an additional cost for immersion hot water
+ combinations = []
+ for comb in combinations_untrimmed:
+ if "Flat" in comb[0] and comb[1] == 199:
+ continue
+
+ # If we have a flat, not too much difference if it's in a conservation area or not
+ if "Flat" in comb[0] and comb[4] is True:
+ continue
+ combinations.append(comb)
+
+ risk_matrix = []
+ for combination in combinations:
+ n_floors = num_floors_map[combination[0]]
+ bf = built_form_map[combination[0]]
+ pt = "House" if "Flat" not in combination[0] else "Flat"
+ # Model the home as a box
+ ground_floor_area = combination[1] / n_floors
+ perimeter = np.sqrt(ground_floor_area) * 4
+
+ # This is the amount of insulation required
+ external_wall_area = estimate_external_wall_area(
+ num_floors=n_floors,
+ floor_height=2.5,
+ perimeter=perimeter,
+ built_form=bf
+ )
+
+ n_rooms = np.floor(combination[1] / 15)
+
+ n_windows = estimate_windows(
+ property_type=pt,
+ built_form=bf,
+ construction_age_band="",
+ floor_area=combination[1],
+ number_habitable_rooms=n_rooms
+ )
+
+ # We determine the exact upgrade pathway for this combination, guided by the generic upgrade pathway
+ combination_upgrade_pathway = []
+ for upgrade in upgrade_path:
+ if upgrade == "wall_insulation":
+ if combination[2] == "cavity":
+ combination_upgrade_pathway.append("cavity_wall_insulation")
+ else:
+ combination_upgrade_pathway.append("solid_wall_insulation")
+ continue
+
+ if upgrade == "roof_insulation":
+ if combination[3] == "pitched":
+ combination_upgrade_pathway.append("loft_insulation")
+ else:
+ combination_upgrade_pathway.append("non_pitched_roof_insualtion")
+ continue
+
+ if upgrade == "ventilation":
+ combination_upgrade_pathway.append("ventilation")
+ continue
+
+ if upgrade == "low_energy_lighting":
+ combination_upgrade_pathway.append("low_energy_lighting")
+ continue
+
+ if upgrade == "windows":
+ if not combination[4]:
+ combination_upgrade_pathway.append("double_glazing")
+ else:
+ combination_upgrade_pathway.append("secondary_glazing")
+ continue
+
+ if upgrade == "heating":
+ if combination[0] in ["Semi Detached House", "Detached House"]:
+ combination_upgrade_pathway.append("high_heat_retention_storage")
+ else:
+ combination_upgrade_pathway.append("air_source_heat_pump")
+ continue
+
+ if upgrade == "solar":
+ if combination[0] in ["Semi Detached House", "Detached House", "Mid Terrace House"]:
+ combination_upgrade_pathway.append("solar_pv")
+ continue
+
+ combination_costs = []
+ for measure in combination_upgrade_pathway:
+ unit_cost = pricing_matrix[measure]
+ # Wall insulation
+ if measure in ["cavity_wall_insulation", "internal_wall_insulation", "external_wall_insulation"]:
+ cost = unit_cost * external_wall_area
+ elif measure in ["loft_insulation"]:
+ cost = unit_cost * ground_floor_area
+ elif measure == "ventilation":
+ if combination[1] == 73:
+ cost = unit_cost * 2
+ elif combination[1] == 97:
+ cost = unit_cost * 3
+ else:
+ cost = unit_cost * 4
+ elif measure == "low_energy_lighting":
+ n_lights = lighting_count[combination[0]]
+ if combination[1] == 73:
+ inflation = 1
+ elif combination[1] == 97:
+ inflation = 1.2
+ else:
+ inflation = 1.5
+ cost = unit_cost * n_lights * inflation
+ elif measure in ["double_glazing", "secondary_glazing"]:
+ cost = unit_cost * n_windows
+ elif measure == "high_heat_retention_storage":
+ cost = unit_cost * n_rooms
+ elif measure in ["air_source_heat_pump", "solar_pv"]:
+ cost = unit_cost
+ else:
+ raise NotImplementedError("Implement: %s" % measure)
+
+ combination_costs.append(
+ {
+ "measure": measure,
+ "cost": cost
+ }
+ )
+
+ combination_costs = pd.DataFrame(combination_costs)
+
+ contingency = 0.26
+
+ epr_data = pd.read_excel(
+ "/Users/khalimconn-kowlessar/Documents/hestia/Customers/L&G/Risk Matrix/EPR Data.xlsx", header=1
+ )
+ epr_data["Measure added"].value_counts()
+ epr_data["row_id"] = epr_data.index
+ # We need to calculate the costs
+ cost_data = []
+ for _, row in epr_data.iterrows():
+ epc = row["EPC"][0]
+ sap = int(row["EPC"][1:])
+
+ n_floors = num_floors_map[row["Property Type"]]
+ bf = built_form_map[row["Property Type"]]
+ pt = "House" if "flat" not in row["Property Type"].lower() else "Flat"
+ # Model the home as a box
+ ground_floor_area = row["area"] / n_floors
+ perimeter = np.sqrt(ground_floor_area) * 4
+
+ # This is the amount of insulation required
+ external_wall_area = estimate_external_wall_area(
+ num_floors=n_floors,
+ floor_height=2.5,
+ perimeter=perimeter,
+ built_form=bf
+ )
+
+ n_rooms = np.floor(row["area"] / 15)
+
+ n_windows = estimate_windows(
+ property_type=pt,
+ built_form=bf,
+ construction_age_band="",
+ floor_area=row["area"],
+ number_habitable_rooms=n_rooms
+ )
+ cost_upper_bound = None
+ if pd.isnull(row["Measure added"]):
+ unit_cost = None
+ else:
+ measure = row["Measure added"]
+ unit_cost = pricing_matrix[measure]
+
+ if pd.isnull(row["Measure added"]):
+ cost = None
+ elif row["Measure added"] == "Loft insulation":
+ cost = unit_cost * ground_floor_area
+ elif row["Measure added"] in ["Cavity wall insulation", "Internal wall insulation"]:
+ cost = unit_cost * external_wall_area + pricing_matrix["ventilation"] * 3
+ elif row["Measure added"] == "Solid wall insulation":
+ cost = unit_cost * external_wall_area + pricing_matrix["ventilation"] * 3
+ cost_upper_bound = pricing_matrix["External wall insulation"] * external_wall_area + pricing_matrix[
+ "ventilation"] * 3
+ elif row["Measure added"] == "Double Glazing":
+ cost = unit_cost * n_windows
+ elif row["Measure added"] == "LEDs":
+ cost = unit_cost * lighting_count[row["Property Type"]]
+ elif row["Measure added"] in [
+ '5kw ASHP feeding heating & Hot water (dual tariff)',
+ '11.2kw ASHP feeding heating & Hot water (dual tariff)',
+ "3 kWp Solar PV",
+ '4 kWp Solar PV',
+ "4.3 kWp Solar PV",
+ '4.8 kWp Solar PV',
+ '5 kWp Solar PV',
+ '5.5 kWp Solar PV'
+ ]:
+ cost = unit_cost
+ elif row["Measure added"] == "HHRSH (dual tariff)":
+ cost = unit_cost * (n_rooms + 1)
+ elif row["Measure added"] == "Suspended floor insulation":
+ cost = unit_cost * ground_floor_area
+ else:
+ raise Exception()
+
+ cost_data.append(
+ {
+ "row_id": row["row_id"],
+ "epc": epc,
+ "sap": sap,
+ "cost": cost,
+ "cost upper bound": cost_upper_bound
+ }
+ )
+
+ cost_data = pd.DataFrame(cost_data)
+
+ risk_matrix = pd.merge(
+ epr_data,
+ cost_data,
+ on="row_id",
+ )
+
+ risk_matrix["contingency"] = risk_matrix["cost"] * contingency
+ risk_matrix["upper bound coningency"] = risk_matrix["cost upper bound"] * contingency
+
+ pricing_df = pd.DataFrame(
+ [
+ {
+ "Measure": k,
+ "Unit Cost": v
+ }
+ for k, v in pricing_matrix.items()
+ ]
+ )
+
+ with pd.ExcelWriter(
+ "/Users/khalimconn-kowlessar/Documents/hestia/Customers/L&G/Risk Matrix/risk_matrix.xlsx") as writer:
+ risk_matrix.to_excel(writer, sheet_name="Risk Matrix", index=False)
+ pricing_df.to_excel(writer, sheet_name="Pricing Assumptions", index=False)
diff --git a/etl/epc/Record.py b/etl/epc/Record.py
index 9ff1de0a..1ed33567 100644
--- a/etl/epc/Record.py
+++ b/etl/epc/Record.py
@@ -355,6 +355,7 @@ class EPCRecord:
self._clean_floor_level()
self._clean_floor_height()
self._clean_constituency()
+ self._clean_new_build_descriptions()
# self._clean_potential_energy_efficiency()
# self._clean_environment_impact_potential()
@@ -397,6 +398,10 @@ class EPCRecord:
if self.prepared_epc["floor-height"] <= 1.665:
self.prepared_epc["floor-height"] = average
+ def _clean_new_build_descriptions(self):
+ for col in ['roof-description', 'walls-description', 'floor-description']:
+ self.prepared_epc[col] = self.prepared_epc[col].replace("W/m²K", "W/m-¦K")
+
def _clean_constituency(self):
"""
We handle the single case of finding a missing constituency by using the local authority
diff --git a/recommendations/RoofRecommendations.py b/recommendations/RoofRecommendations.py
index cd7f82c4..fa8b831c 100644
--- a/recommendations/RoofRecommendations.py
+++ b/recommendations/RoofRecommendations.py
@@ -163,15 +163,12 @@ class RoofRecommendations:
if self.property.roof["is_thatched"]:
return
- # If we have a u-value already, need to implement this
- if u_value:
- if u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE:
- # The Roof is already compliant
- return
-
- if self.property.data["transaction-type"] in ["new dwelling", "not sale or rental"]:
- return
- raise NotImplementedError("Implement me")
+ # If we have a u-value and we don't have a non-invasive recommendation, we can't recommend anything
+ if u_value and not any(
+ x in MEASURE_MAP["roof_insulation"] for x in [r["type"] for r in self.property.non_invasive_recommendations]
+ ):
+ # We don't have enough information to provide a recommendation
+ return
u_value = get_roof_u_value(
insulation_thickness=self.property.roof["insulation_thickness"],
diff --git a/recommendations/WallRecommendations.py b/recommendations/WallRecommendations.py
index 92147fb8..dbb7d674 100644
--- a/recommendations/WallRecommendations.py
+++ b/recommendations/WallRecommendations.py
@@ -227,24 +227,14 @@ class WallRecommendations(Definitions):
# external wall insulation
if (
(not is_cavity_wall)
- and (self.property.year_built >= self.YEAR_WALLS_BUILT_WITH_INSULATION)
and (u_value >= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE)
):
# Recommend insulation
- self.find_insulation(u_value, phase, measures)
+ self.find_insulation(u_value, phase, measures=measures, default_u_values=default_u_values)
return
- # We can't detect it's a cavity wall, but it was built after 1990 so likely built with insulation already
- # + it already has a U-value better than the building regulations, so we don't need to recommend anything
- if (
- (not is_cavity_wall)
- and ((self.property.year_built >= self.YEAR_WALLS_BUILT_WITH_INSULATION)
- or (u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE))
- ):
- # Recommend nothing
- return
-
- raise NotImplementedError("Not implemented yet")
+ # We have a sufficiently low U-value
+ return
u_value = get_wall_u_value(
clean_description=self.property.walls["clean_description"],
@@ -626,7 +616,7 @@ class WallRecommendations(Definitions):
"walls_thermal_transmittance_ending": new_u_value
}
- if default_u_values:
+ if default_u_values and "Average thermal transmittance" not in new_description:
# If we're using default U-values, we overwrite new_u_value
new_u_value = get_wall_u_value(
clean_description=new_description,