Merge pull request #421 from Hestia-Homes/remote-assessment-api

Handing new build for existing property edge case

.idea/Model.iml

@@ -7,7 +7,7 @@
       <sourceFolder url="file://$MODULE_DIR$/open_uprn" isTestSource="false" />
       <sourceFolder url="file://$MODULE_DIR$/recommendations" isTestSource="false" />
     </content>
-    <orderEntry type="jdk" jdkName="AssetList" jdkType="Python SDK" />
+    <orderEntry type="jdk" jdkName="Fastapi-backend" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
   <component name="PyNamespacePackagesService">

asset_list/AssetList.py

@@ -444,6 +444,25 @@ class AssetList:
                 self.standardised_asset_list[self.address1_colname].copy()
             )
 
+        # Handle the case where the property type column and built form are missing
+        if self.landlord_property_type is None and self.landlord_built_form is None:
+            if "Archetype" in self.raw_asset_list.columns:
+                # We use the non-intrusives as our property type and built form
+                self.landlord_property_type = self.STANDARD_PROPERTY_TYPE
+                self.landlord_built_form = self.STANDARD_BUILT_FORM
+                self.standardised_asset_list[self.landlord_property_type] = (
+                    self.standardised_asset_list["Archetype"].copy()
+                )
+                self.standardised_asset_list[self.landlord_built_form] = (
+                    self.standardised_asset_list["Archetype"].copy()
+                )
+            else:
+                # We use the EPC data as our property type and built form
+                self.landlord_property_type = self.STANDARD_PROPERTY_TYPE
+                self.landlord_built_form = self.STANDARD_BUILT_FORM
+                self.standardised_asset_list[self.landlord_property_type] = None
+                self.standardised_asset_list[self.landlord_built_form] = None
+
         # Handle the case where the property type column is the same as the built type
         if self.landlord_property_type == self.landlord_built_form:
             self.landlord_built_form = self.STANDARD_BUILT_FORM
@@ -1342,7 +1361,7 @@ class AssetList:
         )
         self.standardised_asset_list["solar_landlord_data_indicates_needs_heating_upgrade"] = (
             self.standardised_asset_list[self.STANDARD_HEATING_SYSTEM].isin(
-                ["electric storage heaters", "room heaters", "electric radiators", "no heating"]
+                ["electric storage heaters", "room heaters", "electric radiators", "no heating", "electric fuel"]
             )
         )
 
@@ -1651,7 +1670,7 @@ class AssetList:
                     "SAP Category"],
                 self.standardised_asset_list["cavity_reason"]
             )
-        else:
+        elif self.non_intrusives_present:
             self.standardised_asset_list["cavity_reason"] = np.where(
                 (
                     self.standardised_asset_list["epc_indicates_empty_cavity"] &
@@ -1675,6 +1694,16 @@ class AssetList:
                     "SAP Category"],
                 self.standardised_asset_list["cavity_reason"]
             )
+        else:
+            self.standardised_asset_list["cavity_reason"] = np.where(
+                (
+                    self.standardised_asset_list["epc_indicates_empty_cavity"] &
+                    ~self.standardised_asset_list["non_intrusive_indicates_empty_cavity"] &
+                    pd.isnull(self.standardised_asset_list["cavity_reason"])
+                ),
+                "EPC Shows Empty Cavity: " + self.standardised_asset_list["SAP Category"],
+                self.standardised_asset_list["cavity_reason"]
+            )
 
         self.standardised_asset_list["cavity_reason"] = np.where(
             (
@@ -1716,17 +1745,18 @@ class AssetList:
         self.standardised_asset_list["solar_reason"] = None
 
         # Map of variables and fill values for the solar_reason variable
+        # ordering of this map is important, where we flag our prioritised work types first
         solar_reason_map = {
             "solar_eligible": "Solar Eligible: ",
+            "solar_eligible_solid_wall_uninsulated": "Solar Eligible, Solid Wall Uninsulated, EPC E or Below: ",
             "solar_eligible_needs_heating_upgrade": (
                 "Solar Eligible, Needs Heating Upgrade: "
-            ),
+            )
-            "solar_eligible_solid_wall_uninsulated": "Solar Eligible, Solid Wall Uninsulated, EPC E or Below: ",
         }
 
         for variable, reason in solar_reason_map.items():
             self.standardised_asset_list["solar_reason"] = np.where(
-                self.standardised_asset_list[variable],
+                self.standardised_asset_list[variable] & pd.isnull(self.standardised_asset_list["solar_reason"]),
                 reason + self.standardised_asset_list["SAP Category"],
                 self.standardised_asset_list["solar_reason"]
             )
@@ -2401,6 +2431,7 @@ class AssetList:
             master_data = pd.read_csv(filepath)
             # Strip columns
             master_data.columns = [c.strip() for c in master_data.columns]
+            master_data.columns = [re.sub(r'\s+', ' ', c) for c in master_data.columns]
 
             if not id_map.empty:
                 master_data = master_data.merge(
@@ -2537,8 +2568,8 @@ class AssetList:
             ]
 
             scheme_col = (
-                "AFFORDABLE WARMTH  OR EPC FOR HOUSING ASSOCIATION" if
+                "AFFORDABLE WARMTH OR EPC FOR HOUSING ASSOCIATION" if
-                "AFFORDABLE WARMTH  OR EPC FOR HOUSING ASSOCIATION" in master_data.columns else "AFFORDABLE WARMTH"
+                "AFFORDABLE WARMTH OR EPC FOR HOUSING ASSOCIATION" in master_data.columns else "AFFORDABLE WARMTH"
             )
             # The columns are massively different - we take just a few
             unmatched_df = unmatched_df[

asset_list/app.py

@@ -89,43 +89,6 @@ def app():
     # - We want: fully insulated property (all wall types), EPC D or below (floors should be solid)
     # - Or the insulation required is loft/cavity (floors should be solid)
 
-    # Bromford
-    data_folder = ("/Users/khalimconn-kowlessar/Documents/hestia/Customers/Bromford/Apr 2025 Programme "
-                   "Rebuild/Prepared data/")
-    data_filename = "asset_list.xlsx"
-    sheet_name = "Sheet1"
-    postcode_column = 'PostCode'
-    fulladdress_column = "FullAddress"
-    address1_column = None
-    address1_method = "house_number_extraction"
-    address_cols_to_concat = []
-    missing_postcodes_method = None
-    landlord_year_built = "ConYear"
-    landlord_os_uprn = None
-    landlord_property_type = "AssetTypeDesc"
-    landlord_built_form = "PropTypeDesc"
-    landlord_wall_construction = "Construction type"
-    landlord_roof_construction = None
-    landlord_heating_system = "Heating Type"
-    landlord_existing_pv = None
-    landlord_property_id = "Asset"
-    landlord_sap = None
-    outcomes_filename = "outcomes.xlsx"
-    outcomes_sheetname = "Sheet1"
-    outcomes_postcode = "Postcode"
-    outcomes_houseno = "No"
-    outcomes_id = None
-    outcomes_address = "Address"
-    master_filepaths = [
-        "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Bromford/Apr 2025 Programme Rebuild/Prepared data/ECO "
-        "3 submissions.csv",
-        "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Bromford/Apr 2025 Programme Rebuild/Prepared data/ECO "
-        "4 submissions.csv",
-    ]
-    master_to_asset_list_filepath = None
-    phase = False
-    ecosurv_landlords = "paul butler|bromford"
-    
     # Torus
     data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Torus/Phase 1"
     data_filename = "Torus Property Asset List - Phase 1.xlsx"
@@ -156,33 +119,6 @@ def app():
     master_to_asset_list_filepath = None
     phase = True
 
-    # Ealing - houses
-    data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Ealing"
-    data_filename = "Ealing_rechecked_cleaned_05042025.csv"
-    sheet_name = None
-    postcode_column = 'Postcode'
-    fulladdress_column = "Address"
-    address1_column = None
-    address1_method = "house_number_extraction"
-    address_cols_to_concat = []
-    missing_postcodes_method = None
-    landlord_year_built = "Year Built"
-    landlord_os_uprn = None
-    landlord_property_type = "Property Type Code"
-    landlord_built_form = None
-    landlord_wall_construction = None
-    landlord_heating_system = None
-    landlord_existing_pv = None
-    landlord_property_id = "Property ref"
-    outcomes_filename = None
-    outcomes_sheetname = None
-    outcomes_postcode = None
-    outcomes_houseno = None
-    outcomes_id = None
-    outcomes_address = None
-    master_filepaths = []
-    master_to_asset_list_filepath = None
-
     # Southern Midlands
     data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Southern/Midlands Properties - Apr 2025"
     data_filename = "Southern Housing Midlands Property List - combined.xlsx"
@@ -210,67 +146,6 @@ def app():
     master_filepaths = []
     master_to_asset_list_filepath = None
 
-    # Live West (2018 Asset list)
-    data_folder = (
-        "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Livewest/Programme Update - March 2025/2018 Asset List"
-    )
-    data_filename = "LIVEWEST  STOCK - 23rd October 2018.xlsx"
-    sheet_name = "Assets"
-    postcode_column = 'Postcode'
-    fulladdress_column = "Address"
-    address1_column = None
-    address1_method = "house_number_extraction"
-    address_cols_to_concat = []
-    missing_postcodes_method = None
-    landlord_year_built = "Build Year"
-    landlord_os_uprn = None
-    landlord_property_type = "Property Archetype"
-    landlord_built_form = None
-    landlord_wall_construction = None
-    landlord_heating_system = "Heating Fuel Type"
-    landlord_existing_pv = None
-    landlord_property_id = "Uprn - DO NOT DELETE"
-    outcomes_filename = "RT - LiveWest.xlsx"
-    outcomes_sheetname = "Feedback"
-    outcomes_postcode = "Poscode"
-    outcomes_houseno = "No."
-    outcomes_id = "UPRN"
-    master_filepaths = [
-        "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Livewest/Programme Update - March 2025/Rolling Master "
-        "- redacted for analysis/CAVITY-Table 1.csv"
-    ]
-    master_to_asset_list_filepath = None
-
-    # Live West (South West asset list)
-    data_folder = ("/Users/khalimconn-kowlessar/Documents/hestia/Customers/Livewest/Programme Update - March "
-                   "2025/Livewest Asset List (Original) - csv")
-    data_filename = "Report-Table 1.csv"
-    sheet_name = None
-    postcode_column = 'Postcode'
-    fulladdress_column = "T1_Address"
-    address1_column = None
-    address1_method = "house_number_extraction"
-    address_cols_to_concat = []
-    missing_postcodes_method = None
-    landlord_year_built = "Build Yr"
-    landlord_os_uprn = None
-    landlord_property_type = "T1_AssetType"
-    landlord_built_form = "T1_AssetType"
-    landlord_wall_construction = "Wall Type Cavity"
-    landlord_heating_system = "Heating Fuel"
-    landlord_existing_pv = None
-    landlord_property_id = "T1_UPRN"
-    outcomes_filename = "RT - LiveWest.xlsx"
-    outcomes_sheetname = "Feedback"
-    outcomes_postcode = "Poscode"
-    outcomes_houseno = "No."
-    outcomes_id = "UPRN"
-    master_filepaths = [
-        "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Livewest/Programme Update - March 2025/Rolling Master "
-        "- redacted for analysis/CAVITY-Table 1.csv"
-    ]
-    master_to_asset_list_filepath = None
-
     # PFP London
     data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Places For People/London"
     data_filename = "PFP AREAS SURROUNDING LONDON - JAY, RUTH & LANE.xlsx"
@@ -459,6 +334,8 @@ def app():
     landlord_heating_system = "Heat Source"
     landlord_existing_pv = "PV (Y/N)"
     landlord_property_id = "Place ref"
+    landlord_roof_construction = None
+    landlord_sap = None
     outcomes_filename = None
     outcomes_sheetname = None
     outcomes_postcode = None
@@ -466,6 +343,9 @@ def app():
     master_filepaths = []
     master_to_asset_list_filepath = None
     outcomes_id = None
+    outcomes_address = None
+    phase = False
+    ecosurv_landlords = None
 
     # For ACIS - programme re-build
     # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/ACIS/ACIS Full Programme Review March 2025"
@@ -483,17 +363,23 @@ def app():
     # landlord_property_type = "Property type"
     # landlord_built_form = None
     # landlord_wall_construction = "Wall Constuction"
+    # landlord_roof_construction = None
+    # landlord_sap = None
     # landlord_heating_system = "Heating"
     # landlord_existing_pv = None
     # outcomes_filename = "ACIS Group - 25.11.2024 - outcomes.xlsx"
     # outcomes_sheetname = "Feedback"
     # outcomes_postcode = "Postcode"
+    # outcomes_address = "Address"
     # outcomes_houseno = "No"
+    # outcomes_id = None
     # master_filepaths = [
     #     os.path.join(data_folder, "ECO 3 -Table 1.csv"),
     #     os.path.join(data_folder, "ECO 4 -Table 1.csv"),
     # ]
     # master_to_asset_list_filepath = None
+    # phase = False
+    # ecosurv_landlords = None
 
     # For plus dane
     data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Plus Dane"
@@ -697,6 +583,9 @@ def app():
         epc_data.append(csv_data)
 
     epc_df = pd.concat(epc_data)
+    if "estimated" not in epc_df.columns:
+        epc_df["estimated"] = False
+
     epc_df["estimated"] = epc_df["estimated"].fillna(False)
 
     # We expand out the recommendations
@@ -804,98 +693,6 @@ def app():
 
     asset_list.flat_analysis()
 
-    ################################################################
-    # WESTWARD - comparison between Kieran's method & automated
-    ################################################################
-
-    # Check 1)
-    cavity_fills = pd.read_excel(
-        os.path.join(data_folder, "WESTWARD - Route March Prep.xlsx"),
-        sheet_name="Straight Fill"
-    )
-    cavity_fills = cavity_fills.merge(
-        asset_list.standardised_asset_list[
-            [asset_list.STANDARD_LANDLORD_PROPERTY_ID, "cavity_reason"]
-        ],
-        how="left",
-        left_on=asset_list.landlord_property_id,
-        right_on=asset_list.STANDARD_LANDLORD_PROPERTY_ID
-    )
-    cavity_fills["cavity_reason"] = cavity_fills["cavity_reason"].fillna("Not identified")
-    print(cavity_fills["cavity_reason"].value_counts())
-    # Didn't identify 3 properties because they're bedsits
-    # 4 properties were identified, not based on the non-intrusives but instead because
-    # Westward said they were built in 2003/2007. Have adjusted this to use the age from the
-    # epc as well, as EPC says 1975 and they look like 1975 properties
-    # 37 properties flagged as already having solar - these are all because the landlord said they have solar
-    # e.g.
-    # https://earth.google.com/web/search/11+Winsland+Avenue+TOTNES+TQ9+5FT/@50.43354465,-3.71318276,46.57468503a,
-    # 59.14004365d,35y,0h,0t,
-    # 0r/data=CpABGmISXAolMHg0ODZkMWQxOGE4NWRiZjdkOjB4YjBhM2E5M2Q3YWVlMWEwYhlZYgp7fzdJQCHFfC9027QNwCohMTEgV2luc2xhbmQgQXZlbnVlIFRPVE5FUyBUUTkgNUZUGAIgASImCiQJbxsQEoo3SUARXQcp_HE3SUAZBmiZGJ6yDcAhCA0fqq63DcBCAggBOgMKATBCAggASg0I____________ARAA
-    # https://earth.google.com/web/search/15+St+Anne%27s+Ct,+Newton+Abbot+TQ12+1TL/@50.53068337,-3.61611128,
-    # 11.74908956a,135.73212429d,35y,0h,0t,
-    # 0r/data=CpUBGmcSYQolMHg0ODZkMDVkMjFhODhjZjgxOjB4MjBmMzE2Zjc3MGI2NGMwYxlCxHLw8UNJQCFZqyzALe4MwComMTUgU3QgQW5uZSdzIEN0LCBOZXd0b24gQWJib3QgVFExMiAxVEwYAiABIiYKJAm-r6U2iDdJQBHS5ICRdDdJQBmYGVpmiLINwCG8wcrtqbYNwEICCAE6AwoBMEICCABKDQj___________8BEAA
-
-    # Check 2)
-    cavity_fills_with_solar = pd.read_excel(
-        os.path.join(data_folder, "WESTWARD - Route March Prep.xlsx"),
-        sheet_name="Solar PV - Straight Fill"
-    )
-    cavity_fills_with_solar = cavity_fills_with_solar.merge(
-        asset_list.standardised_asset_list[
-            [asset_list.STANDARD_LANDLORD_PROPERTY_ID, "cavity_reason"]
-        ],
-        how="left",
-        left_on=asset_list.landlord_property_id,
-        right_on=asset_list.STANDARD_LANDLORD_PROPERTY_ID
-    )
-    cavity_fills_with_solar["cavity_reason"] = cavity_fills_with_solar["cavity_reason"].fillna("Not identified")
-    print(cavity_fills_with_solar["cavity_reason"].value_counts())
-    # 203 properties total
-    # 140 properties were flagged up based on non-intrusives (Non-Intrusive Data Showed Empty Cavity)
-    # 63 property already has solar
-
-    # Check 3) RDF
-    rdf = pd.read_excel(
-        os.path.join(data_folder, "WESTWARD - Route March Prep.xlsx"),
-        sheet_name="RDF CIGA checks"
-    )
-    rdf = rdf.merge(
-        asset_list.standardised_asset_list[
-            [asset_list.STANDARD_LANDLORD_PROPERTY_ID, "cavity_reason", "solar_reason"]
-        ],
-        how="left",
-        left_on=asset_list.landlord_property_id,
-        right_on=asset_list.STANDARD_LANDLORD_PROPERTY_ID
-    )
-    rdf["cavity_reason"] = rdf["cavity_reason"].fillna("Not identified")
-    print(rdf["cavity_reason"].value_counts())
-    # 264 properties are not identified, 261 of which are due to the fact they contain materials
-    # The other 3 were determined to be eligible for solar instead
-    # Many of these units that were identified for rdf works could be solar jobs
-
-    rdf_with_solar = pd.read_excel(
-        os.path.join(data_folder, "WESTWARD - Route March Prep.xlsx"),
-        sheet_name="Solar PV - RDF CIGA Checks"
-    )
-    rdf_with_solar = rdf_with_solar.merge(
-        asset_list.standardised_asset_list[
-            [asset_list.STANDARD_LANDLORD_PROPERTY_ID, "cavity_reason", "solar_reason"]
-        ],
-        how="left",
-        left_on=asset_list.landlord_property_id,
-        right_on=asset_list.STANDARD_LANDLORD_PROPERTY_ID
-    )
-    rdf_with_solar["cavity_reason"] = rdf_with_solar["cavity_reason"].fillna("Not identified")
-    rdf_with_solar["cavity_reason"].value_counts()
-
-    # All others identified - some flagged as empties due to EPC or landlord data suggesting as much
-    # 5 not identified due to containing COMPACTED BEAD
-
-    asset_list.standardised_asset_list = asset_list.standardised_asset_list[
-        asset_list.standardised_asset_list[asset_list.landlord_property_id]
-    ]
-
     asset_list.load_contact_details(
         local_filepath=os.path.join(data_folder, "Full property list wth D&V report V look up 12.2.25.xlsx"),
         sheet_name="Report 1",

asset_list/mappings/built_form.py

@@ -143,6 +143,74 @@ BUILT_FORM_MAPPINGS = {
     'Sixth Floor': 'top-floor',
     'Sheltered Bung': 'semi-detached',
     'Guest': 'unknown',
-    'Fifth Floor': 'mid-floor'
+    'Fifth Floor': 'mid-floor',
-
+    'Flat Within Block': 'mid-floor',
+    'Coach House with Garage': 'detached',
+    'Over Garage House': 'top-floor',
+    'Apartment': 'mid-floor',
+    'Flat Over Shop': 'top-floor',
+    'Flat Over Garage': 'top-floor',
+    'Bridge Flat': 'mid-floor',
+    'House Mid Terrace': 'mid-terrace',
+    'Semi-detached house': 'semi-detached',
+    'House Semi Detached': 'semi-detached',
+    'House Detached': 'detached',
+    'Detached house': 'detached',
+    'House End Terrace': 'end-terrace',
+    'Flat Ground Floor Mr': 'ground floor',
+    'Mais Flat 1St Fl Mr': 'mid-floor',
+    'Top-floor maisonette': 'top-floor',
+    'Flat 1St Warden Lr': 'mid-floor',
+    'Cranwell': 'unknown',
+    'No Fines': 'unknown',
+    'Flat 1St Elderly Mr': 'mid-floor',
+    'Stent Mod': 'unknown',
+    'Mais Flat Grd Fl Mr': 'ground floor',
+    'Flat 1St Floor Mr': 'mid-floor',
+    'Mid-terrace house': 'mid-terrace',
+    'Stent Unmod': 'unknown',
+    'Flat 2Nd Floor Mr': 'mid-floor',
+    'Studio Grd Warden Lr': 'ground floor',
+    'Flat Grd Elderly Mr': 'ground floor',
+    'Studio Fl Grd Eld Lr': 'ground floor',
+    'Scottwood': 'unknown',
+    'Airey': 'unknown',
+    'Studio Flat 1Stfl Lr': 'mid-floor',
+    'Studio Flat 1Stfl Mr': 'mid-floor',
+    'Flat Grd Elderly Lr': 'ground floor',
+    'Trusteel MKII': 'unknown',
+    'No-Fines Concrete': 'unknown',
+    'Crosswall': 'unknown',
+    'Fidler': 'unknown',
+    'Ground-floor maisonette': 'ground floor',
+    'Studio Flat Grdfl Mr': 'ground floor',
+    'Studio Flat Grd Lr': 'ground floor',
+    'Studio Fl Grd Eld Mr': 'ground floor',
+    'Bungalow Eld Person': 'unknown',
+    'Cornish': 'unknown',
+    'B.I.S.F.': 'unknown',
+    'Flat 1St Floor Lr': 'mid-floor',
+    'Mid-floor flat': 'mid-floor',
+    'Bsit Bung Warden Sch': 'unknown',
+    'Hawksley': 'unknown',
+    'Orlit': 'unknown',
+    'Mid-floor maisonette': 'mid-floor',
+    'Ground-floor flat': 'ground floor',
+    'Flat Grd Floor Lr': 'ground floor',
+    'Studio 1St Warden Lr': 'mid-floor',
+    'Flat Grd Warden Lr': 'ground floor',
+    'end-terrace house': 'end-terrace',
+    'Top-floor flat': 'top-floor',
+    'End-terrace house': 'end-terrace',
+    'Mais Flat 2Nd Fl Mr': 'mid-floor',
+    'Flat 1St Elderly Lr': 'mid-floor',
+    'Bfly Bung Bed Sitter': 'unknown',
+    'Swedish': 'unknown',
+    'Bungalow Semi Detach': 'semi-detached',
+    '4 Ext. Wall Flat': 'unknown',
+    '6 Ext. Wall Flat': 'unknown',
+    '5 Ext. Wall Flat': 'unknown',
+    'Unknown': 'unknown',
+    'Enclosed mid-terrace': 'mid-terrace',
+    'Enclosed end-terrace': 'end-terrace'
 }

asset_list/mappings/heating_systems.py

@@ -2,8 +2,10 @@ import numpy as np
 
 STANDARD_HEATING_SYSTEMS = {
     "gas combi boiler",
+    "gas boiler, radiators",
     "electric storage heaters",
     "district heating",
+    "communal heating"
     "gas condensing boiler",
     "oil boiler",
     "gas condensing combi",
@@ -24,7 +26,8 @@ STANDARD_HEATING_SYSTEMS = {
     'unknown',
     "electric ceiling",
     "electric underfloor",
-    "no heating"
+    "no heating",
+    "non-electric underfloor"
 }
 
 HEATING_MAPPINGS = {
@@ -202,5 +205,20 @@ HEATING_MAPPINGS = {
     'Wet - Underfloor Solar': 'other',
     'No Heating Required Gas': 'unknown',
     'Electric - Storage/Panel Heaters Gas': 'electric storage heaters',
-    'Electric - Storage/Panel Heaters Solid': 'electric storage heaters'
+    'Electric - Storage/Panel Heaters Solid': 'electric storage heaters',
+    'District Heat Network': 'district heating',
+    'Not Applicable': 'no heating',
+    'Not Responsible': 'unknown',
+    'Communal Oil': 'communal heating',
+    'Communal Electric': 'communal heating',
+    'Renewables (Air / Ground Source Pumps)': 'air source heat pump',
+    'Communal Renewable': 'air source heat pump',
+    'Room heaters/ Electric': 'room heaters',
+    'Room heaters/ Gas': 'room heaters',
+    'Radiator system': "gas boiler, radiators",
+    'Drilled and filled': 'unknown',
+    'Boiler/ underfloor': 'electric underfloor',
+    'Storage system': "non-electric underfloor",
+    'BOILER': 'gas combi boiler',
+    'SPACE_HEATER': 'room heaters'
 }

asset_list/mappings/property_type.py

@@ -178,5 +178,21 @@ PROPERTY_MAPPING = {
     'Parking Space': 'other',
     'Community Centre': 'other',
     'Communal Facility': 'other',
-    'Semi': 'house'
+    'Semi': 'house',
+    'House with Compulsory Garage': 'house',
+    'Flat with Compulsory Garage': 'flat',
+    'Other': 'other',
+    'Maisonette with Compulsory Garage': 'maisonette',
+    'Room in Shared Property': 'other',
+    'Bungalow Mid Terrace': 'bungalow',
+    '4 Ext. Wall Flat': 'flat',
+    '3 Ext. Wall Flat': 'flat',
+    'Bungalow End Terrace': 'bungalow',
+    '6 Ext. Wall Flat': 'flat',
+    'Bungalow Detached': 'bungalow',
+    'Maisonette 3 Ext. Wall': 'maisonette',
+    'Maisonette 2 Ext. Wall': 'maisonette',
+    '5 Ext. Wall Flat': 'flat',
+    'Bungalow Semi Detached': 'bungalow',
+    'COMINT': 'unknown'
 }

asset_list/mappings/walls.py

@@ -158,7 +158,6 @@ WALL_CONSTRUCTION_MAPPINGS = {
     '2017 onwards': 'new build - average thermal transmittance',
     'ND (inferred)': 'unknown',
     'Flat / maisonette': 'other',
-
     'Other': 'other',
     'Timber Frame': 'timber frame unknown insulation',
     'Cavity Wall': 'cavity unknown insulation',
@@ -166,5 +165,59 @@ WALL_CONSTRUCTION_MAPPINGS = {
     'PRC': 'system built',
     'Cross Wall': 'system built',
     'Solid Wall': 'solid brick unknown insulation',
-    'Traditional': 'other'
+    'Traditional': 'unknown',
+
+    'Solid': 'solid brick unknown insulation',
+    'Wates no fines': 'system built',
+    'Concrete Frame': 'system built',
+    'PRCWATES': 'system built',
+    'Refurbished Cornish': 'system built',
+    'Bailey Stratton': 'other',
+    'Refurbished Reema': 'system built',
+    'PRCREEMA': 'system built',
+    'Trustsell Type': 'system built',
+    'Petra Nissan': 'unknown',
+    'Reinstated Airey': 'system built',
+    'Refurbished Airey': 'system built',
+    # From Abri- slightly unclear on types but not a large portion of the data
+    'No Fines Type': 'system built',
+    'Refurbished Unity': 'system built',
+    'Timber Framed': 'timber frame unknown insulation',
+    'Refurbished Woolaway': 'system built',
+    'Modern Methods of Construction': 'other',
+    'BISF - Brit Iron & Steel Federation': 'system built',
+    'Steel Framed': 'system built',
+    'Timber Framed with confirmed Fire Stopping': 'timber frame unknown insulation',
+    'Sipporex': 'system built',
+
+    'Wates': 'system built',
+    'Bryants': 'system built',
+    'Gregory (Crosswall)': 'system built',
+    'Rsmit': 'system built',
+    'Dorman Long': 'system built',
+    'Tarmac': 'system built',
+    'RBIS': 'system built',
+    'Five Oaks': 'system built',
+    'Not known': 'unknown',
+    'Smiths': 'system built',
+    'Kendrick': 'system built',
+    'IDC': 'system built',
+    'Wimpey (Part Brick)': 'system built',
+    'Whitehall': 'system built',
+    'Wimpey': 'system built',
+    'Bison': 'system built',
+    'Zinns': 'system built',
+    'Bisf': 'system built',
+    'Integer': 'system built',
+    'Cornish': 'system built',
+    'Rwate': 'system built',
+    'Hill Presweld Steel': 'system built',
+
+    'Cavity Filled Cavity': 'filled cavity',
+    'Cavity Unknown': 'cavity unknown insulation',
+    'Cavity Filled Cavity (internal)': 'filled cavity',
+    '': 'unknown',
+    'Cavity Internal Insulation': 'filled cavity',
+    'Cavity As Built': "uninsulated cavity"
+
 }

backend/Property.py

@@ -1261,6 +1261,11 @@ class Property:
             # may be installed such that they are not visible from the street
             return False
 
+        if (self.data["property-type"] in ["House", "Bungalow"]) and (
+            not pd.isnull(self.roof["thermal_transmittance"])
+        ):
+            return True
+
         is_valid_property_type = self.data["property-type"] in ["House", "Bungalow", "Maisonette"]
         is_valid_roof_type = (
             self.roof["is_flat"] or self.roof["is_pitched"] or self.roof["is_roof_room"]

backend/engine/engine.py

@@ -411,7 +411,7 @@ def get_funding_data():
 
 
 async def model_engine(body: PlanTriggerRequest):
-    logger.info("Model Engine triggered with body: %s", body.model_dump_json())
+    logger.info("Model Engine triggered with body: %s", json.loads(body.model_dump_json()))
 
     logger.info("Connecting to db")
     session = sessionmaker(bind=db_engine)()

etl/customers/l_and_g/risk_matrix.py

@@ -0,0 +1,323 @@
+from itertools import product
+from recommendations.recommendation_utils import estimate_external_wall_area, estimate_windows
+
+import numpy as np
+
+import pandas as pd
+
+
+def app():
+    # Given a combination of variables, this code attempts to break down the costs of works to achieve upgrade
+    # targets
+
+    upgrade_path = [
+        "wall_insulation", "roof_insulation", "ventilation", "windows", "low_energy_lighting",
+        "heating", "solar"
+    ]
+
+    pricing_matrix = {
+        "Cavity wall insulation": 14.5,
+        "ventilation": 350,
+        "Room Roof Insulation": 210,
+        "Loft insulation": 15,
+        "Internal wall insulation": 215,
+        "External wall insulation": 298.35,
+        "Solid wall insulation": 215,
+        "LEDs": 35,  # per light
+        "Flat Roof Insulation": 195,
+        "Double Glazing": 1140,
+        "secondary_glazing": 970,
+        "5kw ASHP feeding heating & Hot water (dual tariff)": 14738,
+        "11.2kw ASHP feeding heating & Hot water (dual tariff)": 16541,
+        "3 kWp Solar PV": 4552.32,
+        "4 kWp Solar PV": 4892.8,
+        "4.3 kWp Solar PV": 4961.44,
+        "4.8 kWp Solar PV": 5414,
+        '5 kWp Solar PV': 5509.71,
+        '5.5 kWp Solar PV': 5631.92,
+        "HHRSH (dual tariff)": 1000,  # per heater
+        "Suspended floor insulation": 75
+    }
+    dwelling_types = [
+        "Semi Detached House",
+        "Detached house",
+        "Mid Terrace house",
+        "Mid Floor Flat",
+        "Top Floor Flat",
+        "Ground Floor Flat"
+    ]
+    num_floors_map = {
+        "Semi-detached house": 2,
+        "Detached house": 2,
+        "Mid Terrace house": 2,
+        "Mid Floor Flat": 1,
+        "Top Floor Flat": 1,
+        "Ground Floor Flat": 1
+    }
+    built_form_map = {
+        "Semi-detached house": "Semi-Detached",
+        "Detached house": "Detached",
+        "Mid Terrace house": "Mid Terrace",
+        "Mid Floor Flat": "Semi-Detached",
+        "Top Floor Flat": "Semi-Detached",
+        "Ground Floor Flat": "Semi-Detached"
+    }
+    lighting_count = {
+        "Semi-detached house": 15,
+        "Detached house": 19,
+        "Mid Terrace house": 12,
+        "Mid Floor Flat": 10,
+        "Top Floor Flat": 10,
+        "Ground Floor Flat": 10
+    }
+
+    # If we have a flat, we won't use the 199m2 floor area
+    floor_areas = [73, 97, 199]
+    # We remove age bracket, as we ended up with 360 combinations
+    # age_brackets = ["1945-1970", "1971-2002", "Post 2002"]
+    wall_type = ["cavity", "non-cavity"]
+    roof_type = ["pitched", "other"]
+    planning_constraints = [True, False]
+
+    # This is the list of all combinations of the above variables
+    combinations_untrimmed = product(
+        *[
+            dwelling_types, floor_areas, wall_type, roof_type, planning_constraints
+        ]
+    )
+
+    # TODO: Possibly need to add an additional cost for immersion hot water
+    combinations = []
+    for comb in combinations_untrimmed:
+        if "Flat" in comb[0] and comb[1] == 199:
+            continue
+
+        # If we have a flat, not too much difference if it's in a conservation area or not
+        if "Flat" in comb[0] and comb[4] is True:
+            continue
+        combinations.append(comb)
+
+    risk_matrix = []
+    for combination in combinations:
+        n_floors = num_floors_map[combination[0]]
+        bf = built_form_map[combination[0]]
+        pt = "House" if "Flat" not in combination[0] else "Flat"
+        # Model the home as a box
+        ground_floor_area = combination[1] / n_floors
+        perimeter = np.sqrt(ground_floor_area) * 4
+
+        # This is the amount of insulation required
+        external_wall_area = estimate_external_wall_area(
+            num_floors=n_floors,
+            floor_height=2.5,
+            perimeter=perimeter,
+            built_form=bf
+        )
+
+        n_rooms = np.floor(combination[1] / 15)
+
+        n_windows = estimate_windows(
+            property_type=pt,
+            built_form=bf,
+            construction_age_band="",
+            floor_area=combination[1],
+            number_habitable_rooms=n_rooms
+        )
+
+        # We determine the exact upgrade pathway for this combination, guided by the generic upgrade pathway
+        combination_upgrade_pathway = []
+        for upgrade in upgrade_path:
+            if upgrade == "wall_insulation":
+                if combination[2] == "cavity":
+                    combination_upgrade_pathway.append("cavity_wall_insulation")
+                else:
+                    combination_upgrade_pathway.append("solid_wall_insulation")
+                continue
+
+            if upgrade == "roof_insulation":
+                if combination[3] == "pitched":
+                    combination_upgrade_pathway.append("loft_insulation")
+                else:
+                    combination_upgrade_pathway.append("non_pitched_roof_insualtion")
+                continue
+
+            if upgrade == "ventilation":
+                combination_upgrade_pathway.append("ventilation")
+                continue
+
+            if upgrade == "low_energy_lighting":
+                combination_upgrade_pathway.append("low_energy_lighting")
+                continue
+
+            if upgrade == "windows":
+                if not combination[4]:
+                    combination_upgrade_pathway.append("double_glazing")
+                else:
+                    combination_upgrade_pathway.append("secondary_glazing")
+                continue
+
+            if upgrade == "heating":
+                if combination[0] in ["Semi Detached House", "Detached House"]:
+                    combination_upgrade_pathway.append("high_heat_retention_storage")
+                else:
+                    combination_upgrade_pathway.append("air_source_heat_pump")
+                continue
+
+            if upgrade == "solar":
+                if combination[0] in ["Semi Detached House", "Detached House", "Mid Terrace House"]:
+                    combination_upgrade_pathway.append("solar_pv")
+                continue
+
+        combination_costs = []
+        for measure in combination_upgrade_pathway:
+            unit_cost = pricing_matrix[measure]
+            # Wall insulation
+            if measure in ["cavity_wall_insulation", "internal_wall_insulation", "external_wall_insulation"]:
+                cost = unit_cost * external_wall_area
+            elif measure in ["loft_insulation"]:
+                cost = unit_cost * ground_floor_area
+            elif measure == "ventilation":
+                if combination[1] == 73:
+                    cost = unit_cost * 2
+                elif combination[1] == 97:
+                    cost = unit_cost * 3
+                else:
+                    cost = unit_cost * 4
+            elif measure == "low_energy_lighting":
+                n_lights = lighting_count[combination[0]]
+                if combination[1] == 73:
+                    inflation = 1
+                elif combination[1] == 97:
+                    inflation = 1.2
+                else:
+                    inflation = 1.5
+                cost = unit_cost * n_lights * inflation
+            elif measure in ["double_glazing", "secondary_glazing"]:
+                cost = unit_cost * n_windows
+            elif measure == "high_heat_retention_storage":
+                cost = unit_cost * n_rooms
+            elif measure in ["air_source_heat_pump", "solar_pv"]:
+                cost = unit_cost
+            else:
+                raise NotImplementedError("Implement: %s" % measure)
+
+            combination_costs.append(
+                {
+                    "measure": measure,
+                    "cost": cost
+                }
+            )
+
+        combination_costs = pd.DataFrame(combination_costs)
+
+    contingency = 0.26
+
+    epr_data = pd.read_excel(
+        "/Users/khalimconn-kowlessar/Documents/hestia/Customers/L&G/Risk Matrix/EPR Data.xlsx", header=1
+    )
+    epr_data["Measure added"].value_counts()
+    epr_data["row_id"] = epr_data.index
+    # We need to calculate the costs
+    cost_data = []
+    for _, row in epr_data.iterrows():
+        epc = row["EPC"][0]
+        sap = int(row["EPC"][1:])
+
+        n_floors = num_floors_map[row["Property Type"]]
+        bf = built_form_map[row["Property Type"]]
+        pt = "House" if "flat" not in row["Property Type"].lower() else "Flat"
+        # Model the home as a box
+        ground_floor_area = row["area"] / n_floors
+        perimeter = np.sqrt(ground_floor_area) * 4
+
+        # This is the amount of insulation required
+        external_wall_area = estimate_external_wall_area(
+            num_floors=n_floors,
+            floor_height=2.5,
+            perimeter=perimeter,
+            built_form=bf
+        )
+
+        n_rooms = np.floor(row["area"] / 15)
+
+        n_windows = estimate_windows(
+            property_type=pt,
+            built_form=bf,
+            construction_age_band="",
+            floor_area=row["area"],
+            number_habitable_rooms=n_rooms
+        )
+        cost_upper_bound = None
+        if pd.isnull(row["Measure added"]):
+            unit_cost = None
+        else:
+            measure = row["Measure added"]
+            unit_cost = pricing_matrix[measure]
+
+        if pd.isnull(row["Measure added"]):
+            cost = None
+        elif row["Measure added"] == "Loft insulation":
+            cost = unit_cost * ground_floor_area
+        elif row["Measure added"] in ["Cavity wall insulation", "Internal wall insulation"]:
+            cost = unit_cost * external_wall_area + pricing_matrix["ventilation"] * 3
+        elif row["Measure added"] == "Solid wall insulation":
+            cost = unit_cost * external_wall_area + pricing_matrix["ventilation"] * 3
+            cost_upper_bound = pricing_matrix["External wall insulation"] * external_wall_area + pricing_matrix[
+                "ventilation"] * 3
+        elif row["Measure added"] == "Double Glazing":
+            cost = unit_cost * n_windows
+        elif row["Measure added"] == "LEDs":
+            cost = unit_cost * lighting_count[row["Property Type"]]
+        elif row["Measure added"] in [
+            '5kw ASHP feeding heating & Hot water (dual tariff)',
+            '11.2kw ASHP feeding heating & Hot water (dual tariff)',
+            "3 kWp Solar PV",
+            '4 kWp Solar PV',
+            "4.3 kWp Solar PV",
+            '4.8 kWp Solar PV',
+            '5 kWp Solar PV',
+            '5.5 kWp Solar PV'
+        ]:
+            cost = unit_cost
+        elif row["Measure added"] == "HHRSH (dual tariff)":
+            cost = unit_cost * (n_rooms + 1)
+        elif row["Measure added"] == "Suspended floor insulation":
+            cost = unit_cost * ground_floor_area
+        else:
+            raise Exception()
+
+        cost_data.append(
+            {
+                "row_id": row["row_id"],
+                "epc": epc,
+                "sap": sap,
+                "cost": cost,
+                "cost upper bound": cost_upper_bound
+            }
+        )
+
+    cost_data = pd.DataFrame(cost_data)
+
+    risk_matrix = pd.merge(
+        epr_data,
+        cost_data,
+        on="row_id",
+    )
+
+    risk_matrix["contingency"] = risk_matrix["cost"] * contingency
+    risk_matrix["upper bound coningency"] = risk_matrix["cost upper bound"] * contingency
+
+    pricing_df = pd.DataFrame(
+        [
+            {
+                "Measure": k,
+                "Unit Cost": v
+            }
+            for k, v in pricing_matrix.items()
+        ]
+    )
+
+    with pd.ExcelWriter(
+        "/Users/khalimconn-kowlessar/Documents/hestia/Customers/L&G/Risk Matrix/risk_matrix.xlsx") as writer:
+        risk_matrix.to_excel(writer, sheet_name="Risk Matrix", index=False)
+        pricing_df.to_excel(writer, sheet_name="Pricing Assumptions", index=False)

etl/epc/Record.py

@@ -355,6 +355,7 @@ class EPCRecord:
         self._clean_floor_level()
         self._clean_floor_height()
         self._clean_constituency()
+        self._clean_new_build_descriptions()
 
         # self._clean_potential_energy_efficiency()
         # self._clean_environment_impact_potential()
@@ -397,6 +398,10 @@ class EPCRecord:
         if self.prepared_epc["floor-height"] <= 1.665:
             self.prepared_epc["floor-height"] = average
 
+    def _clean_new_build_descriptions(self):
+        for col in ['roof-description', 'walls-description', 'floor-description']:
+            self.prepared_epc[col] = self.prepared_epc[col].replace("W/m²K", "W/m-¦K")
+
     def _clean_constituency(self):
         """
         We handle the single case of finding a missing constituency by using the local authority

recommendations/RoofRecommendations.py

@@ -163,15 +163,12 @@ class RoofRecommendations:
         if self.property.roof["is_thatched"]:
             return
 
-        # If we have a u-value already, need to implement this
+        # If we have a u-value and we don't have a non-invasive recommendation, we can't recommend anything
-        if u_value:
+        if u_value and not any(
-            if u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE:
+            x in MEASURE_MAP["roof_insulation"] for x in [r["type"] for r in self.property.non_invasive_recommendations]
-                # The Roof is already compliant
+        ):
-                return
+            # We don't have enough information to provide a recommendation
-
+            return
-            if self.property.data["transaction-type"] in ["new dwelling", "not sale or rental"]:
-                return
-            raise NotImplementedError("Implement me")
 
         u_value = get_roof_u_value(
             insulation_thickness=self.property.roof["insulation_thickness"],

recommendations/WallRecommendations.py

@@ -227,24 +227,14 @@ class WallRecommendations(Definitions):
             # external wall insulation
             if (
                 (not is_cavity_wall)
-                and (self.property.year_built >= self.YEAR_WALLS_BUILT_WITH_INSULATION)
                 and (u_value >= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE)
             ):
                 # Recommend insulation
-                self.find_insulation(u_value, phase, measures)
+                self.find_insulation(u_value, phase, measures=measures, default_u_values=default_u_values)
                 return
 
-            # We can't detect it's a cavity wall, but it was built after 1990 so likely built with insulation already
+            # We have a sufficiently low U-value
-            # + it already has a U-value better than the building regulations, so we don't need to recommend anything
+            return
-            if (
-                (not is_cavity_wall)
-                and ((self.property.year_built >= self.YEAR_WALLS_BUILT_WITH_INSULATION)
-                     or (u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE))
-            ):
-                # Recommend nothing
-                return
-
-            raise NotImplementedError("Not implemented yet")
 
         u_value = get_wall_u_value(
             clean_description=self.property.walls["clean_description"],
@@ -626,7 +616,7 @@ class WallRecommendations(Definitions):
                         "walls_thermal_transmittance_ending": new_u_value
                     }
 
-                    if default_u_values:
+                    if default_u_values and "Average thermal transmittance" not in new_description:
                         # If we're using default U-values, we overwrite new_u_value
                         new_u_value = get_wall_u_value(
                             clean_description=new_description,