Merge branch 'main' of github.com:Hestia-Homes/Model into etl-michael

asset_list/AssetList.py

@@ -303,7 +303,7 @@ class AssetList:
 
     # Another version of non-intrusives:
     NON_INTRUSIVES_NEW_FORMAT_COLNAMES_V2 = [
-        'Archetype', 'Archetype 2', 'Construction', 'Insulated', 'Material', 'Boroscoped?',
+        'Archetype', 'Archetype 2', 'Construction', 'Insulated', 'Material', 'Borescoped?',
         'CIGA Check Required', 'ROOF ORIENTATION', 'TILE HUNG', 'RENDERED',
         'CLADDING', 'ACCESS ISSUES', 'FURTHER SURVEYOR NOTES', 'DATE',
         'NAME OF SURVEYOR'
@@ -860,7 +860,7 @@ class AssetList:
                         return date_str.year
 
                     # Handle numeric year (float or int)
-                    if isinstance(date_str, (int, float)):
+                    if isinstance(date_str, (int, float, np.int_)):
                         if 1000 <= int(date_str) <= 2100:
                             return int(date_str)
 
@@ -887,9 +887,6 @@ class AssetList:
                     self.landlord_year_built
                 ].apply(extract_year)
 
-                for x in self.standardised_asset_list[self.landlord_year_built].values:
-                    extract_year(x)
-
         # We now create standard lookups
         to_remap = {
             self.landlord_property_type: {
@@ -949,9 +946,19 @@ class AssetList:
 
         if self.phase:
             # We filter on just the properties that have had an inspection
-            self.standardised_asset_list = self.standardised_asset_list[
-                ~self.standardised_asset_list['Surveyors Name'].isin(["YET TO BE SURVEYED"])
-            ]
+            if self.new_format_non_insturives_present_v2:
+                self.standardised_asset_list = self.standardised_asset_list[
+                    ~self.standardised_asset_list['NAME OF SURVEYOR'].isin(
+                        ["YET TO BE SURVEYED", "", None]
+                    )
+                ]
+                self.standardised_asset_list = self.standardised_asset_list[
+                    ~pd.isnull(self.standardised_asset_list["NAME OF SURVEYOR"])
+                ]
+            else:
+                self.standardised_asset_list = self.standardised_asset_list[
+                    ~self.standardised_asset_list['Surveyors Name'].isin(["YET TO BE SURVEYED"])
+                ]
 
         if not self.variable_mappings and not override_empty_mappings:
             raise ValueError("Please run init_standardise first")
@@ -1336,7 +1343,9 @@ class AssetList:
 
             if self.new_format_non_insturives_present_v2:
                 existing_solar_non_intrusives_check = (
-                    self.standardised_asset_list["non-intrusives: ROOF ORIENTATION"] == "ALREADY HAS SOLAR PV"
+                    self.standardised_asset_list["non-intrusives: ROOF ORIENTATION"].str.strip().isin(
+                        ["ALREADY HAS SOLAR PV"]
+                    )
                 )
             else:
                 existing_solar_non_intrusives_check = (
@@ -2110,6 +2119,7 @@ class AssetList:
         RANGE_RE = re.compile(r'\b(\d+[A-Za-z]?)\s*[-–]\s*(\d+[A-Za-z]?)\b')
         NUM_RE = re.compile(r'\b\d+[A-Za-z]?\b')  # captures 12, 12A, etc.
         TO_RANGE_RE = re.compile(r'\b(\d+[A-Za-z]?)\s+(?:to|To|TO)\s+(\d+[A-Za-z]?)\b')  # captures "13 to 15"
+        LETTER_RANGE_RE = re.compile(r'\b(\d+)([A-Za-z]?)\s*[-–]\s*(\d+)([A-Za-z]?)\b')  # captures "1A-3B"
 
         expanded_rows = []
 
@@ -2166,7 +2176,7 @@ class AssetList:
                     # We update the full address
 
                     new[self.DOMNA_PROPERTY_ID] = f"{row[self.DOMNA_PROPERTY_ID]}-{new_addr}"
-                    expanded_rows.append(new)
+                    expanded_rows.append(new.to_dict())
                 continue
 
             # 2 ─ Explicit list  (e.g. 1, 2, 5 Block) or split by an ampersand (e.g. 1 & 2 Block)
@@ -2177,12 +2187,36 @@ class AssetList:
                     new_addr = re.sub(NUM_RE, n, addr, count=1)  # replace the first number only
                     new[self.STANDARD_ADDRESS_1] = new_addr
                     new[self.DOMNA_PROPERTY_ID] = f"{row[self.DOMNA_PROPERTY_ID]}-{new_addr}"
-                    expanded_rows.append(new)
+                    expanded_rows.append(new.to_dict())
                 continue
 
-            # 3 ─ Single number or no number, treat as individual dwelling
+            # Check for a range of lettered addresses e.g 31A - 31D
+            letter_range = LETTER_RANGE_RE.search(full_addr)
+            if letter_range:
+                start_num, start_letter, end_num, end_letter = letter_range.groups()
+                start_num, end_num = int(start_num), int(end_num)
+                if start_num != end_num:
+                    raise NotImplementedError(f"Unusual range - handle me")
+
+                # We define the looping range on whether we have odd, even or all numbers
+                house_number_range = range(start_num, end_num + 1)
+                if has_odd:
+                    house_number_range = [x for x in house_number_range if x % 2 != 0]
+                if has_even:
+                    house_number_range = [x for x in house_number_range if x % 2 == 0]
+
+                for n in house_number_range:
+                    for letter in range(ord(start_letter), ord(end_letter) + 1):
+                        new = row.copy()
+                        new_addr = f"{n}{chr(letter)}"
+                        new[self.STANDARD_ADDRESS_1] = new_addr
+                        new[self.DOMNA_PROPERTY_ID] = f"{row[self.DOMNA_PROPERTY_ID]}-{new_addr}"
+                        expanded_rows.append(new.to_dict())
+                continue
+
+            # 4 ─ Single number or no number, treat as individual dwelling
             if (len(nums) == 1) or not nums:
-                expanded_rows.append(row)
+                expanded_rows.append(row.to_dict())
                 continue
 
             # Anything else with digits is unrecognised

asset_list/app.py

@@ -59,324 +59,426 @@ def app():
     Property UPRN
     """
 
-    data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Broadlands"
-    data_filename = "Broadlands Asset List.xlsx"
-    sheet_name = "Assets"
-    postcode_column = 'POSTCODE'
-    fulladdress_column = None
-    address1_column = "Address1"
+    # Colchester
+    data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Colchester/Aug2025 202 inspections"
+    data_filename = "Colchester Borough Homes - Inspections - Additional 202 Addresses JW 280725 copy.xlsx"
+    sheet_name = "Extra 202 Colchester Addresses"
+    postcode_column = 'domna_postcode'
+    address1_column = "domna_address_1"
     address1_method = None
-    address_cols_to_concat = ["Address1"]
+    fulladdress_column = "domna_full_address"
+    address_cols_to_concat = []
     missing_postcodes_method = None
-    landlord_year_built = "DATEBUILT"
+    landlord_year_built = None
     landlord_os_uprn = None
-    landlord_property_type = "PropertyType"
-    landlord_built_form = "PropertyType"
+    landlord_property_type = "landlord_property_type"
+    landlord_built_form = "landlord_built_form"
     landlord_wall_construction = None
-    landlord_heating_system = "Heating Fuel"
-    landlord_existing_pv = None
-    landlord_property_id = "Row ID"
-    outcomes_filename = [os.path.join(data_folder, "outcomes.xlsx")]
-    outcomes_sheetname = ["Sheet1"]
-    outcomes_postcode = ["Postcode"]
-    outcomes_houseno = ["No."]
-    outcomes_address = ["Address"]
-    outcomes_id = [None]
-    master_filepaths = [
-        os.path.join(data_folder, "eco3 submissions.csv"),
-        os.path.join(data_folder, "eco4 submissions.csv"),
-    ]
-    master_to_asset_list_filepath = None
-    asset_list_header = 0
-    landlord_block_reference = None
-    master_id_colnames = [None, None]
     landlord_roof_construction = None
-    phase = False
+    landlord_heating_system = None
+    landlord_existing_pv = None
+    landlord_property_id = "landlord_property_id"
     landlord_sap = None
-    ecosurv_landlords = "broadland"
+    outcomes_filename = None
+    outcomes_sheetname = None
+    outcomes_postcode = None
+    outcomes_houseno = None
+    outcomes_id = None
+    outcomes_address = None
+    master_filepaths = []
+    master_id_colnames = []
+    master_to_asset_list_filepath = None
+    phase = False
+    ecosurv_landlords = None
+    asset_list_header = 0
+    landlord_block_reference = "landlord_block_reference"
+
+    # # Abri
+    # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Abri/Post Inspections"
+    # data_filename = "Desktop ABRI data - Standardised After Programmes (2).xlsx"
+    # sheet_name = "Reviewed List"
+    # postcode_column = 'domna_postcode'
+    # address1_column = "domna_address_1"
+    # address1_method = None
+    # fulladdress_column = "domna_full_address"
+    # address_cols_to_concat = []
+    # missing_postcodes_method = None
+    # landlord_year_built = "landlord_year_built"
+    # landlord_os_uprn = None
+    # landlord_property_type = "PropertyType_original_from_landlord"
+    # landlord_built_form = "BuildForm_original_from_landlord"
+    # landlord_wall_construction = "Wall Construction_original_from_landlord"
+    # landlord_roof_construction = None
+    # landlord_heating_system = "HeatingType_original_from_landlord"
+    # landlord_existing_pv = None
+    # landlord_property_id = "landlord_property_id"
+    # landlord_sap = None
+    # outcomes_filename = None
+    # outcomes_sheetname = None
+    # outcomes_postcode = None
+    # outcomes_houseno = None
+    # outcomes_id = None
+    # outcomes_address = None
+    # master_filepaths = []
+    # master_id_colnames = []
+    # master_to_asset_list_filepath = None
+    # phase = False
+    # ecosurv_landlords = None
+    # asset_list_header = 0
+    # landlord_block_reference = None
+
+    # Freebridge
+    # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Freebridge"
+    # data_filename = "Domna - FCH property data May 25 copy.xlsx"
+    # sheet_name = "EPC Data"
+    # postcode_column = 'Post Code'
+    # address1_column = "Address 1"
+    # address1_method = None
+    # fulladdress_column = None
+    # address_cols_to_concat = ["Address 1", "Address 4"]
+    # missing_postcodes_method = None
+    # landlord_year_built = "Build Date"
+    # landlord_os_uprn = None
+    # landlord_property_type = "Property Type"
+    # landlord_built_form = None
+    # landlord_wall_construction = "Walls Description"
+    # landlord_heating_system = "Heating Type"
+    # landlord_existing_pv = None
+    # landlord_property_id = "Place Ref"
+    # landlord_roof_construction = "Roof Description"
+    # landlord_sap = "Current SAP"
+    # outcomes_filename = []
+    # outcomes_sheetname = []
+    # outcomes_postcode = []
+    # outcomes_houseno = []
+    # outcomes_address = []
+    # outcomes_id = []
+    # master_filepaths = []
+    # master_to_asset_list_filepath = None
+    # asset_list_header = 0
+    # landlord_block_reference = None
+    # master_id_colnames = []
+    # phase = True  # Inspections not complete, produce a partial view
+    # ecosurv_landlords = None
+
+    # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Broadlands"
+    # data_filename = "Broadlands Asset List.xlsx"
+    # sheet_name = "Assets"
+    # postcode_column = 'POSTCODE'
+    # fulladdress_column = None
+    # address1_column = "Address1"
+    # address1_method = None
+    # address_cols_to_concat = ["Address1"]
+    # missing_postcodes_method = None
+    # landlord_year_built = "DATEBUILT"
+    # landlord_os_uprn = None
+    # landlord_property_type = "PropertyType"
+    # landlord_built_form = "PropertyType"
+    # landlord_wall_construction = None
+    # landlord_heating_system = "Heating Fuel"
+    # landlord_existing_pv = None
+    # landlord_property_id = "Row ID"
+    # outcomes_filename = [os.path.join(data_folder, "outcomes.xlsx")]
+    # outcomes_sheetname = ["Sheet1"]
+    # outcomes_postcode = ["Postcode"]
+    # outcomes_houseno = ["No."]
+    # outcomes_address = ["Address"]
+    # outcomes_id = [None]
+    # master_filepaths = [
+    #     os.path.join(data_folder, "eco3 submissions.csv"),
+    #     os.path.join(data_folder, "eco4 submissions.csv"),
+    # ]
+    # master_to_asset_list_filepath = None
+    # asset_list_header = 0
+    # landlord_block_reference = None
+    # master_id_colnames = [None, None]
+    # landlord_roof_construction = None
+    # phase = False
+    # landlord_sap = None
+    # ecosurv_landlords = "broadland"
+    # #
     #
-
-    # Community:
-    data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Community Housing/New Programme"
-    data_filename = "SUB EPC C to DOMNA - 24.07.25.xlsx"
-    sheet_name = "Sheet1"
-    postcode_column = 'POSTCODE'
-    fulladdress_column = "ADDRESS"
-    address1_column = None
-    address1_method = "house_number_extraction"
-    address_cols_to_concat = []
-    missing_postcodes_method = None
-    landlord_year_built = "BUILD DATE"
-    landlord_os_uprn = None
-    landlord_property_type = "PROPERTY TYPE"
-    landlord_built_form = "Archetype"  # Using the inspections archetype
-    landlord_wall_construction = "CONSTRUCTION TYPE"
-    landlord_roof_construction = None
-    landlord_heating_system = None
-    landlord_existing_pv = None
-    landlord_property_id = "UPRN"
-    landlord_sap = None
-    outcomes_filename = []
-    outcomes_sheetname = []
-    outcomes_postcode = []
-    outcomes_houseno = []
-    outcomes_id = []
-    outcomes_address = []
-    master_filepaths = []
-    master_to_asset_list_filepath = None
-    phase = False
-    ecosurv_landlords = None
-    asset_list_header = 1
-    landlord_block_reference = None
-    master_id_colnames = []
-
-    data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Ealing/Programme Analysis"
-    data_filename = "EalingProjectRebuildJW210725.xlsx"
-    sheet_name = "Refine & Houses"
-    postcode_column = 'Postcode'
-    fulladdress_column = "Address"
-    address1_column = None
-    address1_method = "house_number_extraction"
-    address_cols_to_concat = []
-    missing_postcodes_method = None
-    landlord_year_built = None
-    landlord_os_uprn = None
-    landlord_property_type = None  # Using the inspections property type
-    landlord_built_form = None
-    landlord_wall_construction = None
-    landlord_roof_construction = None
-    landlord_heating_system = None
-    landlord_existing_pv = None
-    landlord_property_id = "Property ref"
-    landlord_sap = None
-    outcomes_filename = []
-    outcomes_sheetname = []
-    outcomes_postcode = []
-    outcomes_houseno = []
-    outcomes_id = []
-    outcomes_address = []
-    master_filepaths = []
-    master_to_asset_list_filepath = None
-    phase = False
-    ecosurv_landlords = None
-    asset_list_header = 0
-    landlord_block_reference = "Block Reference"
-    master_id_colnames = []
-
-    # TODO: Delete me
-    data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/NRLA/"
-    data_filename = "20250716 Asset List.xlsx"
-    sheet_name = "Sheet 1"
-    postcode_column = 'Postcode'
-    fulladdress_column = "Full Address"
-    address1_column = None
-    address1_method = "house_number_extraction"
-    address_cols_to_concat = []
-    missing_postcodes_method = None
-    landlord_year_built = None
-    landlord_os_uprn = None
-    landlord_property_type = None
-    landlord_built_form = None
-    landlord_wall_construction = None
-    landlord_heating_system = None
-    landlord_existing_pv = None
-    landlord_property_id = "Row ID"
-    outcomes_filename = []
-    outcomes_sheetname = []
-    outcomes_postcode = []
-    outcomes_houseno = []
-    outcomes_address = []
-    outcomes_id = []
-    master_filepaths = []
-    master_to_asset_list_filepath = None
-    asset_list_header = 0
-    landlord_block_reference = None
-    master_id_colnames = []
-    landlord_roof_construction = None
-    phase = False
-    landlord_sap = None
-    ecosurv_landlords = None
-
-    # Southend
-    data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Southend/July 2025 Programme"
-    data_filename = "SOUTHEND - RYAN.xlsx"
-    sheet_name = "July 2025 Surveys"
-    postcode_column = 'Postcode'
-    fulladdress_column = "Full postal address"
-    address1_column = None
-    address1_method = "house_number_extraction"
-    address_cols_to_concat = []
-    missing_postcodes_method = None
-    landlord_year_built = "Property age"
-    landlord_os_uprn = None
-    landlord_property_type = "Property type"
-    landlord_built_form = "Property type"
-    landlord_wall_construction = None
-    landlord_heating_system = None
-    landlord_existing_pv = None
-    landlord_property_id = "ID"
-    outcomes_filename = []
-    outcomes_sheetname = []
-    outcomes_postcode = []
-    outcomes_houseno = []
-    outcomes_address = []
-    outcomes_id = []
-    master_filepaths = []
-    master_to_asset_list_filepath = None
-    asset_list_header = 0
-    landlord_block_reference = None
-    master_id_colnames = []
-    landlord_roof_construction = None
-    phase = False
-    landlord_sap = None
-    ecosurv_landlords = None
-
-    # For Rooftop
-    data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Rooftop"
-    data_filename = "Rooftop Asset List - July 2025.xlsx"
-    sheet_name = "Sheet1"
-    postcode_column = 'post_code'
-    fulladdress_column = None
-    address1_column = "add_1"
-    address1_method = None
-    address_cols_to_concat = [
-        "add_1", "add_2", "add_3", "add_4"
-    ]
-    missing_postcodes_method = None
-    landlord_year_built = "date_built"
-    landlord_os_uprn = None
-    landlord_property_type = "ConstructionStyle"
-    landlord_built_form = "ConstructionStyle"
-    landlord_wall_construction = None
-    landlord_heating_system = "Description"
-    landlord_existing_pv = None
-    landlord_property_id = "PropertyCode"
-    outcomes_filename = [os.path.join(data_folder, "Rooftop_Outcomes.xlsx")]
-    outcomes_sheetname = ["OUTCOMES"]
-    outcomes_postcode = ["POSTCODE"]
-    outcomes_houseno = ["NO"]
-    outcomes_address = ["ADDRESS"]
-    outcomes_id = [None]
-    master_filepaths = [os.path.join(data_folder, "Master.csv")]
-    master_to_asset_list_filepath = None
-    asset_list_header = 1
-    landlord_block_reference = "bl_rec_ref"
-    master_id_colnames = [None]
-    landlord_roof_construction = None
-    phase = False
-    landlord_sap = None
-    ecosurv_landlords = "rooftop"
-
-    # For Housing
-    data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/For Housing/New Programme July 2025"
-    data_filename = "FOR HOUSING Asset List (Combined).xlsx"
-    sheet_name = "Asset List"
-    postcode_column = 'Postcode'
-    fulladdress_column = "Address"
-    address1_column = None
-    address1_method = "house_number_extraction"
-    address_cols_to_concat = []
-    missing_postcodes_method = None
-    landlord_year_built = None
-    landlord_os_uprn = None
-    landlord_property_type = "Type"
-    landlord_built_form = "Type"
-    landlord_wall_construction = None
-    landlord_heating_system = "Heating - full"
-    landlord_existing_pv = None
-    landlord_property_id = "UPRN"
-    outcomes_filename = [os.path.join(data_folder, "Khalim Combined - for analysis.xlsx")]
-    outcomes_sheetname = ["Sheet1"]
-    outcomes_postcode = ["POSTCODE"]
-    outcomes_houseno = ["NO"]
-    outcomes_address = ["ADDRESS"]
-    outcomes_id = [None]
-    master_filepaths = [os.path.join(data_folder, "submissions.csv")]
-    master_to_asset_list_filepath = None
-    asset_list_header = 0
-    landlord_block_reference = None
-    master_id_colnames = [None]
-    landlord_roof_construction = None
-    phase = False
-    landlord_sap = "SAP"
-    ecosurv_landlords = "for housing"
-
-    # CDS
-    data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/CDS"
-    data_filename = "Founder Estates - Asset List.xlsx"
-    sheet_name = "Combined"
-    postcode_column = 'Postcode'
-    fulladdress_column = "Address"
-    address1_column = None
-    address1_method = "house_number_extraction"
-    address_cols_to_concat = []
-    missing_postcodes_method = None
-    landlord_year_built = None
-    landlord_os_uprn = None
-    landlord_property_type = None
-    landlord_built_form = None
-    landlord_wall_construction = None
-    landlord_heating_system = "Heating Type"
-    landlord_existing_pv = None
-    landlord_property_id = "Row ID"
-    outcomes_filename = []
-    outcomes_sheetname = []
-    outcomes_postcode = []
-    outcomes_houseno = []
-    outcomes_address = []
-    outcomes_id = []
-    master_filepaths = [os.path.join(data_folder, "submissions.csv")]
-    master_to_asset_list_filepath = None
-    asset_list_header = 0
-    landlord_block_reference = None
-    master_id_colnames = [None]
-    landlord_roof_construction = None
-    phase = False
-    landlord_sap = None
-    ecosurv_landlords = "cds"
-
-    # Plus Dane
-    data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Plus Dane/New Programme July 2025/"
-    data_filename = "20250711 Plus Dane Asset List.xlsx"
-    sheet_name = "Sheet1"
-    postcode_column = 'Postcode'
-    fulladdress_column = "Address"
-    address1_column = None
-    address1_method = "house_number_extraction"
-    address_cols_to_concat = []
-    missing_postcodes_method = None
-    landlord_year_built = "Property Age"
-    landlord_os_uprn = None
-    landlord_property_type = "Property Type"
-    landlord_built_form = "Built Form"
-    landlord_wall_construction = "Wall Construction"
-    landlord_heating_system = "Full Heating System"
-    landlord_existing_pv = None
-    landlord_property_id = "UPRN"
-    outcomes_filename = [
-        os.path.join(data_folder, "Outcomes - Plus Dane_CWI_2024.xlsx"),
-        os.path.join(data_folder, "Outcomes - Plus Dane_CWI_2025.xlsx"),
-        os.path.join(data_folder, "Outcomes - Plus Dane_PV_2025.xlsx"),
-    ]
-    outcomes_sheetname = [
-        "CWI & LI - 2024", "2025 - CWI", "PV - 2025",
-    ]
-    outcomes_postcode = ["Postcode", "Postcode", "Postcode"]
-    outcomes_houseno = ["No.", "No", "No"]
-    outcomes_address = ["Address", "Address", "Address"]
-    outcomes_id = ["Asset Reference", "LL UPRN", "LL UPRN"]
-    master_filepaths = [
-        os.path.join(data_folder, "submissions/JJC-Table 1.csv"),
-        os.path.join(data_folder, "submissions/SCIS-Table 1.csv")
-    ]
-    master_to_asset_list_filepath = None
-    asset_list_header = 1
-    landlord_block_reference = None
-    master_id_colnames = [None, None]
-    landlord_roof_construction = None
-    phase = False
-    landlord_sap = "SAP Rating"
-    ecosurv_landlords = "plus dane"
+    # # Community:
+    # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Community Housing/New Programme"
+    # data_filename = "SUB EPC C to DOMNA - 24.07.25.xlsx"
+    # sheet_name = "Sheet1"
+    # postcode_column = 'POSTCODE'
+    # fulladdress_column = "ADDRESS"
+    # address1_column = None
+    # address1_method = "house_number_extraction"
+    # address_cols_to_concat = []
+    # missing_postcodes_method = None
+    # landlord_year_built = "BUILD DATE"
+    # landlord_os_uprn = None
+    # landlord_property_type = "PROPERTY TYPE"
+    # landlord_built_form = "Archetype"  # Using the inspections archetype
+    # landlord_wall_construction = "CONSTRUCTION TYPE"
+    # landlord_roof_construction = None
+    # landlord_heating_system = None
+    # landlord_existing_pv = None
+    # landlord_property_id = "UPRN"
+    # landlord_sap = None
+    # outcomes_filename = []
+    # outcomes_sheetname = []
+    # outcomes_postcode = []
+    # outcomes_houseno = []
+    # outcomes_id = []
+    # outcomes_address = []
+    # master_filepaths = []
+    # master_to_asset_list_filepath = None
+    # phase = False
+    # ecosurv_landlords = None
+    # asset_list_header = 1
+    # landlord_block_reference = None
+    # master_id_colnames = []
+    #
+    # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Ealing/Programme Analysis"
+    # data_filename = "EalingProjectRebuildJW210725.xlsx"
+    # sheet_name = "Refine & Houses"
+    # postcode_column = 'Postcode'
+    # fulladdress_column = "Address"
+    # address1_column = None
+    # address1_method = "house_number_extraction"
+    # address_cols_to_concat = []
+    # missing_postcodes_method = None
+    # landlord_year_built = None
+    # landlord_os_uprn = None
+    # landlord_property_type = None  # Using the inspections property type
+    # landlord_built_form = None
+    # landlord_wall_construction = None
+    # landlord_roof_construction = None
+    # landlord_heating_system = None
+    # landlord_existing_pv = None
+    # landlord_property_id = "Property ref"
+    # landlord_sap = None
+    # outcomes_filename = []
+    # outcomes_sheetname = []
+    # outcomes_postcode = []
+    # outcomes_houseno = []
+    # outcomes_id = []
+    # outcomes_address = []
+    # master_filepaths = []
+    # master_to_asset_list_filepath = None
+    # phase = False
+    # ecosurv_landlords = None
+    # asset_list_header = 0
+    # landlord_block_reference = "Block Reference"
+    # master_id_colnames = []
+    #
+    # # TODO: Delete me
+    # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/NRLA/"
+    # data_filename = "20250716 Asset List.xlsx"
+    # sheet_name = "Sheet 1"
+    # postcode_column = 'Postcode'
+    # fulladdress_column = "Full Address"
+    # address1_column = None
+    # address1_method = "house_number_extraction"
+    # address_cols_to_concat = []
+    # missing_postcodes_method = None
+    # landlord_year_built = None
+    # landlord_os_uprn = None
+    # landlord_property_type = None
+    # landlord_built_form = None
+    # landlord_wall_construction = None
+    # landlord_heating_system = None
+    # landlord_existing_pv = None
+    # landlord_property_id = "Row ID"
+    # outcomes_filename = []
+    # outcomes_sheetname = []
+    # outcomes_postcode = []
+    # outcomes_houseno = []
+    # outcomes_address = []
+    # outcomes_id = []
+    # master_filepaths = []
+    # master_to_asset_list_filepath = None
+    # asset_list_header = 0
+    # landlord_block_reference = None
+    # master_id_colnames = []
+    # landlord_roof_construction = None
+    # phase = False
+    # landlord_sap = None
+    # ecosurv_landlords = None
+    #
+    # # Southend
+    # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Southend/July 2025 Programme"
+    # data_filename = "SOUTHEND - RYAN.xlsx"
+    # sheet_name = "July 2025 Surveys"
+    # postcode_column = 'Postcode'
+    # fulladdress_column = "Full postal address"
+    # address1_column = None
+    # address1_method = "house_number_extraction"
+    # address_cols_to_concat = []
+    # missing_postcodes_method = None
+    # landlord_year_built = "Property age"
+    # landlord_os_uprn = None
+    # landlord_property_type = "Property type"
+    # landlord_built_form = "Property type"
+    # landlord_wall_construction = None
+    # landlord_heating_system = None
+    # landlord_existing_pv = None
+    # landlord_property_id = "ID"
+    # outcomes_filename = []
+    # outcomes_sheetname = []
+    # outcomes_postcode = []
+    # outcomes_houseno = []
+    # outcomes_address = []
+    # outcomes_id = []
+    # master_filepaths = []
+    # master_to_asset_list_filepath = None
+    # asset_list_header = 0
+    # landlord_block_reference = None
+    # master_id_colnames = []
+    # landlord_roof_construction = None
+    # phase = False
+    # landlord_sap = None
+    # ecosurv_landlords = None
+    #
+    # # For Rooftop
+    # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Rooftop"
+    # data_filename = "Rooftop Asset List - July 2025.xlsx"
+    # sheet_name = "Sheet1"
+    # postcode_column = 'post_code'
+    # fulladdress_column = None
+    # address1_column = "add_1"
+    # address1_method = None
+    # address_cols_to_concat = [
+    #     "add_1", "add_2", "add_3", "add_4"
+    # ]
+    # missing_postcodes_method = None
+    # landlord_year_built = "date_built"
+    # landlord_os_uprn = None
+    # landlord_property_type = "ConstructionStyle"
+    # landlord_built_form = "ConstructionStyle"
+    # landlord_wall_construction = None
+    # landlord_heating_system = "Description"
+    # landlord_existing_pv = None
+    # landlord_property_id = "PropertyCode"
+    # outcomes_filename = [os.path.join(data_folder, "Rooftop_Outcomes.xlsx")]
+    # outcomes_sheetname = ["OUTCOMES"]
+    # outcomes_postcode = ["POSTCODE"]
+    # outcomes_houseno = ["NO"]
+    # outcomes_address = ["ADDRESS"]
+    # outcomes_id = [None]
+    # master_filepaths = [os.path.join(data_folder, "Master.csv")]
+    # master_to_asset_list_filepath = None
+    # asset_list_header = 1
+    # landlord_block_reference = "bl_rec_ref"
+    # master_id_colnames = [None]
+    # landlord_roof_construction = None
+    # phase = False
+    # landlord_sap = None
+    # ecosurv_landlords = "rooftop"
+    #
+    # # For Housing
+    # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/For Housing/New Programme July 2025"
+    # data_filename = "FOR HOUSING Asset List (Combined).xlsx"
+    # sheet_name = "Asset List"
+    # postcode_column = 'Postcode'
+    # fulladdress_column = "Address"
+    # address1_column = None
+    # address1_method = "house_number_extraction"
+    # address_cols_to_concat = []
+    # missing_postcodes_method = None
+    # landlord_year_built = None
+    # landlord_os_uprn = None
+    # landlord_property_type = "Type"
+    # landlord_built_form = "Type"
+    # landlord_wall_construction = None
+    # landlord_heating_system = "Heating - full"
+    # landlord_existing_pv = None
+    # landlord_property_id = "UPRN"
+    # outcomes_filename = [os.path.join(data_folder, "Khalim Combined - for analysis.xlsx")]
+    # outcomes_sheetname = ["Sheet1"]
+    # outcomes_postcode = ["POSTCODE"]
+    # outcomes_houseno = ["NO"]
+    # outcomes_address = ["ADDRESS"]
+    # outcomes_id = [None]
+    # master_filepaths = [os.path.join(data_folder, "submissions.csv")]
+    # master_to_asset_list_filepath = None
+    # asset_list_header = 0
+    # landlord_block_reference = None
+    # master_id_colnames = [None]
+    # landlord_roof_construction = None
+    # phase = False
+    # landlord_sap = "SAP"
+    # ecosurv_landlords = "for housing"
+    #
+    # # CDS
+    # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/CDS"
+    # data_filename = "Founder Estates - Asset List.xlsx"
+    # sheet_name = "Combined"
+    # postcode_column = 'Postcode'
+    # fulladdress_column = "Address"
+    # address1_column = None
+    # address1_method = "house_number_extraction"
+    # address_cols_to_concat = []
+    # missing_postcodes_method = None
+    # landlord_year_built = None
+    # landlord_os_uprn = None
+    # landlord_property_type = None
+    # landlord_built_form = None
+    # landlord_wall_construction = None
+    # landlord_heating_system = "Heating Type"
+    # landlord_existing_pv = None
+    # landlord_property_id = "Row ID"
+    # outcomes_filename = []
+    # outcomes_sheetname = []
+    # outcomes_postcode = []
+    # outcomes_houseno = []
+    # outcomes_address = []
+    # outcomes_id = []
+    # master_filepaths = [os.path.join(data_folder, "submissions.csv")]
+    # master_to_asset_list_filepath = None
+    # asset_list_header = 0
+    # landlord_block_reference = None
+    # master_id_colnames = [None]
+    # landlord_roof_construction = None
+    # phase = False
+    # landlord_sap = None
+    # ecosurv_landlords = "cds"
+    #
+    # # Plus Dane
+    # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Plus Dane/New Programme July 2025/"
+    # data_filename = "20250711 Plus Dane Asset List.xlsx"
+    # sheet_name = "Sheet1"
+    # postcode_column = 'Postcode'
+    # fulladdress_column = "Address"
+    # address1_column = None
+    # address1_method = "house_number_extraction"
+    # address_cols_to_concat = []
+    # missing_postcodes_method = None
+    # landlord_year_built = "Property Age"
+    # landlord_os_uprn = None
+    # landlord_property_type = "Property Type"
+    # landlord_built_form = "Built Form"
+    # landlord_wall_construction = "Wall Construction"
+    # landlord_heating_system = "Full Heating System"
+    # landlord_existing_pv = None
+    # landlord_property_id = "UPRN"
+    # outcomes_filename = [
+    #     os.path.join(data_folder, "Outcomes - Plus Dane_CWI_2024.xlsx"),
+    #     os.path.join(data_folder, "Outcomes - Plus Dane_CWI_2025.xlsx"),
+    #     os.path.join(data_folder, "Outcomes - Plus Dane_PV_2025.xlsx"),
+    # ]
+    # outcomes_sheetname = [
+    #     "CWI & LI - 2024", "2025 - CWI", "PV - 2025",
+    # ]
+    # outcomes_postcode = ["Postcode", "Postcode", "Postcode"]
+    # outcomes_houseno = ["No.", "No", "No"]
+    # outcomes_address = ["Address", "Address", "Address"]
+    # outcomes_id = ["Asset Reference", "LL UPRN", "LL UPRN"]
+    # master_filepaths = [
+    #     os.path.join(data_folder, "submissions/JJC-Table 1.csv"),
+    #     os.path.join(data_folder, "submissions/SCIS-Table 1.csv")
+    # ]
+    # master_to_asset_list_filepath = None
+    # asset_list_header = 1
+    # landlord_block_reference = None
+    # master_id_colnames = [None, None]
+    # landlord_roof_construction = None
+    # phase = False
+    # landlord_sap = "SAP Rating"
+    # ecosurv_landlords = "plus dane"
 
     # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Brentwood/July 2025 New Programme"
     # data_filename = "20250710 Asset List Brentwood.xlsx"

asset_list/hubspot/config.py

@@ -1,6 +1,6 @@
 from enum import IntEnum, Enum
 
-CRM_PIPELINE_NAME = 'Operations - Housing Associations'
+CRM_PIPELINE_NAME = 'Operations - Social Housing'
 
 
 class HubspotProcessStatus(IntEnum):

asset_list/hubspot/prepare_for_hubspot.py

@@ -44,26 +44,27 @@ def app():
     """
 
     # inputs:
-    reconcile_programme = True  # If True, the hubspot upload will include all properties with a project code
-    customer_domain = "https://southend.gov.uk"
-    installer_name = "J & J CRUMP"
+    reconcile_programme = False  # If True, the hubspot upload will include all properties with a project code
+    customer_domain = "https://shgroup.org.uk"
+    installer_name = "SCIS"
     asset_list_filepath = (
-        "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Southend/July 2025 Programme/SOUTHEND - RYAN - "
-        "Standardised 2.xlsx"
+        "/Users/khalimconn-kowlessar/Downloads/20250701 Optivo Southern - Standardised.xlsx"
     )
-    asset_list_sheet_name = "Standardised Asset List"
+    asset_list_sheet_name = "Solar Route Revised (100)"
     asset_list_header = 0
 
-    contact_details_filepath = None
-    contacts_sheet_name = "Sheet 1"
-    contacts_landlord_property_id = "UPRN"
-    contacts_phone_number_column = "phone_number"
-    contacts_secondary_phone_number_column = "secondary_phone_number"
-    contacts_secondary_contact_full_name = "secondary_contact_full_name"
-    contacts_email_column = "email"
-    contacts_fullname_column = "fullname"
-    contacts_firstname_column = "First Name"
-    contacts_lastname_column = "Last Name"
+    contact_details_filepath = (
+        "/Users/khalimconn-kowlessar/Downloads/southern_optivo_solar_pv.xlsx"
+    )
+    contacts_sheet_name = "Sheet1"
+    contacts_landlord_property_id = "landlord_property_id"
+    contacts_phone_number_column = "Primary phone number"
+    contacts_secondary_phone_number_column = "Secondary phone number"
+    contacts_secondary_contact_full_name = None
+    contacts_email_column = "Email Address"
+    contacts_fullname_column = None
+    contacts_firstname_column = "Name"
+    contacts_lastname_column = None
 
     existing_programme_filepath = None
 
@@ -89,6 +90,18 @@ def app():
         reconcile_programme=reconcile_programme
     )
 
+    for x in asset_list.hubspot_data["Phone <CONTACT phone>"].values:
+        normalize_uk_phone(x)
+
+    asset_list.hubspot_data["Phone <CONTACT phone>"] = (
+        asset_list.hubspot_data["Phone <CONTACT phone>"].astype("Int64").astype(str).apply(normalize_uk_phone)
+    )
+    asset_list.hubspot_data["Secondary Phone <CONTACT secondary_phone_number>"] = asset_list.hubspot_data[
+        "Secondary Phone <CONTACT secondary_phone_number>"].astype(
+        "Int64").astype(
+        str).apply(
+        normalize_uk_phone)
+
     # Remove the existing programme
     # existing_programme = pd.read_csv(existing_programme_filepath, encoding="utf-8-sig")
     # asset_list.hubspot_data = asset_list.hubspot_data[

asset_list/mappings/heating_systems.py

@@ -431,6 +431,47 @@ HEATING_MAPPINGS = {
     'Mains Electric': 'electric fuel',
     'Unvented cylinder': 'other',
     'MVHR & Heat Recovery': 'other',
-    'Solar': 'other'
+    'Solar': 'other',
 
+    'Electric storage heaters, Electric storage heaters': 'electric storage heaters',
+    'Room heaters, electric': 'room heaters',
+    'Room heaters, mains gas, Room heaters, electric': 'room heaters',
+    'Air source heat pump, underfloor, electric': 'air source heat pump',
+    'Air source heat pump, radiators, electric': 'air source heat pump',
+    'Air source heat pump, Systems with radiators, electric': 'air source heat pump',
+    'Electric storage heaters': 'electric storage heaters',
+    'Air source heat pump, Underfloor heating and radiators, pipes in screed above insulation, electric': 'air source '
+                                                                                                          'heat pump',
+    'Room heaters, coal': 'room heaters',
+    'Room heaters, electric, Electric storage heaters': 'electric storage heaters',
+    'Air source heat pump, fan coil units, electric': 'air source heat pump',
+    'Boiler and radiators, mains gas': 'gas boiler, radiators',
+    'Boiler and radiators, mains gas, Electric storage heaters': 'condensing boiler, radiators',
+    'Room heaters, mains gas': 'room heaters',
+    'Air source heat pump, radiators, electric, Air source heat pump, fan coil units, electric': 'air source heat pump',
+    'Air source heat pump, warm air, electric': 'air source heat pump',
+    'Electric ceiling heating, electric': 'electric ceiling',
+    'Electric storage heaters, Room heaters, electric': 'electric storage heaters',
+    'Room heaters, dual fuel (mineral and wood)': 'room heaters',
+    'Water source heat pump, radiators, electric': 'other',
+    'Warm air, electric': 'warm air heating',
+    'Boiler and radiators, wood logs': 'solid fuel',
+    'Boiler and radiators, dual fuel (mineral and wood)': 'solid fuel',
+    'Boiler & underfloor, mains gas': 'gas boiler, radiators',
+    'Boiler and underfloor heating, mains gas': 'gas boiler, radiators',
+    'Community scheme': 'communal heating',
+    'Warm air, Electricaire': 'warm air heating',
+    'Boiler and radiators, smokeless fuel': 'solid fuel',
+    'Warm air, mains gas': 'warm air heating',
+    'Warm air , electric': 'warm air heating',
+    'Boiler and radiators, LPG': 'boiler - other fuel',
+    'Boiler & underfloor, oil': 'oil boiler',
+    'Boiler and radiators, bottled LPG': 'boiler - other fuel',
+    'Boiler and underfloor heating, oil': 'oil boiler',
+    'SAP05:Main-Heating': 'unknown',
+    'Boiler and radiators, coal': 'solid fuel',
+    'Boiler and radiators, oil': 'oil boiler',
+    'Boiler and radiators, electric': 'electric boiler',
+    'No system present: electric heaters assumed': 'electric radiators',
+    'Boiler and radiators, anthracite': 'solid fuel'
 }

asset_list/mappings/property_type.py

@@ -337,5 +337,9 @@ PROPERTY_MAPPING = {
     'Maisonette - Mid Terrace': 'maisonette',
     'Chalet - Wheelchair': 'other',
     'Amenity Block - Detached': 'other',
-    'Amenity Block - Semi detached': 'other'
+    'Amenity Block - Semi detached': 'other',
+    'house': 'house',
+    'block of flats': 'block of flats',
+    'bungalow': 'bungalow',
+    'flat': 'flat'
 }

asset_list/mappings/roof.py

@@ -10,8 +10,12 @@ STANDARD_ROOF_CONSTRUCTIONS = {
     "another dwelling above",
     "flat unknown insulation",
     "flat insulated",
+    "flat uninsulated",
     "unknown insulated",
     "unknown",
+    "room roof insulated",
+    "room roof uninsulated",
+    "average thermal transmittance",
 }
 
 ROOF_CONSTRUCTION_MAPPINGS = {
@@ -173,6 +177,73 @@ ROOF_CONSTRUCTION_MAPPINGS = {
 
     'PitchedNormalNoLoftAccess: Unknown': 'pitched no access to loft',
     'PitchedNormalLoftAccess: Unknown': 'pitched unknown insulation',
-    'AnotherDwellingAbove: Unknown': 'another dwelling above'
+    'AnotherDwellingAbove: Unknown': 'another dwelling above',
+
+    'Flat, insulated': 'flat insulated',
+    'Pitched, insulated (assumed)': 'pitched insulated',
+    'Flat, insulated (assumed)': 'flat insulated',
+    '(another dwelling above)': 'another dwelling above',
+    'Pitched, insulated at rafters': 'pitched insulated',
+    '(other premises above)': 'another dwelling above',
+    'Average thermal transmittance 0.15 W/m-¦K': 'average thermal transmittance',
+    'Pitched, 25 mm loft insulation': 'pitched less than 100mm insulation',
+    'Roof room(s), insulated (assumed)': 'room roof insulated',
+    'Pitched, limited insulation (assumed)': 'pitched less than 100mm insulation',
+    'Pitched, 270 mm loft insulation': 'pitched insulated',
+    'Pitched, 250 mm loft insulation': 'pitched insulated',
+    'Pitched, 200mm loft insulation': 'pitched insulated',
+    'Flat, no insulation': 'flat uninsulated',
+
+    'Pitched, 75 mm loft insulation': 'pitched less than 100mm insulation',
+    'Average thermal transmittance 0.09 W/m-¦K': 'average thermal transmittance',
+
+    'SAP05:Roof': 'unknown',
+    'Pitched, 400 mm loft insulation': 'pitched insulated',
+    'Pitched, 150mm loft insulation': 'pitched insulated',
+    'Average thermal transmittance 0.11 W/m-¦K': 'unknown',
+    'Pitched, 100 mm loft insulation': 'pitched less than 100mm insulation',
+    'Pitched, 300 mm loft insulation': 'pitched insulated',
+    'Pitched, 75mm loft insulation': 'pitched less than 100mm insulation',
+    'Pitched, 300+mm loft insulation': 'pitched insulated',
+    'Pitched, 300+  mm loft insulation': 'pitched insulated',
+    'Average thermal transmittance 0.11 W/m?K': 'average thermal transmittance',
+    'Average thermal transmittance 0.10 W/m?K': 'average thermal transmittance',
+    'Pitched, 250mm loft insulation': 'pitched insulated',
+    'Pitched, 300+ mm loft insulation': 'pitched insulated',
+    'Average thermal transmittance 0.1 W/m-¦K': 'average thermal transmittance',
+    'Pitched, *** INVALID INPUT Code : 57 *** loft insulation': 'unknown',
+    'Pitched, 100mm loft insulation': 'pitched less than 100mm insulation',
+    'Pitched, loft insulation': 'pitched less than 100mm insulation',
+    'Average thermal transmittance 0.20 W/m?K': 'average thermal transmittance',
+    'Average thermal transmittance 0.1 W/m?K': 'average thermal transmittance',
+    'Average thermal transmittance 0.16 W/m-¦K': 'average thermal transmittance',
+    'Average thermal transmittance 0.14 W/m?K': 'average thermal transmittance',
+    'Pitched, 50 mm loft insulation': 'pitched less than 100mm insulation',
+    'Flat, limited insulation': 'flat uninsulated',
+    'Average thermal transmittance 0.12 W/m?K': 'average thermal transmittance',
+    'Roof room(s), ceiling insulated': 'room roof insulated',
+    'Average thermal transmittance 0.18 W/m?K': 'average thermal transmittance',
+    'Average thermal transmittance 0.10 W/m-¦K': 'average thermal transmittance',
+    'Pitched, 400+ mm loft insulation': 'pitched insulated',
+    'Average thermal transmittance 0.14 W/m²K': 'average thermal transmittance',
+    'Pitched, no insulation (assumed)': 'pitched less than 100mm insulation',
+    'Average thermal transmittance 0.16 W/m?K': 'average thermal transmittance',
+    'Average thermal transmittance 0.21 W/m?K': 'average thermal transmittance',
+    'Flat, no insulation (assumed)': 'flat uninsulated',
+    'Pitched, no insulation': 'pitched less than 100mm insulation',
+
+    'Average thermal transmittance 0.12 W/m-¦K': 'average thermal transmittance',
+    'Pitched, 12 mm loft insulation': 'pitched less than 100mm insulation',
+    'Average thermal transmittance 0.07 W/m-¦K': 'average thermal transmittance',
+    'Roof room(s), no insulation (assumed)': 'room roof uninsulated',
+    'Pitched, no insulation(assumed)': 'pitched less than 100mm insulation',
+    'Average thermal transmittance 0.13 W/m-¦K': 'average thermal transmittance',
+    'Average thermal transmittance 0.08 W/m-¦K': 'average thermal transmittance',
+    'Average thermal transmittance 0.14 W/m-¦K': 'average thermal transmittance',
+    'Pitched, 350 mm loft insulation': 'pitched insulated',
+    'Average thermal transmittance 0 W/m-¦K': 'average thermal transmittance',
+    'Pitched, 200 mm loft insulation': 'pitched insulated',
+    'Pitched, 150 mm loft insulation': 'pitched insulated',
+    'Flat, limited insulation (assumed)': 'flat uninsulated',
 
 }

asset_list/mappings/walls.py

@@ -334,4 +334,13 @@ WALL_CONSTRUCTION_MAPPINGS = {
     'Cavity: FilledCavity, TimberFrame: AsBuilt': 'filled cavity',
     'Cavity: FilledCavity, SolidBrick: AsBuilt, SolidBrick: Internal': 'filled cavity',
     'Cavity: Internal, SolidBrick: AsBuilt': 'filled cavity',
+
+    'Timber frame, filled cavity': 'filled cavity',
+    'Cob, as built': 'cob',
+    'Cavity wall, filled cavity and internal insulation': 'filled cavity',
+    'SAP05:Walls': 'other',
+    'Solid brick, as built, partial insulation (assumed)': 'insulated solid brick',
+    'Sandstone, as built, no insulation (assumed)': 'uninsulated sandstone or limestone',
+    'System built, as built, partial insulation (assumed)': 'system built unknown insulation',
+    'Timber frame, with external insulation': 'insulated timber frame'
 }

backend/Property.py

@@ -213,9 +213,16 @@ class Property:
         self.parse_kwargs(kwargs)
 
         # Funding
-        self.gbis_eligibiltiy = None
-        self.eco4_eligibility = None
-        self.whlg_eligibility = None
+        # self.gbis_eligibiltiy = None
+        # self.eco4_eligibility = None
+        # self.whlg_eligibility = None
+        self.scheme = None
+        self.funded_measures = None
+        self.project_funding = None
+        self.total_uplift = None
+        self.full_project_score = None
+        self.partial_project_score = None
+        self.uplift_project_score = None
 
         # Ventilation
         self.has_ventilation = self.identify_ventilation()
@@ -531,7 +538,7 @@ class Property:
                 "internal_wall_insulation", "external_wall_insulation", "cavity_wall_insulation",
                 "cylinder_thermostat", "loft_insulation", "room_roof_insulation", "flat_roof_insulation",
                 "solid_floor_insulation", "suspended_floor_insulation", "mixed_glazing",
-                "windows_glazing", "mechanical_ventilation"
+                "windows_glazing", "mechanical_ventilation", "solar_pv"
             ]:
                 # We update the data, as defined in the recommendaton
                 for prefix in ["walls", "roof", "floor"]:
@@ -547,9 +554,6 @@ class Property:
 
                 output.update(simulation_config)
 
-            if recommendation["type"] == "solar_pv":
-                output["photo_supply_ending"] = recommendation["photo_supply"]
-
             if recommendation["type"] not in [
                 "sealing_open_fireplace", "low_energy_lighting",
                 "internal_wall_insulation", "external_wall_insulation", "cavity_wall_insulation",
@@ -1241,15 +1245,11 @@ class Property:
         ):
             return True
 
-        suitable_house = self.data["property-type"] == "House" and self.data["built-form"] in [
-            "Detached", "Semi-Detached", "End-Terrace",
-        ]
+        suitable_property_type = (
+            self.data["property-type"] in ["House", "Bungalow"] and
+            self.data["built-form"] not in ["Enclosed Mid-Terrace", "Enclosed End-Terrace"]
+        )
 
-        suitable_bungalow = self.data["property-type"] == "Bungalow" and self.data["built-form"] in [
-            "Detached", "Semi-Detached"
-        ]
-
-        suitable_property_type = suitable_house or suitable_bungalow
         has_air_source_heat_pump = self.main_heating["has_air_source_heat_pump"]
 
         return suitable_property_type and not has_air_source_heat_pump
@@ -1343,13 +1343,26 @@ class Property:
 
         return electric_consumption
 
-    def insert_funding(self, funding_calulator: Funding):
+    def insert_funding(
+        self,
+        scheme,
+        funded_measures,
+        project_funding,
+        total_uplift,
+        full_project_score,
+        partial_project_score,
+        uplift_project_score
+    ):
         """
         This method inserts the funding into the property object
         """
-        self.gbis_eligibiltiy = funding_calulator.gbis_eligibiltiy
-        self.eco4_eligibility = funding_calulator.eco4_eligibility
-        self.whlg_eligibility = funding_calulator.whlg_eligibility
+        self.scheme = scheme
+        self.funded_measures = funded_measures
+        self.project_funding = project_funding
+        self.total_uplift = total_uplift
+        self.full_project_score = full_project_score
+        self.partial_project_score = partial_project_score
+        self.uplift_project_score = uplift_project_score
 
     def identify_ventilation(self):
 

backend/apis/GoogleSolarApi.py

@@ -60,7 +60,7 @@ class GoogleSolarApi:
     # Error Messages
     ENTITY_NOT_FOUND_ERROR = 'Requested entity was not found.'
 
-    def __init__(self, api_key, max_retries=5):
+    def __init__(self, api_key, solar_materials: list, max_retries=5):
         """
         Initialize the GoogleSolarApi class with the provided API key and maximum retries.
 
@@ -77,7 +77,6 @@ class GoogleSolarApi:
         # property attributes:
         self.floor_area = None
         self.roof_area = None
-        self.roof_segment_indexes = None
         self.panel_area = assumptions.RDSAP_AREA_PER_PANEL
         self.panel_wattage = None
         self.panel_performance = None
@@ -87,6 +86,9 @@ class GoogleSolarApi:
 
         # Indicates if we think we have both units attached to a semi-detached property
         self.double_property = False
+        self.solar_materials = solar_materials
+
+        self.allowed_segment_indices = None
 
     def get_building_insights(self, longitude, latitude, required_quality="MEDIUM", max_retries=None):
         """
@@ -202,19 +204,17 @@ class GoogleSolarApi:
             # It should be straightforward, but I'd rather see an actual instance of this happening
             raise NotImplementedError("Panel wattage is not 400W - implement me")
 
-        self.roof_segment_indexes = [segment['segmentIndex'] for segment in self.roof_segments]
-
         # We now start finding the solar panel configurations
         self.optimise_solar_configuration(
             energy_consumption=energy_consumption,
             is_building=is_building,
-            property_instance=property_instance
+            property_instance=property_instance,
         )
 
         # Finally, if we have a double property, we half the data we stored area
         if self.double_property:
             self.roof_area = self.roof_area / 2
-            self.floor_area = self.floor_area / 2
+            self.floor_area = float(self.floor_area) / 2
 
     def save_to_db(self, session, uprns_to_location, scenario_type):
         if self.insights_data is None:
@@ -279,7 +279,9 @@ class GoogleSolarApi:
                 installation_life_span)) /
             (1 - efficiency_depreciation_factor))
 
-    def optimise_solar_configuration(self, energy_consumption, is_building=False, property_instance=None):
+    def optimise_solar_configuration(
+        self, energy_consumption, is_building=False, property_instance=None
+    ):
         """
         Optimise the solar panel configuration for the building.
         :return:
@@ -291,10 +293,10 @@ class GoogleSolarApi:
         # Remove any north facing roof segments
         panel_performance = []
         for config in self.insights_data["solarPotential"].get("solarPanelConfigs", []):
-            roof_segment_summaries = config["roofSegmentSummaries"]
-            # Filter on just the segments in self.roof_segment_indexes
+
             roof_segment_summaries = [
-                segment for segment in roof_segment_summaries if segment["segmentIndex"] in self.roof_segment_indexes
+                s for s in config.get("roofSegmentSummaries", [])
+                if s["segmentIndex"] in self.allowed_segment_indices
             ]
 
             roi_summary = []
@@ -321,9 +323,25 @@ class GoogleSolarApi:
             if roi_summary["n_panels"].sum() < min_panels:
                 continue
 
+            total_panels = roi_summary["n_panels"].sum()
+            # find a product which is suitable for the ROI calc, without a battery. 400 Watts for the baseline
+            solar_product = next(
+                (m for m in self.solar_materials if m["type"] == "solar_pv" and
+                 abs(m["size"] - (400 * total_panels) / 1000) < 0.1 and not m["includes_battery"]),
+                None
+            )
+
+            if solar_product is None:
+                logger.info("No suitable solar product found for the configuration with %d panels.", total_panels)
+                continue
+
             total_cost = Costs.solar_pv(
-                n_panels=roi_summary["n_panels"].sum(),
-                has_battery=False,
+                solar_product=solar_product,
+                # We don't actually need scaffolding for the ROI calc
+                scaffolding_options=[
+                    {"total_cost": 1000, "size": property_instance.number_of_floors},
+                    {"total_cost": 1000, "size": 3}
+                ],
                 # Assume the most amount of scaffolding
                 n_floors=3 if property_instance is None else property_instance.number_of_floors
             )["total"]
@@ -525,23 +543,24 @@ class GoogleSolarApi:
 
     def exclude_north_facing_segments(self, property_instance):
         """
-        Filter out any north-facing roof segments from the roof_segments attribute.
-
-        North-facing segments are defined as those with an azimuth between -30 and 30 degrees.
+        Filter out any north-facing roof segments from self.roof_segments.
+        Keep API's original segmentIndex; optionally add a localIndex.
         """
+        is_flat = property_instance.roof["is_flat"]
 
-        filtered_segments = []
-        for segment_index, segment in enumerate(self.roof_segments):
-            segment["segmentIndex"] = segment_index
-            # Check if the segment is north-facing
-            if (
-                self.NORTH_FACING_AZIMUTH_RANGE[0] <= segment['azimuthDegrees'] <= self.NORTH_FACING_AZIMUTH_RANGE[1]
-            ) and not property_instance.roof["is_flat"]:
+        kept = []
+        allowed = set()
+        for i, seg in enumerate(self.roof_segments):  # i is the API segmentIndex
+            if not is_flat and (
+                self.NORTH_FACING_AZIMUTH_RANGE[0] <= seg['azimuthDegrees'] <= self.NORTH_FACING_AZIMUTH_RANGE[1]):
                 continue
+            s = dict(seg)
+            s["localIndex"] = len(kept)  # for charts/UI only
+            kept.append(s)
+            allowed.add(i)  # this i IS the API segmentIndex
 
-            filtered_segments.append(segment)
-
-        self.roof_segments = filtered_segments
+        self.roof_segments = kept
+        self.allowed_segment_indices = allowed
 
     @staticmethod
     def haversine(lat1, lon1, lat2, lon2):
@@ -723,7 +742,8 @@ class GoogleSolarApi:
 
     @classmethod
     def building_solar_analysis(
-        cls, building_solar_config: List, input_properties: List[Property], session, google_solar_api_key: str
+        cls, building_solar_config: List, input_properties: List[Property], session, google_solar_api_key: str,
+        solar_materials: list
     ):
         """
         Perform the solar analysis for the building level
@@ -731,6 +751,7 @@ class GoogleSolarApi:
         :param input_properties: List of properties
         :param session: Database session
         :param google_solar_api_key: Google Solar API key
+        :param solar_materials: List of solar materials
         :return:
         """
 
@@ -769,7 +790,7 @@ class GoogleSolarApi:
             energy_consumption = sum(
                 [entry['energy_consumption'] for entry in building_solar_config if entry['building_id'] == building_id]
             )
-            solar_api_client = cls(api_key=google_solar_api_key)
+            solar_api_client = cls(api_key=google_solar_api_key, solar_materials=solar_materials)
             solar_api_client.get(
                 longitude=coordinates["longitude"],
                 latitude=coordinates["latitude"],
@@ -805,7 +826,8 @@ class GoogleSolarApi:
 
     @classmethod
     def unit_solar_analysis(
-        cls, unit_solar_config: List, input_properties: List[Property], session, body, google_solar_api_key: str
+        cls, unit_solar_config: List, input_properties: List[Property], session, body, google_solar_api_key: str,
+        solar_materials: list
     ):
 
         if not unit_solar_config:
@@ -844,7 +866,7 @@ class GoogleSolarApi:
                 )
                 continue
 
-            solar_api_client = cls(api_key=google_solar_api_key)
+            solar_api_client = cls(api_key=google_solar_api_key, solar_materials=solar_materials)
             solar_api_client.get(
                 longitude=unit["longitude"],
                 latitude=unit["latitude"],
@@ -852,7 +874,7 @@ class GoogleSolarApi:
                 is_building=False,
                 session=session,
                 uprn=unit["uprn"],
-                property_instance=property_instance
+                property_instance=property_instance,
             )
 
             # Store the data in the database

backend/app/db/functions/funding_functions.py

@@ -0,0 +1,68 @@
+from sqlalchemy.orm import Session
+from sqlalchemy.exc import SQLAlchemyError
+from backend.app.db.models.funding import FundingPackage, FundingPackageMeasures
+
+
+def upload_funding(session: Session, p, plan_id, recommendations_to_upload):
+    try:
+        # Prepare data for bulk insert for Recommendation
+        funding_package_data = {
+            "plan_id": plan_id,
+            "scheme": p.scheme,
+            "project_funding": float(p.project_funding),
+            "total_uplift": float(p.total_uplift),
+            "full_project_score": float(p.full_project_score),
+            "partial_project_score": float(p.partial_project_score),
+            "uplift_project_score": float(p.uplift_project_score)
+        }
+
+        # upload the funding package data and get back the ID
+        new_funding_package = FundingPackage(**funding_package_data)
+        session.add(new_funding_package)
+        session.flush()
+        session.commit()
+
+        funding_package_id = new_funding_package.id
+
+        # We now prepare the list of funding measures to be uploaded
+        funding_measures_data = []
+        for part in p.funded_measures:
+            recommendation_id = part["id"]
+            recommendation = next(
+                (x for x in recommendations_to_upload if x["recommendation_id"] == recommendation_id), {}
+            )
+            material_id = None
+            if recommendation["parts"]:
+                material_id = recommendation["parts"][0]["id"]
+
+            part_type = part["type"]
+            if part_type == "extension_cavity_wall_insulation":
+                part_type = "cavity_wall_insulation"
+            if part_type == "sealing_open_fireplace":
+                part_type = "sealing_fireplace"
+            funding_measures_data.append({
+                "funding_package_id": funding_package_id,
+                "measure": part_type,
+                "material_id": material_id,
+                "innovation_uplift": float(part["innovation_uplift"]),
+                "partial_project_score": float(part["partial_project_score"]),
+                "uplift_project_score": float(part["uplift_project_score"])
+            })
+
+        # Bulk insert the funding measures data
+        if funding_measures_data:
+            session.bulk_insert_mappings(FundingPackageMeasures, funding_measures_data)
+
+            # flush the changes to get the newly created IDs
+            session.flush()
+
+            # Commit the transaction
+            session.commit()
+
+        return True
+
+    except SQLAlchemyError as e:
+        # Rollback the transaction in case of an error
+        session.rollback()
+        print(f"An error occurred: {e}")
+        return False

backend/app/db/functions/portfolio_functions.py

@@ -29,6 +29,7 @@ def aggregate_portfolio_recommendations(
         .one()
     )
 
+    # Contingeny and funding are in the aggregated data
     aggregates_dict = {
         "cost": aggregates.cost or 0,
         "total_work_hours": aggregates.total_work_hours or 0,

backend/app/db/functions/recommendations_functions.py

@@ -7,6 +7,7 @@ from backend.app.db.models.recommendations import (
 from backend.app.db.models.portfolio import (
     PropertyModel, PropertyTargetsModel, PropertyDetailsEpcModel
 )
+from backend.app.db.models.funding import FundingPackageMeasures, FundingPackage
 
 
 def create_plan(session: Session, plan):
@@ -138,9 +139,9 @@ def upload_recommendations(session: Session, recommendations_to_upload, property
                 "recommendation_id": recommendation_id,
                 "material_id": part["id"],
                 "depth": int(part["depth"]) if part["depth"] else None,
-                "quantity": float(part["quantity"]),
-                "quantity_unit": part["quantity_unit"],
-                "estimated_cost": part["total"],
+                "quantity": float(part["quantity"]) if part.get("quantity") else None,
+                "quantity_unit": part.get("quantity_unit", None),
+                "estimated_cost": float(part.get("total", part.get("total_cost"))),
             }
             for rec, recommendation_id in zip(recommendations_to_upload, uploaded_recommendation_ids)
             for part in rec["parts"]
@@ -176,6 +177,10 @@ def clear_portfolio(session: Session, portfolio_id: int):
     recommendation_ids = session.query(Recommendation.id).filter(Recommendation.property_id.in_(property_ids)).all()
     recommendation_ids = [r.id for r in recommendation_ids]
 
+    # Fetch all plan IDs associated with the portfolio
+    plan_ids = session.query(Plan.id).filter(Plan.portfolio_id == portfolio_id).all()
+    plan_ids = [p.id for p in plan_ids]
+
     # Delete all entries from RecommendationMaterials for these recommendations
     session.execute(
         delete(RecommendationMaterials).where(RecommendationMaterials.recommendation_id.in_(recommendation_ids))
@@ -186,6 +191,16 @@ def clear_portfolio(session: Session, portfolio_id: int):
         session.query(Plan.id).filter(Plan.portfolio_id == portfolio_id).subquery().as_scalar()
     )))
 
+    # Delete FundingPackageMeasures → FundingPackage → Plan
+    session.execute(
+        delete(FundingPackageMeasures).where(FundingPackageMeasures.funding_package_id.in_(
+            session.query(FundingPackage.id).filter(FundingPackage.plan_id.in_(plan_ids))
+        ))
+    )
+    session.execute(
+        delete(FundingPackage).where(FundingPackage.plan_id.in_(plan_ids))
+    )
+
     # Delete all Plans associated with the portfolio
     session.execute(delete(Plan).where(Plan.portfolio_id == portfolio_id))
 

backend/app/db/models/funding.py

@@ -0,0 +1,48 @@
+import enum
+
+from sqlalchemy import Column, Integer, String, Float, Enum, TIMESTAMP, BigInteger, ForeignKey
+from sqlalchemy.orm import declarative_base
+from sqlalchemy.sql import func
+from backend.app.db.models.recommendations import Plan
+from backend.app.db.models.materials import MaterialType, Material
+
+Base = declarative_base()
+
+
+class SchemeEnum(enum.Enum):
+    eco4 = "eco4"
+    gbis = "gbis"
+    whlg = "whlg"
+    none = "none"
+
+
+class FundingPackage(Base):
+    __tablename__ = 'funding_package'
+
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    plan_id = Column(BigInteger, ForeignKey(Plan.id), nullable=False)
+    scheme = Column(
+        Enum(SchemeEnum, values_callable=lambda x: [e.value for e in x], create_constraint=False),
+        nullable=False
+    )
+    created_at = Column(TIMESTAMP, nullable=False, server_default=func.now())
+    project_funding = Column(Float)
+    total_uplift = Column(Float)
+    full_project_score = Column(Float)
+    partial_project_score = Column(Float)
+    uplift_project_score = Column(Float)
+
+
+class FundingPackageMeasures(Base):
+    __tablename__ = 'funding_package_measures'
+
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    funding_package_id = Column(BigInteger, ForeignKey(FundingPackage.id), nullable=False)
+    measure = Column(
+        Enum(MaterialType, values_callable=lambda x: [e.value for e in x], create_constraint=False),
+        nullable=False
+    )
+    material_id = Column(BigInteger, ForeignKey(Material.id), nullable=False)  # Assuming material table exists
+    innovation_uplift = Column(Float)
+    partial_project_score = Column(Float)
+    uplift_project_score = Column(Float)

backend/app/db/models/materials.py

@@ -38,12 +38,27 @@ class MaterialType(enum.Enum):
     flat_roof_preparation = "flat_roof_preparation"
     flat_roof_vapour_barrier = "flat_roof_vapour_barrier"
     flat_roof_waterproofing = "flat_roof_waterproofing"
+    trickle_vent = "trickle_vent"
+    door_undercut = "door_undercut"
+    solar_pv = "solar_pv"
+    solar_battery = "solar_battery"
+    scaffolding = "scaffolding"
+    high_heat_retention_storage_heaters = "high_heat_retention_storage_heaters"
+    sealing_fireplace = "sealing_fireplace"
 
 
 class DepthUnit(enum.Enum):
     mm = "mm"
 
 
+class SizeUnit(enum.Enum):
+    # ["kWp", "kW", "watt", "storey"]
+    kWp = "kWp"
+    kW = "kW"
+    watt = "watt"
+    storey = "storey"
+
+
 class CostUnit(enum.Enum):
     gbp_sq_meter = "gbp_sq_meter"
     gbp_per_unit = "gbp_per_unit"
@@ -90,3 +105,11 @@ class Material(Base):
     total_cost = Column(Float)
     notes = Column(String)
     is_installer_quote = Column(Boolean, nullable=False, default=False)
+    innovation_rate = Column(Float, default=0.0)
+    size = Column(Float)
+    size_unit = Column(
+        Enum(SizeUnit, values_callable=lambda x: [e.value for e in x]), nullable=True
+    )
+    includes_scaffolding = Column(Boolean, default=False)
+    includes_battery = Column(Boolean, default=False)
+    battery_size = Column(Float)

backend/app/db/models/recommendations.py

@@ -91,6 +91,8 @@ class Scenario(Base):
 
     # Add in the fields we need, which were previously sitting at the portfolio level
     cost = Column(Float)
+    contingency = Column(Float)
+    funding = Column(Float)
     total_work_hours = Column(Float)
     energy_savings = Column(Float)
     co2_equivalent_savings = Column(Float)

backend/app/plan/schemas.py

@@ -8,15 +8,26 @@ TYPICAL_MEASURE_TYPES = [
     "secondary_heating", "solar_pv"
 ]
 
-SPECIFIC_MEASURES = [
-    "internal_wall_insulation", "external_wall_insulation", "cavity_wall_insulation",
-    "loft_insulation", "flat_roof_insulation", "room_roof_insulation",
-    "suspended_floor_insulation", "solid_floor_insulation",
-    "boiler_upgrade", "high_heat_retention_storage_heater", "air_source_heat_pump",
-    "secondary_heating", "solar_pv", "double_glazing", "secondary_glazing",
-    "ventilation", "low_energy_lighting", "fireplace", "hot_water_tank_insulation",
-    "cylinder_thermostat"
+WALL_INSULATION_MEASURES = ["internal_wall_insulation", "external_wall_insulation", "cavity_wall_insulation"]
+ROOF_INSULATION_MEASURES = ["loft_insulation", "flat_roof_insulation", "room_roof_insulation"]
+
+# Both all and roof insulaiton measures are eligible for ECO4. These are the remaining fabric and heating measures
+# This is based on th measures we have recommendations for
+ECO4_ELIGIBILE_FABRIC_MEASURES = [
+    "suspended_floor_insulation", "solid_floor_insulation", "double_glazing", "secondary_glazing"
 ]
+ECO4_ELIGIBLE_HEATING_MEASURES = [
+    "boiler_upgrade", "high_heat_retention_storage_heater", "air_source_heat_pump", "solar_pv"
+]
+
+SPECIFIC_MEASURES = (
+    WALL_INSULATION_MEASURES + ROOF_INSULATION_MEASURES + ECO4_ELIGIBILE_FABRIC_MEASURES +
+    ECO4_ELIGIBLE_HEATING_MEASURES + [
+        "secondary_heating", "ventilation", "low_energy_lighting", "fireplace",
+        "hot_water_tank_insulation",
+        "cylinder_thermostat"
+    ]
+)
 
 INSULATION_MEASURES = [
     "internal_wall_insulation", "external_wall_insulation", "cavity_wall_insulation",
@@ -96,7 +107,6 @@ class PlanTriggerRequest(BaseModel):
     scenario_name: Optional[str] = ""
     scenario_id: Optional[str | int] = None  # Used to utilise and existing scenario for a engine run
     multi_plan: Optional[bool] = False
-    optimise: Optional[bool] = True
     default_u_values: Optional[bool] = True
 
     ashp_cop: Optional[float] = 2.8

backend/engine/engine.py

@@ -1,5 +1,6 @@
 import ast
 import json
+from copy import deepcopy
 from datetime import datetime
 
 from tqdm import tqdm
@@ -22,9 +23,10 @@ from backend.app.db.functions.property_functions import (
 from backend.app.db.functions.recommendations_functions import (
     create_plan, upload_recommendations, create_scenario
 )
+from backend.app.db.functions.funding_functions import upload_funding
 from backend.app.db.functions.energy_assessment_functions import get_latest_assessment_by_uprn
 from backend.app.db.models.portfolio import rating_lookup
-from backend.app.plan.schemas import PlanTriggerRequest
+from backend.app.plan.schemas import PlanTriggerRequest, WALL_INSULATION_MEASURES, ROOF_INSULATION_MEASURES
 from backend.app.plan.utils import get_cleaned
 from backend.app.utils import sap_to_epc
 import backend.app.assumptions as assumptions
@@ -45,6 +47,10 @@ from etl.bill_savings.KwhData import KwhData
 from etl.spatial.OpenUprnClient import OpenUprnClient
 from etl.find_my_epc.RetrieveFindMyEpc import RetrieveFindMyEpc
 
+from backend.Funding import Funding
+from recommendations.optimiser.funding_optimiser import optimise_with_funding_paths
+from recommendations.recommendation_utils import convert_thickness_to_numeric, get_wall_u_value
+
 logger = setup_logger()
 
 BATCH_SIZE = 5
@@ -161,7 +167,9 @@ def extract_portfolio_aggregation_data(
             "sap_point_improvement": sap_point_improvement,
             "lower_bound_valuation_uplift": lower_bound_valuation_uplift,
             "upper_bound_valuation_uplift": upper_bound_valuation_uplift,
-            "has_recommendations": has_recommendations
+            "has_recommendations": has_recommendations,
+            "funding": float(p.project_funding) if p.project_funding is not None else 0,
+            "contingency": float(sum([x.get("contingency", 0) for x in default_recommendations]))
         })
 
     agg_data = pd.DataFrame(agg_data)
@@ -209,6 +217,9 @@ def extract_portfolio_aggregation_data(
     cost_per_sap_point = agg_data["cost"].sum() / total_sap_points if total_sap_points > 0 else 0
     cost_per_sap_point = format_money(cost_per_sap_point)
 
+    total_funding = agg_data["funding"].sum()
+    total_contingency = agg_data["contingency"].sum()
+
     aggregation_data = {
         "epc_breakdown_pre_retrofit": json.dumps(
             reformat_epc_data(agg_data["pre_retrofit_epc"].value_counts().to_dict())
@@ -231,6 +242,8 @@ def extract_portfolio_aggregation_data(
         "cost_per_co2_saved": cost_per_co2_saved,
         "cost_per_sap_point": cost_per_sap_point,
         "valuation_return_on_investment": valuation_return_on_investment,
+        "funding": float(total_funding),
+        "contingency": float(total_contingency)
     }
 
     return aggregation_data
@@ -415,13 +428,29 @@ def get_funding_data():
     project_scores_matrix.columns = ['Floor Area Segment', 'Starting Band', 'Finishing Band', 'Cost Savings']
     project_scores_matrix["Cost Savings"] = project_scores_matrix["Cost Savings"].astype(float)
 
+    partial_project_scores_matrix = read_csv_from_s3(
+        bucket_name=get_settings().DATA_BUCKET,
+        filepath="funding/ECO4_Partial_Project_Scores_Matrix_v6.csv",
+    )
+    partial_project_scores_matrix = pd.DataFrame(partial_project_scores_matrix)
+    partial_project_scores_matrix.columns = [
+        'Measure category', 'Measure_Type', 'Pre_Main_Heating_Source',
+        'Post_Main_Heating_Source', 'Total Floor Area Band', 'Starting Band',
+        'Average Treatable Factor', 'Cost Savings', 'SAP Savings'
+    ]
+    # Replace 200 with 200+ in floor area band
+    partial_project_scores_matrix["Total Floor Area Band"] = partial_project_scores_matrix[
+        "Total Floor Area Band"
+    ].replace({"200": "200+"})
+    partial_project_scores_matrix["Cost Savings"] = partial_project_scores_matrix["Cost Savings"].astype(float)
+
     whlg_eligible_postcodes = read_csv_from_s3(
         bucket_name=get_settings().DATA_BUCKET,
         filepath="funding/whlg eligible postcodes.csv",
     )
     whlg_eligible_postcodes = pd.DataFrame(whlg_eligible_postcodes)
 
-    return project_scores_matrix, whlg_eligible_postcodes
+    return project_scores_matrix, partial_project_scores_matrix, whlg_eligible_postcodes
 
 
 async def model_engine(body: PlanTriggerRequest):
@@ -535,7 +564,6 @@ async def model_engine(body: PlanTriggerRequest):
             epc_searcher.ordnance_survey_client.property_type = config.get("property_type", None)
             # For the moment, our OS API access is unavailable, so we skip and interpolate
             epc_searcher.find_property(skip_os=True)
-            # TODO: Placeholder
             if epc_searcher.newest_epc.get("estimated") and body.file_format == "domna_asset_list":
                 epc_searcher.newest_epc["uprn-source"] = epc_searcher.UPRN_SOURCE_SIMULATED
 
@@ -648,7 +676,7 @@ async def model_engine(body: PlanTriggerRequest):
         logger.info("Reading in materials and cleaned datasets")
         materials = get_materials(session)
         cleaned = get_cleaned()
-        eco_project_scores_matrix, whlg_eligible_postcodes = get_funding_data()
+        project_scores_matrix, partial_project_scores_matrix, whlg_eligible_postcodes = get_funding_data()
 
         kwh_client = KwhData(bucket=get_settings().DATA_BUCKET, read_consumption_data=True)
 
@@ -689,7 +717,8 @@ async def model_engine(body: PlanTriggerRequest):
             building_solar_config=building_solar_config,
             input_properties=input_properties,
             session=session,
-            google_solar_api_key=get_settings().GOOGLE_SOLAR_API_KEY
+            google_solar_api_key=get_settings().GOOGLE_SOLAR_API_KEY,
+            solar_materials=[m for m in materials if m["type"] == "solar_pv"]
         )
 
         input_properties = GoogleSolarApi.unit_solar_analysis(
@@ -697,7 +726,8 @@ async def model_engine(body: PlanTriggerRequest):
             input_properties=input_properties,
             session=session,
             body=body,
-            google_solar_api_key=get_settings().GOOGLE_SOLAR_API_KEY
+            solar_materials=[m for m in materials if m["type"] == "solar_pv"],
+            google_solar_api_key=get_settings().GOOGLE_SOLAR_API_KEY,
         )
 
         logger.info("Identifying property recommendations")
@@ -807,11 +837,7 @@ async def model_engine(body: PlanTriggerRequest):
                 x in property_measure_types for x in assumptions.measures_needing_ventilation
             ) and not p.has_ventilation
 
-            input_measures = optimiser_functions.prepare_input_measures(
-                measures_to_optimise, body.goal, needs_ventilation
-            )
-
-            if not input_measures[0]:
+            if not measures_to_optimise:
                 # Nothing to do, we just reshape the recommendations
                 recommendations[p.id] = optimiser_functions.flatten_recommendations_with_defaults(
                     p.id, recommendations, set()
@@ -823,20 +849,174 @@ async def model_engine(body: PlanTriggerRequest):
             )
             gain = optimiser_functions.calculate_gain(body=body, p=p, fixed_gain=fixed_gain)
 
-            if not body.optimise:
-                if body.goal != "Increasing EPC":
-                    raise NotImplementedError("Only EPC optimisation is currently supported")
-                solution = [max(sub_list, key=lambda x: (x['gain'], -x['cost'])) for sub_list in input_measures]
+            funding = Funding(
+                tenure=body.housing_type,
+                project_scores_matrix=project_scores_matrix,
+                partial_project_scores_matrix=partial_project_scores_matrix,
+                whlg_eligible_postcodes=whlg_eligible_postcodes,
+                eco4_social_cavity_abs_rate=12.5,
+                eco4_social_solid_abs_rate=17,
+                eco4_private_cavity_abs_rate=12.5,
+                eco4_private_solid_abs_rate=17,
+                gbis_social_cavity_abs_rate=21,
+                gbis_social_solid_abs_rate=25,
+                gbis_private_cavity_abs_rate=21,
+                gbis_private_solid_abs_rate=28,
+            )
+
+            li_thickness = convert_thickness_to_numeric(
+                p.roof["insulation_thickness"], p.roof["is_pitched"], p.roof["is_flat"]
+            )
+            current_wall_u_value = p.walls["thermal_transmittance"]
+            if current_wall_u_value is None:
+                current_wall_u_value = get_wall_u_value(
+                    clean_description=p.walls["clean_description"],
+                    age_band=p.age_band,
+                    is_granite_or_whinstone=p.walls["is_granite_or_whinstone"],
+                    is_sandstone_or_limestone=p.walls["is_sandstone_or_limestone"],
+                )
+
+            # We insert the innovation uplift
+            measures_to_optimise_with_uplift = deepcopy(measures_to_optimise)
+
+            # TODO: Turn this into a function and store the innovaiton uplift
+            for group in measures_to_optimise_with_uplift:
+                for r in group:
+
+                    if r["type"] in ["mechanical_ventilation", "low_energy_lighting", "secondary_heating",
+                                     "extension_cavity_wall_insulation", "draught_proofing", "sealing_open_fireplace"]:
+                        (
+                            r["partial_project_score"],
+                            r["partial_project_funding"],
+                            r["innovation_uplift"],
+                            r["uplift_project_score"],
+                        ) = (
+                            0, 0, 0, 0
+                        )
+                        continue
+                    (
+                        r["partial_project_score"], r["partial_project_funding"], r["innovation_uplift"],
+                        r["uplift_project_score"]
+                    ) = funding.get_innovation_uplift(
+                        measure=r,
+                        starting_sap=p.data["current-energy-efficiency"],
+                        floor_area=p.floor_area,
+                        is_cavity=p.walls["is_cavity_wall"],
+                        current_wall_uvalue=current_wall_u_value,
+                        is_partial="partial" in p.walls["clean_description"].lower(),
+                        existing_li_thickness=li_thickness,
+                        mainheating=p.main_heating,
+                        main_fuel=p.main_fuel,
+                        mainheat_energy_eff=p.data["mainheat-energy-eff"],
+                    )
+
+            input_measures = optimiser_functions.prepare_input_measures(
+                measures_to_optimise_with_uplift, body.goal, needs_ventilation, funding=True
+            )
+
+            # When the goal is Increasing EPC, we can run the funding optimiser
+            if body.goal == "Increasing EPC":
+
+                solutions = optimise_with_funding_paths(
+                    p=p,
+                    input_measures=input_measures,
+                    housing_type=body.housing_type,
+                    budget=body.budget,
+                    target_gain=gain,
+                    funding=funding
+                )
+
+                # Given the solutions we select the optimal one
+                solutions["cost_less_full_project_funding"] = np.where(
+                    solutions["scheme"] == "eco4",
+                    solutions["total_cost"] - solutions["full_project_funding"] - solutions["total_uplift"],
+                    solutions["total_cost"] - solutions["partial_project_funding"] - solutions["total_uplift"]
+                )
+
+                solutions["cost_less_full_project_funding"] = (
+                    solutions["total_cost"] - solutions["full_project_funding"] - solutions["total_uplift"]
+                )
+                solutions = solutions.sort_values("cost_less_full_project_funding", ascending=True)
+
+                if solutions["meets_upgrade_target"].any():
+                    # If we have a solution that meets the upgrade target, we select that one
+                    optimal_solution = solutions[solutions["meets_upgrade_target"]].iloc[0]
+                else:
+                    optimal_solution = solutions.iloc[0]
+
+                # This is the list of measures that we will recommend
+                scheme = optimal_solution["scheme"]
+                funded_measures = optimal_solution["items"] if scheme != "none" else []
+                solution = optimal_solution["items"] + optimal_solution["unfunded_items"]
+                # This is the total amount of funding that the project will produce (including uplifts) (£)
+                project_funding = optimal_solution["full_project_funding"]
+                # This is the total amount of funding associated to the uplift (£)
+                total_uplift = optimal_solution["total_uplift"]
+                # This is the funding scheme selected
+                # This is the full project ABS
+                full_project_score = optimal_solution["project_score"]
+                # This is the partial project ABS
+                partial_project_score = optimal_solution["partial_project_score"]
+                # This is the uplift score ABS
+                uplift_project_score = optimal_solution["total_uplift_score"]
             else:
+                # We optimise and then we determine eligibility for funding, based on the measures selected
                 optimiser = (
                     GainOptimiser(
-                        input_measures, max_cost=body.budget, max_gain=gain, allow_slack=body.goal == "Increasing EPC"
+                        input_measures, max_cost=body.budget, max_gain=gain, allow_slack=False
                     ) if body.budget else CostOptimiser(input_measures, min_gain=gain)
                 )
                 optimiser.setup()
                 optimiser.solve()
                 solution = optimiser.solution
 
+                recommendation_types = []
+                for measures in input_measures:
+                    for measure in measures:
+                        recommendation_types.append(measure["type"])
+                recommendation_types = set(recommendation_types)
+
+                has_wall_insulation_recommendation = any(
+                    (m in recommendation_types or "+".join([m, "mechanical_ventilation"])) for m in
+                    WALL_INSULATION_MEASURES
+                )
+                has_roof_insulation_recommendation = any(
+                    (m in recommendation_types or "+".join([m, "mechanical_ventilation"])) for m in
+                    ROOF_INSULATION_MEASURES
+                )
+
+                funding.check_funding(
+                    measures=solution,
+                    starting_sap=p.data["current-energy-efficiency"],
+                    ending_sap=p.data["current-energy-efficiency"] + sum([x["gain"] for x in solution]),
+                    floor_area=p.floor_area,
+                    mainheat_description=p.main_heating["clean_description"],
+                    heating_control_description=p.main_heating_controls["clean_description"],
+                    is_cavity=p.walls["is_cavity_wall"],
+                    current_wall_uvalue=current_wall_u_value,
+                    is_partial="partial" in p.walls["clean_description"].lower(),
+                    existing_li_thickness=li_thickness,
+                    mainheating=p.main_heating,
+                    main_fuel=p.main_fuel,
+                    mainheat_energy_eff=p.data["mainheat-energy-eff"],
+                    has_wall_insulation_recommendation=has_wall_insulation_recommendation,
+                    has_roof_insulation_recommendation=has_roof_insulation_recommendation,
+                )
+
+                # Determine the scheme
+                scheme = "none"
+                if funding.eco4_eligible:
+                    scheme = "eco4"
+                if scheme == "none" and funding.gbis_eligible:
+                    scheme = "gbis"
+
+                funded_measures = solution if scheme in ["gbis", "eco4"] else []
+                project_funding = 0 if funding.full_project_abs is not None else funding.full_project_abs
+                total_uplift = funding.eco4_uplift
+                full_project_score = 0 if funding.full_project_abs is not None else funding.full_project_abs
+                partial_project_score = funding.partial_project_abs
+                uplift_project_score = funding.eco4_uplift if scheme == "eco4" else funding.gbis_uplift
+
             selected = {r["id"] for r in solution}
 
             if property_required_measures:
@@ -852,6 +1032,21 @@ async def model_engine(body: PlanTriggerRequest):
                 p.id, recommendations, selected
             )
 
+            # TODO: functionise
+            for measure in funded_measures:
+                if "+mechanical_ventilation" in measure["type"]:
+                    measure["type"] = measure["type"].split("+mechanical_ventilation")[0]
+
+            p.insert_funding(
+                scheme=scheme,
+                funded_measures=funded_measures,
+                project_funding=project_funding,
+                total_uplift=total_uplift,
+                full_project_score=full_project_score,
+                partial_project_score=partial_project_score,
+                uplift_project_score=uplift_project_score
+            )
+
         # when we have buildings, we tweak our solar PV recommendations as if one unit needs it, we apply it to all
         # of them
         # TODO: We can probably do better and optimise at the building level - this is temp
@@ -878,28 +1073,6 @@ async def model_engine(body: PlanTriggerRequest):
                             # need to be updated
                             rec["default"] = True
 
-        # ~~~~~~~~~~~~~~~~
-        # Funding
-        # ~~~~~~~~~~~~~~~~
-
-        # for p in input_properties:
-        #     funding_calulator = Funding(
-        #         tenure=body.housing_type,
-        #         starting_epc=p.data["current-energy-rating"],
-        #         starting_sap=int(p.data["current-energy-efficiency"]),
-        #         postcode=p.postcode,
-        #         floor_area=p.floor_area,
-        #         council_tax_band=None,  # This is seemingly always None at the moment
-        #         property_recommendations=recommendations[p.id],
-        #         project_scores_matrix=eco_project_scores_matrix,
-        #         whlg_eligible_postcodes=whlg_eligible_postcodes,
-        #         gbis_abs_rate=15,
-        #         eco4_abs_rate=15,
-        #     )
-        #     funding_calulator.check_eligibiltiy()
-        #     # Insert finding
-        #     p.insert_funding(funding_calulator)
-
         logger.info("Uploading recommendations to the database")
         # If we have any work to do, we create a new scenario
         if body.scenario_id:
@@ -988,6 +1161,8 @@ async def model_engine(body: PlanTriggerRequest):
                         session, recommendations_to_upload, p.id, new_plan_id
                     )
 
+                    upload_funding(session, p, new_plan_id, recommendations_to_upload)
+
                     property_valuation_increases.append(
                         valuations["average_increased_value"] - valuations["current_value"]
                     )
@@ -1000,6 +1175,7 @@ async def model_engine(body: PlanTriggerRequest):
                 session.rollback()
                 print("Failed i = %s" % str(i))
                 logger.error(f"An error occurred during batch starting at index {i}: {e}")
+                logger.error(f"property is uprn {p.uprn} id {p.id} address {p.address}")
 
         logger.info("Creating portfolio aggregations")
         # We implement this in the simplest way possible which will be just to query the database for all

backend/tests/test_data/heating_scenarios.py

@@ -0,0 +1,104 @@
+from backend.Funding import EligibilityCaveats
+
+heating_scenarios = [
+    {
+        "description": "EPC D with ASHP and no insulation at all — fails precondition 1",
+        "measures": [{"type": "air_source_heat_pump"}],
+        "starting_sap": 60,
+        "mainheat_description": "air source heat pump",
+        "heating_control_description": "roomstat_programmer_trvs",
+        "has_wall_insulation_recommendation": True,
+        "has_roof_insulation_recommendation": False,
+        "expected_eligibility": False,
+        "expected_caveats": [EligibilityCaveats.MINIMUM_INSULATION_PRECONDITIONS_NOT_MET],
+    },
+    {
+        "description": "EPC D with ASHP and no insulation at all — fails precondition 1",
+        "measures": [{"type": "air_source_heat_pump"}],
+        "starting_sap": 60,
+        "mainheat_description": "air source heat pump",
+        "heating_control_description": "roomstat_programmer_trvs",
+        "has_wall_insulation_recommendation": False,
+        "has_roof_insulation_recommendation": False,
+        "expected_eligibility": False,
+        "expected_caveats": [EligibilityCaveats.MINIMUM_INSULATION_PRECONDITIONS_NOT_MET],
+    },
+    {
+        "description": "EPC D with ASHP and floor insulation — passes precondition 1",
+        "measures": [
+            {"type": "air_source_heat_pump"},
+            {"type": "suspended_floor_insulation"}
+        ],
+        "starting_sap": 60,
+        "has_wall_insulation_recommendation": False,
+        "has_roof_insulation_recommendation": False,
+        "mainheat_description": "air source heat pump",
+        "heating_control_description": "roomstat_programmer_trvs",
+        "expected_eligibility": True,
+        "expected_caveats": [],
+    },
+    {
+        "description": "EPC E with ASHP and only floor insulation — fails precondition 2 due to missing wall/roof",
+        "measures": [
+            {"type": "air_source_heat_pump"},
+            {"type": "suspended_floor_insulation"}
+        ],
+        "starting_sap": 45,
+        "mainheat_description": "air source heat pump",
+        "heating_control_description": "roomstat_programmer_trvs",
+        "has_wall_insulation_recommendation": True,
+        "has_roof_insulation_recommendation": True,
+        "expected_eligibility": False,
+        "expected_caveats": [EligibilityCaveats.MINIMUM_INSULATION_PRECONDITIONS_NOT_MET],
+    },
+    {
+        "description": "EPC E with ASHP and both wall and roof insulation — passes precondition 2",
+        "measures": [
+            {"type": "air_source_heat_pump"},
+            {"type": "external_wall_insulation"},
+            {"type": "loft_insulation"}
+        ],
+        "starting_sap": 45,
+        "mainheat_description": "air source heat pump",
+        "heating_control_description": "roomstat_programmer_trvs",
+        "has_wall_insulation_recommendation": True,
+        "has_roof_insulation_recommendation": True,
+        "expected_eligibility": True,
+        "expected_caveats": [],
+    },
+    {
+        "description": "EPC D with FTCH and no insulation — still passes (exempt from precondition 1)",
+        "measures": [{"type": "first_time_central_heating"}],
+        "starting_sap": 60,
+        "mainheat_description": "none",
+        "heating_control_description": "none",
+        "expected_eligibility": True,
+        "expected_caveats": [],
+    },
+    {
+        "description": "EPC E with FTCH and no insulation — fails precondition 2",
+        "measures": [{"type": "first_time_central_heating"}],
+        "starting_sap": 45,
+        "mainheat_description": "none",
+        "heating_control_description": "none",
+        "has_wall_insulation_recommendation": True,
+        "has_roof_insulation_recommendation": True,
+        "expected_eligibility": False,
+        "expected_caveats": [EligibilityCaveats.MINIMUM_INSULATION_PRECONDITIONS_NOT_MET],
+    },
+    {
+        "description": "EPC E with FTCH and wall/roof insulation — passes precondition 2",
+        "measures": [
+            {"type": "first_time_central_heating"},
+            {"type": "external_wall_insulation"},
+            {"type": "loft_insulation"},
+        ],
+        "starting_sap": 45,
+        "mainheat_description": "none",
+        "heating_control_description": "none",
+        "has_wall_insulation_recommendation": True,
+        "has_roof_insulation_recommendation": True,
+        "expected_eligibility": True,
+        "expected_caveats": [],
+    },
+]

backend/tests/test_data/innovation_measure_fixtures.py

@@ -0,0 +1,170 @@
+from backend.Funding import EligibilityCaveats
+
+innovation_scenarios = [
+    # 1) Innovation PV, non-eligible heating system in place, EPC D - not eligible
+    {
+        "description": "Innovation PV, non-eligible heating system in place, EPC D",
+        "measures": [{"type": "solar_pv", "is_innovation": True, "uplift": 0.45}],
+        "starting_sap": 60,
+        "mainheat_description": "Electric storage heaters",
+        "heating_control_description": "Manual charge control",
+        "has_wall_insulation_recommendation": False,
+        "has_roof_insulation_recommendation": False,
+        "expected_eligibility": False,
+        "expected_caveats": [EligibilityCaveats.SOLAR_NEEDS_HEATING],
+    },
+    # 2) Innovation PV, eligible heating system in place, EPC D - eligible
+    {
+        "description": "Innovation PV, eligible heating system in place, EPC D",
+        "measures": [{"type": "solar_pv", "is_innovation": True, "uplift": 0.45}],
+        "starting_sap": 60,
+        "mainheat_description": "Air source heat pump, radiators",
+        "heating_control_description": "Programmer, room thermostat and TRVs",
+        "has_wall_insulation_recommendation": False,
+        "has_roof_insulation_recommendation": False,
+        "expected_eligibility": True,
+        "expected_caveats": [],
+    },
+    # 3) Innovation PV, non-eligible heating system, heating upgrade to HHRSH, EPC E - eligible
+    {
+        "description": "Innovation PV + HHRSH upgrade, EPC E",
+        "measures": [
+            {"type": "solar_pv", "is_innovation": True, "uplift": 0.45},
+            {"type": "high_heat_retention_storage_heater", "is_innovation": True, "uplift": 0.1}
+        ],
+        "starting_sap": 50,
+        "mainheat_description": "Electric storage heaters",
+        "heating_control_description": "Manual charge control",
+        "has_wall_insulation_recommendation": False,
+        "has_roof_insulation_recommendation": False,
+        "expected_eligibility": True,
+        "expected_caveats": [],
+    },
+    # 4) Innovation PV + HHRSH upgrade
+    {
+        "description": "Innovation PV + HHRSH upgrade, EPC E",
+        "measures": [
+            {"type": "solar_pv", "is_innovation": True, "uplift": 0.45},
+            {"type": "high_heat_retention_storage_heater", "is_innovation": True, "uplift": 0.1}
+        ],
+        "starting_sap": 50,
+        "mainheat_description": "Electric storage heaters",
+        "heating_control_description": "Manual charge control",
+        "has_wall_insulation_recommendation": False,
+        "has_roof_insulation_recommendation": False,
+        "expected_eligibility": True,
+        "expected_caveats": [],
+    },
+    # 5) Innovation PV, needs wall insulation, no wall insulation measure - not eligible
+    {
+        "description": "Innovation PV, wall insulation recommended, but not installed",
+        "measures": [{"type": "solar_pv", "is_innovation": True, "uplift": 0.45}],
+        "starting_sap": 60,
+        "mainheat_description": "Air source heat pump, radiators",
+        "heating_control_description": "Programmer, room thermostat and TRVs",
+        "has_wall_insulation_recommendation": True,
+        "has_roof_insulation_recommendation": False,
+        "expected_eligibility": False,
+        "expected_caveats": [EligibilityCaveats.MINIMUM_INSULATION_PRECONDITIONS_NOT_MET],
+    },
+    # 6) Innovation PV, wall insulation recommended and installed - eligible
+    {
+        "description": "Innovation PV, wall insulation recommended and installed",
+        "measures": [
+            {"type": "solar_pv", "is_innovation": True, "uplift": 0.45},
+            {"type": "internal_wall_insulation", "is_innovation": False, "uplift": 0.25}
+        ],
+        "starting_sap": 60,
+        "mainheat_description": "Air source heat pump, radiators",
+        "heating_control_description": "Programmer, room thermostat and TRVs",
+        "has_wall_insulation_recommendation": True,
+        "has_roof_insulation_recommendation": False,
+        "expected_eligibility": True,
+        "expected_caveats": [],
+    },
+    # 7) Innovation PV, needs roof insulation, no roof insulation measure - not eligible
+    {
+        "description": "Innovation PV, roof insulation recommended, not installed",
+        "measures": [{"type": "solar_pv", "is_innovation": True, "uplift": 0.45}],
+        "starting_sap": 60,
+        "mainheat_description": "Air source heat pump, radiators",
+        "heating_control_description": "Programmer, room thermostat and TRVs",
+        "has_wall_insulation_recommendation": False,
+        "has_roof_insulation_recommendation": True,
+        "expected_eligibility": False,
+        "expected_caveats": [EligibilityCaveats.MINIMUM_INSULATION_PRECONDITIONS_NOT_MET],
+    },
+    # 8) Innovation PV, roof insulation recommended and installed - eligible
+    {
+        "description": "Innovation PV, roof insulation recommended and installed",
+        "measures": [
+            {"type": "solar_pv", "is_innovation": True, "uplift": 0.45},
+            {"type": "loft_insulation", "is_innovation": False, "uplift": 0}
+        ],
+        "starting_sap": 60,
+        "mainheat_description": "Air source heat pump, radiators",
+        "heating_control_description": "Programmer, room thermostat and TRVs",
+        "has_wall_insulation_recommendation": False,
+        "has_roof_insulation_recommendation": True,
+        "expected_eligibility": True,
+        "expected_caveats": [],
+    },
+    # 9) Innovation PV, needs both roof + wall insulation, no insulation - not eligible
+    {
+        "description": "Innovation PV, both insulations recommended, none installed",
+        "measures": [{"type": "solar_pv", "is_innovation": True, "uplift": 0.45}],
+        "starting_sap": 60,
+        "mainheat_description": "Air source heat pump, radiators",
+        "heating_control_description": "Programmer, room thermostat and TRVs",
+        "has_wall_insulation_recommendation": True,
+        "has_roof_insulation_recommendation": True,
+        "expected_eligibility": False,
+        "expected_caveats": [EligibilityCaveats.MINIMUM_INSULATION_PRECONDITIONS_NOT_MET],
+    },
+    # 10) Innovation PV, both recommended, only wall insulation installed - not eligible
+    {
+        "description": "Innovation PV, both insulations recommended, only wall done",
+        "measures": [
+            {"type": "solar_pv", "is_innovation": True, "uplift": 0.45},
+            {"type": "internal_wall_insulation", "is_innovation": False, "uplift": 0.25}
+        ],
+        "starting_sap": 60,
+        "mainheat_description": "Air source heat pump, radiators",
+        "heating_control_description": "Programmer, room thermostat and TRVs",
+        "has_wall_insulation_recommendation": True,
+        "has_roof_insulation_recommendation": True,
+        "expected_eligibility": False,
+        "expected_caveats": [EligibilityCaveats.MINIMUM_INSULATION_PRECONDITIONS_NOT_MET],
+    },
+    # 11) Innovation PV, both recommended, only roof insulation installed - not eligible
+    {
+        "description": "Innovation PV, both insulations recommended, only roof done",
+        "measures": [
+            {"type": "solar_pv", "is_innovation": True, "uplift": 0.45},
+            {"type": "loft_insulation", "is_innovation": False, "uplift": 0}
+        ],
+        "starting_sap": 60,
+        "mainheat_description": "Air source heat pump, radiators",
+        "heating_control_description": "Programmer, room thermostat and TRVs",
+        "has_wall_insulation_recommendation": True,
+        "has_roof_insulation_recommendation": True,
+        "expected_eligibility": False,
+        "expected_caveats": [EligibilityCaveats.MINIMUM_INSULATION_PRECONDITIONS_NOT_MET],
+    },
+    # 12) Innovation PV, both recommended, both installed - eligible
+    {
+        "description": "Innovation PV, both insulations recommended and installed",
+        "measures": [
+            {"type": "solar_pv", "is_innovation": True, "uplift": 0.45},
+            {"type": "internal_wall_insulation", "is_innovation": False, "uplift": 0.25},
+            {"type": "loft_insulation", "is_innovation": False, "uplift": 0}
+        ],
+        "starting_sap": 60,
+        "mainheat_description": "Air source heat pump, radiators",
+        "heating_control_description": "Programmer, room thermostat and TRVs",
+        "has_wall_insulation_recommendation": True,
+        "has_roof_insulation_recommendation": True,
+        "expected_eligibility": True,
+        "expected_caveats": [],
+    },
+]

backend/tests/test_data/pre_heating_scenarios.py

@@ -0,0 +1,144 @@
+# Each scenario: super explicit about inputs and expected mapping
+pre_main_heating_scenarios = [
+    # --- Mains gas boilers (radiators) ---
+    {
+        "description": "Boiler and radiators, mains gas (condensing expected)",
+        "MAINHEAT_DESCRIPTION": "Boiler and radiators, mains gas",
+        "MAIN_FUEL": "mains gas (not community)",
+        "MAINHEAT_ENERGY_EFF": "Good",
+        "expected": "Condensing Gas Boiler",
+    },
+    {
+        "description": "Boiler and radiators, mains gas (non-condensing expected)",
+        "MAINHEAT_DESCRIPTION": "Boiler and radiators, mains gas",
+        "MAIN_FUEL": "mains gas - this is for backwards compatibility only and should not be used",
+        "MAINHEAT_ENERGY_EFF": "Average",
+        "expected": "Non Condensing Gas Boiler",
+    },
+    {
+        "description": "Boiler and radiators, mains gas (very poor => back boiler to rads)",
+        "MAINHEAT_DESCRIPTION": "Boiler and radiators, mains gas",
+        "MAIN_FUEL": "Gas: mains gas",
+        "MAINHEAT_ENERGY_EFF": "Very Poor",
+        "expected": "Gas Back Boiler to Radiators",
+    },
+
+    # --- Warm air (treated like gas boiler family in your mapper) ---
+    {
+        "description": "Warm air, mains gas (good => condensing)",
+        "MAINHEAT_DESCRIPTION": "Warm air, mains gas",
+        "MAIN_FUEL": "mains gas (not community)",
+        "MAINHEAT_ENERGY_EFF": "Good",
+        "expected": "Condensing Gas Boiler",
+    },
+
+    # --- Community scheme (CHP vs non-CHP depends on energy eff) ---
+    {
+        "description": "Community scheme (gas, good => CHP)",
+        "MAINHEAT_DESCRIPTION": "Community scheme",
+        "MAIN_FUEL": "mains gas (community)",
+        "MAINHEAT_ENERGY_EFF": "Good",
+        "expected": "DHS CHP",
+    },
+    {
+        "description": "Community scheme (gas, average => non-CHP)",
+        "MAINHEAT_DESCRIPTION": "Community scheme",
+        "MAIN_FUEL": "mains gas (community)",
+        "MAINHEAT_ENERGY_EFF": "Average",
+        "expected": "DHS non-CHP",
+    },
+    {
+        "description": "Community scheme (no fuel data, good => CHP)",
+        "MAINHEAT_DESCRIPTION": "Community scheme",
+        "MAIN_FUEL": "NO DATA!",
+        "MAINHEAT_ENERGY_EFF": "Good",
+        "expected": "DHS CHP",
+    },
+
+    # --- Electric storage heaters (ESH responsiveness split) ---
+    {
+        "description": "Electric storage heaters (average => responsiveness > 0.2)",
+        "MAINHEAT_DESCRIPTION": "Electric storage heaters",
+        "MAIN_FUEL": "electricity (not community)",
+        "MAINHEAT_ENERGY_EFF": "Average",
+        "expected": "Electric Storage Heaters Responsiveness >0.2",
+    },
+    {
+        "description": "Electric storage heaters (poor => responsiveness > 0.2)",
+        "MAINHEAT_DESCRIPTION": "Electric storage heaters",
+        "MAIN_FUEL": "electricity - this is for backwards compatibility only and should not be used",
+        "MAINHEAT_ENERGY_EFF": "Poor",
+        "expected": "Electric Storage Heaters Responsiveness >0.2",
+    },
+    {
+        "description": "Electric storage heaters (very poor => responsiveness <= 0.2)",
+        "MAINHEAT_DESCRIPTION": "Electric storage heaters",
+        "MAIN_FUEL": "electricity (not community)",
+        "MAINHEAT_ENERGY_EFF": "Very Poor",
+        "expected": "Electric Storage Heaters Responsiveness <=0.2",
+    },
+
+    # --- Electric direct-acting / room heaters ---
+    {
+        "description": "Room heaters, electric (very poor)",
+        "MAINHEAT_DESCRIPTION": "Room heaters, electric",
+        "MAIN_FUEL": "electricity (not community)",
+        "MAINHEAT_ENERGY_EFF": "Very Poor",
+        "expected": "Electric Room Heaters",
+    },
+    {
+        "description": "Room heaters, electric (poor, unspecified tariff)",
+        "MAINHEAT_DESCRIPTION": "Room heaters, electric",
+        "MAIN_FUEL": "Electricity: electricity, unspecified tariff",
+        "MAINHEAT_ENERGY_EFF": "Poor",
+        "expected": "Electric Room Heaters",
+    },
+    {
+        "description": "Portable electric heaters assumed for most rooms (maps to electric room heaters)",
+        "MAINHEAT_DESCRIPTION": "Portable electric heaters assumed for most rooms",
+        "MAIN_FUEL": "mains gas (not community)",  # weird in EPCs, mapper forces electric room heaters here
+        "MAINHEAT_ENERGY_EFF": "Very Poor",
+        "expected": "Electric Room Heaters",
+    },
+    {
+        "description": "No system present: electric heaters assumed",
+        "MAINHEAT_DESCRIPTION": "No system present: electric heaters assumed",
+        "MAIN_FUEL": "To be used only when there is no heating/hot-water system",
+        "MAINHEAT_ENERGY_EFF": "Very Poor",
+        "expected": "Electric Room Heaters",
+    },
+    {
+        "description": "Electric underfloor heating => direct-acting electric",
+        "MAINHEAT_DESCRIPTION": "Electric underfloor heating",
+        "MAIN_FUEL": "electricity (not community)",
+        "MAINHEAT_ENERGY_EFF": "Average",
+        "expected": "Electric Room Heaters",
+    },
+
+    # --- Gas room heaters ---
+    {
+        "description": "Room heaters, mains gas (average)",
+        "MAINHEAT_DESCRIPTION": "Room heaters, mains gas",
+        "MAIN_FUEL": "mains gas (not community)",
+        "MAINHEAT_ENERGY_EFF": "Average",
+        "expected": "Gas Room Heaters",
+    },
+
+    # --- Electric boiler ---
+    {
+        "description": "Boiler and radiators, electric (very poor => electric boiler)",
+        "MAINHEAT_DESCRIPTION": "Boiler and radiators, electric",
+        "MAIN_FUEL": "electricity (not community)",
+        "MAINHEAT_ENERGY_EFF": "Very Poor",
+        "expected": "Electric Boiler",
+    },
+
+    # --- Gas boiler + UFH (still ‘boiler’ logic) ---
+    {
+        "description": "Boiler and underfloor heating, mains gas (good => condensing)",
+        "MAINHEAT_DESCRIPTION": "Boiler and underfloor heating, mains gas",
+        "MAIN_FUEL": "mains gas (not community)",
+        "MAINHEAT_ENERGY_EFF": "Good",
+        "expected": "Condensing Gas Boiler",
+    },
+]

etl/costs/app.py

@@ -11,7 +11,7 @@ import inspect
 
 src_file_path = inspect.getfile(lambda: None)
 
-DATA_DIRECTORY = Path(src_file_path).parent / "local_data" / "20250316 Domna Materials.xlsx"
+DATA_DIRECTORY = Path(src_file_path).parent / "local_data" / "20250815 Domna Materials.xlsx"
 # Environment file is at the same level as this file
 ENV_FILE = Path(src_file_path).parent / "etl" / "costs" / ".env"
 dotenv.load_dotenv(ENV_FILE)
@@ -92,6 +92,10 @@ def app():
     flat_roof_costs = pd.read_excel(DATA_DIRECTORY, sheet_name="flat_roof_insulation", header=0)
     window_costs = pd.read_excel(DATA_DIRECTORY, sheet_name="window_glazing", header=0)
     rir_insulation_costs = pd.read_excel(DATA_DIRECTORY, sheet_name="room_roof_insulation", header=0)
+    solar_pv = pd.read_excel(DATA_DIRECTORY, sheet_name="solar_pv", header=0)
+    hhrsh = pd.read_excel(DATA_DIRECTORY, sheet_name="hhrsh", header=0)
+    scaffolding = pd.read_excel(DATA_DIRECTORY, sheet_name="scaffolding", header=0)
+    fireplaces = pd.read_excel(DATA_DIRECTORY, sheet_name="fireplaces", header=0)
 
     # Form a single table to be uploaded
     costs = pd.concat(
@@ -107,6 +111,10 @@ def app():
             flat_roof_costs,
             window_costs,
             rir_insulation_costs,
+            solar_pv,
+            hhrsh,
+            scaffolding,
+            fireplaces
         ]
     )
 

etl/customers/remote_assessments/app.py

@@ -4,7 +4,7 @@ from dotenv import load_dotenv
 from utils.s3 import save_csv_to_s3
 from etl.find_my_epc.AssetListEpcData import AssetListEpcData
 
-PORTFOLIO_ID = 212
+PORTFOLIO_ID = 235
 USER_ID = 8
 
 load_dotenv(dotenv_path="backend/.env")
@@ -17,15 +17,45 @@ def app():
     :return:
     """
 
-    asset_list = pd.read_excel(
-        "/Users/khalimconn-kowlessar/Downloads/Energy Information MASTER June 2025 - Standardised.xlsx",
-        sheet_name="Solar Properties",
-    )
-    asset_list = asset_list[~asset_list["estimated"]]
-    asset_list["domna_address_1"] = asset_list["domna_address_1"].astype(str)
-    asset_list = asset_list[["domna_address_1", "domna_postcode", "epc_os_uprn"]].rename(
-        columns={"domna_address_1": "address", "domna_postcode": "postcode", "epc_os_uprn": "uprn"}
-    )
+    # asset_list = pd.read_excel(
+    #     "/Users/khalimconn-kowlessar/Downloads/Energy Information MASTER June 2025 - Standardised.xlsx",
+    #     sheet_name="Solar Properties",
+    # )
+    # asset_list = asset_list[~asset_list["estimated"]]
+    # asset_list["domna_address_1"] = asset_list["domna_address_1"].astype(str)
+    # asset_list = asset_list[["domna_address_1", "domna_postcode", "epc_os_uprn"]].rename(
+    #     columns={"domna_address_1": "address", "domna_postcode": "postcode", "epc_os_uprn": "uprn"}
+    # )
+
+    asset_list = [
+        {
+            "address": "9 Reeds Place",
+            "postcode": "PO12 3HR",
+            "uprn": 37017508
+        },
+        {
+            "address": "7 Crawley Road",
+            "postcode": "N22 6AN",
+            "uprn": 100021169757
+        },
+        {
+            "address": "20 Main Street",
+            "postcode": "NG32 1SE",
+            "uprn": 200002698370
+        },
+        {
+            "address": "19 Wolley Avenue",
+            "postcode": "LS12 5DX",
+            "uprn": 72234517
+        },
+        {
+            "address": "45 Bolton Lane, Hose",
+            "postcode": "LE14 4JE",
+            "uprn": 100030535501
+        }
+    ]
+
+    asset_list = pd.DataFrame(asset_list)
 
     # Store the asset list in s3
     filename = f"{USER_ID}/{PORTFOLIO_ID}/asset_list.csv"
@@ -64,16 +94,24 @@ def app():
 
     valuation_data = [
         {
-            "valuation": 339_000,
-            "uprn": 200003423454,
+            "valuation": 201000,
+            "uprn": 37017508,
         },
         {
-            "valuation": 374_000,
-            "uprn": 200003423194
+            "valuation": 810000,
+            "uprn": 100021169757,
         },
         {
-            "valuation": 719_000,
-            "uprn": 200003423607
+            "valuation": 228_000,
+            "uprn": 72234517
+        },
+        {
+            "valuation": 236_000,
+            "uprn": 100030535501
+        },
+        {
+            "valuation": 509000,
+            "uprn": 200002698370
         },
     ]
     # Store valuation data to s3
@@ -84,19 +122,42 @@ def app():
         file_name=valuation_filename
     )
 
-    body = {
+    body1 = {
         "portfolio_id": str(PORTFOLIO_ID),
-        "housing_type": "Private",
+        "housing_type": "Social",
         "goal": "Increasing EPC",
-        "goal_value": "A",
+        "goal_value": "B",
         "trigger_file_path": filename,
         "already_installed_file_path": "",
-        "patches_file_path": patches_filename,
-        "non_invasive_recommendations_file_path": non_invasive_recommendations_filename,
-        "valuation_file_path": "",
-        "scenario_name": "Full package remote assessment",
+        "patches_file_path": "",
+        "non_invasive_recommendations_file_path": "",
+        "valuation_file_path": valuation_filename,
+        "scenario_name": "EPC B",
         "multi_plan": True,
         "budget": None,
-        "inclusions": ["cavity_wall_insulation", "ventilation"]
+        "ashp_cop": 3.5,
+        "event_type": "remote_assessment",
+        "default_u_values": True,
+
     }
-    print(body)
+    print(body1)
+
+    body2 = {
+        "portfolio_id": str(PORTFOLIO_ID),
+        "housing_type": "Social",
+        "goal": "Increasing EPC",
+        "goal_value": "C",
+        "trigger_file_path": filename,
+        "already_installed_file_path": "",
+        "patches_file_path": "",
+        "non_invasive_recommendations_file_path": "",
+        "valuation_file_path": valuation_filename,
+        "scenario_name": "EPC C",
+        "multi_plan": True,
+        "budget": None,
+        "ashp_cop": 3.5,
+        "event_type": "remote_assessment",
+        "default_u_values": True,
+
+    }
+    print(body2)

etl/epc/DataProcessor.py

@@ -11,7 +11,6 @@ from etl.epc.settings import (
     IGNORED_TENURES,
     FULLY_GLAZED_DESCRIPTIONS,
     AVERAGE_FIXED_FEATURES,
-    BUILT_FORM_REMAP,
     COLUMNS_TO_MERGE_ON,
     FIXED_FEATURES,
     COLUMNTYPES,
@@ -125,7 +124,6 @@ class EPCDataProcessor:
         self.confine_data(ignore_step=ignore_step)
         self.remap_anomalies()
         self.remap_floor_level(ignore_step=ignore_step)
-        self.remap_build_form()
         self.cast_data_column_values_to_lower()
         self.standardise_construction_age_band(ignore_step=ignore_step)
         self.clean_missing_rooms(ignore_step=ignore_step)
@@ -242,13 +240,6 @@ class EPCDataProcessor:
         for col in convert_to_lower:
             self.data[col] = self.data[col].str.lower()
 
-    def remap_build_form(self):
-        """
-        Remap build form to standard values
-        No Violation mode or newdata modes required
-        """
-        self.data["BUILT_FORM"] = self.data["BUILT_FORM"].replace(BUILT_FORM_REMAP)
-
     def remap_anomalies(self):
         """
         Remap anomalies to None

etl/epc/Record.py

@@ -7,7 +7,7 @@ from etl.epc.ValidationConfiguration import (
 )
 from etl.epc.DataProcessor import EPCDataProcessor
 from recommendations.rdsap_tables import england_wales_age_band_lookup, FLOOR_LEVEL_MAP
-from etl.epc.settings import DATA_ANOMALY_MATCHES, BUILT_FORM_REMAP
+from etl.epc.settings import DATA_ANOMALY_MATCHES
 import re
 import os
 import numpy as np
@@ -748,10 +748,6 @@ class EPCRecord:
         if not self.prepared_epc:
             raise ValueError("EPC Recrod doesn not contain epc data")
 
-        self.prepared_epc["built-form"] = BUILT_FORM_REMAP.get(
-            self.prepared_epc["built-form"], self.prepared_epc["built-form"]
-        )
-
         if self.prepared_epc["built-form"] in DATA_ANOMALY_MATCHES:
             if self.prepared_epc["property-type"] in ["Flat", "Maisonette"]:
                 self.prepared_epc["built-form"] = "End-Terrace"

etl/epc_clean/EpcClean.py

@@ -75,6 +75,9 @@ class EpcClean:
             ]
         ]
 
+        # Average
+        filtered_data.groupby("lighting-description")["low-energy-lighting"].mean().reset_index()
+
         # Convert low-energy-lighting to float
         for row in filtered_data:
             row["low-energy-lighting"] = float(row["low-energy-lighting"])
@@ -88,9 +91,10 @@ class EpcClean:
             sums[description] += row["low-energy-lighting"]
             counts[description] += 1
 
+        # Scale to between 0 and 1
         averages = [{
-            "lighting-description": correct_spelling(description.lower()),
-            "low-energy-lighting": total / counts[description]
+            "lighting-description": correct_spelling(description.lower()) / 100,
+            "low-energy-lighting": total / counts[description] / 100
         } for description, total in sums.items()]
 
         return averages

etl/epc_clean/app.py

@@ -3,11 +3,12 @@ import os
 import pandas as pd
 import msgpack
 import inspect
+from datetime import datetime
 
 from etl.epc_clean.EpcClean import EpcClean
 from etl.epc.settings import EARLIEST_EPC_DATE
 from pathlib import Path
-from utils.s3 import save_data_to_s3
+from utils.s3 import save_data_to_s3, read_from_s3
 
 src_file_path = inspect.getfile(lambda: None)
 
@@ -22,7 +23,7 @@ LAND_REGISTRY_PATHS = [
     os.path.abspath(os.path.dirname(src_file_path)) + "/model_data/local_data/pp-2017-part2.csv",
 ]
 
-EPC_DIRECTORY = Path(src_file_path).parent / "local_data" / "all-domestic-certificates"
+EPC_DIRECTORY = Path("/Users/khalimconn-kowlessar/Downloads") / "all-domestic-certificates"
 
 ENVIRONMENT = os.getenv("ENVIRONMENT", "dev")
 
@@ -39,28 +40,35 @@ def app():
     cleaned_data = {}
     epc_directories = [entry for entry in EPC_DIRECTORY.iterdir() if entry.is_dir()]
 
+    errors = []
     for directory in tqdm(epc_directories):
-        data = pd.read_csv(directory / "certificates.csv", low_memory=False)
-        # Rename the columns to the same format as the api returns
-        data.columns = [c.replace("_", "-").lower() for c in data.columns]
-        # Take just date before the date threshold
-        data = data[data["lodgement-date"] >= "2011-01-01"]
+        try:
+            data = pd.read_csv(directory / "certificates.csv", low_memory=False)
+            # Rename the columns to the same format as the api returns
+            data.columns = [c.replace("_", "-").lower() for c in data.columns]
+            # Take just date before the date threshold
+            data = data[data["lodgement-date"] >= "2011-01-01"]
 
-        # Convert to list of dictioaries as returned by the api
-        data = data.to_dict("records")
+            # Convert to list of dictioaries as returned by the api
+            data = data.to_dict("records")
 
-        # Incorporate input data into cleaning
-        cleaner = EpcClean(data)
+            # Incorporate input data into cleaning
+            cleaner = EpcClean(data)
 
-        cleaner.clean()
-        # Extended cleaned_data
-        for k, data in cleaner.cleaned.items():
-            if k not in cleaned_data:
-                cleaned_data[k] = data
-            else:
-                existing_descriptions = [x["original_description"] for x in cleaned_data[k]]
-                new_data = [x for x in data if x["original_description"] not in existing_descriptions]
-                cleaned_data[k].extend(new_data)
+            cleaner.clean()
+            # Extended cleaned_data
+            for k, data in cleaner.cleaned.items():
+                if k not in cleaned_data:
+                    cleaned_data[k] = data
+                else:
+                    existing_descriptions = [x["original_description"] for x in cleaned_data[k]]
+                    new_data = [x for x in data if x["original_description"] not in existing_descriptions]
+                    cleaned_data[k].extend(new_data)
+        except Exception as e:
+            errors.append(directory)
+
+    if errors:
+        raise ValueError("We have errors")
 
     # Basic check to make sure all descriptions are unique
     for _, cleaned in cleaned_data.items():
@@ -74,6 +82,17 @@ def app():
     # data being read in will be extremely small, meaning quicker load times. We'll begin by storing as a single
     # file and monitor usage patterns to see if it makes sense to split the data up
 
+    cleaned_historic = read_from_s3(
+        s3_file_name="cleaned_epc_data/cleaned.bson",
+        bucket_name=f"retrofit-data-{ENVIRONMENT}"
+    )
+    cleaned_historic = msgpack.unpackb(cleaned_historic, raw=False)
+    save_data_to_s3(
+        data=msgpack.packb(cleaned_historic, use_bin_type=True),
+        s3_file_name=f"cleaned_epc_data/archive/{str(datetime.now())} - cleaned.bson",
+        bucket_name=f"retrofit-data-{ENVIRONMENT}"
+    )
+
     save_data_to_s3(
         data=msgpack.packb(cleaned_data, use_bin_type=True),
         s3_file_name="cleaned_epc_data/cleaned.bson",

etl/epc_clean/epc_attributes/FloorAttributes.py

@@ -34,6 +34,8 @@ class FloorAttributes(Definitions):
         "i ofod heb ei wresogi, dim inswleiddio (rhagdybiaeth)": "to unheated space, no insulation (assumed)",
         "i ofod heb ei wresogi, heb ei inswleiddio (rhagdybiaeth)": "to unheated space, no insulation (assumed)",
         "i ofod heb ei wresogi, dim inswleiddio": "to unheated space, no insulation",
+        "igçör awyr y tu allan, wedigçöi inswleiddio (rhagdybiaeth)": "to external air, insulated (assumed)",
+        "crog, inswleiddio cyfyngedig (rhagdybiaeth)": "suspended, limited insulation (assumed)"
     }
 
     def __init__(self, description: str):

etl/epc_clean/epc_attributes/HotWaterAttributes.py

@@ -130,6 +130,7 @@ class HotWaterAttributes(Definitions):
         "o r brif system, gydag ynni r haul, dim thermostat ar y silindr": "from main system, plus solar, no cylinder "
                                                                            "thermostat",
         "o r brif system, gydag ynni r haul": "from main system, plus solar",
+        "pwmp gwres": "heat pump"
     }
 
     NODATA_DESCRIPTIONS = [

etl/epc_clean/epc_attributes/LightingAttributes.py

@@ -9,7 +9,10 @@ class LightingAttributes(Definitions):
         "goleuadau ynni-isel ym mhob un ogçör mannau gosod": "low energy lighting in all fixed outlets",
         "goleuadau ynni-isel ym mhob un o r mannau gosod": "low energy lighting in all fixed outlets",
         "dim goleuadau ynni-isel": "no low energy lighting",
-        "goleuadau ynni-isel ym mhob un o'r mannau gosod": 'Low energy lighting in all fixed outlets'
+        "goleuadau ynni-isel ym mhob un o'r mannau gosod": 'Low energy lighting in all fixed outlets',
+        "effeithlonrwydd goleuo da": 'good lighting efficiency',
+        "effeithlonrwydd goleuo is na'r cyfartaledd": 'below average lighting efficiency',
+        "effeithlonrwydd goleuo rhagorol": "excellent lighting efficiency"
     }
 
     OBSERVED_ERRORS = []

etl/epc_clean/epc_attributes/MainheatAttributes.py

@@ -16,9 +16,11 @@ class MainHeatAttributes(Definitions):
         "solar assisted heat pump",
         "exhaust source heat pump",
         "community heat pump",
+        "hot-water-only"
     ]
     FUEL_TYPES = ["electric", "mains gas", "wood logs", "coal", "oil", "wood pellets", "anthracite",
-                  "dual fuel mineral and wood", "smokeless fuel", "lpg", "b30k"]
+                  "dual fuel mineral and wood", "smokeless fuel", "lpg", "b30k", "mineral and wood",
+                  "dual fuel appliance"]
     DISTRIBUTION_SYSTEMS = ["radiators", "fan coil units", "pipes in screed above insulation",
                             "pipes in insulated timber floor", "pipes in concrete slab"]
     OTHERS = ["assumed", "electricaire", "assumed for most rooms"]
@@ -72,6 +74,10 @@ class MainHeatAttributes(Definitions):
         "dim system ar gael, rhagdybir bod gwresogyddion trydan, trydan": "no system present, electric heaters assumed",
         # Should be handled by edge cases
         ", trydan": ", electric",
+        'awyr gynnes, nwy prif gyflenwad': 'warm air, mains gas',
+        "bwyler a rheiddiaduron, nwy prif gyflenwad, gwresogyddion ystafell, trydan": "Boiler and radiators, "
+                                                                                      "mains gas, Room heaters, "
+                                                                                      "electric"
     }
 
     REMAP = {
@@ -86,6 +92,9 @@ class MainHeatAttributes(Definitions):
         "gas-fired heat pumps, electric": "air source heat pump, electric",
         "radiator heating, heat from boilers - gas": "boiler and radiators, mains gas",
         "heat pump, warm air, mains gas": "air source heat pump, warm air, mains gas",
+        "air sourceheat pump, radiators, electric": "air source heat pump, radiators, electric",
+        "bwyler gyda rheiddiaduron a gwres dan y llawr, nwy prif gyflenwad": "Boiler and radiators, mains gas, "
+                                                                             "Boiler and underfloor heating, mains gas",
     }
 
     edge_case_result = {}

etl/epc_clean/epc_attributes/MainheatControlAttributes.py

@@ -75,6 +75,7 @@ class MainheatControlAttributes(Definitions):
     TO_REMAP = {
         "celect control": 'celect-type control',
         "celect controls": 'celect-type control',
+        "celect type controls": 'celect-type control',
         "trv's, program & flow switch": 'trvs, programmer & flow switch',
         'appliance thermostat': 'appliance thermostats',
     }
@@ -118,6 +119,7 @@ class MainheatControlAttributes(Definitions):
         'rheoli r tal a llaw': 'manual charge control',
         'tal un gyfradd, thermostat ystafell yn unig': 'flat rate charging, room thermostat only',
         "rheoli'r t l llaw": "manual charge control",
+        "2205 rhaglennydd ac o leiaf ddau thermostat ystafell": "programmer and at least two room thermostats"
     }
 
     NO_DATA_DESCRIPTIONS = [

etl/epc_clean/epc_attributes/attribute_utils.py

@@ -108,9 +108,9 @@ def process_part(result: Dict[str, Union[str, bool]], part: str, attr_list: List
         if set(attr_words).issubset(set(part_words)):
             result[f'{prefix}{attr.replace(" ", "_")}'] = True
 
-    at_least_one_attribute_true = any(result.values())
-    if not at_least_one_attribute_true:
-        raise ValueError("No attribute matches found")
+    # at_least_one_attribute_true = any(result.values())
+    # if not at_least_one_attribute_true:
+    #     raise ValueError("No attribute matches found")
 
     return result
 

etl/epc_clean/requirements.txt

@@ -0,0 +1,5 @@
+tqdm
+pandas
+msgpack
+textblob
+boto3

etl/epc_clean/tests/test_attribute_utils.py

@@ -53,12 +53,11 @@ def test_process_part_value_errors():
     with pytest.raises(ValueError):
         attribute_utils.process_part(result, part, attr_list, prefix)
 
-
-# Test for no attribute matches found
-def test_process_part_no_matches():
-    result = {'has_glazing': False, 'has_glazed': False, 'has_glaze': False}
-    part = 'high performance coating'
-    attr_list = ['glazing', 'glazed', 'glaze']
-    prefix = 'has_'
-    with pytest.raises(ValueError):
-        attribute_utils.process_part(result, part, attr_list, prefix)
+# Test for no attribute matches found - we don't raise this error any more
+# def test_process_part_no_matches():
+#     result = {'has_glazing': False, 'has_glazed': False, 'has_glaze': False}
+#     part = 'high performance coating'
+#     attr_list = ['glazing', 'glazed', 'glaze']
+#     prefix = 'has_'
+#     with pytest.raises(ValueError):
+#         attribute_utils.process_part(result, part, attr_list, prefix)

etl/epc_clean/tests/test_data/test_mainheat_attributes_cases.py

@@ -864,7 +864,7 @@ mainheat_cases = [
      'has_wood_pellets': False, 'has_anthracite': False, 'has_dual_fuel_mineral_and_wood': True,
      'has_smokeless_fuel': False, 'has_lpg': False, 'has_assumed': False, 'has_electricaire': False,
      'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False, "has_electric_heat_pumps": False,
-     "has_micro-cogeneration": False},
+     "has_micro-cogeneration": False, 'has_mineral_and_wood': True},
     {'original_description': 'Room heaters, electric', 'has_radiators': False, 'has_fan_coil_units': False,
      'has_pipes_in_screed_above_insulation': False, 'has_pipes_in_insulated_timber_floor': False,
      'has_pipes_in_concrete_slab': False, 'has_boiler': False, 'has_air_source_heat_pump': False,
@@ -1455,8 +1455,7 @@ mainheat_cases = [
      'has_coal': False, 'has_oil': False, 'has_wood_pellets': False, 'has_anthracite': False,
      'has_dual_fuel_mineral_and_wood': True, 'has_smokeless_fuel': False, 'has_lpg': False, 'has_assumed': False,
      'has_electricaire': False, 'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False,
-     "has_electric_heat_pumps": False,
-     "has_micro-cogeneration": False},
+     "has_electric_heat_pumps": False, "has_micro-cogeneration": False, "has_mineral_and_wood": True},
     {'original_description': 'Bwyler a rheiddiaduron, dau danwydd (mwynau a choed)', 'has_radiators': True,
      'has_fan_coil_units': False,
      'has_pipes_in_screed_above_insulation': False, 'has_pipes_in_insulated_timber_floor': False,
@@ -1468,8 +1467,8 @@ mainheat_cases = [
      'has_coal': False, 'has_oil': False, 'has_wood_pellets': False, 'has_anthracite': False,
      'has_dual_fuel_mineral_and_wood': True, 'has_smokeless_fuel': False, 'has_lpg': False, 'has_assumed': False,
      'has_electricaire': False, 'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False,
-     "has_electric_heat_pumps": False,
-     "has_micro-cogeneration": False},
+     "has_electric_heat_pumps": False, "has_micro-cogeneration": False, "has_mineral_and_wood": True
+     },
     {'original_description': 'Pwmp gwres syGÇÖn tarddu yn y ddaear, dan y llawr, trydan', 'has_radiators': False,
      'has_fan_coil_units': False, 'has_pipes_in_screed_above_insulation': False,
      'has_pipes_in_insulated_timber_floor': False, 'has_pipes_in_concrete_slab': False, 'has_boiler': False,
@@ -1541,7 +1540,7 @@ mainheat_cases = [
      'has_wood_pellets': False, 'has_anthracite': False, 'has_dual_fuel_mineral_and_wood': True,
      'has_smokeless_fuel': False, 'has_lpg': False, 'has_assumed': False, 'has_electricaire': False,
      'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False, "has_electric_heat_pumps": False,
-     "has_micro-cogeneration": False},
+     "has_micro-cogeneration": False, "has_mineral_and_wood": True},
     {'original_description': 'Room heaters, wood pellets', 'has_radiators': False, 'has_fan_coil_units': False,
      'has_pipes_in_screed_above_insulation': False, 'has_pipes_in_insulated_timber_floor': False,
      'has_pipes_in_concrete_slab': False, 'has_boiler': False, 'has_air_source_heat_pump': False,
@@ -1707,6 +1706,52 @@ mainheat_cases = [
         'has_dual_fuel_mineral_and_wood': False, 'has_smokeless_fuel': False, 'has_lpg': False, 'has_b30k': False,
         'has_assumed': False, 'has_electricaire': False, 'has_assumed_for_most_rooms': False,
         'has_underfloor_heating': False
+    },
+    {
+        "original_description": "Boiler and radiators, dual fuel (mineral and wood)",
+        'has_radiators': True, 'has_fan_coil_units': False, 'has_pipes_in_screed_above_insulation': False,
+        'has_pipes_in_insulated_timber_floor': False, 'has_pipes_in_concrete_slab': False, 'has_boiler': True,
+        'has_air_source_heat_pump': False, 'has_room_heaters': False, 'has_electric_storage_heaters': False,
+        'has_warm_air': False, 'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False,
+        'has_community_scheme': False, 'has_ground_source_heat_pump': False, 'has_no_system_present': False,
+        'has_portable_electric_heaters': False, 'has_water_source_heat_pump': False, 'has_electric_heat_pump': False,
+        'has_micro-cogeneration': False, 'has_solar_assisted_heat_pump': False, 'has_exhaust_source_heat_pump': False,
+        'has_community_heat_pump': False, 'has_electric': False, 'has_mains_gas': False, 'has_wood_logs': False,
+        'has_coal': False, 'has_oil': False, 'has_wood_pellets': False, 'has_anthracite': False,
+        'has_dual_fuel_mineral_and_wood': True, 'has_smokeless_fuel': False, 'has_lpg': False, 'has_b30k': False,
+        'has_mineral_and_wood': True, 'has_assumed': False, 'has_electricaire': False,
+        'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False
+    },
+    {
+        "original_description": "Room heaters, dual fuel appliance",
+        'has_radiators': False, 'has_fan_coil_units': False, 'has_pipes_in_screed_above_insulation': False,
+        'has_pipes_in_insulated_timber_floor': False, 'has_pipes_in_concrete_slab': False, 'has_boiler': False,
+        'has_air_source_heat_pump': False, 'has_room_heaters': True, 'has_electric_storage_heaters': False,
+        'has_warm_air': False, 'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False,
+        'has_community_scheme': False, 'has_ground_source_heat_pump': False, 'has_no_system_present': False,
+        'has_portable_electric_heaters': False, 'has_water_source_heat_pump': False, 'has_electric_heat_pump': False,
+        'has_micro-cogeneration': False, 'has_solar_assisted_heat_pump': False, 'has_exhaust_source_heat_pump': False,
+        'has_community_heat_pump': False, 'has_electric': False, 'has_mains_gas': False, 'has_wood_logs': False,
+        'has_coal': False, 'has_oil': False, 'has_wood_pellets': False, 'has_anthracite': False,
+        'has_dual_fuel_mineral_and_wood': False, 'has_smokeless_fuel': False, 'has_lpg': False, 'has_b30k': False,
+        'has_mineral_and_wood': False, 'has_dual_fuel_appliance': True, 'has_assumed': False, 'has_electricaire': False,
+        'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False
+
+    },
+    {
+        "original_description": "Hot-Water-Only Systems, electric",
+        'has_radiators': False, 'has_fan_coil_units': False, 'has_pipes_in_screed_above_insulation': False,
+        'has_pipes_in_insulated_timber_floor': False, 'has_pipes_in_concrete_slab': False, 'has_boiler': False,
+        'has_air_source_heat_pump': False, 'has_room_heaters': False, 'has_electric_storage_heaters': False,
+        'has_warm_air': False, 'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False,
+        'has_community_scheme': False, 'has_ground_source_heat_pump': False, 'has_no_system_present': False,
+        'has_portable_electric_heaters': False, 'has_water_source_heat_pump': False, 'has_electric_heat_pump': False,
+        'has_micro-cogeneration': False, 'has_solar_assisted_heat_pump': False, 'has_exhaust_source_heat_pump': False,
+        'has_community_heat_pump': False, 'has_hot-water-only': True, 'has_electric': True, 'has_mains_gas': False,
+        'has_wood_logs': False, 'has_coal': False, 'has_oil': False, 'has_wood_pellets': False, 'has_anthracite': False,
+        'has_dual_fuel_mineral_and_wood': False, 'has_smokeless_fuel': False, 'has_lpg': False, 'has_b30k': False,
+        'has_mineral_and_wood': False, 'has_dual_fuel_appliance': False, 'has_assumed': False,
+        'has_electricaire': False, 'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False
 
     }
 ]

recommendations/Costs.py

@@ -57,8 +57,6 @@ INSTALLER_SOLAR_COSTS = [
     {'n_panels': 17, 'array_kwp': 17 * PANEL_SIZE, 'cost': 6637.95, 'installer': 'CEG'},
     {'n_panels': 18, 'array_kwp': 18 * PANEL_SIZE, 'cost': 6792.57, 'installer': 'CEG'}
 ]
-# This is the maximum number of panels that we have a cost from the installers for
-INSTALLER_MAX_PANELS = 18
 
 # CEG uses use Solshare as an inverter to provide solar PV to multiple flats. This costs £7500 for the inverter alone
 # https://midsummerwholesale.co.uk/buy/solshare
@@ -101,8 +99,6 @@ INSTALLER_ASHP_COSTS = [
     {'capacity_kw': None, 'brand': '2 x cascaded ASHPs', 'tank_size_liters': 500, 'cost': 22950.00, 'installer': 'CEG'}
 ]
 
-BOILER_UPGRADE_SCHEME_ASHP_VALUE = 7500
-
 INSTALLER_SOLAR_BATTERY_COSTS = [
     {'capacity_kwh': 5, 'description': 'Battery Add on', 'cost': 3769.89, 'installer': 'JJC'},
     # {'capacity_kwh': 10, 'description': 'Battery Add on', 'cost': 4300.00, 'installer': 'CEG'},
@@ -131,19 +127,10 @@ TTZC_ROOM_TEMPERATURE_SENSOR_LABOUR_HOURS = 0.17  # (Assume ~ 10 mins install pe
 TTZC_SMART_RADIATOR_VALUES = 50
 TTZC_SMART_RADIATOR_VALUES_LABOUR_HOURS = 0.37  # (Assume ~ 15-30 mins install per valve)
 
-# Low carbon combi boiler - median value based on £2200 - £3000 range
-LOW_CARBON_COMBI_BOILER = 2200
-
 # boiler prices based on
-# https://www.greenmatch.co.uk/boilers/30kw-boiler
-# https://www.greenmatch.co.uk/boilers/35kw-boiler
-# https://www.greenmatch.co.uk/boilers/40kw-boiler
+# This is the cost of a firs time central heating install from The Warm Front rate card
 # These are exclusive of installation costs
-CONDENSING_BOILER_COSTS = {
-    "30kw": 1550,
-    "35kw": 1610,
-    "40kw": 1625
-}
+CONDENSING_BOILER_COST = 2600
 
 # Electric boiler prices base on
 # https://www.greenmatch.co.uk/boilers/combi-boilers/electric-combi-boilers
@@ -155,10 +142,6 @@ ELECTRIC_BOILER_COSTS = 1800
 ROOM_HEATER_REMOVAL_COST = 25
 ROOM_HEATER_REMOVAL_LABOUR_HOURS = 3
 
-# This is a cost quoted by Jim for a system flush - existig system will run more efficiently
-SYSTEM_FLUSH_COST = 250
-
-SINGLE_RADIATOR_COST = 150
 DOUBLE_RADIATOR_COST = 300
 FLUE_COST = 600
 PIPEWORK_COST = 750  # Min cost is £500
@@ -185,33 +168,27 @@ class Costs:
     # We assume a conservative 10% contingency for all works which is a rate defined by SPONs
     CONTINGENCY = 0.1
 
-    # For flat roof, we assume it's a high risk project as it's very weather dependent and also is heavily
-    # dependent on the quality of the existing roof
-    FLAT_ROOF_CONTINGENCY = 0.15
-
-    # We use a higher contingency rate for internal wall insulation because of the potential for issues with moving
-    # fittings and trimming doors, as well as scope for damage to the existing wall during preparation.
-    IWI_CONTINGENCY = 0.2
-
-    # For air source heat pumps, we inflate the assume cost by quite a bit to account for design and installation
-    ASHP_CONTINGENCY = 0.25
-    # Where there is more uncertainty, a higher contingency rate is used
-    HIGH_RISK_CONTINGENCY = 0.2
-    # When there is less uncertainty, a lower contingency rate is used
-    LOW_RISK_CONTINGENCY = 0.05
+    # Measure level contingency
+    CONTINGENCIES = {
+        "cavity_wall_insulation": 0.1,
+        "internal_wall_insulation": 0.26,
+        "external_wall_insulation": 0.26,
+        "loft_insulation": 0.1,
+        "solar_pv": 0.15,
+        "air_source_heat_pump": 0.25,
+        "flat_roof_insulation": 0.26,
+        "suspended_floor_insulation": 0.2,
+        "solid_floor_insulation": 0.26,
+        "low_energy_lighting": 0.26,
+        "high_heat_retention_storage_heaters": 0.1,
+        "windows_glazing": 0.15,
+    }
 
     # Preliminaries are a percentage of the total cost of the work and covers the cost of site-specific costs
     # such as site preparation, safety measures and project management. This rate can vary but we'll assume a 10%
     # rate, on the total cost before VAT, as recommended by SPONs
     PRELIMINARIES = 0.1
 
-    # For higher risk projects, a higher preliminaries rate is used. SPONs indicates that a higher risk project might
-    # have a preliminaries of 12-14% so we use 12% as the median for the preliminaries rate.
-    # For External wall insulation (EWI), we use 15% as the preliminaries rate if we think the property might
-    # need scaffolding, otherwise we use 12%. This is to account for any site preparation that might be required
-    EWI_NO_SCAFFOLDING_PRELIMINARIES = 0.2
-    EWI_SCAFFOLDING_PRELIMINARIES = 0.25
-
     VAT_RATE = 0.2
     PROFIT_MARGIN = 0.2
 
@@ -273,33 +250,14 @@ class Costs:
         labour_hours = 8
         labour_days = 1
 
-        # if the material is based on an installer cost, we return the flat price
-        if material["is_installer_quote"]:
-            total_cost = material["total_cost"] * wall_area
-
-            return {
-                "total": total_cost,
-                "labour_hours": labour_hours,
-                "labour_days": labour_days,
-            }
-
-        total_including_vat = material["total_cost"] * wall_area
-
-        if is_extraction_and_refill:
-            total_including_vat = CAVITY_EXTRACTION_COST * wall_area
-            # Additional 2 days work
-            labour_hours += + (2 * 8)
-            labour_days += + 2
-
-        total_excluding_vat = total_including_vat / (1 + self.VAT_RATE)
-        vat_cost = total_including_vat - total_excluding_vat
+        total_cost = material["total_cost"] * wall_area
 
         return {
-            "total": total_including_vat,
-            "subtotal": total_excluding_vat,
-            "vat": vat_cost,
+            "total": total_cost,
+            "contingency": self.CONTINGENCIES["cavity_wall_insulation"] * total_cost,
+            "contingency_rate": self.CONTINGENCIES["cavity_wall_insulation"],
             "labour_hours": labour_hours,
-            "labour_days": labour_days
+            "labour_days": labour_days,
         }
 
     def loft_and_flat_insulation(self, floor_area, material):
@@ -310,25 +268,20 @@ class Costs:
         :return: A dictionary containing detailed cost breakdown.
         """
 
-        if material["is_installer_quote"]:
-            total_cost = material["total_cost"] * floor_area
-
-            return {
-                "total": total_cost,
-                "labour_hours": 8,
-                "labour_days": 1,
-            }
-
-        total_including_vat = material["total_cost"] * floor_area
-        total_excluding_vat = total_including_vat / (1 + self.VAT_RATE)
-        vat_cost = total_including_vat - total_excluding_vat
+        total_cost = material["total_cost"] * floor_area
+        if material["type"] == "loft_insulation":
+            contingency_rate = self.CONTINGENCIES["loft_insulation"]
+            contingency = contingency_rate * total_cost
+        else:
+            contingency_rate = self.CONTINGENCIES["flat_roof_insulation"]
+            contingency = contingency_rate * total_cost
 
         return {
-            "total": total_including_vat,
-            "subtotal": total_excluding_vat,
-            "vat": vat_cost,
+            "total": total_cost,
+            "contingency": contingency,
+            "contingency_rate": contingency_rate,
             "labour_hours": 8,
-            "labour_days": 1
+            "labour_days": 1,
         }
 
     def solid_wall_insulation(self, wall_area, material):
@@ -338,469 +291,155 @@ class Costs:
         """
 
         # if the material is based on an installer cost, we return the flat price
-        if material["is_installer_quote"]:
-            total_cost = material["total_cost"] * wall_area
+        total_cost = material["total_cost"] * wall_area
 
-            labour_hours = material["labour_hours_per_unit"] * wall_area
+        if material["type"] == "internal_wall_insulation":
+            contingency_rate = self.CONTINGENCIES["internal_wall_insulation"]
+        else:
+            contingency_rate = self.CONTINGENCIES["external_wall_insulation"]
 
-            # To install internal wall insulation, a small to medium size project might be conducted by a team of 3-5
-            # people
-            labour_days = (labour_hours / 8) / 4
+        labour_hours = material["labour_hours_per_unit"] * wall_area
 
-            return {
-                "total": total_cost,
-                "labour_hours": labour_hours,
-                "labour_days": labour_days,
-            }
-
-        # Break out the individual material costs
-        # Since we don't know the exact wall construction, we take an average for demolition costs, since
-        # the cost will depend on the type of wall construction
-
-        total_including_vat = material["total_cost"] * wall_area
-        total_excluding_vat = total_including_vat / (1 + self.VAT_RATE)
-        vat_cost = total_including_vat - total_excluding_vat
-
-        # We estimate 1 weeks worth of work
-        labour_hours = 160
-        # To install internal wall insulation, a small to medium size project might be conducted by a team of 3-5 people
+        # To install internal wall insulation, a small to medium size project might be conducted by a team of 3-5
+        # people
         labour_days = (labour_hours / 8) / 4
 
         return {
-            "total": total_including_vat,
-            "subtotal": total_excluding_vat,
-            "vat": vat_cost,
+            "total": total_cost,
+            "contingency": contingency_rate * total_cost,
+            "contingency_rate": contingency_rate,
             "labour_hours": labour_hours,
             "labour_days": labour_days,
         }
 
-    def suspended_floor_insulation(self, insulation_floor_area, material, non_insulation_materials):
+    def suspended_floor_insulation(self, insulation_floor_area, material):
         """
-        We characterise the steps for suspended floor insulation as the following tasks:
-
-        1) Removal of Carpet and Underfelt: Where necessary, remove existing floor coverings to access the floorboards.
-        2) Removal of Floor Boarding: Carefully remove floorboards to access the space beneath for insulation.
-        3) Installation of Vapour Barrier: Install a vapour barrier to prevent moisture from affecting
-        the insulation and floor structure.
-        4) Installation of Insulation: Fit the chosen insulation material between the joists in the floor void.
-        5) Refixing Floorboards: Replace and secure the floorboards after insulation installation.
-        6) Re-carpeting: Lay down the carpet or other floor coverings once the insulation and floorboards are in place.
-        :return:
+        Given an installer cost for the works, produces an estimate for the cost of works.
+        Includes contingency
         """
 
         # if the material is based on an installer cost, we return the flat price
-        if material["is_installer_quote"]:
-            total_cost = material["total_cost"] * insulation_floor_area
+        total_cost = material["total_cost"] * insulation_floor_area
 
-            labour_hours = material["labour_hours_per_unit"] * insulation_floor_area
-            # To install suspended floor insulation, a small to medium size project might be conducted by a team of 3
-            # people
-            labour_days = (labour_hours / 8) / 3
-
-            return {
-                "total": total_cost,
-                "labour_hours": labour_hours,
-                "labour_days": labour_days,
-            }
-
-        demolition_data = [x for x in non_insulation_materials if x["type"] == "suspended_floor_demolition"]
-        vapour_barrier_data = [x for x in non_insulation_materials if x["type"] == "suspended_floor_vapour_barrier"]
-        redecoration_data = [x for x in non_insulation_materials if x["type"] == "suspended_floor_redecoration"]
-
-        if (len(demolition_data) != 2) or (len(vapour_barrier_data) != 1) or (len(redecoration_data) != 2):
-            raise ValueError("Incorrect number of data entries for non-insulation materials")
-
-        # Break out the individual material costs
-        demolition_material_costs = sum([x["material_cost"] * insulation_floor_area for x in demolition_data])
-        insulation_material_costs = material["material_cost"] * insulation_floor_area
-        vapour_barrier_material_costs = vapour_barrier_data[0]["material_cost"] * insulation_floor_area
-        redecoration_material_costs = sum([x["material_cost"] * insulation_floor_area for x in redecoration_data])
-
-        demolition_labour_costs = sum([x["labour_cost"] * insulation_floor_area for x in demolition_data])
-        insulation_labour_costs = material["labour_cost"] * insulation_floor_area
-        vapour_barrier_labour_costs = vapour_barrier_data[0]["labour_cost"] * insulation_floor_area
-        redecoration_labour_costs = sum([x["labour_cost"] * insulation_floor_area for x in redecoration_data])
-
-        labour_costs = (demolition_labour_costs + insulation_labour_costs + vapour_barrier_labour_costs +
-                        redecoration_labour_costs)
-
-        labour_costs = labour_costs * self.labour_adjustment_factor
-
-        materials_costs = (demolition_material_costs + insulation_material_costs + vapour_barrier_material_costs +
-                           redecoration_material_costs)
-
-        subtotal_before_profit = labour_costs + materials_costs
-
-        # Because of the possiblity of damage to the existing floor, or difficulties associated to moving fittings,
-        # we use a higher contingency rate
-        contingency_cost = subtotal_before_profit * self.HIGH_RISK_CONTINGENCY
-        preliminaries_cost = subtotal_before_profit * self.PRELIMINARIES
-        profit_cost = subtotal_before_profit * self.PROFIT_MARGIN
-
-        subtotal_before_vat = subtotal_before_profit + contingency_cost + preliminaries_cost + profit_cost
-
-        vat_cost = subtotal_before_vat * self.VAT_RATE
-
-        total_cost = subtotal_before_vat + vat_cost
-
-        demolition_labour_hours = sum([x["labour_hours_per_unit"] * insulation_floor_area for x in demolition_data])
-        insulation_labour_hours = material["labour_hours_per_unit"] * insulation_floor_area
-        vapour_barrier_labour_hours = vapour_barrier_data[0]["labour_hours_per_unit"] * insulation_floor_area
-        redecoration_labour_hours = sum([x["labour_hours_per_unit"] * insulation_floor_area for x in redecoration_data])
-
-        labour_hours = (demolition_labour_hours + insulation_labour_hours + vapour_barrier_labour_hours +
-                        redecoration_labour_hours)
-
-        # Assume a team of 3 people for a small to medium size project
+        labour_hours = material["labour_hours_per_unit"] * insulation_floor_area
+        # To install suspended floor insulation, a small to medium size project might be conducted by a team of 3
+        # people
         labour_days = (labour_hours / 8) / 3
 
         return {
             "total": total_cost,
-            "subtotal": subtotal_before_vat,
-            "vat": vat_cost,
-            "contingency": contingency_cost,
-            "preliminaries": preliminaries_cost,
-            "material": materials_costs,
-            "profit": profit_cost,
+            "contengency": self.CONTINGENCIES["suspended_floor_insulation"] * total_cost,
+            "contingency_rate": self.CONTINGENCIES["suspended_floor_insulation"],
             "labour_hours": labour_hours,
             "labour_days": labour_days,
-            "labour_cost": labour_costs
         }
 
-    def solid_floor_insulation(self, insulation_floor_area, material, non_insulation_materials):
+    def solid_floor_insulation(self, insulation_floor_area, material):
         """
-        We characterise the steps for solid floor insulation as the following tasks:
-
-        1) Removal of Carpet and Underfelt: This is the initial stage where any existing floor coverings, like carpets,
-        tiles, or linoleum, are carefully removed. This exposes the solid floor beneath, which is typically concrete.
-
-        2) Preparation of Flooring: This step is critical. It involves:
-        - Cleaning the existing floor surface thoroughly to remove debris and ensure a flat surface.
-        - Assessing and repairing any damage to the concrete floor. This might include filling cracks or leveling
-        uneven areas.
-
-        3) Installation of a Damp Proof Membrane (DPM): Before installing insulation, a DPM is often laid down to
-        prevent moisture from rising into the insulation and the interior space. This step is crucial in areas prone to
-        dampness.
-
-        4) Install Insulation: The insulation, often in the form of rigid foam boards, is laid over the DPM.
-        The choice of insulation material will depend on the desired thermal properties and the available floor height.
-        Care is taken to minimize thermal bridges and ensure a snug fit between insulation boards.
-
-        5) Laying a New Subfloor: Over the insulation, a new subfloor is often installed. This could be a layer of
-        screed (a type of concrete) or wooden boarding, depending on the specific requirements and preferences.
-
-        6) Re-decoration and Finishing Touches: Once the subfloor is in place and has set or dried (if necessary),
-        the final floor finish can be applied. This might involve:
-        - Laying new tiles, wooden flooring, or other chosen materials.
-        - If you're planning to re-carpet, this would be the stage to do it.
-        - Skirting boards may need to be refitted or replaced.
-
-        7) Considerations for Doors and Fixtures: It's important to note that raising the floor level can affect door
-        thresholds and other fixtures. Doors may need to be trimmed, and fixtures might need adjustments.
+        based on costing data from installers, produces an estimate for the cost of works. Returns contingency
 
         :param insulation_floor_area: Area of the floor to be insulated
         :param material: Selected insulation material
-        :param non_insulation_materials: Non-insulation materials required for the job
         :return:
         """
 
-        # if the material is based on an installer cost, we return the flat price
-        if material["is_installer_quote"]:
-            total_cost = material["total_cost"] * insulation_floor_area
+        total_cost = material["total_cost"] * insulation_floor_area
 
-            labour_hours = material["labour_hours_per_unit"] * insulation_floor_area
-            # To install suspended floor insulation, a small to medium size project might be conducted by a team of 3
-            # people
-            labour_days = (labour_hours / 8) / 3
-
-            return {
-                "total": total_cost,
-                "labour_hours": labour_hours,
-                "labour_days": labour_days,
-            }
-
-        demolition_data = [x for x in non_insulation_materials if x["type"] == "solid_floor_demolition"]
-        preparation_data = [x for x in non_insulation_materials if x["type"] == "solid_floor_preparation"]
-        vapour_barrier_data = [x for x in non_insulation_materials if x["type"] == "solid_floor_vapour_barrier"]
-        redecoration_data = [x for x in non_insulation_materials if x["type"] == "solid_floor_redecoration"]
-
-        if ((len(demolition_data) != 1) or (len(preparation_data) != 2) or (len(vapour_barrier_data) != 1) or
-            (len(redecoration_data) != 3)):
-            raise ValueError("Incorrect number of data entries for non-insulation materials")
-
-        # Break out the individual material costs
-        preparation_material_costs = sum([x["material_cost"] * insulation_floor_area for x in preparation_data])
-        insulation_material_costs = material["material_cost"] * insulation_floor_area
-        vapour_barrier_material_costs = vapour_barrier_data[0]["material_cost"] * insulation_floor_area
-        redecoration_material_costs = sum([x["material_cost"] * insulation_floor_area for x in redecoration_data])
-
-        demolition_labour_costs = sum([x["labour_cost"] * insulation_floor_area for x in demolition_data])
-        preparation_labour_costs = sum([x["labour_cost"] * insulation_floor_area for x in preparation_data])
-        insulation_labour_costs = material["labour_cost"] * insulation_floor_area
-        vapour_barrier_labour_costs = vapour_barrier_data[0]["labour_cost"] * insulation_floor_area
-        redecoration_labour_costs = sum([x["labour_cost"] * insulation_floor_area for x in redecoration_data])
-
-        preparation_plant_costs = sum([x["plant_cost"] * insulation_floor_area for x in preparation_data])
-
-        labour_costs = (demolition_labour_costs + insulation_labour_costs + vapour_barrier_labour_costs +
-                        redecoration_labour_costs + preparation_labour_costs)
-
-        labour_costs = labour_costs * self.labour_adjustment_factor
-
-        materials_cost = (preparation_material_costs + insulation_material_costs + vapour_barrier_material_costs +
-                          redecoration_material_costs)
-
-        subtotal_before_profit = labour_costs + materials_cost + preparation_plant_costs
-
-        # We use HIGH_RISH_CONTINGENCY because of the potential for issues with moving fittings and trimming doors,
-        # as well as scope for damage to the existing floor during preparation.
-        contingency_cost = subtotal_before_profit * self.HIGH_RISK_CONTINGENCY
-        preliminaries_cost = subtotal_before_profit * self.PRELIMINARIES
-        profit_cost = subtotal_before_profit * self.PROFIT_MARGIN
-
-        subtotal_before_vat = subtotal_before_profit + contingency_cost + preliminaries_cost + profit_cost
-        vat_cost = subtotal_before_vat * self.VAT_RATE
-        total_cost = subtotal_before_vat + vat_cost
-
-        demolition_labour_hours = sum([x["labour_hours_per_unit"] * insulation_floor_area for x in demolition_data])
-        preparation_labour_hours = sum([x["labour_hours_per_unit"] * insulation_floor_area for x in preparation_data])
-        insulation_labour_hours = material["labour_hours_per_unit"] * insulation_floor_area
-        vapour_barrier_labour_hours = vapour_barrier_data[0]["labour_hours_per_unit"] * insulation_floor_area
-        redecoration_labour_hours = sum([x["labour_hours_per_unit"] * insulation_floor_area for x in redecoration_data])
-
-        labour_hours = (demolition_labour_hours + insulation_labour_hours + vapour_barrier_labour_hours +
-                        redecoration_labour_hours + preparation_labour_hours)
-
-        # Assume a team of 3 people for a small to medium size project
+        labour_hours = material["labour_hours_per_unit"] * insulation_floor_area
+        # To install suspended floor insulation, a small to medium size project might be conducted by a team of 3
+        # people
         labour_days = (labour_hours / 8) / 3
 
         return {
             "total": total_cost,
-            "subtotal": subtotal_before_vat,
-            "vat": vat_cost,
-            "contingency": contingency_cost,
-            "preliminaries": preliminaries_cost,
-            "material": materials_cost,
-            "profit": profit_cost,
+            "contingency": self.CONTINGENCIES["solid_floor_insulation"] * total_cost,
+            "contingency_rate": self.CONTINGENCIES["solid_floor_insulation"],
             "labour_hours": labour_hours,
             "labour_days": labour_days,
-            "labour_cost": labour_costs
         }
 
-    def low_energy_lighting(self, number_of_lights, number_current_lel_lights, material):
+    def low_energy_lighting(self, number_of_lights, material):
 
         """
         Calculates the total cost for low energy lighting based on material and labor costs,
         including contingency, preliminaries, profit, and VAT.
 
         :param number_of_lights: Int, number of light
-        :param number_current_lel_lights:  Int, number of low energy lights currently installed in the home
         :material: Dict, material data containing costs of fittings
         """
 
         # If there are no lights fitted in the property, we increase the contingency in case there are potential wiring
         # blockers
 
-        if number_current_lel_lights == 0:
-            contingency = self.HIGH_RISK_CONTINGENCY
-        else:
-            contingency = self.CONTINGENCY
+        total_cost = material["total_cost"] * number_of_lights
 
-        if material["is_installer_quote"]:
-            total_cost = material["total_cost"] * number_of_lights * (1 + contingency)
-
-            labour_hours = 1
-            labour_days = (labour_hours / 8)
-
-            return {
-                "total": total_cost,
-                "labour_hours": labour_hours,
-                "labour_days": labour_days,
-            }
-
-        material_cost = material["material_cost"] * number_of_lights
-        labour_cost = material["labour_cost"] * number_of_lights * self.labour_adjustment_factor
-
-        subtotal_before_profit = material_cost + labour_cost
-
-        contingency_cost = subtotal_before_profit * contingency
-
-        subtotal_before_vat = subtotal_before_profit + contingency_cost
-        vat_cost = subtotal_before_vat * self.VAT_RATE
-        total_cost = subtotal_before_vat + vat_cost
-
-        labour_hours = material["labour_hours_per_unit"] * number_of_lights
-        # Assume a single electrician installing
+        labour_hours = 1
         labour_days = (labour_hours / 8)
 
         return {
             "total": total_cost,
-            "subtotal": subtotal_before_vat,
-            "vat": vat_cost,
-            "contingency": contingency_cost,
-            "material": material_cost,
+            "contingency": self.CONTINGENCIES["low_energy_lighting"] * total_cost,
+            "contingency_rate": self.CONTINGENCIES["low_energy_lighting"],
             "labour_hours": labour_hours,
             "labour_days": labour_days,
-            "labour_cost": labour_cost
         }
 
     def window_glazing(self, number_of_windows, material, is_secondary_glazing=False):
         """
-        We characterise the jobs to be done for window glazing as the following:
-        1) Initial Assessment and Measurements: Before removing the existing window, it's essential to assess the
-        condition of the window frame and opening. Precise measurements are taken to ensure the new double glazed
-        windows fit perfectly.
-
-        2) Remove the Existing Window: This involves carefully dismantling and removing the old single glazed window. It
-        requires skill to avoid damaging the surrounding wall and the window frame (if it's to be reused).
-
-        3) Dispose of the Existing Window: The old window, especially if it's a single glazed unit, needs to be
-        disposed of responsibly. Glass and other materials should be recycled where possible.
-
-        4) Surface Preparation: The window opening might need some preparation, especially if there's damage or if
-        adjustments are needed to accommodate the new window. This can include repairing or replacing parts of the
-        window frame, sealing gaps, and ensuring the opening is level and square.
-
-        5) Install the Window Frame (if new frames are used): In many cases, double glazed windows come with their
-        frames. These need to be installed securely into the window opening. This process involves aligning, leveling,
-        and fixing the frame in place.
-
-        6) Install the Window Sill: If a new window sill is required, it is installed at this stage. It needs to be
-        correctly aligned with the frame and securely attached.
-
-        7) Install the Double Glazed Glass Units: The glass units are carefully inserted into the frame. This step
-        requires precision to ensure a snug fit without causing stress on the glass, which could lead to cracking or
-        breaking.
-
-        8) Sealing and Weatherproofing: After the glass units are in place, it's crucial to seal around the frame and
-        between the glass and frame to ensure there are no drafts and that the installation is weather-tight. This
-        typically involves applying silicone sealant or other appropriate sealing materials.
-
-        9) Finishing Touches: This includes any cosmetic work, such as trimming, painting, or staining the frame and
-        sill to match the rest of the property. It might also involve cleaning up any mess created during the
-        installation.
-
-        10) Inspection and Testing: Finally, the new windows should be inspected to ensure they open, close, and lock
-        correctly. This is also a good time to check for any gaps or issues with the sealing.
-
-        For this cost estimation process, we factor in initial assement into the preliminaries
+        Given an isntaller quote, produces an estimate for the cost of works.
 
         """
 
-        if material["is_installer_quote"]:
-            total_cost = material["total_cost"] * number_of_windows
-
-            labour_hours = material["labour_hours_per_unit"] * number_of_windows
-            # To install windows, a small to medium size project might be conducted by a team of 2-3 people
-            labour_days = (labour_hours / 8) / 2
-
-            return {
-                "total": total_cost,
-                "labour_hours": labour_hours,
-                "labour_days": labour_days,
-            }
-
-        material_cost = material["material_cost"] * number_of_windows
-
-        labour_cost = (
-            material["labour_cost"] * number_of_windows * self.labour_adjustment_factor
-        )
-        multiplier = self.SECONDARY_GLAZING_SCALING_FACTOR if is_secondary_glazing else (
-            self.SASH_WINDOW_INFLATION_FACTOR)
-
-        subtotal = (material_cost + labour_cost) * multiplier
-
-        contingency_cost = subtotal * self.CONTINGENCY
-        preliminaries_cost = subtotal * self.PRELIMINARIES
-        profit_cost = subtotal * self.PROFIT_MARGIN
-
-        subtotal_before_vat = subtotal + contingency_cost + preliminaries_cost + profit_cost
-
-        vat_cost = subtotal_before_vat * self.VAT_RATE
-
-        total_cost = subtotal_before_vat + vat_cost
+        total_cost = material["total_cost"] * number_of_windows
 
         labour_hours = material["labour_hours_per_unit"] * number_of_windows
-        labour_hours = labour_hours * self.SECONDARY_GLAZING_SCALING_FACTOR if is_secondary_glazing else labour_hours
-
-        # Assume a team of 2
+        # To install windows, a small to medium size project might be conducted by a team of 2-3 people
         labour_days = (labour_hours / 8) / 2
 
         return {
             "total": total_cost,
-            "subtotal": subtotal_before_vat,
-            "vat": vat_cost,
-            "contingency": contingency_cost,
-            "preliminaries": preliminaries_cost,
-            "material": material_cost,
-            "profit": profit_cost,
+            "contingency": self.CONTINGENCIES["windows_glazing"] * total_cost,
+            "contingency_rate": self.CONTINGENCIES["windows_glazing"],
             "labour_hours": labour_hours,
-            "labour_cost": labour_cost,
-            "labour_days": labour_days
+            "labour_days": labour_days,
         }
 
     @classmethod
     def solar_pv(
         cls,
-        n_panels: int | float,
-        has_battery: bool = False,
-        array_cost=None,
-        n_floors: int = 1,
-        battery_kwh: int = 5,
-        needs_inverter=False
+        solar_product,
+        scaffolding_options,
+        n_floors
     ):
 
         """
-        Calculates the total cost for solar PV based data provided by the MCS dashboard, which contains
-        costing data for installations of renewable and clean energy measures.
 
-        The data in the dashboard is filtered on domestic building installations and then the data across the
-        various regions is manually collected. There is currently no automated way to get the data from the MCS
-        dashboard
-
-        Price can also be benchmarked against this checkatrade article:
-        https://www.checkatrade.com/blog/cost-guides/cost-of-solar-panel-installation/
-        :param n_panels: Number of solar panels
-        :param has_battery: Bool, whether the system includes a battery
-        :param array_cost: float, containing the cost of the solar PV array
-        :param n_floors: int, number of floors in the property, used to estimate the cost of scaffolding
-        :param battery_kwh: int, capacity of the battery in kWh. Defaulted to 5
-        :param needs_inverter: Bool, whether the system needs an inverter, where the solar panels are feeding multiple
-                               units
         """
 
-        if n_panels > INSTALLER_MAX_PANELS:
-            base_cost = [c for c in INSTALLER_SOLAR_COSTS if c["n_panels"] == INSTALLER_MAX_PANELS][0]["cost"]
-            cost_per_panel = [
-                c for c in INSTALLER_SOLAR_COSTS if c["n_panels"] == (INSTALLER_MAX_PANELS - 1)
-            ][0]["cost"]
-            cost_per_panel = base_cost - cost_per_panel
-            system_cost = base_cost + (n_panels - INSTALLER_MAX_PANELS) * cost_per_panel
-        else:
-            system_cost = [c for c in INSTALLER_SOLAR_COSTS if c["n_panels"] == n_panels][0]["cost"]
+        system_cost = solar_product["total_cost"]
 
-        subtotal = array_cost if array_cost is not None else system_cost
+        if not solar_product["includes_scaffolding"]:
+            # We base this on the number of floors
+            scaffolding = [x["total_cost"] for x in scaffolding_options if x["size"] == n_floors]
+            if not scaffolding:
+                # If we have no options, handle this
+                if n_floors <= 3:
+                    raise ValueError("No scaffolding options available for 3 or fewer floors")
+                # We take the largest scaffolding option available
+                scaffolding_cost = max([x["total_cost"] for x in scaffolding_options])
+            else:
+                scaffolding_cost = min(scaffolding)
 
-        if has_battery:
-            battery_cost = [c for c in INSTALLER_SOLAR_BATTERY_COSTS if c["capacity_kwh"] == battery_kwh][0]["cost"]
-            subtotal += battery_cost
-
-        if needs_inverter:
-            subtotal += INSTALLER_SOLAR_PV_INVERTER_COST
-            # We also add an additional labour cost
-            subtotal += INSTALLER_SOLAR_PV_INVERTER_LABOUR_COST
-
-        # Solar doesn't have VAT but we add a high risk contingency
-        # to account for design variation that we see in practice
-        total_cost = subtotal * (1 + cls.HIGH_RISK_CONTINGENCY)
+            system_cost += scaffolding_cost
 
         # Labour hours are based on estimates from online research but an average team seems to consist of 3 people
         # and most jobs take around 2 days. Assuming an 8 hour day for 3 people across 2 days, gives us 48 hours of
         # labour
         return {
-            "total": total_cost,
-            "subtotal": subtotal,
+            "total": system_cost,
+            "subtotal": system_cost,
+            "contingency": system_cost * cls.CONTINGENCIES["solar_pv"],
+            "contingency_rate": cls.CONTINGENCIES["solar_pv"],
             "vat": 0,
             "labour_hours": 48,
             "labour_days": 2,
@@ -826,6 +465,8 @@ class Costs:
         # We estimate the cost of an appliance thermostat at £400, which is the upper end of the range
         return {
             "total": total_cost,
+            "contengency": total_cost * self.CONTINGENCY,
+            "contingency_rate": self.CONTINGENCY,
             "subtotal": subtotal_before_vat,
             "vat": vat,
             "labour_hours": labour_hours,
@@ -852,13 +493,19 @@ class Costs:
 
         return {
             "total": total_cost,
+            "contingency": total_cost * self.CONTINGENCY,
+            "contingency_rate": self.CONTINGENCY,
             "subtotal": subtotal_before_vat,
             "vat": vat,
             "labour_hours": labour_hours,
             "labour_days": labour_days,
         }
 
-    def high_heat_electric_storage_heaters(self, number_heated_rooms, needs_cylinder):
+    def high_heat_electric_storage_heaters(
+        self, number_heated_rooms: int,
+        needs_cylinder: bool,
+        product: dict | None = None
+    ):
 
         """
         We base the estimates for the cost of electric storage heaters on the cost per room as estimated by the
@@ -867,23 +514,27 @@ class Costs:
 
         The cost is based on the number of heated rooms
         :param number_heated_rooms: int, number of rooms to be heated
+        :param needs_cylinder: bool, whether the property needs a hot water cylinder
+        :param product: dict, product data containing costs of heaters
         """
 
         if needs_cylinder:
-            # 1000 is the cost of a new hot water cylinder
-            total_cost = 1300 * number_heated_rooms + 1000
+            # 1500 is the cost of a new hot water cylinder
+            total_cost = product["total_cost"] * number_heated_rooms + 1500
         else:
             # 500 is the cost of a dual immersion heater - a rough estimate
-            total_cost = 1300 * number_heated_rooms + 500
+            total_cost = product["total_cost"] * number_heated_rooms + 500
+
         subtotal_before_vat = total_cost / (1 + self.VAT_RATE)
         vat = total_cost - subtotal_before_vat
 
-        # TODO: Rough estimate to be reviewed
         labour_hours = 3 * number_heated_rooms
         labour_days = np.ceil(labour_hours / 8)
 
         return {
             "total": total_cost,
+            "contingency": total_cost * self.CONTINGENCIES["high_heat_retention_storage_heaters"],
+            "contingency_rate": self.CONTINGENCIES["high_heat_retention_storage_heaters"],
             "subtotal": subtotal_before_vat,
             "vat": vat,
             "labour_hours": labour_hours,
@@ -903,6 +554,8 @@ class Costs:
         # We estimate the labour hours to be 4
         return {
             "total": total_cost,
+            "contingency": total_cost * self.CONTINGENCY,
+            "contingency_rate": self.CONTINGENCY,
             "subtotal": subtotal_before_vat,
             "vat": vat,
             "labour_hours": 4,
@@ -922,6 +575,8 @@ class Costs:
         # We estimate the labour hours to be 2
         return {
             "total": total_cost,
+            "contingency": total_cost * self.CONTINGENCY,
+            "contingency_rate": self.CONTINGENCY,
             "subtotal": subtotal_before_vat,
             "vat": vat,
             "labour_hours": 2,
@@ -940,6 +595,8 @@ class Costs:
 
         return {
             "total": total_cost,
+            "contingency": total_cost * self.CONTINGENCY,
+            "contingency_rate": self.CONTINGENCY,
             "subtotal": subtotal_before_vat,
             "vat": vat,
             "labour_hours": 0,
@@ -974,6 +631,8 @@ class Costs:
 
         return {
             "total": total_cost,
+            "contingency": total_cost * self.CONTINGENCY,
+            "contingency_rate": self.CONTINGENCY,
             "subtotal": subtotal_before_vat,
             "vat": vat,
             "labour_hours": labour_hours,
@@ -1006,6 +665,8 @@ class Costs:
 
         return {
             "total": total_cost,
+            "contingency": total_cost * self.CONTINGENCY,
+            "contingency_rate": self.CONTINGENCY,
             "subtotal": subtotal_before_vat,
             "vat": vat,
             "labour_hours": labour_hours,
@@ -1034,6 +695,8 @@ class Costs:
 
         return {
             "total": total_cost,
+            "contingency": total_cost * self.CONTINGENCY,
+            "contingency_rate": self.CONTINGENCY,
             "subtotal": subtotal_before_vat,
             "vat": vat,
             "labour_hours": labour_hours,
@@ -1056,6 +719,8 @@ class Costs:
 
         return {
             "total": total_cost,
+            "contingency": total_cost * self.CONTINGENCY,
+            "contingency_rate": self.CONTINGENCY,
             "subtotal": subtotal_before_vat,
             "vat": vat,
             "labour_hours": removal_labour_hours,
@@ -1096,7 +761,7 @@ class Costs:
         estimated_radiators = max(total_radiators_based_on_power, base_radiators + additional_radiators)
         return round(estimated_radiators)
 
-    def boiler(self, size, exising_room_heaters, system_change, n_heated_rooms, n_rooms, is_electric=False):
+    def boiler(self, exising_room_heaters, system_change, n_heated_rooms, n_rooms, is_electric=False):
         """
         Based on a basic estimate of median value £2600 to install a low carbon combi boiler
         First time central heating vosts can als be found here:
@@ -1105,7 +770,7 @@ class Costs:
         """
 
         if not is_electric:
-            unit_cost = CONDENSING_BOILER_COSTS[size]
+            unit_cost = CONDENSING_BOILER_COST
         else:
             unit_cost = ELECTRIC_BOILER_COSTS
         # The unit cost is the cost without VAT
@@ -1119,7 +784,6 @@ class Costs:
         # To be pessimistic, assume 2 days work
         labour_cost = labour_rate * self.labour_adjustment_factor * labour_days
         # Add contingency and preliminaries
-        labour_cost = labour_cost * (1 + self.CONTINGENCY + self.PRELIMINARIES)
 
         vat = labour_cost * self.VAT_RATE
 
@@ -1159,6 +823,8 @@ class Costs:
 
         return {
             "total": total_cost,
+            "contingency": total_cost * self.CONTINGENCY,
+            "contingency_rate": self.CONTINGENCY,
             "subtotal": subtotal_before_vat,
             "vat": vat,
             "labour_hours": labour_hours,
@@ -1178,19 +844,18 @@ class Costs:
         else:
             cost = [x for x in INSTALLER_ASHP_COSTS if x][0]["cost"]
 
-        # We add some contingency since there are additional costs such as resizing radiators, that could be required
-        subtotal = cost * (1 + self.ASHP_CONTINGENCY)
         # The costs from installers exclude VAT
-        vat = subtotal * self.VAT_RATE
-        total_cost = subtotal + vat
+        vat = cost * self.VAT_RATE
+        cost = cost + vat
 
         # We assume 5 days installation
         labour_days = 5
         labour_hours = labour_days * 8
 
         return {
-            "total": total_cost,
-            "subtotal": subtotal,
+            "total": cost,
+            "contingency": cost * self.CONTINGENCIES["air_source_heat_pump"],
+            "contingency_rate": self.CONTINGENCIES["air_source_heat_pump"],
             "vat": vat,
             "labour_hours": labour_hours,
             "labour_days": labour_days,

recommendations/FireplaceRecommendations.py

@@ -8,17 +8,18 @@ class FireplaceRecommendations(Definitions):
     For properties that have open fireplaces, we recommend sealing the fireplaces
     """
 
-    # This is our base assumption for the cost of the work
-    COST_OF_WORK = 235
-
     def __init__(
         self,
         property_instance: Property,
+        materials: list,
     ):
         self.property = property_instance
 
         self.has_ventilaion = None
         self.recommendation = None
+        self.materials = [m for m in materials if m["type"] == "sealing_fireplace"]
+        if len(self.materials) != 1:
+            raise ValueError("Incorrect number of sealing fireplace materials specified")
 
     def recommend(self, phase=0):
         """
@@ -32,14 +33,16 @@ class FireplaceRecommendations(Definitions):
         if number_open_fireplaces == 0:
             return
 
+        material = self.materials[0]
+
         already_installed = "sealing_open_fireplace" in self.property.already_installed
-        estimated_cost = number_open_fireplaces * self.COST_OF_WORK if not already_installed else 0
+        estimated_cost = number_open_fireplaces * material["total_cost"] if not already_installed else 0
 
         # We recommend installing two mechanical ventilation systems
         self.recommendation = [
             {
                 "phase": phase,
-                "parts": [],
+                "parts": [material],
                 "type": "sealing_open_fireplace",
                 "measure_type": "sealing_open_fireplace",
                 "description": "Seal %s open fireplaces" % str(number_open_fireplaces),
@@ -53,6 +56,7 @@ class FireplaceRecommendations(Definitions):
                 "labour_days": 6 * number_open_fireplaces / 8,  # Assume 8 hour day
                 "description_simulation": {
                     "number-open-fireplaces": 0
-                }
+                },
+                "innovation_rate": material["innovation_rate"],
             }
         ]

recommendations/FloorRecommendations.py

@@ -200,7 +200,6 @@ class FloorRecommendations(Definitions):
                         cost_result = self.costs.suspended_floor_insulation(
                             insulation_floor_area=self.property.insulation_floor_area,
                             material=material.to_dict(),
-                            non_insulation_materials=non_insulation_materials
                         )
 
                         already_installed = "suspended_floor_insulation" in self.property.already_installed
@@ -213,7 +212,6 @@ class FloorRecommendations(Definitions):
                         cost_result = self.costs.solid_floor_insulation(
                             insulation_floor_area=self.property.insulation_floor_area,
                             material=material.to_dict(),
-                            non_insulation_materials=non_insulation_materials
                         )
 
                         already_installed = "solid_floor_insulation" in self.property.already_installed
@@ -264,6 +262,7 @@ class FloorRecommendations(Definitions):
                                 material["type"] == "solid_floor_insulation"
                                 else "Suspended, insulated"
                             },
-                            **cost_result
+                            **cost_result,
+                            "innovation_rate": material["innovation_rate"],
                         }
                     )

recommendations/HeatingControlRecommender.py

@@ -93,7 +93,8 @@ class HeatingControlRecommender:
                     "measure_type": "programmer_appliance_thermostat",
                     "description": "upgrade heating controls to Programmer and Appliance or Smart Thermostats",
                     **self.costs.programmer_and_appliance_thermostat(has_programmer=has_programmer),
-                    "simulation_config": simulation_config
+                    "simulation_config": simulation_config,
+                    "innovation_rate": 0.0,
                 }
             )
 
@@ -142,7 +143,8 @@ class HeatingControlRecommender:
                 "description": "Upgrade heating controls to High Heat Retention Storage Heater Controls",
                 **self.costs.celect_type_controls(),
                 "simulation_config": simulation_config,
-                "description_simulation": description_simulation
+                "description_simulation": description_simulation,
+                "innovation_rate": 0.0,
             }
         )
 
@@ -232,7 +234,8 @@ class HeatingControlRecommender:
                 "sap_points": None,
                 "already_installed": already_installed,
                 "simulation_config": simulation_config,
-                "description_simulation": description_simulation
+                "description_simulation": description_simulation,
+                "innovation_rate": 0.0
             }
         )
 
@@ -307,7 +310,8 @@ class HeatingControlRecommender:
                 "sap_points": None,
                 "already_installed": already_installed,
                 "simulation_config": simulation_config,
-                "description_simulation": description_simulation
+                "description_simulation": description_simulation,
+                "innovation_rate": 0.0
             }
         )
 

recommendations/HeatingRecommender.py

@@ -1,11 +1,11 @@
 import re
 import backend.app.assumptions as assumptions
-from recommendations.Costs import Costs, BOILER_UPGRADE_SCHEME_ASHP_VALUE
 from recommendations.recommendation_utils import (
     check_simulation_difference, override_costs, combine_recommendation_configs
 )
 from backend.Property import Property
 from backend.app.plan.schemas import MEASURE_MAP
+from recommendations.Costs import Costs
 from etl.epc_clean.epc_attributes.MainheatAttributes import MainHeatAttributes
 from etl.epc_clean.epc_attributes.HotWaterAttributes import HotWaterAttributes
 from etl.epc_clean.epc_attributes.MainFuelAttributes import MainFuelAttributes
@@ -85,7 +85,7 @@ class HeatingRecommender:
         }
     }
 
-    def __init__(self, property_instance: Property):
+    def __init__(self, property_instance: Property, materials: list):
         self.property = property_instance
         self.costs = Costs(self.property)
 
@@ -103,6 +103,11 @@ class HeatingRecommender:
 
         self.dual_heating = self.identify_dual_heating()
 
+        # Split out the different materials
+        self.hhrsh_products = [
+            product for product in materials if product["type"] == "high_heat_retention_storage_heaters"
+        ]
+
     def identify_dual_heating(self):
         # All heat systems are in here so we identify whether two of these are true
         # MainHeatAttributes.HEAT_SYSTEMS
@@ -521,9 +526,8 @@ class HeatingRecommender:
             ashp_descriptions = {
                 "Time and temperature zone control": (
                     f"Install two cascaded air source heat pumps, and upgrade heating controls to Smart Thermostats, "
-                    "room sensors and smart radiator valves (time & temperature zone control). Ensure you have an 18 "
-                    "or "
-                    "24 hour tariff"
+                    "room sensors and smart radiator valves (time & temperature zone control). Ensure you have single "
+                    "tariff"
                 )
             }
         else:
@@ -531,9 +535,8 @@ class HeatingRecommender:
             ashp_descriptions = {
                 "Time and temperature zone control": (
                     f"Install a {ashp_size}KW air source heat pump, and upgrade heating controls to Smart Thermostats, "
-                    "room sensors and smart radiator valves (time & temperature zone control). Ensure you have an 18 "
-                    "or "
-                    "24 hour tariff"
+                    "room sensors and smart radiator valves (time & temperature zone control). Ensure you have a "
+                    "single tariff"
                 ),
                 "Programmer, TRVs and bypass": (
                     f"Install a {ashp_size}KW air source heat pump, with programmer, TRVs and a Bypass valve. Ensure "
@@ -555,7 +558,7 @@ class HeatingRecommender:
                         ashp_costs_with_controls[key] += controls_rec[key]
 
             if controls_rec is None:
-                description = f"Install a {ashp_size}KW Air source heat pump. Ensure you have an 18 or 24 hour tariff"
+                description = f"Install a {ashp_size}KW Air source heat pump. Ensure you have a single tariff"
             elif already_installed:
                 description = "The property already has an air source heat pump, no further action needed."
             else:
@@ -568,17 +571,18 @@ class HeatingRecommender:
                 if has_cavity_or_loft_recommendations:
                     description = description + (
                         f" You must ensure that the property has an insulated cavity and "
-                        f"270mm+ loft insulation to qualify for the grant, to claim £"
-                        f"{BOILER_UPGRADE_SCHEME_ASHP_VALUE} of funding from the boiler upgrade scheme grant. "
+                        f"270mm+ loft insulation to qualify for the grant, to claim £7,500"
+                        f" of funding from the boiler upgrade scheme grant. "
                     )
                 else:
                     description = description + (
-                        f" £{BOILER_UPGRADE_SCHEME_ASHP_VALUE} of funding can be claimed from the boiler upgrade scheme"
+                        f" £7,500 of funding can be claimed from the boiler upgrade scheme"
                     )
 
+            # These are the impacts based on a single tariff with an ashp
             simulation_config = {
-                "mainheat_energy_eff_ending": "Very Good",
-                "hot_water_energy_eff_ending": "Very Good"
+                "mainheat_energy_eff_ending": "Good",
+                "hot_water_energy_eff_ending": "Average"
             }
             description_simulation = {
                 "mainheat-description": new_heating_description,
@@ -644,7 +648,8 @@ class HeatingRecommender:
                 "already_installed": already_installed,
                 "simulation_config": simulation_config,
                 "description_simulation": description_simulation,
-                **ashp_costs_with_controls
+                **ashp_costs_with_controls,
+                "innovation_rate": 0
             }
 
             ashp_recommendations.append(ashp_recommendation)
@@ -678,7 +683,7 @@ class HeatingRecommender:
         heating_controls_only,
         system_change,
         system_type,
-        measure_type,
+        heating_product,
         non_intrusive_recommendation=None
     ):
         """
@@ -695,8 +700,7 @@ class HeatingRecommender:
         current system. If we have a system change and we have a heat control recommendation, we only recommend
         both heating and controls together
         :param system_type: The type of heating system we are recommending
-        :param measure_type: The type of measure we are recommending - more granular than the "type" field, allowing us
-        to distinguish between different types of heating recommendations
+        :param heating_product: The heating product we are recommending, used to determine the system type
         :param non_intrusive_recommendation: A non-intrusive recommendation, which may specify the number of SAP points
                                              or a cost for this recommendation
         """
@@ -747,7 +751,7 @@ class HeatingRecommender:
                 "phase": phase,
                 "parts": [],
                 "type": "heating",
-                "measure_type": measure_type,
+                "measure_type": heating_product["type"],
                 "description": recommendation_description,
                 "starting_u_value": None,
                 "new_u_value": None,
@@ -758,7 +762,10 @@ class HeatingRecommender:
                 "description_simulation": recommendation_description_simulation,
                 # We insert the heating system type here
                 "system_type": system_type,
-                "survey": non_intrusive_recommendation.get("survey", False)
+                "survey": non_intrusive_recommendation.get("survey", False),
+                # In this instance, we are recommending an entire heating system so the innovation rate is becased
+                # on the heating system as whole
+                "innovation_rate": heating_product["innovation_rate"],
             }
 
             output.append(recommendation)
@@ -921,10 +928,14 @@ class HeatingRecommender:
         if (number_heated_rooms == 1) and (self.property.number_of_rooms > 2):
             number_heated_rooms = self.property.number_of_rooms - 1
 
+        # We focus on the 700 watt product
+        hhrsh_product = next((x for x in self.hhrsh_products if x["size"] == 700), {})
+
         # Upgrade to electric storage heaters
         costs = self.costs.high_heat_electric_storage_heaters(
             number_heated_rooms=number_heated_rooms,
-            needs_cylinder=self.property.hotwater["system_type"] == "from main system"
+            needs_cylinder=self.property.hotwater["system_type"] == "from main system",
+            product=hhrsh_product
         )
         if self.dual_heating:
             description = self.DUAL_HEATING_DESCRIPTIONS[
@@ -960,8 +971,8 @@ class HeatingRecommender:
             heating_controls_only=heating_controls_only,
             system_change=system_change,
             system_type="high_heat_retention_storage_heater",
-            measure_type="high_heat_retention_storage_heater",
-            non_intrusive_recommendation=non_intrusive_recommendation
+            non_intrusive_recommendation=non_intrusive_recommendation,
+            heating_product=hhrsh_product
         )
         if _return:
             return recommendations
@@ -1053,13 +1064,6 @@ class HeatingRecommender:
         ), {})
 
         if has_inefficient_space_heating or has_inefficient_water:
-            boiler_size = self.estimate_boiler_size(
-                property_type=self.property.data["property-type"],
-                built_form=self.property.data["built-form"],
-                floor_area=self.property.floor_area,
-                floor_height=self.property.floor_height,
-                num_heated_rooms=self.property.data["number-heated-rooms"],
-            )
 
             if self.dual_heating:
                 description = self.DUAL_HEATING_DESCRIPTIONS[
@@ -1130,7 +1134,6 @@ class HeatingRecommender:
                 }
 
             boiler_costs = self.costs.boiler(
-                size=f"{boiler_size}kw",
                 exising_room_heaters=exising_room_heaters,
                 system_change=system_change,
                 n_heated_rooms=self.property.data["number-heated-rooms"],
@@ -1156,7 +1159,8 @@ class HeatingRecommender:
                 "description_simulation": description_simulation,
                 **boiler_costs,
                 "system_type": "boiler_upgrade",
-                "survey": non_invasive_recommendation.get("survey", None)
+                "survey": non_invasive_recommendation.get("survey", None),
+                "innovation_rate": 0,
             }
 
         # We recommend the heating controls
@@ -1197,7 +1201,10 @@ class HeatingRecommender:
                     heating_controls_only=False,
                     system_change=True,
                     system_type="boiler_upgrade",
-                    measure_type="boiler_upgrade",
+                    heating_product={  # temp until we do another product database update
+                        "type": "boiler_upgrade",
+                        "innovation_rate": 0
+                    }
                 )
                 combined_recommendations.extend(combined_recommendation)
 

recommendations/HotwaterRecommendations.py

@@ -110,7 +110,8 @@ class HotwaterRecommendations:
             "description_simulation": {
                 "hot-water-energy-eff": "Poor"
             },
-            "survey": survey
+            "survey": survey,
+            "innovation_rate": 0
         }
         if _return:
             return to_append
@@ -160,7 +161,8 @@ class HotwaterRecommendations:
                 "hot-water-energy-eff": self.property.data["hot-water-energy-eff"],
                 "hotwater-description": new_epc_description,
             },
-            "survey": survey
+            "survey": survey,
+            "innovation_rate": 0
         }
         if _return:
             return to_append
@@ -222,7 +224,8 @@ class HotwaterRecommendations:
                 "hot-water-energy-eff": simulation_config["hot_water_energy_eff_ending"],
                 "hotwater-description": new_epc_description,
             },
-            "survey": False
+            "survey": False,
+            "innovation_rate": 0
         }
 
         self.recommendations.append(to_append)

recommendations/LightingRecommendations.py

@@ -100,7 +100,7 @@ class LightingRecommendations:
         :return:
         """
 
-        if self.property.lighting["low_energy_proportion"] == 100:
+        if self.property.lighting["low_energy_proportion"] >= 1:
             return
 
         leds_recommendation_config = next(
@@ -126,7 +126,6 @@ class LightingRecommendations:
 
         cost_result = self.costs.low_energy_lighting(
             number_of_lights=number_non_lel_outlets,
-            number_current_lel_lights=number_lighting_outlets - number_non_lel_outlets,
             material=self.material
         )
 
@@ -170,6 +169,7 @@ class LightingRecommendations:
                     "low-energy-lighting": 100,
                 },
                 **cost_result,
-                "survey": leds_recommendation_config.get("survey", False)
+                "survey": leds_recommendation_config.get("survey", False),
+                "innovation_rate": self.material["innovation_rate"],
             }
         ]

recommendations/Recommendations.py

@@ -65,11 +65,11 @@ class Recommendations:
             property_instance=property_instance, materials=materials
         )
         self.draught_proofing_recommender = DraughtProofingRecommendations(property_instance=property_instance)
-        self.fireplace_recommender = FireplaceRecommendations(property_instance=property_instance)
+        self.fireplace_recommender = FireplaceRecommendations(property_instance=property_instance, materials=materials)
         self.lighting_recommender = LightingRecommendations(property_instance=property_instance, materials=materials)
         self.windows_recommender = WindowsRecommendations(property_instance=property_instance, materials=materials)
-        self.solar_recommender = SolarPvRecommendations(property_instance=property_instance)
-        self.heating_recommender = HeatingRecommender(property_instance=property_instance)
+        self.solar_recommender = SolarPvRecommendations(property_instance=property_instance, materials=materials)
+        self.heating_recommender = HeatingRecommender(property_instance=property_instance, materials=materials)
         self.hotwater_recommender = HotwaterRecommendations(property_instance=property_instance)
         self.secondary_heating_recommender = SecondaryHeating(property_instance=property_instance)
 

recommendations/RoofRecommendations.py

@@ -80,20 +80,6 @@ class RoofRecommendations:
 
         return None
 
-    def mds_loft_insulation(self, phase):
-        """
-        For usages within the mds report
-        :param phase:
-        :return:
-        """
-        self.recommendations = []
-
-        u_value = get_roof_u_value(**{**self.property.roof, "age_band": self.property.age_band})
-
-        self.recommend_roof_insulation(u_value, self.insulation_thickness, self.property.roof, phase)
-
-        return self.recommendations
-
     def is_loft_already_insulated(self, measures):
         """
         Check if the loft is already insulated
@@ -459,7 +445,8 @@ class RoofRecommendations:
                                 "roof-energy-eff": new_efficiency
                             },
                             **cost_result,
-                            "survey": non_invasive_recommendations.get("survey", False)
+                            "survey": non_invasive_recommendations.get("survey", False),
+                            "innovation_rate": material.to_dict()["innovation_rate"]
                         }
                     )
 
@@ -593,7 +580,8 @@ class RoofRecommendations:
                         },
                         **cost_result,
                         "already_installed": already_installed,
-                        "survey": rir_non_invasive_recommendation.get("survey", None)
+                        "survey": rir_non_invasive_recommendation.get("survey", None),
+                        "innovation_rate": material.to_dict()["innovation_rate"]
                     }
                 )
 

recommendations/SecondaryHeating.py

@@ -50,6 +50,7 @@ class SecondaryHeating:
                 },
                 "description_simulation": {
                     "secondheat-description": "None"
-                }
+                },
+                "innovation_rate": 0.0,  # No innovation rate for this measure
             }
         )

recommendations/SolarPvRecommendations.py

@@ -34,7 +34,9 @@ class SolarPvRecommendations:
         ]
     )
 
-    def __init__(self, property_instance):
+    PANEL_SIZES = [400, 435, 440, 445]
+
+    def __init__(self, property_instance, materials: list):
         """
         :param property_instance: Instance of the Property class, for the home associated to property_id
         """
@@ -44,6 +46,14 @@ class SolarPvRecommendations:
 
         self.recommendation = []
 
+        self.panels_products = [
+            material for material in materials if material["type"] == "solar_pv"
+        ]
+
+        self.scaffolding_options = [
+            material for material in materials if material["type"] == "scaffolding"
+        ]
+
     @staticmethod
     def trim_solar_wattage_options(scenarios_with_wattage):
         # Initialize the list with the first element, assuming the list is not empty
@@ -85,17 +95,25 @@ class SolarPvRecommendations:
             else:
                 raise Exception("IMPLEMENT ME")
 
+            # We get solar PV options
+            solar_product = [x for x in self.panels_products if x["id"] == recommendation_config["solar_product_id"]]
+            if not solar_product:
+                raise NotImplementedError(
+                    f"Solar product with id {recommendation_config['solar_product_id']} not found in "
+                    "panels_products"
+                )
+            solar_product = solar_product[0]
+
             n_floors = (
                 self.property.number_of_storeys["number_of_storeys"] if
                 self.property.number_of_storeys["number_of_storeys"] is not None else 3
             )
 
             total_cost = self.costs.solar_pv(
-                array_cost=recommendation_config.get("cost", None),
-                n_panels=recommendation_config["n_panels"],
+                solar_product=solar_product,
+                scaffolding_options=self.scaffolding_options,
                 n_floors=n_floors,
-                needs_inverter=True,
-            )["total"] / n_units
+            )["total"]
 
             kw = np.floor(recommendation_config["array_wattage"] / 100) / 10
             # Default to a weeks work for a team of 3 people doing 8 hour days
@@ -125,12 +143,34 @@ class SolarPvRecommendations:
                     # back up here
                     "photo_supply": roof_coverage_percent,
                     "has_battery": False,
+                    "simulation_config": {
+                        "photo_supply_ending": roof_coverage_percent
+                    },
                     "initial_ac_kwh_per_year": initial_ac_kwh_per_year,
                     "description_simulation": {"photo-supply": roof_coverage_percent},
-                    "rank": rank  # Rank is used to get the representative recommendation - rank 0 will be chosen
+                    "rank": rank,  # Rank is used to get the representative recommendation - rank 0 will be chosen
+                    "innovation_rate": solar_product["innovation_rate"],
                 }
             )
 
+    def _get_available_products(self, n_panels):
+        """
+        Utility function to get the available solar PV products based on the number of panels
+        :param n_panels:
+        :return:
+        """
+        available_products = []
+        for panel_size in self.PANEL_SIZES:
+            system_size = (n_panels * panel_size) / 1000
+            prods = [
+                x for x in self.panels_products if abs(x["size"] - system_size) < 0.01
+            ]
+            for x in prods:
+                x["panel_size"] = panel_size
+            available_products.extend(prods)
+
+        return available_products
+
     def recommend(self, phase):
         """
         We check if a property is potentially suitable for solar PV based on the following criteria:
@@ -184,37 +224,47 @@ class SolarPvRecommendations:
 
         # We combine each of these configurations with estimates with and without a battery
         for rank, recommendation_config in solar_configurations.iterrows():
-            roof_coverage_percent = round(recommendation_config["panneled_roof_area"] / roof_area * 100)
-            # We round up to the nearest 5
-            roof_coverage_percent = np.ceil(roof_coverage_percent / 5) * 5
 
-            # Typically, we've observed that every 5% of additional roof coverage will result in at least
-            # an additional 1 SAP points (though often 2 points) Given this, we can add a reasonable minimum
-            # for the number of SAP points we might expect. We've observed that for some cases where properties
-            # are hitting the higher SAP scores (e.g. EPC A and above), the model can sometimes under-predict
-            # the number of SAP points. This appears to be due to a relatively small number of properties
-            # actually achieving the upper echelons of EPC rating. This can be the case if we're simulating a
-            # whole house retrofit where the home is getting complete insulation, a heat pump and solar panels.
-            # Because panels are the final recommendation, they are often the measure that takes the home
-            # into the medium to high EPC A ranges and so because of a lack of training data, this means that
-            # we might sometime under-predict. This minimum is intended to try and reduce the negative impact
-            # of this. This minimum is used in Recommendations.calculate_recommendation_impact
-            minimum_sap_points = (roof_coverage_percent / 5) * self.SAP_POINTS_PER_5_PERCENT_ROOF_COVERAGE
+            n_panels = recommendation_config["n_panels"]
+
+            available_products = self._get_available_products(n_panels)
+
+            # Given the available products in the database, we product the possible array of recommendations
+            for solar_pv_product in available_products:
+
+                # we take the paneled roof area and this tells us the roof coverage, based on 400W panels
+                # We then look at the equivalent for larger panels, which will produce more energy in the same area
+
+                paneled_roof_area = recommendation_config["panneled_roof_area"]
+
+                roof_coverage_percent = round(
+                    ((paneled_roof_area / 400) * solar_pv_product["panel_size"]) / roof_area * 100
+                )
+                # We round up to the nearest 5
+                roof_coverage_percent = np.ceil(roof_coverage_percent / 5) * 5
+
+                # Note roof_coverage_percent is based on 400 watt panels, so we need to scale it up based on
+                # largest panels that will produce more energy in the same area
+
+                # Typically, we've observed that every 5% of additional roof coverage will result in at least
+                # an additional 1 SAP points (though often 2 points) Given this, we can add a reasonable minimum
+                # for the number of SAP points we might expect. We've observed that for some cases where properties
+                # are hitting the higher SAP scores (e.g. EPC A and above), the model can sometimes under-predict
+                # the number of SAP points. This appears to be due to a relatively small number of properties
+                # actually achieving the upper echelons of EPC rating. This can be the case if we're simulating a
+                # whole house retrofit where the home is getting complete insulation, a heat pump and solar panels.
+                # Because panels are the final recommendation, they are often the measure that takes the home
+                # into the medium to high EPC A ranges and so because of a lack of training data, this means that
+                # we might sometime under-predict. This minimum is intended to try and reduce the negative impact
+                # of this. This minimum is used in Recommendations.calculate_recommendation_impact
+                minimum_sap_points = (roof_coverage_percent / 5) * self.SAP_POINTS_PER_5_PERCENT_ROOF_COVERAGE
 
-            for has_battery in [False, True]:
                 cost_result = self.costs.solar_pv(
-                    has_battery=has_battery,
-                    array_cost=non_invasive_recommendation.get("cost", None),
-                    n_panels=recommendation_config["n_panels"],
+                    solar_product=solar_pv_product,
+                    scaffolding_options=self.scaffolding_options,
                     n_floors=self.property.number_of_floors
                 )
-                kw = np.floor(recommendation_config["array_wattage"] / 100) / 10
-                if has_battery:
-                    description = (
-                        f"Install a {kw} kilowatt-peak (kWp) solar panel system, with a battery."
-                    )
-                else:
-                    description = f"Install a {kw} kilowatt-peak (kWp) solar panel system."
+                description = f"{solar_pv_product['description']} - {solar_pv_product['size']} kWp system"
 
                 if self.property.in_conservation_area:
                     description += " Property is in a consevation area - please check with local planning authority."
@@ -226,7 +276,7 @@ class SolarPvRecommendations:
                 self.recommendation.append(
                     {
                         "phase": phase,
-                        "parts": [],
+                        "parts": [solar_pv_product],
                         "type": "solar_pv",
                         "measure_type": "solar_pv",
                         "description": description,
@@ -235,12 +285,12 @@ class SolarPvRecommendations:
                         "sap_points": minimum_sap_points,
                         "already_installed": already_installed,
                         **cost_result,
-                        # This is required for simulating the SAP impact. solar_pv_percentage is between 0 & 1 so we
-                        # scale
-                        # back up here
-                        "photo_supply": roof_coverage_percent,
-                        "has_battery": has_battery,
+                        "has_battery": solar_pv_product["includes_battery"],
+                        "simulation_config": {
+                            "photo_supply_ending": roof_coverage_percent
+                        },
                         "initial_ac_kwh_per_year": recommendation_config["initial_ac_kwh_per_year"],
                         "description_simulation": {"photo-supply": roof_coverage_percent},
+                        "innovation_rate": solar_pv_product["innovation_rate"],
                     }
                 )

recommendations/VentilationRecommendations.py

@@ -18,7 +18,8 @@ class VentilationRecommendations(Definitions):
         self.property = property_instance
 
         self.recommendation = None
-        self.materials = [part for part in materials if part["type"] == "mechanical_ventilation"]
+        self.mechanical_ventilation_materials = [part for part in materials if part["type"] == "mechanical_ventilation"]
+        self.trickle_vent_materials = [part for part in materials if part["type"] == "trickle_vent"]
 
     def recommend(self, phase):
         """
@@ -33,32 +34,32 @@ class VentilationRecommendations(Definitions):
         if self.property.has_ventilation:
             return
 
-        if len(self.materials) != 1:
-            raise NotImplementedError("Only handled the case of having one venilation option")
-
         # We recommend installing 2 units
         n_units = 2
 
-        part = self.materials.copy()
+        parts = self.mechanical_ventilation_materials.copy()
 
         already_installed = "cavity_wall_insulation" in self.property.already_installed
 
-        estimated_cost = n_units * part[0]["total_cost"] if not already_installed else 0
+        # TODO: We now have multiple ventilation options - we default to selecting the cheapest option
+        part = min(parts, key=lambda x: x['total_cost'])
+
+        estimated_cost = n_units * part["total_cost"] if not already_installed else 0
         labour_hours = 4 * n_units if not already_installed else 0
         labour_days = 4 * n_units / 8.0 if not already_installed else 0
 
-        part[0]["total"] = estimated_cost
-        part[0]["quantity"] = n_units
-        part[0]["quantity_unit"] = "part"
+        part["total"] = estimated_cost
+        part["quantity"] = n_units
+        part["quantity_unit"] = "part"
 
         # We recommend installing two mechanical ventilation systems
         self.recommendation = [
             {
                 "phase": phase,
-                "parts": part,
-                "type": part[0]["type"],
+                "parts": [part],
+                "type": part["type"],
                 "measure_type": "mechanical_ventilation",
-                "description": f"Install {n_units} {part[0]['description']} units",
+                "description": f"Install {n_units} {part['description']} units",
                 "starting_u_value": None,
                 "new_u_value": None,
                 "already_installed": already_installed,
@@ -76,7 +77,8 @@ class VentilationRecommendations(Definitions):
                 },
                 "description_simulation": {
                     "mechanical-ventilation": "mechanical, extract only"
-                }
+                },
+                "innovation_rate": part["innovation_rate"],
             }
         ]
 
@@ -99,6 +101,10 @@ class VentilationRecommendations(Definitions):
             else trickle_vents_recommendation_config["description"]
         )
 
+        cheapest_trickle_vent = min(
+            self.trickle_vent_materials, key=lambda x: x["total_cost"]
+        )
+
         return [
             {
                 "phase": None,
@@ -114,7 +120,7 @@ class VentilationRecommendations(Definitions):
                 "kwh_savings": 0,
                 "co2_equivalent_savings": 0,
                 "energy_cost_savings": 0,
-                "total": trickle_vents_recommendation_config["cost"],
+                "total": cheapest_trickle_vent["total_cost"] * self.property.number_of_windows,
                 # We use a very simple and rough estimate of 4 hours per unit
                 "labour_hours": trickle_vents_recommendation_config.get("labour_hours", 8),
                 "labour_days": trickle_vents_recommendation_config.get("labour_days", 1),  # Assume 8 hour day

recommendations/WallRecommendations.py

@@ -142,46 +142,6 @@ class WallRecommendations(Definitions):
 
         return True
 
-    def mds_recommend_cavity_wall_insulation(self, phase=None):
-        # Function specifically for cavity wall insulation, for usage in the mds report
-        self.recommendations = []
-        insulation_thickness = self.property.walls["insulation_thickness"]
-
-        u_value = get_wall_u_value(
-            clean_description=self.property.walls["clean_description"],
-            age_band=self.property.age_band,
-            is_granite_or_whinstone=self.property.walls["is_granite_or_whinstone"],
-            is_sandstone_or_limestone=self.property.walls["is_sandstone_or_limestone"],
-        )
-
-        # Test filling cavity
-        self.find_cavity_insulation(u_value, insulation_thickness, phase, measures)
-
-        return self.recommendations
-
-    def mds_recommend_ewi(self, phase=None):
-        # Function specifically for external wall insulation, for usage in the mds report
-        self.recommendations = []
-
-        u_value = self.property.walls["thermal_transmittance"]
-
-        if u_value is None:
-            u_value = get_wall_u_value(
-                clean_description=self.property.walls["clean_description"],
-                age_band=self.property.age_band,
-                is_granite_or_whinstone=self.property.walls["is_granite_or_whinstone"],
-                is_sandstone_or_limestone=self.property.walls["is_sandstone_or_limestone"],
-            )
-
-        # EWI
-        ewi_recommendations = self._find_insulation(
-            u_value=u_value,
-            insulation_materials=pd.DataFrame(self.external_wall_insulation_materials),
-            phase=phase
-        )
-
-        return ewi_recommendations
-
     def recommend(self, phase=0, measures=None, default_u_values=False):
         # if building built after 1990 + we're able to identify U-value +
         # U-value less than 0.18 and if in or close to a conversation area,
@@ -478,7 +438,8 @@ class WallRecommendations(Definitions):
                             "walls-energy-eff": "Good"
                         },
                         **cost_result,
-                        "survey": non_invasive_recommendations.get("survey", False)
+                        "survey": non_invasive_recommendations.get("survey", False),
+                        "innovation_rate": material.to_dict()["innovation_rate"]
                     }
                 )
 
@@ -658,7 +619,8 @@ class WallRecommendations(Definitions):
                                 "walls-energy-eff": simulation_config["walls_energy_eff_ending"]
                             },
                             **cost_result,
-                            "survey": survey
+                            "survey": survey,
+                            "innovation_rate": material.to_dict()["innovation_rate"]
                         }
                     )
 

recommendations/WindowsRecommendations.py

@@ -238,6 +238,7 @@ class WindowsRecommendations:
                 "description_simulation": description_simulation,
                 "simulation_config": simulation_config,
                 "survey": non_invasive_recommendation.get("survey", None),
+                "innovation_rate": self.glazing_material["innovation_rate"],
             }
         ]
 

recommendations/optimiser/CostOptimiser.py

@@ -12,7 +12,7 @@ class CostOptimiser:
     # We add an optional buffer to the minimum gain to allow for slack in the optimisation
     BUFFER = 0.2
 
-    def __init__(self, components, min_gain):
+    def __init__(self, components, min_gain, verbose=False):
         self.components = components
         self.min_gain = min_gain
         self.gain_constraint = None
@@ -22,6 +22,7 @@ class CostOptimiser:
 
         self.solution_cost = None
         self.solution_gain = None
+        self.verbose = verbose
 
     @classmethod
     def calculate_sap_gain_with_slack(cls, min_gain: int | float):
@@ -42,6 +43,8 @@ class CostOptimiser:
     def setup(self):
         # Initialize Model
         self.m = Model("knapsack")
+        # Set the verbosity level
+        self.m.verbose = 1 if self.verbose else 0
 
         # Create variables
         self.variables = [

recommendations/optimiser/GainOptimiser.py

@@ -9,7 +9,7 @@ class GainOptimiser:
     This class is used to maximise gain, given a constrained cost
     """
 
-    def __init__(self, components, max_cost, max_gain, allow_slack=True):
+    def __init__(self, components, max_cost, max_gain, allow_slack=True, verbose=False):
         """
         This function will try and maximise the gain, given a constrained cost. If we specific a max_gain, then the
         optimisation routine is constained to try not to exceed a maximum increase
@@ -23,6 +23,7 @@ class GainOptimiser:
         :param max_gain: Maximum gain constraint
         :param allow_slack: If True, allows the model to use slack variables to relax the cost constraint if the model
                             is infeasible. Defaults to True.
+        :param verbose: If True, enables verbose logging
         """
         self.components = components
         self.max_cost = max_cost
@@ -35,11 +36,15 @@ class GainOptimiser:
         self.solution_gain = None
         self.solution_cost = None
         self.allow_slack = allow_slack
+        self.verbose = verbose
 
     def setup(self):
         # Initialize Model
         self.m = Model("knapsack")
 
+        # Set the verbosity level
+        self.m.verbose = 1 if self.verbose else 0
+
         # Create variables
         self.variables = [
             [self.m.add_var(var_type=BINARY, name=str(component["id"])) for component in group] for group in

recommendations/optimiser/funding_optimiser.py

@@ -0,0 +1,892 @@
+"""
+This script contains a number of functions which are designed to enable optimisation and selection of funding options
+for individual properties to improve their energy efficiency
+
+The main entry point to this is optimise_with_funding_paths
+
+In the future, we will adapt this into a class-based structure to allow for more flexibility and reusability
+"""
+
+from copy import deepcopy
+import pandas as pd
+
+from backend.app.plan.schemas import (
+    WALL_INSULATION_MEASURES, ROOF_INSULATION_MEASURES, ECO4_ELIGIBILE_FABRIC_MEASURES
+)
+from recommendations.optimiser.CostOptimiser import CostOptimiser
+from recommendations.optimiser.GainOptimiser import GainOptimiser
+from utils.logger import setup_logger
+from backend.Funding import Funding
+
+logger = setup_logger()
+
+# measures we DO NOT treat as fundable in the ECO4 'funded' pass
+_ECO4_EXCLUDE_TYPES = {"secondary_heating", "extension_cavity_wall_insulation", "sealing_open_fireplace"}
+
+
+def _path_scheme(path_spec):
+    """
+    Infer scheme from any 'reference' tag in the path.
+    Defaults to 'eco4' if not specified.
+    """
+    for elem in path_spec or []:
+        ref = elem.get("reference")
+        if isinstance(ref, str):
+            if ref.endswith(":gbis"):
+                return "gbis"
+            if ref.endswith(":eco4"):
+                return "eco4"
+    return "eco4"
+
+
+def _filter_fundable_subgroups(groups, scheme):
+    """
+    Keep only options eligible for the funded pass of the given scheme.
+    - ECO4: drop excluded types (e.g., secondary_heating)
+    - GBIS: funded pass is the GBIS fixed measure only, so return empty sub-groups
+    """
+    if scheme == "gbis":
+        return []  # we won't optimise 'the rest' under GBIS here
+
+    # ECO4 case
+    filtered = []
+    for grp in groups:
+        kept = [opt for opt in grp
+                if not any(ex in opt["type"] for ex in _ECO4_EXCLUDE_TYPES)]
+        if kept:
+            filtered.append(kept)
+    return filtered
+
+
+def _sum_cost_gain_with_scheme(items, scheme):
+    """
+    Sum cost/gain of fixed items, adjusting for scheme rules.
+    - GBIS: strip innovation uplift from GBIS-funded fixed measures only.
+    """
+    total_cost = 0.0
+    total_gain = 0.0
+    for it in items:
+        cost = float(it["cost"])
+        if scheme == "gbis":
+            # innovation uplifts are not paid under GBIS
+            cost -= float(it.get("innovation_uplift", 0.0))
+        total_cost += cost
+        total_gain += float(it["gain"])
+    return total_cost, total_gain
+
+
+def violates_min_insulation(fixed):
+    """Return True if fixed selection includes a heating/PV measure but no required insulation."""
+    picked_types = {opt["type"] for (_, _, opt) in fixed}
+
+    def has_any(substrs):
+        return any(any(s in t for s in substrs) for t in picked_types)
+
+    # heating (incl. PV) flags
+    is_heating = has_any([
+        "air_source_heat_pump",
+        "high_heat_retention_storage_heater",
+        "boiler_upgrade",
+        "electric_boiler",
+        "time_temperature_zone_control",
+        "secondary_heating",
+        "solar_pv",  # PV treated as heating for MIR
+    ])
+
+    # MIR insulation (the ones you’re using in path construction)
+    has_insul = has_any([
+        "external_wall_insulation",
+        "internal_wall_insulation",
+        "cavity_wall_insulation",
+        "extension_cavity_wall_insulation",
+        "loft_insulation",
+        "flat_roof_insulation",
+        "room_roof_insulation",
+    ])
+
+    return is_heating and not has_insul
+
+
+# Treat "type" like "external_wall_insulation+mechanical_ventilation" → "external_wall_insulation"
+def _base_type(s: str) -> str:
+    return s.split("+", 1)[0]
+
+
+def _filter_measures_by_types(input_measures, allowed_types):
+    """
+    Keep only groups that have ≥1 allowed option; inside each group keep only allowed options.
+    """
+    allowed_set = set(allowed_types)
+    filtered = []
+    for group in input_measures:
+        kept_opts = [opt for opt in group if _base_type(opt["type"]) in allowed_set]
+        if kept_opts:
+            filtered.append(kept_opts)
+    return filtered
+
+
+def _is_eligible_funding_package(scheme, starting_sap, total_gain):
+    if scheme == "eco4":
+        # We check if we meet the upgrade requirements
+        # If the property is an E or above, we need to upgrade to a C or above
+        if starting_sap >= 39:  # ie. EPC C or above
+            return starting_sap + total_gain >= 69
+
+    if scheme == "gbis":
+        # GBIS is a fixed measure only, so we don't check the gain
+        return True
+
+
+def _prs_solution_ok(items, p, funding):
+    # items: list of picked option dicts (after optimisation)
+    # treat "type" possibly like "x+y" -> split and look at base tokens
+    types = set()
+    for opt in items:
+        for t in opt["type"].split("+"):
+            types.add(t)
+
+    has_solid_wall = ("external_wall_insulation" in types) or ("internal_wall_insulation" in types)
+
+    # renewable set:
+    has_ashp = ("air_source_heat_pump" in types)  # ASHP alone is renewable
+    has_solar = ("solar_pv" in types)
+    has_hhrsh = ("high_heat_retention_storage_heater" in types)  # only counts *with* solar
+
+    # solar PV qualifies if paired with eligible existing heating
+    solar_ok_existing = has_solar and funding.check_solar_eligible_heating_system(
+        p.main_heating["clean_description"], p.main_heating_controls["clean_description"]
+    )
+
+    # or paired with ASHP/HHRSH in the same package
+    solar_ok_with_installed = has_solar and (has_ashp or has_hhrsh)
+
+    renewable_ok = has_ashp or solar_ok_existing or solar_ok_with_installed
+
+    return has_solid_wall or renewable_ok
+
+
+def _ensure_unfunded_costs(groups):
+    """Make sure each option’s cost is base+uplift (i.e., no funding).
+    Safe if fields already match; works on a deepcopy.
+    """
+    for grp in groups:
+        for opt in grp:
+            base = opt.get("cost_minus_uplift")
+            if base is not None:
+                opt["cost"] = opt["raw_cost"]
+    return groups
+
+
+def optimise_with_funding_paths(p, input_measures, housing_type, funding: Funding, budget=None, target_gain=None):
+    """
+    run_optimizer(sub_measures, budget, target_gain) -> (picked_options, sub_cost, sub_gain)
+    """
+
+    solutions = []
+
+    # unfunded - we utilise all measures
+    unfunded_measures = input_measures.copy()
+    unfunded_measures = _ensure_unfunded_costs(unfunded_measures)
+    picked, total_cost, total_gain = run_optimizer(
+        unfunded_measures,
+        budget=budget,
+        sub_target_gain=target_gain
+    )
+    if picked is not None:
+        solutions.append({
+            "fixed_ids": [],
+            "items": picked,
+            "total_cost": total_cost,
+            "total_gain": total_gain,
+            "path": {"reference": "unfunded:all"},
+            "scheme": "none",
+            "is_eligible": False,  # no funding scheme applied
+            "unfunded_items": []
+        })
+
+    # This function will filter down on innovation measures if we are social EPC D
+    funding_paths, optimisation_input_measures = make_funding_paths(p, input_measures, housing_type, funding)
+
+    # We now produce a fabric only path for ECO4
+    # We add in generic insulation funding paths (where there is no fixed measure)
+    # Heating controls are only eligible if installed as part of a heating upgrade and so we do not include them
+    # here
+    if housing_type == "Social":
+        funding_paths = (
+            [
+                {
+                    'reference': 'fabric-only:eco4',
+                    "allowed_types": WALL_INSULATION_MEASURES + ROOF_INSULATION_MEASURES +
+                                     ECO4_ELIGIBILE_FABRIC_MEASURES
+                }
+            ] + funding_paths
+        )
+
+    for path_spec in funding_paths:
+
+        # ECO4 fabric only path = special case
+        if isinstance(path_spec, dict) and path_spec.get("reference") == "fabric-only:eco4":
+            sub_measures = _filter_measures_by_types(optimisation_input_measures, path_spec["allowed_types"])
+            # If the property is EPC D and socil, we also include just innovation measures
+            if housing_type == "Social" and p.data["current-energy-rating"] == "D":
+                # We add in a second option which is just innovation measures
+                sub_measures_innovation = []
+                for measures in sub_measures:
+                    group = []
+                    for measure in measures:
+                        if measure["innovation_uplift"]:
+                            group.append(measure)
+                    if group:
+                        sub_measures_innovation.append(group)
+                sub_measures = deepcopy(sub_measures_innovation)
+
+            if not sub_measures:
+                continue
+
+            picked, sub_cost, sub_gain = run_optimizer(
+                sub_measures,
+                budget=budget,  # no fixed items; budget unchanged
+                sub_target_gain=target_gain
+            )
+
+            if picked is None:
+                continue
+
+            scheme = _path_scheme([path_spec])
+
+            solutions.append(
+                {
+                    "fixed_ids": [],
+                    "items": picked,
+                    "total_cost": sub_cost,
+                    "total_gain": sub_gain,
+                    "path": path_spec,
+                    "scheme": scheme,
+                    "is_eligible": _is_eligible_funding_package(scheme, p.data["current-energy-efficiency"], sub_gain),
+                    "unfunded_items": []
+                }
+            )
+
+            continue
+
+        # 1) expand fixed selections for this path
+        fixed_selections = expand_funding_path(optimisation_input_measures, path_spec) if path_spec else [[]]
+        if not fixed_selections:
+            continue
+
+        for fixed in fixed_selections:
+
+            if violates_min_insulation(fixed):
+                # We log an error and skip this - we should not see any errors but we can probably get a reasonable
+                # outcome for the end user without a complete termination of the process
+                logger.error("Skipping fixed selection due to minimum insulation violation: %s", fixed)
+                continue
+
+            scheme = _path_scheme(path_spec)
+
+            # 3) compute fixed cost/gain, and strip those groups from subproblem
+            fixed_items = [opt for (_, _, opt) in fixed]
+
+            if scheme == "gbis":
+                # Re-set costs as the only funding we get is the PPS
+                for x in fixed_items:
+                    x["cost"] = x["raw_cost"]
+
+            fixed_ids = [opt['id'] for opt in fixed_items]
+            fixed_cost, fixed_gain = sum_cost_gain(fixed_items)
+            fixed_groups = {gi for (gi, _, _) in fixed}
+
+            sub_measures = deepcopy(
+                [grp for gi, grp in enumerate(optimisation_input_measures) if gi not in fixed_groups]
+            )
+
+            if scheme == "gbis":
+                # Then for the sub-measures, we need to strip the innovation uplift from the GBIS fixed measures. We
+                # do this by adding innovation back onto the cost
+                for grp in sub_measures:
+                    for opt in grp:
+                        opt["cost"] = opt["raw_cost"]
+
+            if scheme == "eco4":
+                # Need to strip out any measure types that are not eligible for ECO4 funding (e.g. secondary heating)
+                sub_measures = _filter_fundable_subgroups(sub_measures, scheme)
+
+            # 4) run your existing optimiser for the remaining groups
+            # If we have a budget, we need to ensure the subproblem respects it so we remove the fixed cost (which
+            # may already be over budget) and the fixed gain (which may not be achievable)
+
+            if fixed_gain > target_gain:
+                picked, sub_cost, sub_gain = ([], 0.0, 0.0)
+            elif fixed_gain < target_gain and not sub_measures:
+                picked, sub_cost, sub_gain = ([], 0.0, 0.0)
+            else:
+                picked, sub_cost, sub_gain = run_optimizer(
+                    sub_measures,
+                    budget - fixed_cost if budget is not None else None,
+                    sub_target_gain=target_gain - fixed_gain if target_gain is not None else None
+                )
+
+            if picked is None:
+                continue
+
+            total_cost = fixed_cost + sub_cost
+            total_gain = fixed_gain + sub_gain
+            total_picks = fixed_items + picked
+
+            if housing_type == "Private":
+                if not _prs_solution_ok(total_picks, p, funding):
+                    logger.error(
+                        "Found a solution that does not meet the PRS requirements: %s - this shouldn't be happening",
+                        total_picks
+                    )
+                    continue
+
+            scheme = _path_scheme(path_spec)
+
+            unfunded_picked = []
+            if total_gain - target_gain < -0.1:
+                # In this case, we have a funded package that does not meet the target gain, so we look at the remaining
+                # measures and see if we can include them
+                picked_types = {opt["type"] for opt in total_picks}
+
+                # We find the indexes of the picked types
+                picked_group_index = {}
+                for pt in picked_types:
+                    for gi, grp in enumerate(input_measures):
+                        if any(pt in opt["type"] for opt in grp):
+                            picked_group_index[pt] = gi
+                            break
+                # We get the remaining types
+                # ECO4 case
+                remaining = []
+                for i, grp in enumerate(input_measures):
+                    if i in picked_group_index.values():
+                        continue
+                    keep = [x for x in grp if x["type"] not in picked_types]
+                    if keep:
+                        for x in keep:
+                            # Adjust to raw cost (without funding)
+                            x["cost"] = x["raw_cost"]
+                        remaining.append(keep)
+
+                if remaining:
+                    # If we have remaining measures we can optimise, we run them down an unfunded route
+                    unfunded_picked, unfunded_cost, unfunded_gain = run_optimizer(
+                        remaining,
+                        budget - total_cost if budget is not None else None,
+                        sub_target_gain=target_gain - total_gain if target_gain is not None else None
+                    )
+
+                    total_cost += unfunded_cost
+                    total_gain += unfunded_gain
+
+            solutions.append({
+                "fixed_ids": fixed_ids,
+                "items": total_picks,
+                "total_cost": total_cost,
+                "total_gain": total_gain,
+                "path": path_spec,
+                "scheme": scheme,
+                "is_eligible": _is_eligible_funding_package(scheme, p.data["current-energy-efficiency"], total_gain),
+                "unfunded_items": unfunded_picked,
+            })
+
+    solutions = pd.DataFrame(solutions)
+
+    # Given the scheme, we now check if the packages are eligible. If they *are* eligible, but they don't meet the
+    # final upgrade target, we then look to perform a final optimisation pass to meet the target gain.
+    solutions["meets_upgrade_target"] = solutions["total_gain"] >= target_gain - 0.1
+
+    # If we have packages that are fundable, but do not meet the upgrade target, we can run a final optimisation pass
+    if not solutions[solutions["is_eligible"] & ~solutions["meets_upgrade_target"]].empty:
+        logger.info("We have some packages that are fundable but do not meet the target gain")
+
+    # We now can calculate the project ABS, which subtracts from the cost, but this is only relevant for ECO4
+    solutions["starting_sap"] = p.data["current-energy-efficiency"]
+    solutions["floor_area"] = p.floor_area
+    solutions["ending_sap"] = solutions["starting_sap"] + solutions["total_gain"]
+    solutions["starting_band"] = solutions["starting_sap"].apply(funding.get_sap_band)
+    solutions["ending_band"] = solutions["ending_sap"].apply(funding.get_sap_band)
+    solutions["floor_area_band"] = solutions["floor_area"].apply(funding.get_floor_area_band)
+    solutions["project_score"] = solutions.apply(
+        lambda x: funding._calculate_full_project_abs(
+            floor_area_band=x["floor_area_band"],
+            starting_sap_band=x["starting_band"],
+            ending_sap_band=x["ending_band"],
+        ),
+        axis=1
+    )
+
+    rate = funding.get_eco4_abs_rate(is_cavity=p.walls["is_cavity_wall"])
+    solutions["full_project_funding"] = solutions["project_score"] * rate
+    # if the scheme is not ECO4, we set the funding to 0 with iloc
+    solutions.loc[solutions["scheme"] != "eco4", "full_project_funding"] = 0.0
+    solutions["partial_project_funding"] = solutions.apply(lambda x: get_gbis_pp_funding(x), axis=1)
+    solutions["partial_project_score"] = solutions.apply(lambda x: get_gbis_pps(x), axis=1)
+
+    # We pull out uplifts
+    solutions["total_uplift"] = solutions.apply(lambda x: get_total_uplift(x), axis=1)
+    solutions["total_uplift_score"] = solutions.apply(lambda x: get_total_innovation_score(x), axis=1)
+
+    return solutions
+
+
+# ---- helpers -------------------------------------------------------------
+
+
+def get_gbis_pp_funding(x):
+    if x["scheme"] != "gbis":
+        return 0
+    fixed_ids = x["fixed_ids"]
+    if len(fixed_ids) != 1:
+        raise ValueError("More than one fixed ID for GBIS")
+    return [x for x in x["items"] if x["id"] in fixed_ids][0]["partial_project_funding"]
+
+
+def get_gbis_pps(x):
+    if x["scheme"] != "gbis":
+        return 0
+    fixed_ids = x["fixed_ids"]
+    if len(fixed_ids) != 1:
+        raise ValueError("More than one fixed ID for GBIS")
+    return [x for x in x["items"] if x["id"] in fixed_ids][0]["partial_project_score"]
+
+
+def get_total_uplift(x):
+    return sum([y["innovation_uplift"] for y in x["items"]])
+
+
+def get_total_innovation_score(x):
+    return sum([y["uplift_project_score"] for y in x["items"]])
+
+
+def sum_cost_gain(items):
+    c = sum(float(x['cost']) for x in items)
+    g = sum(float(x['gain']) for x in items)
+    return c, g
+
+
+# ---- candidate expansion -------------------------------------------------
+def type_matches(option_type: str, required: str) -> bool:
+    # substring match so "external_wall_insulation+mechanical_ventilation" satisfies "external_wall_insulation"
+    return required in option_type
+
+
+def candidates_for_type(input_measures, required_type):
+    """
+    Return a list of (gi, oi, opt) where opt['type'] contains required_type.
+    gi = group index, oi = option index inside that group.
+    """
+    cands = []
+    for gi, group in enumerate(input_measures):
+        for oi, opt in enumerate(group):
+            if type_matches(opt["type"], required_type):
+                cands.append((gi, oi, opt))
+    return cands
+
+
+def iter_or_candidates(input_measures, types_list):
+    """
+    For OR: pick exactly ONE candidate whose type matches ANY in types_list.
+    Return a list of dicts: {"fixed": [(gi, oi, opt)]}
+    """
+    union = []
+    seen_ids = set()
+    for t in types_list:
+        for tup in candidates_for_type(input_measures, t):
+            # de-dupe by the option id so the same physical option (with multi-type name) isn’t repeated
+            if tup[2]["id"] not in seen_ids:
+                seen_ids.add(tup[2]["id"])
+                union.append(tup)
+    return [{"fixed": [t]} for t in union]
+
+
+def iter_and_candidates(input_measures, types_list):
+    """
+    For AND: we must cover ALL required types.
+    We allow a single option to satisfy multiple types.
+    We build a simple product but collapse duplicates by (gi, oi).
+    """
+    # Build candidate pools per required type
+    pools = [candidates_for_type(input_measures, t) for t in types_list]
+    if any(len(pool) == 0 for pool in pools):
+        return []  # impossible to satisfy AND
+
+    # Start with one empty selection; accumulate per pool
+    selections = [[]]  # each selection is a list of (gi, oi, opt)
+    for pool in pools:
+        new_selections = []
+        for sel in selections:
+            for cand in pool:
+                # Try adding cand; collapse duplicates by (gi,oi)
+                gi, oi, opt = cand
+                replaced = False
+                conflict = False
+                merged = []
+                for (sgi, soi, sopt) in sel:
+                    if (sgi, soi) == (gi, oi):
+                        # same exact option already in selection (satisfies another required type) – keep one
+                        replaced = True
+                        # keep the existing one (identical)
+                        merged.append((sgi, soi, sopt))
+                    else:
+                        merged.append((sgi, soi, sopt))
+                if not replaced:
+                    merged.append(cand)
+                if not conflict:
+                    new_selections.append(merged)
+        selections = new_selections
+        if not selections:
+            return []
+
+    # After accumulation, we may still have duplicate groups with different options (conflict). Drop those.
+    cleaned = []
+    for sel in selections:
+        seen_by_group = {}
+        ok = True
+        for gi, oi, opt in sel:
+            if gi in seen_by_group and seen_by_group[gi] != oi:
+                # same group, different option -> conflict for AND; invalid selection
+                ok = False
+                break
+            seen_by_group[gi] = oi
+        if ok:
+            # ensure stable order and unique by (gi,oi)
+            uniq = {}
+            for gi, oi, opt in sel:
+                uniq[(gi, oi)] = opt
+            cleaned.append([(gi, oi, opt) for (gi, oi), opt in uniq.items()])
+    return [{"fixed": c} for c in cleaned]
+
+
+def expand_funding_path(input_measures, path_spec):
+    """
+    path_spec is a list of elements; each element is either:
+      {"OR": [type1, type2, ...], "reference": "..."}  or
+      {"AND": [type1, type2, ...], "reference": "..."}
+    We cross-product across elements (all required), and produce selections as lists of (gi, oi, opt).
+    """
+    selections = [[]]  # list[list[(gi,oi,opt)]]
+    for elem in path_spec:
+        if "OR" in elem:
+            cands = iter_or_candidates(input_measures, elem["OR"])
+        elif "AND" in elem:
+            cands = iter_and_candidates(input_measures, elem["AND"])
+        else:
+            raise ValueError("unknown path element; expected 'OR' or 'AND'")
+
+        if not cands:
+            return []
+
+        new_selections = []
+        for base in selections:
+            for cand in cands:
+                # merge base + cand["fixed"], collapsing duplicate same-option picks
+                combined = list(base)
+                # reject if combined picks two different options from the same group
+                groups_to_oi = {(gi,): oi for gi, oi, _ in combined}  # temporary; we’ll refactor below
+                conflict = False
+                # simpler: build a dict by group -> (oi, opt), conflict if group exists with different oi
+                gmap = {gi: (oi, opt) for gi, oi, opt in combined}
+                for gi, oi, opt in cand["fixed"]:
+                    if gi in gmap:
+                        prev_oi, _ = gmap[gi]
+                        if prev_oi != oi:
+                            conflict = True
+                            break
+                    gmap[gi] = (oi, opt)
+                if conflict:
+                    continue
+                # back to list
+                merged = [(gi, oi, opt) for gi, (oi, opt) in gmap.items()]
+                new_selections.append(merged)
+        selections = new_selections
+        if not selections:
+            return []
+
+    # Final tidy: ensure no duplicate groups with different options (already protected), keep stable ordering
+    deduped = []
+    for sel in selections:
+        gmap = {}
+        for gi, oi, opt in sel:
+            # keep the first occurrence
+            if gi not in gmap:
+                gmap[gi] = (oi, opt)
+            else:
+                # same group, different oi would have been filtered; if same oi, ignore duplicate
+                pass
+        deduped.append([(gi, oi, opt) for gi, (oi, opt) in gmap.items()])
+    return deduped
+
+
+# ---- tiny utilities ----------------------------------------------------------
+
+def parse_types(t):
+    # e.g. "external_wall_insulation+mechanical_ventilation" -> {"external_wall_insulation","mechanical_ventilation"}
+    return set(map(str.strip, t.split("+"))) if isinstance(t, str) else set()
+
+
+def includes_heating(opt_types):
+    return any(x in opt_types for x in {
+        "air_source_heat_pump",
+        "high_heat_retention_storage_heater",
+        "time_temperature_zone_control",  # controls count as a heating measure in your pipeline
+        "solar_pv"  # you treat PV as heating for funding logic
+    })
+
+
+def contributes_min_insulation(opt_types):
+    # MIR satisfiers you mentioned (extend as needed)
+    return any(x in opt_types for x in {
+        "external_wall_insulation",
+        "internal_wall_insulation",
+        "loft_insulation",
+        "cavity_wall_insulation",
+    })
+
+
+def run_optimizer(input_measures, budget=None, sub_target_gain=None, allow_slack=False):
+    """
+    Thin wrapper over your optimisers.
+    Returns: list[dict] selected_options
+    """
+    if budget is not None:
+        opt = GainOptimiser(
+            input_measures, max_cost=budget, max_gain=(sub_target_gain or float("inf")),
+            allow_slack=allow_slack
+        )
+    else:
+        if sub_target_gain is None:
+            raise ValueError("Either budget or target_gain must be provided.")
+        opt = CostOptimiser(input_measures, min_gain=sub_target_gain)
+
+    opt.setup()
+    opt.solve()
+    cost = sum([x["cost"] for x in opt.solution])
+    return opt.solution, cost, opt.solution_gain
+
+
+# ---- Define optimisation paths ----------------------------------------------------------
+
+def _find_measure(input_measures, measure_type):
+    for measures in input_measures:
+        for m in measures:
+            if measure_type in m["type"]:
+                return True
+    return False
+
+
+def _make_solar_heating_funding_paths(
+    p, input_measures, funding_paths, remaining_insulation_type, housing_type, funding: Funding
+):
+    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    # Solar PV with existing eligible heating system
+    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    has_eligible_heating_system = funding.check_solar_eligible_heating_system(
+        mainheat_description=p.main_heating["clean_description"].lower(),
+        heating_control_description=p.main_heating_controls["clean_description"].lower()
+    )
+
+    if has_eligible_heating_system and _find_measure(input_measures, "solar_pv"):
+        single_solar_template = [{"AND": ["solar_pv"], "reference": None}]
+        # We now look to pair this with any lingering insulation measures
+        solar_paths = []
+        for insulation_measure in remaining_insulation_type:
+            new_solar_path = deepcopy(single_solar_template)
+            new_solar_path[0]["AND"].append(insulation_measure)
+            # Make a specific reference for this path
+            new_solar_path[0]["reference"] = "solar_pv+" + insulation_measure + ":eco4"
+            solar_paths.append(new_solar_path)
+
+        if solar_paths:
+            funding_paths.extend(solar_paths)
+        else:
+            # If we have no insulation measures, we just add the solar PV path
+            funding_paths.append([{"AND": ["solar_pv"], "reference": "solar_pv:eco4"}])
+
+    # For each of these, because there is a heating measure begin implemented, we check for minimum insulation
+    # requirements.
+    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    # Solar PV + Heating Upgrade combos
+    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    # We don't include electric boilers as they are not eligible for ECO4 funding
+    solar_heating_combos = [
+        ("high_heat_retention_storage_heater", "solar_pv+hhrsh:eco4"),
+        ("air_source_heat_pump", "solar_pv+ashp:eco4"),
+    ]
+    if _find_measure(input_measures, "solar_pv"):
+        for heat_type, ref in solar_heating_combos:
+            if _find_measure(input_measures, heat_type):
+                if remaining_insulation_type:
+                    for insulation_measure in remaining_insulation_type:
+                        funding_paths.append(
+                            [{"AND": ["solar_pv", heat_type, insulation_measure],
+                              "reference": f"{ref[:-5]}+{insulation_measure}:eco4"}]  # keeps naming consistent
+                        )
+                else:
+                    funding_paths.append([{"AND": ["solar_pv", heat_type], "reference": ref}])
+
+    # We've actually covered all possible options where solar PV can be included in a funded package, so where
+    # solar PV is not in a reference, we can exclude it
+
+    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    # Heating Upgrades
+    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    # Must have an existing eligible heating system
+
+    # For private, HHRSH alone, or a boiler upgrade is NOT eligible for ECO4 funding. Boiler upgrade also doesn't
+    # count as an eligible heating system
+    if housing_type == "Private":
+        single_heating_measures = ["air_source_heat_pump"]
+    else:
+        single_heating_measures = [
+            "boiler_upgrade", "high_heat_retention_storage_heater", "air_source_heat_pump"
+        ]
+    measure_references = {
+        "boiler_upgrade": "boiler_upgrade",
+        "high_heat_retention_storage_heater": "hhrsh",
+        "air_source_heat_pump": "ashp"
+    }
+    for heating_upgrade in single_heating_measures:
+        if _find_measure(input_measures, heating_upgrade):
+            if remaining_insulation_type:
+                for insulation_measure in remaining_insulation_type:
+                    path = [
+                        {
+                            "AND": [heating_upgrade, insulation_measure],
+                            "reference": f"{measure_references[heating_upgrade]}+{insulation_measure}:eco4"
+                        }
+                    ]
+                    funding_paths.append(path)
+            else:
+                funding_paths.append(
+                    [{"AND": [heating_upgrade], "reference": f"{measure_references[heating_upgrade]}:eco4"}]
+                )
+
+    return funding_paths
+
+
+def _make_generic_gbis_funding_paths(input_gbis_measures, funding_paths):
+    """
+    For GBIS, the packages are single insulation measure.
+
+    We also have potential GBIS packages that allow heating controls as a secondary measure, however this
+    is not currently implemented in the optimiser due to not being certain about the heating controls pre conditions
+    :param input_gbis_measures:
+    :param funding_paths:
+    :return:
+    """
+
+    gbis_funding_paths = []
+    for input_measure in input_gbis_measures:
+        for measure in input_measure:
+            # We create a path for each measure
+            gbis_funding_paths.append([{"AND": [measure["type"]], "reference": measure["type"] + ":gbis"}])
+
+    return funding_paths + gbis_funding_paths
+
+
+def make_funding_paths(p, input_measures, housing_type, funding: Funding):
+    """
+    This function generates funding paths based on the input measures and the tenure of the property.
+    It checks for the presence of specific measures and creates paths that include necessary insulation measures
+    to meet minimum insulation requirements, particularly when a heating system is recommended.
+
+    Remaining measures that are not fixed as part of the package are then optimised
+    :param p: The property object containing details about the property, including main heating and controls.
+    :param input_measures:
+    :param housing_type:
+    :return:
+    """
+    # We handle the case of minimum insulation requirements. Whenever we have a heating system recommendation,
+    # we *must* include an additional insulation measure, unless the property already has sufficient insulation.
+
+    # We determine which insulation measures need to be included
+    wall_insulation_measures = [
+        "internal_wall_insulation", "external_wall_insulation", "cavity_wall_insulation",
+        "extension_cavity_wall_insulation"
+    ]
+    roof_insulation_measures = [
+        "loft_insulation", "flat_roof_insulation", "room_roof_insulation"
+    ]
+    other_gbis_insulation_measures = [
+        "suspended_floor_insulation", "solid_floor_insulation",
+    ]
+    # These are the insulation measures that the property still needs and so will be considered for
+    # filling the minimum insulation requirements
+    remaining_insulation_type = []
+    for insulation_measure in wall_insulation_measures + roof_insulation_measures:
+        if _find_measure(input_measures, insulation_measure):
+            remaining_insulation_type.append(insulation_measure)
+
+    remaining_insulation_type = list(set(remaining_insulation_type))
+
+    funding_paths = []
+
+    if housing_type == "Social" and p.data["current-energy-rating"] == "D":
+        # If the property is currently EPC D, we can only include innovation measures or measures to meet the
+        # minimum insulation requirements
+        input_measures_innovation = []
+        input_gbis_measures_innovation = []
+        for measures in input_measures:
+            group_of_innovation_measures = []
+            group_of_gbis_innovation_measures = []
+            for measure in measures:
+                if measure["innovation_uplift"] or measure["type"] in remaining_insulation_type:
+                    group_of_innovation_measures.append(measure)
+
+                if measure["innovation_uplift"] and measure["type"] in (
+                    remaining_insulation_type + other_gbis_insulation_measures
+                ):
+                    group_of_gbis_innovation_measures.append([measure])
+
+            if group_of_innovation_measures:
+                input_measures_innovation.append(group_of_innovation_measures)
+
+            if group_of_gbis_innovation_measures:
+                input_gbis_measures_innovation.extend(group_of_gbis_innovation_measures)
+
+        funding_paths = _make_solar_heating_funding_paths(
+            p, input_measures_innovation, funding_paths, remaining_insulation_type, housing_type, funding
+        )
+
+        # Can only be innovation GBIS measures
+        funding_paths = _make_generic_gbis_funding_paths(input_gbis_measures_innovation, funding_paths)
+        return funding_paths, input_measures_innovation
+
+    if housing_type == "Private":
+        # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+        # EWI or IWI
+        # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+        # 1) The package must include EWI or IWI if the property is private rental sector
+        # We check if we have any EWI or IWI measures available
+        ewi_or_iwi = [{"OR": []}]
+        reference_measures = []
+        # If we have EWI we add it in
+        if _find_measure(input_measures, "external_wall_insulation"):
+            ewi_or_iwi[0]["OR"].append("external_wall_insulation")
+            reference_measures.append("ewi")
+
+        if _find_measure(input_measures, "internal_wall_insulation"):
+            ewi_or_iwi[0]["OR"].append("internal_wall_insulation")
+            reference_measures.append("iwi")
+
+        if ewi_or_iwi[0]["OR"]:
+            ewi_or_iwi[0]["reference"] = "+".join(reference_measures) + ":eco4"
+            funding_paths.append(ewi_or_iwi)
+
+    funding_paths = _make_solar_heating_funding_paths(
+        p, input_measures, funding_paths, remaining_insulation_type, housing_type, funding
+    )
+
+    # If we have any remaining insulation measures, we add them to the funding paths
+    input_gbis_measures = []
+    for measures in input_measures:
+        for measure in measures:
+            type_to_check = measure["type"].split("+")[0] if "+" in measure["type"] else measure["type"]
+            if type_to_check in remaining_insulation_type + other_gbis_insulation_measures:
+                input_gbis_measures.append([measure])
+
+    funding_paths = _make_generic_gbis_funding_paths(input_gbis_measures, funding_paths)
+
+    return funding_paths, input_measures

recommendations/optimiser/optimiser_functions.py

@@ -6,7 +6,7 @@ from backend.app.utils import epc_to_sap_lower_bound
 from recommendations.optimiser.CostOptimiser import CostOptimiser
 
 
-def prepare_input_measures(property_recommendations, goal, needs_ventilation):
+def prepare_input_measures(property_recommendations, goal, needs_ventilation, funding=False):
     """
     Prepares a nested list of measure options for optimisation.
 
@@ -34,6 +34,9 @@ def prepare_input_measures(property_recommendations, goal, needs_ventilation):
         Optimisation goal, one of: "Increasing EPC", "Energy Savings", "Reducing CO2 emissions".
     needs_ventilation : bool
         Whether the property requires mechanical ventilation to accompany certain measures.
+    funding: bool, optional
+        If true, the function will include the innovation uplift in the total cost calculation. If false, this is
+        excluded, since innovation uplift cannot be claimed where funding is not available.
 
     Returns
     -------
@@ -75,24 +78,49 @@ def prepare_input_measures(property_recommendations, goal, needs_ventilation):
         # Build enriched measure data
         to_append = []
         for rec in recs:
-            total = (
-                rec["total"] + ventilation_recommendation["total"]
-                if rec["type"] in assumptions.measures_needing_ventilation and needs_ventilation
-                else rec["total"]
-            )
+
+            raw_cost = rec["total"]
+
+            if funding:
+                total = (
+                    rec["total"] - rec["innovation_uplift"] + ventilation_recommendation["total"]
+                    if rec["measure_type"] in assumptions.measures_needing_ventilation and needs_ventilation
+                    else rec["total"] - rec["innovation_uplift"]
+                )
+            else:
+                total = (
+                    rec["total"] + ventilation_recommendation["total"]
+                    if rec["measure_type"] in assumptions.measures_needing_ventilation and needs_ventilation
+                    else rec["total"]
+                )
+
+            # If the innovation uplift being removed make this negative, we keep the total so we can re-engineer
+            # the original cost
+            non_negative_total = 0 if total < 0 else total
             gain = (
                 rec[goal_key] + ventilation_recommendation[goal_key]
-                if rec["type"] in assumptions.measures_needing_ventilation and needs_ventilation
+                if rec["measure_type"] in assumptions.measures_needing_ventilation and needs_ventilation
                 else rec[goal_key]
             )
             rec_type = (
-                f"{rec['type']}+{ventilation_recommendation['type']}"
-                if rec["type"] in assumptions.measures_needing_ventilation and needs_ventilation
-                else rec["type"]
+                f"{rec['measure_type']}+{ventilation_recommendation['measure_type']}"
+                if rec["measure_type"] in assumptions.measures_needing_ventilation and needs_ventilation
+                else rec["measure_type"]
             )
 
+            # We also include the innovation uplift
             to_append.append(
-                {"id": rec["recommendation_id"], "cost": total, "gain": gain, "type": rec_type}
+                {
+                    "id": rec["recommendation_id"],
+                    "cost": non_negative_total,
+                    "gain": gain, "type": rec_type,
+                    "innovation_uplift": rec["innovation_uplift"] if funding else 0,
+                    "cost_minus_uplift": total,
+                    "raw_cost": raw_cost,
+                    "partial_project_funding": rec["partial_project_funding"],
+                    "partial_project_score": rec["partial_project_score"],
+                    "uplift_project_score": rec["uplift_project_score"],
+                }
             )
 
         input_measures.append(to_append)

recommendations/tests/test_heating_recommendations.py

@@ -51,6 +51,12 @@ class TestHeatingRecommendations:
         :return:
         """
 
+        # We patch an old version of cleaned which is missing some attributes for 'mainheat-description'
+        for x in cleaned['mainheat-description']:
+            x["has_hot-water-only"] = False
+            x["has_mineral_and_wood"] = False
+            x["has_dual_fuel_appliance"] = False
+
         epc_records = {"original_epc": test_case["epc"].copy(), "full_sap_epc": {}, "old_data": []}
 
         epc_record = EPCRecord(

recommendations/tests/test_optimiser_functions.py

@@ -12,7 +12,7 @@ class TestPrepareInputMeasures:
         recs = [
             [  # loft insulation measure
                 {"recommendation_id": "loft1", "type": "loft_insulation", "total": 100, "kwh_savings": 200,
-                 "energy_cost_savings": 10, "has_battery": False},
+                 "energy_cost_savings": 10, "has_battery": False, "measure_type": "loft_insulation"},
             ],
         ]
         measures = optimiser_functions.prepare_input_measures(recs, goal="Energy Savings", needs_ventilation=False)
@@ -27,9 +27,9 @@ class TestPrepareInputMeasures:
                             ["internal_wall_insulation"])
         recs = [
             [{"recommendation_id": "wall1", "type": "internal_wall_insulation", "total": 500, "kwh_savings": 300,
-              "energy_cost_savings": 5, "has_battery": False}],
+              "energy_cost_savings": 5, "has_battery": False, "measure_type": "internal_wall_insulation"}],
             [{"recommendation_id": "vent1", "type": "mechanical_ventilation", "total": 50, "kwh_savings": 30,
-              "energy_cost_savings": 5, "has_battery": False}]
+              "energy_cost_savings": 5, "has_battery": False, "measure_type": "mechanical_ventilation"}],
         ]
         measures = optimiser_functions.prepare_input_measures(recs, goal="Energy Savings", needs_ventilation=True)
         wall_option = measures[0][0]

recommendations/tests/test_optimisers.py

@@ -0,0 +1,665 @@
+import numpy as np
+# import pandas as pd
+from pandas import Timestamp
+from numpy import nan
+import datetime
+
+# import backend.app.assumptions as assumptions
+# import recommendations.optimiser.optimiser_functions as optimiser_functions
+#
+# from backend.Funding import Funding
+#
+# project_scores_matrix = pd.read_csv("/Users/khalimconn-kowlessar/Downloads/ECO4 Full Project Scores Matrix.csv")
+# partial_project_scores_matrix = pd.read_csv("backend/tests/test_data/ECO4_Partial_Project_Scores_Matrix_v6.csv")
+# partial_project_scores_matrix.columns = ['Measure category', 'Measure_Type', 'Pre_Main_Heating_Source',
+#                                          'Post_Main_Heating_Source', 'Total Floor Area Band', 'Starting Band',
+#                                          'Average Treatable Factor', 'Cost Savings', 'SAP Savings']
+# whlg_eligible_postcodes = pd.DataFrame([{"Postcode": "ab12cd"}])
+#
+# funding = Funding(
+#     project_scores_matrix=project_scores_matrix,
+#     partial_project_scores_matrix=partial_project_scores_matrix,
+#     whlg_eligible_postcodes=whlg_eligible_postcodes,
+#     eco4_social_cavity_abs_rate=13.5,
+#     eco4_social_solid_abs_rate=17,
+#     eco4_private_cavity_abs_rate=13.5,
+#     eco4_private_solid_abs_rate=17,
+#     gbis_social_cavity_abs_rate=21,
+#     gbis_social_solid_abs_rate=25,
+#     gbis_private_cavity_abs_rate=22,
+#     gbis_private_solid_abs_rate=28,
+#     tenure="Social"
+# )
+#
+# # Assume these costs have been adjusted
+
+
+#
+# # Insert the funding uplifts
+# for recs in property_recommendations:
+#     for r in recs:
+#         # Insert randomly
+#         # Select one of 0, 0.25 or 0.45
+#         r["uplift"] = np.random.choice([0, 0.25, 0.45])
+#
+# # We calculate the innovation uplift against each measure
+# for recs in property_recommendations:
+#     for r in recs:
+#         if r["type"] in ["mechanical_ventilation", "low_energy_lighting", "secondary_heating"]:
+#             r["innovation_uplift"] = 0
+#             continue
+#         r["innovation_uplift"] = funding.get_innovation_uplift(
+#             measure=r,
+#             starting_sap=p.data["current-energy-efficiency"],
+#             floor_area=p.floor_area,
+#             is_cavity=False,
+#             current_wall_uvalue=1.7,
+#             is_partial=False,
+#             existing_li_thickness=150,
+#             mainheating=p.main_heating,
+#             main_fuel=p.main_fuel,
+#             mainheat_energy_eff=p.data["mainheat-energy-eff"],
+#         )
+#         print(r["innovation_uplift"])
+#
+# property_measure_types = {rec["type"] for recs in property_recommendations for rec in recs}
+# property_required_measures = [m for m in property_recommendations if m[0]["type"] in []]
+# measures_to_optimise = [m for m in property_recommendations if m[0]["type"] not in []]
+#
+# # If a measure requiring ventilation is selected, and the property does not have ventilation, we enfore
+# # its inclusion
+# needs_ventilation = any(
+#     x in property_measure_types for x in assumptions.measures_needing_ventilation
+# ) and not p.has_ventilation
+#
+# input_measures = optimiser_functions.prepare_input_measures(
+#     measures_to_optimise, "Increasing EPC", needs_ventilation, True
+# )
+#
+# # ---- main wrapper around your optimiser ----------------------------------
+#
+# # Run inputs:
+# target_gain = 18.5
+#
+# # Run the optimiser with these inouts
+
+
+# tests/test_social_fabric_only.py
+import numpy as np
+import pandas as pd
+import pytest
+from copy import deepcopy
+
+from recommendations.optimiser import optimiser_functions
+from recommendations.optimiser.funding_optimiser import optimise_with_funding_paths  # wherever you defined it
+from backend.Funding import Funding
+from backend.app.plan.schemas import WALL_INSULATION_MEASURES, ROOF_INSULATION_MEASURES, ECO4_ELIGIBILE_FABRIC_MEASURES
+
+ALLOWED_FABRIC_TYPES = set(WALL_INSULATION_MEASURES + ROOF_INSULATION_MEASURES + ECO4_ELIGIBILE_FABRIC_MEASURES)
+
+
+@pytest.fixture
+def mock_project_scores_matrix():
+    data = []
+    floor_segments = ["0-72", "73-97", "98-199", "200"]
+    bands = [
+        "Low_G", "High_G", "Low_F", "High_F", "Low_E", "High_E", "Low_D", "High_D", "Low_C", "High_C", "Low_B",
+        "High_B", "Low_A", "High_A"
+    ]
+
+    cost = 50.0
+    for floor in floor_segments:
+        for start in bands:
+            for finish in bands:
+                if start != finish:  # skip identical start/finish (no SAP movement)
+                    data.append({
+                        "Floor Area Segment": floor,
+                        "Starting Band": start,
+                        "Finishing Band": finish,
+                        "Cost Savings": cost
+                    })
+                    cost += 5.0  # increment to create variety
+
+    return pd.DataFrame(data)
+
+
+@pytest.fixture
+def mock_partial_scores_matrix():
+    df = pd.read_csv("backend/tests/test_data/ECO4_Partial_Project_Scores_Matrix_v6.csv")
+    df.columns = ['Measure category', 'Measure_Type', 'Pre_Main_Heating_Source',
+                  'Post_Main_Heating_Source', 'Total Floor Area Band', 'Starting Band',
+                  'Average Treatable Factor', 'Cost Savings', 'SAP Savings']
+    return df
+
+
+class DummyProp:
+    """Minimal property stub exposing just what your code reads."""
+
+    def __init__(self):
+        self.data = {
+            "current-energy-rating": "E",  # or "D" for the special Social+D path
+            "current-energy-efficiency": 55,  # numeric SAP points used in eligibility calc
+            "mainheat-energy-eff": "Very Good",
+        }
+        self.has_ventilation = False
+        self.floor_area = 70.0
+        self.main_heating_controls = {"clean_description": "time and temperature zone control"}
+
+        self.main_heating = {
+            'original_description': 'Boiler and radiators, mains gas',
+            'clean_description': 'Boiler and radiators, mains gas',
+            'has_radiators': True, 'has_fan_coil_units': False, 'has_pipes_in_screed_above_insulation': False,
+            'has_pipes_in_insulated_timber_floor': False, 'has_pipes_in_concrete_slab': False, 'has_boiler': True,
+            'has_air_source_heat_pump': False, 'has_room_heaters': False, 'has_electric_storage_heaters': False,
+            'has_warm_air': False, 'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False,
+            'has_community_scheme': False, 'has_ground_source_heat_pump': False, 'has_no_system_present': False,
+            'has_portable_electric_heaters': False, 'has_water_source_heat_pump': False, 'has_electric_heat_pump':
+                False,
+            'has_micro-cogeneration': False, 'has_solar_assisted_heat_pump': False, 'has_exhaust_source_heat_pump':
+                False,
+            'has_community_heat_pump': False, 'has_hot-water-only': False, 'has_electric': False, 'has_mains_gas':
+                True,
+            'has_wood_logs': False, 'has_coal': False, 'has_oil': False, 'has_wood_pellets': False,
+            'has_anthracite': False,
+            'has_dual_fuel_mineral_and_wood': False, 'has_smokeless_fuel': False, 'has_lpg': False, 'has_b30k': False,
+            'has_mineral_and_wood': False, 'has_dual_fuel_appliance': False, 'has_assumed': False,
+            'has_electricaire': False,
+            'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False
+        }
+
+        self.main_fuel = {
+            'original_description': 'mains gas (not community)', 'clean_description': 'Mains gas not community',
+            'fuel_type': 'mains gas', 'tariff_type': None, 'is_community': False,
+            'no_individual_heating_or_community_network': False, 'complex_fuel_type': None
+        }
+
+
+@pytest.fixture
+def p():
+    return DummyProp()
+
+
+@pytest.fixture
+def funding(monkeypatch, mock_partial_scores_matrix, mock_project_scores_matrix):
+    """Simple Funding that returns zero uplift so costs stay as provided."""
+    # Build the Funding with tiny in-memory frames (avoid test I/O)
+
+    f = Funding(
+        project_scores_matrix=mock_project_scores_matrix,
+        partial_project_scores_matrix=mock_partial_scores_matrix,
+        whlg_eligible_postcodes=pd.DataFrame([{"Postcode": "ab12cd"}]),
+        eco4_social_cavity_abs_rate=13.5, eco4_social_solid_abs_rate=17,
+        eco4_private_cavity_abs_rate=13.5, eco4_private_solid_abs_rate=17,
+        gbis_social_cavity_abs_rate=21, gbis_social_solid_abs_rate=25,
+        gbis_private_cavity_abs_rate=22, gbis_private_solid_abs_rate=28,
+        tenure="Social"
+    )
+
+    # Keep innovation_uplift simple for the first test
+    # monkeypatch.setattr(f, "get_innovation_uplift", lambda *args, **kwargs: 0.0)
+
+    # If your solar precondition matters, you can force True/False here:
+    # monkeypatch.setattr(
+    #     __import__("backend").Funding, "check_solar_eligible_heating_system",
+    #     staticmethod(lambda mainheat_description, heating_control_description: False)
+    # )
+
+    return f
+
+
+@pytest.fixture
+def property_recommendations():
+    """Short sample; replace with your full block if you want."""
+    recs = [
+        [{'phase': 0, 'parts': [{'id': 2466, 'type': 'external_wall_insulation',
+                                 'description': 'EWI Pro EPS external wall insulation system with '
+                                                'Brick Slip finish',
+                                 'depth': 150.0, 'depth_unit': 'mm', 'cost': None,
+                                 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.02631579,
+                                 'r_value_unit': 'square_meter_kelvin_per_watt',
+                                 'thermal_conductivity': 0.038,
+                                 'thermal_conductivity_unit': 'watt_per_meter_kelvin',
+                                 'link': 'SCIS',
+                                 'created_at': Timestamp('2025-03-16 15:26:22.379496'),
+                                 'is_active': True, 'prime_material_cost': None,
+                                 'material_cost': 0.0, 'labour_cost': 0.0,
+                                 'labour_hours_per_unit': 0.0, 'plant_cost': 0.0,
+                                 'total_cost': 298.35,
+                                 'notes': 'This is the quoted value from SCIS',
+                                 'is_installer_quote': True, 'quantity': 63.98796761892035,
+                                 'quantity_unit': 'm2', 'total': 19090.810139104888,
+                                 'labour_hours': 0.0, 'labour_days': 0.0}],
+          'type': 'external_wall_insulation', 'measure_type': 'external_wall_insulation',
+          'description': 'Install 150mm EWI Pro EPS external wall insulation system with Brick '
+                         'Slip finish on external walls',
+          'starting_u_value': 1.7, 'new_u_value': 0.32, 'already_installed': False,
+          'sap_points': np.float64(9.6),
+          'simulation_config': {'is_as_built_ending': False, 'walls_is_assumed_ending': False,
+                                'walls_insulation_thickness_ending': 'average',
+                                'external_insulation_ending': True,
+                                'walls_energy_eff_ending': 'Good',
+                                'walls_thermal_transmittance_ending': 0.23},
+          'description_simulation': {'walls-description': 'Solid brick, with external insulation',
+                                     'walls-energy-eff': 'Good'}, 'total': 19090.810139104888,
+          'labour_hours': 0.0, 'labour_days': 0.0, 'survey': False,
+          'recommendation_id': '0_phase=0', 'efficiency': 11229.568317120522,
+          'co2_equivalent_savings': np.float64(0.5), 'heat_demand': np.float64(37.099999999999994),
+          'kwh_savings': np.float64(1827.8999999999996),
+          'energy_cost_savings': np.float64(136.1247882352941)}, {'phase': 0, 'parts': [
+            {'id': 2373, 'type': 'internal_wall_insulation', 'description': 'SWIP EcoBatt & Plastered finish',
+             'depth': 95.0,
+             'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.03125,
+             'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.032,
+             'thermal_conductivity_unit': None,
+             'link': 'SCIS', 'created_at': Timestamp('2025-03-16 15:26:22.379496'), 'is_active': True,
+             'prime_material_cost': None, 'material_cost': 0.0, 'labour_cost': 0.0, 'labour_hours_per_unit': 2.1,
+             'plant_cost': 0.0, 'total_cost': 89.0, 'notes': None, 'is_installer_quote': True,
+             'quantity': 63.98796761892035,
+             'quantity_unit': 'm2', 'total': 5694.929118083911, 'labour_hours': 134.37473199973275,
+             'labour_days': 4.199210374991648}], 'type': 'internal_wall_insulation',
+                                                                  'measure_type': 'internal_wall_insulation',
+                                                                  'description': 'Install 95mm '
+                                                                                 'SWIP EcoBatt & '
+                                                                                 'Plastered '
+                                                                                 'finish on '
+                                                                                 'internal walls',
+                                                                  'starting_u_value': 1.7,
+                                                                  'new_u_value': 0.32,
+                                                                  'already_installed': False,
+                                                                  'sap_points': 6,
+                                                                  'simulation_config': {
+                                                                      'is_as_built_ending': False,
+                                                                      'walls_is_assumed_ending':
+                                                                          False,
+                                                                      'walls_insulation_thickness_ending': 'average',
+                                                                      'internal_insulation_ending': True,
+                                                                      'walls_energy_eff_ending':
+                                                                          'Good',
+                                                                      'walls_thermal_transmittance_ending': 0.29},
+                                                                  'description_simulation': {
+                                                                      'walls-description': 'Solid '
+                                                                                           'brick, with internal '
+                                                                                           'insulation',
+                                                                      'walls-energy-eff': 'Good'},
+                                                                  'total': 5694.929118083911,
+                                                                  'labour_hours': 134.37473199973275,
+                                                                  'labour_days': 4.199210374991648,
+                                                                  'survey': True,
+                                                                  'recommendation_id': '1_phase=0',
+                                                                  'efficiency': 3349.6383047552417,
+                                                                  'co2_equivalent_savings': np.float64(
+                                                                      0.5),
+                                                                  'heat_demand': np.float64(
+                                                                      35.30000000000001),
+                                                                  'kwh_savings': np.float64(
+                                                                      1432.3999999999996),
+                                                                  'energy_cost_savings': np.float64(
+                                                                      106.67167058823532)}], [
+            {'phase': 1, 'parts': [{'id': 2351, 'type': 'loft_insulation',
+                                    'description': 'Knauf Loft Roll 44 glass fibre roll',
+                                    'depth': 300.0, 'depth_unit': 'mm', 'cost': None,
+                                    'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.022727273,
+                                    'r_value_unit': 'square_meter_kelvin_per_watt',
+                                    'thermal_conductivity': 0.044,
+                                    'thermal_conductivity_unit': 'watt_per_meter_kelvin',
+                                    'link': 'SCIS',
+                                    'created_at': Timestamp('2025-03-16 15:26:22.379496'),
+                                    'is_active': True, 'prime_material_cost': None,
+                                    'material_cost': 0.0, 'labour_cost': 0.0,
+                                    'labour_hours_per_unit': 0.11, 'plant_cost': 0.0,
+                                    'total_cost': 15.0,
+                                    'notes': 'This is the cost if there is less than 100mm '
+                                             'existing insulation',
+                                    'is_installer_quote': True, 'quantity': 63.98796761892035,
+                                    'quantity_unit': 'm2', 'total': 645.0, 'labour_hours': 8,
+                                    'labour_days': 1}], 'type': 'loft_insulation',
+             'measure_type': 'loft_insulation',
+             'description': 'Install 300mm of Knauf Loft Roll 44 glass fibre roll in your loft',
+             'starting_u_value': 2.3, 'new_u_value': 2.3, 'sap_points': np.float64(2.4),
+             'already_installed': False,
+             'simulation_config': {'is_loft_ending': True, 'roof_is_assumed_ending': False,
+                                   'roof_insulation_thickness_ending': '300',
+                                   'roof_thermal_transmittance_ending': 2.3,
+                                   'roof_energy_eff_ending': 'Very Good'},
+             'description_simulation': {'roof-description': 'Pitched, 300mm loft insulation',
+                                        'roof-energy-eff': 'Very Good'}, 'total': 645.0,
+             'labour_hours': 8, 'labour_days': 1, 'survey': False, 'recommendation_id': '2_phase=1',
+             'efficiency': 278.1347826086957,
+             'co2_equivalent_savings': np.float64(0.10000000000000009),
+             'heat_demand': np.float64(1.5), 'kwh_savings': np.float64(566.1499999999996),
+             'energy_cost_savings': np.float64(42.16152352941185)}], [{'phase': 2, 'parts': [
+            {'id': 2329, 'type': 'mechanical_ventilation', 'description': 'Mechanical Extract Ventilation',
+             'depth': 0.0,
+             'depth_unit': None, 'cost': None, 'cost_unit': 'gbp_per_unit', 'r_value_per_mm': nan,
+             'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None,
+             'thermal_conductivity_unit': None,
+             'link': 'SCIS', 'created_at': datetime.datetime(2025, 3, 16, 15, 26, 22, 379496), 'is_active': True,
+             'prime_material_cost': None, 'material_cost': 0.0, 'labour_cost': 0.0, 'labour_hours_per_unit': 0.0,
+             'plant_cost': 0.0, 'total_cost': 350.0, 'notes': None, 'is_installer_quote': True, 'total': 700.0,
+             'quantity': 2,
+             'quantity_unit': 'part'}], 'type': 'mechanical_ventilation', 'measure_type': 'mechanical_ventilation',
+                                                                       'description': 'Install 2 '
+                                                                                      'Mechanical '
+                                                                                      'Extract '
+                                                                                      'Ventilation units',
+                                                                       'starting_u_value': None,
+                                                                       'new_u_value': None,
+                                                                       'already_installed': False,
+                                                                       'sap_points': np.float64(
+                                                                           -0.10000000000000142),
+                                                                       'heat_demand': np.float64(
+                                                                           -3.3999999999999773),
+                                                                       'kwh_savings': np.float64(
+                                                                           -53.80000000000018),
+                                                                       'co2_equivalent_savings': np.float64(
+                                                                           0.0),
+                                                                       'energy_cost_savings': np.float64(
+                                                                           -4.0065176470588995),
+                                                                       'total': 700.0,
+                                                                       'labour_hours': 8,
+                                                                       'labour_days': 1.0,
+                                                                       'simulation_config': {
+                                                                           'mechanical_ventilation_ending':
+                                                                               'mechanical, '
+                                                                               'extract '
+                                                                               'only'},
+                                                                       'description_simulation': {
+                                                                           'mechanical-ventilation': 'mechanical, '
+                                                                                                     'extract only'},
+                                                                       'recommendation_id': '3_phase=2',
+                                                                       'efficiency': 0}], [
+            {'phase': 3, 'parts': [{'id': 2409, 'type': 'suspended_floor_insulation',
+                                    'description': 'Q-bot underfloor insulation', 'depth': 75.0,
+                                    'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2',
+                                    'r_value_per_mm': 0.045454547,
+                                    'r_value_unit': 'square_meter_kelvin_per_watt',
+                                    'thermal_conductivity': 0.022,
+                                    'thermal_conductivity_unit': 'watt_per_meter_kelvin',
+                                    'link': 'SCIS',
+                                    'created_at': Timestamp('2025-03-16 15:26:22.379496'),
+                                    'is_active': True, 'prime_material_cost': None,
+                                    'material_cost': 0.0, 'labour_cost': 0.0,
+                                    'labour_hours_per_unit': 1.63, 'plant_cost': 0.0,
+                                    'total_cost': 93.75,
+                                    'notes': 'Linearly interpolated based on Qbot costs',
+                                    'is_installer_quote': True, 'quantity': 43.0,
+                                    'quantity_unit': 'm2', 'total': 4031.25,
+                                    'labour_hours': 70.08999999999999,
+                                    'labour_days': 2.920416666666666}],
+             'type': 'suspended_floor_insulation', 'measure_type': 'suspended_floor_insulation',
+             'description': 'Install 75mm Q-bot underfloor insulation insulation in suspended '
+                            'floor',
+             'starting_u_value': 0.83, 'new_u_value': 0.22, 'sap_points': 2, 'survey': True,
+             'already_installed': False, 'simulation_config': {'floor_is_assumed_ending': False,
+                                                               'floor_insulation_thickness_ending': 'average',
+                                                               'floor_thermal_transmittance_ending': 0.685593},
+             'description_simulation': {'floor-description': 'Suspended, insulated'},
+             'total': 4031.25, 'labour_hours': 70.08999999999999, 'labour_days': 2.920416666666666,
+             'recommendation_id': '4_phase=3', 'efficiency': 4856.707710843373,
+             'co2_equivalent_savings': np.float64(0.20000000000000018),
+             'heat_demand': np.float64(33.5), 'kwh_savings': np.float64(1021.1999999999998),
+             'energy_cost_savings': np.float64(76.04936470588231)}], [
+            {'phase': 4, 'parts': [], 'type': 'low_energy_lighting',
+             'measure_type': 'low_energy_lighting',
+             'description': 'Install low energy lighting in -886 outlets', 'starting_u_value': None,
+             'new_u_value': None, 'already_installed': False, 'sap_points': 2,
+             'kwh_savings': -48508.5, 'energy_cost_savings': -12481.237049999998,
+             'co2_equivalent_savings': -7.858377,
+             'description_simulation': {'lighting-energy-eff': 'Very Good',
+                                        'lighting-description': 'Low energy lighting in all fixed'
+                                                                ' outlets',
+                                        'low-energy-lighting': 100}, 'total': -3411.1000000000004,
+             'labour_hours': 1, 'labour_days': 0.125, 'survey': True,
+             'recommendation_id': '5_phase=4', 'efficiency': -1705.5500000000002,
+             'heat_demand': np.float64(5.099999999999994)}], [
+            {'type': 'heating', 'phase': 5, 'measure_type': 'time_temperature_zone_control',
+             'parts': [],
+             'description': 'Upgrade heating controls to Smart Thermostats, room sensors and '
+                            'smart radiator valves (time & temperature zone control)',
+             'total': 739.576, 'subtotal': 700.48, 'vat': 39.096000000000004,
+             'labour_hours': 3.6199999999999997, 'labour_days': np.float64(1.0),
+             'starting_u_value': None, 'new_u_value': None, 'sap_points': np.float64(2.9),
+             'already_installed': False, 'simulation_config': {
+                'thermostatic_control_ending': 'time and temperature zone control',
+                'switch_system_ending': None, 'trvs_ending': None,
+                'mainheatc_energy_eff_ending': 'Very Good'}, 'description_simulation': {
+                'mainheatcont-description': 'Time and temperature zone control',
+                'mainheatc-energy-eff': 'Very Good'}, 'recommendation_id': '6_phase=5',
+             'efficiency': 739.576, 'co2_equivalent_savings': np.float64(0.30000000000000027),
+             'heat_demand': np.float64(6.599999999999994),
+             'kwh_savings': np.float64(876.8000000000002),
+             'energy_cost_savings': np.float64(65.29581176470589)}], [
+            {'phase': 6, 'parts': [], 'type': 'secondary_heating',
+             'measure_type': 'secondary_heating',
+             'description': 'Remove the secondary heating system', 'starting_u_value': None,
+             'new_u_value': None, 'sap_points': np.float64(3.6), 'already_installed': False,
+             'total': 30.0, 'subtotal': 25.0, 'vat': 5.0, 'labour_hours': 3.0,
+             'labour_days': np.float64(1.0),
+             'simulation_config': {'secondheat_description_ending': 'None'},
+             'description_simulation': {'secondheat-description': 'None'},
+             'recommendation_id': '7_phase=6', 'efficiency': 30.0,
+             'co2_equivalent_savings': np.float64(0.10000000000000009),
+             'heat_demand': np.float64(15.400000000000006),
+             'kwh_savings': np.float64(196.29999999999927),
+             'energy_cost_savings': np.float64(14.61857647058821)}], [
+            {'phase': 7, 'parts': [], 'type': 'solar_pv', 'measure_type': 'solar_pv',
+             'description': 'Install a 4.0 kilowatt-peak (kWp) solar panel system.',
+             'starting_u_value': None, 'new_u_value': None, 'sap_points': np.float64(13.0),
+             'already_installed': False, 'total': 6013.139999999999, 'subtotal': 5010.95, 'vat': 0,
+             'labour_hours': 48, 'labour_days': 2, 'photo_supply': np.float64(65.0),
+             'has_battery': False, 'initial_ac_kwh_per_year': np.float64(4081.7132614999996),
+             'description_simulation': {'photo-supply': np.float64(65.0)},
+             'recommendation_id': '8_phase=7', 'efficiency': np.float64(462.54923076923075),
+             'co2_equivalent_savings': np.float64(0.47347873833399995),
+             'heat_demand': np.float64(88.69999999999999),
+             'kwh_savings': np.float64(2040.8566307499998),
+             'energy_cost_savings': np.float64(525.1124110919749)},
+            {'phase': 7, 'parts': [], 'type': 'solar_pv', 'measure_type': 'solar_pv',
+             'description': 'Install a 4.0 kilowatt-peak (kWp) solar panel system, with a battery.',
+             'starting_u_value': None, 'new_u_value': None, 'sap_points': np.float64(13.0),
+             'already_installed': False, 'total': 10537.008, 'subtotal': 8780.84, 'vat': 0,
+             'labour_hours': 48, 'labour_days': 2, 'photo_supply': np.float64(65.0),
+             'has_battery': True, 'initial_ac_kwh_per_year': np.float64(4081.7132614999996),
+             'description_simulation': {'photo-supply': np.float64(65.0)},
+             'recommendation_id': '9_phase=7', 'efficiency': np.float64(810.5390769230769),
+             'co2_equivalent_savings': np.float64(0.6628702336675999),
+             'heat_demand': np.float64(88.69999999999999),
+             'kwh_savings': np.float64(2857.1992830499994),
+             'energy_cost_savings': np.float64(735.1573755287648)},
+            {'phase': 7, 'parts': [], 'type': 'solar_pv', 'measure_type': 'solar_pv',
+             'description': 'Install a 3.6 kilowatt-peak (kWp) solar panel system.',
+             'starting_u_value': None, 'new_u_value': None, 'sap_points': np.float64(12.0),
+             'already_installed': False, 'total': 5826.491999999999, 'subtotal': 4855.41, 'vat': 0,
+             'labour_hours': 48, 'labour_days': 2, 'photo_supply': np.float64(60.0),
+             'has_battery': False, 'initial_ac_kwh_per_year': np.float64(3692.66794),
+             'description_simulation': {'photo-supply': np.float64(60.0)},
+             'recommendation_id': '10_phase=7', 'efficiency': np.float64(485.54099999999994),
+             'co2_equivalent_savings': np.float64(0.42834948104),
+             'heat_demand': np.float64(83.69999999999999), 'kwh_savings': np.float64(1846.33397),
+             'energy_cost_savings': np.float64(475.0617304809999)},
+            {'phase': 7, 'parts': [], 'type': 'solar_pv', 'measure_type': 'solar_pv',
+             'description': 'Install a 3.6 kilowatt-peak (kWp) solar panel system, with a battery.',
+             'starting_u_value': None, 'new_u_value': None, 'sap_points': np.float64(12.0),
+             'already_installed': False, 'total': 10350.359999999999, 'subtotal': 8625.3, 'vat': 0,
+             'labour_hours': 48, 'labour_days': 2, 'photo_supply': np.float64(60.0),
+             'has_battery': True, 'initial_ac_kwh_per_year': np.float64(3692.66794),
+             'description_simulation': {'photo-supply': np.float64(60.0)},
+             'recommendation_id': '11_phase=7', 'efficiency': np.float64(862.5299999999999),
+             'co2_equivalent_savings': np.float64(0.599689273456),
+             'heat_demand': np.float64(83.69999999999999), 'kwh_savings': np.float64(2584.867558),
+             'energy_cost_savings': np.float64(665.0864226734)},
+            {'phase': 7, 'parts': [], 'type': 'solar_pv', 'measure_type': 'solar_pv',
+             'description': 'Install a 3.2 kilowatt-peak (kWp) solar panel system.',
+             'starting_u_value': None, 'new_u_value': None, 'sap_points': np.float64(11.0),
+             'already_installed': False, 'total': 5642.604, 'subtotal': 4702.17, 'vat': 0,
+             'labour_hours': 48, 'labour_days': 2, 'photo_supply': np.float64(55.0),
+             'has_battery': False, 'initial_ac_kwh_per_year': np.float64(3300.5416548),
+             'description_simulation': {'photo-supply': np.float64(55.0)},
+             'recommendation_id': '12_phase=7', 'efficiency': np.float64(512.964),
+             'co2_equivalent_savings': np.float64(0.3828628319568), 'heat_demand': np.float64(78.3),
+             'kwh_savings': np.float64(1650.2708274),
+             'energy_cost_savings': np.float64(424.61468389001993)},
+            {'phase': 7, 'parts': [], 'type': 'solar_pv', 'measure_type': 'solar_pv',
+             'description': 'Install a 3.2 kilowatt-peak (kWp) solar panel system, with a battery.',
+             'starting_u_value': None, 'new_u_value': None, 'sap_points': np.float64(11.0),
+             'already_installed': False, 'total': 10166.472, 'subtotal': 8472.06, 'vat': 0,
+             'labour_hours': 48, 'labour_days': 2, 'photo_supply': np.float64(55.0),
+             'has_battery': True, 'initial_ac_kwh_per_year': np.float64(3300.5416548),
+             'description_simulation': {'photo-supply': np.float64(55.0)},
+             'recommendation_id': '13_phase=7', 'efficiency': np.float64(924.2247272727273),
+             'co2_equivalent_savings': np.float64(0.53600796473952),
+             'heat_demand': np.float64(78.3), 'kwh_savings': np.float64(2310.3791583599996),
+             'energy_cost_savings': np.float64(594.4605574460278)},
+            {'phase': 7, 'parts': [], 'type': 'solar_pv', 'measure_type': 'solar_pv',
+             'description': 'Install a 2.8 kilowatt-peak (kWp) solar panel system.',
+             'starting_u_value': None, 'new_u_value': None, 'sap_points': np.float64(9.0),
+             'already_installed': False, 'total': 5458.727999999999, 'subtotal': 4548.94, 'vat': 0,
+             'labour_hours': 48, 'labour_days': 2, 'photo_supply': np.float64(45.0),
+             'has_battery': False, 'initial_ac_kwh_per_year': np.float64(2907.1867812),
+             'description_simulation': {'photo-supply': np.float64(45.0)},
+             'recommendation_id': '14_phase=7', 'efficiency': np.float64(606.5253333333333),
+             'co2_equivalent_savings': np.float64(0.3372336666192), 'heat_demand': np.float64(64.0),
+             'kwh_savings': np.float64(1453.5933906),
+             'energy_cost_savings': np.float64(374.00957940138)},
+            {'phase': 7, 'parts': [], 'type': 'solar_pv', 'measure_type': 'solar_pv',
+             'description': 'Install a 2.8 kilowatt-peak (kWp) solar panel system, with a battery.',
+             'starting_u_value': None, 'new_u_value': None, 'sap_points': np.float64(9.0),
+             'already_installed': False, 'total': 9982.596, 'subtotal': 8318.83, 'vat': 0,
+             'labour_hours': 48, 'labour_days': 2, 'photo_supply': np.float64(45.0),
+             'has_battery': True, 'initial_ac_kwh_per_year': np.float64(2907.1867812),
+             'description_simulation': {'photo-supply': np.float64(45.0)},
+             'recommendation_id': '15_phase=7', 'efficiency': np.float64(1109.1773333333333),
+             'co2_equivalent_savings': np.float64(0.47212713326688),
+             'heat_demand': np.float64(64.0), 'kwh_savings': np.float64(2035.03074684),
+             'energy_cost_savings': np.float64(523.6134111619319)},
+            {'phase': 7, 'parts': [], 'type': 'solar_pv', 'measure_type': 'solar_pv',
+             'description': 'Install a 2.4 kilowatt-peak (kWp) solar panel system.',
+             'starting_u_value': None, 'new_u_value': None, 'sap_points': np.float64(8.0),
+             'already_installed': False, 'total': 5274.852, 'subtotal': 4395.71, 'vat': 0,
+             'labour_hours': 48, 'labour_days': 2, 'photo_supply': np.float64(40.0),
+             'has_battery': False, 'initial_ac_kwh_per_year': np.float64(2510.25188),
+             'description_simulation': {'photo-supply': np.float64(40.0)},
+             'recommendation_id': '16_phase=7', 'efficiency': np.float64(659.3565),
+             'co2_equivalent_savings': np.float64(0.29118921808), 'heat_demand': np.float64(54.3),
+             'kwh_savings': np.float64(1255.12594),
+             'energy_cost_savings': np.float64(322.94390436199996)},
+            {'phase': 7, 'parts': [], 'type': 'solar_pv', 'measure_type': 'solar_pv',
+             'description': 'Install a 2.4 kilowatt-peak (kWp) solar panel system, with a battery.',
+             'starting_u_value': None, 'new_u_value': None, 'sap_points': np.float64(8.0),
+             'already_installed': False, 'total': 9798.72, 'subtotal': 8165.6, 'vat': 0,
+             'labour_hours': 48, 'labour_days': 2, 'photo_supply': np.float64(40.0),
+             'has_battery': True, 'initial_ac_kwh_per_year': np.float64(2510.25188),
+             'description_simulation': {'photo-supply': np.float64(40.0)},
+             'recommendation_id': '17_phase=7', 'efficiency': np.float64(1224.84),
+             'co2_equivalent_savings': np.float64(0.40766490531199995),
+             'heat_demand': np.float64(54.3), 'kwh_savings': np.float64(1757.1763159999998),
+             'energy_cost_savings': np.float64(452.1214661067999)},
+            {'phase': 7, 'parts': [], 'type': 'solar_pv', 'measure_type': 'solar_pv',
+             'description': 'Install a 2.0 kilowatt-peak (kWp) solar panel system.',
+             'starting_u_value': None, 'new_u_value': None, 'sap_points': np.float64(7.0),
+             'already_installed': False, 'total': 5090.976, 'subtotal': 4242.48, 'vat': 0,
+             'labour_hours': 48, 'labour_days': 2, 'photo_supply': np.float64(35.0),
+             'has_battery': False, 'initial_ac_kwh_per_year': np.float64(2096.682636),
+             'description_simulation': {'photo-supply': np.float64(35.0)},
+             'recommendation_id': '18_phase=7', 'efficiency': np.float64(727.2822857142856),
+             'co2_equivalent_savings': np.float64(0.243215185776), 'heat_demand': np.float64(48.5),
+             'kwh_savings': np.float64(1048.341318),
+             'energy_cost_savings': np.float64(269.7382211214)},
+            {'phase': 7, 'parts': [], 'type': 'solar_pv', 'measure_type': 'solar_pv',
+             'description': 'Install a 2.0 kilowatt-peak (kWp) solar panel system, with a battery.',
+             'starting_u_value': None, 'new_u_value': None, 'sap_points': np.float64(7.0),
+             'already_installed': False, 'total': 9614.844, 'subtotal': 8012.369999999999, 'vat': 0,
+             'labour_hours': 48, 'labour_days': 2, 'photo_supply': np.float64(35.0),
+             'has_battery': True, 'initial_ac_kwh_per_year': np.float64(2096.682636),
+             'description_simulation': {'photo-supply': np.float64(35.0)},
+             'recommendation_id': '19_phase=7', 'efficiency': np.float64(1373.5491428571427),
+             'co2_equivalent_savings': np.float64(0.3405012600864), 'heat_demand': np.float64(48.5),
+             'kwh_savings': np.float64(1467.6778451999999),
+             'energy_cost_savings': np.float64(377.6335095699599)}]
+    ]
+    return recs
+
+
+def _attach_costs_and_uplifts(recs, funding, p):
+    """Mimic what your script did: add cost fields & innovation uplift."""
+    out = deepcopy(recs)
+    for group in out:
+        for r in group:
+            if r["type"] in ["mechanical_ventilation", "low_energy_lighting", "secondary_heating"]:
+                r["innovation_uplift"] = 0
+                continue
+            r["uplift"] = 0.0  # fixed for determinism in test
+            r["innovation_uplift"] = funding.get_innovation_uplift(
+                measure=r,
+                starting_sap=55,
+                floor_area=70.0,
+                is_cavity=False,
+                current_wall_uvalue=1.7,
+                is_partial=False,
+                existing_li_thickness=150,
+                mainheating=p.main_heating,
+                main_fuel=p.main_fuel,
+                mainheat_energy_eff="Very Good",
+            )
+            # the optimiser_functions.prepare_input_measures will translate these to input format; but
+            # for safety add explicit cost fields some downstream code expects:
+            r["total"] = float(r["total"])
+    return out
+
+
+def _to_input_measures(recs, p):
+    """Use your own helper so we test the full pipeline."""
+    property_measure_types = {rec["type"] for grp in recs for rec in grp}
+    needs_ventilation = any(
+        x in property_measure_types for x in optimiser_functions.assumptions.measures_needing_ventilation
+    ) and not getattr(p, "has_ventilation", False)
+
+    # goal="Increasing EPC", add_uplift=True for Social path
+    return optimiser_functions.prepare_input_measures(
+        recs, goal="Increasing EPC", needs_ventilation=needs_ventilation, funding=True
+    )
+
+
+def _types_of(picked_items):
+    return {item["type"] for item in picked_items}
+
+
+def test_social_fabric_only_returns_only_fabric_types(p, funding, property_recommendations, monkeypatch):
+    # 1) prepare data like your script
+    recs = _attach_costs_and_uplifts(property_recommendations, funding, p)
+    input_measures = _to_input_measures(recs, p)
+
+    # 2) run optimiser wrapper (budget and target_gain can be modest for the test)
+    budget = 30000.0
+    target_gain = 8.0
+
+    solutions = optimise_with_funding_paths(
+        p=p,
+        input_measures=input_measures,
+        housing_type="Social",
+        budget=budget,
+        target_gain=target_gain,
+        funding=funding
+    )
+
+    # 3) basic shape assertions
+    assert isinstance(solutions, pd.DataFrame)
+    assert not solutions.empty
+
+    # 4) find the fabric-only ECO4 row
+    fabric_rows = solutions[
+        solutions["path"].apply(lambda x: isinstance(x, dict) and x.get("reference") == "fabric-only:eco4")]
+    assert not fabric_rows.empty, "Expected a fabric-only:eco4 solution for Social tenure"
+
+    # 5) ensure only fabric measure types are present in that solution
+    picked_types = _types_of(fabric_rows.iloc[0]["items"])
+    assert picked_types == {'internal_wall_insulation+mechanical_ventilation',
+                            'suspended_floor_insulation'}, "incorrect types selected"
+
+    # 6) respect budget
+    assert fabric_rows.iloc[0]["total_cost"] <= budget + 1e-9
+
+    # (optional) ensure unfunded baseline also appears
+    unfunded_rows = solutions[
+        solutions["path"].apply(lambda x: isinstance(x, dict) and x.get("reference") == "unfunded:all")]
+    assert not unfunded_rows.empty