diff --git a/.idea/Model.iml b/.idea/Model.iml
index c6561970..09f2e496 100644
--- a/.idea/Model.iml
+++ b/.idea/Model.iml
@@ -7,7 +7,7 @@
-
+
\ No newline at end of file
diff --git a/.idea/misc.xml b/.idea/misc.xml
index 50cad4ca..fb10c6b0 100644
--- a/.idea/misc.xml
+++ b/.idea/misc.xml
@@ -3,7 +3,7 @@
-
+
diff --git a/asset_list/AssetList.py b/asset_list/AssetList.py
index eca4ae1f..3c5627fc 100644
--- a/asset_list/AssetList.py
+++ b/asset_list/AssetList.py
@@ -303,7 +303,7 @@ class AssetList:
# Another version of non-intrusives:
NON_INTRUSIVES_NEW_FORMAT_COLNAMES_V2 = [
- 'Archetype', 'Archetype 2', 'Construction', 'Insulated', 'Material', 'Boroscoped?',
+ 'Archetype', 'Archetype 2', 'Construction', 'Insulated', 'Material', 'Borescoped?',
'CIGA Check Required', 'ROOF ORIENTATION', 'TILE HUNG', 'RENDERED',
'CLADDING', 'ACCESS ISSUES', 'FURTHER SURVEYOR NOTES', 'DATE',
'NAME OF SURVEYOR'
@@ -2119,6 +2119,7 @@ class AssetList:
RANGE_RE = re.compile(r'\b(\d+[A-Za-z]?)\s*[-–]\s*(\d+[A-Za-z]?)\b')
NUM_RE = re.compile(r'\b\d+[A-Za-z]?\b') # captures 12, 12A, etc.
TO_RANGE_RE = re.compile(r'\b(\d+[A-Za-z]?)\s+(?:to|To|TO)\s+(\d+[A-Za-z]?)\b') # captures "13 to 15"
+ LETTER_RANGE_RE = re.compile(r'\b(\d+)([A-Za-z]?)\s*[-–]\s*(\d+)([A-Za-z]?)\b') # captures "1A-3B"
expanded_rows = []
@@ -2175,7 +2176,7 @@ class AssetList:
# We update the full address
new[self.DOMNA_PROPERTY_ID] = f"{row[self.DOMNA_PROPERTY_ID]}-{new_addr}"
- expanded_rows.append(new)
+ expanded_rows.append(new.to_dict())
continue
# 2 ─ Explicit list (e.g. 1, 2, 5 Block) or split by an ampersand (e.g. 1 & 2 Block)
@@ -2186,12 +2187,36 @@ class AssetList:
new_addr = re.sub(NUM_RE, n, addr, count=1) # replace the first number only
new[self.STANDARD_ADDRESS_1] = new_addr
new[self.DOMNA_PROPERTY_ID] = f"{row[self.DOMNA_PROPERTY_ID]}-{new_addr}"
- expanded_rows.append(new)
+ expanded_rows.append(new.to_dict())
continue
- # 3 ─ Single number or no number, treat as individual dwelling
+ # Check for a range of lettered addresses e.g 31A - 31D
+ letter_range = LETTER_RANGE_RE.search(full_addr)
+ if letter_range:
+ start_num, start_letter, end_num, end_letter = letter_range.groups()
+ start_num, end_num = int(start_num), int(end_num)
+ if start_num != end_num:
+ raise NotImplementedError(f"Unusual range - handle me")
+
+ # We define the looping range on whether we have odd, even or all numbers
+ house_number_range = range(start_num, end_num + 1)
+ if has_odd:
+ house_number_range = [x for x in house_number_range if x % 2 != 0]
+ if has_even:
+ house_number_range = [x for x in house_number_range if x % 2 == 0]
+
+ for n in house_number_range:
+ for letter in range(ord(start_letter), ord(end_letter) + 1):
+ new = row.copy()
+ new_addr = f"{n}{chr(letter)}"
+ new[self.STANDARD_ADDRESS_1] = new_addr
+ new[self.DOMNA_PROPERTY_ID] = f"{row[self.DOMNA_PROPERTY_ID]}-{new_addr}"
+ expanded_rows.append(new.to_dict())
+ continue
+
+ # 4 ─ Single number or no number, treat as individual dwelling
if (len(nums) == 1) or not nums:
- expanded_rows.append(row)
+ expanded_rows.append(row.to_dict())
continue
# Anything else with digits is unrecognised
diff --git a/asset_list/app.py b/asset_list/app.py
index 763f245f..cf64a02d 100644
--- a/asset_list/app.py
+++ b/asset_list/app.py
@@ -58,23 +58,24 @@ def app():
EPC recommendations
Property UPRN
"""
- # Abri
- data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Abri/Post Inspections"
- data_filename = "Desktop ABRI data - Standardised After Programmes (2).xlsx"
- sheet_name = "Reviewed List"
+
+ # Colchester
+ data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Colchester/Aug2025 202 inspections"
+ data_filename = "Colchester Borough Homes - Inspections - Additional 202 Addresses JW 280725 copy.xlsx"
+ sheet_name = "Extra 202 Colchester Addresses"
postcode_column = 'domna_postcode'
address1_column = "domna_address_1"
address1_method = None
fulladdress_column = "domna_full_address"
address_cols_to_concat = []
missing_postcodes_method = None
- landlord_year_built = "landlord_year_built"
+ landlord_year_built = None
landlord_os_uprn = None
- landlord_property_type = "PropertyType_original_from_landlord"
- landlord_built_form = "BuildForm_original_from_landlord"
- landlord_wall_construction = "Wall Construction_original_from_landlord"
+ landlord_property_type = "landlord_property_type"
+ landlord_built_form = "landlord_built_form"
+ landlord_wall_construction = None
landlord_roof_construction = None
- landlord_heating_system = "HeatingType_original_from_landlord"
+ landlord_heating_system = None
landlord_existing_pv = None
landlord_property_id = "landlord_property_id"
landlord_sap = None
@@ -90,360 +91,394 @@ def app():
phase = False
ecosurv_landlords = None
asset_list_header = 0
- landlord_block_reference = None
+ landlord_block_reference = "landlord_block_reference"
+
+ # # Abri
+ # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Abri/Post Inspections"
+ # data_filename = "Desktop ABRI data - Standardised After Programmes (2).xlsx"
+ # sheet_name = "Reviewed List"
+ # postcode_column = 'domna_postcode'
+ # address1_column = "domna_address_1"
+ # address1_method = None
+ # fulladdress_column = "domna_full_address"
+ # address_cols_to_concat = []
+ # missing_postcodes_method = None
+ # landlord_year_built = "landlord_year_built"
+ # landlord_os_uprn = None
+ # landlord_property_type = "PropertyType_original_from_landlord"
+ # landlord_built_form = "BuildForm_original_from_landlord"
+ # landlord_wall_construction = "Wall Construction_original_from_landlord"
+ # landlord_roof_construction = None
+ # landlord_heating_system = "HeatingType_original_from_landlord"
+ # landlord_existing_pv = None
+ # landlord_property_id = "landlord_property_id"
+ # landlord_sap = None
+ # outcomes_filename = None
+ # outcomes_sheetname = None
+ # outcomes_postcode = None
+ # outcomes_houseno = None
+ # outcomes_id = None
+ # outcomes_address = None
+ # master_filepaths = []
+ # master_id_colnames = []
+ # master_to_asset_list_filepath = None
+ # phase = False
+ # ecosurv_landlords = None
+ # asset_list_header = 0
+ # landlord_block_reference = None
# Freebridge
- data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Freebridge"
- data_filename = "Domna - FCH property data May 25 copy.xlsx"
- sheet_name = "EPC Data"
- postcode_column = 'Post Code'
- address1_column = "Address 1"
- address1_method = None
- fulladdress_column = None
- address_cols_to_concat = ["Address 1", "Address 4"]
- missing_postcodes_method = None
- landlord_year_built = "Build Date"
- landlord_os_uprn = None
- landlord_property_type = "Property Type"
- landlord_built_form = None
- landlord_wall_construction = "Walls Description"
- landlord_heating_system = "Heating Type"
- landlord_existing_pv = None
- landlord_property_id = "Place Ref"
- landlord_roof_construction = "Roof Description"
- landlord_sap = "Current SAP"
- outcomes_filename = []
- outcomes_sheetname = []
- outcomes_postcode = []
- outcomes_houseno = []
- outcomes_address = []
- outcomes_id = []
- master_filepaths = []
- master_to_asset_list_filepath = None
- asset_list_header = 0
- landlord_block_reference = None
- master_id_colnames = []
- phase = True # Inspections not complete, produce a partial view
- ecosurv_landlords = None
+ # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Freebridge"
+ # data_filename = "Domna - FCH property data May 25 copy.xlsx"
+ # sheet_name = "EPC Data"
+ # postcode_column = 'Post Code'
+ # address1_column = "Address 1"
+ # address1_method = None
+ # fulladdress_column = None
+ # address_cols_to_concat = ["Address 1", "Address 4"]
+ # missing_postcodes_method = None
+ # landlord_year_built = "Build Date"
+ # landlord_os_uprn = None
+ # landlord_property_type = "Property Type"
+ # landlord_built_form = None
+ # landlord_wall_construction = "Walls Description"
+ # landlord_heating_system = "Heating Type"
+ # landlord_existing_pv = None
+ # landlord_property_id = "Place Ref"
+ # landlord_roof_construction = "Roof Description"
+ # landlord_sap = "Current SAP"
+ # outcomes_filename = []
+ # outcomes_sheetname = []
+ # outcomes_postcode = []
+ # outcomes_houseno = []
+ # outcomes_address = []
+ # outcomes_id = []
+ # master_filepaths = []
+ # master_to_asset_list_filepath = None
+ # asset_list_header = 0
+ # landlord_block_reference = None
+ # master_id_colnames = []
+ # phase = True # Inspections not complete, produce a partial view
+ # ecosurv_landlords = None
- data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Broadlands"
- data_filename = "Broadlands Asset List.xlsx"
- sheet_name = "Assets"
- postcode_column = 'POSTCODE'
- fulladdress_column = None
- address1_column = "Address1"
- address1_method = None
- address_cols_to_concat = ["Address1"]
- missing_postcodes_method = None
- landlord_year_built = "DATEBUILT"
- landlord_os_uprn = None
- landlord_property_type = "PropertyType"
- landlord_built_form = "PropertyType"
- landlord_wall_construction = None
- landlord_heating_system = "Heating Fuel"
- landlord_existing_pv = None
- landlord_property_id = "Row ID"
- outcomes_filename = [os.path.join(data_folder, "outcomes.xlsx")]
- outcomes_sheetname = ["Sheet1"]
- outcomes_postcode = ["Postcode"]
- outcomes_houseno = ["No."]
- outcomes_address = ["Address"]
- outcomes_id = [None]
- master_filepaths = [
- os.path.join(data_folder, "eco3 submissions.csv"),
- os.path.join(data_folder, "eco4 submissions.csv"),
- ]
- master_to_asset_list_filepath = None
- asset_list_header = 0
- landlord_block_reference = None
- master_id_colnames = [None, None]
- landlord_roof_construction = None
- phase = False
- landlord_sap = None
- ecosurv_landlords = "broadland"
+ # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Broadlands"
+ # data_filename = "Broadlands Asset List.xlsx"
+ # sheet_name = "Assets"
+ # postcode_column = 'POSTCODE'
+ # fulladdress_column = None
+ # address1_column = "Address1"
+ # address1_method = None
+ # address_cols_to_concat = ["Address1"]
+ # missing_postcodes_method = None
+ # landlord_year_built = "DATEBUILT"
+ # landlord_os_uprn = None
+ # landlord_property_type = "PropertyType"
+ # landlord_built_form = "PropertyType"
+ # landlord_wall_construction = None
+ # landlord_heating_system = "Heating Fuel"
+ # landlord_existing_pv = None
+ # landlord_property_id = "Row ID"
+ # outcomes_filename = [os.path.join(data_folder, "outcomes.xlsx")]
+ # outcomes_sheetname = ["Sheet1"]
+ # outcomes_postcode = ["Postcode"]
+ # outcomes_houseno = ["No."]
+ # outcomes_address = ["Address"]
+ # outcomes_id = [None]
+ # master_filepaths = [
+ # os.path.join(data_folder, "eco3 submissions.csv"),
+ # os.path.join(data_folder, "eco4 submissions.csv"),
+ # ]
+ # master_to_asset_list_filepath = None
+ # asset_list_header = 0
+ # landlord_block_reference = None
+ # master_id_colnames = [None, None]
+ # landlord_roof_construction = None
+ # phase = False
+ # landlord_sap = None
+ # ecosurv_landlords = "broadland"
+ # #
#
-
- # Community:
- data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Community Housing/New Programme"
- data_filename = "SUB EPC C to DOMNA - 24.07.25.xlsx"
- sheet_name = "Sheet1"
- postcode_column = 'POSTCODE'
- fulladdress_column = "ADDRESS"
- address1_column = None
- address1_method = "house_number_extraction"
- address_cols_to_concat = []
- missing_postcodes_method = None
- landlord_year_built = "BUILD DATE"
- landlord_os_uprn = None
- landlord_property_type = "PROPERTY TYPE"
- landlord_built_form = "Archetype" # Using the inspections archetype
- landlord_wall_construction = "CONSTRUCTION TYPE"
- landlord_roof_construction = None
- landlord_heating_system = None
- landlord_existing_pv = None
- landlord_property_id = "UPRN"
- landlord_sap = None
- outcomes_filename = []
- outcomes_sheetname = []
- outcomes_postcode = []
- outcomes_houseno = []
- outcomes_id = []
- outcomes_address = []
- master_filepaths = []
- master_to_asset_list_filepath = None
- phase = False
- ecosurv_landlords = None
- asset_list_header = 1
- landlord_block_reference = None
- master_id_colnames = []
-
- data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Ealing/Programme Analysis"
- data_filename = "EalingProjectRebuildJW210725.xlsx"
- sheet_name = "Refine & Houses"
- postcode_column = 'Postcode'
- fulladdress_column = "Address"
- address1_column = None
- address1_method = "house_number_extraction"
- address_cols_to_concat = []
- missing_postcodes_method = None
- landlord_year_built = None
- landlord_os_uprn = None
- landlord_property_type = None # Using the inspections property type
- landlord_built_form = None
- landlord_wall_construction = None
- landlord_roof_construction = None
- landlord_heating_system = None
- landlord_existing_pv = None
- landlord_property_id = "Property ref"
- landlord_sap = None
- outcomes_filename = []
- outcomes_sheetname = []
- outcomes_postcode = []
- outcomes_houseno = []
- outcomes_id = []
- outcomes_address = []
- master_filepaths = []
- master_to_asset_list_filepath = None
- phase = False
- ecosurv_landlords = None
- asset_list_header = 0
- landlord_block_reference = "Block Reference"
- master_id_colnames = []
-
- # TODO: Delete me
- data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/NRLA/"
- data_filename = "20250716 Asset List.xlsx"
- sheet_name = "Sheet 1"
- postcode_column = 'Postcode'
- fulladdress_column = "Full Address"
- address1_column = None
- address1_method = "house_number_extraction"
- address_cols_to_concat = []
- missing_postcodes_method = None
- landlord_year_built = None
- landlord_os_uprn = None
- landlord_property_type = None
- landlord_built_form = None
- landlord_wall_construction = None
- landlord_heating_system = None
- landlord_existing_pv = None
- landlord_property_id = "Row ID"
- outcomes_filename = []
- outcomes_sheetname = []
- outcomes_postcode = []
- outcomes_houseno = []
- outcomes_address = []
- outcomes_id = []
- master_filepaths = []
- master_to_asset_list_filepath = None
- asset_list_header = 0
- landlord_block_reference = None
- master_id_colnames = []
- landlord_roof_construction = None
- phase = False
- landlord_sap = None
- ecosurv_landlords = None
-
- # Southend
- data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Southend/July 2025 Programme"
- data_filename = "SOUTHEND - RYAN.xlsx"
- sheet_name = "July 2025 Surveys"
- postcode_column = 'Postcode'
- fulladdress_column = "Full postal address"
- address1_column = None
- address1_method = "house_number_extraction"
- address_cols_to_concat = []
- missing_postcodes_method = None
- landlord_year_built = "Property age"
- landlord_os_uprn = None
- landlord_property_type = "Property type"
- landlord_built_form = "Property type"
- landlord_wall_construction = None
- landlord_heating_system = None
- landlord_existing_pv = None
- landlord_property_id = "ID"
- outcomes_filename = []
- outcomes_sheetname = []
- outcomes_postcode = []
- outcomes_houseno = []
- outcomes_address = []
- outcomes_id = []
- master_filepaths = []
- master_to_asset_list_filepath = None
- asset_list_header = 0
- landlord_block_reference = None
- master_id_colnames = []
- landlord_roof_construction = None
- phase = False
- landlord_sap = None
- ecosurv_landlords = None
-
- # For Rooftop
- data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Rooftop"
- data_filename = "Rooftop Asset List - July 2025.xlsx"
- sheet_name = "Sheet1"
- postcode_column = 'post_code'
- fulladdress_column = None
- address1_column = "add_1"
- address1_method = None
- address_cols_to_concat = [
- "add_1", "add_2", "add_3", "add_4"
- ]
- missing_postcodes_method = None
- landlord_year_built = "date_built"
- landlord_os_uprn = None
- landlord_property_type = "ConstructionStyle"
- landlord_built_form = "ConstructionStyle"
- landlord_wall_construction = None
- landlord_heating_system = "Description"
- landlord_existing_pv = None
- landlord_property_id = "PropertyCode"
- outcomes_filename = [os.path.join(data_folder, "Rooftop_Outcomes.xlsx")]
- outcomes_sheetname = ["OUTCOMES"]
- outcomes_postcode = ["POSTCODE"]
- outcomes_houseno = ["NO"]
- outcomes_address = ["ADDRESS"]
- outcomes_id = [None]
- master_filepaths = [os.path.join(data_folder, "Master.csv")]
- master_to_asset_list_filepath = None
- asset_list_header = 1
- landlord_block_reference = "bl_rec_ref"
- master_id_colnames = [None]
- landlord_roof_construction = None
- phase = False
- landlord_sap = None
- ecosurv_landlords = "rooftop"
-
- # For Housing
- data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/For Housing/New Programme July 2025"
- data_filename = "FOR HOUSING Asset List (Combined).xlsx"
- sheet_name = "Asset List"
- postcode_column = 'Postcode'
- fulladdress_column = "Address"
- address1_column = None
- address1_method = "house_number_extraction"
- address_cols_to_concat = []
- missing_postcodes_method = None
- landlord_year_built = None
- landlord_os_uprn = None
- landlord_property_type = "Type"
- landlord_built_form = "Type"
- landlord_wall_construction = None
- landlord_heating_system = "Heating - full"
- landlord_existing_pv = None
- landlord_property_id = "UPRN"
- outcomes_filename = [os.path.join(data_folder, "Khalim Combined - for analysis.xlsx")]
- outcomes_sheetname = ["Sheet1"]
- outcomes_postcode = ["POSTCODE"]
- outcomes_houseno = ["NO"]
- outcomes_address = ["ADDRESS"]
- outcomes_id = [None]
- master_filepaths = [os.path.join(data_folder, "submissions.csv")]
- master_to_asset_list_filepath = None
- asset_list_header = 0
- landlord_block_reference = None
- master_id_colnames = [None]
- landlord_roof_construction = None
- phase = False
- landlord_sap = "SAP"
- ecosurv_landlords = "for housing"
-
- # CDS
- data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/CDS"
- data_filename = "Founder Estates - Asset List.xlsx"
- sheet_name = "Combined"
- postcode_column = 'Postcode'
- fulladdress_column = "Address"
- address1_column = None
- address1_method = "house_number_extraction"
- address_cols_to_concat = []
- missing_postcodes_method = None
- landlord_year_built = None
- landlord_os_uprn = None
- landlord_property_type = None
- landlord_built_form = None
- landlord_wall_construction = None
- landlord_heating_system = "Heating Type"
- landlord_existing_pv = None
- landlord_property_id = "Row ID"
- outcomes_filename = []
- outcomes_sheetname = []
- outcomes_postcode = []
- outcomes_houseno = []
- outcomes_address = []
- outcomes_id = []
- master_filepaths = [os.path.join(data_folder, "submissions.csv")]
- master_to_asset_list_filepath = None
- asset_list_header = 0
- landlord_block_reference = None
- master_id_colnames = [None]
- landlord_roof_construction = None
- phase = False
- landlord_sap = None
- ecosurv_landlords = "cds"
-
- # Plus Dane
- data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Plus Dane/New Programme July 2025/"
- data_filename = "20250711 Plus Dane Asset List.xlsx"
- sheet_name = "Sheet1"
- postcode_column = 'Postcode'
- fulladdress_column = "Address"
- address1_column = None
- address1_method = "house_number_extraction"
- address_cols_to_concat = []
- missing_postcodes_method = None
- landlord_year_built = "Property Age"
- landlord_os_uprn = None
- landlord_property_type = "Property Type"
- landlord_built_form = "Built Form"
- landlord_wall_construction = "Wall Construction"
- landlord_heating_system = "Full Heating System"
- landlord_existing_pv = None
- landlord_property_id = "UPRN"
- outcomes_filename = [
- os.path.join(data_folder, "Outcomes - Plus Dane_CWI_2024.xlsx"),
- os.path.join(data_folder, "Outcomes - Plus Dane_CWI_2025.xlsx"),
- os.path.join(data_folder, "Outcomes - Plus Dane_PV_2025.xlsx"),
- ]
- outcomes_sheetname = [
- "CWI & LI - 2024", "2025 - CWI", "PV - 2025",
- ]
- outcomes_postcode = ["Postcode", "Postcode", "Postcode"]
- outcomes_houseno = ["No.", "No", "No"]
- outcomes_address = ["Address", "Address", "Address"]
- outcomes_id = ["Asset Reference", "LL UPRN", "LL UPRN"]
- master_filepaths = [
- os.path.join(data_folder, "submissions/JJC-Table 1.csv"),
- os.path.join(data_folder, "submissions/SCIS-Table 1.csv")
- ]
- master_to_asset_list_filepath = None
- asset_list_header = 1
- landlord_block_reference = None
- master_id_colnames = [None, None]
- landlord_roof_construction = None
- phase = False
- landlord_sap = "SAP Rating"
- ecosurv_landlords = "plus dane"
+ # # Community:
+ # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Community Housing/New Programme"
+ # data_filename = "SUB EPC C to DOMNA - 24.07.25.xlsx"
+ # sheet_name = "Sheet1"
+ # postcode_column = 'POSTCODE'
+ # fulladdress_column = "ADDRESS"
+ # address1_column = None
+ # address1_method = "house_number_extraction"
+ # address_cols_to_concat = []
+ # missing_postcodes_method = None
+ # landlord_year_built = "BUILD DATE"
+ # landlord_os_uprn = None
+ # landlord_property_type = "PROPERTY TYPE"
+ # landlord_built_form = "Archetype" # Using the inspections archetype
+ # landlord_wall_construction = "CONSTRUCTION TYPE"
+ # landlord_roof_construction = None
+ # landlord_heating_system = None
+ # landlord_existing_pv = None
+ # landlord_property_id = "UPRN"
+ # landlord_sap = None
+ # outcomes_filename = []
+ # outcomes_sheetname = []
+ # outcomes_postcode = []
+ # outcomes_houseno = []
+ # outcomes_id = []
+ # outcomes_address = []
+ # master_filepaths = []
+ # master_to_asset_list_filepath = None
+ # phase = False
+ # ecosurv_landlords = None
+ # asset_list_header = 1
+ # landlord_block_reference = None
+ # master_id_colnames = []
+ #
+ # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Ealing/Programme Analysis"
+ # data_filename = "EalingProjectRebuildJW210725.xlsx"
+ # sheet_name = "Refine & Houses"
+ # postcode_column = 'Postcode'
+ # fulladdress_column = "Address"
+ # address1_column = None
+ # address1_method = "house_number_extraction"
+ # address_cols_to_concat = []
+ # missing_postcodes_method = None
+ # landlord_year_built = None
+ # landlord_os_uprn = None
+ # landlord_property_type = None # Using the inspections property type
+ # landlord_built_form = None
+ # landlord_wall_construction = None
+ # landlord_roof_construction = None
+ # landlord_heating_system = None
+ # landlord_existing_pv = None
+ # landlord_property_id = "Property ref"
+ # landlord_sap = None
+ # outcomes_filename = []
+ # outcomes_sheetname = []
+ # outcomes_postcode = []
+ # outcomes_houseno = []
+ # outcomes_id = []
+ # outcomes_address = []
+ # master_filepaths = []
+ # master_to_asset_list_filepath = None
+ # phase = False
+ # ecosurv_landlords = None
+ # asset_list_header = 0
+ # landlord_block_reference = "Block Reference"
+ # master_id_colnames = []
+ #
+ # # TODO: Delete me
+ # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/NRLA/"
+ # data_filename = "20250716 Asset List.xlsx"
+ # sheet_name = "Sheet 1"
+ # postcode_column = 'Postcode'
+ # fulladdress_column = "Full Address"
+ # address1_column = None
+ # address1_method = "house_number_extraction"
+ # address_cols_to_concat = []
+ # missing_postcodes_method = None
+ # landlord_year_built = None
+ # landlord_os_uprn = None
+ # landlord_property_type = None
+ # landlord_built_form = None
+ # landlord_wall_construction = None
+ # landlord_heating_system = None
+ # landlord_existing_pv = None
+ # landlord_property_id = "Row ID"
+ # outcomes_filename = []
+ # outcomes_sheetname = []
+ # outcomes_postcode = []
+ # outcomes_houseno = []
+ # outcomes_address = []
+ # outcomes_id = []
+ # master_filepaths = []
+ # master_to_asset_list_filepath = None
+ # asset_list_header = 0
+ # landlord_block_reference = None
+ # master_id_colnames = []
+ # landlord_roof_construction = None
+ # phase = False
+ # landlord_sap = None
+ # ecosurv_landlords = None
+ #
+ # # Southend
+ # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Southend/July 2025 Programme"
+ # data_filename = "SOUTHEND - RYAN.xlsx"
+ # sheet_name = "July 2025 Surveys"
+ # postcode_column = 'Postcode'
+ # fulladdress_column = "Full postal address"
+ # address1_column = None
+ # address1_method = "house_number_extraction"
+ # address_cols_to_concat = []
+ # missing_postcodes_method = None
+ # landlord_year_built = "Property age"
+ # landlord_os_uprn = None
+ # landlord_property_type = "Property type"
+ # landlord_built_form = "Property type"
+ # landlord_wall_construction = None
+ # landlord_heating_system = None
+ # landlord_existing_pv = None
+ # landlord_property_id = "ID"
+ # outcomes_filename = []
+ # outcomes_sheetname = []
+ # outcomes_postcode = []
+ # outcomes_houseno = []
+ # outcomes_address = []
+ # outcomes_id = []
+ # master_filepaths = []
+ # master_to_asset_list_filepath = None
+ # asset_list_header = 0
+ # landlord_block_reference = None
+ # master_id_colnames = []
+ # landlord_roof_construction = None
+ # phase = False
+ # landlord_sap = None
+ # ecosurv_landlords = None
+ #
+ # # For Rooftop
+ # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Rooftop"
+ # data_filename = "Rooftop Asset List - July 2025.xlsx"
+ # sheet_name = "Sheet1"
+ # postcode_column = 'post_code'
+ # fulladdress_column = None
+ # address1_column = "add_1"
+ # address1_method = None
+ # address_cols_to_concat = [
+ # "add_1", "add_2", "add_3", "add_4"
+ # ]
+ # missing_postcodes_method = None
+ # landlord_year_built = "date_built"
+ # landlord_os_uprn = None
+ # landlord_property_type = "ConstructionStyle"
+ # landlord_built_form = "ConstructionStyle"
+ # landlord_wall_construction = None
+ # landlord_heating_system = "Description"
+ # landlord_existing_pv = None
+ # landlord_property_id = "PropertyCode"
+ # outcomes_filename = [os.path.join(data_folder, "Rooftop_Outcomes.xlsx")]
+ # outcomes_sheetname = ["OUTCOMES"]
+ # outcomes_postcode = ["POSTCODE"]
+ # outcomes_houseno = ["NO"]
+ # outcomes_address = ["ADDRESS"]
+ # outcomes_id = [None]
+ # master_filepaths = [os.path.join(data_folder, "Master.csv")]
+ # master_to_asset_list_filepath = None
+ # asset_list_header = 1
+ # landlord_block_reference = "bl_rec_ref"
+ # master_id_colnames = [None]
+ # landlord_roof_construction = None
+ # phase = False
+ # landlord_sap = None
+ # ecosurv_landlords = "rooftop"
+ #
+ # # For Housing
+ # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/For Housing/New Programme July 2025"
+ # data_filename = "FOR HOUSING Asset List (Combined).xlsx"
+ # sheet_name = "Asset List"
+ # postcode_column = 'Postcode'
+ # fulladdress_column = "Address"
+ # address1_column = None
+ # address1_method = "house_number_extraction"
+ # address_cols_to_concat = []
+ # missing_postcodes_method = None
+ # landlord_year_built = None
+ # landlord_os_uprn = None
+ # landlord_property_type = "Type"
+ # landlord_built_form = "Type"
+ # landlord_wall_construction = None
+ # landlord_heating_system = "Heating - full"
+ # landlord_existing_pv = None
+ # landlord_property_id = "UPRN"
+ # outcomes_filename = [os.path.join(data_folder, "Khalim Combined - for analysis.xlsx")]
+ # outcomes_sheetname = ["Sheet1"]
+ # outcomes_postcode = ["POSTCODE"]
+ # outcomes_houseno = ["NO"]
+ # outcomes_address = ["ADDRESS"]
+ # outcomes_id = [None]
+ # master_filepaths = [os.path.join(data_folder, "submissions.csv")]
+ # master_to_asset_list_filepath = None
+ # asset_list_header = 0
+ # landlord_block_reference = None
+ # master_id_colnames = [None]
+ # landlord_roof_construction = None
+ # phase = False
+ # landlord_sap = "SAP"
+ # ecosurv_landlords = "for housing"
+ #
+ # # CDS
+ # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/CDS"
+ # data_filename = "Founder Estates - Asset List.xlsx"
+ # sheet_name = "Combined"
+ # postcode_column = 'Postcode'
+ # fulladdress_column = "Address"
+ # address1_column = None
+ # address1_method = "house_number_extraction"
+ # address_cols_to_concat = []
+ # missing_postcodes_method = None
+ # landlord_year_built = None
+ # landlord_os_uprn = None
+ # landlord_property_type = None
+ # landlord_built_form = None
+ # landlord_wall_construction = None
+ # landlord_heating_system = "Heating Type"
+ # landlord_existing_pv = None
+ # landlord_property_id = "Row ID"
+ # outcomes_filename = []
+ # outcomes_sheetname = []
+ # outcomes_postcode = []
+ # outcomes_houseno = []
+ # outcomes_address = []
+ # outcomes_id = []
+ # master_filepaths = [os.path.join(data_folder, "submissions.csv")]
+ # master_to_asset_list_filepath = None
+ # asset_list_header = 0
+ # landlord_block_reference = None
+ # master_id_colnames = [None]
+ # landlord_roof_construction = None
+ # phase = False
+ # landlord_sap = None
+ # ecosurv_landlords = "cds"
+ #
+ # # Plus Dane
+ # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Plus Dane/New Programme July 2025/"
+ # data_filename = "20250711 Plus Dane Asset List.xlsx"
+ # sheet_name = "Sheet1"
+ # postcode_column = 'Postcode'
+ # fulladdress_column = "Address"
+ # address1_column = None
+ # address1_method = "house_number_extraction"
+ # address_cols_to_concat = []
+ # missing_postcodes_method = None
+ # landlord_year_built = "Property Age"
+ # landlord_os_uprn = None
+ # landlord_property_type = "Property Type"
+ # landlord_built_form = "Built Form"
+ # landlord_wall_construction = "Wall Construction"
+ # landlord_heating_system = "Full Heating System"
+ # landlord_existing_pv = None
+ # landlord_property_id = "UPRN"
+ # outcomes_filename = [
+ # os.path.join(data_folder, "Outcomes - Plus Dane_CWI_2024.xlsx"),
+ # os.path.join(data_folder, "Outcomes - Plus Dane_CWI_2025.xlsx"),
+ # os.path.join(data_folder, "Outcomes - Plus Dane_PV_2025.xlsx"),
+ # ]
+ # outcomes_sheetname = [
+ # "CWI & LI - 2024", "2025 - CWI", "PV - 2025",
+ # ]
+ # outcomes_postcode = ["Postcode", "Postcode", "Postcode"]
+ # outcomes_houseno = ["No.", "No", "No"]
+ # outcomes_address = ["Address", "Address", "Address"]
+ # outcomes_id = ["Asset Reference", "LL UPRN", "LL UPRN"]
+ # master_filepaths = [
+ # os.path.join(data_folder, "submissions/JJC-Table 1.csv"),
+ # os.path.join(data_folder, "submissions/SCIS-Table 1.csv")
+ # ]
+ # master_to_asset_list_filepath = None
+ # asset_list_header = 1
+ # landlord_block_reference = None
+ # master_id_colnames = [None, None]
+ # landlord_roof_construction = None
+ # phase = False
+ # landlord_sap = "SAP Rating"
+ # ecosurv_landlords = "plus dane"
# data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Brentwood/July 2025 New Programme"
# data_filename = "20250710 Asset List Brentwood.xlsx"
diff --git a/asset_list/mappings/property_type.py b/asset_list/mappings/property_type.py
index c6539465..5c3a2b29 100644
--- a/asset_list/mappings/property_type.py
+++ b/asset_list/mappings/property_type.py
@@ -337,5 +337,9 @@ PROPERTY_MAPPING = {
'Maisonette - Mid Terrace': 'maisonette',
'Chalet - Wheelchair': 'other',
'Amenity Block - Detached': 'other',
- 'Amenity Block - Semi detached': 'other'
+ 'Amenity Block - Semi detached': 'other',
+ 'house': 'house',
+ 'block of flats': 'block of flats',
+ 'bungalow': 'bungalow',
+ 'flat': 'flat'
}
diff --git a/backend/Property.py b/backend/Property.py
index 82c60439..66ca84e3 100644
--- a/backend/Property.py
+++ b/backend/Property.py
@@ -531,7 +531,7 @@ class Property:
"internal_wall_insulation", "external_wall_insulation", "cavity_wall_insulation",
"cylinder_thermostat", "loft_insulation", "room_roof_insulation", "flat_roof_insulation",
"solid_floor_insulation", "suspended_floor_insulation", "mixed_glazing",
- "windows_glazing", "mechanical_ventilation"
+ "windows_glazing", "mechanical_ventilation", "solar_pv"
]:
# We update the data, as defined in the recommendaton
for prefix in ["walls", "roof", "floor"]:
@@ -547,9 +547,6 @@ class Property:
output.update(simulation_config)
- if recommendation["type"] == "solar_pv":
- output["photo_supply_ending"] = recommendation["photo_supply"]
-
if recommendation["type"] not in [
"sealing_open_fireplace", "low_energy_lighting",
"internal_wall_insulation", "external_wall_insulation", "cavity_wall_insulation",
diff --git a/backend/apis/GoogleSolarApi.py b/backend/apis/GoogleSolarApi.py
index 51e9c893..9d136b22 100644
--- a/backend/apis/GoogleSolarApi.py
+++ b/backend/apis/GoogleSolarApi.py
@@ -77,7 +77,6 @@ class GoogleSolarApi:
# property attributes:
self.floor_area = None
self.roof_area = None
- self.roof_segment_indexes = None
self.panel_area = assumptions.RDSAP_AREA_PER_PANEL
self.panel_wattage = None
self.panel_performance = None
@@ -89,6 +88,8 @@ class GoogleSolarApi:
self.double_property = False
self.solar_materials = solar_materials
+ self.allowed_segment_indices = None
+
def get_building_insights(self, longitude, latitude, required_quality="MEDIUM", max_retries=None):
"""
Make an API request to retrieve building insights based on the given longitude and latitude, with retry
@@ -203,8 +204,6 @@ class GoogleSolarApi:
# It should be straightforward, but I'd rather see an actual instance of this happening
raise NotImplementedError("Panel wattage is not 400W - implement me")
- self.roof_segment_indexes = [segment['segmentIndex'] for segment in self.roof_segments]
-
# We now start finding the solar panel configurations
self.optimise_solar_configuration(
energy_consumption=energy_consumption,
@@ -294,10 +293,10 @@ class GoogleSolarApi:
# Remove any north facing roof segments
panel_performance = []
for config in self.insights_data["solarPotential"].get("solarPanelConfigs", []):
- roof_segment_summaries = config["roofSegmentSummaries"]
- # Filter on just the segments in self.roof_segment_indexes
+
roof_segment_summaries = [
- segment for segment in roof_segment_summaries if segment["segmentIndex"] in self.roof_segment_indexes
+ s for s in config.get("roofSegmentSummaries", [])
+ if s["segmentIndex"] in self.allowed_segment_indices
]
roi_summary = []
@@ -544,24 +543,28 @@ class GoogleSolarApi:
def exclude_north_facing_segments(self, property_instance):
"""
- Filter out any north-facing roof segments from the roof_segments attribute.
-
- North-facing segments are defined as those with an azimuth between -30 and 30 degrees.
+ Filter out any north-facing roof segments from self.roof_segments.
+ Keep API's original segmentIndex; optionally add a localIndex.
"""
+ is_flat = property_instance.roof["is_flat"]
filtered_segments = []
- for segment_index, segment in enumerate(self.roof_segments):
- segment["segmentIndex"] = segment_index
- # Check if the segment is north-facing
- if (
- self.NORTH_FACING_AZIMUTH_RANGE[0] <= segment['azimuthDegrees'] <= self.NORTH_FACING_AZIMUTH_RANGE[1]
- ) and not property_instance.roof["is_flat"]:
- continue
+ for i, seg in enumerate(self.roof_segments):
+ # DO NOT overwrite seg["segmentIndex"]
+ keep = True
+ if not is_flat:
+ if self.NORTH_FACING_AZIMUTH_RANGE[0] <= seg['azimuthDegrees'] <= self.NORTH_FACING_AZIMUTH_RANGE[1]:
+ keep = False
- filtered_segments.append(segment)
+ if keep:
+ seg = dict(**seg) # shallow copy
+ seg["localIndex"] = i # optional local index as a reference from this loop
+ filtered_segments.append(seg)
self.roof_segments = filtered_segments
+ self.allowed_segment_indices = {s["segmentIndex"] for s in self.roof_segments}
+
@staticmethod
def haversine(lat1, lon1, lat2, lon2):
"""
@@ -742,7 +745,8 @@ class GoogleSolarApi:
@classmethod
def building_solar_analysis(
- cls, building_solar_config: List, input_properties: List[Property], session, google_solar_api_key: str
+ cls, building_solar_config: List, input_properties: List[Property], session, google_solar_api_key: str,
+ solar_materials: list
):
"""
Perform the solar analysis for the building level
@@ -750,6 +754,7 @@ class GoogleSolarApi:
:param input_properties: List of properties
:param session: Database session
:param google_solar_api_key: Google Solar API key
+ :param solar_materials: List of solar materials
:return:
"""
@@ -788,7 +793,7 @@ class GoogleSolarApi:
energy_consumption = sum(
[entry['energy_consumption'] for entry in building_solar_config if entry['building_id'] == building_id]
)
- solar_api_client = cls(api_key=google_solar_api_key)
+ solar_api_client = cls(api_key=google_solar_api_key, solar_materials=solar_materials)
solar_api_client.get(
longitude=coordinates["longitude"],
latitude=coordinates["latitude"],
diff --git a/backend/engine/engine.py b/backend/engine/engine.py
index 14e5d85a..026b0405 100644
--- a/backend/engine/engine.py
+++ b/backend/engine/engine.py
@@ -415,13 +415,25 @@ def get_funding_data():
project_scores_matrix.columns = ['Floor Area Segment', 'Starting Band', 'Finishing Band', 'Cost Savings']
project_scores_matrix["Cost Savings"] = project_scores_matrix["Cost Savings"].astype(float)
+ partial_project_scores_matrix = read_csv_from_s3(
+ bucket_name=get_settings().DATA_BUCKET,
+ filepath="funding/ECO4_Partial_Project_Scores_Matrix_v6.csv",
+ )
+ partial_project_scores_matrix = pd.DataFrame(partial_project_scores_matrix)
+ partial_project_scores_matrix.columns = [
+ 'Measure category', 'Measure_Type', 'Pre_Main_Heating_Source',
+ 'Post_Main_Heating_Source', 'Total Floor Area Band', 'Starting Band',
+ 'Average Treatable Factor', 'Cost Savings', 'SAP Savings'
+ ]
+ partial_project_scores_matrix["Cost Savings"] = partial_project_scores_matrix["Cost Savings"].astype(float)
+
whlg_eligible_postcodes = read_csv_from_s3(
bucket_name=get_settings().DATA_BUCKET,
filepath="funding/whlg eligible postcodes.csv",
)
whlg_eligible_postcodes = pd.DataFrame(whlg_eligible_postcodes)
- return project_scores_matrix, whlg_eligible_postcodes
+ return project_scores_matrix, partial_project_scores_matrix, whlg_eligible_postcodes
async def model_engine(body: PlanTriggerRequest):
@@ -648,7 +660,7 @@ async def model_engine(body: PlanTriggerRequest):
logger.info("Reading in materials and cleaned datasets")
materials = get_materials(session)
cleaned = get_cleaned()
- eco_project_scores_matrix, whlg_eligible_postcodes = get_funding_data()
+ project_scores_matrix, partial_project_scores_matrix, whlg_eligible_postcodes = get_funding_data()
kwh_client = KwhData(bucket=get_settings().DATA_BUCKET, read_consumption_data=True)
@@ -690,6 +702,7 @@ async def model_engine(body: PlanTriggerRequest):
input_properties=input_properties,
session=session,
google_solar_api_key=get_settings().GOOGLE_SOLAR_API_KEY,
+ solar_materials=[m for m in materials if m["type"] == "solar_pv"]
)
input_properties = GoogleSolarApi.unit_solar_analysis(
@@ -819,21 +832,24 @@ async def model_engine(body: PlanTriggerRequest):
)
continue
- # We layer funding on top of the recommendations
- # We take one of these options
- funding_paths = [
- [["internal_wall_insulation", "external_wall_insulation"]],
- ["air_source_heat_pump"],
- # We must have both of these options (though we check if the property doesn't already have HHRSH and
- # is recommended it
- [["solar_pv"], ["high_heat_retention_storage_heaters"]]
- ]
-
fixed_gain = optimiser_functions.calculate_fixed_gain(
property_required_measures, recommendations, p, needs_ventilation
)
gain = optimiser_functions.calculate_gain(body=body, p=p, fixed_gain=fixed_gain)
+ from backend.Funding import Funding
+ funding = Funding(
+ project_scores_matrix=project_scores_matrix,
+ partial_project_scores_matrix=partial_project_scores_matrix,
+ whlg_eligible_postcodes=whlg_eligible_postcodes,
+ eco4_social_cavity_abs_rate=13,
+ eco4_social_solid_abs_rate=17,
+ gbis_social_cavity_abs_rate=21,
+ gbis_social_solid_abs_rate=25,
+ gbis_private_cavity_abs_rate=21,
+ gbis_private_solid_abs_rate=28,
+ )
+
if not body.optimise:
if body.goal != "Increasing EPC":
raise NotImplementedError("Only EPC optimisation is currently supported")
diff --git a/recommendations/SolarPvRecommendations.py b/recommendations/SolarPvRecommendations.py
index 3f32da25..1a9b47ac 100644
--- a/recommendations/SolarPvRecommendations.py
+++ b/recommendations/SolarPvRecommendations.py
@@ -143,6 +143,9 @@ class SolarPvRecommendations:
# back up here
"photo_supply": roof_coverage_percent,
"has_battery": False,
+ "simulation_config": {
+ "photo_supply_ending": roof_coverage_percent
+ },
"initial_ac_kwh_per_year": initial_ac_kwh_per_year,
"description_simulation": {"photo-supply": roof_coverage_percent},
"rank": rank, # Rank is used to get the representative recommendation - rank 0 will be chosen
@@ -282,7 +285,10 @@ class SolarPvRecommendations:
"sap_points": minimum_sap_points,
"already_installed": already_installed,
**cost_result,
- "has_battery": has_battery,
+ "has_battery": solar_pv_product["includes_battery"],
+ "simulation_config": {
+ "photo_supply_ending": roof_coverage_percent
+ },
"initial_ac_kwh_per_year": recommendation_config["initial_ac_kwh_per_year"],
"description_simulation": {"photo-supply": roof_coverage_percent},
}