implementing funding optimiser and fixing some bugs in sal

This commit is contained in:
Khalim Conn-Kowlessar 2025-08-19 11:27:17 +01:00
parent 8e5388b1ea
commit 7b1b1e0c11
9 changed files with 491 additions and 403 deletions

2
.idea/Model.iml generated
View file

@ -7,7 +7,7 @@
<sourceFolder url="file://$MODULE_DIR$/open_uprn" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/recommendations" isTestSource="false" />
</content>
<orderEntry type="jdk" jdkName="Fastapi-backend" jdkType="Python SDK" />
<orderEntry type="jdk" jdkName="AssetList" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>

2
.idea/misc.xml generated
View file

@ -3,7 +3,7 @@
<component name="Black">
<option name="sdkName" value="Python 3.10 (backend)" />
</component>
<component name="ProjectRootManager" version="2" project-jdk-name="Fastapi-backend" project-jdk-type="Python SDK" />
<component name="ProjectRootManager" version="2" project-jdk-name="AssetList" project-jdk-type="Python SDK" />
<component name="PyCharmProfessionalAdvertiser">
<option name="shown" value="true" />
</component>

View file

@ -303,7 +303,7 @@ class AssetList:
# Another version of non-intrusives:
NON_INTRUSIVES_NEW_FORMAT_COLNAMES_V2 = [
'Archetype', 'Archetype 2', 'Construction', 'Insulated', 'Material', 'Boroscoped?',
'Archetype', 'Archetype 2', 'Construction', 'Insulated', 'Material', 'Borescoped?',
'CIGA Check Required', 'ROOF ORIENTATION', 'TILE HUNG', 'RENDERED',
'CLADDING', 'ACCESS ISSUES', 'FURTHER SURVEYOR NOTES', 'DATE',
'NAME OF SURVEYOR'
@ -2119,6 +2119,7 @@ class AssetList:
RANGE_RE = re.compile(r'\b(\d+[A-Za-z]?)\s*[-]\s*(\d+[A-Za-z]?)\b')
NUM_RE = re.compile(r'\b\d+[A-Za-z]?\b') # captures 12, 12A, etc.
TO_RANGE_RE = re.compile(r'\b(\d+[A-Za-z]?)\s+(?:to|To|TO)\s+(\d+[A-Za-z]?)\b') # captures "13 to 15"
LETTER_RANGE_RE = re.compile(r'\b(\d+)([A-Za-z]?)\s*[-]\s*(\d+)([A-Za-z]?)\b') # captures "1A-3B"
expanded_rows = []
@ -2175,7 +2176,7 @@ class AssetList:
# We update the full address
new[self.DOMNA_PROPERTY_ID] = f"{row[self.DOMNA_PROPERTY_ID]}-{new_addr}"
expanded_rows.append(new)
expanded_rows.append(new.to_dict())
continue
# 2 ─ Explicit list (e.g. 1, 2, 5 Block) or split by an ampersand (e.g. 1 & 2 Block)
@ -2186,12 +2187,36 @@ class AssetList:
new_addr = re.sub(NUM_RE, n, addr, count=1) # replace the first number only
new[self.STANDARD_ADDRESS_1] = new_addr
new[self.DOMNA_PROPERTY_ID] = f"{row[self.DOMNA_PROPERTY_ID]}-{new_addr}"
expanded_rows.append(new)
expanded_rows.append(new.to_dict())
continue
# 3 ─ Single number or no number, treat as individual dwelling
# Check for a range of lettered addresses e.g 31A - 31D
letter_range = LETTER_RANGE_RE.search(full_addr)
if letter_range:
start_num, start_letter, end_num, end_letter = letter_range.groups()
start_num, end_num = int(start_num), int(end_num)
if start_num != end_num:
raise NotImplementedError(f"Unusual range - handle me")
# We define the looping range on whether we have odd, even or all numbers
house_number_range = range(start_num, end_num + 1)
if has_odd:
house_number_range = [x for x in house_number_range if x % 2 != 0]
if has_even:
house_number_range = [x for x in house_number_range if x % 2 == 0]
for n in house_number_range:
for letter in range(ord(start_letter), ord(end_letter) + 1):
new = row.copy()
new_addr = f"{n}{chr(letter)}"
new[self.STANDARD_ADDRESS_1] = new_addr
new[self.DOMNA_PROPERTY_ID] = f"{row[self.DOMNA_PROPERTY_ID]}-{new_addr}"
expanded_rows.append(new.to_dict())
continue
# 4 ─ Single number or no number, treat as individual dwelling
if (len(nums) == 1) or not nums:
expanded_rows.append(row)
expanded_rows.append(row.to_dict())
continue
# Anything else with digits is unrecognised

View file

@ -58,23 +58,24 @@ def app():
EPC recommendations
Property UPRN
"""
# Abri
data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Abri/Post Inspections"
data_filename = "Desktop ABRI data - Standardised After Programmes (2).xlsx"
sheet_name = "Reviewed List"
# Colchester
data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Colchester/Aug2025 202 inspections"
data_filename = "Colchester Borough Homes - Inspections - Additional 202 Addresses JW 280725 copy.xlsx"
sheet_name = "Extra 202 Colchester Addresses"
postcode_column = 'domna_postcode'
address1_column = "domna_address_1"
address1_method = None
fulladdress_column = "domna_full_address"
address_cols_to_concat = []
missing_postcodes_method = None
landlord_year_built = "landlord_year_built"
landlord_year_built = None
landlord_os_uprn = None
landlord_property_type = "PropertyType_original_from_landlord"
landlord_built_form = "BuildForm_original_from_landlord"
landlord_wall_construction = "Wall Construction_original_from_landlord"
landlord_property_type = "landlord_property_type"
landlord_built_form = "landlord_built_form"
landlord_wall_construction = None
landlord_roof_construction = None
landlord_heating_system = "HeatingType_original_from_landlord"
landlord_heating_system = None
landlord_existing_pv = None
landlord_property_id = "landlord_property_id"
landlord_sap = None
@ -90,360 +91,394 @@ def app():
phase = False
ecosurv_landlords = None
asset_list_header = 0
landlord_block_reference = None
landlord_block_reference = "landlord_block_reference"
# # Abri
# data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Abri/Post Inspections"
# data_filename = "Desktop ABRI data - Standardised After Programmes (2).xlsx"
# sheet_name = "Reviewed List"
# postcode_column = 'domna_postcode'
# address1_column = "domna_address_1"
# address1_method = None
# fulladdress_column = "domna_full_address"
# address_cols_to_concat = []
# missing_postcodes_method = None
# landlord_year_built = "landlord_year_built"
# landlord_os_uprn = None
# landlord_property_type = "PropertyType_original_from_landlord"
# landlord_built_form = "BuildForm_original_from_landlord"
# landlord_wall_construction = "Wall Construction_original_from_landlord"
# landlord_roof_construction = None
# landlord_heating_system = "HeatingType_original_from_landlord"
# landlord_existing_pv = None
# landlord_property_id = "landlord_property_id"
# landlord_sap = None
# outcomes_filename = None
# outcomes_sheetname = None
# outcomes_postcode = None
# outcomes_houseno = None
# outcomes_id = None
# outcomes_address = None
# master_filepaths = []
# master_id_colnames = []
# master_to_asset_list_filepath = None
# phase = False
# ecosurv_landlords = None
# asset_list_header = 0
# landlord_block_reference = None
# Freebridge
data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Freebridge"
data_filename = "Domna - FCH property data May 25 copy.xlsx"
sheet_name = "EPC Data"
postcode_column = 'Post Code'
address1_column = "Address 1"
address1_method = None
fulladdress_column = None
address_cols_to_concat = ["Address 1", "Address 4"]
missing_postcodes_method = None
landlord_year_built = "Build Date"
landlord_os_uprn = None
landlord_property_type = "Property Type"
landlord_built_form = None
landlord_wall_construction = "Walls Description"
landlord_heating_system = "Heating Type"
landlord_existing_pv = None
landlord_property_id = "Place Ref"
landlord_roof_construction = "Roof Description"
landlord_sap = "Current SAP"
outcomes_filename = []
outcomes_sheetname = []
outcomes_postcode = []
outcomes_houseno = []
outcomes_address = []
outcomes_id = []
master_filepaths = []
master_to_asset_list_filepath = None
asset_list_header = 0
landlord_block_reference = None
master_id_colnames = []
phase = True # Inspections not complete, produce a partial view
ecosurv_landlords = None
# data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Freebridge"
# data_filename = "Domna - FCH property data May 25 copy.xlsx"
# sheet_name = "EPC Data"
# postcode_column = 'Post Code'
# address1_column = "Address 1"
# address1_method = None
# fulladdress_column = None
# address_cols_to_concat = ["Address 1", "Address 4"]
# missing_postcodes_method = None
# landlord_year_built = "Build Date"
# landlord_os_uprn = None
# landlord_property_type = "Property Type"
# landlord_built_form = None
# landlord_wall_construction = "Walls Description"
# landlord_heating_system = "Heating Type"
# landlord_existing_pv = None
# landlord_property_id = "Place Ref"
# landlord_roof_construction = "Roof Description"
# landlord_sap = "Current SAP"
# outcomes_filename = []
# outcomes_sheetname = []
# outcomes_postcode = []
# outcomes_houseno = []
# outcomes_address = []
# outcomes_id = []
# master_filepaths = []
# master_to_asset_list_filepath = None
# asset_list_header = 0
# landlord_block_reference = None
# master_id_colnames = []
# phase = True # Inspections not complete, produce a partial view
# ecosurv_landlords = None
data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Broadlands"
data_filename = "Broadlands Asset List.xlsx"
sheet_name = "Assets"
postcode_column = 'POSTCODE'
fulladdress_column = None
address1_column = "Address1"
address1_method = None
address_cols_to_concat = ["Address1"]
missing_postcodes_method = None
landlord_year_built = "DATEBUILT"
landlord_os_uprn = None
landlord_property_type = "PropertyType"
landlord_built_form = "PropertyType"
landlord_wall_construction = None
landlord_heating_system = "Heating Fuel"
landlord_existing_pv = None
landlord_property_id = "Row ID"
outcomes_filename = [os.path.join(data_folder, "outcomes.xlsx")]
outcomes_sheetname = ["Sheet1"]
outcomes_postcode = ["Postcode"]
outcomes_houseno = ["No."]
outcomes_address = ["Address"]
outcomes_id = [None]
master_filepaths = [
os.path.join(data_folder, "eco3 submissions.csv"),
os.path.join(data_folder, "eco4 submissions.csv"),
]
master_to_asset_list_filepath = None
asset_list_header = 0
landlord_block_reference = None
master_id_colnames = [None, None]
landlord_roof_construction = None
phase = False
landlord_sap = None
ecosurv_landlords = "broadland"
# data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Broadlands"
# data_filename = "Broadlands Asset List.xlsx"
# sheet_name = "Assets"
# postcode_column = 'POSTCODE'
# fulladdress_column = None
# address1_column = "Address1"
# address1_method = None
# address_cols_to_concat = ["Address1"]
# missing_postcodes_method = None
# landlord_year_built = "DATEBUILT"
# landlord_os_uprn = None
# landlord_property_type = "PropertyType"
# landlord_built_form = "PropertyType"
# landlord_wall_construction = None
# landlord_heating_system = "Heating Fuel"
# landlord_existing_pv = None
# landlord_property_id = "Row ID"
# outcomes_filename = [os.path.join(data_folder, "outcomes.xlsx")]
# outcomes_sheetname = ["Sheet1"]
# outcomes_postcode = ["Postcode"]
# outcomes_houseno = ["No."]
# outcomes_address = ["Address"]
# outcomes_id = [None]
# master_filepaths = [
# os.path.join(data_folder, "eco3 submissions.csv"),
# os.path.join(data_folder, "eco4 submissions.csv"),
# ]
# master_to_asset_list_filepath = None
# asset_list_header = 0
# landlord_block_reference = None
# master_id_colnames = [None, None]
# landlord_roof_construction = None
# phase = False
# landlord_sap = None
# ecosurv_landlords = "broadland"
# #
#
# Community:
data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Community Housing/New Programme"
data_filename = "SUB EPC C to DOMNA - 24.07.25.xlsx"
sheet_name = "Sheet1"
postcode_column = 'POSTCODE'
fulladdress_column = "ADDRESS"
address1_column = None
address1_method = "house_number_extraction"
address_cols_to_concat = []
missing_postcodes_method = None
landlord_year_built = "BUILD DATE"
landlord_os_uprn = None
landlord_property_type = "PROPERTY TYPE"
landlord_built_form = "Archetype" # Using the inspections archetype
landlord_wall_construction = "CONSTRUCTION TYPE"
landlord_roof_construction = None
landlord_heating_system = None
landlord_existing_pv = None
landlord_property_id = "UPRN"
landlord_sap = None
outcomes_filename = []
outcomes_sheetname = []
outcomes_postcode = []
outcomes_houseno = []
outcomes_id = []
outcomes_address = []
master_filepaths = []
master_to_asset_list_filepath = None
phase = False
ecosurv_landlords = None
asset_list_header = 1
landlord_block_reference = None
master_id_colnames = []
data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Ealing/Programme Analysis"
data_filename = "EalingProjectRebuildJW210725.xlsx"
sheet_name = "Refine & Houses"
postcode_column = 'Postcode'
fulladdress_column = "Address"
address1_column = None
address1_method = "house_number_extraction"
address_cols_to_concat = []
missing_postcodes_method = None
landlord_year_built = None
landlord_os_uprn = None
landlord_property_type = None # Using the inspections property type
landlord_built_form = None
landlord_wall_construction = None
landlord_roof_construction = None
landlord_heating_system = None
landlord_existing_pv = None
landlord_property_id = "Property ref"
landlord_sap = None
outcomes_filename = []
outcomes_sheetname = []
outcomes_postcode = []
outcomes_houseno = []
outcomes_id = []
outcomes_address = []
master_filepaths = []
master_to_asset_list_filepath = None
phase = False
ecosurv_landlords = None
asset_list_header = 0
landlord_block_reference = "Block Reference"
master_id_colnames = []
# TODO: Delete me
data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/NRLA/"
data_filename = "20250716 Asset List.xlsx"
sheet_name = "Sheet 1"
postcode_column = 'Postcode'
fulladdress_column = "Full Address"
address1_column = None
address1_method = "house_number_extraction"
address_cols_to_concat = []
missing_postcodes_method = None
landlord_year_built = None
landlord_os_uprn = None
landlord_property_type = None
landlord_built_form = None
landlord_wall_construction = None
landlord_heating_system = None
landlord_existing_pv = None
landlord_property_id = "Row ID"
outcomes_filename = []
outcomes_sheetname = []
outcomes_postcode = []
outcomes_houseno = []
outcomes_address = []
outcomes_id = []
master_filepaths = []
master_to_asset_list_filepath = None
asset_list_header = 0
landlord_block_reference = None
master_id_colnames = []
landlord_roof_construction = None
phase = False
landlord_sap = None
ecosurv_landlords = None
# Southend
data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Southend/July 2025 Programme"
data_filename = "SOUTHEND - RYAN.xlsx"
sheet_name = "July 2025 Surveys"
postcode_column = 'Postcode'
fulladdress_column = "Full postal address"
address1_column = None
address1_method = "house_number_extraction"
address_cols_to_concat = []
missing_postcodes_method = None
landlord_year_built = "Property age"
landlord_os_uprn = None
landlord_property_type = "Property type"
landlord_built_form = "Property type"
landlord_wall_construction = None
landlord_heating_system = None
landlord_existing_pv = None
landlord_property_id = "ID"
outcomes_filename = []
outcomes_sheetname = []
outcomes_postcode = []
outcomes_houseno = []
outcomes_address = []
outcomes_id = []
master_filepaths = []
master_to_asset_list_filepath = None
asset_list_header = 0
landlord_block_reference = None
master_id_colnames = []
landlord_roof_construction = None
phase = False
landlord_sap = None
ecosurv_landlords = None
# For Rooftop
data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Rooftop"
data_filename = "Rooftop Asset List - July 2025.xlsx"
sheet_name = "Sheet1"
postcode_column = 'post_code'
fulladdress_column = None
address1_column = "add_1"
address1_method = None
address_cols_to_concat = [
"add_1", "add_2", "add_3", "add_4"
]
missing_postcodes_method = None
landlord_year_built = "date_built"
landlord_os_uprn = None
landlord_property_type = "ConstructionStyle"
landlord_built_form = "ConstructionStyle"
landlord_wall_construction = None
landlord_heating_system = "Description"
landlord_existing_pv = None
landlord_property_id = "PropertyCode"
outcomes_filename = [os.path.join(data_folder, "Rooftop_Outcomes.xlsx")]
outcomes_sheetname = ["OUTCOMES"]
outcomes_postcode = ["POSTCODE"]
outcomes_houseno = ["NO"]
outcomes_address = ["ADDRESS"]
outcomes_id = [None]
master_filepaths = [os.path.join(data_folder, "Master.csv")]
master_to_asset_list_filepath = None
asset_list_header = 1
landlord_block_reference = "bl_rec_ref"
master_id_colnames = [None]
landlord_roof_construction = None
phase = False
landlord_sap = None
ecosurv_landlords = "rooftop"
# For Housing
data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/For Housing/New Programme July 2025"
data_filename = "FOR HOUSING Asset List (Combined).xlsx"
sheet_name = "Asset List"
postcode_column = 'Postcode'
fulladdress_column = "Address"
address1_column = None
address1_method = "house_number_extraction"
address_cols_to_concat = []
missing_postcodes_method = None
landlord_year_built = None
landlord_os_uprn = None
landlord_property_type = "Type"
landlord_built_form = "Type"
landlord_wall_construction = None
landlord_heating_system = "Heating - full"
landlord_existing_pv = None
landlord_property_id = "UPRN"
outcomes_filename = [os.path.join(data_folder, "Khalim Combined - for analysis.xlsx")]
outcomes_sheetname = ["Sheet1"]
outcomes_postcode = ["POSTCODE"]
outcomes_houseno = ["NO"]
outcomes_address = ["ADDRESS"]
outcomes_id = [None]
master_filepaths = [os.path.join(data_folder, "submissions.csv")]
master_to_asset_list_filepath = None
asset_list_header = 0
landlord_block_reference = None
master_id_colnames = [None]
landlord_roof_construction = None
phase = False
landlord_sap = "SAP"
ecosurv_landlords = "for housing"
# CDS
data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/CDS"
data_filename = "Founder Estates - Asset List.xlsx"
sheet_name = "Combined"
postcode_column = 'Postcode'
fulladdress_column = "Address"
address1_column = None
address1_method = "house_number_extraction"
address_cols_to_concat = []
missing_postcodes_method = None
landlord_year_built = None
landlord_os_uprn = None
landlord_property_type = None
landlord_built_form = None
landlord_wall_construction = None
landlord_heating_system = "Heating Type"
landlord_existing_pv = None
landlord_property_id = "Row ID"
outcomes_filename = []
outcomes_sheetname = []
outcomes_postcode = []
outcomes_houseno = []
outcomes_address = []
outcomes_id = []
master_filepaths = [os.path.join(data_folder, "submissions.csv")]
master_to_asset_list_filepath = None
asset_list_header = 0
landlord_block_reference = None
master_id_colnames = [None]
landlord_roof_construction = None
phase = False
landlord_sap = None
ecosurv_landlords = "cds"
# Plus Dane
data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Plus Dane/New Programme July 2025/"
data_filename = "20250711 Plus Dane Asset List.xlsx"
sheet_name = "Sheet1"
postcode_column = 'Postcode'
fulladdress_column = "Address"
address1_column = None
address1_method = "house_number_extraction"
address_cols_to_concat = []
missing_postcodes_method = None
landlord_year_built = "Property Age"
landlord_os_uprn = None
landlord_property_type = "Property Type"
landlord_built_form = "Built Form"
landlord_wall_construction = "Wall Construction"
landlord_heating_system = "Full Heating System"
landlord_existing_pv = None
landlord_property_id = "UPRN"
outcomes_filename = [
os.path.join(data_folder, "Outcomes - Plus Dane_CWI_2024.xlsx"),
os.path.join(data_folder, "Outcomes - Plus Dane_CWI_2025.xlsx"),
os.path.join(data_folder, "Outcomes - Plus Dane_PV_2025.xlsx"),
]
outcomes_sheetname = [
"CWI & LI - 2024", "2025 - CWI", "PV - 2025",
]
outcomes_postcode = ["Postcode", "Postcode", "Postcode"]
outcomes_houseno = ["No.", "No", "No"]
outcomes_address = ["Address", "Address", "Address"]
outcomes_id = ["Asset Reference", "LL UPRN", "LL UPRN"]
master_filepaths = [
os.path.join(data_folder, "submissions/JJC-Table 1.csv"),
os.path.join(data_folder, "submissions/SCIS-Table 1.csv")
]
master_to_asset_list_filepath = None
asset_list_header = 1
landlord_block_reference = None
master_id_colnames = [None, None]
landlord_roof_construction = None
phase = False
landlord_sap = "SAP Rating"
ecosurv_landlords = "plus dane"
# # Community:
# data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Community Housing/New Programme"
# data_filename = "SUB EPC C to DOMNA - 24.07.25.xlsx"
# sheet_name = "Sheet1"
# postcode_column = 'POSTCODE'
# fulladdress_column = "ADDRESS"
# address1_column = None
# address1_method = "house_number_extraction"
# address_cols_to_concat = []
# missing_postcodes_method = None
# landlord_year_built = "BUILD DATE"
# landlord_os_uprn = None
# landlord_property_type = "PROPERTY TYPE"
# landlord_built_form = "Archetype" # Using the inspections archetype
# landlord_wall_construction = "CONSTRUCTION TYPE"
# landlord_roof_construction = None
# landlord_heating_system = None
# landlord_existing_pv = None
# landlord_property_id = "UPRN"
# landlord_sap = None
# outcomes_filename = []
# outcomes_sheetname = []
# outcomes_postcode = []
# outcomes_houseno = []
# outcomes_id = []
# outcomes_address = []
# master_filepaths = []
# master_to_asset_list_filepath = None
# phase = False
# ecosurv_landlords = None
# asset_list_header = 1
# landlord_block_reference = None
# master_id_colnames = []
#
# data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Ealing/Programme Analysis"
# data_filename = "EalingProjectRebuildJW210725.xlsx"
# sheet_name = "Refine & Houses"
# postcode_column = 'Postcode'
# fulladdress_column = "Address"
# address1_column = None
# address1_method = "house_number_extraction"
# address_cols_to_concat = []
# missing_postcodes_method = None
# landlord_year_built = None
# landlord_os_uprn = None
# landlord_property_type = None # Using the inspections property type
# landlord_built_form = None
# landlord_wall_construction = None
# landlord_roof_construction = None
# landlord_heating_system = None
# landlord_existing_pv = None
# landlord_property_id = "Property ref"
# landlord_sap = None
# outcomes_filename = []
# outcomes_sheetname = []
# outcomes_postcode = []
# outcomes_houseno = []
# outcomes_id = []
# outcomes_address = []
# master_filepaths = []
# master_to_asset_list_filepath = None
# phase = False
# ecosurv_landlords = None
# asset_list_header = 0
# landlord_block_reference = "Block Reference"
# master_id_colnames = []
#
# # TODO: Delete me
# data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/NRLA/"
# data_filename = "20250716 Asset List.xlsx"
# sheet_name = "Sheet 1"
# postcode_column = 'Postcode'
# fulladdress_column = "Full Address"
# address1_column = None
# address1_method = "house_number_extraction"
# address_cols_to_concat = []
# missing_postcodes_method = None
# landlord_year_built = None
# landlord_os_uprn = None
# landlord_property_type = None
# landlord_built_form = None
# landlord_wall_construction = None
# landlord_heating_system = None
# landlord_existing_pv = None
# landlord_property_id = "Row ID"
# outcomes_filename = []
# outcomes_sheetname = []
# outcomes_postcode = []
# outcomes_houseno = []
# outcomes_address = []
# outcomes_id = []
# master_filepaths = []
# master_to_asset_list_filepath = None
# asset_list_header = 0
# landlord_block_reference = None
# master_id_colnames = []
# landlord_roof_construction = None
# phase = False
# landlord_sap = None
# ecosurv_landlords = None
#
# # Southend
# data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Southend/July 2025 Programme"
# data_filename = "SOUTHEND - RYAN.xlsx"
# sheet_name = "July 2025 Surveys"
# postcode_column = 'Postcode'
# fulladdress_column = "Full postal address"
# address1_column = None
# address1_method = "house_number_extraction"
# address_cols_to_concat = []
# missing_postcodes_method = None
# landlord_year_built = "Property age"
# landlord_os_uprn = None
# landlord_property_type = "Property type"
# landlord_built_form = "Property type"
# landlord_wall_construction = None
# landlord_heating_system = None
# landlord_existing_pv = None
# landlord_property_id = "ID"
# outcomes_filename = []
# outcomes_sheetname = []
# outcomes_postcode = []
# outcomes_houseno = []
# outcomes_address = []
# outcomes_id = []
# master_filepaths = []
# master_to_asset_list_filepath = None
# asset_list_header = 0
# landlord_block_reference = None
# master_id_colnames = []
# landlord_roof_construction = None
# phase = False
# landlord_sap = None
# ecosurv_landlords = None
#
# # For Rooftop
# data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Rooftop"
# data_filename = "Rooftop Asset List - July 2025.xlsx"
# sheet_name = "Sheet1"
# postcode_column = 'post_code'
# fulladdress_column = None
# address1_column = "add_1"
# address1_method = None
# address_cols_to_concat = [
# "add_1", "add_2", "add_3", "add_4"
# ]
# missing_postcodes_method = None
# landlord_year_built = "date_built"
# landlord_os_uprn = None
# landlord_property_type = "ConstructionStyle"
# landlord_built_form = "ConstructionStyle"
# landlord_wall_construction = None
# landlord_heating_system = "Description"
# landlord_existing_pv = None
# landlord_property_id = "PropertyCode"
# outcomes_filename = [os.path.join(data_folder, "Rooftop_Outcomes.xlsx")]
# outcomes_sheetname = ["OUTCOMES"]
# outcomes_postcode = ["POSTCODE"]
# outcomes_houseno = ["NO"]
# outcomes_address = ["ADDRESS"]
# outcomes_id = [None]
# master_filepaths = [os.path.join(data_folder, "Master.csv")]
# master_to_asset_list_filepath = None
# asset_list_header = 1
# landlord_block_reference = "bl_rec_ref"
# master_id_colnames = [None]
# landlord_roof_construction = None
# phase = False
# landlord_sap = None
# ecosurv_landlords = "rooftop"
#
# # For Housing
# data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/For Housing/New Programme July 2025"
# data_filename = "FOR HOUSING Asset List (Combined).xlsx"
# sheet_name = "Asset List"
# postcode_column = 'Postcode'
# fulladdress_column = "Address"
# address1_column = None
# address1_method = "house_number_extraction"
# address_cols_to_concat = []
# missing_postcodes_method = None
# landlord_year_built = None
# landlord_os_uprn = None
# landlord_property_type = "Type"
# landlord_built_form = "Type"
# landlord_wall_construction = None
# landlord_heating_system = "Heating - full"
# landlord_existing_pv = None
# landlord_property_id = "UPRN"
# outcomes_filename = [os.path.join(data_folder, "Khalim Combined - for analysis.xlsx")]
# outcomes_sheetname = ["Sheet1"]
# outcomes_postcode = ["POSTCODE"]
# outcomes_houseno = ["NO"]
# outcomes_address = ["ADDRESS"]
# outcomes_id = [None]
# master_filepaths = [os.path.join(data_folder, "submissions.csv")]
# master_to_asset_list_filepath = None
# asset_list_header = 0
# landlord_block_reference = None
# master_id_colnames = [None]
# landlord_roof_construction = None
# phase = False
# landlord_sap = "SAP"
# ecosurv_landlords = "for housing"
#
# # CDS
# data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/CDS"
# data_filename = "Founder Estates - Asset List.xlsx"
# sheet_name = "Combined"
# postcode_column = 'Postcode'
# fulladdress_column = "Address"
# address1_column = None
# address1_method = "house_number_extraction"
# address_cols_to_concat = []
# missing_postcodes_method = None
# landlord_year_built = None
# landlord_os_uprn = None
# landlord_property_type = None
# landlord_built_form = None
# landlord_wall_construction = None
# landlord_heating_system = "Heating Type"
# landlord_existing_pv = None
# landlord_property_id = "Row ID"
# outcomes_filename = []
# outcomes_sheetname = []
# outcomes_postcode = []
# outcomes_houseno = []
# outcomes_address = []
# outcomes_id = []
# master_filepaths = [os.path.join(data_folder, "submissions.csv")]
# master_to_asset_list_filepath = None
# asset_list_header = 0
# landlord_block_reference = None
# master_id_colnames = [None]
# landlord_roof_construction = None
# phase = False
# landlord_sap = None
# ecosurv_landlords = "cds"
#
# # Plus Dane
# data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Plus Dane/New Programme July 2025/"
# data_filename = "20250711 Plus Dane Asset List.xlsx"
# sheet_name = "Sheet1"
# postcode_column = 'Postcode'
# fulladdress_column = "Address"
# address1_column = None
# address1_method = "house_number_extraction"
# address_cols_to_concat = []
# missing_postcodes_method = None
# landlord_year_built = "Property Age"
# landlord_os_uprn = None
# landlord_property_type = "Property Type"
# landlord_built_form = "Built Form"
# landlord_wall_construction = "Wall Construction"
# landlord_heating_system = "Full Heating System"
# landlord_existing_pv = None
# landlord_property_id = "UPRN"
# outcomes_filename = [
# os.path.join(data_folder, "Outcomes - Plus Dane_CWI_2024.xlsx"),
# os.path.join(data_folder, "Outcomes - Plus Dane_CWI_2025.xlsx"),
# os.path.join(data_folder, "Outcomes - Plus Dane_PV_2025.xlsx"),
# ]
# outcomes_sheetname = [
# "CWI & LI - 2024", "2025 - CWI", "PV - 2025",
# ]
# outcomes_postcode = ["Postcode", "Postcode", "Postcode"]
# outcomes_houseno = ["No.", "No", "No"]
# outcomes_address = ["Address", "Address", "Address"]
# outcomes_id = ["Asset Reference", "LL UPRN", "LL UPRN"]
# master_filepaths = [
# os.path.join(data_folder, "submissions/JJC-Table 1.csv"),
# os.path.join(data_folder, "submissions/SCIS-Table 1.csv")
# ]
# master_to_asset_list_filepath = None
# asset_list_header = 1
# landlord_block_reference = None
# master_id_colnames = [None, None]
# landlord_roof_construction = None
# phase = False
# landlord_sap = "SAP Rating"
# ecosurv_landlords = "plus dane"
# data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Brentwood/July 2025 New Programme"
# data_filename = "20250710 Asset List Brentwood.xlsx"

View file

@ -337,5 +337,9 @@ PROPERTY_MAPPING = {
'Maisonette - Mid Terrace': 'maisonette',
'Chalet - Wheelchair': 'other',
'Amenity Block - Detached': 'other',
'Amenity Block - Semi detached': 'other'
'Amenity Block - Semi detached': 'other',
'house': 'house',
'block of flats': 'block of flats',
'bungalow': 'bungalow',
'flat': 'flat'
}

View file

@ -531,7 +531,7 @@ class Property:
"internal_wall_insulation", "external_wall_insulation", "cavity_wall_insulation",
"cylinder_thermostat", "loft_insulation", "room_roof_insulation", "flat_roof_insulation",
"solid_floor_insulation", "suspended_floor_insulation", "mixed_glazing",
"windows_glazing", "mechanical_ventilation"
"windows_glazing", "mechanical_ventilation", "solar_pv"
]:
# We update the data, as defined in the recommendaton
for prefix in ["walls", "roof", "floor"]:
@ -547,9 +547,6 @@ class Property:
output.update(simulation_config)
if recommendation["type"] == "solar_pv":
output["photo_supply_ending"] = recommendation["photo_supply"]
if recommendation["type"] not in [
"sealing_open_fireplace", "low_energy_lighting",
"internal_wall_insulation", "external_wall_insulation", "cavity_wall_insulation",

View file

@ -77,7 +77,6 @@ class GoogleSolarApi:
# property attributes:
self.floor_area = None
self.roof_area = None
self.roof_segment_indexes = None
self.panel_area = assumptions.RDSAP_AREA_PER_PANEL
self.panel_wattage = None
self.panel_performance = None
@ -89,6 +88,8 @@ class GoogleSolarApi:
self.double_property = False
self.solar_materials = solar_materials
self.allowed_segment_indices = None
def get_building_insights(self, longitude, latitude, required_quality="MEDIUM", max_retries=None):
"""
Make an API request to retrieve building insights based on the given longitude and latitude, with retry
@ -203,8 +204,6 @@ class GoogleSolarApi:
# It should be straightforward, but I'd rather see an actual instance of this happening
raise NotImplementedError("Panel wattage is not 400W - implement me")
self.roof_segment_indexes = [segment['segmentIndex'] for segment in self.roof_segments]
# We now start finding the solar panel configurations
self.optimise_solar_configuration(
energy_consumption=energy_consumption,
@ -294,10 +293,10 @@ class GoogleSolarApi:
# Remove any north facing roof segments
panel_performance = []
for config in self.insights_data["solarPotential"].get("solarPanelConfigs", []):
roof_segment_summaries = config["roofSegmentSummaries"]
# Filter on just the segments in self.roof_segment_indexes
roof_segment_summaries = [
segment for segment in roof_segment_summaries if segment["segmentIndex"] in self.roof_segment_indexes
s for s in config.get("roofSegmentSummaries", [])
if s["segmentIndex"] in self.allowed_segment_indices
]
roi_summary = []
@ -544,24 +543,28 @@ class GoogleSolarApi:
def exclude_north_facing_segments(self, property_instance):
"""
Filter out any north-facing roof segments from the roof_segments attribute.
North-facing segments are defined as those with an azimuth between -30 and 30 degrees.
Filter out any north-facing roof segments from self.roof_segments.
Keep API's original segmentIndex; optionally add a localIndex.
"""
is_flat = property_instance.roof["is_flat"]
filtered_segments = []
for segment_index, segment in enumerate(self.roof_segments):
segment["segmentIndex"] = segment_index
# Check if the segment is north-facing
if (
self.NORTH_FACING_AZIMUTH_RANGE[0] <= segment['azimuthDegrees'] <= self.NORTH_FACING_AZIMUTH_RANGE[1]
) and not property_instance.roof["is_flat"]:
continue
for i, seg in enumerate(self.roof_segments):
# DO NOT overwrite seg["segmentIndex"]
keep = True
if not is_flat:
if self.NORTH_FACING_AZIMUTH_RANGE[0] <= seg['azimuthDegrees'] <= self.NORTH_FACING_AZIMUTH_RANGE[1]:
keep = False
filtered_segments.append(segment)
if keep:
seg = dict(**seg) # shallow copy
seg["localIndex"] = i # optional local index as a reference from this loop
filtered_segments.append(seg)
self.roof_segments = filtered_segments
self.allowed_segment_indices = {s["segmentIndex"] for s in self.roof_segments}
@staticmethod
def haversine(lat1, lon1, lat2, lon2):
"""
@ -742,7 +745,8 @@ class GoogleSolarApi:
@classmethod
def building_solar_analysis(
cls, building_solar_config: List, input_properties: List[Property], session, google_solar_api_key: str
cls, building_solar_config: List, input_properties: List[Property], session, google_solar_api_key: str,
solar_materials: list
):
"""
Perform the solar analysis for the building level
@ -750,6 +754,7 @@ class GoogleSolarApi:
:param input_properties: List of properties
:param session: Database session
:param google_solar_api_key: Google Solar API key
:param solar_materials: List of solar materials
:return:
"""
@ -788,7 +793,7 @@ class GoogleSolarApi:
energy_consumption = sum(
[entry['energy_consumption'] for entry in building_solar_config if entry['building_id'] == building_id]
)
solar_api_client = cls(api_key=google_solar_api_key)
solar_api_client = cls(api_key=google_solar_api_key, solar_materials=solar_materials)
solar_api_client.get(
longitude=coordinates["longitude"],
latitude=coordinates["latitude"],

View file

@ -415,13 +415,25 @@ def get_funding_data():
project_scores_matrix.columns = ['Floor Area Segment', 'Starting Band', 'Finishing Band', 'Cost Savings']
project_scores_matrix["Cost Savings"] = project_scores_matrix["Cost Savings"].astype(float)
partial_project_scores_matrix = read_csv_from_s3(
bucket_name=get_settings().DATA_BUCKET,
filepath="funding/ECO4_Partial_Project_Scores_Matrix_v6.csv",
)
partial_project_scores_matrix = pd.DataFrame(partial_project_scores_matrix)
partial_project_scores_matrix.columns = [
'Measure category', 'Measure_Type', 'Pre_Main_Heating_Source',
'Post_Main_Heating_Source', 'Total Floor Area Band', 'Starting Band',
'Average Treatable Factor', 'Cost Savings', 'SAP Savings'
]
partial_project_scores_matrix["Cost Savings"] = partial_project_scores_matrix["Cost Savings"].astype(float)
whlg_eligible_postcodes = read_csv_from_s3(
bucket_name=get_settings().DATA_BUCKET,
filepath="funding/whlg eligible postcodes.csv",
)
whlg_eligible_postcodes = pd.DataFrame(whlg_eligible_postcodes)
return project_scores_matrix, whlg_eligible_postcodes
return project_scores_matrix, partial_project_scores_matrix, whlg_eligible_postcodes
async def model_engine(body: PlanTriggerRequest):
@ -648,7 +660,7 @@ async def model_engine(body: PlanTriggerRequest):
logger.info("Reading in materials and cleaned datasets")
materials = get_materials(session)
cleaned = get_cleaned()
eco_project_scores_matrix, whlg_eligible_postcodes = get_funding_data()
project_scores_matrix, partial_project_scores_matrix, whlg_eligible_postcodes = get_funding_data()
kwh_client = KwhData(bucket=get_settings().DATA_BUCKET, read_consumption_data=True)
@ -690,6 +702,7 @@ async def model_engine(body: PlanTriggerRequest):
input_properties=input_properties,
session=session,
google_solar_api_key=get_settings().GOOGLE_SOLAR_API_KEY,
solar_materials=[m for m in materials if m["type"] == "solar_pv"]
)
input_properties = GoogleSolarApi.unit_solar_analysis(
@ -819,21 +832,24 @@ async def model_engine(body: PlanTriggerRequest):
)
continue
# We layer funding on top of the recommendations
# We take one of these options
funding_paths = [
[["internal_wall_insulation", "external_wall_insulation"]],
["air_source_heat_pump"],
# We must have both of these options (though we check if the property doesn't already have HHRSH and
# is recommended it
[["solar_pv"], ["high_heat_retention_storage_heaters"]]
]
fixed_gain = optimiser_functions.calculate_fixed_gain(
property_required_measures, recommendations, p, needs_ventilation
)
gain = optimiser_functions.calculate_gain(body=body, p=p, fixed_gain=fixed_gain)
from backend.Funding import Funding
funding = Funding(
project_scores_matrix=project_scores_matrix,
partial_project_scores_matrix=partial_project_scores_matrix,
whlg_eligible_postcodes=whlg_eligible_postcodes,
eco4_social_cavity_abs_rate=13,
eco4_social_solid_abs_rate=17,
gbis_social_cavity_abs_rate=21,
gbis_social_solid_abs_rate=25,
gbis_private_cavity_abs_rate=21,
gbis_private_solid_abs_rate=28,
)
if not body.optimise:
if body.goal != "Increasing EPC":
raise NotImplementedError("Only EPC optimisation is currently supported")

View file

@ -143,6 +143,9 @@ class SolarPvRecommendations:
# back up here
"photo_supply": roof_coverage_percent,
"has_battery": False,
"simulation_config": {
"photo_supply_ending": roof_coverage_percent
},
"initial_ac_kwh_per_year": initial_ac_kwh_per_year,
"description_simulation": {"photo-supply": roof_coverage_percent},
"rank": rank, # Rank is used to get the representative recommendation - rank 0 will be chosen
@ -282,7 +285,10 @@ class SolarPvRecommendations:
"sap_points": minimum_sap_points,
"already_installed": already_installed,
**cost_result,
"has_battery": has_battery,
"has_battery": solar_pv_product["includes_battery"],
"simulation_config": {
"photo_supply_ending": roof_coverage_percent
},
"initial_ac_kwh_per_year": recommendation_config["initial_ac_kwh_per_year"],
"description_simulation": {"photo-supply": roof_coverage_percent},
}