junte/Model
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/Model.git synced 2026-06-08 11:17:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

adding project codes for blocks

Browse source
This commit is contained in:
Khalim Conn-Kowlessar 2025-06-08 16:15:23 +01:00
parent dd2a04f05e
commit d8b0662422
8 changed files with 226 additions and 119 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

92
asset_list/AssetList.py
View file

@ -526,6 +526,23 @@ class AssetList:
self.standardised_asset_list["Archetype"].copy()
)
self.prefixes_to_products = {
# Empty
self.EMPTY_CAVITY_NON_INTRUSIVE: self.CRM_PRODUCTS["Empty Cavity - ECO4"],
self.EPC_EMPTY_INSPECTIONS_RETRO_DRILLED: self.CRM_PRODUCTS["Empty Cavity - ECO4"],
self.EPC_EMPTY_INSPECTIONS_FILLED: self.CRM_PRODUCTS["Empty Cavity - ECO4"],
self.EPC_EMPTY_INSPECTIONS_FILLED_AT_BUILD: self.CRM_PRODUCTS["Empty Cavity - ECO4"],
self.EPC_EMPTY_INSPECTIONS_NON_CAVITY: self.CRM_PRODUCTS["Empty Cavity - ECO4"],
self.EPC_EMPTY: self.CRM_PRODUCTS["Empty Cavity - ECO4"],
self.LANDLORD_EMPTY_INSPECTIONS_OTHER: self.CRM_PRODUCTS["Empty Cavity - ECO4"],
# Extraction
self.EXTRACTION_NON_INTRUSIVE: self.CRM_PRODUCTS["Extract & Fill - ECO4"],
# Solar
self.SOLAR_ELIGIBLE: self.CRM_PRODUCTS["Solar PV - ECO4"],
self.SOLAR_ELIGIBLE_SOLID_WALL_UNINSULATED: self.CRM_PRODUCTS["Solar PV - ECO4"],
self.SOLAR_ELIGIBLE_NEEDS_HEATING_UPGRADE: self.CRM_PRODUCTS["Solar PV + Heating Upgrade - ECO4"],
}
def _extract_address1(self, asset_list, full_address_col, postcode_col, method="first_two_words"):
if method not in self.ADDRESS_1_CLEANING_METHODS:
@ -1752,7 +1769,7 @@ class AssetList:
self.standardised_asset_list["cavity_reason"] = None
empty_cavity_map = {
"non_intrusive_indicates_empty_cavity": self.EMPTY_CAVITY_NON_INTRUSIVE_PREFIX + ": ",
"non_intrusive_indicates_empty_cavity": self.EMPTY_CAVITY_NON_INTRUSIVE + ": ",
"non_intrusive_indicates_empty_cavity_has_solar": f"{self.EMPTY_CAVITY_NON_INTRUSIVE} - property "
"already has solar: ",
"non_intrusive_indicates_empty_cavity_no_year_filter": f"{self.EMPTY_CAVITY_NON_INTRUSIVE}, "
@ -1780,7 +1797,7 @@ class AssetList:
)) &
pd.isnull(self.standardised_asset_list["cavity_reason"])
),
f"{EPC_EMPTY_INSPECTIONS_RETRO_DRILLED}: " + self.standardised_asset_list[
f"{self.EPC_EMPTY_INSPECTIONS_RETRO_DRILLED}: " + self.standardised_asset_list[
"SAP Category"],
self.standardised_asset_list["cavity_reason"]
)
@ -1979,6 +1996,22 @@ class AssetList:
self.outcomes[self.DOMNA_PROPERTY_ID].isin(identified_work)
]
# Finally, direct operations feedback has suggested that if a property is a flat that has a SAP rating of
# 76 or above, we should exclude it because it's likely not going to be eligible for anyting
self.standardised_asset_list["cavity_reason"] = np.where(
(self.standardised_asset_list[self.STANDARD_PROPERTY_TYPE] == "flat") &
(self.standardised_asset_list["SAP Category"] == "SAP Rating 76 or more"),
None,
self.standardised_asset_list["cavity_reason"]
)
# Split cavity_reason on the colon and check if the first part is equal to one of the two options above
# that indicates empties
self.standardised_asset_list["identified_empty_cavity"] = (
self.standardised_asset_list["cavity_reason"].str.split(":").str[0].isin(
[self.EMPTY_CAVITY_NON_INTRUSIVE, self.EPC_EMPTY]
)
)
def label_property_status(self):
"""
This function is designed to be run after identify_worktypes() has been run, and will create a "property_status"
@ -2015,6 +2048,28 @@ class AssetList:
get_max_status_from_columns, axis=1
)
self.standardised_asset_list["project_code"] = None
# if we have any blocks, where work is eligible, we flag them now
if self.landlord_block_reference is not None:
# For blocks that have a 50% allocation, we create project codes
self.block_analysis()
# find any block refs with more than 50% emptires
viable_empty_blocks = self.block_analysis_df[
self.block_analysis_df['Percentage of Empties'] >= 0.50
]
if not viable_empty_blocks.empty:
project_code_lookup = viable_empty_blocks[["Block Reference"]].copy()
self.standardised_asset_list = self.standardised_asset_list.merge(
project_code_lookup, how="left", left_on=self.STANDARD_BLOCK_REFERENCE, right_on="Block Reference"
)
self.standardised_asset_list["project_code"] = np.where(
~pd.isnull(self.standardised_asset_list["Block Reference"]),
self.standardised_asset_list["Block Reference"],
self.standardised_asset_list["project_code"]
)
self.standardised_asset_list = self.standardised_asset_list.drop(columns=["Block Reference"])
def block_analysis(self):
if self.landlord_block_reference is None:
@ -2024,7 +2079,7 @@ class AssetList:
# Reverse mapping: label -> enum
LABEL_TO_ENUM = {e.label: e for e in hubspot_config.HubspotProcessStatus}
# Threshold status - anythign that is at this stage or beyond is considered surveyed
# Threshold status - anything that is at this stage or beyond is considered surveyed
threshold = hubspot_config.HubspotProcessStatus.SURVEYED_COMPLETED_SIGNED_OFF.value
block_analysis = []
@ -2034,15 +2089,21 @@ class AssetList:
if all(cavity_breakdown.index == "No Eligibility"):
continue
# We check the % of empty vs not empty as right now, we're focused on empty
n_empties = ((group["identified_empty_cavity"] == True) & (~pd.isnull(group["cavity_reason"]))).sum()
works = group["hubspot_status"]
above_threshold = works.map(LABEL_TO_ENUM.get).dropna()
count_above = (above_threshold >= threshold).sum()
proportion = count_above / len(works)
proportion_surveyed = count_above / len(works)
proportion_empty = n_empties / len(works)
# We auto-populate any blocks that have greater than 50% proportion empty
block_analysis.append(
{
"Block Reference": block_reference,
"Proportion of properties suryeyed": proportion,
"Proportion of properties suryeyed": proportion_surveyed,
"Percentage of Empties": proportion_empty,
**cavity_breakdown.to_dict(),
}
)
@ -2050,6 +2111,8 @@ class AssetList:
block_analysis = pd.DataFrame(block_analysis)
block_analysis = block_analysis.fillna(0)
# We flag which properties are eligible for works. We need at least 50%
self.block_analysis_df = block_analysis
@staticmethod
@ -2161,23 +2224,6 @@ class AssetList:
if not hubspot_config.Installer.is_valid_value(installer_name):
raise ValueError(f"Installer name {installer_name} is not valid. Please check the installer name.")
prefixes_to_products = {
# Empty
self.EMPTY_CAVITY_NON_INTRUSIVE: self.CRM_PRODUCTS["Empty Cavity - ECO4"],
self.EPC_EMPTY_INSPECTIONS_RETRO_DRILLED: self.CRM_PRODUCTS["Empty Cavity - ECO4"],
self.EPC_EMPTY_INSPECTIONS_FILLED: self.CRM_PRODUCTS["Empty Cavity - ECO4"],
self.EPC_EMPTY_INSPECTIONS_FILLED_AT_BUILD: self.CRM_PRODUCTS["Empty Cavity - ECO4"],
self.EPC_EMPTY_INSPECTIONS_NON_CAVITY: self.CRM_PRODUCTS["Empty Cavity - ECO4"],
self.EPC_EMPTY: self.CRM_PRODUCTS["Empty Cavity - ECO4"],
self.LANDLORD_EMPTY_INSPECTIONS_OTHER: self.CRM_PRODUCTS["Empty Cavity - ECO4"],
# Extraction
self.EXTRACTION_NON_INTRUSIVE: self.CRM_PRODUCTS["Extract & Fill - ECO4"],
# Solar
self.SOLAR_ELIGIBLE: self.CRM_PRODUCTS["Solar PV - ECO4"],
self.SOLAR_ELIGIBLE_SOLID_WALL_UNINSULATED: self.CRM_PRODUCTS["Solar PV - ECO4"],
self.SOLAR_ELIGIBLE_NEEDS_HEATING_UPGRADE: self.CRM_PRODUCTS["Solar PV + Heating Upgrade - ECO4"],
}
# We check if all products are covered in the lookup table
cavity_products = self.standardised_asset_list["cavity_reason"].unique().tolist()
solar_products = self.standardised_asset_list["solar_reason"].unique().tolist()
@ -2188,7 +2234,7 @@ class AssetList:
continue
matched_product = None
for product_prefix, crm_product in prefixes_to_products.items():
for product_prefix, crm_product in self.prefixes_to_products.items():
if identified_product.startswith(product_prefix):
matched_product = crm_product

152
asset_list/app.py
View file

@ -62,77 +62,77 @@ def app():
Property UPRN
"""
# Thrive - reconciliation
data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Thrive/Programme Reconciliation"
data_filename = "Thrive Asset List - Complete - Updated May 2025.xlsx"
sheet_name = "Sheet1"
postcode_column = 'postcode'
fulladdress_column = "full_address"
address1_column = "address_line_1"
address1_method = None
# Stori
data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Storicymru"
data_filename = "Asset list - for analysis.xlsx"
sheet_name = "SAP and Costs Calculations"
postcode_column = 'Postcode'
fulladdress_column = "Address1"
address1_column = None
address1_method = "house_number_extraction"
address_cols_to_concat = []
missing_postcodes_method = None
landlord_year_built = "age_band_calculated"
landlord_year_built = "Age"
landlord_os_uprn = None
landlord_property_type = "property_type"
landlord_built_form = "build_form"
landlord_wall_construction = None
landlord_roof_construction = "assumed_loft_insulation_thickness_updated"
landlord_heating_system = "heating_type_updated"
landlord_existing_pv = None
landlord_property_id = "thrive_property_id"
landlord_sap = "sap_rating_updated"
landlord_block_reference = "block_reference"
outcomes_filename = [
os.path.join(data_folder, "Thrive - Outcomes - April 24-March25 - Corrected.xlsx")
]
outcomes_sheetname = ["Sheet1"]
outcomes_postcode = ["postcode"]
outcomes_houseno = ["No."]
outcomes_id = ["thrive_property_id"]
outcomes_address = ["address"]
master_filepaths = [
os.path.join(data_folder, "Thrive Submissions ECO3 - with IDS.csv"),
os.path.join(data_folder, "Thrive Submissions ECO4 - with IDS.csv"),
]
master_to_asset_list_filepath = None
master_id_colnames = ["thrive_property_id", "thrive_property_id"]
phase = False
ecosurv_landlords = "thrive"
# Torus
data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Torus/Phase 2"
data_filename = "Torus Property Asset List - INSPECTIONS.xlsx"
sheet_name = "TORUS"
postcode_column = 'Postcode'
fulladdress_column = None
address1_column = "AddressLine1"
address1_method = None
address_cols_to_concat = ["AddressLine1", "AddressLine2", "AddressLine3"]
missing_postcodes_method = None
landlord_year_built = "Property Age"
landlord_os_uprn = "NatUPRN"
landlord_property_type = "Property Type"
landlord_built_form = "Built Form"
landlord_wall_construction = "Wall Construction"
landlord_roof_construction = "Roof Construction"
landlord_heating_system = "Space Heating Source"
landlord_existing_pv = "Low Carbon Technology (Solar PV)"
landlord_property_type = "TYPE"
landlord_built_form = "AGE / DETACHMENT"
landlord_wall_construction = "WALL"
landlord_roof_construction = "LOFT INSULATION"
landlord_heating_system = "BOILER"
landlord_existing_pv = "SOLAR PV"
landlord_property_id = "UPRN"
landlord_sap = "SAP Score"
landlord_sap = "Current SAP Rating"
landlord_block_reference = None
outcomes_filename = None
outcomes_sheetname = None
outcomes_postcode = None
outcomes_houseno = None
outcomes_id = None
outcomes_address = None
outcomes_filename = []
outcomes_sheetname = []
outcomes_postcode = []
outcomes_houseno = []
outcomes_id = []
outcomes_address = []
master_filepaths = []
master_to_asset_list_filepath = None
master_id_colnames = []
phase = True
phase = False
ecosurv_landlords = None
# Thrive - reconciliation
# data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Thrive/Programme Reconciliation"
# data_filename = "Thrive Asset List - Complete - Updated May 2025.xlsx"
# sheet_name = "Sheet1"
# postcode_column = 'postcode'
# fulladdress_column = "full_address"
# address1_column = "address_line_1"
# address1_method = None
# address_cols_to_concat = []
# missing_postcodes_method = None
# landlord_year_built = "age_band_calculated"
# landlord_os_uprn = None
# landlord_property_type = "property_type"
# landlord_built_form = "build_form"
# landlord_wall_construction = None
# landlord_roof_construction = "assumed_loft_insulation_thickness_updated"
# landlord_heating_system = "heating_type_updated"
# landlord_existing_pv = None
# landlord_property_id = "thrive_property_id"
# landlord_sap = "sap_rating_updated"
# landlord_block_reference = "block_reference"
# outcomes_filename = [
# os.path.join(data_folder, "Thrive - Outcomes - April 24-March25 - Corrected.xlsx")
# ]
# outcomes_sheetname = ["Sheet1"]
# outcomes_postcode = ["postcode"]
# outcomes_houseno = ["No."]
# outcomes_id = ["thrive_property_id"]
# outcomes_address = ["address"]
# master_filepaths = [
# os.path.join(data_folder, "Thrive Submissions ECO3 - with IDS.csv"),
# os.path.join(data_folder, "Thrive Submissions ECO4 - with IDS.csv"),
# ]
# master_to_asset_list_filepath = None
# master_id_colnames = ["thrive_property_id", "thrive_property_id"]
# phase = False
# ecosurv_landlords = "thrive"
# Southern Midlands
# data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Southern/Midlands Properties - Apr 2025"
# data_filename = "Southern Housing Midlands Property List - combined.xlsx"
@ -160,34 +160,6 @@ def app():
# master_filepaths = []
# master_to_asset_list_filepath = None
data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Places For People/North-West"
data_filename = "Places for People NORTH WEST - INSPECTIONS MASTER - UPDATE.xlsx"
sheet_name = "CHECKED"
postcode_column = 'Postcode'
fulladdress_column = None
address1_column = "AddressLine1"
address1_method = None
address_cols_to_concat = ["AddressLine1", "AddressLine2", "AddressLine3"]
missing_postcodes_method = None
landlord_year_built = None
landlord_os_uprn = None
landlord_property_type = "Archetype (PFP)"
landlord_built_form = "Archetype (PFP)"
landlord_wall_construction = None
landlord_roof_construction = None
landlord_heating_system = None
landlord_existing_pv = None
landlord_property_id = "Uprn"
outcomes_filename = None
outcomes_sheetname = None
outcomes_postcode = None
outcomes_houseno = None
outcomes_id = None
master_filepaths = []
master_to_asset_list_filepath = None
landlord_sap = None
phase = None
# Maps addresses to uprn in problematic cases
manual_uprn_map = {}
@ -482,8 +454,6 @@ def app():
# We now flag the status of the property
asset_list.label_property_status()
asset_list.block_analysis()
# Store as an excel
filename = os.path.join(data_folder, ".".join(data_filename.split(".")[:-1])) + " - Standardised.xlsx"
# Store the data in two tabs. One for the asset list with the EPC data and the second with the flat data

29
asset_list/mappings/built_form.py
View file

@ -331,4 +331,33 @@ BUILT_FORM_MAPPINGS = {
'Low Rise': 'low rise',
'Upper Floor': 'top-floor',
'High Rise': 'high rise',
'2012 ONWARDS DETACHED': 'detached',
'1950-66 END TERRACE': 'end-terrace',
'1976-82 MID TERRACED': 'mid-terrace',
'1950-66 MID TERRACE': 'mid-terrace',
'1991-95 DETACHED': 'detached',
'1976-82 END TERRACED': 'end-terrace',
'1967-75 DETACHED': 'detached',
'PRE 1900 DETACHED': 'detached',
'PRE 1900 MID TERRACE': 'mid-terrace',
'1900 DET': 'detached',
'1967-75 MID TERR': 'mid-terrace',
'1930-49 SEMI DET': 'semi-detached',
'1900-29 SEMI DET': 'semi-detached',
'1900-29 MID TERR': 'mid-terrace',
'1983- 90 MID TERR': 'mid-terrace',
'1976-82 MID TERR': 'mid-terrace',
'1983-90 END TERR': 'end-terrace',
'1991-95 SEMI DET': 'semi-detached',
'1983-90 SEMI DET': 'semi-detached',
'1991-95 MID TERR': 'mid-terrace',
'1950-66 SEMI DET': 'semi-detached',
'1900 MID TERR': 'mid-terrace',
'1967-75 SEMI DET': 'semi-detached',
'1983- 90 SEMI DET': 'semi-detached',
'1983-90 MID TERR': 'mid-terrace',
'1976-82 SEMI DET': 'semi-detached',
'PRE 1900 MID TERR': 'mid-terrace'
}

3
asset_list/mappings/exising_pv.py
View file

@ -16,5 +16,6 @@ EXISTING_PV_MAPPINGS = {
'PV: 25% roof area, PV: 3.6kWp array': 'already has PV',
'PV: 10% roof area, PV: 2kWp array': 'already has PV',
'PV: 50% roof area': 'already has PV',
'Solar PV': 'already has PV'
'Solar PV': 'already has PV',
'SOLAR PV': 'already has PV'
}

34
asset_list/mappings/heating_systems.py
View file

@ -293,5 +293,37 @@ HEATING_MAPPINGS = {
'No Data': 'unknown',
'Boiler System': 'gas condensing boiler',
'Storage heating': 'electric storage heaters',
'Storage heating (HHRSH)': 'high heat retention storage heaters'
'Storage heating (HHRSH)': 'high heat retention storage heaters',
'ELECTRIC BOILER': 'electric boiler',
'STORAGE HEATERS': 'electric storage heaters',
'GREENSTAR 24I JUNIOR': 'gas combi boiler',
'generic cond combi post98': 'gas condensing combi',
'SAP TABLE REG COND +98 NO PICTURE OF BOILER': 'gas condensing boiler',
'ECO TEC PRO 28 H COMBI A': 'gas combi boiler',
'GREENSTAR 25I ErP': 'gas combi boiler',
'IDEAL LOGIC MAX COMBI C30': 'gas combi boiler',
'ECO TEC PRO 28 (286/5-3)': 'gas combi boiler',
'IDEAL LOGIC HEAT 30': 'gas boiler, radiators',
'WORCESTER 240': 'gas boiler, radiators',
'ECO TEC PRO 24 (246/5-3)': 'gas combi boiler',
'ECO TEC PRO 28 (OLD)': 'gas combi boiler',
'LOGIC COMBI2 C30': 'gas combi boiler',
'GREENSTAR 28I JUNIOR': 'gas combi boiler',
'WORCESTER 24i': 'gas combi boiler',
'GREENSTAR 30I ErP': 'gas combi boiler',
'25 CDI': 'gas combi boiler',
'GREENSTAR 28CDI COMPACT ErP': 'gas combi boiler',
'GREENSTAR 24 RI': 'gas boiler, radiators',
'BAXI COMBI 105 HE': 'gas combi boiler',
'ECO TEC PRO 28 (OLD TYPE)': 'gas combi boiler',
'WORCESTER 28 SI ll RSF': 'gas combi boiler',
'GREENSTAR 30SI COMPACT ErP': 'gas combi boiler',
'SAP TABLE REG COND +98 NO PICTURE OF CYLINDER': 'gas condensing boiler',
'WORCESTER 24 SI ll RSF': 'gas combi boiler',
'GREENSTAR 4000': 'gas combi boiler',
'GREENSTAR 24i JUNIOR': 'gas combi boiler',
'ECO TEC PRO 24 (OLD TYPE)': 'gas combi boiler',
'GREENSTAR 30SI COMPACT': 'gas combi boiler',
'BAXI DUO TEC 28 COMBI ErP': 'gas combi boiler'
}

5
asset_list/mappings/property_type.py
View file

@ -252,5 +252,8 @@ PROPERTY_MAPPING = {
'Bedsit bungalow semi detached': 'bedsit',
'Bedsit Flat': 'bedsit',
'Semi detached house': 'house',
'Unit': 'unknown'
'Unit': 'unknown',
'HOUSE (3 STOREY)': 'house',
'FLAT GROUND FLOOR': 'flat',
'FLAT TOP FLOOR': 'flat'
}

9
asset_list/mappings/roof.py
View file

@ -43,6 +43,13 @@ ROOF_CONSTRUCTION_MAPPINGS = {
'Non-joist': 'unknown',
'25mm': 'pitched less than 100mm insulation',
'400mm+': 'pitched insulated',
'12mm': 'pitched less than 100mm insulation'
'12mm': 'pitched less than 100mm insulation',
'150MM': 'pitched insulated',
'200MM': 'pitched insulated',
'250MM': 'pitched insulated',
'100MM': 'pitched less than 100mm insulation',
'U/K': 'unknown',
'U/K - 250MM RIR FLAT CEILING': 'flat unknown insulation',
'U/K - 200MM RIR FLAT CEILING': 'flat unknown insulation'
}

21
asset_list/mappings/walls.py
View file

@ -224,5 +224,24 @@ WALL_CONSTRUCTION_MAPPINGS = {
'Traditional Cavity Brickwork': 'cavity unknown insulation',
'System build (undefined)': 'system built',
'Non Trad Wimpey': 'system built',
'Non Trad Wates': 'system built'
'Non Trad Wates': 'system built',
'CAVITY FILLED 270MM': 'filled cavity',
'CAVITY FILLED 270MM': 'filled cavity',
'CAVITY FILLED 250MM': 'filled cavity',
'CAVITY FILLED 260MM': 'filled cavity',
'CAVITY FILLED 260MM': 'filled cavity',
'SOLID A/B 220MM': 'solid brick unknown insulation',
'CAVITY A/B 300MM': "uninsulated cavity",
'CAVITY A/B 250MM': "uninsulated cavity",
'CAVITY A/B 260MM': "uninsulated cavity",
'CAVITY A/B 270MM': "uninsulated cavity",
'SOLID BRICK/CAVITY EXT': 'solid brick unknown insulation',
'CAVITY EWI': 'filled cavity',
'SANDSTONE/CAVITY EXT': 'sandstone or limestone',
'SYSTEM BUILD 100MM EWI': 'system built',
'CAVITY A/B 260MM': "uninsulated cavity",
'CAVITY A/B 270MM': "uninsulated cavity",
'CAVITY A/B 250MM': "uninsulated cavity"
}