From 45b372b9ae43dc1c867f5ff425d8fc4b3e6e5c91 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 10 Mar 2025 19:11:09 +0000 Subject: [PATCH] refining detection for solar and breakdown counts --- asset_list/AssetList.py | 409 +++++++++++++------- asset_list/app.py | 72 +++- asset_list/mappings/heating_systems.py | 13 +- asset_list/mappings/property_type.py | 10 +- asset_list/mappings/walls.py | 23 +- etl/customers/mod/pilot/1. Create Sample.py | 34 ++ 6 files changed, 406 insertions(+), 155 deletions(-) create mode 100644 etl/customers/mod/pilot/1. Create Sample.py diff --git a/asset_list/AssetList.py b/asset_list/AssetList.py index 689e752b..dc22a8a2 100644 --- a/asset_list/AssetList.py +++ b/asset_list/AssetList.py @@ -1012,7 +1012,8 @@ class AssetList: ) ) else: - raise NotImplementedError("need to implement the case for non-intrusives") + # We set the filter to False, as we have no non-intrusives + non_intrusives_wall_filter = False self.standardised_asset_list["non_intrusive_indicates_empty_cavity"] = ( (~self.standardised_asset_list[self.STANDARD_PROPERTY_TYPE].isin(["bedsit"])) & @@ -1110,7 +1111,8 @@ class AssetList: ) else: - raise NotImplementedError("need to implement the case for non-intrusives") + self.standardised_asset_list["non_intrusive_indicates_cavity_extraction"] = False + self.standardised_asset_list["non_intrusive_indicates_cavity_extraction_no_sap_filter"] = False # Adjust self.standardised_asset_list["non_intrusive_indicates_cavity_extraction_no_sap_filter"] = np.where( @@ -1131,7 +1133,7 @@ class AssetList: ) self.standardised_asset_list["solar_landlord_data_indicates_needs_heating_upgrade"] = ( self.standardised_asset_list[self.STANDARD_HEATING_SYSTEM].isin( - ["electric storage heaters", "room heaters"] + ["electric storage heaters", "room heaters", "electric radiators"] ) ) @@ -1179,7 +1181,8 @@ class AssetList: ) ) else: - raise NotImplementedError("need to implement the case for non-intrusives") + # We don't have an indication + existing_solar_non_intrusives_check = False self.standardised_asset_list["property_has_solar"] = ( (self.standardised_asset_list[self.STANDARD_EXISTING_PV] == "already has PV") | @@ -1208,7 +1211,7 @@ class AssetList: print("Should we include cavity properties where they might be uninsulated?") self.standardised_asset_list["solar_landlord_walls_insulated"] = ( self.standardised_asset_list[self.STANDARD_WALL_CONSTRUCTION].isin( - ["filled cavity", "insulated solid brick"] + ["filled cavity", "insulated solid brick", "insulated timber frame"] ) ) @@ -1225,7 +1228,7 @@ class AssetList: ) ) else: - raise NotImplementedError("need to implement the case for non-intrusives") + self.standardised_asset_list["solar_non_intrusives_walls_insulated"] = False # TODO: We don't have information about the roof from this landlord @@ -1294,7 +1297,6 @@ class AssetList: lambda x: int(x) < 200 if str(x).isdigit() else False ) - # TODO: Fill with False - should be temp! self.standardised_asset_list["epc_has_floor_recommendation"] = ( self.standardised_asset_list["epc_has_floor_recommendation"].fillna(False) ) @@ -1339,36 +1341,6 @@ class AssetList: ) ) - # Check for other floor types, insulated - self.standardised_asset_list["solar_epc_floor_is_other_insulated"] = ( - # The floor is suspended and insulated - ( - ( - self.standardised_asset_list[self.EPC_API_DATA_NAMES["floor-description"]].str - .lower().str.contains("suspended") - ) & ( - ~self.standardised_asset_list["epc_has_floor_recommendation"] - ) & ( - # We do not utilise estimated EPCs for this method because we will always find that - # "epc_has_floor_recommendation" is False - self.standardised_asset_list["estimated"] == False - ) - ) | ( - ( - self.standardised_asset_list[ - self.EPC_API_DATA_NAMES["floor-description"] - ].str.lower().str.contains("suspended") - ) & ( - self.standardised_asset_list[ - self.EPC_API_DATA_NAMES["floor-description"] - ].str.lower().str.contains(", insulated") - ) - ) | ( - self.standardised_asset_list["floor_u_value"].apply( - lambda x: x <= 0.5 if not pd.isnull(x) else False - ) - ) - ) #################################### # Check solar eligibility #################################### @@ -1390,7 +1362,13 @@ class AssetList: self.standardised_asset_list["solar_non_intrusives_walls_insulated"] ) + not_a_flat = ( + self.standardised_asset_list[self.STANDARD_PROPERTY_TYPE] != "flat" + ) + self.standardised_asset_list["solar_eligible_solid_floor"] = ( + # Property isn't a flag + not_a_flat & # Landlord data or EPC data indicates the heating system is appropriate correct_heating_system & # The property doesn't currently have solar @@ -1399,11 +1377,32 @@ class AssetList: walls_are_insulated & # Roof is insulated self.standardised_asset_list["solar_epc_roof_insulated"] & - self.standardised_asset_list["solar_epc_floor_is_solid_no_recommendation"] + # Floor type check + self.standardised_asset_list["solar_epc_floor_is_solid_no_recommendation"] & + # SAP below threshold + self.standardised_asset_list[self.ATTRIBUTE_SAP_THRESHOLD_AND_BELOW] + ) + + self.standardised_asset_list["solar_eligible_solid_floor_sap_above_threshold"] = ( + # Property isn't a flag + not_a_flat & + # Landlord data or EPC data indicates the heating system is appropriate + correct_heating_system & + # The property doesn't currently have solar + ~self.standardised_asset_list["property_has_solar"] & + # The walls are insulated + walls_are_insulated & + # Roof is insulated + self.standardised_asset_list["solar_epc_roof_insulated"] & + # Floor type check + self.standardised_asset_list["solar_epc_floor_is_solid_no_recommendation"] & + # SAP above threshold + ~self.standardised_asset_list[self.ATTRIBUTE_SAP_THRESHOLD_AND_BELOW] ) # With heating upgrade self.standardised_asset_list["solar_eligible_solid_floor_needs_heating_upgrade"] = ( + not_a_flat & # Needs heating upgrade needs_heating_upgrade & # The property doesn't currently have solar @@ -1412,14 +1411,43 @@ class AssetList: walls_are_insulated & # Roof is insulated self.standardised_asset_list["solar_epc_roof_insulated"] & - self.standardised_asset_list["solar_epc_floor_is_solid_no_recommendation"] + # Floor type check + self.standardised_asset_list["solar_epc_floor_is_solid_no_recommendation"] & + # SAP Below threshold + self.standardised_asset_list[self.ATTRIBUTE_SAP_THRESHOLD_AND_BELOW] & + # SAP above threshold + self.standardised_asset_list[self.ATTRIBUTE_SAP_THRESHOLD_AND_BELOW] ) + # With heating upgrade, above threshold + self.standardised_asset_list["solar_eligible_solid_floor_needs_heating_upgrade_sap_above_threshold"] = ( + not_a_flat & + # Needs heating upgrade + needs_heating_upgrade & + # The property doesn't currently have solar + ~self.standardised_asset_list["property_has_solar"] & + # The walls are insulated + walls_are_insulated & + # Roof is insulated + self.standardised_asset_list["solar_epc_roof_insulated"] & + # Floor type check + self.standardised_asset_list["solar_epc_floor_is_solid_no_recommendation"] & + # SAP Below threshold + self.standardised_asset_list[self.ATTRIBUTE_SAP_THRESHOLD_AND_BELOW] & + # SAP above threshold + ~self.standardised_asset_list[self.ATTRIBUTE_SAP_THRESHOLD_AND_BELOW] + ) + # Because the EPC data can be contradictrory, we remove any overlap self.standardised_asset_list["solar_eligible_solid_floor_needs_heating_upgrade"] = np.where( self.standardised_asset_list["solar_eligible_solid_floor"], False, self.standardised_asset_list["solar_eligible_solid_floor_needs_heating_upgrade"] ) + self.standardised_asset_list["solar_eligible_solid_floor_needs_heating_upgrade_sap_above_threshold"] = np.where( + self.standardised_asset_list["solar_eligible_solid_floor_sap_above_threshold"], + False, + self.standardised_asset_list["solar_eligible_solid_floor_needs_heating_upgrade_sap_above_threshold"] + ) # We shouldn't have an overlap if ( @@ -1430,6 +1458,7 @@ class AssetList: # Solid floor but needs a loft top-up self.standardised_asset_list["solar_eligible_solid_floor_needs_loft"] = ( + not_a_flat & # Landlord data or EPC data indicates the heating system is appropriate correct_heating_system & # The property doesn't currently have solar @@ -1438,10 +1467,31 @@ class AssetList: walls_are_insulated & # Roof is insulated self.standardised_asset_list["solar_epc_loft_needs_topup"] & - self.standardised_asset_list["solar_epc_floor_is_solid_no_recommendation"] + # Check floor + self.standardised_asset_list["solar_epc_floor_is_solid_no_recommendation"] & + # SAP below threshold + self.standardised_asset_list[self.ATTRIBUTE_SAP_THRESHOLD_AND_BELOW] ) + # Solid floor, needs loft, above SAP thresold + self.standardised_asset_list["solar_eligible_solid_floor_needs_loft_sap_above_threshold"] = ( + not_a_flat & + # Landlord data or EPC data indicates the heating system is appropriate + correct_heating_system & + # The property doesn't currently have solar + ~self.standardised_asset_list["property_has_solar"] & + # The walls are insulated + walls_are_insulated & + # Roof is insulated + self.standardised_asset_list["solar_epc_loft_needs_topup"] & + # Check floor + self.standardised_asset_list["solar_epc_floor_is_solid_no_recommendation"] & + # SAP above threshold + ~self.standardised_asset_list[self.ATTRIBUTE_SAP_THRESHOLD_AND_BELOW] + ) + # Needs loft & heating self.standardised_asset_list["solar_eligible_solid_floor_needs_loft_needs_heating_upgrade"] = ( + not_a_flat & # Needs heating upgrade needs_heating_upgrade & # The property doesn't currently have solar @@ -1450,11 +1500,33 @@ class AssetList: walls_are_insulated & # Roof is insulated self.standardised_asset_list["solar_epc_loft_needs_topup"] & - self.standardised_asset_list["solar_epc_floor_is_solid_no_recommendation"] + # Floor type + self.standardised_asset_list["solar_epc_floor_is_solid_no_recommendation"] & + # SAP below threshold + self.standardised_asset_list[self.ATTRIBUTE_SAP_THRESHOLD_AND_BELOW] + ) + + self.standardised_asset_list[ + "solar_eligible_solid_floor_needs_loft_needs_heating_upgrade_sap_above_threshold" + ] = ( + not_a_flat & + # Needs heating upgrade + needs_heating_upgrade & + # The property doesn't currently have solar + ~self.standardised_asset_list["property_has_solar"] & + # The walls are insulated + walls_are_insulated & + # Roof is insulated + self.standardised_asset_list["solar_epc_loft_needs_topup"] & + # Floor type + self.standardised_asset_list["solar_epc_floor_is_solid_no_recommendation"] & + # SAP above threshold + ~self.standardised_asset_list[self.ATTRIBUTE_SAP_THRESHOLD_AND_BELOW] ) # Other floor type, fully insulated self.standardised_asset_list["solar_eligible_other_floor"] = ( + not_a_flat & # Landlord data or EPC data indicates the heating system is appropriate correct_heating_system & # The property doesn't currently have solar @@ -1463,11 +1535,30 @@ class AssetList: walls_are_insulated & # Roof is insulated self.standardised_asset_list["solar_epc_roof_insulated"] & - self.standardised_asset_list["solar_epc_floor_is_other_insulated"] + # Floor type + ~self.standardised_asset_list["solar_epc_floor_is_solid_no_recommendation"] & + # SAP below threshold + self.standardised_asset_list[self.ATTRIBUTE_SAP_THRESHOLD_AND_BELOW] + ) + self.standardised_asset_list["solar_eligible_other_floor_sap_above_threshold"] = ( + not_a_flat & + # Landlord data or EPC data indicates the heating system is appropriate + correct_heating_system & + # The property doesn't currently have solar + ~self.standardised_asset_list["property_has_solar"] & + # The walls are insulated + walls_are_insulated & + # Roof is insulated + self.standardised_asset_list["solar_epc_roof_insulated"] & + # Floor type - other types + ~self.standardised_asset_list["solar_epc_floor_is_solid_no_recommendation"] & + # SAP above threshold + ~self.standardised_asset_list[self.ATTRIBUTE_SAP_THRESHOLD_AND_BELOW] ) # With heating upgrade self.standardised_asset_list["solar_eligible_other_floor_needs_heating_upgrade"] = ( + not_a_flat & # Needs heating upgrade needs_heating_upgrade & # The property doesn't currently have solar @@ -1476,11 +1567,37 @@ class AssetList: walls_are_insulated & # Roof is insulated self.standardised_asset_list["solar_epc_roof_insulated"] & - self.standardised_asset_list["solar_epc_floor_is_other_insulated"] + # Other floor types + ~self.standardised_asset_list["solar_epc_floor_is_solid_no_recommendation"] & + # SAP below threshold + self.standardised_asset_list[self.ATTRIBUTE_SAP_THRESHOLD_AND_BELOW] ) + # With heating upgrade, SAP above threshold + self.standardised_asset_list["solar_eligible_other_floor_needs_heating_upgrade_sap_above_threshold"] = ( + not_a_flat & + # Needs heating upgrade + needs_heating_upgrade & + # The property doesn't currently have solar + ~self.standardised_asset_list["property_has_solar"] & + # The walls are insulated + walls_are_insulated & + # Roof is insulated + self.standardised_asset_list["solar_epc_roof_insulated"] & + # Other floor types + ~self.standardised_asset_list["solar_epc_floor_is_solid_no_recommendation"] & + # SAP above threshold + ~self.standardised_asset_list[self.ATTRIBUTE_SAP_THRESHOLD_AND_BELOW] + ) + # Check for overlap + if ( + self.standardised_asset_list["solar_eligible_other_floor_needs_heating_upgrade"] & + self.standardised_asset_list["solar_eligible_other_floor_needs_heating_upgrade_sap_above_threshold"] + ).sum(): + raise ValueError("Both heating upgrade and no heating upgrade are true - this should not be possible") # Other floor type, needs loft top-up self.standardised_asset_list["solar_eligible_other_floor_needs_loft"] = ( + not_a_flat & # Landlord data or EPC data indicates the heating system is appropriate correct_heating_system & # The property doesn't currently have solar @@ -1489,12 +1606,31 @@ class AssetList: walls_are_insulated & # Roof need loft top-up self.standardised_asset_list["solar_epc_loft_needs_topup"] & - # Floor is not solid, but is insulated - self.standardised_asset_list["solar_epc_floor_is_other_insulated"] + # Other floor types + ~self.standardised_asset_list["solar_epc_floor_is_solid_no_recommendation"] & + # SAP below threshold + self.standardised_asset_list[self.ATTRIBUTE_SAP_THRESHOLD_AND_BELOW] + ) + # Other floor type, needs loft top-up, SAP above threshold + self.standardised_asset_list["solar_eligible_other_floor_needs_loft_sap_above_threshold"] = ( + not_a_flat & + # Landlord data or EPC data indicates the heating system is appropriate + correct_heating_system & + # The property doesn't currently have solar + ~self.standardised_asset_list["property_has_solar"] & + # The walls are insulated + walls_are_insulated & + # Roof need loft top-up + self.standardised_asset_list["solar_epc_loft_needs_topup"] & + # Other floor types + ~self.standardised_asset_list["solar_epc_floor_is_solid_no_recommendation"] & + # SAP above threshold + ~self.standardised_asset_list[self.ATTRIBUTE_SAP_THRESHOLD_AND_BELOW] ) # With heating upgrade self.standardised_asset_list["solar_eligible_other_floor_needs_loft_needs_heating_upgrade"] = ( + not_a_flat & # Landlord data or EPC data indicates the heating system is appropriate needs_heating_upgrade & # The property doesn't currently have solar @@ -1503,8 +1639,28 @@ class AssetList: walls_are_insulated & # Roof need loft top-up self.standardised_asset_list["solar_epc_loft_needs_topup"] & - # Floor is not solid, but is insulated - self.standardised_asset_list["solar_epc_floor_is_other_insulated"] + # Other floor types + ~self.standardised_asset_list["solar_epc_floor_is_solid_no_recommendation"] & + # SAP below threshold + self.standardised_asset_list[self.ATTRIBUTE_SAP_THRESHOLD_AND_BELOW] + ) + + self.standardised_asset_list[ + "solar_eligible_other_floor_needs_loft_needs_heating_upgrade_sap_above_threshold" + ] = ( + not_a_flat & + # Landlord data or EPC data indicates the heating system is appropriate + needs_heating_upgrade & + # The property doesn't currently have solar + ~self.standardised_asset_list["property_has_solar"] & + # The walls are insulated + walls_are_insulated & + # Roof need loft top-up + self.standardised_asset_list["solar_epc_loft_needs_topup"] & + # Other floor types + ~self.standardised_asset_list["solar_epc_floor_is_solid_no_recommendation"] & + # SAP above threshold + ~self.standardised_asset_list[self.ATTRIBUTE_SAP_THRESHOLD_AND_BELOW] ) # Drop anything we don't need @@ -1529,66 +1685,6 @@ class AssetList: self.standardised_asset_list[self.STANDARD_PROPERTY_TYPE] != "block of flats" ] - # Produce some aggregate figures - self.work_type_figures = { - # Empty cavity from non-intrusives - "Empty Cavity (non-intrusives)": non_blocks_of_flats["non_intrusive_indicates_empty_cavity"].sum(), - "Empty Cavity (non-intrusives, blocks of flats)": ( - blocks_of_flats["non_intrusive_indicates_empty_cavity"].sum() - ), - "Empty Cavity (non-intrusives, no SAP filter)": ( - non_blocks_of_flats["non_intrusive_indicates_empty_cavity_no_sap_filter"].sum() - ), - "Empty Cavity (non-intrusives, no SAP filter, blocks of flats)": ( - blocks_of_flats["non_intrusive_indicates_empty_cavity_no_sap_filter"].sum() - ), - "Empty Cavity (EPC)": ( - ( - non_blocks_of_flats["epc_indicates_empty_cavity"] & - ~non_blocks_of_flats["non_intrusive_indicates_empty_cavity"] - ).sum() - ), - "Empty Cavity (EPC, blocks of flat)": ( - ( - blocks_of_flats["epc_indicates_empty_cavity"] & - ~blocks_of_flats["non_intrusive_indicates_empty_cavity"] - ).sum() - ), - "Cavity Extraction": ( - ( - ~non_blocks_of_flats["non_intrusive_indicates_empty_cavity"] & - ~non_blocks_of_flats["epc_indicates_empty_cavity"] & - non_blocks_of_flats["non_intrusive_indicates_cavity_extraction"] - ).sum() - ), - "Cavity Extraction (blocks of flats)": ( - ( - ~blocks_of_flats["non_intrusive_indicates_empty_cavity"] & - ~blocks_of_flats["epc_indicates_empty_cavity"] & - blocks_of_flats["non_intrusive_indicates_cavity_extraction"] - ).sum() - ), - "Cavity Extraction (no SAP filter)": ( - ( - ~self.standardised_asset_list["non_intrusive_indicates_empty_cavity"] & - ~self.standardised_asset_list["epc_indicates_empty_cavity"] & - self.standardised_asset_list["non_intrusive_indicates_cavity_extraction_no_sap_filter"] - ).sum() - ), - "Solar PV (Solid Floor)": ( - self.standardised_asset_list["solar_eligible_solid_floor"].sum() - ), - "Solar PV (Solid Floor, Needs Loft Top-up)": ( - self.standardised_asset_list["solar_eligible_solid_floor_needs_loft"].sum() - ), - "Solar PV (Other Floor)": ( - self.standardised_asset_list["solar_eligible_other_floor"].sum() - ), - "Solar PV (Other Floor, Needs Loft Top-up)": ( - self.standardised_asset_list["solar_eligible_other_floor_needs_loft"].sum() - ) - } - # Finally, we note why each property has been flagged self.standardised_asset_list["cavity_reason"] = None self.standardised_asset_list["cavity_reason"] = np.where( @@ -1628,51 +1724,55 @@ class AssetList: self.standardised_asset_list["cavity_reason"] ) + ###################################################### # Flag solar + ###################################################### self.standardised_asset_list["solar_reason"] = None - self.standardised_asset_list["solar_reason"] = np.where( - self.standardised_asset_list["solar_eligible_solid_floor"], - "Solid Floor, Insulated, No Existing Solar", - self.standardised_asset_list["solar_reason"] - ) - self.standardised_asset_list["solar_reason"] = np.where( - self.standardised_asset_list["solar_eligible_solid_floor_needs_heating_upgrade"], - "Solid Floor, Insulated, No Existing Solar, Needs Heating Upgrade", - self.standardised_asset_list["solar_reason"] - ) - self.standardised_asset_list["solar_reason"] = np.where( - self.standardised_asset_list["solar_eligible_solid_floor_needs_loft"], - "Solid Floor, Insulated, Needs Loft, No Existing Solar", - self.standardised_asset_list["solar_reason"] - ) - self.standardised_asset_list["solar_reason"] = np.where( - self.standardised_asset_list["solar_eligible_solid_floor_needs_loft_needs_heating_upgrade"], - "Solid Floor, Insulated, Needs Loft, No Existing Solar, Needs Heating Upgrade", - self.standardised_asset_list["solar_reason"] - ) + # Map of variables and fill values for the solar_reason variable + solar_reason_map = { + "solar_eligible_solid_floor": "Solar Eligible, Solid Floor", + "solar_eligible_solid_floor_sap_above_threshold": "Solar Eligible, Solid Floor, SAP Above Threshold", + "solar_eligible_solid_floor_needs_heating_upgrade": ( + "Solar Eligible, Solid Floor, Needs Heating Upgrade" + ), + "solar_eligible_solid_floor_needs_heating_upgrade_sap_above_threshold": ( + "Solar Eligible, Solid Floor, Needs Heating Upgrade, SAP Above Threshold" + ), + "solar_eligible_solid_floor_needs_loft": "Solar Eligible, Solid Floor, Needs Loft", + "solar_eligible_solid_floor_needs_loft_sap_above_threshold": ( + "Solar Eligible, Solid Floor, Needs Loft, SAP Above Threshold" + ), + "solar_eligible_solid_floor_needs_loft_needs_heating_upgrade": ( + "Solar Eligible, Solid Floor, Needs Loft, Needs Heating Upgrade" + ), + "solar_eligible_solid_floor_needs_loft_needs_heating_upgrade_sap_above_threshold": ( + "Solar Eligible, Solid Floor, Needs Loft, Needs Heating Upgrade, SAP Above Threshold" + ), + "solar_eligible_other_floor": "Solar Eligible, Other Floor", + "solar_eligible_other_floor_sap_above_threshold": "Solar Eligible, Other Floor, SAP Above Threshold", + "solar_eligible_other_floor_needs_heating_upgrade": "Solar Eligible, Other Floor, Needs Heating Upgrade", + "solar_eligible_other_floor_needs_heating_upgrade_sap_above_threshold": ( + "Solar Eligible, Other Floor, Needs Heating Upgrade, SAP Above Threshold" + ), + "solar_eligible_other_floor_needs_loft": "Solar Eligible, Other Floor, Needs Loft", + "solar_eligible_other_floor_needs_loft_sap_above_threshold": ( + "Solar Eligible, Other Floor, Needs Loft, SAP Above Threshold" + ), + "solar_eligible_other_floor_needs_loft_needs_heating_upgrade": ( + "Solar Eligible, Other Floor, Needs Loft, Needs Heating Upgrade" + ), + "solar_eligible_other_floor_needs_loft_needs_heating_upgrade_sap_above_threshold": ( + "Solar Eligible, Other Floor, Needs Loft, Needs Heating Upgrade, SAP Above Threshold" + ) + } - self.standardised_asset_list["solar_reason"] = np.where( - self.standardised_asset_list["solar_eligible_other_floor"], - "Other Floor, Insulated, No Existing Solar", - self.standardised_asset_list["solar_reason"] - ) - self.standardised_asset_list["solar_reason"] = np.where( - self.standardised_asset_list["solar_eligible_other_floor_needs_heating_upgrade"], - "Other Floor, Insulated, No Existing Solar, Needs Heating Upgrade", - self.standardised_asset_list["solar_reason"] - ) - - self.standardised_asset_list["solar_reason"] = np.where( - self.standardised_asset_list["solar_eligible_other_floor_needs_loft"], - "Other Floor, Insulated, Needs Loft, No Existing Solar", - self.standardised_asset_list["solar_reason"] - ) - self.standardised_asset_list["solar_reason"] = np.where( - self.standardised_asset_list["solar_eligible_other_floor_needs_loft_needs_heating_upgrade"], - "Other Floor, Insulated, Needs Loft, No Existing Solar, Needs Heating Upgrade", - self.standardised_asset_list["solar_reason"] - ) + for variable, reason in solar_reason_map.items(): + self.standardised_asset_list["solar_reason"] = np.where( + self.standardised_asset_list[variable], + reason, + self.standardised_asset_list["solar_reason"] + ) # Flag anything that has existing outcomes if self.outcomes is not None: @@ -1694,6 +1794,12 @@ class AssetList: self.standardised_asset_list["cavity_reason"] ) + # Produce some aggregate figures + self.work_type_figures = { + **self.standardised_asset_list["cavity_reason"].value_counts().to_dict(), + **self.standardised_asset_list["solar_reason"].value_counts().to_dict() + } + def flat_analysis(self): # We need to deduce the building name - we strip out the house number @@ -2028,7 +2134,7 @@ class AssetList: outcomes_houseno ): if outcomes_filepath is None: - pass + return # ToDO: Parameterise for future use? self.outcomes = pd.read_excel(outcomes_filepath, sheet_name=outcomes_sheetname) @@ -2119,6 +2225,8 @@ class AssetList: self.outcomes[["row_id", "Outcome", "Notes", date_col]], how="left", on="row_id" ) + df = lookup[lookup["domna_property_id"] == "44beckettavenuegainsboroughdn211en-1d4811cbb046"] + visit_counts = ( lookup.groupby(self.DOMNA_PROPERTY_ID)["row_id"] .count() @@ -2153,6 +2261,9 @@ class AssetList: ): # TODO: This probably needs further expansion + if not master_filepaths: + return + if master_to_asset_list_filepath is not None: id_map = pd.read_csv(master_to_asset_list_filepath) else: diff --git a/asset_list/app.py b/asset_list/app.py index 63ca40d8..1a6dbc6b 100644 --- a/asset_list/app.py +++ b/asset_list/app.py @@ -247,6 +247,30 @@ def app(): # - We want: fully insulated property (all wall types), EPC D or below (floors should be solid) # - Or the insulation required is loft/cavity (floors should be solid) + # Wates + data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Wates - " + data_filename = "ECO 4 Wates.xlsx" + sheet_name = "Roadmap Homes" + postcode_column = 'Postcode' + fulladdress_column = None + address1_column = "Address Line 1" + address1_method = None + address_cols_to_concat = ["Address Line 1", "Address Line 2", "Address Line 3"] + missing_postcodes_method = None + landlord_year_built = "Build Year" + landlord_os_uprn = None + landlord_property_type = "Archetype" + landlord_wall_construction = "Wall" + landlord_heating_system = "Heating Type" + landlord_existing_pv = None + landlord_property_id = "UPRN" + outcomes_filename = None + outcomes_sheetname = None + outcomes_postcode = None + outcomes_houseno = None + master_filepaths = [] + master_to_asset_list_filepath = None + # Ealing # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Ealing/Programme data - 04032025" # data_filename = "Ealing BC - Property Plus Tenure 25.02.2025.xlsx" @@ -265,6 +289,29 @@ def app(): # landlord_existing_pv = None # landlord_property_id = "Property ref" + # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Colchester" + # data_filename = "Warmfront data- Colchester Borough Homes (Complete).xlsx" + # sheet_name = "Sheet1" + # postcode_column = 'Full Address.1' + # fulladdress_column = "Full Address" + # address1_column = None + # address1_method = "first_word" + # address_cols_to_concat = [] + # missing_postcodes_method = None + # landlord_year_built = "Build Date" + # landlord_os_uprn = None + # landlord_property_type = "Property Type" + # landlord_wall_construction = "Wallinsul" + # landlord_heating_system = "HeatSorc" + # landlord_existing_pv = None + # landlord_property_id = "Property Reference" + # outcomes_filename = None + # outcomes_sheetname = None + # outcomes_postcode = None + # outcomes_houseno = None + # master_filepaths = [] + # master_to_asset_list_filepath = None + # For Westward # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Westward" # data_filename = "WESTWARD - completed list..xlsx" @@ -282,6 +329,12 @@ def app(): # landlord_heating_system = "Heat Source" # landlord_existing_pv = "PV (Y/N)" # landlord_property_id = "Place ref" + # outcomes_filename = None + # outcomes_sheetname = None + # outcomes_postcode = None + # outcomes_houseno = None + # master_filepaths = [] + # master_to_asset_list_filepath = None # For ACIS - programme re-build # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/ACIS/ACIS Full Programme Review March 2025" @@ -393,7 +446,7 @@ def app(): # We now flag properties that have been treated under existing programmes asset_list.flag_outcomes( - outcomes_filepath=os.path.join(data_folder, outcomes_filename), + outcomes_filepath=os.path.join(data_folder, outcomes_filename) if outcomes_filename else None, outcomes_sheetname=outcomes_sheetname, outcomes_postcode=outcomes_postcode, outcomes_houseno=outcomes_houseno @@ -566,6 +619,22 @@ def app(): pprint(asset_list.work_type_figures) + # TODO: Characterise the properties that didn't qualify + eg = asset_list.standardised_asset_list[ + pd.isnull(asset_list.standardised_asset_list["solar_reason"]) + ] + eg[asset_list.EPC_API_DATA_NAMES["floor-description"]].value_counts() + + # TODO: Look into the estimated ones + eg["estimated"].value_counts() + + eg = eg[eg[asset_list.STANDARD_HEATING_SYSTEM] == "high heat retention storage heaters"] + eg[asset_list.STANDARD_WALL_CONSTRUCTION].value_counts() + eg = eg[eg[asset_list.STANDARD_WALL_CONSTRUCTION] == "filled cavity"] + eg[asset_list.EPC_API_DATA_NAMES["roof-description"]].value_counts() + eg[asset_list.EPC_API_DATA_NAMES["floor-description"]].value_counts() + eg["epc_has_floor_recommendation"].value_counts() + asset_list.flat_analysis() asset_list.load_contact_details( @@ -614,6 +683,7 @@ def app(): with pd.ExcelWriter(filename) as writer: asset_list.standardised_asset_list.to_excel(writer, sheet_name="Standardised Asset List", index=False) asset_list.flat_data.to_excel(writer, sheet_name="Flat Data", index=False) + # If we have outcomes, we add a tab with the outcomes # Store the Hubspot export as a csv hubspot_data.to_csv(os.path.join(data_folder, "Hubspot Export.csv"), index=False) diff --git a/asset_list/mappings/heating_systems.py b/asset_list/mappings/heating_systems.py index f397391c..73e2679e 100644 --- a/asset_list/mappings/heating_systems.py +++ b/asset_list/mappings/heating_systems.py @@ -95,5 +95,16 @@ HEATING_MAPPINGS = { 'Boiler Solid fuel': 'boiler - other fuel', 'Community heating Community (mains gas)': 'communal gas boiler', 'Boiler Biomass': 'boiler - other fuel', - 'No heating system Mains gas': 'unknown' + 'No heating system Mains gas': 'unknown', + + 'Storage heaters': 'electric storage heaters', + 'Air Source': 'air source heat pump', + 'Ground source': 'ground source heat pump', + 'OIl': 'boiler - other fuel', + 'Quantum storage heaters (old sh on EPC)': 'high heat retention storage heaters', + 'Quanum Storage heaters': 'high heat retention storage heaters', + 'Quantum storage heaters (Old SH on EPC)': 'high heat retention storage heaters', + 'Quantum storage heaters': 'high heat retention storage heaters', + 'Air Source (EPC says SH)': 'air source heat pump', + 'ASHP - Was logged as oil': 'air source heat pump' } diff --git a/asset_list/mappings/property_type.py b/asset_list/mappings/property_type.py index ccee5d3e..3182bd45 100644 --- a/asset_list/mappings/property_type.py +++ b/asset_list/mappings/property_type.py @@ -63,5 +63,13 @@ PROPERTY_MAPPING = { '2 Bed 1st Floor Sheltered Flat': 'flat', '1 Bed First Floor Flat': 'flat', '3 Bed First Floor Flat': 'flat', - 'ND': 'unknown' + 'ND': 'unknown', + 'House (Mid Terrace)': 'house', + 'First Floor Flat General': 'flat', + 'House (End Terrace)': 'house', + 'House (Mid terrace)': 'house', + 'Bungalow (Semi)': 'bungalow', + 'Ground Floor Flat General': 'flat', + 'House (Semi)': 'house' + } diff --git a/asset_list/mappings/walls.py b/asset_list/mappings/walls.py index 2313f063..89c97d7e 100644 --- a/asset_list/mappings/walls.py +++ b/asset_list/mappings/walls.py @@ -1,10 +1,14 @@ import numpy as np STANDARD_WALL_CONSTRUCTIONS = { + # Cavity "uninsulated cavity", "filled cavity", "partial insulated cavity", "cavity unknown insulation", + # Solic Brick "uninsulated solid brick", "insulated solid brick", "solid brick unknown insulation", - "timber frame", - "system built", "granite or whinstone", "other", "unknown", "sandstone or limestone", + # Timber Frame + "timber frame unknown insulation", "insulated timber frame", "uninsulated timber frame", + "system built", "granite or whinstone", "other", + "unknown", "sandstone or limestone", "cob", "new build - average thermal transmittance", } @@ -117,5 +121,18 @@ WALL_CONSTRUCTION_MAPPINGS = { 'Solid brick Internal': 'insulated solid brick', 'Cavity Internal': 'filled cavity', 'System build Internal': 'system built', - 'Solid brick As-built': 'solid brick unknown insulation' + 'Solid brick As-built': 'solid brick unknown insulation', + + 'Cavity ': 'cavity unknown insulation', + 'Solid brick ': 'solid brick unknown insulation', + 'Timber frame Timber frame (good insulation)': 'insulated timber frame', + ' ': 'unknown', + 'Cavity No data': 'cavity unknown insulation', + 'Non trad ': 'other', + 'Solid brick / Multiple Attributes ': 'solid brick unknown insulation', + 'Cavity Believe CWI done by Dyson': 'filled cavity', + 'Cavity CWI required': 'uninsulated cavity', + 'Solid brick EWI installed': 'insulated solid brick', + 'Cavity Cavity batts': 'filled cavity', + 'Cavity CWI Completed by Dyson': 'filled cavity' } diff --git a/etl/customers/mod/pilot/1. Create Sample.py b/etl/customers/mod/pilot/1. Create Sample.py new file mode 100644 index 00000000..e1f9b444 --- /dev/null +++ b/etl/customers/mod/pilot/1. Create Sample.py @@ -0,0 +1,34 @@ +import pandas as pd + + +def app(): + """ + Given the sample data and additonal properties, this function prepares the data + :return: + """ + folder_path = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/MOD/Pilot Programme" + sample_list = pd.read_excel(f"{folder_path}/20250227_DIO_Accommodation_Sample_Properties.xlsx") + asset_data = pd.read_excel(f"{folder_path}/20250303_DIO_Accommodation_Property_Attribution.xlsx") + asset_data["BLNDG_GOVERMENT_UPRN"] = asset_data["BLNDG_GOVERMENT_UPRN"].astype("Int64") + + asset_data["BLNDG_GOVERMENT_UPRN"].nunique() + for _id in asset_data["ESTB_ID"].unique(): + data = asset_data[asset_data["ESTB_ID"] == _id] + z = data["BLNDG_GOVERMENT_UPRN"] + + data["BLNDG_GOVERMENT_UPRN"].unique() + + asset_data["BLNDG_GOVERMENT_UPRN"].unique() + + df = asset_data.groupby("BLNDG_GOVERMENT_UPRN")["ESTB_ID"].nunique().sort_values(ascending=False).reset_index() + + example = asset_data[asset_data["BLNDG_GOVERMENT_UPRN"] == df.head(1)["BLNDG_GOVERMENT_UPRN"].values[0]] + + asset_data[asset_data["BLNDG_GOVERMENT_UPRN"]] + + asset_data = asset_data[asset_data["ESTB_ID"].isin(sample_list["ESTB_ID"].values)] + asset_data.drop_duplicates("ESTB_ID", inplace=True) + + [x for x in asset_data.columns if "uprn" in x.lower()] + + example = asset_data[asset_data["ESTB_ID"] == 1547072]