diff --git a/backend/diagnostics/portfolio_diagnostics.py b/backend/diagnostics/portfolio_diagnostics.py new file mode 100644 index 00000000..bcdec24e --- /dev/null +++ b/backend/diagnostics/portfolio_diagnostics.py @@ -0,0 +1,3 @@ +""" +This script is set up to perform broad portfolio diagnostics to identify potential issues +""" diff --git a/etl/customers/peabody/Nov 2025 Consulting Project/f_diagnostics.py b/etl/customers/peabody/Nov 2025 Consulting Project/f_diagnostics.py new file mode 100644 index 00000000..4c2a49ca --- /dev/null +++ b/etl/customers/peabody/Nov 2025 Consulting Project/f_diagnostics.py @@ -0,0 +1,63 @@ +""" +This script performs a deep dive into the various scenarios and checks fundamental things +This includes: +1) Do properties that should have a plan, have a plan? E.g. if the property is EPC D, and has a plan getting up to +# EPC C, there should be a plan +2) If the plan is fabric first, make sure they are actually fabric first +""" +import pandas as pd + +scenario_names = { + 871: "EPC C, fabric first, no solid floor, ashp 3.0", + 863: "EPC B, No EWI IWI, No Solid Floor, ASHP 3.0 COP", + 862: "EPC B, No solid floor, ASHP COP 3.0", + 861: "EPC C, No EWI IWI, No Solid Floor, ASHP 3.0 COP", + 859: "EPC C, no solid floor, ashp 3.0", +} + +scenario_sap_targets = { + 871: 69, + 863: 81, + 862: 81, + 861: 69, + 859: 69, +} + +for scenario_id, scenario_name in scenario_names.items(): + # Read in the recommended measures + + df = pd.read_excel( + f"/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting Project/" + f"{scenario_name}.xlsx" + ) + + # find properties that are below the scenario sap target, but have no recommended measures + df["below_scenario_target"] = df["current_sap_points"] < scenario_sap_targets[scenario_id] + df["no_recommended_measures"] = df["sap_points"] == 0 + + problematic_properties = df[ + df["below_scenario_target"] & df["no_recommended_measures"] + ] + + # show all columns + # Source - https://stackoverflow.com/a + # Posted by YOLO, modified by community. See post 'Timeline' for change history + # Retrieved 2026-01-06, License - CC BY-SA 4.0 + + pd.set_option('display.max_rows', 500) + pd.set_option('display.max_columns', 500) + pd.set_option('display.width', 1000) + problematic_properties.head(len(problematic_properties)) + + # + + plan_input = [ + { + "uprn": 100022725126, + "address": "FLAT 5 Daveys Court", + "postcode": "WC2N 4BW" + } + ] + +# Plan notes: +# UPRN: 5870109770, property ID: 281244 - need to delete and re-build all scenarios diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index b1f6205c..c7c5895d 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -167,9 +167,12 @@ class HeatingRecommender: hhr_suitable = no_mains or self.has_electric_heating_description or self.has_room_heaters + # If the property has community heating heaters in place, we don't recommend HHRSH + has_community_heating = self.property.main_fuel["is_community"] + hhr_suitable = hhr_suitable and ( "underfloor heating" not in self.property.main_heating["clean_description"] - ) + ) and not has_community_heating # If the property has a ground source heat pump, or air source heat pump, we don't recommend HHRSH diff --git a/recommendations/Recommendations.py b/recommendations/Recommendations.py index 0b3d1635..29ba267a 100644 --- a/recommendations/Recommendations.py +++ b/recommendations/Recommendations.py @@ -718,7 +718,8 @@ class Recommendations: ): # Handle the case of community schemes - if (heating_description == "Community scheme") or (hotwater_description == "Community scheme") and ( + if (heating_description in ["Community scheme", 'Community scheme, plus solar']) or ( + hotwater_description in ["Community scheme", 'Community scheme, plus solar']) and ( "not community" not in main_fuel_description ): if main_fuel_description in ["mains gas (community)", "UNKNOWN"]: @@ -742,6 +743,18 @@ class Recommendations: "heating_cop": 0.85, "hotwater_cop": 0.85 } + + # Handling specific case + if main_fuel_description in ["To be used only when there is no heating/hot-water system"] and ( + "electric heaters" in heating_description.lower() + ): + return { + "heating_fuel_type": "Electricity", + "hotwater_fuel_type": "Electricity", + "heating_cop": 1, + "hotwater_cop": 1 + } + logger.warning( "Unhandled community fuel." f"Fuel: {main_fuel_description}" diff --git a/sfr/principal_pitch/2_export_data.py b/sfr/principal_pitch/2_export_data.py index d05275ea..7574414c 100644 --- a/sfr/principal_pitch/2_export_data.py +++ b/sfr/principal_pitch/2_export_data.py @@ -11,8 +11,21 @@ from backend.app.db.models.portfolio import PropertyModel, PropertyDetailsEpcMod # PORTFOLIO_ID = 206 # SCENARIOS = [389] -PORTFOLIO_ID = 404 -SCENARIOS = [829] +PORTFOLIO_ID = 419 # Peabody +SCENARIOS = [ + 871, # EPC C - fabric first, no solid floor, ashp 3.0 + 863, # EPC B, No EWI/IWI, No Solid Floor, ASHP 3.0 COP + 862, # EPC B - No solid floor, ASHP COP 3.0 + 861, # EPC C, No EWI/IWI, No Solid Floor, ASHP 3.0 COP + 859, # EPC C - no solid floor, ashp 3.0 +] +scenario_names = { + 871: "EPC C, fabric first, no solid floor, ashp 3.0", + 863: "EPC B, No EWI IWI, No Solid Floor, ASHP 3.0 COP", + 862: "EPC B, No solid floor, ASHP COP 3.0", + 861: "EPC C, No EWI IWI, No Solid Floor, ASHP 3.0 COP", + 859: "EPC C, no solid floor, ashp 3.0", +} def get_data(portfolio_id, scenario_ids): @@ -84,88 +97,96 @@ properties_df = pd.DataFrame(properties_data) plans_df = pd.DataFrame(plans_data) recommendations_df = pd.DataFrame(recommendations_data) -recommended_measures_df = recommendations_df[ - ["property_id", "measure_type", "estimated_cost", "default"] -] -recommended_measures_df = recommended_measures_df[recommended_measures_df["default"]] -recommended_measures_df = recommended_measures_df.drop(columns=["default"]) - -post_install_sap = recommendations_df[["property_id", "default", "sap_points"]] -post_install_sap = post_install_sap[post_install_sap["default"]] -# Sum up the sap points by property id -post_install_sap = post_install_sap.groupby("property_id")[["sap_points"]].sum().reset_index() - -# Find dupes by property id and measure type -dupes = recommended_measures_df.duplicated( - subset=["property_id", "measure_type"], keep=False -) -dupe_df = recommended_measures_df[dupes] - -if dupe_df.shape: - # Drop dupes - happened due to a funny bug - recommended_measures_df = recommended_measures_df.drop_duplicates( - subset=["property_id", "measure_type"], keep='first' - ) - -recommendations_measures_pivot = recommended_measures_df.pivot( - index='property_id', - columns='measure_type', - values='estimated_cost' -) -recommendations_measures_pivot = recommendations_measures_pivot.reset_index() - -# Total cost is the row sum, excluding the property_id column -recommendations_measures_pivot["total_retrofit_cost"] = recommendations_measures_pivot.drop( - columns=["property_id"] -).sum(axis=1) - -df = properties_df[ - [ - "landlord_property_id", "property_id", "uprn", "address", "postcode", "property_type", "walls", "roof", - "heating", "windows", - "current_epc_rating", - "current_sap_points", "total_floor_area", "number_of_rooms", - ] -].merge( - recommendations_measures_pivot, how="left", on="property_id" -).merge( - post_install_sap, how="left", on="property_id" -) - -df = df.drop(columns=["property_id"]) -df["sap_points"] = df["sap_points"].fillna(0) - -df["predicted_post_works_sap"] = df["current_sap_points"] + df["sap_points"] -df["predicted_post_works_sap"] = df["predicted_post_works_sap"].round() -df["predicted_post_works_epc"] = df["predicted_post_works_sap"].apply(lambda x: sap_to_epc(x)) - -# We merge this back to the main dataframe, which will contain the bathrooms from utils.s3 import read_csv_from_s3, read_excel_from_s3 -# asset_list = read_csv_from_s3(bucket_name="retrofit-plan-inputs-dev", filepath='8/206/asset_list.csv') -asset_list = read_excel_from_s3( - bucket_name="retrofit-plan-inputs-dev", file_key="2/404/20251211T163200754Z/asset_list.xlsx", - header_row=0, sheet_name="Standardised Asset List" -) -asset_list = pd.DataFrame(asset_list) -asset_list = asset_list.rename( - columns={ - "postcode": "domna_postcode" - } -) -if "domna_full_address": - # For Peabody - asset_list["domna_full_address"] = asset_list["domna_address_1"] +# asset_list = read_excel_from_s3( +# bucket_name="retrofit-plan-inputs-dev", file_key="2/404/20251211T163200754Z/asset_list.xlsx", +# header_row=0, sheet_name="Standardised Asset List" +# ) -asset_list = asset_list[["domna_full_address", "domna_postcode", "epc_os_uprn", ]].copy() -asset_list = asset_list.rename(columns={"epc_os_uprn": "uprn"}) -df["uprn"] = df["uprn"].astype(str) -asset_list["uprn"] = asset_list["uprn"].astype("Int64").astype(str) -asset_list = asset_list.merge( - df.drop(columns=["address", "postcode", "property_type", "total_floor_area"]), - how="left", - on="uprn" -) + +for scenario_id in SCENARIOS: + # Get recs for this scenario + recommended_measures_df = recommendations_df[recommendations_df["Scenario ID"] == scenario_id][ + ["property_id", "measure_type", "estimated_cost", "default"] + ] + recommended_measures_df = recommended_measures_df[recommended_measures_df["default"]] + recommended_measures_df = recommended_measures_df.drop(columns=["default"]) + + post_install_sap = recommendations_df[recommendations_df["Scenario ID"] == scenario_id][ + ["property_id", "default", "sap_points"]] + post_install_sap = post_install_sap[post_install_sap["default"]] + # Sum up the sap points by property id + post_install_sap = post_install_sap.groupby(["property_id"])[["sap_points"]].sum().reset_index() + + # Find dupes by property id and measure type + dupes = recommended_measures_df.duplicated(subset=["property_id", "measure_type"], keep=False) + dupe_df = recommended_measures_df[dupes] + + if dupe_df.shape: + # Drop dupes - happened due to a funny bug + recommended_measures_df = recommended_measures_df.drop_duplicates( + subset=["property_id", "measure_type"], keep='first' + ) + + recommendations_measures_pivot = recommended_measures_df.pivot( + index='property_id', + columns='measure_type', + values='estimated_cost' + ) + recommendations_measures_pivot = recommendations_measures_pivot.reset_index() + + # Total cost is the row sum, excluding the property_id column + recommendations_measures_pivot["total_retrofit_cost"] = recommendations_measures_pivot.drop( + columns=["property_id"] + ).sum(axis=1) + + df = properties_df[ + [ + "landlord_property_id", "property_id", "uprn", "address", "postcode", "property_type", "walls", "roof", + "heating", "windows", + "current_epc_rating", + "current_sap_points", "total_floor_area", "number_of_rooms", + ] + ].merge( + recommendations_measures_pivot, how="left", on="property_id" + ).merge( + post_install_sap, how="left", on="property_id" + ) + + df = df.drop(columns=["property_id"]) + df["sap_points"] = df["sap_points"].fillna(0) + + df["predicted_post_works_sap"] = df["current_sap_points"] + df["sap_points"] + df["predicted_post_works_sap"] = df["predicted_post_works_sap"].round() + df["predicted_post_works_epc"] = df["predicted_post_works_sap"].apply(lambda x: sap_to_epc(x)) + df["uprn"] = df["uprn"].astype(str) + + # Create excel to store to + filename = ("/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting " + f"Project/{scenario_names[scenario_id]}.xlsx") + with pd.ExcelWriter(filename) as writer: + df.to_excel(writer, sheet_name="properties", index=False) + + +# asset_list = pd.DataFrame(asset_list) +# asset_list = asset_list.rename( +# columns={ +# "postcode": "domna_postcode" +# } +# ) +# if "domna_full_address": +# # For Peabody +# asset_list["domna_full_address"] = asset_list["domna_address_1"] +# +# asset_list = asset_list[["domna_full_address", "domna_postcode", "epc_os_uprn", ]].copy() +# asset_list = asset_list.rename(columns={"epc_os_uprn": "uprn"}) +# asset_list["uprn"] = asset_list["uprn"].astype("Int64").astype(str) +# asset_list = asset_list.merge( +# df.drop(columns=["address", "postcode", "property_type", "total_floor_area"]), +# how="left", +# on="uprn" +# ) # Get conservation area data from property details spatial. based on the UPRNs