From 519dc6cfcb31ce4093ae0e6cace03ba30920e5e7 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 3 Apr 2024 19:17:27 +0100 Subject: [PATCH] added off-gas property recommendations --- backend/app/plan/router.py | 1 + etl/customers/gla_croydon_demo/asset_list.py | 42 +++- etl/customers/gla_croydon_demo/slides.py | 200 ++++++++++++++++++- recommendations/HeatingControlRecommender.py | 2 +- recommendations/HeatingRecommender.py | 12 +- 5 files changed, 247 insertions(+), 10 deletions(-) diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 50b8a837..4868749d 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -389,6 +389,7 @@ async def trigger_plan(body: PlanTriggerRequest): # Commit final changes session.commit() + except IntegrityError: logger.error("Database integrity error occurred", exc_info=True) session.rollback() diff --git a/etl/customers/gla_croydon_demo/asset_list.py b/etl/customers/gla_croydon_demo/asset_list.py index 3a3f02a3..52e9422c 100644 --- a/etl/customers/gla_croydon_demo/asset_list.py +++ b/etl/customers/gla_croydon_demo/asset_list.py @@ -4,6 +4,23 @@ from utils.s3 import save_csv_to_s3 USER_ID = 8 PORTFOLIO_ID = 67 +archetype_1_uprns = [100020604138, 200001188299, 100020578756, 200001187196, 200001192253, 100020581792, 200001188304, + 100020625813, 100020618060, 100020585305, 100020617489, 100020615039, 100020618076, 100020588913, + 200001187197, 100020671205, 100020576940, 100020619814, 100020576472, 100020618083] +archetype_2_uprns = [100020698027, 10001007455, 100020653785, 10090383198, 100020665632, 100020620659, 100020615603, + 100020609610, 100020625597, 100020665656, 100020665640, 100020587905, 100020665630, 100020624351, + 100020625451, 100020624348, 100020666735, 100020653786, 100020576458, 100020657902, 100020624350, + 100020637405, 100020666734, 100020616325, 100020666716, 100020653783, 100020665645, 100020642337, + 100020665638, 100022904981, 100020688226, 100020630285, 100020626800, 100020665634, 100022907528, + 100020665652, 100020624347, 100020666721, 100020585002, 10014055968, 10001008257, 100020621438, + 100020576459, 100020665643, 100020665654, 100022917303] +archetype_3_uprns = [100020577523, 100020616446, 100020605342, 100020594652, 100020585394, 100020601138, 100020597485, + 100020614883, 100020633162, 100020697787, 200001185785, 100020646842, 100020581449, 100020595611, + 100020641814, 100020575611, 100020652986, 100020654671, 100020647336, 100020610518, 100020607980, + 100020692380, 100020581690] +archetype_4_uprns = [100020650603, 100020582907, 100020605116, 100020650607, 100020589325, 100020655500, 100020642537, + 200001187539, 100020631683, 100020610165, 100020596436, 100020598277, 100020660228] + def app(): """ @@ -84,14 +101,15 @@ def app(): archetype_2_sample_asset_list["ARCHETYPE"] = "Archetype 2" # Archetype 3: defined below: - # 1) EPC F + # 1) EPC E or below # 2) Solid brick wall # 3) House # 4) Pitched roof with no insulation # Just 7 properties (more expensive to retrofit) archetype_3_sample = epc_data[ epc_data["PROPERTY_TYPE"].isin(["House"]) & - (epc_data["CURRENT_ENERGY_RATING"].isin(["F", "G"])) & + (epc_data["CURRENT_ENERGY_RATING"].isin(["E", "F", "G"])) & + epc_data["WALLS_DESCRIPTION"].isin(["Solid brick, as built, no insulation (assumed)"]) & epc_data["ROOF_DESCRIPTION"].isin( [ "Pitched, no insulation", @@ -119,7 +137,6 @@ def app(): archetype_4_sample_asset_list = archetype_4_sample[["UPRN", "ADDRESS1", "POSTCODE"]].copy() archetype_4_sample_asset_list["ARCHETYPE"] = "Archetype 4" - # 104 total properties asset_list = pd.concat( [ archetype_1_sample_asset_list, @@ -152,6 +169,25 @@ def app(): ] ) ] + # We have slightly too many properties, so we take a random sample of each archetype + # achetype_1_size = 20 + # achetype_2_size = 46 + # achetype_3_size = 23 + # achetype_4_size = 13 + # archetype_1_uprns = asset_list[asset_list["archetype"] == "Archetype 1"]["uprn"].sample( + # int(achetype_1_size) + # ).tolist() + # archetype_2_uprns = asset_list[asset_list["archetype"] == "Archetype 2"]["uprn"].sample( + # int(achetype_2_size) + # ).tolist() + # archetype_3_uprns = asset_list[asset_list["archetype"] == "Archetype 3"]["uprn"].sample( + # int(achetype_3_size) + # ).tolist() + # archetype_4_uprns = asset_list[asset_list["archetype"] == "Archetype 4"]["uprn"].sample( + # int(achetype_4_size) + # ).tolist() + uprns_to_keep = archetype_1_uprns + archetype_2_uprns + archetype_3_uprns + archetype_4_uprns + asset_list = asset_list[asset_list["uprn"].isin(uprns_to_keep)] filename = f"{USER_ID}/{PORTFOLIO_ID}/inputs.csv" save_csv_to_s3( diff --git a/etl/customers/gla_croydon_demo/slides.py b/etl/customers/gla_croydon_demo/slides.py index 1d217226..e6c4b5b8 100644 --- a/etl/customers/gla_croydon_demo/slides.py +++ b/etl/customers/gla_croydon_demo/slides.py @@ -27,8 +27,24 @@ SAP_TARGET_1 = 69 SAP_TARGET_2 = 100 CUSTOMER_KEY = "gla-demo" +# Sample UPRNS +archetype_1_sample = ['100020618076', '100020619814', '100020581792', '100020671205', '100020585305', '100020606853', + '100020625813', '100020618042', '200001188304', '200001187196', '100020603026', '100020604138', + '100020615039', '200001188299', '100020618060', '200001192253'] -def app(): +archetype_2_sample = ['100020616325', '100020665634', '100020665654', '100020665638', '100020587936', '100020587905', + '100020665645', '100020625597', '100022907528', '100020665630', '100020624348', '10001008257', + '100020666735', '100020698027', '100020624351', '100020665656', '100020666716', '100020665632', + '100020666715', '100020645639', '200001191309', '100020625451', '100020624347', '100020665658', + '100020585002', '100022917303', '100020665650', '100020667737', '100020620659', '100022904981', + '100020642337', '100020657902', '100020615603', '100020626800', '100020665647', '100020665643'] + +archetype_3_sample = ['100020607980', '200001193193', '100020581690', '100020665611'] +archetype_4_sample = ['100020631683', '100020607667', '100020660228', '100020605116', '200001187539', '100020582907', + '100020610165', '100020650607', '100020655500', '100020598277', '100020642537'] + + +def scenario_1(): # Connect to database session = sessionmaker(bind=db_engine)() @@ -36,9 +52,7 @@ def app(): # Get the data we need ######################################################################## - # TODO: Update to portfolio desired - # portfolio_id = PORTFOLIO_ID_1 - portfolio_id = PORTFOLIO_ID_2 + portfolio_id = PORTFOLIO_ID_1 # Get the asset list asset_list = read_csv_from_s3( @@ -157,3 +171,181 @@ def app(): # Overview ######################## overview_totals = recommendations_summary.sum() + + +def make_sample(): + # sample_proportion = 67 / 102 + # Get the asset list + asset_list = read_csv_from_s3( + "retrofit-plan-inputs-dev", f"{USER_ID}/67/inputs.csv" + ) + asset_list = pd.DataFrame(asset_list) + + # From the asset list, we deduce how many properties we need + archetype_1_sample_size = 16 + archetype_2_sample_size = 36 + archetype_3_sample_size = 4 + archetype_4_sample_size = 11 + + # We take the sample and we'll keep the uprns static + archetype_1_sample = asset_list[ + asset_list["archetype"] == "Archetype 1" + ].sample(archetype_1_sample_size)["uprn"].to_list() + + archetype_2_sample = asset_list[ + asset_list["archetype"] == "Archetype 2" + ].sample(archetype_2_sample_size)["uprn"].to_list() + + archetype_3_sample = asset_list[ + asset_list["archetype"] == "Archetype 3" + ].sample(archetype_3_sample_size)["uprn"].to_list() + + archetype_4_sample = asset_list[ + asset_list["archetype"] == "Archetype 4" + ].sample(archetype_4_sample_size)["uprn"].to_list() + + +def scenario_2(): + # Connect to database + session = sessionmaker(bind=db_engine)() + + ######################################################################## + # Get the data we need + ######################################################################## + + portfolio_id = PORTFOLIO_ID_2 + + # Get the asset list + asset_list = read_csv_from_s3( + "retrofit-plan-inputs-dev", f"{USER_ID}/67/inputs.csv" + ) + asset_list = pd.DataFrame(asset_list) + + sample_uprns = archetype_1_sample + archetype_2_sample + archetype_3_sample + archetype_4_sample + + # Filter on sample uprns + asset_list = asset_list[asset_list["uprn"].astype(str).isin(sample_uprns)] + + # Get the properties for the portfolio + properties = get_properties_with_default_recommendations(session, portfolio_id) + properties_df = pd.DataFrame(properties) + properties_df = properties_df[properties_df["uprn"].astype(str).isin(sample_uprns)] + + # We now pull the data for the property details + property_details = get_property_details_by_portfolio_id(session, portfolio_id) + property_details_df = pd.DataFrame(property_details) + property_details_df = property_details_df[property_details_df["property_id"].isin(properties_df["id"].values)] + # We estimate bills based on the adjusted_energy_consumption + property_details_df["energy_bill"] = property_details_df["adjusted_energy_consumption"].apply( + lambda x: AnnualBillSavings.calculate_annual_bill(x) + ) + # Merge on uprn + property_details_df = property_details_df.merge( + properties_df[["uprn", "id"]].rename(columns={"id": "property_id"}), + on="property_id" + ) + + plans = get_plan_by_portfolio_id(session, portfolio_id) + plans_df = pd.DataFrame(plans) + + # Unnest the recommendations. Each recommendation is a list of dictionaries + recommendations_exploded = properties_df["recommendations"].explode().tolist() + recommendations_df = pd.DataFrame([r for r in recommendations_exploded if not pd.isnull(r)]) + # Add uprn on + recommendations_df = recommendations_df.merge( + properties_df[["uprn", "id"]].rename(columns={"id": "property_id"}), + how="left", + on="property_id" + ) + + recommendations_summary = create_recommendations_summary( + recommendations_df, + properties_df, + property_details_df, + SAP_TARGET_1 + ) + + # Calculate % changes of energ, co2 and abs + recommendations_summary["carbon_percent_change"] = ( + recommendations_summary["total_carbon"] / recommendations_summary["current_co2"] + ) + + recommendations_summary["energy_percent_change"] = ( + recommendations_summary["adjusted_heat_demand"] / recommendations_summary["current_energy"] + ) + + recommendations_summary["bills_percent_change"] = ( + recommendations_summary["total_bill_savings"] / recommendations_summary["current_energy_bill"] + ) + + ######################## + # Overview + ######################## + overview_totals = recommendations_summary.sum() + overview_means = recommendations_summary.mean() + + ######################## + # Measures + ######################## + measures_count = recommendations_df.groupby("type")["id"].count().reset_index() + + z = recommendations_df[recommendations_df["uprn"].astype(str).isin(archetype_3_sample)] + + recommendations_df[recommendations_df["uprn"].astype(str).isin(archetype_3_sample)]["type"].value_counts() + + # Summary information by each archetype + ######################## + # Archetype 1 + ######################## + archetype_1 = asset_list[asset_list["archetype"] == "Archetype 1"] + recommendations_arch_1_summary = recommendations_summary[ + recommendations_summary["uprn"].astype(str).isin(archetype_1["uprn"].values) + ] + + # Take the mean, median and maximum of each value + arch_1_recommendation_min = recommendations_arch_1_summary.min() + arch_1_recommendation_max = recommendations_arch_1_summary.max() + arch_1_recommendation_means = recommendations_arch_1_summary.mean() + + ######################## + # Archetype 2 + ######################## + archetype_2 = asset_list[asset_list["archetype"] == "Archetype 2"] + recommendations_arch_2_summary = recommendations_summary[ + recommendations_summary["uprn"].astype(str).isin(archetype_2["uprn"].values) + ] + + # Take the mean, median and maximum of each value + arch_2_recommendation_min = recommendations_arch_2_summary.min() + arch_2_recommendation_max = recommendations_arch_2_summary.max() + arch_2_recommendation_means = recommendations_arch_2_summary.mean().round(2) + + ######################## + # Archetype 3 + ######################## + archetype_3 = asset_list[asset_list["archetype"] == "Archetype 3"] + recommendations_arch_3_summary = recommendations_summary[ + recommendations_summary["uprn"].astype(str).isin(archetype_3["uprn"].values) + ] + + # Take the mean, median and maximum of each value + arch_3_recommendation_min = recommendations_arch_3_summary.min() + arch_3_recommendation_max = recommendations_arch_3_summary.max() + arch_3_recommendation_means = recommendations_arch_3_summary.mean() + + ######################## + # Archetype 4 + ######################## + archetype_4 = asset_list[asset_list["archetype"] == "Archetype 4"] + recommendations_arch_4_summary = recommendations_summary[ + recommendations_summary["uprn"].astype(str).isin(archetype_4["uprn"].values) + ] + + # Take the mean, median and maximum of each value + arch_4_recommendation_min = recommendations_arch_4_summary.min() + arch_4_recommendation_max = recommendations_arch_4_summary.max() + arch_4_recommendation_means = recommendations_arch_4_summary.mean() + + property_details_df[ + property_details_df["uprn"].astype(str).isin(archetype_4["uprn"].values) + ]["total_floor_area"].mean() diff --git a/recommendations/HeatingControlRecommender.py b/recommendations/HeatingControlRecommender.py index 7010ad53..95b5e3b1 100644 --- a/recommendations/HeatingControlRecommender.py +++ b/recommendations/HeatingControlRecommender.py @@ -215,7 +215,7 @@ class HeatingControlRecommender: { "type": "heating_control", "parts": [], - "description": "Upgrade heating controls to Time and Temperature Zone Controls", + "description": "Upgrade heating controls to Smart Thermostats, room sensors and smart radiator valves", **self.costs.time_and_temperature_zone_control( number_heated_rooms=int(self.property.data["number-heated-rooms"]) ), diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index 9658aaa3..8b20c0cd 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -19,9 +19,17 @@ class HeatingRecommender: self.recommendations = [] # This first iteration of the recommender will provide very basic recommendation # We recommend heating controls based on the main heating system - if self.property.main_heating["clean_description"] in [ + + has_electric_heating_description = self.property.main_heating["clean_description"] in [ "Room heaters, electric", "Electric storage heaters", "Electric storage heaters, radiators" - ]: + ] + + no_heating_no_mains = ( + self.property.main_heating["clean_description"] in ["No system present, electric heaters assumed"] and + not self.property.data["mains-gas-flag"] + ) + + if has_electric_heating_description or no_heating_no_mains: # Recommend high heat retention storage heaters self.recommend_electric_storage_heaters(phase=phase, system_change=True, heating_controls_only=False) return