From d97a91eec7feb60d38c65f93c0a7fb4eb0cd0d25 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 28 Nov 2023 22:32:12 +0000 Subject: [PATCH] Put in placeholder method to break down carbon and energy savings --- .../app/db/functions/portfolio_functions.py | 9 +- backend/app/plan/router.py | 175 ++++++++++++++++-- backend/ml_models/Valuation.py | 35 +--- recommendations/LightingRecommendations.py | 2 +- 4 files changed, 170 insertions(+), 51 deletions(-) diff --git a/backend/app/db/functions/portfolio_functions.py b/backend/app/db/functions/portfolio_functions.py index b3936eb9..a8a882bd 100644 --- a/backend/app/db/functions/portfolio_functions.py +++ b/backend/app/db/functions/portfolio_functions.py @@ -3,7 +3,9 @@ from backend.app.db.models.recommendations import Plan, PlanRecommendations, Rec from backend.app.db.models.portfolio import Portfolio -def aggregate_portfolio_recommendations(session, portfolio_id: int, total_valuation_increase: float): +def aggregate_portfolio_recommendations( + session, portfolio_id: int, total_valuation_increase: float, labour_days: float +): # Aggregate multiple fields aggregates = ( session.query( @@ -12,7 +14,6 @@ def aggregate_portfolio_recommendations(session, portfolio_id: int, total_valuat func.sum(Recommendation.heat_demand).label("energy_savings"), func.sum(Recommendation.co2_equivalent_savings).label("co2_equivalent_savings"), func.sum(Recommendation.energy_cost_savings).label("energy_cost_savings"), - func.sum(Recommendation.labour_days).label("labour_days"), ) .join(PlanRecommendations, PlanRecommendations.recommendation_id == Recommendation.id) .join(Plan, Plan.id == PlanRecommendations.plan_id) @@ -26,7 +27,6 @@ def aggregate_portfolio_recommendations(session, portfolio_id: int, total_valuat "energy_savings": aggregates.energy_savings or 0, "co2_equivalent_savings": aggregates.co2_equivalent_savings or 0, "energy_cost_savings": aggregates.energy_cost_savings or 0, - "labour_days": aggregates.labour_days or 0, } # Get the portfolio and update the fields @@ -35,8 +35,9 @@ def aggregate_portfolio_recommendations(session, portfolio_id: int, total_valuat for key, value in aggregates_dict.items(): setattr(portfolio, key, value) - # Insert total valuation increase + # Insert total valuation increase and labour days portfolio.property_valuation_increase = total_valuation_increase + portfolio.labour_days = labour_days # Merge the updated portfolio back into the session session.merge(portfolio) diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 7a50ac98..0c086a87 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -1,5 +1,6 @@ from datetime import datetime +import numpy as np import pandas as pd from epc_api.client import EpcClient from fastapi import APIRouter, Depends @@ -34,6 +35,7 @@ from recommendations.Recommendations import Recommendations from utils.logger import setup_logger from utils.s3 import read_dataframe_from_s3_parquet from backend.ml_models.Valuation import PropertyValuation +from backend.ml_models.AnnualBillSavings import AnnualBillSavings logger = setup_logger() @@ -118,6 +120,7 @@ async def trigger_plan(body: PlanTriggerRequest): recommendations = {} recommendations_scoring_data = [] + property_scoring_data = {} for p in input_properties: @@ -156,6 +159,12 @@ async def trigger_plan(body: PlanTriggerRequest): # We update the ending record with the recommended updates and we set lodgement date to today ending_epc_data["DAYS_TO_ENDING"] = data_processor.calculate_days_to(created_at) + property_scoring_data[p.id] = { + "starting_epc_data": starting_epc_data, + "ending_epc_data": ending_epc_data, + "fixed_data": fixed_data + } + for recommendations_by_type in property_recommendations: for i, rec in enumerate(recommendations_by_type): scoring_dict = create_recommendation_scoring_data( @@ -219,22 +228,6 @@ async def trigger_plan(body: PlanTriggerRequest): recommendations=recommendations ) - print("GET RID OF ME!") - rec_df = [] - for rec_group in recommendations_with_impact: - for rec in rec_group: - rec_df.append( - { - "type": rec["type"], - "description": rec["description"], - "sap": rec["sap_points"], - "total": rec["total"], - "co2_equivalent_savings": rec["co2_equivalent_savings"], - "heat_demand": rec["heat_demand"] - } - ) - rec_df = pd.DataFrame(rec_df) - input_measures = prepare_input_measures(recommendations_with_impact, body.goal) if body.budget: @@ -270,6 +263,148 @@ async def trigger_plan(body: PlanTriggerRequest): ] recommendations[property_id] = final_recommendations + # This is a temporary step, to estimate the impact of the measured on heat demand and carbon + combined_recommendations_scoring_data = [] + representative_recs = {} + for property_id, property_recommendations in recommendations.items(): + default_recommendations = [r for r in property_recommendations if r["default"]] + default_types = {x["type"] for x in default_recommendations} + + # Missing types + missing_types = list(set([r["type"] for r in property_recommendations if r["type"] not in default_types])) + if missing_types: + for missed_type in missing_types: + missed = [r for r in property_recommendations if r["type"] == missed_type] + median_cost = np.median([r["total"] for r in missed]) + # Grab a representative, based on median cost + representative_rec = [r for r in property_recommendations if r["total"] == median_cost] + default_recommendations.append(representative_rec[0]) + + representative_recs[property_id] = default_recommendations + + property_instance = [p for p in input_properties if p.id == property_id][0] + + property_scoring_datasets = property_scoring_data[property_id] + starting_epc_data = property_scoring_datasets["starting_epc_data"].copy() + ending_epc_data = property_scoring_datasets["ending_epc_data"].copy() + fixed_data = property_scoring_datasets["fixed_data"].copy() + + scoring_dict = {} + for rec in default_recommendations: + scoring_dict = create_recommendation_scoring_data( + property=property_instance, + recommendation=rec, + starting_epc_data=starting_epc_data, + ending_epc_data=ending_epc_data, + fixed_data=fixed_data, + ) + # At each iteration, we want to update the ending_epc_data, so in the end, ending_epc_data contains + # all of the updates + for k in scoring_dict.keys(): + if k in ending_epc_data.columns: + ending_epc_data[k] = scoring_dict[k] + + combined_recommendations_scoring_data.append(scoring_dict) + + # PERFORM SAME STEPS AGAIN - TODO: TO BE REMOVED + combined_recommendations_scoring_data = pd.DataFrame(combined_recommendations_scoring_data) + + # Perform the same cleaning as in the model - first clean number of room variables though + combined_recommendations_scoring_data = DataProcessor.apply_averages_cleaning( + data_to_clean=combined_recommendations_scoring_data, + cleaning_data=cleaning_data, + cols_to_merge_on=['PROPERTY_TYPE', 'BUILT_FORM', 'CONSTRUCTION_AGE_BAND', 'LOCAL_AUTHORITY'], + colnames=["NUMBER_HABITABLE_ROOMS", "NUMBER_HEATED_ROOMS"], + ) + + combined_recommendations_scoring_data = DataProcessor.apply_averages_cleaning( + data_to_clean=combined_recommendations_scoring_data, + cleaning_data=cleaning_data, + cols_to_merge_on=COLUMNS_TO_MERGE_ON + ["LOCAL_AUTHORITY"], + ).drop(columns=["LOCAL_AUTHORITY"]) + + combined_recommendations_scoring_data = DataProcessor.clean_missings_after_description_process( + combined_recommendations_scoring_data, + ignore_cols=[c for c in combined_recommendations_scoring_data.columns if ("thermal_transmittance" in c) or ( + "insulation_thickness" in c) or ("ENERGY_EFF" in c)] + ) + + combined_recommendations_scoring_data = DataProcessor.clean_efficiency_variables( + combined_recommendations_scoring_data + ) + + model_api = ModelApi(portfolio_id=body.portfolio_id, timestamp=created_at) + all_combined_predictions = model_api.predict_all( + df=combined_recommendations_scoring_data, + bucket=get_settings().DATA_BUCKET, + prediction_buckets={ + "sap_change_predictions": get_settings().SAP_PREDICTIONS_BUCKET, + "heat_demand_predictions": get_settings().HEAT_PREDICTIONS_BUCKET, + "carbon_change_predictions": get_settings().CARBON_PREDICTIONS_BUCKET + } + ) + + # We update the carbon and heat demand predictions + for property_id, property_recommendations in recommendations.items(): + combined_heat_demand = all_combined_predictions["heat_demand_predictions"] + combined_heat_demand = combined_heat_demand[combined_heat_demand["property_id"] == str(property_id)] + + combined_carbon = all_combined_predictions["carbon_change_predictions"] + combined_carbon = combined_carbon[combined_carbon["property_id"] == str(property_id)] + + property_instance = [p for p in input_properties if p.id == property_id][0] + + carbon_change = float( + property_instance.data["co2-emissions-current"] + ) - combined_carbon["predictions"].values[0] + + heat_demand_change = ( + (float(property_instance.data["energy-consumption-current"]) - + combined_heat_demand["predictions"].values[0]) + * property_instance.floor_area + ) + + # update the recommendations + # We need to totals for the representative recommendations + representative_rec_data = [ + { + "recommendation_id": r["recommendation_id"], + "co2_equivalent_savings": r["co2_equivalent_savings"], + "heat_demand": r["heat_demand"], + "type": r["type"] + } for r + in representative_recs[property_id] + ] + representative_rec_data = pd.DataFrame(representative_rec_data) + # Convert co2 and heat demand to proportions of their column sums + representative_rec_data["co2_equivalent_savings_percent"] = ( + representative_rec_data["co2_equivalent_savings"] / + representative_rec_data["co2_equivalent_savings"].sum() + ) + + representative_rec_data["heat_demand_percent"] = ( + representative_rec_data["heat_demand"] / representative_rec_data["heat_demand"].sum() + ) + + # We'll use the proportions to update the carbon and heat demand + representative_rec_data["co2_equivalent_savings"] = ( + carbon_change * representative_rec_data["co2_equivalent_savings_percent"] + ) + + representative_rec_data["heat_demand"] = ( + heat_demand_change * representative_rec_data["heat_demand_percent"] + ) + + # Finally, insert these values into the final recommendations + for rec in property_recommendations: + change_data = representative_rec_data[representative_rec_data["type"] == rec["type"]] + rec["co2_equivalent_savings"] = change_data["co2_equivalent_savings"].values[0] + rec["heat_demand"] = change_data["heat_demand"].values[0] + rec["energy_cost_savings"] = AnnualBillSavings.estimate(rec["heat_demand"]) + + # Update recommendations + recommendations[property_id] = property_recommendations + # 1) the property data # 2) the property details (epc) # 3) the recommendations @@ -342,9 +477,15 @@ async def trigger_plan(body: PlanTriggerRequest): # the portfolion level impact total_valuation_increase = sum(property_valuation_increases) + labour_days = max( + [sum(r["labour_days"] for r in rec_group if r["default"]) for p_id, rec_group in recommendations.items()] + ) aggregate_portfolio_recommendations( - session, portfolio_id=body.portfolio_id, total_valuation_increase=total_valuation_increase + session, + portfolio_id=body.portfolio_id, + total_valuation_increase=total_valuation_increase, + labour_days=labour_days ) # Commit final changes diff --git a/backend/ml_models/Valuation.py b/backend/ml_models/Valuation.py index 4a720fc8..0db2a082 100644 --- a/backend/ml_models/Valuation.py +++ b/backend/ml_models/Valuation.py @@ -4,42 +4,19 @@ class PropertyValuation: """ UPRN_VALUE_LOOKUP = { - 15038202: 202000, - 37024763: 213000, + 15038202: {"current_value": 202000, "increase_percentage": 0.05725}, + 37024763: {"current_value": 213000, "increase_percentage": 0.03625}, } - VALUE_INCREASE_MAPPING = [ - { - "starting_epc": "D", - "ending_epc": "C", - "increase_percentage": 0.03625, - }, - { - "starting_epc": "D", - "ending_epc": "B", - "increase_percentage": 0.05725, - }, - ] - @classmethod def estimate(cls, property_instance, target_epc): - current_value = cls.UPRN_VALUE_LOOKUP.get(property_instance.uprn) + data = cls.UPRN_VALUE_LOOKUP.get(property_instance.uprn) - raise ValueError("NEED TO UPDATE THIS") - - if not current_value: + if not data: raise ValueError("Have not implemented valuation for this property") - valuation_increases = [ - v for v in cls.VALUE_INCREASE_MAPPING if - v["starting_epc"] == property_instance.data["current-energy-rating"] and v["ending_epc"] == target_epc - ] + new_valuation = (1 + data["increase_percentage"]) * data["current_value"] - if len(valuation_increases) != 1: - raise ValueError("Valuation increase mapping not found") - - new_valuation = (1 + valuation_increases[0]["increase_percentage"]) * current_value - - increase = round(new_valuation - current_value, 2) + increase = round(new_valuation - data["current_value"], 2) return increase diff --git a/recommendations/LightingRecommendations.py b/recommendations/LightingRecommendations.py index f898eaf5..82baf5d6 100644 --- a/recommendations/LightingRecommendations.py +++ b/recommendations/LightingRecommendations.py @@ -54,7 +54,7 @@ class LightingRecommendations: ) if number_non_lel_outlets == 1: - description = "Install low energy lighting in %s 1 remaining outlet" + description = "Install low energy lighting in 1 remaining outlet" else: description = "Install low energy lighting in %s outlets" % str(number_non_lel_outlets)