From aa391966efe99d0697277923ebe9f9d872ae78d3 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 9 Aug 2024 15:15:26 +0100 Subject: [PATCH] recommendation fuel wip --- backend/app/plan/router.py | 125 +++++++++++++++---------------------- backend/ml_models/api.py | 7 ++- 2 files changed, 56 insertions(+), 76 deletions(-) diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index fbdc2323..e75e65a1 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -444,14 +444,17 @@ async def trigger_plan(body: PlanTriggerRequest): kwh_client = KwhData(bucket=get_settings().DATA_BUCKET, read_consumption_data=True) - model_api = ModelApi(portfolio_id=body.portfolio_id, timestamp=created_at) + model_api = ModelApi( + portfolio_id=body.portfolio_id, + timestamp=created_at, + prediction_buckets=get_prediction_buckets() + ) epcs_for_scoring = kwh_client.transform(data=kwh_client.prepare_epc(input_properties), cleaned=cleaned) kwh_preds = model_api.predict_all( df=epcs_for_scoring, bucket=get_settings().DATA_BUCKET, - prediction_buckets=get_prediction_buckets(), model_prefixes=["heating_kwh_predictions", "hotwater_kwh_predictions"], extract_ids=False ) @@ -687,11 +690,57 @@ async def trigger_plan(body: PlanTriggerRequest): kwh_simulation_predictions = model_api.predict_all( df=scoring_epcs, bucket=get_settings().DATA_BUCKET, - prediction_buckets=get_prediction_buckets(), model_prefixes=["heating_kwh_predictions", "hotwater_kwh_predictions"], ) # TODO: Costing model, which should include today's costs! + # We now insert into the recommendations + for property_id in recommendations.keys(): + property_recommendations = recommendations[property_id] + property_instance = [p for p in input_properties if p.id == property_id][0] + + kwh_impact_table = kwh_simulation_predictions["heating_kwh_predictions"][ + kwh_simulation_predictions["heating_kwh_predictions"]["property_id"] == str(property_id) + ].merge( + kwh_simulation_predictions["hotwater_kwh_predictions"].drop( + columns=["property_id", "recommendation_id", "phase"] + ), + how="inner", + on="id", + suffixes=("_heating", "_hotwater") + ) + + property_kwh = property_instance.energy_consumption_estimates["unadjusted"] + + kwh_impact_table = pd.concat( + [ + pd.DataFrame( + [ + { + "id": None, + "predictions_heating": property_kwh["heating"], + "predictions_hotwater": property_kwh["hot_water"], + } + ] + ), + kwh_impact_table + ] + ) + # We adjust the predictions with the UCL model + for k in ["heating", "hotwater"]: + kwh_impact_table[f"adjusted_{k}"] = kwh_impact_table[f"predictions_{k}"].apply( + lambda x: AnnualBillSavings.adjust_energy_to_metered( + epc_energy=x, current_epc_rating=property_instance.data["current-energy-rating"] + ) + ) + + kwh_impact_table["heating_fuel"] = property_instance.heating_energy_source + kwh_impact_table["hotwater_fuel"] = property_instance.hot_water_energy_source + + # We now deduce if any of the recommendations result in a change of fuel type + for recs in property_recommendations: + for rec in recs: + print(rec["description_simulation"]) # Insert the predictions into the recommendations and run the optimiser # TODO: If a recommendation has a negative impact on SAP, we should remove it - this seems to have become a @@ -754,70 +803,6 @@ async def trigger_plan(body: PlanTriggerRequest): ] recommendations[p.id] = final_recommendations - # We now insert into the recommendations - for property_id in recommendations.keys(): - property_recommendations = recommendations[property_id] - property_instance = [p for p in input_properties if p.id == property_id][0] - # The predicted kwhs are without appliances - consumption = property_instance.energy_consumption_estimates["adjusted"] - # Starting consumption is the sum of the consumption values, without appliances - starting_heating = consumption["heating"] - starting_hotwater = consumption["hot_water"] - property_kwh_predictions = { - k: kwh_simulation_predictions[k][kwh_simulation_predictions[k]["property_id"] == str(property_id)] - for k in ['heating_kwh_predictions', 'hotwater_kwh_predictions'] - } - # We adjust the predictions - from backend.ml_models.AnnualBillSavings import AnnualBillSavings - for k in ["heating_kwh_predictions", "hotwater_kwh_predictions"]: - property_kwh_predictions[k]["adjusted"] = property_kwh_predictions[k]["predictions"].apply( - lambda x: AnnualBillSavings.adjust_energy_to_metered( - epc_energy=x, current_epc_rating=property_instance.data["current-energy-rating"] - ) - ) - - # For each recommendation, we difference the predictions - property_kwh_predictions["heating_kwh_predictions"]["savings"] = np.diff( - property_kwh_predictions["heating_kwh_predictions"]["adjusted"], prepend=starting_heating - ) - property_kwh_predictions["hotwater_kwh_predictions"]["savings"] = np.diff( - property_kwh_predictions["hotwater_kwh_predictions"]["adjusted"], prepend=starting_hotwater - ) - - for recommendations_by_type in property_recommendations: - for rec in recommendations_by_type: - # In the case of mechanical ventilation, there is no impact, and for low energy lighting we - # calculate the savings inside of the recommendation itself - if rec["type"] in ["mechanical_ventilation", "low_energy_lighing"]: - continue - - heating_kwh_savings = property_kwh_predictions["heating_kwh_predictions"][ - ( - property_kwh_predictions["heating_kwh_predictions"]["recommendation_id"] == - rec["recommendation_id"] - ) - ]["savings"].values[0] - # This should be negative - if heating_kwh_savings > 0: - print("Positive heating kwh savings") - # TODO: Raise an exception to investigate - # raise Exception("Positive heating kwh savings") - - hot_water_kwh_savings = property_kwh_predictions["hotwater_kwh_predictions"][ - ( - property_kwh_predictions["hotwater_kwh_predictions"]["recommendation_id"] == - rec["recommendation_id"] - ) - ]["savings"].values[0] - - # This should be negative - if hot_water_kwh_savings > 0: - print("Positive hot water kwh savings") - # TODO: Raise an exception to investigate - # raise Exception("Positive hot water kwh savings") - - rec["kwh_savings"] = abs(heating_kwh_savings + hot_water_kwh_savings) - # 1) the property data # 2) the property details (epc) # 3) the recommendations @@ -1154,12 +1139,6 @@ async def build_mds(body: MdsRequest): for chunk in tqdm(to_loop_over, total=len(to_loop_over)): predictions_dict = model_api.predict_all( df=recommendations_scoring_data.iloc[chunk:chunk + SCORING_BATCH_SIZE], - bucket=get_settings().DATA_BUCKET, - prediction_buckets={ - "sap_change_predictions": get_settings().SAP_PREDICTIONS_BUCKET, - "heat_demand_predictions": get_settings().HEAT_PREDICTIONS_BUCKET, - "carbon_change_predictions": get_settings().CARBON_PREDICTIONS_BUCKET - } ) # Append the predictions to the predictions dictionary diff --git a/backend/ml_models/api.py b/backend/ml_models/api.py index c401e0f4..fab28e89 100644 --- a/backend/ml_models/api.py +++ b/backend/ml_models/api.py @@ -32,6 +32,7 @@ class ModelApi: self, portfolio_id, timestamp, + prediction_buckets, base_url="https://api.dev.hestia.homes", ): """ @@ -46,6 +47,7 @@ class ModelApi: self.base_url = base_url self.portfolio_id = portfolio_id self.timestamp = timestamp + self.prediction_buckets = prediction_buckets @staticmethod def predictions_template(): @@ -125,7 +127,7 @@ class ModelApi: else: return None - def predict_all(self, df, bucket, prediction_buckets, model_prefixes=None, extract_ids=True) -> dict: + def predict_all(self, df, bucket, model_prefixes=None, extract_ids=True) -> dict: """ For each model prefix, this method will upload the scoring data to s3 and then make a request to the @@ -134,7 +136,6 @@ class ModelApi: a dictionary of panaas dataframes :param df: Pandas dataframe with scoring data to be uploaded to s3 :param bucket: Name of the bucket in s3 to upload to - :param prediction_buckets: Dictionary containing the prediction buckets for each model prefix :param model_prefixes: List of model prefixes to generate predictions for. If None, all model prefixes will be used :param extract_ids: Boolean to determine if the property_id and recommendation_id should be extracted from the @@ -152,7 +153,7 @@ class ModelApi: "s3://{DATA_BUCKET}/".format(DATA_BUCKET=bucket) + file_location, model_prefix ) - predictions_bucket = prediction_buckets[model_prefix] + predictions_bucket = self.prediction_buckets[model_prefix] # Retrieve the predictions predictions_df = pd.DataFrame(