From 7ff3d8799991f1f1a46344c4d4b5c3dfba820e77 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 16 Feb 2024 17:04:43 +0000 Subject: [PATCH] Trying and failing to implement the newest model --- backend/Property.py | 2 ++ backend/app/plan/router.py | 2 ++ etl/epc/generate_scenarios_data.py | 42 ++++++++++++++-------- etl/testing_data/sap_model_simulation.py | 46 +++++++++++++++++------- 4 files changed, 66 insertions(+), 26 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index 75cfaef1..418a35a1 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -230,6 +230,8 @@ class Property: # The upgrade made here is to the u-value of the walls and the description of the # insulation thickness output["walls_thermal_transmittance_ending"] = recommendation["new_u_value"] + # Setting the insulation thickness here to above average should be tested further because we + # don't see a high volume of instances for this output["walls_insulation_thickness_ending"] = "above average" output["walls_energy_eff_ending"] = "Good" diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 28dcaa68..07fdbe94 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -160,6 +160,8 @@ async def trigger_plan(body: PlanTriggerRequest): recommendations_scoring_data.extend(p.recommendations_scoring_data) + # TODO: Make sure that number_habitable_rooms has been dropped + logger.info("Preparing data for scoring in sap change api") recommendations_scoring_data = pd.DataFrame(recommendations_scoring_data) recommendations_scoring_data = recommendations_scoring_data.drop( diff --git a/etl/epc/generate_scenarios_data.py b/etl/epc/generate_scenarios_data.py index af3ed4a4..afe9ab98 100644 --- a/etl/epc/generate_scenarios_data.py +++ b/etl/epc/generate_scenarios_data.py @@ -50,9 +50,9 @@ scenario_properties = [ "postcode": "NN1 5JY", "lmk-key": "1459796789102016070507274146560098", "measures": [ - [["internal_wall_insulation"], "11", None], - [["external_wall_insulation"], "10", None], - [["solar", "windows"], "12-15", {"photo_supply_ending": 50}], + [["internal_wall_insulation"], "11", None, [0]], + [["external_wall_insulation"], "10", None, [0]], + [["solar", "windows"], "12-15", {"photo_supply_ending": 50}, [0, 1]], ], }, { @@ -60,7 +60,7 @@ scenario_properties = [ "postcode": "HP1 2HA", "lmk-key": "c14029235739827d5f627dc8aa9bb567d026b267e851e0db0001db24638667b1", "measures": [ - [["cavity_wall_insulation", "loft_insulation"], "15", None], + [["cavity_wall_insulation", "loft_insulation"], "15", None, [0, 1]], ], }, { @@ -68,7 +68,7 @@ scenario_properties = [ "postcode": "HP1 2HE", "lmk-key": "99296a6dda21314fef3a61cda59e441e9a2aacf115eb96f4a0fa85696bf7b117", "measures": [ - [["cavity_wall_insulation", "loft_insulation"], "15", None], + [["cavity_wall_insulation", "loft_insulation"], "15", None, [0, 1]], ], }, { @@ -76,7 +76,7 @@ scenario_properties = [ "postcode": "HP1 2AN", "lmk-key": "d1e0534be3a44c33003323b21d0e322e3daddc65b5ee71936f89c59ddab96b50", "measures": [ - [["cavity_wall_insulation", "loft_insulation"], "15", None], + [["cavity_wall_insulation", "loft_insulation"], "15", None, [0, 1]], ], }, { @@ -84,7 +84,7 @@ scenario_properties = [ "postcode": "HP1 2HX", "lmk-key": "1eae354db522a95188018d9cd0502ed8c609910b6c88f8797d3a25f59b11770a", "measures": [ - [["cavity_wall_insulation", "loft_insulation"], "15", None], + [["cavity_wall_insulation", "loft_insulation"], "15", None, [0, 1]], ], }, ] @@ -195,18 +195,18 @@ for scenario_property in scenario_properties: recommendation_record = p.base_difference_record.df.to_dict("records")[ 0 ].copy() - for rec in combi: - recommendation_record = p.create_recommendation_scoring_data( - property_id=rec["type"], - recommendation_record=recommendation_record, - recommendation=rec, - ) + recommendation_record = p.create_recommendation_scoring_data( + property_id=i, + primary_recommendation_id=i, + recommendation_record=recommendation_record, + recommendations=combi, + ) if override is not None: for key, value in override.items(): recommendation_record[key] = value - recommendation_record["id"] = "+".join(measure) + "+" + str(i) + recommendation_record["id"] = "&".join(measure) + "+" + str(i) recommendation_record["impact"] = impact scoring_list.append(recommendation_record) @@ -230,6 +230,20 @@ recommendations_scoring_data.insert(0, "impact", impact_col) id_col = recommendations_scoring_data.pop("id") recommendations_scoring_data.insert(0, "id", id_col) +from backend.ml_models.api import ModelApi + +model_api = ModelApi(portfolio_id="generate-scenarios-data", timestamp=created_at) + +all_predictions = model_api.predict_all( + df=recommendations_scoring_data, + bucket=get_settings().DATA_BUCKET, + prediction_buckets={ + "sap_change_predictions": get_settings().SAP_PREDICTIONS_BUCKET, + "heat_demand_predictions": get_settings().HEAT_PREDICTIONS_BUCKET, + "carbon_change_predictions": get_settings().CARBON_PREDICTIONS_BUCKET + } +) + save_dataframe_to_s3_parquet( recommendations_scoring_data, "retrofit-data-dev", diff --git a/etl/testing_data/sap_model_simulation.py b/etl/testing_data/sap_model_simulation.py index 726c2428..12b26249 100644 --- a/etl/testing_data/sap_model_simulation.py +++ b/etl/testing_data/sap_model_simulation.py @@ -1935,10 +1935,10 @@ def app(): "address": "73 Long Chaulden", "postcode": "HP1 2HX", }, - { - "address": "77 Simmons Drive", - "postcode": "B32 1SL", - }, + # { + # "address": "77 Simmons Drive", + # "postcode": "B32 1SL", + # }, { "address": "139 School Road", "postcode": "B28 8JF", @@ -1984,6 +1984,19 @@ def app(): floor_recommendations = recommender.floor_recommender.recommendations solar_recommendations = recommender.solar_recommender.recommendation + # We set wall recommendations to phase 0, loft to phase 1, floor to phase 2 and solar to phase 3 + for rec in wall_recommendations: + rec["phase"] = 0 + + for rec in loft_recommendations: + rec["phase"] = 1 + + for rec in floor_recommendations: + rec["phase"] = 2 + + for rec in solar_recommendations: + rec["phase"] = 3 + # TODO: TEMP! solar_recommendations[0]["photo_supply"] = 50 @@ -1994,7 +2007,8 @@ def app(): for wall_rec in wall_recommendations: recommendation_record = p.base_difference_record.df.to_dict("records")[0].copy() scoring_dict = p.create_recommendation_scoring_data( - property_id=p.id, recommendation_record=recommendation_record, recommendation=wall_rec, + property_id=p.id, recommendation_record=recommendation_record, recommendations=[wall_rec], + primary_recommendation_id=wall_rec["recommendation_id"] ) wall_scoring_data.append(scoring_dict) @@ -2002,7 +2016,8 @@ def app(): for roof_rec in loft_recommendations: recommendation_record = p.base_difference_record.df.to_dict("records")[0].copy() scoring_dict = p.create_recommendation_scoring_data( - property_id=p.id, recommendation_record=recommendation_record, recommendation=roof_rec, + property_id=p.id, recommendation_record=recommendation_record, recommendations=[roof_rec], + primary_recommendation_id=roof_rec["recommendation_id"] ) roof_scoring_data.append(scoring_dict) @@ -2010,7 +2025,8 @@ def app(): for floor_rec in floor_recommendations: recommendation_record = p.base_difference_record.df.to_dict("records")[0].copy() scoring_dict = p.create_recommendation_scoring_data( - property_id=p.id, recommendation_record=recommendation_record, recommendation=floor_rec, + property_id=p.id, recommendation_record=recommendation_record, recommendations=[floor_rec], + primary_recommendation_id=floor_rec["recommendation_id"] ) floor_scoring_data.append(scoring_dict) @@ -2018,7 +2034,8 @@ def app(): for solar_rec in solar_recommendations: recommendation_record = p.base_difference_record.df.to_dict("records")[0].copy() scoring_dict = p.create_recommendation_scoring_data( - property_id=p.id, recommendation_record=recommendation_record, recommendation=solar_rec, + property_id=p.id, recommendation_record=recommendation_record, recommendations=[solar_rec], + primary_recommendation_id=solar_rec["recommendation_id"] ) solar_scoring_data.append(scoring_dict) @@ -2029,25 +2046,30 @@ def app(): scoring_dict_with_wall = p.create_recommendation_scoring_data( property_id=p.id, recommendation_record=starting_record.copy(), - recommendation=wall_recommendations[0], + recommendations=[wall_recommendations[0]], + primary_recommendation_id=wall_recommendations[0]["recommendation_id"] ) if wall_recommendations else starting_record.copy() scoring_dict_with_wall_and_roof = p.create_recommendation_scoring_data( property_id=p.id, recommendation_record=scoring_dict_with_wall.copy(), - recommendation=loft_recommendations[0], + recommendations=[wall_recommendations[0], loft_recommendations[0], ], + primary_recommendation_id=loft_recommendations[0]["recommendation_id"] ) if loft_recommendations else scoring_dict_with_wall.copy() scoring_dict_with_wall_roof_floor = p.create_recommendation_scoring_data( property_id=p.id, recommendation_record=scoring_dict_with_wall_and_roof.copy(), - recommendation=floor_recommendations[0], + recommendations=[wall_recommendations[0], loft_recommendations[0], floor_recommendations[0]], + primary_recommendation_id=floor_recommendations[0]["recommendation_id"] ) if floor_recommendations else scoring_dict_with_wall_and_roof.copy() scoring_dict_with_wall_roof_floor_solar = p.create_recommendation_scoring_data( property_id=p.id, recommendation_record=scoring_dict_with_wall_roof_floor.copy(), - recommendation=solar_recommendations[0], + recommendations=[wall_recommendations[0], loft_recommendations[0], floor_recommendations[0], + solar_recommendations[0]], + primary_recommendation_id=solar_recommendations[0]["recommendation_id"] ) if solar_recommendations else scoring_dict_with_wall_roof_floor.copy() # We score each dataset with the model