From 637ada9cb3b9fdf48371205c368e2b01d9135893 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 5 Sep 2023 15:47:26 +0100 Subject: [PATCH] Implementing sap model api to backend wip --- backend/app/plan/router.py | 110 +++++++++++++++++- backend/requirements/base.txt | 2 + model_data/simulation_system/core/Settings.py | 1 - recommendations/WallRecommendations.py | 4 + 4 files changed, 115 insertions(+), 2 deletions(-) diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index 15646c00..7db2f31f 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -15,6 +15,8 @@ from backend.app.db.utils import row2dict from starlette.responses import Response from sqlalchemy.orm import sessionmaker from sqlalchemy.exc import IntegrityError, OperationalError +from datetime import datetime +import pandas as pd # database interaction functions from backend.app.db.functions.property_functions import ( @@ -212,6 +214,8 @@ async def trigger_plan(body: PlanTriggerRequest): # TODO: Move this to a class. We probably was a Recommender class which takes the injects the optimisers # in as a dependency and then the optimisers can take the input measures in as part of the setup() method recommendations = {} + recommendations_scoring_data = [] + for p in input_properties: property_recommendations = [] @@ -323,7 +327,111 @@ async def trigger_plan(body: PlanTriggerRequest): recommendations[p.id] = property_recommendations - # Once we're done, we'll store: + # Finally, we'll prepare data for predicting the impact on SAP + from model_data.simulation_system.core.Settings import FIXED_FEATURES, COMPONENT_FEATURES + epc_data = p.data.copy() + epc_data = pd.DataFrame([epc_data]) + epc_data.columns = [col.upper().replace("-", "_") for col in epc_data.columns] + starting_epc_data = epc_data[COMPONENT_FEATURES + ["LODGEMENT_DATE"]].copy().add_suffix("_STARTING") + ending_epc_data = epc_data[COMPONENT_FEATURES + ["LODGEMENT_DATE"]].copy().add_suffix("_ENDING") + fixed_data = epc_data[FIXED_FEATURES] + + # We update the ending record with the recommended updates and we set lodgement date to today + ending_epc_data["LODGEMENT_DATE_ENDING"] = datetime.now().strftime("%Y-%m-%d") + + scoring_map = { + 'Solid brick, as built, no insulation (assumed)': 'Solid brick, as built, insulated (assumed)', + 'Suspended, no insulation (assumed)': 'Suspended, insulated (assumed)', + 'Solid, no insulation (assumed)': 'Solid, insulated (assumed)', + } + for rec in property_recommendations: + scoring_dict = { + "UPRN": p.data["uprn"], + "id": "+".join([str(p.id), str(rec["recommendation_id"])]), + **starting_epc_data.to_dict("records")[0], + **ending_epc_data.to_dict("records")[0], + **fixed_data.to_dict("records")[0] + } + + # We update the description to indicate it's insulated + if rec["type"] == "wall_insulation": + scoring_dict["WALLS_DESCRIPTION_ENDING"] = scoring_map[p.walls["clean_description"]] + elif rec["type"] == "floor_insulation": + scoring_dict["FLOOR_DESCRIPTION_ENDING"] = scoring_map[p.floor["clean_description"]] + else: + raise NotImplementedError("Implement me") + + recommendations_scoring_data.append(scoring_dict) + + recommendations_scoring_data = pd.DataFrame(recommendations_scoring_data) + # TODO: We need to clean the data + + column_types = data.dtypes.to_dict() + columntypes = {} + for k, v in column_types.items(): + if k not in recommendations_scoring_data.columns: + continue + columntypes[k] = v.name + + columntypes["id"] = "object" + for col in recommendations_scoring_data.columns: + recommendations_scoring_data[col] = recommendations_scoring_data[col].astype(columntypes[col]) + + recommendations_scoring_data = recommendations_scoring_data.astype(columntypes) + + + column_types = {'UPRN': dtype('O'), 'RDSAP_CHANGE': dtype('int64'), 'HEAT_DEMAND_CHANGE': dtype('int64'), + 'TOTAL_FLOOR_AREA': dtype('float64'), 'FLOOR_HEIGHT': dtype('float64'), 'PROPERTY_TYPE': dtype('O'), + 'BUILT_FORM': dtype('O'), 'CONSTITUENCY': dtype('O'), 'NUMBER_HABITABLE_ROOMS': dtype('float64'), + 'NUMBER_HEATED_ROOMS': dtype('float64'), 'FIXED_LIGHTING_OUTLETS_COUNT': dtype('float64'), + 'FLOOR_LEVEL': dtype('float64'), 'CONSTRUCTION_AGE_BAND': dtype('O'), 'TRANSACTION_TYPE_STARTING': dtype('O'), + 'WALLS_DESCRIPTION_STARTING': dtype('O'), 'FLOOR_DESCRIPTION_STARTING': dtype('O'), + 'LIGHTING_DESCRIPTION_STARTING': dtype('O'), 'ROOF_DESCRIPTION_STARTING': dtype('O'), + 'MAINHEAT_DESCRIPTION_STARTING': dtype('O'), 'HOTWATER_DESCRIPTION_STARTING': dtype('O'), + 'MAIN_FUEL_STARTING': dtype('O'), 'MECHANICAL_VENTILATION_STARTING': dtype('O'), + 'SECONDHEAT_DESCRIPTION_STARTING': dtype('O'), 'ENERGY_TARIFF_STARTING': dtype('O'), + 'SOLAR_WATER_HEATING_FLAG_STARTING': dtype('O'), 'PHOTO_SUPPLY_STARTING': dtype('float64'), + 'WINDOWS_DESCRIPTION_STARTING': dtype('O'), 'GLAZED_TYPE_STARTING': dtype('O'), + 'MULTI_GLAZE_PROPORTION_STARTING': dtype('float64'), 'LOW_ENERGY_LIGHTING_STARTING': dtype('float64'), + 'NUMBER_OPEN_FIREPLACES_STARTING': dtype('float64'), 'MAINHEATCONT_DESCRIPTION_STARTING': dtype('O'), + 'EXTENSION_COUNT_STARTING': dtype('float64'), 'LODGEMENT_DATE_STARTING': dtype('O'), + 'TRANSACTION_TYPE_ENDING': dtype('O'), 'WALLS_DESCRIPTION_ENDING': dtype('O'), + 'FLOOR_DESCRIPTION_ENDING': dtype('O'), 'LIGHTING_DESCRIPTION_ENDING': dtype('O'), + 'ROOF_DESCRIPTION_ENDING': dtype('O'), 'MAINHEAT_DESCRIPTION_ENDING': dtype('O'), + 'HOTWATER_DESCRIPTION_ENDING': dtype('O'), 'MAIN_FUEL_ENDING': dtype('O'), + 'MECHANICAL_VENTILATION_ENDING': dtype('O'), 'SECONDHEAT_DESCRIPTION_ENDING': dtype('O'), + 'ENERGY_TARIFF_ENDING': dtype('O'), 'SOLAR_WATER_HEATING_FLAG_ENDING': dtype('O'), + 'PHOTO_SUPPLY_ENDING': dtype('float64'), 'WINDOWS_DESCRIPTION_ENDING': dtype('O'), + 'GLAZED_TYPE_ENDING': dtype('O'), 'MULTI_GLAZE_PROPORTION_ENDING': dtype('float64'), + 'LOW_ENERGY_LIGHTING_ENDING': dtype('float64'), 'NUMBER_OPEN_FIREPLACES_ENDING': dtype('float64'), + 'MAINHEATCONT_DESCRIPTION_ENDING': dtype('O'), 'EXTENSION_COUNT_ENDING': dtype('float64'), + 'LODGEMENT_DATE_ENDING': dtype('O'), 'id': dtype('int64')} + + # Example data file + + from io import BytesIO + import boto3 + def read_parquet_from_s3(bucket_name, file_key): + client = boto3.client('s3') + + # Get the object + s3_object = client.get_object(Bucket=bucket_name, Key=file_key) + + # Read the CSV body into a DataFrame + csv_body = s3_object["Body"].read() + df = pd.read_parquet(BytesIO(csv_body)) + + return df + data = read_parquet_from_s3( + bucket_name="retrofit-data-dev", file_key="model_build_data/change_data/rdsap_full/test_data_with_id.parquet" + ) + data = data.head(5) + + + # We query the sap difference model api to get the estimated impact on sap + for property_id, recommendations in recommendations.items(): + + # 1) the property data # 2) the property details (epc) # 3) the recommendations diff --git a/backend/requirements/base.txt b/backend/requirements/base.txt index 0b7337e4..de173db2 100644 --- a/backend/requirements/base.txt +++ b/backend/requirements/base.txt @@ -32,3 +32,5 @@ psycopg2-binary pytz==2023.3 mip==1.15.0 boto3==1.28.3 +pandas==1.5.3 +pyarrow==12.0.1 \ No newline at end of file diff --git a/model_data/simulation_system/core/Settings.py b/model_data/simulation_system/core/Settings.py index 7765f64a..9f6c2e12 100644 --- a/model_data/simulation_system/core/Settings.py +++ b/model_data/simulation_system/core/Settings.py @@ -99,7 +99,6 @@ COMPONENT_FEATURES = [ "WINDOWS_DESCRIPTION", "GLAZED_TYPE", "MULTI_GLAZE_PROPORTION", - "LIGHTING_DESCRIPTION", "LOW_ENERGY_LIGHTING", "NUMBER_OPEN_FIREPLACES", "MAINHEATCONT_DESCRIPTION", diff --git a/recommendations/WallRecommendations.py b/recommendations/WallRecommendations.py index fdd271be..570a46d7 100644 --- a/recommendations/WallRecommendations.py +++ b/recommendations/WallRecommendations.py @@ -256,6 +256,10 @@ class WallRecommendations(Definitions): is_solid_brick = self.property.walls["is_solid_brick"] insulation_thickness = self.property.walls["insulation_thickness"] + # We check if the wall is already insulated and if so, we exit + if insulation_thickness in ["average", "above average"]: + return + if u_value: if self.property.walls["thermal_transmittance_unit"] != self.U_VALUE_UNIT: raise NotImplementedError("Haven't handled the case of other u value units yet")