diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index a0d93190..2067d796 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -24,7 +24,7 @@ from backend.app.db.models.portfolio import rating_lookup from backend.app.dependencies import validate_token from backend.app.plan.schemas import PlanTriggerRequest from backend.app.plan.utils import get_cleaned -from backend.app.utils import epc_to_sap_lower_bound, read_csv_from_s3, sap_to_epc +from backend.app.utils import epc_to_sap_lower_bound, sap_to_epc from backend.ml_models.api import ModelApi from backend.Property import Property @@ -35,7 +35,7 @@ from recommendations.optimiser.GainOptimiser import GainOptimiser from recommendations.optimiser.optimiser_functions import prepare_input_measures from recommendations.Recommendations import Recommendations from utils.logger import setup_logger -from utils.s3 import read_dataframe_from_s3_parquet +from utils.s3 import read_dataframe_from_s3_parquet, read_csv_from_s3 from backend.ml_models.Valuation import PropertyValuation logger = setup_logger() @@ -196,7 +196,7 @@ async def trigger_plan(body: PlanTriggerRequest): ) model_api = ModelApi(portfolio_id=body.portfolio_id, timestamp=created_at) - # model_api.MODEL_PREFIXES = ["sap_change_predictions"] + # model_api.MODEL_PREFIXES = ['sap_change_predictions', 'carbon_change_predictions'] all_predictions = { "sap_change_predictions": pd.DataFrame(), @@ -221,7 +221,6 @@ async def trigger_plan(body: PlanTriggerRequest): # TODO: TEMP # all_predictions["heat_demand_predictions"] = all_predictions["sap_change_predictions"].copy() - # all_predictions["carbon_change_predictions"] = all_predictions["sap_change_predictions"].copy() # Insert the predictions into the recommendations and run the optimiser # TODO: If a recommendation has a negative impact on SAP, we should remove it - this seems to have become a diff --git a/backend/app/utils.py b/backend/app/utils.py index ba5509e1..b3843206 100644 --- a/backend/app/utils.py +++ b/backend/app/utils.py @@ -1,6 +1,4 @@ import boto3 -import csv -from io import StringIO import string import secrets import logging @@ -41,25 +39,6 @@ def setup_logger(log_file=None, level=logging.INFO, overwrite_handler=False): return logger -def read_csv_from_s3(bucket_name, filepath): - s3 = boto3.client('s3') - - # Get the object from s3 - s3_object = s3.get_object(Bucket=bucket_name, Key=filepath) - - # Read the CSV body from the s3 object - body = s3_object['Body'].read() - - # Use StringIO to create a file-like object from the string - csv_data = StringIO(body.decode('utf-8')) - - # Use csv library to read it into a list of dictionaries - reader = csv.DictReader(csv_data) - data = list(reader) - - return data - - def generate_api_key(): # Define the characters that will be used to generate the api key characters = string.ascii_letters + string.digits diff --git a/etl/customers/gla_croydon_demo/slides.py b/etl/customers/gla_croydon_demo/slides.py index e69de29b..5954f604 100644 --- a/etl/customers/gla_croydon_demo/slides.py +++ b/etl/customers/gla_croydon_demo/slides.py @@ -0,0 +1,44 @@ +""" +This script contains the code to generate the data required to populate the slides +We connect to the database amd extract the data for the portfolio needed so it is recommended to use +a environment akin to the backend to run this script +""" +import pandas as pd +import numpy as np +from backend.app.db.connection import db_engine +from sqlalchemy.orm import sessionmaker +from utils.s3 import read_csv_from_s3 +from etl.customers.slide_utils import ( + plot_epc_distribution, + get_property_details_by_portfolio_id, + get_plan_by_portfolio_id, + get_properties_with_default_recommendations, + create_powerpoint, + create_recommendations_summary +) + +USER_ID = 8 +PORTFOLIO_ID_1 = 67 +EPC_TARGET_1 = "C" +SAP_TARGET_1 = 69 +CUSTOMER_KEY = "gla-demo" + + +def app(): + # Connect to database + session = sessionmaker(bind=db_engine)() + + ######################################################################## + # Get the data we need + ######################################################################## + + portfolio_id = PORTFOLIO_ID_1 + + # Get the asset list + asset_list = read_csv_from_s3( + "retrofit-plan-inputs-dev", f"{USER_ID}/{portfolio_id}/inputs.csv" + ) + + # Get the properties for the portfolio + properties = get_properties_with_default_recommendations(session, portfolio_id) + properties_df = pd.DataFrame(properties) diff --git a/utils/s3.py b/utils/s3.py index 8d36bdb3..fd5992ce 100644 --- a/utils/s3.py +++ b/utils/s3.py @@ -1,9 +1,10 @@ import pickle import boto3 -from io import BytesIO, StringIO -from botocore.exceptions import NoCredentialsError, PartialCredentialsError +import csv import pandas as pd +from io import BytesIO, StringIO from utils.logger import setup_logger +from botocore.exceptions import NoCredentialsError, PartialCredentialsError logger = setup_logger() @@ -224,3 +225,22 @@ def read_excel_from_s3(bucket_name, file_key, header_row): df.reset_index(drop=True, inplace=True) return df + + +def read_csv_from_s3(bucket_name, filepath): + s3 = boto3.client('s3') + + # Get the object from s3 + s3_object = s3.get_object(Bucket=bucket_name, Key=filepath) + + # Read the CSV body from the s3 object + body = s3_object['Body'].read() + + # Use StringIO to create a file-like object from the string + csv_data = StringIO(body.decode('utf-8')) + + # Use csv library to read it into a list of dictionaries + reader = csv.DictReader(csv_data) + data = list(reader) + + return data