moed reading csv function

backend/app/plan/router.py

@@ -24,7 +24,7 @@ from backend.app.db.models.portfolio import rating_lookup
 from backend.app.dependencies import validate_token
 from backend.app.plan.schemas import PlanTriggerRequest
 from backend.app.plan.utils import get_cleaned
-from backend.app.utils import epc_to_sap_lower_bound, read_csv_from_s3, sap_to_epc
+from backend.app.utils import epc_to_sap_lower_bound, sap_to_epc
 
 from backend.ml_models.api import ModelApi
 from backend.Property import Property
@@ -35,7 +35,7 @@ from recommendations.optimiser.GainOptimiser import GainOptimiser
 from recommendations.optimiser.optimiser_functions import prepare_input_measures
 from recommendations.Recommendations import Recommendations
 from utils.logger import setup_logger
-from utils.s3 import read_dataframe_from_s3_parquet
+from utils.s3 import read_dataframe_from_s3_parquet, read_csv_from_s3
 from backend.ml_models.Valuation import PropertyValuation
 
 logger = setup_logger()
@@ -196,7 +196,7 @@ async def trigger_plan(body: PlanTriggerRequest):
         )
 
         model_api = ModelApi(portfolio_id=body.portfolio_id, timestamp=created_at)
-        # model_api.MODEL_PREFIXES = ["sap_change_predictions"]
+        # model_api.MODEL_PREFIXES = ['sap_change_predictions', 'carbon_change_predictions']
 
         all_predictions = {
             "sap_change_predictions": pd.DataFrame(),
@@ -221,7 +221,6 @@ async def trigger_plan(body: PlanTriggerRequest):
 
         # TODO: TEMP
         # all_predictions["heat_demand_predictions"] = all_predictions["sap_change_predictions"].copy()
-        # all_predictions["carbon_change_predictions"] = all_predictions["sap_change_predictions"].copy()
 
         # Insert the predictions into the recommendations and run the optimiser
         # TODO: If a recommendation has a negative impact on SAP, we should remove it - this seems to have become a

backend/app/utils.py

@@ -1,6 +1,4 @@
 import boto3
-import csv
-from io import StringIO
 import string
 import secrets
 import logging
@@ -41,25 +39,6 @@ def setup_logger(log_file=None, level=logging.INFO, overwrite_handler=False):
     return logger
 
 
-def read_csv_from_s3(bucket_name, filepath):
-    s3 = boto3.client('s3')
-
-    # Get the object from s3
-    s3_object = s3.get_object(Bucket=bucket_name, Key=filepath)
-
-    # Read the CSV body from the s3 object
-    body = s3_object['Body'].read()
-
-    # Use StringIO to create a file-like object from the string
-    csv_data = StringIO(body.decode('utf-8'))
-
-    # Use csv library to read it into a list of dictionaries
-    reader = csv.DictReader(csv_data)
-    data = list(reader)
-
-    return data
-
-
 def generate_api_key():
     # Define the characters that will be used to generate the api key
     characters = string.ascii_letters + string.digits

etl/customers/gla_croydon_demo/slides.py

@@ -0,0 +1,44 @@
+"""
+This script contains the code to generate the data required to populate the slides
+We connect to the database amd extract the data for the portfolio needed so it is recommended to use
+a environment akin to the backend to run this script
+"""
+import pandas as pd
+import numpy as np
+from backend.app.db.connection import db_engine
+from sqlalchemy.orm import sessionmaker
+from utils.s3 import read_csv_from_s3
+from etl.customers.slide_utils import (
+    plot_epc_distribution,
+    get_property_details_by_portfolio_id,
+    get_plan_by_portfolio_id,
+    get_properties_with_default_recommendations,
+    create_powerpoint,
+    create_recommendations_summary
+)
+
+USER_ID = 8
+PORTFOLIO_ID_1 = 67
+EPC_TARGET_1 = "C"
+SAP_TARGET_1 = 69
+CUSTOMER_KEY = "gla-demo"
+
+
+def app():
+    # Connect to database
+    session = sessionmaker(bind=db_engine)()
+
+    ########################################################################
+    # Get the data we need
+    ########################################################################
+
+    portfolio_id = PORTFOLIO_ID_1
+
+    # Get the asset list
+    asset_list = read_csv_from_s3(
+        "retrofit-plan-inputs-dev", f"{USER_ID}/{portfolio_id}/inputs.csv"
+    )
+
+    # Get the properties for the portfolio
+    properties = get_properties_with_default_recommendations(session, portfolio_id)
+    properties_df = pd.DataFrame(properties)

utils/s3.py

@@ -1,9 +1,10 @@
 import pickle
 import boto3
-from io import BytesIO, StringIO
+import csv
-from botocore.exceptions import NoCredentialsError, PartialCredentialsError
 import pandas as pd
+from io import BytesIO, StringIO
 from utils.logger import setup_logger
+from botocore.exceptions import NoCredentialsError, PartialCredentialsError
 
 logger = setup_logger()
 
@@ -224,3 +225,22 @@ def read_excel_from_s3(bucket_name, file_key, header_row):
     df.reset_index(drop=True, inplace=True)
 
     return df
+
+
+def read_csv_from_s3(bucket_name, filepath):
+    s3 = boto3.client('s3')
+
+    # Get the object from s3
+    s3_object = s3.get_object(Bucket=bucket_name, Key=filepath)
+
+    # Read the CSV body from the s3 object
+    body = s3_object['Body'].read()
+
+    # Use StringIO to create a file-like object from the string
+    csv_data = StringIO(body.decode('utf-8'))
+
+    # Use csv library to read it into a list of dictionaries
+    reader = csv.DictReader(csv_data)
+    data = list(reader)
+
+    return data