Model/model_data/simulation_system/training.py

import argparse

# import boto3
from pathlib import Path
from datetime import datetime
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from MLModel.Models import AutogluonModel
from core.Logger import logger
from core.DataLoader import DataLoader
from core.FeatureProcessor import FeatureProcessor
from core.Settings import (
    MODEL_DIRECTORY,
    BASE_REGISTRY_PATH,
    REGISTRY_FILE,
    MODEL_FOLDER,
    METRICS_FOLDER,
    DEPLOYMENT_FOLDER,
    SUBSAMPLE_FACTOR,
    MODEL_HYPERPARAMETERS,
    TIMESTAMP_FORMAT,
    RESIDUAL_TRUE_LABEL,
    RESIDUAL_PREDICTION_LABEL,
    RESIDUAL_FILE,
    SEABORN_RESIDUAL_AXIS_FONTSIZE,
    SEABORN_RESIDUAL_TITLE_FONTSIZE,
    SEABORN_RESIDUAL_STYLE,
    SEABORN_RESIDUAL_ASPECT_RATIO,
    SEABORN_RESIDUAL_PLOT_DPI,
    SEABORN_RESIDUAL_RANGE,
    SEABORN_RESIDUAL_LINE_COLOUR,
    SEABORN_RESIDUAL_LINE_WIDTH,
)

TIMESTAMP = datetime.now().strftime(TIMESTAMP_FORMAT)

# FOR TESTING
# train_filepath = "./model_build_data/change_data/rdsap_full/train_validation_data.parquet"
# test_filepath = "./model_build_data/change_data/rdsap_full/test_data.parquet"
# target_column = "RDSAP_CHANGE"
# model_type = "autogluon"
# hyperparameter = HYPERPARAMETERS
# SUBSAMPLE_FACTOR = 200

# SESSION = boto3.Session()

# S3_CLIENT = SESSION.client(
#     service_name="s3",
#     aws_access_key_id=os.environ.get("AWS_ACCESS_KEY_ID", 'admin'),
#     aws_secret_access_key=os.environ.get("AWS_SECRET_ACCESS_KEY", 'password'),
#     endpoint_url=os.environ.get("ENDPOINT_URL", "http://localhost:9000")
# )

# S3_CLIENT.create_bucket
# S3_CLIENT.list_buckets()


def ingest_arguments() -> argparse.Namespace:
    """
    Helper function to take in arguments from script start
    """

    parser = argparse.ArgumentParser(description="Inputs for training script")

    parser.add_argument(
        "--train-filepath",
        type=str,
        help="Location of Parquet dataset to load for training",
        required=True,
    )
    parser.add_argument(
        "--test-filepath",
        type=str,
        help="Location of Parquet dataset to load for testing",
        required=True,
    )
    parser.add_argument(
        "--model-type",
        type=str,
        help="The type of model to train",
        choices=["autogluon"],
        default="autogluon",
    )
    parser.add_argument(
        "--target-column",
        type=str,
        help="The response variable",
        choices=["RDSAP_CHANGE", "HEAT_DEMAND_CHANGE"],
        default="RDSAP_CHANGE",
    )

    args = parser.parse_args()

    return args


def training(
    train_filepath: str,
    test_filepath: str,
    target_column: str = "RDSAP_CHANGE",
    model_type: str = "autogluon",
    hyperparameters: dict = None,
) -> None:
    """
    Pipeline to run training on the dataset
    """

    logger.info("--- Loading data ---")
    dataloader = DataLoader()
    train_df = dataloader.load(filepath=train_filepath)
    test_df = dataloader.load(filepath=test_filepath)

    logger.info("--- Feature processing ---")

    feature_processor = FeatureProcessor()

    subsample_amount = round(len(train_df) / SUBSAMPLE_FACTOR)

    train_df = feature_processor.process(
        train_df, target_column=target_column, subsample_amount=subsample_amount
    )
    test_df = feature_processor.process(test_df, target_column=target_column)

    logger.info("--- Build Model ---")

    logger.info("--- Load Hyperparameters ---")

    if hyperparameters is None:
        logger.info("Use base hyperparameters in settings")
        hyperparameters = MODEL_HYPERPARAMETERS[model_type]
        logger.info(f"Hyperparameters are: {hyperparameters}")

    if model_type == "autogluon":
        model_root = f"{target_column}-{hyperparameters['presets']}-{hyperparameters['time_limit']}-{TIMESTAMP}".lower()
        output_base = Path(MODEL_DIRECTORY) / target_column / model_type / model_root

        model = AutogluonModel(output_filepath=output_base / MODEL_FOLDER)
    else:
        raise ValueError("No alternative model implemented yet")

    model.train_model(
        data=train_df, target_column=target_column, hyperparameters=hyperparameters
    )

    logger.info("--- Save Model ---")
    model.save_model(output_filepath=model.output_filepath)

    logger.info("--- Generate evaluation metrics ---")
    metrics_df = model.model_evaluation(
        validation_data=test_df,
        target_column=target_column,
        metrics_location=output_base / METRICS_FOLDER,
    )

    logger.info("--- Generate metric outputs using predictions ---")
    # TODO: can have a model.metric_outputs method
    # FOr not just do it here
    residual_df = pd.DataFrame(
        list(zip(test_df[target_column], model.predictions)),
        columns=[RESIDUAL_TRUE_LABEL, RESIDUAL_PREDICTION_LABEL],
    )

    # image formatting
    # TODO: move to settings file , AXIS_FONT, TITLE_FONT
    sns.set(style=SEABORN_RESIDUAL_STYLE)
    ax = sns.scatterplot(
        x=RESIDUAL_TRUE_LABEL, y=RESIDUAL_PREDICTION_LABEL, data=residual_df
    )
    ax.set_aspect(SEABORN_RESIDUAL_ASPECT_RATIO)
    ax.set_xlabel(f"True {target_column}", fontsize=SEABORN_RESIDUAL_AXIS_FONTSIZE)
    ax.set_ylabel(
        f"Predicted {target_column}", fontsize=SEABORN_RESIDUAL_AXIS_FONTSIZE
    )  # ylabel
    ax.set_title("Residuals", fontsize=SEABORN_RESIDUAL_TITLE_FONTSIZE)

    # Square aspect ratio
    ax.plot(
        SEABORN_RESIDUAL_RANGE,
        SEABORN_RESIDUAL_RANGE,
        SEABORN_RESIDUAL_LINE_COLOUR,
        linewidth=SEABORN_RESIDUAL_LINE_WIDTH,
    )

    plt.tight_layout()
    plt.savefig(
        output_base / METRICS_FOLDER / RESIDUAL_FILE, dpi=SEABORN_RESIDUAL_PLOT_DPI
    )

    # TODO: for cml, we might want to have class that outputs all data and plots to add to the report
    # If we want residual plot/ any plots, we will need to self host
    # plt.savefig(RESIDUAL_FILE, dpi=120)

    # TODO: introduce a seperate script for model optimisation, and from there, optimise for deployment
    # Imagining for now that the model trained here is the best model amongst all models built

    logger.info("--- Optimising model for deployment ---")

    deployment_model_path = model.optimise_model_for_deployment(
        deployment_path=output_base / DEPLOYMENT_FOLDER
    )
    logger.info(
        f"Optimised version of best model can be found at: {deployment_model_path}"
    )

    # TODO: Need a model registry - for now have this as a CSV
    # Save this in the model directory
    logger.info("--- Append registry with new model ---")
    registry_path = BASE_REGISTRY_PATH / target_column / REGISTRY_FILE

    if registry_path.exists():
        logger.info("Registry file found - Loading into Dataframe")
        registry_df = pd.read_csv(registry_path, index_col=None)
    else:
        # TODO: Moved columns into settings: MODEL_DETAILS and Metrics class columns
        registry_df = pd.DataFrame(
            columns=[
                "model_type",
                "model_name",
                "model_location",
                "mean_absolute_error",
                "root_mean_squared_error",
                "mean_squared_error",
                "r2",
                "pearsonr",
                "median_absolute_error",
                "mape",
                "best_model",
            ]
        )

    model_details_df = pd.DataFrame(
        [
            {
                "model_type": model_type,
                "model_name": model_root,
                "model_location": deployment_model_path,
            }
        ]
    )

    registry_row = pd.concat([model_details_df, metrics_df], axis=1)
    registry_df = pd.concat([registry_df, registry_row], axis=0).reset_index(drop=True)

    # TODO: will need a rebuild script metric script -i.e. if we add new metrics, we will want to load models and
    #       regenerate new metrics
    # TODO: decide metric to optimise to
    registry_df = registry_df.sort_values(
        "mean_absolute_error", ascending=False
    ).reset_index(drop=True)
    registry_df["best_model"] = [False] * len(registry_df)
    registry_df.loc[0, "best_model"] = True

    logger.info("--- Saving new model to registry ---")
    # Ensure the directory exists
    registry_path.parent.mkdir(parents=True, exist_ok=True)
    registry_df.to_csv(registry_path, index=False)

    logger.info("--- Training Pipeline Complete --- ")


if __name__ == "__main__":

    logger.info("---Begin Pipeline---")

    logger.info("---Ingest Arguments---")
    args = ingest_arguments()

    # To run script: python3 training.py --train-filepath
    # ./model_build_data/change_data/rdsap_full/train_validation_data.parquet --test-filepath
    # ./model_build_data/change_data/rdsap_full/test_data.parquet
    # TODO: Ingest hyper parameters from somewhere - currently change at the top of script
    training(
        train_filepath=args.train_filepath,
        test_filepath=args.test_filepath,
        target_column=args.target_column,
        model_type=args.model_type,
    )