""" Generate metrics and enable regeneration of metrics if new metrics are generated Key tasks: - Specify metric functions that take in prediction vs actual to generate a metric value - Given a model and test data, produce a suite of all metrics """ import pandas as pd from core.Settings import OPTIMISE_METRIC from MLModel.BaseMLModel import MLModel def sort_by_metric( data: pd.DataFrame, optimse_metric: str, best_model_column_name: str ) -> pd.DataFrame: """ Helper function to sort data frame by metric and append a best model flag """ data = data.sort_values(optimse_metric, ascending=False).reset_index(drop=True) data[best_model_column_name] = [False] * len(data) data.loc[0, best_model_column_name] = True return data class Metrics: """ All metric functions used to generate a dictionary of metrics """ @staticmethod def metric_1(predictions: pd.Series, actuals: pd.Series) -> float: """ Can leverage ML packages like sklearn for individual metrics like MAPE etc """ pass @staticmethod def metric_2(predictions: pd.Series, actuals: pd.Series) -> float: """ Can leverage ML packages like sklearn for individual metrics like MAPE etc """ pass def list_metric_functions(self) -> list: """ Gather all metric functions to run """ pass def generate_metric_suite( self, model: MLModel, data: pd.DataFrame, target_column: str ) -> pd.Series: """ For the model, test data and target, generate predictions and then iterative over all metrics to generate a Series of metric values """ predictions = model.generate_predictions(data=data) actuals = data[target_column] metric_dict = {} for key, metric_function in asd: # TODO: metric_dict[key] = metric_function(predictions, actuals) metrics = pd.Series([metric_dict]) return metrics