From 2221283de40931958dae784d3f2ac671ed7bdf20 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Fri, 16 Feb 2024 16:43:23 +0000 Subject: [PATCH] try the scenario cml --- .github/workflows/MLPipelinePullRequest.yml | 4 + .../src/pipeline/5_generate_scenarios.py | 125 ++++++++++++++++++ modules/ml-pipeline/src/pipeline/config.py | 1 + .../src/pipeline/configs/build_model.yaml | 2 +- .../src/pipeline/configs/scenarios.yaml | 9 ++ .../src/pipeline/core/DataClient.py | 10 +- modules/ml-pipeline/src/pipeline/dvc.lock | 68 ++++++---- modules/ml-pipeline/src/pipeline/dvc.yaml | 11 ++ .../src/pipeline/metrics/.gitignore | 1 + 9 files changed, 205 insertions(+), 26 deletions(-) create mode 100644 modules/ml-pipeline/src/pipeline/5_generate_scenarios.py create mode 100644 modules/ml-pipeline/src/pipeline/configs/scenarios.yaml diff --git a/.github/workflows/MLPipelinePullRequest.yml b/.github/workflows/MLPipelinePullRequest.yml index cbc379d..ceb6800 100644 --- a/.github/workflows/MLPipelinePullRequest.yml +++ b/.github/workflows/MLPipelinePullRequest.yml @@ -98,6 +98,10 @@ jobs: git fetch --depth=1 origin ${TARGET_BRANCH}:${TARGET_BRANCH} dvc metrics diff --md --all ${TARGET_BRANCH} >> report.md + echo "## Scenario metrics" > report.md + + cat metrics/scenarios/scenario_table.md >> report.md + cml comment create report.md # echo "## Residuals plot from model" >> report.md diff --git a/modules/ml-pipeline/src/pipeline/5_generate_scenarios.py b/modules/ml-pipeline/src/pipeline/5_generate_scenarios.py new file mode 100644 index 0000000..28bcb9d --- /dev/null +++ b/modules/ml-pipeline/src/pipeline/5_generate_scenarios.py @@ -0,0 +1,125 @@ +""" +Fourth part of the pipeline: +After the model is built and metrics are generated, +we want to test this model against known scenarios +""" + +import os +import pandas as pd +from core.interface.InterfaceModels import MLModel +from core.interface.InterfaceDataClient import DataClient +from configs.post_prediction_logic import post_prediction_logic +from core.DataClient import dataclient_factory +from core.MLModels import model_factory +from core.Logger import logger +from config import settings + +logger.info(f"--- Initiate Parameters ---") + +RUNTIME_ENVIRONMENT = os.environ.get("RUNTIME_ENVIRONMENT", "local") + +client_params = settings.client +prepare_data_params = settings.prepare_data +build_model_params = settings.build_model +generate_predictions_params = settings.generate_predictions +generate_metrics_params = settings.generate_metrics +feature_process_params = settings.feature_processor +scenarios_params = settings.scenarios + +model_filepath = build_model_params["model_save_filepath"] +target = feature_process_params["feature_processor_config"]["target"] +scenario_data_filepaths = scenarios_params["scenario_data_filepaths"] +predictions_column_name = generate_predictions_params["predictions_column_name"] +output_filepath = scenarios_params["output_filepath"] + +logger.info(f"--- Initiate MLModel ---") + +model = model_factory(build_model_params["model_type"]) + +logger.info(f"--- Initiate DataClient ---") + +# Use data client for input and output, as we use dvc to cache later to the cloud +input_dataclient_type = scenarios_params["input_dataclient_type"] +input_dataclient = dataclient_factory( + dataclient_type=input_dataclient_type, + dataclient_config=client_params[input_dataclient_type], +) + +output_dataclient_type = scenarios_params["output_dataclient_type"] +output_dataclient = dataclient_factory( + dataclient_type=output_dataclient_type, + dataclient_config=client_params[output_dataclient_type], +) + + +def generate_scenario_predictions( + input_dataclient: DataClient, + output_dataclient: DataClient, + model: MLModel, + model_filepath: str, + scenario_data_filepaths: list, + predictions_column_name: str, + output_filepath: str, +): + """ + Given the new model, we generate prediction for expected scenarios + """ + + logger.info("--- Loading Scenario Data ---") + + scenario_data = pd.DataFrame() + + # Can have multiple scenario data files + for scenario_data_filepath in scenario_data_filepaths: + scenario_data = pd.concat( + [ + scenario_data, + input_dataclient.load_data(scenario_data_filepath, load_config=None), + ] + ) + + logger.info("--- Loading Model ---") + + model.load_model(model_filepath) + + logger.info("--- Generating Predictions ---") + + predictions = model.predict( + data=scenario_data, post_prediction_logic=post_prediction_logic + ) + + logger.info("--- Generate Scenario Predicted Impact ---") + + predictions_df = pd.DataFrame(predictions) + predictions_df.columns = [predictions_column_name] + + scenario_data = pd.concat([scenario_data, predictions_df], axis=1) + scenario_data["predicted_impact"] = abs( + scenario_data[predictions_column_name] - scenario_data["sap_starting"] + ) + + logger.info("--- Save prediction into metrics ---") + + output_df = scenario_data[["uprn", "id", "impact", "predicted_impact"]] + + output_dataclient.save_data( + obj=output_df, location=output_filepath, save_config=None + ) + + +if __name__ == "__main__": + logger.info(f"--- {__file__} - Start! ---") + + logger.info(f"--- Generate Scenario Predictions ---") + + generate_scenario_predictions( + input_dataclient=input_dataclient, + output_dataclient=output_dataclient, + model=model, + model_filepath=model_filepath, + scenario_data_filepaths=scenario_data_filepaths, + predictions_column_name=predictions_column_name, + output_filepath=output_filepath, + ) + + logger.info(f"--- {__file__} - Complete! ---") diff --git a/modules/ml-pipeline/src/pipeline/config.py b/modules/ml-pipeline/src/pipeline/config.py index 7a7366b..bac430c 100644 --- a/modules/ml-pipeline/src/pipeline/config.py +++ b/modules/ml-pipeline/src/pipeline/config.py @@ -7,6 +7,7 @@ settings = Dynaconf( "./configs/settings.yaml", "./configs/build_model.yaml", "./configs/analysis.yaml", + "./configs/scenarios.yaml", ], ) diff --git a/modules/ml-pipeline/src/pipeline/configs/build_model.yaml b/modules/ml-pipeline/src/pipeline/configs/build_model.yaml index 1acea2a..add3da1 100644 --- a/modules/ml-pipeline/src/pipeline/configs/build_model.yaml +++ b/modules/ml-pipeline/src/pipeline/configs/build_model.yaml @@ -14,7 +14,7 @@ default: output_filepath: ./data/model/allmodels/ problem_type: regression eval_metric: mean_squared_error #mean_absolute_error - time_limit: 4000 + time_limit: 60 presets: medium_quality excluded_model_types: ['RF', 'NN_TORCH', 'KNN', 'XT', 'CAT', 'FASTAI'] infer_limit: 0.05 diff --git a/modules/ml-pipeline/src/pipeline/configs/scenarios.yaml b/modules/ml-pipeline/src/pipeline/configs/scenarios.yaml new file mode 100644 index 0000000..29b6672 --- /dev/null +++ b/modules/ml-pipeline/src/pipeline/configs/scenarios.yaml @@ -0,0 +1,9 @@ +default: + scenarios: + input_dataclient_type: aws-s3 + output_dataclient_type: local + scenario_data_filepaths: + [ + s3://retrofit-data-dev/scenario_data/recommendations_scoring_data.parquet, + ] + output_filepath: ./metrics/scenario_table.md diff --git a/modules/ml-pipeline/src/pipeline/core/DataClient.py b/modules/ml-pipeline/src/pipeline/core/DataClient.py index 53f4072..b38ca32 100644 --- a/modules/ml-pipeline/src/pipeline/core/DataClient.py +++ b/modules/ml-pipeline/src/pipeline/core/DataClient.py @@ -245,7 +245,8 @@ class LocalClient: save_methods = { ".parquet": self._save_parquet, - ".json": self._save_json + ".json": self._save_json, + ".md": self._save_md, # "": _save_directory(**save_config), # ADD MORE save_methods HERE } @@ -294,3 +295,10 @@ class LocalClient: # Write the contents of the buffer to the local file with open(location, "wb") as f: f.write(buffer.getvalue()) + + def _save_md(self, obj: pd.DataFrame, location: str, save_config: dict): + """ + Save object as markdown + """ + + obj.to_markdown(location, **save_config) diff --git a/modules/ml-pipeline/src/pipeline/dvc.lock b/modules/ml-pipeline/src/pipeline/dvc.lock index 2f513d4..5959200 100644 --- a/modules/ml-pipeline/src/pipeline/dvc.lock +++ b/modules/ml-pipeline/src/pipeline/dvc.lock @@ -31,8 +31,8 @@ stages: outs: - path: data/prepared_data/ hash: md5 - md5: 8f0f5481075094460ab852ace2fa9b7a.dir - size: 43692138 + md5: 86d085385f7e170d951e95d5e9d0f0bc.dir + size: 43684784 nfiles: 2 build_model: cmd: python 2_build_model.py @@ -43,8 +43,8 @@ stages: size: 4820 - path: data/prepared_data hash: md5 - md5: 8f0f5481075094460ab852ace2fa9b7a.dir - size: 43692138 + md5: 86d085385f7e170d951e95d5e9d0f0bc.dir + size: 43684784 nfiles: 2 params: configs/build_model.yaml: @@ -61,7 +61,7 @@ stages: output_filepath: ./data/model/allmodels/ problem_type: regression eval_metric: mean_squared_error - time_limit: 4000 + time_limit: 60 presets: medium_quality excluded_model_types: - RF @@ -75,17 +75,17 @@ stages: outs: - path: data/fit_predictions/ hash: md5 - md5: e2a05a84a14d35516a6cda8e0a1e963c.dir - size: 3681005 + md5: 69cbcceee3e360e0040a7c45ed72ef7f.dir + size: 3674358 nfiles: 1 - path: data/model/ hash: md5 - md5: 7b0382d001ed2bd7aec5c8112f69d129.dir - size: 793365790 - nfiles: 30 + md5: 09757210fdbaa9ad216a84285cf1cbf2.dir + size: 353975267 + nfiles: 21 - path: metrics/fit_metrics.json hash: md5 - md5: bcfd8d3bd3af858fa3dc26433bc8cd9e + md5: 69be95e8d60eb7cef41ec1e69fa9d2ce size: 224 generate_predictions: cmd: python 3_generate_predictions.py @@ -96,13 +96,13 @@ stages: size: 2464 - path: data/model hash: md5 - md5: 7b0382d001ed2bd7aec5c8112f69d129.dir - size: 793365790 - nfiles: 30 + md5: 09757210fdbaa9ad216a84285cf1cbf2.dir + size: 353975267 + nfiles: 21 - path: data/prepared_data hash: md5 - md5: 8f0f5481075094460ab852ace2fa9b7a.dir - size: 43692138 + md5: 86d085385f7e170d951e95d5e9d0f0bc.dir + size: 43684784 nfiles: 2 params: configs/settings.yaml: @@ -114,8 +114,8 @@ stages: outs: - path: data/predictions/ hash: md5 - md5: 90b5275b5d9829a42573ade3f5a025d2.dir - size: 648526 + md5: 2a0421436d59d95e52a51571c34e0ce9.dir + size: 647012 nfiles: 1 generate_metrics: cmd: python 4_generate_metrics.py @@ -126,13 +126,13 @@ stages: size: 3484 - path: data/predictions hash: md5 - md5: 90b5275b5d9829a42573ade3f5a025d2.dir - size: 648526 + md5: 2a0421436d59d95e52a51571c34e0ce9.dir + size: 647012 nfiles: 1 - path: data/prepared_data hash: md5 - md5: 8f0f5481075094460ab852ace2fa9b7a.dir - size: 43692138 + md5: 86d085385f7e170d951e95d5e9d0f0bc.dir + size: 43684784 nfiles: 2 params: configs/settings.yaml: @@ -142,8 +142,8 @@ stages: outs: - path: metrics/metrics.json hash: md5 - md5: be48389ba2755e6c18e41243aaa9bb81 - size: 226 + md5: 83698142cedb9fb4df5ab82f408690a2 + size: 222 startup_cleanup: cmd: python 0_startup_cleanup.py deps: @@ -155,3 +155,23 @@ stages: configs/settings.yaml: default.startup_cleanup.artefacts: ./data default.startup_cleanup.metrics: ./metrics + generate_scenerio_metrics: + cmd: python 5_generate_scenarios.py + deps: + - path: 5_generate_scenarios.py + hash: md5 + md5: 30f80ffeb6ee50c5f7b82943a4dc7702 + size: 4014 + params: + configs/scenarios.yaml: + default.scenarios: + input_dataclient_type: aws-s3 + output_dataclient_type: local + scenario_data_filepaths: + - s3://retrofit-data-dev/scenario_data/recommendations_scoring_data.parquet + output_filepath: ./metrics/scenario_table.md + outs: + - path: metrics/scenario_table.md + hash: md5 + md5: 36b1b26224ebbbfd5b2bbb15ae173247 + size: 1648 diff --git a/modules/ml-pipeline/src/pipeline/dvc.yaml b/modules/ml-pipeline/src/pipeline/dvc.yaml index 58889cc..b513184 100644 --- a/modules/ml-pipeline/src/pipeline/dvc.yaml +++ b/modules/ml-pipeline/src/pipeline/dvc.yaml @@ -71,6 +71,17 @@ stages: outs: - metrics/metrics.json always_changed: true + generate_scenerio_metrics: + cmd: python 5_generate_scenarios.py + deps: + - 5_generate_scenarios.py + params: + - configs/scenarios.yaml: + - default.scenarios + outs: + - metrics/scenario_table.md + always_changed: true metrics: - metrics/metrics.json - metrics/fit_metrics.json + - metrics/scenario_table.md diff --git a/modules/ml-pipeline/src/pipeline/metrics/.gitignore b/modules/ml-pipeline/src/pipeline/metrics/.gitignore index e6fbc8d..189c2ee 100644 --- a/modules/ml-pipeline/src/pipeline/metrics/.gitignore +++ b/modules/ml-pipeline/src/pipeline/metrics/.gitignore @@ -1,2 +1,3 @@ /fit_metrics.json /metrics.json +/scenario_table.md