From f4e91162ec588a730785c42286372e34cfd97f03 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Wed, 11 Oct 2023 13:23:54 +0000 Subject: [PATCH 01/23] initial model --- .../src/pipeline/configs/build_model.yaml | 2 +- .../configs/feature_processor_logic.py | 6 +++ .../pipeline/configs/post_prediction_logic.py | 6 ++- .../src/pipeline/configs/settings.yaml | 4 +- modules/ml-pipeline/src/pipeline/dvc.lock | 50 +++++++++---------- 5 files changed, 38 insertions(+), 30 deletions(-) diff --git a/modules/ml-pipeline/src/pipeline/configs/build_model.yaml b/modules/ml-pipeline/src/pipeline/configs/build_model.yaml index d296e6a..ee7394e 100644 --- a/modules/ml-pipeline/src/pipeline/configs/build_model.yaml +++ b/modules/ml-pipeline/src/pipeline/configs/build_model.yaml @@ -13,6 +13,6 @@ default: output_filepath: ./data/model/allmodels/ problem_type: regression eval_metric: mean_squared_error #mean_absolute_error - time_limit: 4000 + time_limit: 60 presets: medium_quality excluded_model_types: ['KNN', 'RF'] diff --git a/modules/ml-pipeline/src/pipeline/configs/feature_processor_logic.py b/modules/ml-pipeline/src/pipeline/configs/feature_processor_logic.py index c32d2fe..7b131dc 100644 --- a/modules/ml-pipeline/src/pipeline/configs/feature_processor_logic.py +++ b/modules/ml-pipeline/src/pipeline/configs/feature_processor_logic.py @@ -18,6 +18,11 @@ def remove_starting_columns(df): return df +def keep_negative_carbon_change(df): + df = df[df["CARBON_CHANGE"] < 0] + return df + + # def keep_ending_columns(df): # ending_column_index = [ col_name.endswith("_ENDING") for col_name in list(df.columns)] # keep_columns = df.columns[ending_column_index].to_list() @@ -27,6 +32,7 @@ def remove_starting_columns(df): # return df business_logic = { + "keep_negative_carbon_change": keep_negative_carbon_change # "remove_starting_columns": remove_starting_columns # "keep_ENDING_COLUMNS": keep_ending_columns } diff --git a/modules/ml-pipeline/src/pipeline/configs/post_prediction_logic.py b/modules/ml-pipeline/src/pipeline/configs/post_prediction_logic.py index b85d3a4..bb36713 100644 --- a/modules/ml-pipeline/src/pipeline/configs/post_prediction_logic.py +++ b/modules/ml-pipeline/src/pipeline/configs/post_prediction_logic.py @@ -12,9 +12,11 @@ def clip_predictions_to_minimum_value( predictions.name = "predictions" predictions_df = pd.concat([data, predictions], axis=1) # We expect all prediction to be atleast one point improvement - replace_index = predictions_df["SAP_STARTING"] + 1 > predictions_df["predictions"] + replace_index = ( + predictions_df["predictions"] > predictions_df["CARBON_STARTING"] - 1 + ) predictions_df.loc[replace_index, "predictions"] = ( - predictions_df.loc[replace_index, "SAP_STARTING"] + minimum_value + predictions_df.loc[replace_index, "CARBON_STARTING"] - minimum_value ) predictions_new = predictions_df["predictions"] diff --git a/modules/ml-pipeline/src/pipeline/configs/settings.yaml b/modules/ml-pipeline/src/pipeline/configs/settings.yaml index ce7ed2c..4f3ebce 100644 --- a/modules/ml-pipeline/src/pipeline/configs/settings.yaml +++ b/modules/ml-pipeline/src/pipeline/configs/settings.yaml @@ -31,9 +31,9 @@ default: feature_processor_config: subsample_amount: null subsample_seed: 0 - target: SAP_ENDING + target: CARBON_ENDING identifier_columns: ["UPRN"] - drop_columns: ["HEAT_DEMAND_CHANGE", "CARBON_CHANGE", "RDSAP_CHANGE", "HEAT_DEMAND_ENDING", "CARBON_ENDING"] + drop_columns: ["HEAT_DEMAND_CHANGE", "CARBON_CHANGE", "RDSAP_CHANGE", "HEAT_DEMAND_ENDING", "SAP_ENDING"] # retain_features: ["SAP_STARTING", "TOTAL_FLOOR_AREA_DIFF"] retain_features: null diff --git a/modules/ml-pipeline/src/pipeline/dvc.lock b/modules/ml-pipeline/src/pipeline/dvc.lock index c499874..d2291d8 100644 --- a/modules/ml-pipeline/src/pipeline/dvc.lock +++ b/modules/ml-pipeline/src/pipeline/dvc.lock @@ -14,11 +14,11 @@ stages: - CARBON_CHANGE - RDSAP_CHANGE - HEAT_DEMAND_ENDING - - CARBON_ENDING + - SAP_ENDING default.feature_processor.feature_processor_config.retain_features: default.feature_processor.feature_processor_config.subsample_amount: default.feature_processor.feature_processor_config.subsample_seed: 0 - default.feature_processor.feature_processor_config.target: SAP_ENDING + default.feature_processor.feature_processor_config.target: CARBON_ENDING default.feature_processor.feature_processor_type: dataframe default.prepare_data.data_filepath: s3://retrofit-data-dev/sap_change_model/dataset.parquet default.prepare_data.input_dataclient_type: aws-s3 @@ -29,8 +29,8 @@ stages: outs: - path: data/prepared_data/ hash: md5 - md5: 9ce5c45722da7fc40491b5a4d00daf9e.dir - size: 33881619 + md5: cd9a3d5e6208c1fd5de513b4d5c51b5b.dir + size: 30121189 nfiles: 2 build_model: cmd: python 2_build_model.py @@ -41,8 +41,8 @@ stages: size: 5359 - path: data/prepared_data hash: md5 - md5: 9ce5c45722da7fc40491b5a4d00daf9e.dir - size: 33881619 + md5: cd9a3d5e6208c1fd5de513b4d5c51b5b.dir + size: 30121189 nfiles: 2 params: configs/build_model.yaml: @@ -58,7 +58,7 @@ stages: output_filepath: ./data/model/allmodels/ problem_type: regression eval_metric: mean_squared_error - time_limit: 4000 + time_limit: 60 presets: medium_quality excluded_model_types: - KNN @@ -66,13 +66,13 @@ stages: outs: - path: data/model/ hash: md5 - md5: 7bb5156243b4db39349e80a01ffecde4.dir - size: 473398662 - nfiles: 27 + md5: 0d43e4ac3985da215dadf5fed8e68200.dir + size: 210841782 + nfiles: 21 - path: metrics/fit_metrics.json hash: md5 - md5: 2bb16ac67de8778fbc08171d562b34d5 - size: 184 + md5: 06f8bb0d004b91c33493dbee9a8763e7 + size: 206 generate_predictions: cmd: python 3_generate_predictions.py deps: @@ -82,13 +82,13 @@ stages: size: 3028 - path: data/model hash: md5 - md5: 7bb5156243b4db39349e80a01ffecde4.dir - size: 473398662 - nfiles: 27 + md5: 0d43e4ac3985da215dadf5fed8e68200.dir + size: 210841782 + nfiles: 21 - path: data/prepared_data hash: md5 - md5: 9ce5c45722da7fc40491b5a4d00daf9e.dir - size: 33881619 + md5: cd9a3d5e6208c1fd5de513b4d5c51b5b.dir + size: 30121189 nfiles: 2 params: configs/settings.yaml: @@ -100,8 +100,8 @@ stages: outs: - path: data/predictions/ hash: md5 - md5: 0bb3cf991906953def81c8204cdcfaf0.dir - size: 374532 + md5: acdcb06ee7574672b1148c10c37a868b.dir + size: 275959 nfiles: 1 generate_metrics: cmd: python 4_generate_metrics.py @@ -112,13 +112,13 @@ stages: size: 4487 - path: data/predictions hash: md5 - md5: 0bb3cf991906953def81c8204cdcfaf0.dir - size: 374532 + md5: acdcb06ee7574672b1148c10c37a868b.dir + size: 275959 nfiles: 1 - path: data/prepared_data hash: md5 - md5: 9ce5c45722da7fc40491b5a4d00daf9e.dir - size: 33881619 + md5: cd9a3d5e6208c1fd5de513b4d5c51b5b.dir + size: 30121189 nfiles: 2 params: configs/settings.yaml: @@ -128,8 +128,8 @@ stages: outs: - path: metrics/metrics.json hash: md5 - md5: 2e13ae67759a64261d03224f1c0d4bf4 - size: 185 + md5: e3bdc173023a7d909704f0313aa1609f + size: 219 startup_cleanup: cmd: python 0_startup_cleanup.py deps: From 680e879503c5b4dafb4702a1ad9cb2391cb500fa Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Wed, 11 Oct 2023 15:38:55 +0000 Subject: [PATCH 02/23] 400 second model --- .../src/pipeline/configs/build_model.yaml | 2 +- .../pipeline/configs/post_prediction_logic.py | 13 +++--- modules/ml-pipeline/src/pipeline/dvc.lock | 46 +++++++++---------- 3 files changed, 30 insertions(+), 31 deletions(-) diff --git a/modules/ml-pipeline/src/pipeline/configs/build_model.yaml b/modules/ml-pipeline/src/pipeline/configs/build_model.yaml index ee7394e..7e409bf 100644 --- a/modules/ml-pipeline/src/pipeline/configs/build_model.yaml +++ b/modules/ml-pipeline/src/pipeline/configs/build_model.yaml @@ -13,6 +13,6 @@ default: output_filepath: ./data/model/allmodels/ problem_type: regression eval_metric: mean_squared_error #mean_absolute_error - time_limit: 60 + time_limit: 400 presets: medium_quality excluded_model_types: ['KNN', 'RF'] diff --git a/modules/ml-pipeline/src/pipeline/configs/post_prediction_logic.py b/modules/ml-pipeline/src/pipeline/configs/post_prediction_logic.py index bb36713..1ffab90 100644 --- a/modules/ml-pipeline/src/pipeline/configs/post_prediction_logic.py +++ b/modules/ml-pipeline/src/pipeline/configs/post_prediction_logic.py @@ -5,19 +5,18 @@ import pandas as pd def clip_predictions_to_minimum_value( - data: pd.DataFrame, predictions: pd.Series, minimum_value: int = 1 + data: pd.DataFrame, + predictions: pd.Series, ) -> pd.Series: series_name = predictions.name predictions.name = "predictions" predictions_df = pd.concat([data, predictions], axis=1) # We expect all prediction to be atleast one point improvement - replace_index = ( - predictions_df["predictions"] > predictions_df["CARBON_STARTING"] - 1 - ) - predictions_df.loc[replace_index, "predictions"] = ( - predictions_df.loc[replace_index, "CARBON_STARTING"] - minimum_value - ) + replace_index = predictions_df["predictions"] > predictions_df["CARBON_STARTING"] + predictions_df.loc[replace_index, "predictions"] = predictions_df.loc[ + replace_index, "CARBON_STARTING" + ] predictions_new = predictions_df["predictions"] predictions_new.name = series_name diff --git a/modules/ml-pipeline/src/pipeline/dvc.lock b/modules/ml-pipeline/src/pipeline/dvc.lock index d2291d8..2056834 100644 --- a/modules/ml-pipeline/src/pipeline/dvc.lock +++ b/modules/ml-pipeline/src/pipeline/dvc.lock @@ -29,8 +29,8 @@ stages: outs: - path: data/prepared_data/ hash: md5 - md5: cd9a3d5e6208c1fd5de513b4d5c51b5b.dir - size: 30121189 + md5: 5fd3c01804ee2994ee77fc501d178be4.dir + size: 30137355 nfiles: 2 build_model: cmd: python 2_build_model.py @@ -41,8 +41,8 @@ stages: size: 5359 - path: data/prepared_data hash: md5 - md5: cd9a3d5e6208c1fd5de513b4d5c51b5b.dir - size: 30121189 + md5: 5fd3c01804ee2994ee77fc501d178be4.dir + size: 30137355 nfiles: 2 params: configs/build_model.yaml: @@ -58,7 +58,7 @@ stages: output_filepath: ./data/model/allmodels/ problem_type: regression eval_metric: mean_squared_error - time_limit: 60 + time_limit: 400 presets: medium_quality excluded_model_types: - KNN @@ -66,13 +66,13 @@ stages: outs: - path: data/model/ hash: md5 - md5: 0d43e4ac3985da215dadf5fed8e68200.dir - size: 210841782 - nfiles: 21 + md5: 4b49c12395a645e35e50a9de8840f08d.dir + size: 282024140 + nfiles: 24 - path: metrics/fit_metrics.json hash: md5 - md5: 06f8bb0d004b91c33493dbee9a8763e7 - size: 206 + md5: a6d139fa59f5ddf75023bb7d3364f6d2 + size: 225 generate_predictions: cmd: python 3_generate_predictions.py deps: @@ -82,13 +82,13 @@ stages: size: 3028 - path: data/model hash: md5 - md5: 0d43e4ac3985da215dadf5fed8e68200.dir - size: 210841782 - nfiles: 21 + md5: 4b49c12395a645e35e50a9de8840f08d.dir + size: 282024140 + nfiles: 24 - path: data/prepared_data hash: md5 - md5: cd9a3d5e6208c1fd5de513b4d5c51b5b.dir - size: 30121189 + md5: 5fd3c01804ee2994ee77fc501d178be4.dir + size: 30137355 nfiles: 2 params: configs/settings.yaml: @@ -100,8 +100,8 @@ stages: outs: - path: data/predictions/ hash: md5 - md5: acdcb06ee7574672b1148c10c37a868b.dir - size: 275959 + md5: 8f724261b3d17bf87067e91a1ff99077.dir + size: 441423 nfiles: 1 generate_metrics: cmd: python 4_generate_metrics.py @@ -112,13 +112,13 @@ stages: size: 4487 - path: data/predictions hash: md5 - md5: acdcb06ee7574672b1148c10c37a868b.dir - size: 275959 + md5: 8f724261b3d17bf87067e91a1ff99077.dir + size: 441423 nfiles: 1 - path: data/prepared_data hash: md5 - md5: cd9a3d5e6208c1fd5de513b4d5c51b5b.dir - size: 30121189 + md5: 5fd3c01804ee2994ee77fc501d178be4.dir + size: 30137355 nfiles: 2 params: configs/settings.yaml: @@ -128,8 +128,8 @@ stages: outs: - path: metrics/metrics.json hash: md5 - md5: e3bdc173023a7d909704f0313aa1609f - size: 219 + md5: 38787835f838f65c6cc75654843eb311 + size: 223 startup_cleanup: cmd: python 0_startup_cleanup.py deps: From bb2db16f611336d2d88c6406e9f831903f5d6fb4 Mon Sep 17 00:00:00 2001 From: Github-Bot Date: Wed, 11 Oct 2023 15:48:04 +0000 Subject: [PATCH 03/23] Update Registry --- MODEL_REGISTRY.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/MODEL_REGISTRY.md b/MODEL_REGISTRY.md index b3ad75a..db53665 100644 --- a/MODEL_REGISTRY.md +++ b/MODEL_REGISTRY.md @@ -22,5 +22,13 @@ }, "registered": true, "active": true + }, + "carbon": { + "version": "v0.0.1", + "stage": { + "dev": null + }, + "registered": true, + "active": true } } From 2f9092f44740728a7c3fc1833db3c17fdfa890c7 Mon Sep 17 00:00:00 2001 From: Github-Bot Date: Wed, 11 Oct 2023 15:48:52 +0000 Subject: [PATCH 04/23] Update Registry --- MODEL_REGISTRY.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MODEL_REGISTRY.md b/MODEL_REGISTRY.md index db53665..c505c17 100644 --- a/MODEL_REGISTRY.md +++ b/MODEL_REGISTRY.md @@ -26,7 +26,7 @@ "carbon": { "version": "v0.0.1", "stage": { - "dev": null + "dev": "v0.0.1" }, "registered": true, "active": true From f2cc32f4b4b2a8c5c7e7d461744e700815b89def Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Thu, 12 Oct 2023 08:38:55 +0000 Subject: [PATCH 05/23] using good model 4000s --- .../src/pipeline/configs/build_model.yaml | 4 +- modules/ml-pipeline/src/pipeline/dvc.lock | 48 +++++++++---------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/modules/ml-pipeline/src/pipeline/configs/build_model.yaml b/modules/ml-pipeline/src/pipeline/configs/build_model.yaml index 7e409bf..ced4159 100644 --- a/modules/ml-pipeline/src/pipeline/configs/build_model.yaml +++ b/modules/ml-pipeline/src/pipeline/configs/build_model.yaml @@ -13,6 +13,6 @@ default: output_filepath: ./data/model/allmodels/ problem_type: regression eval_metric: mean_squared_error #mean_absolute_error - time_limit: 400 - presets: medium_quality + time_limit: 4000 + presets: good_quality excluded_model_types: ['KNN', 'RF'] diff --git a/modules/ml-pipeline/src/pipeline/dvc.lock b/modules/ml-pipeline/src/pipeline/dvc.lock index 2056834..689bb64 100644 --- a/modules/ml-pipeline/src/pipeline/dvc.lock +++ b/modules/ml-pipeline/src/pipeline/dvc.lock @@ -29,8 +29,8 @@ stages: outs: - path: data/prepared_data/ hash: md5 - md5: 5fd3c01804ee2994ee77fc501d178be4.dir - size: 30137355 + md5: 91407be844d5cfe428bf9d09e980fc0e.dir + size: 30051355 nfiles: 2 build_model: cmd: python 2_build_model.py @@ -41,8 +41,8 @@ stages: size: 5359 - path: data/prepared_data hash: md5 - md5: 5fd3c01804ee2994ee77fc501d178be4.dir - size: 30137355 + md5: 91407be844d5cfe428bf9d09e980fc0e.dir + size: 30051355 nfiles: 2 params: configs/build_model.yaml: @@ -58,21 +58,21 @@ stages: output_filepath: ./data/model/allmodels/ problem_type: regression eval_metric: mean_squared_error - time_limit: 400 - presets: medium_quality + time_limit: 4000 + presets: good_quality excluded_model_types: - KNN - RF outs: - path: data/model/ hash: md5 - md5: 4b49c12395a645e35e50a9de8840f08d.dir - size: 282024140 - nfiles: 24 + md5: 229de034422caa37a4e24366b572bc29.dir + size: 317288619 + nfiles: 138 - path: metrics/fit_metrics.json hash: md5 - md5: a6d139fa59f5ddf75023bb7d3364f6d2 - size: 225 + md5: 50005e7cbac69d6c888cdd0f929b9240 + size: 226 generate_predictions: cmd: python 3_generate_predictions.py deps: @@ -82,13 +82,13 @@ stages: size: 3028 - path: data/model hash: md5 - md5: 4b49c12395a645e35e50a9de8840f08d.dir - size: 282024140 - nfiles: 24 + md5: 229de034422caa37a4e24366b572bc29.dir + size: 317288619 + nfiles: 138 - path: data/prepared_data hash: md5 - md5: 5fd3c01804ee2994ee77fc501d178be4.dir - size: 30137355 + md5: 91407be844d5cfe428bf9d09e980fc0e.dir + size: 30051355 nfiles: 2 params: configs/settings.yaml: @@ -100,8 +100,8 @@ stages: outs: - path: data/predictions/ hash: md5 - md5: 8f724261b3d17bf87067e91a1ff99077.dir - size: 441423 + md5: 40d5bf4d265018e8a181287846e4f892.dir + size: 441331 nfiles: 1 generate_metrics: cmd: python 4_generate_metrics.py @@ -112,13 +112,13 @@ stages: size: 4487 - path: data/predictions hash: md5 - md5: 8f724261b3d17bf87067e91a1ff99077.dir - size: 441423 + md5: 40d5bf4d265018e8a181287846e4f892.dir + size: 441331 nfiles: 1 - path: data/prepared_data hash: md5 - md5: 5fd3c01804ee2994ee77fc501d178be4.dir - size: 30137355 + md5: 91407be844d5cfe428bf9d09e980fc0e.dir + size: 30051355 nfiles: 2 params: configs/settings.yaml: @@ -128,8 +128,8 @@ stages: outs: - path: metrics/metrics.json hash: md5 - md5: 38787835f838f65c6cc75654843eb311 - size: 223 + md5: d6217ba41c60e4ec452670faf07321ab + size: 224 startup_cleanup: cmd: python 0_startup_cleanup.py deps: From 217fb3dca8786b0cd43e7eba9b5c651ad20dfb26 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Mon, 27 Nov 2023 18:52:47 +0000 Subject: [PATCH 06/23] add inference speed check --- deployment/handlers/prediction_app.py | 6 -- .../src/pipeline/0_startup_cleanup.py | 10 --- .../src/pipeline/1_prepare_data.py | 20 ------ .../ml-pipeline/src/pipeline/2_build_model.py | 26 ------- .../src/pipeline/3_generate_predictions.py | 12 ---- .../src/pipeline/4_generate_metrics.py | 22 ------ .../src/pipeline/configs/build_model.yaml | 6 +- .../src/pipeline/configs/settings.yaml | 2 +- .../ml-pipeline/src/pipeline/core/Logger.py | 1 + .../ml-pipeline/src/pipeline/core/MLModels.py | 4 ++ modules/ml-pipeline/src/pipeline/dvc.lock | 68 ++++++++++--------- .../src/pipeline/generate_predictions.py | 8 --- 12 files changed, 45 insertions(+), 140 deletions(-) diff --git a/deployment/handlers/prediction_app.py b/deployment/handlers/prediction_app.py index 36a906c..ac397b9 100644 --- a/deployment/handlers/prediction_app.py +++ b/deployment/handlers/prediction_app.py @@ -69,9 +69,7 @@ def handler(event, context): storage_filepath = f"s3://{PREDICTIONS_BUCKET}/{portfolio_id}/{property_id}/{created_at}.parquet" - logger.info("-------------------------") logger.info(f"--- Initiate MLModel ---") - logger.info("-------------------------") build_model_params = settings.build_model client_params = settings.client @@ -80,17 +78,13 @@ def handler(event, context): model = model_factory(build_model_params["model_type"]) - logger.info("----------------------------") logger.info(f"--- Initiate Input DataClient ---") - logger.info("----------------------------") input_dataclient = dataclient_factory( dataclient_type="aws-s3", dataclient_config=client_params["aws-s3"], ) - logger.info("----------------------------") logger.info(f"--- Initiate Output DataClient ---") - logger.info("----------------------------") output_dataclient = dataclient_factory( dataclient_type="aws-s3", dataclient_config=client_params["aws-s3"], diff --git a/modules/ml-pipeline/src/pipeline/0_startup_cleanup.py b/modules/ml-pipeline/src/pipeline/0_startup_cleanup.py index 0bfa37f..32e8a1b 100644 --- a/modules/ml-pipeline/src/pipeline/0_startup_cleanup.py +++ b/modules/ml-pipeline/src/pipeline/0_startup_cleanup.py @@ -16,13 +16,9 @@ def run_cleanup(artefacts_directory: str, metrics_directory: str) -> None: Remove the directory where artefacts are stored """ - logger.info("---------------------") logger.info(f"--- Run Clean up ---") - logger.info("---------------------") - logger.info("-------------------------") logger.info(f"--- Delete artefacts ---") - logger.info("-------------------------") artefact_directory_path = Path(artefacts_directory) @@ -31,9 +27,7 @@ def run_cleanup(artefacts_directory: str, metrics_directory: str) -> None: logger.info(f"Removing the directory: {artefacts_directory}") shutil.rmtree(artefact_directory_path) - logger.info("-----------------------") logger.info(f"--- Delete metrics ---") - logger.info("-----------------------") metrics_directory_path = Path(metrics_directory) @@ -45,15 +39,11 @@ def run_cleanup(artefacts_directory: str, metrics_directory: str) -> None: if __name__ == "__main__": - logger.info("----------------------------") logger.info(f"--- {__file__} - Start! ---") - logger.info("----------------------------") run_cleanup( artefacts_directory=startup_cleanup_params["artefacts"], metrics_directory=startup_cleanup_params["metrics"], ) - logger.info("-------------------------------") logger.info(f"--- {__file__} - Complete! ---") - logger.info("-------------------------------") diff --git a/modules/ml-pipeline/src/pipeline/1_prepare_data.py b/modules/ml-pipeline/src/pipeline/1_prepare_data.py index 32daa19..ed7e057 100644 --- a/modules/ml-pipeline/src/pipeline/1_prepare_data.py +++ b/modules/ml-pipeline/src/pipeline/1_prepare_data.py @@ -17,9 +17,7 @@ from core.DataClient import dataclient_factory from core.FeatureProcessor import feature_processor_factory from config import settings -logger.info("----------------------------") logger.info(f"--- Initiate Parameters ---") -logger.info("----------------------------") RUNTIME_ENVIRONMENT = os.environ.get("RUNTIME_ENVIRONMENT", "local") @@ -33,9 +31,7 @@ output_train_filepath = prepare_data_params["output_train_filepath"] output_test_filepath = prepare_data_params["output_test_filepath"] feature_processor_config = feature_process_params["feature_processor_config"] -logger.info("----------------------------") logger.info(f"--- Initiate DataClient ---") -logger.info("----------------------------") input_dataclient_type = prepare_data_params["input_dataclient_type"] output_dataclient_type = prepare_data_params["output_dataclient_type"] @@ -49,9 +45,7 @@ output_dataclient = dataclient_factory( dataclient_config=client_params[output_dataclient_type], ) -logger.info("----------------------------------") logger.info(f"--- Initiate FeatureProcessor ---") -logger.info("----------------------------------") feature_processor = feature_processor_factory( feature_process_params["feature_processor_type"] @@ -76,15 +70,11 @@ def prepare_data( :param pipeline_mode: bool, Default False, this caches out the file for experimentation, objects returned in pipeline mode """ - logger.info("--------------------") logger.info("--- Loading data ---") - logger.info("--------------------") data = input_dataclient.load_data(location=data_filepath, load_config={}) - logger.info("--------------------------") logger.info("--- Feature Processing ---") - logger.info("--------------------------") data = feature_processor.feature_process( data, @@ -93,9 +83,7 @@ def prepare_data( new_feature_funcs=new_feature_funcs, ) - logger.info("----------------------") logger.info("--- Splitting data ---") - logger.info("----------------------") if train_proportion == 1: train = data @@ -108,9 +96,7 @@ def prepare_data( train = train.reset_index(drop=True) - logger.info("-----------------------") logger.info("--- Outputting data ---") - logger.info("-----------------------") output_dataclient.save_data( obj=train, location=output_train_filepath, save_config=None @@ -126,13 +112,9 @@ def prepare_data( if __name__ == "__main__": - logger.info("----------------------------") logger.info(f"--- {__file__} - Start! ---") - logger.info("----------------------------") - logger.info("---------------------------") logger.info(f"--- Prepare Data Stage ---") - logger.info("---------------------------") prepare_data( input_dataclient=input_dataclient, @@ -147,6 +129,4 @@ if __name__ == "__main__": new_feature_funcs=new_feature_funcs, ) - logger.info("-------------------------------") logger.info(f"--- {__file__} - Complete! ---") - logger.info("-------------------------------") diff --git a/modules/ml-pipeline/src/pipeline/2_build_model.py b/modules/ml-pipeline/src/pipeline/2_build_model.py index cae5cfd..7ca4951 100644 --- a/modules/ml-pipeline/src/pipeline/2_build_model.py +++ b/modules/ml-pipeline/src/pipeline/2_build_model.py @@ -18,9 +18,7 @@ from core.MLMetrics import metrics_factory from configs.post_prediction_logic import post_prediction_logic from config import settings -logger.info("----------------------------") logger.info(f"--- Initiate Parameters ---") -logger.info("----------------------------") RUNTIME_ENVIRONMENT = os.environ.get("RUNTIME_ENVIRONMENT", "local") @@ -40,22 +38,16 @@ train_filepath = prepare_data_params["output_train_filepath"] test_filepath = prepare_data_params["output_test_filepath"] fit_metrics_filepath = build_model_params["fit_metrics_filepath"] -logger.info("----------------------------") logger.info(f"--- Initiate DataClient ---") -logger.info("----------------------------") # Output of previous prepare data step, will be where the data is dataclient = dataclient_factory(prepare_data_params["output_dataclient_type"]) -logger.info("-------------------------") logger.info(f"--- Initiate MLModel ---") -logger.info("-------------------------") model = model_factory(model_type) -logger.info("-------------------------") logger.info(f"--- Initiate Metrics ---") -logger.info("-------------------------") metrics = metrics_factory(generate_metrics_params["metrics_type"]) @@ -75,9 +67,7 @@ def build_model( test_data: Union[pd.DataFrame, None] = None, pipeline_mode: bool = False, ): - logger.info("--------------------------------------") logger.info("--- Loading Data for build process ---") - logger.info("--------------------------------------") if train_data is None: if train_filepath is None: @@ -89,9 +79,7 @@ def build_model( raise ValueError(f"Need {test_filepath} if no data supplied") test_data = dataclient.load_data(location=test_filepath, load_config=None) - logger.info("----------------------") logger.info("--- Training model ---") - logger.info("----------------------") model.train_model( data=train_data.drop(columns=identifier_columns), @@ -99,32 +87,24 @@ def build_model( model_hyperparameters=model_hyperparameters, ) - logger.info("----------------------------------") logger.info("--- Generating fit predictions ---") - logger.info("----------------------------------") fit_predictions = model.predict( data=train_data, post_prediction_logic=post_prediction_logic ) - logger.info("------------------------------") logger.info("--- Generating fit metrics ---") - logger.info("------------------------------") metrics_output = metrics.generate_metrics( target=train_data[target], predictions=pd.Series(fit_predictions), ) - logger.info("--------------------") logger.info("--- Saving model ---") - logger.info("--------------------") model.save_model(path=Path(model_save_location)) - logger.info("--------------------------") logger.info("--- Saving fit metrics ---") - logger.info("--------------------------") dataclient.save_data( obj=metrics_output, location=fit_metrics_filepath, save_config=None @@ -133,13 +113,9 @@ def build_model( if __name__ == "__main__": - logger.info("----------------------------") logger.info(f"--- {__file__} - Start! ---") - logger.info("----------------------------") - logger.info("--------------------------") logger.info(f"--- Build Model Stage ---") - logger.info("--------------------------") build_model( dataclient=dataclient, @@ -154,6 +130,4 @@ if __name__ == "__main__": fit_metrics_filepath=fit_metrics_filepath, ) - logger.info("-------------------------------") logger.info(f"--- {__file__} - Complete! ---") - logger.info("-------------------------------") diff --git a/modules/ml-pipeline/src/pipeline/3_generate_predictions.py b/modules/ml-pipeline/src/pipeline/3_generate_predictions.py index 9461392..acb9e99 100644 --- a/modules/ml-pipeline/src/pipeline/3_generate_predictions.py +++ b/modules/ml-pipeline/src/pipeline/3_generate_predictions.py @@ -10,9 +10,7 @@ from core.Logger import logger from config import settings from generate_predictions import generate_predictions -logger.info("----------------------------") logger.info(f"--- Initiate Parameters ---") -logger.info("----------------------------") RUNTIME_ENVIRONMENT = os.environ.get("RUNTIME_ENVIRONMENT", "local") @@ -33,15 +31,11 @@ model_filepath = build_model_params["model_save_filepath"] predictions_output_filepath = generate_predictions_params["predictions_output_filepath"] predictions_column_name = generate_predictions_params["predictions_column_name"] -logger.info("-------------------------") logger.info(f"--- Initiate MLModel ---") -logger.info("-------------------------") model = model_factory(build_model_params["model_type"]) -logger.info("----------------------------") logger.info(f"--- Initiate DataClient ---") -logger.info("----------------------------") # We may have different locations of loading hence why we use one specified in generate_predictions.yaml # I.e. for metric runs, this will be a local data client @@ -59,13 +53,9 @@ output_dataclient = dataclient_factory( if __name__ == "__main__": - logger.info("----------------------------") logger.info(f"--- {__file__} - Start! ---") - logger.info("----------------------------") - logger.info("----------------------------------") logger.info(f"--- Generate Predictions Stage---") - logger.info("----------------------------------") generate_predictions( input_dataclient=input_dataclient, @@ -78,6 +68,4 @@ if __name__ == "__main__": predictions_column_name=predictions_column_name, ) - logger.info("-------------------------------") logger.info(f"--- {__file__} - Complete! ---") - logger.info("-------------------------------") diff --git a/modules/ml-pipeline/src/pipeline/4_generate_metrics.py b/modules/ml-pipeline/src/pipeline/4_generate_metrics.py index 7b115a2..937c5be 100644 --- a/modules/ml-pipeline/src/pipeline/4_generate_metrics.py +++ b/modules/ml-pipeline/src/pipeline/4_generate_metrics.py @@ -16,9 +16,7 @@ from core.MLMetrics import metrics_factory from core.Logger import logger from config import settings -logger.info("----------------------------") logger.info(f"--- Initiate Parameters ---") -logger.info("----------------------------") RUNTIME_ENVIRONMENT = os.environ.get("RUNTIME_ENVIRONMENT", "local") @@ -36,15 +34,11 @@ predictions_column_name = generate_predictions_params["predictions_column_name"] metrics_output_filepath = generate_metrics_params["metrics_output_filepath"] -logger.info("-------------------------") logger.info(f"--- Initiate MLModel ---") -logger.info("-------------------------") model = model_factory(build_model_params["model_type"]) -logger.info("----------------------------") logger.info(f"--- Initiate DataClient ---") -logger.info("----------------------------") # Use data client for input and output, as we use dvc to cache later to the cloud dataclient_type = generate_metrics_params["dataclient_type"] @@ -53,9 +47,7 @@ dataclient = dataclient_factory( dataclient_config=client_params[dataclient_type], ) -logger.info("---------------------------") logger.info(f"--- Initiate MLMetrics ---") -logger.info("---------------------------") metrics = metrics_factory(generate_metrics_params["metrics_type"]) @@ -75,34 +67,26 @@ def generate_metrics( For a given model, we generate prediction and evaluate this against the true target """ - logger.info("-------------------------") logger.info("--- Loading test data ---") - logger.info("-------------------------") test_data = input_dataclient.load_data( location=test_data_filepath, load_config=None ) - logger.info("---------------------------") logger.info("--- Loading predictions ---") - logger.info("---------------------------") predictions = input_dataclient.load_data( location=predictions_output_filepath, load_config=None ) - logger.info("--------------------------") logger.info("--- Generating metrics ---") - logger.info("--------------------------") metrics_output = metrics.generate_metrics( target=test_data[target], predictions=pd.Series(predictions[predictions_column_name]), ) - logger.info("----------------------") logger.info("--- Saving metrics ---") - logger.info("----------------------") output_dataclient.save_data( obj=metrics_output, location=metrics_output_filepath, save_config=None @@ -111,13 +95,9 @@ def generate_metrics( if __name__ == "__main__": - logger.info("----------------------------") logger.info(f"--- {__file__} - Start! ---") - logger.info("----------------------------") - logger.info("------------------------------") logger.info(f"--- Generate Metrics Stage---") - logger.info("------------------------------") generate_metrics( input_dataclient=dataclient, @@ -131,6 +111,4 @@ if __name__ == "__main__": metrics_output_filepath=metrics_output_filepath, ) - logger.info("-------------------------------") logger.info(f"--- {__file__} - Complete! ---") - logger.info("-------------------------------") diff --git a/modules/ml-pipeline/src/pipeline/configs/build_model.yaml b/modules/ml-pipeline/src/pipeline/configs/build_model.yaml index ced4159..4c72487 100644 --- a/modules/ml-pipeline/src/pipeline/configs/build_model.yaml +++ b/modules/ml-pipeline/src/pipeline/configs/build_model.yaml @@ -13,6 +13,8 @@ default: output_filepath: ./data/model/allmodels/ problem_type: regression eval_metric: mean_squared_error #mean_absolute_error - time_limit: 4000 - presets: good_quality + time_limit: 400 + presets: medium_quality excluded_model_types: ['KNN', 'RF'] + infer_limit: 0.05 + infer_limit_batch_size: 10000 diff --git a/modules/ml-pipeline/src/pipeline/configs/settings.yaml b/modules/ml-pipeline/src/pipeline/configs/settings.yaml index 4f3ebce..5514406 100644 --- a/modules/ml-pipeline/src/pipeline/configs/settings.yaml +++ b/modules/ml-pipeline/src/pipeline/configs/settings.yaml @@ -21,7 +21,7 @@ default: # data_filepath: s3://retrofit-data-dev/sap_change_model/dataset_with_differencing.parquet # data_filepath: s3://retrofit-data-dev/sap_change_model/floor_area_clean_test.parquet # data_filepath: s3://retrofit-data-dev/sap_change_model/dataset_without_differencing.parquet - data_filepath: s3://retrofit-data-dev/sap_change_model/dataset.parquet + data_filepath: s3://retrofit-data-dev/sap_change_model/dataset_test.parquet train_proportion: 0.9 output_train_filepath: ./data/prepared_data/train.parquet output_test_filepath: ./data/prepared_data/test.parquet diff --git a/modules/ml-pipeline/src/pipeline/core/Logger.py b/modules/ml-pipeline/src/pipeline/core/Logger.py index a0fc231..d2f6c61 100644 --- a/modules/ml-pipeline/src/pipeline/core/Logger.py +++ b/modules/ml-pipeline/src/pipeline/core/Logger.py @@ -21,6 +21,7 @@ def setup_logger(): # Add the stream handler to the logger logger.addHandler(stream_handler) + logger.propagate = False return logger diff --git a/modules/ml-pipeline/src/pipeline/core/MLModels.py b/modules/ml-pipeline/src/pipeline/core/MLModels.py index 4cf8b08..4fc572a 100644 --- a/modules/ml-pipeline/src/pipeline/core/MLModels.py +++ b/modules/ml-pipeline/src/pipeline/core/MLModels.py @@ -149,6 +149,8 @@ class AutogluonAutoML: "time_limit", "presets", "excluded_model_types", + "infer_limit", + "infer_limit_batch_size", ] def load_model(self, path: Union[Path, str]) -> None: @@ -203,6 +205,8 @@ class AutogluonAutoML: time_limit=model_hyperparameters["time_limit"], presets=model_hyperparameters["presets"], excluded_model_types=model_hyperparameters["excluded_model_types"], + infer_limit=model_hyperparameters["infer_limit"], + infer_limit_batch_size=model_hyperparameters["infer_limit_batch_size"], ) def predict( diff --git a/modules/ml-pipeline/src/pipeline/dvc.lock b/modules/ml-pipeline/src/pipeline/dvc.lock index 689bb64..627af99 100644 --- a/modules/ml-pipeline/src/pipeline/dvc.lock +++ b/modules/ml-pipeline/src/pipeline/dvc.lock @@ -5,8 +5,8 @@ stages: deps: - path: 1_prepare_data.py hash: md5 - md5: c9f030df733e318b80d1fa91b7732f79 - size: 5132 + md5: 896d3d88a4a9f68d174efe71dc089517 + size: 4222 params: configs/settings.yaml: default.feature_processor.feature_processor_config.drop_columns: @@ -20,7 +20,7 @@ stages: default.feature_processor.feature_processor_config.subsample_seed: 0 default.feature_processor.feature_processor_config.target: CARBON_ENDING default.feature_processor.feature_processor_type: dataframe - default.prepare_data.data_filepath: s3://retrofit-data-dev/sap_change_model/dataset.parquet + default.prepare_data.data_filepath: s3://retrofit-data-dev/sap_change_model/dataset_test.parquet default.prepare_data.input_dataclient_type: aws-s3 default.prepare_data.output_dataclient_type: local default.prepare_data.output_test_filepath: ./data/prepared_data/test.parquet @@ -29,20 +29,20 @@ stages: outs: - path: data/prepared_data/ hash: md5 - md5: 91407be844d5cfe428bf9d09e980fc0e.dir - size: 30051355 + md5: 73c1f7be21be8358a73c4ab5f9ec8e39.dir + size: 32943109 nfiles: 2 build_model: cmd: python 2_build_model.py deps: - path: 2_build_model.py hash: md5 - md5: 84699d208874c52accaff61c6af9bb0a - size: 5359 + md5: b824822475c222521516493e68eef9c5 + size: 4149 - path: data/prepared_data hash: md5 - md5: 91407be844d5cfe428bf9d09e980fc0e.dir - size: 30051355 + md5: 73c1f7be21be8358a73c4ab5f9ec8e39.dir + size: 32943109 nfiles: 2 params: configs/build_model.yaml: @@ -58,37 +58,39 @@ stages: output_filepath: ./data/model/allmodels/ problem_type: regression eval_metric: mean_squared_error - time_limit: 4000 - presets: good_quality + time_limit: 400 + presets: medium_quality excluded_model_types: - KNN - RF + infer_limit: 0.05 + infer_limit_batch_size: 10000 outs: - path: data/model/ hash: md5 - md5: 229de034422caa37a4e24366b572bc29.dir - size: 317288619 - nfiles: 138 + md5: dee1a60e6a9f4695272da8127196f714.dir + size: 326732699 + nfiles: 24 - path: metrics/fit_metrics.json hash: md5 - md5: 50005e7cbac69d6c888cdd0f929b9240 + md5: 1fefa99c7bc50d09c31bf175d5b9ee9c size: 226 generate_predictions: cmd: python 3_generate_predictions.py deps: - path: 3_generate_predictions.py hash: md5 - md5: 5ef2856a5a977304f1ec01f9b4205262 - size: 3028 + md5: 0a70ad4dfe99414a75d1261c75a177b9 + size: 2464 - path: data/model hash: md5 - md5: 229de034422caa37a4e24366b572bc29.dir - size: 317288619 - nfiles: 138 + md5: dee1a60e6a9f4695272da8127196f714.dir + size: 326732699 + nfiles: 24 - path: data/prepared_data hash: md5 - md5: 91407be844d5cfe428bf9d09e980fc0e.dir - size: 30051355 + md5: 73c1f7be21be8358a73c4ab5f9ec8e39.dir + size: 32943109 nfiles: 2 params: configs/settings.yaml: @@ -100,25 +102,25 @@ stages: outs: - path: data/predictions/ hash: md5 - md5: 40d5bf4d265018e8a181287846e4f892.dir - size: 441331 + md5: d2da3b713811952b66e2c5f8c95f5407.dir + size: 410646 nfiles: 1 generate_metrics: cmd: python 4_generate_metrics.py deps: - path: 4_generate_metrics.py hash: md5 - md5: 2c9fb78955a8c19cff0a098976f81d1b - size: 4487 + md5: d09a80dd55f1f69e2a832b1991b3c406 + size: 3485 - path: data/predictions hash: md5 - md5: 40d5bf4d265018e8a181287846e4f892.dir - size: 441331 + md5: d2da3b713811952b66e2c5f8c95f5407.dir + size: 410646 nfiles: 1 - path: data/prepared_data hash: md5 - md5: 91407be844d5cfe428bf9d09e980fc0e.dir - size: 30051355 + md5: 73c1f7be21be8358a73c4ab5f9ec8e39.dir + size: 32943109 nfiles: 2 params: configs/settings.yaml: @@ -128,15 +130,15 @@ stages: outs: - path: metrics/metrics.json hash: md5 - md5: d6217ba41c60e4ec452670faf07321ab + md5: 4ed2edc06b4dad3c094a2d1be374a5de size: 224 startup_cleanup: cmd: python 0_startup_cleanup.py deps: - path: 0_startup_cleanup.py hash: md5 - md5: fbb7e3b1b98b517c870f3e1df3e7f695 - size: 1676 + md5: b1b12f6b6393fbf8b83d23684df0a3d4 + size: 1220 params: configs/settings.yaml: default.startup_cleanup.artefacts: ./data diff --git a/modules/ml-pipeline/src/pipeline/generate_predictions.py b/modules/ml-pipeline/src/pipeline/generate_predictions.py index 83ea103..59ce732 100644 --- a/modules/ml-pipeline/src/pipeline/generate_predictions.py +++ b/modules/ml-pipeline/src/pipeline/generate_predictions.py @@ -20,23 +20,17 @@ def generate_predictions( For a given model, we generate prediction and evaluate this against the true target """ - logger.info("-------------------------") logger.info("--- Loading test data ---") - logger.info("-------------------------") test_data = input_dataclient.load_data( location=test_data_filepath, load_config=None ) - logger.info("---------------------") logger.info("--- Loading model ---") - logger.info("---------------------") model.load_model(model_filepath) - logger.info("------------------------------") logger.info("--- Generating predictions ---") - logger.info("------------------------------") prediction_data = ( test_data.drop(columns=target) if target in test_data.columns else test_data @@ -46,9 +40,7 @@ def generate_predictions( data=prediction_data, post_prediction_logic=post_prediction_logic ) - logger.info("--------------------------") logger.info("--- Saving predictions ---") - logger.info("--------------------------") predictions_df = pd.DataFrame(predictions) predictions_df.columns = [predictions_column_name] From f29d6af6a28a7816f00a31bf3600cb0a22b0e1e3 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Mon, 27 Nov 2023 19:13:23 +0000 Subject: [PATCH 07/23] change readme --- modules/ml-pipeline/src/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ml-pipeline/src/README.md b/modules/ml-pipeline/src/README.md index d7afc6a..db1b8b4 100644 --- a/modules/ml-pipeline/src/README.md +++ b/modules/ml-pipeline/src/README.md @@ -1,3 +1,3 @@ -# The generic reproducible ML-pipeline +# The generic reproducible ML-pipeline! Pipeline required to build a model to produce an output, that gets hashed via DVC From 03322a13e77950ac3bae28249ac40da5c24584ed Mon Sep 17 00:00:00 2001 From: Github-Bot Date: Mon, 27 Nov 2023 19:21:22 +0000 Subject: [PATCH 08/23] Update Registry --- MODEL_REGISTRY.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/MODEL_REGISTRY.md b/MODEL_REGISTRY.md index c505c17..70fca89 100644 --- a/MODEL_REGISTRY.md +++ b/MODEL_REGISTRY.md @@ -8,9 +8,9 @@ "active": true }, "sap": { - "version": "v0.1.0", + "version": "v0.2.6", "stage": { - "dev": "v0.1.0" + "dev": "v0.2.6" }, "registered": true, "active": true @@ -24,7 +24,7 @@ "active": true }, "carbon": { - "version": "v0.0.1", + "version": "v0.1.0", "stage": { "dev": "v0.0.1" }, From 62f2f83b0a0100ada54f0ee639dc5f6bec9d3c05 Mon Sep 17 00:00:00 2001 From: Github-Bot Date: Mon, 27 Nov 2023 19:22:00 +0000 Subject: [PATCH 09/23] Update Registry --- MODEL_REGISTRY.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MODEL_REGISTRY.md b/MODEL_REGISTRY.md index 70fca89..2dac564 100644 --- a/MODEL_REGISTRY.md +++ b/MODEL_REGISTRY.md @@ -26,7 +26,7 @@ "carbon": { "version": "v0.1.0", "stage": { - "dev": "v0.0.1" + "dev": "v0.1.0" }, "registered": true, "active": true From 2b1e8b912b99cd7fd259da2becff17d76384de05 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 28 Nov 2023 15:13:42 +0000 Subject: [PATCH 10/23] restrict dataset --- .../configs/feature_processor_logic.py | 37 ++++++++++++++++++- modules/ml-pipeline/src/pipeline/dvc.lock | 36 +++++++++--------- 2 files changed, 54 insertions(+), 19 deletions(-) diff --git a/modules/ml-pipeline/src/pipeline/configs/feature_processor_logic.py b/modules/ml-pipeline/src/pipeline/configs/feature_processor_logic.py index 7b131dc..78c29a9 100644 --- a/modules/ml-pipeline/src/pipeline/configs/feature_processor_logic.py +++ b/modules/ml-pipeline/src/pipeline/configs/feature_processor_logic.py @@ -18,11 +18,42 @@ def remove_starting_columns(df): return df +def keep_negative_heat_change(df): + df = df[df["HEAT_DEMAND_CHANGE"] < 0] + return df + + def keep_negative_carbon_change(df): df = df[df["CARBON_CHANGE"] < 0] return df +# TODO: Move to ETL pipeline +def remove_unreasonable_habitable_rooms(df): + """ + Assumption is that proportion of floor area to habitable rooms should be at least 6.5m2 + """ + minimum_room_size_index = ( + df["TOTAL_FLOOR_AREA_ENDING"] / df["NUMBER_HABITABLE_ROOMS"] >= 6.5 + ) + df = df[minimum_room_size_index] + return df + + +def remove_top_1_percent_heat_demand(df): + # threshold_value = df.describe(percentiles=[0.99])['HEAT_DEMAND_STARTING']['99%'] + threshold_value = 860 + df = df[df["HEAT_DEMAND_STARTING"] < threshold_value] + return df + + +def remove_top_1_percent_carbon(df): + # threshold_value = df.describe(percentiles=[0.99])['CARBON_STARTING']['99%'] + threshold_value = 18 + df = df[df["CARBON_STARTING"] < threshold_value] + return df + + # def keep_ending_columns(df): # ending_column_index = [ col_name.endswith("_ENDING") for col_name in list(df.columns)] # keep_columns = df.columns[ending_column_index].to_list() @@ -32,7 +63,11 @@ def keep_negative_carbon_change(df): # return df business_logic = { - "keep_negative_carbon_change": keep_negative_carbon_change + "remove_unreasonable_habitable_rooms": remove_unreasonable_habitable_rooms, + "keep_negative_heat_change": keep_negative_heat_change, + "keep_negative_carbon_change": keep_negative_carbon_change, + "remove_top_1_percent_heat_demand": remove_top_1_percent_heat_demand, + "remove_top_1_percent_carbon": remove_top_1_percent_carbon, # "remove_starting_columns": remove_starting_columns # "keep_ENDING_COLUMNS": keep_ending_columns } diff --git a/modules/ml-pipeline/src/pipeline/dvc.lock b/modules/ml-pipeline/src/pipeline/dvc.lock index 627af99..e65dfe8 100644 --- a/modules/ml-pipeline/src/pipeline/dvc.lock +++ b/modules/ml-pipeline/src/pipeline/dvc.lock @@ -29,8 +29,8 @@ stages: outs: - path: data/prepared_data/ hash: md5 - md5: 73c1f7be21be8358a73c4ab5f9ec8e39.dir - size: 32943109 + md5: ca205aaf77cb9a9414a0c6a1affd8d82.dir + size: 30597800 nfiles: 2 build_model: cmd: python 2_build_model.py @@ -41,8 +41,8 @@ stages: size: 4149 - path: data/prepared_data hash: md5 - md5: 73c1f7be21be8358a73c4ab5f9ec8e39.dir - size: 32943109 + md5: ca205aaf77cb9a9414a0c6a1affd8d82.dir + size: 30597800 nfiles: 2 params: configs/build_model.yaml: @@ -68,12 +68,12 @@ stages: outs: - path: data/model/ hash: md5 - md5: dee1a60e6a9f4695272da8127196f714.dir - size: 326732699 + md5: f3be67a0a80e525d30665f2ffc367d9b.dir + size: 312133166 nfiles: 24 - path: metrics/fit_metrics.json hash: md5 - md5: 1fefa99c7bc50d09c31bf175d5b9ee9c + md5: 36912d423f975802ca3661992103e614 size: 226 generate_predictions: cmd: python 3_generate_predictions.py @@ -84,13 +84,13 @@ stages: size: 2464 - path: data/model hash: md5 - md5: dee1a60e6a9f4695272da8127196f714.dir - size: 326732699 + md5: f3be67a0a80e525d30665f2ffc367d9b.dir + size: 312133166 nfiles: 24 - path: data/prepared_data hash: md5 - md5: 73c1f7be21be8358a73c4ab5f9ec8e39.dir - size: 32943109 + md5: ca205aaf77cb9a9414a0c6a1affd8d82.dir + size: 30597800 nfiles: 2 params: configs/settings.yaml: @@ -102,8 +102,8 @@ stages: outs: - path: data/predictions/ hash: md5 - md5: d2da3b713811952b66e2c5f8c95f5407.dir - size: 410646 + md5: 2ae9ab85ca2551d6b0833337cacbcc3e.dir + size: 389118 nfiles: 1 generate_metrics: cmd: python 4_generate_metrics.py @@ -114,13 +114,13 @@ stages: size: 3485 - path: data/predictions hash: md5 - md5: d2da3b713811952b66e2c5f8c95f5407.dir - size: 410646 + md5: 2ae9ab85ca2551d6b0833337cacbcc3e.dir + size: 389118 nfiles: 1 - path: data/prepared_data hash: md5 - md5: 73c1f7be21be8358a73c4ab5f9ec8e39.dir - size: 32943109 + md5: ca205aaf77cb9a9414a0c6a1affd8d82.dir + size: 30597800 nfiles: 2 params: configs/settings.yaml: @@ -130,7 +130,7 @@ stages: outs: - path: metrics/metrics.json hash: md5 - md5: 4ed2edc06b4dad3c094a2d1be374a5de + md5: 6447c7b2b92a4057aecd3d227de1aadf size: 224 startup_cleanup: cmd: python 0_startup_cleanup.py From 65335468b438f7d6bfa258d2fa656b6ac9502537 Mon Sep 17 00:00:00 2001 From: Github-Bot Date: Tue, 28 Nov 2023 15:26:50 +0000 Subject: [PATCH 11/23] Update Registry --- MODEL_REGISTRY.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/MODEL_REGISTRY.md b/MODEL_REGISTRY.md index 2dac564..7c59593 100644 --- a/MODEL_REGISTRY.md +++ b/MODEL_REGISTRY.md @@ -16,15 +16,15 @@ "active": true }, "heat": { - "version": "v0.0.1", + "version": "v0.2.0", "stage": { - "dev": "v0.0.1" + "dev": "v0.2.0" }, "registered": true, "active": true }, "carbon": { - "version": "v0.1.0", + "version": "v0.2.0", "stage": { "dev": "v0.1.0" }, From db29bece807eb23dedf8638dcab3b4439927b3ad Mon Sep 17 00:00:00 2001 From: Github-Bot Date: Tue, 28 Nov 2023 15:27:34 +0000 Subject: [PATCH 12/23] Update Registry --- MODEL_REGISTRY.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MODEL_REGISTRY.md b/MODEL_REGISTRY.md index 7c59593..118693d 100644 --- a/MODEL_REGISTRY.md +++ b/MODEL_REGISTRY.md @@ -26,7 +26,7 @@ "carbon": { "version": "v0.2.0", "stage": { - "dev": "v0.1.0" + "dev": "v0.2.0" }, "registered": true, "active": true From ae5349974227dd15075d3cca8762c168c75963d9 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Fri, 22 Dec 2023 09:51:57 +0000 Subject: [PATCH 13/23] add keep only non negative carbon change to carbon model --- .../src/pipeline/configs/feature_processor_logic.py | 6 ++++++ modules/ml-pipeline/src/pipeline/example.py | 0 2 files changed, 6 insertions(+) create mode 100644 modules/ml-pipeline/src/pipeline/example.py diff --git a/modules/ml-pipeline/src/pipeline/configs/feature_processor_logic.py b/modules/ml-pipeline/src/pipeline/configs/feature_processor_logic.py index 78c29a9..bce32b6 100644 --- a/modules/ml-pipeline/src/pipeline/configs/feature_processor_logic.py +++ b/modules/ml-pipeline/src/pipeline/configs/feature_processor_logic.py @@ -23,6 +23,11 @@ def keep_negative_heat_change(df): return df +def keep_non_negative_carbon_ending(df): + df = df[df["CARBON_ENDING"] > 0] + return df + + def keep_negative_carbon_change(df): df = df[df["CARBON_CHANGE"] < 0] return df @@ -68,6 +73,7 @@ business_logic = { "keep_negative_carbon_change": keep_negative_carbon_change, "remove_top_1_percent_heat_demand": remove_top_1_percent_heat_demand, "remove_top_1_percent_carbon": remove_top_1_percent_carbon, + "keep_non_negative_carbon_ending": keep_non_negative_carbon_ending # "remove_starting_columns": remove_starting_columns # "keep_ENDING_COLUMNS": keep_ending_columns } diff --git a/modules/ml-pipeline/src/pipeline/example.py b/modules/ml-pipeline/src/pipeline/example.py new file mode 100644 index 0000000..e69de29 From 79a55ba8b5ff5b13a7cd9224033dc3e19363a79c Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Wed, 17 Jan 2024 23:35:50 +0000 Subject: [PATCH 14/23] train 600 second model on new data --- .../src/pipeline/configs/build_model.yaml | 2 +- .../configs/feature_processor_logic.py | 16 ++--- .../pipeline/configs/post_prediction_logic.py | 4 +- .../src/pipeline/configs/settings.yaml | 8 +-- modules/ml-pipeline/src/pipeline/dvc.lock | 60 +++++++++---------- 5 files changed, 45 insertions(+), 45 deletions(-) diff --git a/modules/ml-pipeline/src/pipeline/configs/build_model.yaml b/modules/ml-pipeline/src/pipeline/configs/build_model.yaml index 4c72487..9c97ef0 100644 --- a/modules/ml-pipeline/src/pipeline/configs/build_model.yaml +++ b/modules/ml-pipeline/src/pipeline/configs/build_model.yaml @@ -13,7 +13,7 @@ default: output_filepath: ./data/model/allmodels/ problem_type: regression eval_metric: mean_squared_error #mean_absolute_error - time_limit: 400 + time_limit: 600 presets: medium_quality excluded_model_types: ['KNN', 'RF'] infer_limit: 0.05 diff --git a/modules/ml-pipeline/src/pipeline/configs/feature_processor_logic.py b/modules/ml-pipeline/src/pipeline/configs/feature_processor_logic.py index bce32b6..94d3c6e 100644 --- a/modules/ml-pipeline/src/pipeline/configs/feature_processor_logic.py +++ b/modules/ml-pipeline/src/pipeline/configs/feature_processor_logic.py @@ -9,27 +9,27 @@ Business Logic dict + functions def remove_starting_columns(df): keep_column_index = [ - False if col_name.endswith("_STARTING") else True + False if col_name.endswith("_starting") else True for col_name in list(df.columns) ] keep_columns = df.columns[keep_column_index].to_list() - keep_columns.append("SAP_STARTING") + keep_columns.append("sap_starting") df = df[keep_columns] return df def keep_negative_heat_change(df): - df = df[df["HEAT_DEMAND_CHANGE"] < 0] + df = df[df["heat_demand_change"] < 0] return df def keep_non_negative_carbon_ending(df): - df = df[df["CARBON_ENDING"] > 0] + df = df[df["carbon_ending"] > 0] return df def keep_negative_carbon_change(df): - df = df[df["CARBON_CHANGE"] < 0] + df = df[df["carbon_change"] < 0] return df @@ -39,7 +39,7 @@ def remove_unreasonable_habitable_rooms(df): Assumption is that proportion of floor area to habitable rooms should be at least 6.5m2 """ minimum_room_size_index = ( - df["TOTAL_FLOOR_AREA_ENDING"] / df["NUMBER_HABITABLE_ROOMS"] >= 6.5 + df["total_floor_area_ending"] / df["number_habitable_rooms"] >= 6.5 ) df = df[minimum_room_size_index] return df @@ -48,14 +48,14 @@ def remove_unreasonable_habitable_rooms(df): def remove_top_1_percent_heat_demand(df): # threshold_value = df.describe(percentiles=[0.99])['HEAT_DEMAND_STARTING']['99%'] threshold_value = 860 - df = df[df["HEAT_DEMAND_STARTING"] < threshold_value] + df = df[df["heat_demand_starting"] < threshold_value] return df def remove_top_1_percent_carbon(df): # threshold_value = df.describe(percentiles=[0.99])['CARBON_STARTING']['99%'] threshold_value = 18 - df = df[df["CARBON_STARTING"] < threshold_value] + df = df[df["carbon_starting"] < threshold_value] return df diff --git a/modules/ml-pipeline/src/pipeline/configs/post_prediction_logic.py b/modules/ml-pipeline/src/pipeline/configs/post_prediction_logic.py index 1ffab90..dec7740 100644 --- a/modules/ml-pipeline/src/pipeline/configs/post_prediction_logic.py +++ b/modules/ml-pipeline/src/pipeline/configs/post_prediction_logic.py @@ -13,9 +13,9 @@ def clip_predictions_to_minimum_value( predictions.name = "predictions" predictions_df = pd.concat([data, predictions], axis=1) # We expect all prediction to be atleast one point improvement - replace_index = predictions_df["predictions"] > predictions_df["CARBON_STARTING"] + replace_index = predictions_df["predictions"] > predictions_df["carbon_starting"] predictions_df.loc[replace_index, "predictions"] = predictions_df.loc[ - replace_index, "CARBON_STARTING" + replace_index, "carbon_starting" ] predictions_new = predictions_df["predictions"] diff --git a/modules/ml-pipeline/src/pipeline/configs/settings.yaml b/modules/ml-pipeline/src/pipeline/configs/settings.yaml index 5514406..e4bd13b 100644 --- a/modules/ml-pipeline/src/pipeline/configs/settings.yaml +++ b/modules/ml-pipeline/src/pipeline/configs/settings.yaml @@ -21,7 +21,7 @@ default: # data_filepath: s3://retrofit-data-dev/sap_change_model/dataset_with_differencing.parquet # data_filepath: s3://retrofit-data-dev/sap_change_model/floor_area_clean_test.parquet # data_filepath: s3://retrofit-data-dev/sap_change_model/dataset_without_differencing.parquet - data_filepath: s3://retrofit-data-dev/sap_change_model/dataset_test.parquet + data_filepath: s3://retrofit-data-dev/sap_change_model/dataset.parquet train_proportion: 0.9 output_train_filepath: ./data/prepared_data/train.parquet output_test_filepath: ./data/prepared_data/test.parquet @@ -31,9 +31,9 @@ default: feature_processor_config: subsample_amount: null subsample_seed: 0 - target: CARBON_ENDING - identifier_columns: ["UPRN"] - drop_columns: ["HEAT_DEMAND_CHANGE", "CARBON_CHANGE", "RDSAP_CHANGE", "HEAT_DEMAND_ENDING", "SAP_ENDING"] + target: carbon_ending + identifier_columns: ["uprn"] + drop_columns: ["heat_demand_change", "carbon_change", "rdsap_change", "heat_demand_ending", "sap_ending"] # retain_features: ["SAP_STARTING", "TOTAL_FLOOR_AREA_DIFF"] retain_features: null diff --git a/modules/ml-pipeline/src/pipeline/dvc.lock b/modules/ml-pipeline/src/pipeline/dvc.lock index e65dfe8..d8da73d 100644 --- a/modules/ml-pipeline/src/pipeline/dvc.lock +++ b/modules/ml-pipeline/src/pipeline/dvc.lock @@ -10,17 +10,17 @@ stages: params: configs/settings.yaml: default.feature_processor.feature_processor_config.drop_columns: - - HEAT_DEMAND_CHANGE - - CARBON_CHANGE - - RDSAP_CHANGE - - HEAT_DEMAND_ENDING - - SAP_ENDING + - heat_demand_change + - carbon_change + - rdsap_change + - heat_demand_ending + - sap_ending default.feature_processor.feature_processor_config.retain_features: default.feature_processor.feature_processor_config.subsample_amount: default.feature_processor.feature_processor_config.subsample_seed: 0 - default.feature_processor.feature_processor_config.target: CARBON_ENDING + default.feature_processor.feature_processor_config.target: carbon_ending default.feature_processor.feature_processor_type: dataframe - default.prepare_data.data_filepath: s3://retrofit-data-dev/sap_change_model/dataset_test.parquet + default.prepare_data.data_filepath: s3://retrofit-data-dev/sap_change_model/dataset.parquet default.prepare_data.input_dataclient_type: aws-s3 default.prepare_data.output_dataclient_type: local default.prepare_data.output_test_filepath: ./data/prepared_data/test.parquet @@ -29,8 +29,8 @@ stages: outs: - path: data/prepared_data/ hash: md5 - md5: ca205aaf77cb9a9414a0c6a1affd8d82.dir - size: 30597800 + md5: 70d79ba4a6f0648439dc55031c944d47.dir + size: 32673907 nfiles: 2 build_model: cmd: python 2_build_model.py @@ -41,8 +41,8 @@ stages: size: 4149 - path: data/prepared_data hash: md5 - md5: ca205aaf77cb9a9414a0c6a1affd8d82.dir - size: 30597800 + md5: 70d79ba4a6f0648439dc55031c944d47.dir + size: 32673907 nfiles: 2 params: configs/build_model.yaml: @@ -58,7 +58,7 @@ stages: output_filepath: ./data/model/allmodels/ problem_type: regression eval_metric: mean_squared_error - time_limit: 400 + time_limit: 600 presets: medium_quality excluded_model_types: - KNN @@ -68,13 +68,13 @@ stages: outs: - path: data/model/ hash: md5 - md5: f3be67a0a80e525d30665f2ffc367d9b.dir - size: 312133166 - nfiles: 24 + md5: 2fc9223da8b72e61d81f06665e75019e.dir + size: 324532985 + nfiles: 27 - path: metrics/fit_metrics.json hash: md5 - md5: 36912d423f975802ca3661992103e614 - size: 226 + md5: 7d2f226251ce6f8e92af73d50dadb890 + size: 228 generate_predictions: cmd: python 3_generate_predictions.py deps: @@ -84,13 +84,13 @@ stages: size: 2464 - path: data/model hash: md5 - md5: f3be67a0a80e525d30665f2ffc367d9b.dir - size: 312133166 - nfiles: 24 + md5: 2fc9223da8b72e61d81f06665e75019e.dir + size: 324532985 + nfiles: 27 - path: data/prepared_data hash: md5 - md5: ca205aaf77cb9a9414a0c6a1affd8d82.dir - size: 30597800 + md5: 70d79ba4a6f0648439dc55031c944d47.dir + size: 32673907 nfiles: 2 params: configs/settings.yaml: @@ -102,8 +102,8 @@ stages: outs: - path: data/predictions/ hash: md5 - md5: 2ae9ab85ca2551d6b0833337cacbcc3e.dir - size: 389118 + md5: 8bfc33c14aba5abf5ac4bdba32ff3c4c.dir + size: 412880 nfiles: 1 generate_metrics: cmd: python 4_generate_metrics.py @@ -114,13 +114,13 @@ stages: size: 3485 - path: data/predictions hash: md5 - md5: 2ae9ab85ca2551d6b0833337cacbcc3e.dir - size: 389118 + md5: 8bfc33c14aba5abf5ac4bdba32ff3c4c.dir + size: 412880 nfiles: 1 - path: data/prepared_data hash: md5 - md5: ca205aaf77cb9a9414a0c6a1affd8d82.dir - size: 30597800 + md5: 70d79ba4a6f0648439dc55031c944d47.dir + size: 32673907 nfiles: 2 params: configs/settings.yaml: @@ -130,8 +130,8 @@ stages: outs: - path: metrics/metrics.json hash: md5 - md5: 6447c7b2b92a4057aecd3d227de1aadf - size: 224 + md5: 9a0b57244dfdbd6dab0392a4fd618123 + size: 225 startup_cleanup: cmd: python 0_startup_cleanup.py deps: From 9b29e838af73f1e8113c2bc3981009cbe59a2575 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Wed, 17 Jan 2024 23:45:07 +0000 Subject: [PATCH 15/23] update requirements for dvc --- .../requirements/version_control/requirements.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt b/modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt index 91cb005..a2b9531 100644 --- a/modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt +++ b/modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt @@ -1,4 +1,4 @@ -dvc==3.18.0 -dvc-s3==2.23.0 -gto==1.0.4 -pyOpenSSL==23.2.0 +dvc==3.36.0 +dvc-s3==3.0.1 +gto==1.6.1 +pyOpenSSL==23.3.0 From 5d23992d05cecce1694c54792a1c1190cf37a082 Mon Sep 17 00:00:00 2001 From: Github-Bot Date: Thu, 18 Jan 2024 10:37:29 +0000 Subject: [PATCH 16/23] Update Registry --- MODEL_REGISTRY.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/MODEL_REGISTRY.md b/MODEL_REGISTRY.md index 118693d..7e99fc7 100644 --- a/MODEL_REGISTRY.md +++ b/MODEL_REGISTRY.md @@ -8,9 +8,9 @@ "active": true }, "sap": { - "version": "v0.2.6", + "version": "v0.4.0", "stage": { - "dev": "v0.2.6" + "dev": "v0.3.0" }, "registered": true, "active": true @@ -24,7 +24,7 @@ "active": true }, "carbon": { - "version": "v0.2.0", + "version": "v0.3.0", "stage": { "dev": "v0.2.0" }, From 9880ebed4cf18b706be19c9377cd903b3dd4f532 Mon Sep 17 00:00:00 2001 From: Github-Bot Date: Thu, 18 Jan 2024 10:38:17 +0000 Subject: [PATCH 17/23] Update Registry --- MODEL_REGISTRY.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/MODEL_REGISTRY.md b/MODEL_REGISTRY.md index 7e99fc7..820b75a 100644 --- a/MODEL_REGISTRY.md +++ b/MODEL_REGISTRY.md @@ -10,15 +10,15 @@ "sap": { "version": "v0.4.0", "stage": { - "dev": "v0.3.0" + "dev": "v0.4.0" }, "registered": true, "active": true }, "heat": { - "version": "v0.2.0", + "version": "v0.3.0", "stage": { - "dev": "v0.2.0" + "dev": "v0.3.0" }, "registered": true, "active": true @@ -26,7 +26,7 @@ "carbon": { "version": "v0.3.0", "stage": { - "dev": "v0.2.0" + "dev": "v0.3.0" }, "registered": true, "active": true From 1887a52230b2e7c4ce76ac0cf9f4cd89b59bda7b Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 30 Jan 2024 10:26:28 +0000 Subject: [PATCH 18/23] use new modesl with carbon model --- .../src/pipeline/1_prepare_data.py | 3 +- .../ml-pipeline/src/pipeline/2_build_model.py | 16 +++++ .../src/pipeline/configs/build_model.yaml | 5 +- .../src/pipeline/configs/settings.yaml | 5 +- modules/ml-pipeline/src/pipeline/dvc.lock | 70 +++++++++++-------- modules/ml-pipeline/src/pipeline/dvc.yaml | 1 + 6 files changed, 65 insertions(+), 35 deletions(-) diff --git a/modules/ml-pipeline/src/pipeline/1_prepare_data.py b/modules/ml-pipeline/src/pipeline/1_prepare_data.py index ed7e057..75d784f 100644 --- a/modules/ml-pipeline/src/pipeline/1_prepare_data.py +++ b/modules/ml-pipeline/src/pipeline/1_prepare_data.py @@ -87,7 +87,8 @@ def prepare_data( if train_proportion == 1: train = data - test = None + # Sample 10% of the data for testing + test = data.sample(round(len(data) * 0.1)) else: train, test = train_test_split( data, train_size=train_proportion, test_size=(1 - train_proportion) diff --git a/modules/ml-pipeline/src/pipeline/2_build_model.py b/modules/ml-pipeline/src/pipeline/2_build_model.py index 7ca4951..09e5910 100644 --- a/modules/ml-pipeline/src/pipeline/2_build_model.py +++ b/modules/ml-pipeline/src/pipeline/2_build_model.py @@ -26,9 +26,12 @@ prepare_data_params = settings.prepare_data build_model_params = settings.build_model feature_process_params = settings.feature_processor generate_metrics_params = settings.generate_metrics +generate_predictions_params = settings.generate_predictions model_type = build_model_params["model_type"] target = feature_process_params["feature_processor_config"]["target"] +fit_predictions_filepath = build_model_params["fit_predictions_filepath"] +predictions_column_name = generate_predictions_params["predictions_column_name"] identifier_columns = feature_process_params["feature_processor_config"][ "identifier_columns" ] @@ -60,6 +63,8 @@ def build_model( identifier_columns: List[str], model_save_location: str, model_hyperparameters: dict, + fit_predictions_filepath: str, + predictions_column_name: str, fit_metrics_filepath: str, train_filepath: Union[str, None] = None, test_filepath: Union[str, None] = None, @@ -93,6 +98,15 @@ def build_model( data=train_data, post_prediction_logic=post_prediction_logic ) + logger.info("--- Saving fit predictions ---") + + predictions_df = pd.DataFrame(fit_predictions) + predictions_df.columns = [predictions_column_name] + + dataclient.save_data( + obj=predictions_df, location=fit_predictions_filepath, save_config=None + ) + logger.info("--- Generating fit metrics ---") metrics_output = metrics.generate_metrics( @@ -128,6 +142,8 @@ if __name__ == "__main__": train_filepath=train_filepath, test_filepath=test_filepath, fit_metrics_filepath=fit_metrics_filepath, + fit_predictions_filepath=fit_predictions_filepath, + predictions_column_name=predictions_column_name, ) logger.info(f"--- {__file__} - Complete! ---") diff --git a/modules/ml-pipeline/src/pipeline/configs/build_model.yaml b/modules/ml-pipeline/src/pipeline/configs/build_model.yaml index 9c97ef0..fcec7f7 100644 --- a/modules/ml-pipeline/src/pipeline/configs/build_model.yaml +++ b/modules/ml-pipeline/src/pipeline/configs/build_model.yaml @@ -3,6 +3,7 @@ default: model_type: AutogluonAutoML model_save_filepath: ./data/model/optimised/ fit_metrics_filepath: ./metrics/fit_metrics.json + fit_predictions_filepath: ./data/fit_predictions/predictions.parquet SKLearnLinearRegression: null @@ -13,8 +14,8 @@ default: output_filepath: ./data/model/allmodels/ problem_type: regression eval_metric: mean_squared_error #mean_absolute_error - time_limit: 600 + time_limit: 4000 presets: medium_quality - excluded_model_types: ['KNN', 'RF'] + excluded_model_types: ['RF', 'FASTAI', 'CAT', 'NN_TORCH', 'KNN', 'XT'] infer_limit: 0.05 infer_limit_batch_size: 10000 diff --git a/modules/ml-pipeline/src/pipeline/configs/settings.yaml b/modules/ml-pipeline/src/pipeline/configs/settings.yaml index e4bd13b..051e34e 100644 --- a/modules/ml-pipeline/src/pipeline/configs/settings.yaml +++ b/modules/ml-pipeline/src/pipeline/configs/settings.yaml @@ -21,8 +21,9 @@ default: # data_filepath: s3://retrofit-data-dev/sap_change_model/dataset_with_differencing.parquet # data_filepath: s3://retrofit-data-dev/sap_change_model/floor_area_clean_test.parquet # data_filepath: s3://retrofit-data-dev/sap_change_model/dataset_without_differencing.parquet - data_filepath: s3://retrofit-data-dev/sap_change_model/dataset.parquet - train_proportion: 0.9 + # data_filepath: s3://retrofit-data-dev/sap_change_model/dataset.parquet + data_filepath: s3://retrofit-datalake-dev/dataset_with0perm_all.parquet + train_proportion: 1 output_train_filepath: ./data/prepared_data/train.parquet output_test_filepath: ./data/prepared_data/test.parquet diff --git a/modules/ml-pipeline/src/pipeline/dvc.lock b/modules/ml-pipeline/src/pipeline/dvc.lock index d8da73d..dc996b7 100644 --- a/modules/ml-pipeline/src/pipeline/dvc.lock +++ b/modules/ml-pipeline/src/pipeline/dvc.lock @@ -5,8 +5,8 @@ stages: deps: - path: 1_prepare_data.py hash: md5 - md5: 896d3d88a4a9f68d174efe71dc089517 - size: 4222 + md5: 11a3b8bfdfe199ab7ecc39ccc5652649 + size: 4298 params: configs/settings.yaml: default.feature_processor.feature_processor_config.drop_columns: @@ -20,29 +20,29 @@ stages: default.feature_processor.feature_processor_config.subsample_seed: 0 default.feature_processor.feature_processor_config.target: carbon_ending default.feature_processor.feature_processor_type: dataframe - default.prepare_data.data_filepath: s3://retrofit-data-dev/sap_change_model/dataset.parquet + default.prepare_data.data_filepath: s3://retrofit-datalake-dev/dataset_with0perm_all.parquet default.prepare_data.input_dataclient_type: aws-s3 default.prepare_data.output_dataclient_type: local default.prepare_data.output_test_filepath: ./data/prepared_data/test.parquet default.prepare_data.output_train_filepath: ./data/prepared_data/train.parquet - default.prepare_data.train_proportion: 0.9 + default.prepare_data.train_proportion: 1 outs: - path: data/prepared_data/ hash: md5 - md5: 70d79ba4a6f0648439dc55031c944d47.dir - size: 32673907 + md5: 44737880f5437e23143479a7818a3136.dir + size: 36064622 nfiles: 2 build_model: cmd: python 2_build_model.py deps: - path: 2_build_model.py hash: md5 - md5: b824822475c222521516493e68eef9c5 - size: 4149 + md5: 090bfb7dbaff39f45784b7fe332a9b8e + size: 4819 - path: data/prepared_data hash: md5 - md5: 70d79ba4a6f0648439dc55031c944d47.dir - size: 32673907 + md5: 44737880f5437e23143479a7818a3136.dir + size: 36064622 nfiles: 2 params: configs/build_model.yaml: @@ -51,6 +51,7 @@ stages: model_type: AutogluonAutoML model_save_filepath: ./data/model/optimised/ fit_metrics_filepath: ./metrics/fit_metrics.json + fit_predictions_filepath: ./data/fit_predictions/predictions.parquet SKLearnLinearRegression: SKLearnSVMRegression: kernel: linear @@ -58,23 +59,32 @@ stages: output_filepath: ./data/model/allmodels/ problem_type: regression eval_metric: mean_squared_error - time_limit: 600 + time_limit: 4000 presets: medium_quality excluded_model_types: - - KNN - RF + - FASTAI + - CAT + - NN_TORCH + - KNN + - XT infer_limit: 0.05 infer_limit_batch_size: 10000 outs: + - path: data/fit_predictions/ + hash: md5 + md5: 7b74ae1174ae2c7fab03ee0ce0a8ae71.dir + size: 3877514 + nfiles: 1 - path: data/model/ hash: md5 - md5: 2fc9223da8b72e61d81f06665e75019e.dir - size: 324532985 - nfiles: 27 + md5: d2ebaa73a894387f85083c49e58637bc.dir + size: 798349514 + nfiles: 32 - path: metrics/fit_metrics.json hash: md5 - md5: 7d2f226251ce6f8e92af73d50dadb890 - size: 228 + md5: 51c9c678bbd19bc9f7e16f0bf5df3fef + size: 229 generate_predictions: cmd: python 3_generate_predictions.py deps: @@ -84,13 +94,13 @@ stages: size: 2464 - path: data/model hash: md5 - md5: 2fc9223da8b72e61d81f06665e75019e.dir - size: 324532985 - nfiles: 27 + md5: d2ebaa73a894387f85083c49e58637bc.dir + size: 798349514 + nfiles: 32 - path: data/prepared_data hash: md5 - md5: 70d79ba4a6f0648439dc55031c944d47.dir - size: 32673907 + md5: 44737880f5437e23143479a7818a3136.dir + size: 36064622 nfiles: 2 params: configs/settings.yaml: @@ -102,8 +112,8 @@ stages: outs: - path: data/predictions/ hash: md5 - md5: 8bfc33c14aba5abf5ac4bdba32ff3c4c.dir - size: 412880 + md5: ac0a698f14fb9002b337b1b163997333.dir + size: 638033 nfiles: 1 generate_metrics: cmd: python 4_generate_metrics.py @@ -114,13 +124,13 @@ stages: size: 3485 - path: data/predictions hash: md5 - md5: 8bfc33c14aba5abf5ac4bdba32ff3c4c.dir - size: 412880 + md5: ac0a698f14fb9002b337b1b163997333.dir + size: 638033 nfiles: 1 - path: data/prepared_data hash: md5 - md5: 70d79ba4a6f0648439dc55031c944d47.dir - size: 32673907 + md5: 44737880f5437e23143479a7818a3136.dir + size: 36064622 nfiles: 2 params: configs/settings.yaml: @@ -130,8 +140,8 @@ stages: outs: - path: metrics/metrics.json hash: md5 - md5: 9a0b57244dfdbd6dab0392a4fd618123 - size: 225 + md5: 47aa4601e71a93163d2cc1b85d0eda91 + size: 228 startup_cleanup: cmd: python 0_startup_cleanup.py deps: diff --git a/modules/ml-pipeline/src/pipeline/dvc.yaml b/modules/ml-pipeline/src/pipeline/dvc.yaml index ccdd779..58889cc 100644 --- a/modules/ml-pipeline/src/pipeline/dvc.yaml +++ b/modules/ml-pipeline/src/pipeline/dvc.yaml @@ -38,6 +38,7 @@ stages: - configs/build_model.yaml: outs: - data/model/ + - data/fit_predictions/ - metrics/fit_metrics.json always_changed: true generate_predictions: From 96765cee05322e199a14b56f74a28e9b8fe984c1 Mon Sep 17 00:00:00 2001 From: Github-Bot Date: Tue, 30 Jan 2024 10:38:43 +0000 Subject: [PATCH 19/23] Update Registry --- MODEL_REGISTRY.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/MODEL_REGISTRY.md b/MODEL_REGISTRY.md index 820b75a..0db04e3 100644 --- a/MODEL_REGISTRY.md +++ b/MODEL_REGISTRY.md @@ -8,15 +8,15 @@ "active": true }, "sap": { - "version": "v0.4.0", + "version": "v0.5.0", "stage": { - "dev": "v0.4.0" + "dev": "v0.5.0" }, "registered": true, "active": true }, "heat": { - "version": "v0.3.0", + "version": "v0.4.0", "stage": { "dev": "v0.3.0" }, @@ -24,7 +24,7 @@ "active": true }, "carbon": { - "version": "v0.3.0", + "version": "v0.4.0", "stage": { "dev": "v0.3.0" }, From 869a276d6718643af28ded8cef89ee14bea7ce50 Mon Sep 17 00:00:00 2001 From: Github-Bot Date: Tue, 30 Jan 2024 10:39:26 +0000 Subject: [PATCH 20/23] Update Registry --- MODEL_REGISTRY.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/MODEL_REGISTRY.md b/MODEL_REGISTRY.md index 0db04e3..a593577 100644 --- a/MODEL_REGISTRY.md +++ b/MODEL_REGISTRY.md @@ -18,7 +18,7 @@ "heat": { "version": "v0.4.0", "stage": { - "dev": "v0.3.0" + "dev": "v0.4.0" }, "registered": true, "active": true @@ -26,7 +26,7 @@ "carbon": { "version": "v0.4.0", "stage": { - "dev": "v0.3.0" + "dev": "v0.4.0" }, "registered": true, "active": true From c0dc934be6a61e39c6aae6956c396208eb4c66d1 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Wed, 27 Mar 2024 23:10:36 +0000 Subject: [PATCH 21/23] run carbon model with new data --- MODEL_REGISTRY.md | 4 +- modules/ml-pipeline/src/README.md | 2 +- modules/ml-pipeline/src/pipeline/.gitignore | 1 + .../src/pipeline/configs/build_model.yaml | 1 + .../configs/feature_processor_logic.py | 54 +++++++---- .../pipeline/configs/post_prediction_logic.py | 15 +-- .../src/pipeline/configs/settings.yaml | 15 ++- .../ml-pipeline/src/pipeline/core/MLModels.py | 4 +- modules/ml-pipeline/src/pipeline/dvc.lock | 91 +++++++++++-------- .../predictions/requirements-dev.txt | 4 +- .../requirements/predictions/requirements.txt | 4 +- .../training/requirements-dev.txt | 7 +- .../requirements/training/requirements.txt | 4 +- 13 files changed, 119 insertions(+), 87 deletions(-) diff --git a/MODEL_REGISTRY.md b/MODEL_REGISTRY.md index 1bcceec..820b75a 100644 --- a/MODEL_REGISTRY.md +++ b/MODEL_REGISTRY.md @@ -8,9 +8,9 @@ "active": true }, "sap": { - "version": "v0.5.0", + "version": "v0.4.0", "stage": { - "dev": "v0.5.0" + "dev": "v0.4.0" }, "registered": true, "active": true diff --git a/modules/ml-pipeline/src/README.md b/modules/ml-pipeline/src/README.md index d7afc6a..db1b8b4 100644 --- a/modules/ml-pipeline/src/README.md +++ b/modules/ml-pipeline/src/README.md @@ -1,3 +1,3 @@ -# The generic reproducible ML-pipeline +# The generic reproducible ML-pipeline! Pipeline required to build a model to produce an output, that gets hashed via DVC diff --git a/modules/ml-pipeline/src/pipeline/.gitignore b/modules/ml-pipeline/src/pipeline/.gitignore index bf035d2..ce8309f 100644 --- a/modules/ml-pipeline/src/pipeline/.gitignore +++ b/modules/ml-pipeline/src/pipeline/.gitignore @@ -1,3 +1,4 @@ # Ignore dynaconf secret files .secrets.* +example.py diff --git a/modules/ml-pipeline/src/pipeline/configs/build_model.yaml b/modules/ml-pipeline/src/pipeline/configs/build_model.yaml index fcec7f7..be5ec5c 100644 --- a/modules/ml-pipeline/src/pipeline/configs/build_model.yaml +++ b/modules/ml-pipeline/src/pipeline/configs/build_model.yaml @@ -19,3 +19,4 @@ default: excluded_model_types: ['RF', 'FASTAI', 'CAT', 'NN_TORCH', 'KNN', 'XT'] infer_limit: 0.05 infer_limit_batch_size: 10000 + ag_args_ensemble: {'num_folds_parallel': 2} diff --git a/modules/ml-pipeline/src/pipeline/configs/feature_processor_logic.py b/modules/ml-pipeline/src/pipeline/configs/feature_processor_logic.py index 103168d..bcc53e5 100644 --- a/modules/ml-pipeline/src/pipeline/configs/feature_processor_logic.py +++ b/modules/ml-pipeline/src/pipeline/configs/feature_processor_logic.py @@ -18,30 +18,44 @@ def remove_starting_columns(df): return df -def remove_floor_height_ending(df): - # df.describe(percentiles=[0.005,0.99])['FLOOR_HEIGHT_ENDING'] - # shows bottom 0.5 percentile is 1.665 - # So keep anything above this - df = df[df["floor_height_ending"] > 1.665].reset_index(drop=True) - print("we in here") +def keep_negative_heat_change(df): + df = df[df["heat_demand_change"] < 0] return df -def remove_minimum_habitable_room_size(df): - # Need minimum of 6.5m per habitable room - df = df[ - df["total_floor_area_ending"] / df["number_habitable_rooms"] > 6.5 - ].reset_index(drop=True) +def keep_non_negative_carbon_ending(df): + df = df[df["carbon_ending"] > 0] return df -def keep_flats(df): - df = df[df["property_type"] == "Flat"] +def keep_negative_carbon_change(df): + df = df[df["carbon_change"] < 0] return df -def keep_non_zero_rdsap(df): - df = df[df["rdsap_change"] != 0] +# TODO: Move to ETL pipeline +def remove_unreasonable_habitable_rooms(df): + """ + Assumption is that proportion of floor area to habitable rooms should be at least 6.5m2 + """ + minimum_room_size_index = ( + df["total_floor_area_ending"] / df["number_habitable_rooms"] >= 6.5 + ) + df = df[minimum_room_size_index] + return df + + +def remove_top_1_percent_heat_demand(df): + # threshold_value = df.describe(percentiles=[0.99])['HEAT_DEMAND_STARTING']['99%'] + threshold_value = 860 + df = df[df["heat_demand_starting"] < threshold_value] + return df + + +def remove_top_1_percent_carbon(df): + # threshold_value = df.describe(percentiles=[0.99])['CARBON_STARTING']['99%'] + threshold_value = 18 + df = df[df["carbon_starting"] < threshold_value] return df @@ -54,10 +68,12 @@ def keep_non_zero_rdsap(df): # return df business_logic = { - # "keep_non_zero_rdsap": keep_non_zero_rdsap, - # "keep_flats": keep_flats, - # "remove_minimum_habitable_room_size": remove_minimum_habitable_room_size, - # "remove_floor_height_ending": remove_floor_height_ending + "remove_unreasonable_habitable_rooms": remove_unreasonable_habitable_rooms, + "keep_negative_heat_change": keep_negative_heat_change, + "keep_negative_carbon_change": keep_negative_carbon_change, + "remove_top_1_percent_heat_demand": remove_top_1_percent_heat_demand, + "remove_top_1_percent_carbon": remove_top_1_percent_carbon, + "keep_non_negative_carbon_ending": keep_non_negative_carbon_ending, # "remove_starting_columns": remove_starting_columns # "keep_ENDING_COLUMNS": keep_ending_columns } diff --git a/modules/ml-pipeline/src/pipeline/configs/post_prediction_logic.py b/modules/ml-pipeline/src/pipeline/configs/post_prediction_logic.py index 643231a..2ca8890 100644 --- a/modules/ml-pipeline/src/pipeline/configs/post_prediction_logic.py +++ b/modules/ml-pipeline/src/pipeline/configs/post_prediction_logic.py @@ -1,23 +1,24 @@ """ After predictions, we may want to apply some post processing to the predictions """ + import pandas as pd def clip_predictions_to_minimum_value( - data: pd.DataFrame, predictions: pd.Series, minimum_value: int = 0 + data: pd.DataFrame, + predictions: pd.Series, ) -> pd.Series: series_name = predictions.name predictions.name = "predictions" + predictions = predictions.astype(data["carbon_starting"].dtype) predictions_df = pd.concat([data, predictions], axis=1) # We expect all prediction to be atleast one point improvement - replace_index = ( - predictions_df["sap_starting"] + minimum_value > predictions_df["predictions"] - ) - predictions_df.loc[replace_index, "predictions"] = ( - predictions_df.loc[replace_index, "sap_starting"] + minimum_value - ) + replace_index = predictions_df["predictions"] > predictions_df["carbon_starting"] + predictions_df.loc[replace_index, "predictions"] = predictions_df.loc[ + replace_index, "carbon_starting" + ] predictions_new = predictions_df["predictions"] predictions_new.name = series_name diff --git a/modules/ml-pipeline/src/pipeline/configs/settings.yaml b/modules/ml-pipeline/src/pipeline/configs/settings.yaml index 4327e64..fecdcb0 100644 --- a/modules/ml-pipeline/src/pipeline/configs/settings.yaml +++ b/modules/ml-pipeline/src/pipeline/configs/settings.yaml @@ -18,12 +18,7 @@ default: prepare_data: input_dataclient_type: aws-s3 output_dataclient_type: local - # data_filepath: s3://retrofit-data-dev/sap_change_model/dataset_with_differencing.parquet - # data_filepath: s3://retrofit-data-dev/sap_change_model/floor_area_clean_test.parquet - # data_filepath: s3://retrofit-data-dev/sap_change_model/dataset_without_differencing.parquet - # data_filepath: s3://retrofit-data-dev/sap_change_model/dataset_test.parquet - # data_filepath: s3://retrofit-data-dev/sap_change_model/dataset.parquet - data_filepath: s3://retrofit-datalake-dev/dataset_with0perm_all.parquet + data_filepath: s3://retrofit-data-dev/sap_change_model/2024-03-22-18-56-53/dataset_rooms.parquet train_proportion: 1 output_train_filepath: ./data/prepared_data/train.parquet output_test_filepath: ./data/prepared_data/test.parquet @@ -33,9 +28,13 @@ default: feature_processor_config: subsample_amount: null subsample_seed: 0 - target: sap_ending + target: carbon_ending identifier_columns: ["uprn"] - drop_columns: ["heat_demand_change", "carbon_change", "rdsap_change", "heat_demand_ending", "carbon_ending"] + # drop_columns: ["heat_demand_change", "carbon_change", "rdsap_change", "heat_demand_ending", "sap_ending"] + drop_columns: [ + "heat_demand_change", "carbon_change", "rdsap_change", "heat_demand_ending", "sap_ending", "days_to_starting", "days_to_ending", + 'number_habitable_rooms_starting', 'number_habitable_rooms_ending', 'number_heated_rooms_starting', 'number_heated_rooms_ending', + 'number_habitable_rooms', 'number_heated_rooms'] # retain_features: ["SAP_STARTING", "TOTAL_FLOOR_AREA_DIFF"] retain_features: null diff --git a/modules/ml-pipeline/src/pipeline/core/MLModels.py b/modules/ml-pipeline/src/pipeline/core/MLModels.py index 4fc572a..257261d 100644 --- a/modules/ml-pipeline/src/pipeline/core/MLModels.py +++ b/modules/ml-pipeline/src/pipeline/core/MLModels.py @@ -25,7 +25,7 @@ def model_factory(model_type: str) -> MLModel: models = { "SKLearnLinearRegression": SKLearnLinearRegression(), "SKLearnSVMRegression": SKLearnSVMRegression(), - "AutogluonAutoML": AutogluonAutoML() + "AutogluonAutoML": AutogluonAutoML(), # ADD OTHER MODELS HERE } @@ -151,6 +151,7 @@ class AutogluonAutoML: "excluded_model_types", "infer_limit", "infer_limit_batch_size", + "ag_args_ensemble", ] def load_model(self, path: Union[Path, str]) -> None: @@ -207,6 +208,7 @@ class AutogluonAutoML: excluded_model_types=model_hyperparameters["excluded_model_types"], infer_limit=model_hyperparameters["infer_limit"], infer_limit_batch_size=model_hyperparameters["infer_limit_batch_size"], + ag_args_ensemble=model_hyperparameters["ag_args_ensemble"], ) def predict( diff --git a/modules/ml-pipeline/src/pipeline/dvc.lock b/modules/ml-pipeline/src/pipeline/dvc.lock index f15978f..81224d8 100644 --- a/modules/ml-pipeline/src/pipeline/dvc.lock +++ b/modules/ml-pipeline/src/pipeline/dvc.lock @@ -1,12 +1,23 @@ schema: '2.0' stages: + startup_cleanup: + cmd: python 0_startup_cleanup.py + deps: + - path: 0_startup_cleanup.py + hash: md5 + md5: b1b12f6b6393fbf8b83d23684df0a3d4 + size: 1220 + params: + configs/settings.yaml: + default.startup_cleanup.artefacts: ./data + default.startup_cleanup.metrics: ./metrics prepare_data: cmd: python 1_prepare_data.py deps: - path: 1_prepare_data.py hash: md5 - md5: 1793a35e71751d3c84f9affc67ecb9a8 - size: 4296 + md5: 11a3b8bfdfe199ab7ecc39ccc5652649 + size: 4298 params: configs/settings.yaml: default.feature_processor.feature_processor_config.drop_columns: @@ -14,13 +25,22 @@ stages: - carbon_change - rdsap_change - heat_demand_ending - - carbon_ending + - sap_ending + - days_to_starting + - days_to_ending + - number_habitable_rooms_starting + - number_habitable_rooms_ending + - number_heated_rooms_starting + - number_heated_rooms_ending + - number_habitable_rooms + - number_heated_rooms default.feature_processor.feature_processor_config.retain_features: default.feature_processor.feature_processor_config.subsample_amount: default.feature_processor.feature_processor_config.subsample_seed: 0 - default.feature_processor.feature_processor_config.target: sap_ending + default.feature_processor.feature_processor_config.target: carbon_ending default.feature_processor.feature_processor_type: dataframe - default.prepare_data.data_filepath: s3://retrofit-datalake-dev/dataset_with0perm_all.parquet + default.prepare_data.data_filepath: + s3://retrofit-data-dev/sap_change_model/2024-03-22-18-56-53/dataset_rooms.parquet default.prepare_data.input_dataclient_type: aws-s3 default.prepare_data.output_dataclient_type: local default.prepare_data.output_test_filepath: ./data/prepared_data/test.parquet @@ -29,8 +49,8 @@ stages: outs: - path: data/prepared_data/ hash: md5 - md5: 84fa631bd02686b052d6a7144eafd38e.dir - size: 43859225 + md5: 35d7daa8144434e188ba3b1da4bcf328.dir + size: 33946500 nfiles: 2 build_model: cmd: python 2_build_model.py @@ -41,8 +61,8 @@ stages: size: 4820 - path: data/prepared_data hash: md5 - md5: 84fa631bd02686b052d6a7144eafd38e.dir - size: 43859225 + md5: 35d7daa8144434e188ba3b1da4bcf328.dir + size: 33946500 nfiles: 2 params: configs/build_model.yaml: @@ -70,21 +90,23 @@ stages: - XT infer_limit: 0.05 infer_limit_batch_size: 10000 + ag_args_ensemble: + num_folds_parallel: 2 outs: - path: data/fit_predictions/ hash: md5 - md5: ede187e9d0bffdef054f573f3c2bd222.dir - size: 3578590 + md5: 19d033f5abfa9b064c3e52815e607ced.dir + size: 3927492 nfiles: 1 - path: data/model/ hash: md5 - md5: b2ad0b538dc4aef0de3d431fc9c40c4f.dir - size: 814720415 - nfiles: 31 + md5: f159d40353b01ffdcf1b1b490c019f1f.dir + size: 787748148 + nfiles: 32 - path: metrics/fit_metrics.json hash: md5 - md5: c45b84f12971a0156e4f3d85d3e725f5 - size: 218 + md5: e69d56ab9d82f23f2aa66001bd9bebbc + size: 229 generate_predictions: cmd: python 3_generate_predictions.py deps: @@ -94,13 +116,13 @@ stages: size: 2464 - path: data/model hash: md5 - md5: b2ad0b538dc4aef0de3d431fc9c40c4f.dir - size: 814720415 - nfiles: 31 + md5: f159d40353b01ffdcf1b1b490c019f1f.dir + size: 787748148 + nfiles: 32 - path: data/prepared_data hash: md5 - md5: 84fa631bd02686b052d6a7144eafd38e.dir - size: 43859225 + md5: 35d7daa8144434e188ba3b1da4bcf328.dir + size: 33946500 nfiles: 2 params: configs/settings.yaml: @@ -112,8 +134,8 @@ stages: outs: - path: data/predictions/ hash: md5 - md5: 5e60ca251af51de6fef3d0c659f8bb27.dir - size: 627416 + md5: 50d0c76fc56c6290babeff1c84750316.dir + size: 651956 nfiles: 1 generate_metrics: cmd: python 4_generate_metrics.py @@ -124,13 +146,13 @@ stages: size: 3484 - path: data/predictions hash: md5 - md5: 5e60ca251af51de6fef3d0c659f8bb27.dir - size: 627416 + md5: 50d0c76fc56c6290babeff1c84750316.dir + size: 651956 nfiles: 1 - path: data/prepared_data hash: md5 - md5: 84fa631bd02686b052d6a7144eafd38e.dir - size: 43859225 + md5: 35d7daa8144434e188ba3b1da4bcf328.dir + size: 33946500 nfiles: 2 params: configs/settings.yaml: @@ -140,16 +162,5 @@ stages: outs: - path: metrics/metrics.json hash: md5 - md5: 033efa4d4044b6b6fc92dd37194727fa - size: 225 - startup_cleanup: - cmd: python 0_startup_cleanup.py - deps: - - path: 0_startup_cleanup.py - hash: md5 - md5: b1b12f6b6393fbf8b83d23684df0a3d4 - size: 1220 - params: - configs/settings.yaml: - default.startup_cleanup.artefacts: ./data - default.startup_cleanup.metrics: ./metrics + md5: 542b982d6aa9fe0fdb89611e4299cb1e + size: 228 diff --git a/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements-dev.txt b/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements-dev.txt index 0d259fb..258981d 100644 --- a/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements-dev.txt +++ b/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements-dev.txt @@ -1,7 +1,7 @@ joblib==1.3.2 boto3==1.28.17 -pandas==1.5.3 -autogluon==0.8.2 +pandas==2.1.4 +autogluon==1.0.0 dynaconf==3.2.0 pyarrow==13.0.0 pre-commit==3.3.3 diff --git a/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements.txt b/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements.txt index afad9be..2ab48e9 100644 --- a/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements.txt +++ b/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements.txt @@ -1,7 +1,7 @@ joblib==1.3.2 boto3==1.28.17 -pandas==1.5.3 -autogluon==0.8.2 +pandas==2.1.4 +autogluon==1.0.0 dynaconf==3.2.0 pyarrow==13.0.0 PyYAML==6.0.1 diff --git a/modules/ml-pipeline/src/pipeline/requirements/training/requirements-dev.txt b/modules/ml-pipeline/src/pipeline/requirements/training/requirements-dev.txt index d8c5907..2024d84 100644 --- a/modules/ml-pipeline/src/pipeline/requirements/training/requirements-dev.txt +++ b/modules/ml-pipeline/src/pipeline/requirements/training/requirements-dev.txt @@ -1,9 +1,10 @@ joblib==1.3.2 boto3==1.28.17 -pandas==1.5.3 -autogluon==0.8.2 +pandas==2.1.4 +autogluon==1.0.0 +ray==2.6.3 dynaconf==3.2.0 -alibi==0.9.4 +alibi==0.9.5 shap==0.42.1 pyarrow==13.0.0 pre-commit==3.3.3 diff --git a/modules/ml-pipeline/src/pipeline/requirements/training/requirements.txt b/modules/ml-pipeline/src/pipeline/requirements/training/requirements.txt index bbdc2fa..84452a3 100644 --- a/modules/ml-pipeline/src/pipeline/requirements/training/requirements.txt +++ b/modules/ml-pipeline/src/pipeline/requirements/training/requirements.txt @@ -1,4 +1,4 @@ boto3==1.28.41 -pandas==1.5.3 -autogluon==0.8.2 +pandas==2.1.4 +autogluon==1.0.0 dynaconf==3.2.0 From 78bf0a490dbf9bc281cf773f0d2317ea381ac786 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Wed, 27 Mar 2024 23:43:07 +0000 Subject: [PATCH 22/23] use 0.9 training data --- .../src/pipeline/configs/settings.yaml | 2 +- modules/ml-pipeline/src/pipeline/dvc.lock | 50 +++++++++---------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/modules/ml-pipeline/src/pipeline/configs/settings.yaml b/modules/ml-pipeline/src/pipeline/configs/settings.yaml index fecdcb0..66f7089 100644 --- a/modules/ml-pipeline/src/pipeline/configs/settings.yaml +++ b/modules/ml-pipeline/src/pipeline/configs/settings.yaml @@ -19,7 +19,7 @@ default: input_dataclient_type: aws-s3 output_dataclient_type: local data_filepath: s3://retrofit-data-dev/sap_change_model/2024-03-22-18-56-53/dataset_rooms.parquet - train_proportion: 1 + train_proportion: 0.9 output_train_filepath: ./data/prepared_data/train.parquet output_test_filepath: ./data/prepared_data/test.parquet diff --git a/modules/ml-pipeline/src/pipeline/dvc.lock b/modules/ml-pipeline/src/pipeline/dvc.lock index 81224d8..ef40a2d 100644 --- a/modules/ml-pipeline/src/pipeline/dvc.lock +++ b/modules/ml-pipeline/src/pipeline/dvc.lock @@ -45,12 +45,12 @@ stages: default.prepare_data.output_dataclient_type: local default.prepare_data.output_test_filepath: ./data/prepared_data/test.parquet default.prepare_data.output_train_filepath: ./data/prepared_data/train.parquet - default.prepare_data.train_proportion: 1 + default.prepare_data.train_proportion: 0.9 outs: - path: data/prepared_data/ hash: md5 - md5: 35d7daa8144434e188ba3b1da4bcf328.dir - size: 33946500 + md5: 824541f44e6538d2ef10e9d754c79743.dir + size: 36691842 nfiles: 2 build_model: cmd: python 2_build_model.py @@ -61,8 +61,8 @@ stages: size: 4820 - path: data/prepared_data hash: md5 - md5: 35d7daa8144434e188ba3b1da4bcf328.dir - size: 33946500 + md5: 824541f44e6538d2ef10e9d754c79743.dir + size: 36691842 nfiles: 2 params: configs/build_model.yaml: @@ -95,18 +95,18 @@ stages: outs: - path: data/fit_predictions/ hash: md5 - md5: 19d033f5abfa9b064c3e52815e607ced.dir - size: 3927492 + md5: 5a3091120d3497fa00b994d91bc7e5eb.dir + size: 3664806 nfiles: 1 - path: data/model/ hash: md5 - md5: f159d40353b01ffdcf1b1b490c019f1f.dir - size: 787748148 - nfiles: 32 + md5: 074da8dcfa515b9f3d082b21c7d76616.dir + size: 721558897 + nfiles: 31 - path: metrics/fit_metrics.json hash: md5 - md5: e69d56ab9d82f23f2aa66001bd9bebbc - size: 229 + md5: 728a49dcef5a98182325df455f929a33 + size: 225 generate_predictions: cmd: python 3_generate_predictions.py deps: @@ -116,13 +116,13 @@ stages: size: 2464 - path: data/model hash: md5 - md5: f159d40353b01ffdcf1b1b490c019f1f.dir - size: 787748148 - nfiles: 32 + md5: 074da8dcfa515b9f3d082b21c7d76616.dir + size: 721558897 + nfiles: 31 - path: data/prepared_data hash: md5 - md5: 35d7daa8144434e188ba3b1da4bcf328.dir - size: 33946500 + md5: 824541f44e6538d2ef10e9d754c79743.dir + size: 36691842 nfiles: 2 params: configs/settings.yaml: @@ -134,8 +134,8 @@ stages: outs: - path: data/predictions/ hash: md5 - md5: 50d0c76fc56c6290babeff1c84750316.dir - size: 651956 + md5: 680f51234d214d4cab9e6a064c75fc5d.dir + size: 499546 nfiles: 1 generate_metrics: cmd: python 4_generate_metrics.py @@ -146,13 +146,13 @@ stages: size: 3484 - path: data/predictions hash: md5 - md5: 50d0c76fc56c6290babeff1c84750316.dir - size: 651956 + md5: 680f51234d214d4cab9e6a064c75fc5d.dir + size: 499546 nfiles: 1 - path: data/prepared_data hash: md5 - md5: 35d7daa8144434e188ba3b1da4bcf328.dir - size: 33946500 + md5: 824541f44e6538d2ef10e9d754c79743.dir + size: 36691842 nfiles: 2 params: configs/settings.yaml: @@ -162,5 +162,5 @@ stages: outs: - path: metrics/metrics.json hash: md5 - md5: 542b982d6aa9fe0fdb89611e4299cb1e - size: 228 + md5: 67b7ab30a4b0839d20bc6eb0c84e4dd1 + size: 226 From bb3af26c3f77ea8d13e7679db2a366c56760d501 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Thu, 28 Mar 2024 16:06:43 +0000 Subject: [PATCH 23/23] add binary to prediction docker, change requiremnets --- deployment/Dockerfile.prediction.lambda | 2 +- .../src/pipeline/requirements/predictions/requirements-dev.txt | 2 +- .../src/pipeline/requirements/predictions/requirements.txt | 2 +- .../src/pipeline/requirements/training/requirements-dev.txt | 2 +- .../src/pipeline/requirements/training/requirements.txt | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/deployment/Dockerfile.prediction.lambda b/deployment/Dockerfile.prediction.lambda index a2520ba..f8000bf 100644 --- a/deployment/Dockerfile.prediction.lambda +++ b/deployment/Dockerfile.prediction.lambda @@ -9,7 +9,7 @@ ARG RUNTIME_ENVIRONMENT ENV RUNTIME_ENVIRONMENT=${RUNTIME_ENVIRONMENT} # Install necessary build tools - required to test locally -RUN yum install -y gcc python3-devel +RUN yum install -y gcc python3-devel gcc-c++ # Install python packages COPY modules/ml-pipeline/src/pipeline/requirements/predictions/requirements.txt ./requirements.txt diff --git a/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements-dev.txt b/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements-dev.txt index 258981d..734419a 100644 --- a/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements-dev.txt +++ b/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements-dev.txt @@ -2,6 +2,6 @@ joblib==1.3.2 boto3==1.28.17 pandas==2.1.4 autogluon==1.0.0 -dynaconf==3.2.0 +dynaconf==3.2.1 pyarrow==13.0.0 pre-commit==3.3.3 diff --git a/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements.txt b/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements.txt index 2ab48e9..937b000 100644 --- a/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements.txt +++ b/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements.txt @@ -2,6 +2,6 @@ joblib==1.3.2 boto3==1.28.17 pandas==2.1.4 autogluon==1.0.0 -dynaconf==3.2.0 +dynaconf==3.2.1 pyarrow==13.0.0 PyYAML==6.0.1 diff --git a/modules/ml-pipeline/src/pipeline/requirements/training/requirements-dev.txt b/modules/ml-pipeline/src/pipeline/requirements/training/requirements-dev.txt index 2024d84..fe06a4d 100644 --- a/modules/ml-pipeline/src/pipeline/requirements/training/requirements-dev.txt +++ b/modules/ml-pipeline/src/pipeline/requirements/training/requirements-dev.txt @@ -3,7 +3,7 @@ boto3==1.28.17 pandas==2.1.4 autogluon==1.0.0 ray==2.6.3 -dynaconf==3.2.0 +dynaconf==3.2.1 alibi==0.9.5 shap==0.42.1 pyarrow==13.0.0 diff --git a/modules/ml-pipeline/src/pipeline/requirements/training/requirements.txt b/modules/ml-pipeline/src/pipeline/requirements/training/requirements.txt index 84452a3..a5bccd3 100644 --- a/modules/ml-pipeline/src/pipeline/requirements/training/requirements.txt +++ b/modules/ml-pipeline/src/pipeline/requirements/training/requirements.txt @@ -1,4 +1,4 @@ boto3==1.28.41 pandas==2.1.4 autogluon==1.0.0 -dynaconf==3.2.0 +dynaconf==3.2.1