diff --git a/modules/ml-pipeline/src/pipeline/configs/build_model.yaml b/modules/ml-pipeline/src/pipeline/configs/build_model.yaml index 49fbac4..a5eb62a 100644 --- a/modules/ml-pipeline/src/pipeline/configs/build_model.yaml +++ b/modules/ml-pipeline/src/pipeline/configs/build_model.yaml @@ -12,7 +12,7 @@ default: AutogluonAutoML: output_filepath: ./data/model/autogluonmodel/ problem_type: regression - eval_metric: mean_absolute_error + eval_metric: mean_squared_error #mean_absolute_error time_limit: 1000 presets: medium_quality - excluded_model_types: ['KNN'] + excluded_model_types: ['KNN', 'RF'] diff --git a/modules/ml-pipeline/src/pipeline/configs/feature_processor_logic.py b/modules/ml-pipeline/src/pipeline/configs/feature_processor_logic.py index 1f97005..c32d2fe 100644 --- a/modules/ml-pipeline/src/pipeline/configs/feature_processor_logic.py +++ b/modules/ml-pipeline/src/pipeline/configs/feature_processor_logic.py @@ -27,7 +27,7 @@ def remove_starting_columns(df): # return df business_logic = { - "remove_starting_columns": remove_starting_columns + # "remove_starting_columns": remove_starting_columns # "keep_ENDING_COLUMNS": keep_ending_columns } diff --git a/modules/ml-pipeline/src/pipeline/configs/settings.yaml b/modules/ml-pipeline/src/pipeline/configs/settings.yaml index 588dd9a..1a3c58a 100644 --- a/modules/ml-pipeline/src/pipeline/configs/settings.yaml +++ b/modules/ml-pipeline/src/pipeline/configs/settings.yaml @@ -18,7 +18,10 @@ default: prepare_data: input_dataclient_type: aws-s3 output_dataclient_type: local - data_filepath: s3://retrofit-data-dev/sap_change_model/dataset_without_differencing.parquet + # data_filepath: s3://retrofit-data-dev/sap_change_model/dataset_with_differencing.parquet + # data_filepath: s3://retrofit-data-dev/sap_change_model/floor_area_clean_test.parquet + # data_filepath: s3://retrofit-data-dev/sap_change_model/dataset_without_differencing.parquet + data_filepath: s3://retrofit-data-dev/sap_change_model/dataset.parquet train_proportion: 0.9 output_train_filepath: ./data/prepared_data/train.parquet output_test_filepath: ./data/prepared_data/test.parquet @@ -30,6 +33,7 @@ default: subsample_seed: 0 target: SAP_ENDING drop_columns: ["UPRN", "HEAT_DEMAND_CHANGE", "CARBON_CHANGE", "RDSAP_CHANGE", "HEAT_DEMAND_ENDING", "CARBON_ENDING"] + # retain_features: ["SAP_STARTING", "TOTAL_FLOOR_AREA_DIFF"] retain_features: null generate_predictions: diff --git a/modules/ml-pipeline/src/pipeline/dvc.lock b/modules/ml-pipeline/src/pipeline/dvc.lock index 9b9a3e0..1cc260c 100644 --- a/modules/ml-pipeline/src/pipeline/dvc.lock +++ b/modules/ml-pipeline/src/pipeline/dvc.lock @@ -21,7 +21,7 @@ stages: default.feature_processor.feature_processor_config.subsample_seed: 0 default.feature_processor.feature_processor_config.target: SAP_ENDING default.feature_processor.feature_processor_type: dataframe - default.prepare_data.data_filepath: s3://retrofit-data-dev/sap_change_model/dataset_without_differencing.parquet + default.prepare_data.data_filepath: s3://retrofit-data-dev/sap_change_model/dataset.parquet default.prepare_data.input_dataclient_type: aws-s3 default.prepare_data.output_dataclient_type: local default.prepare_data.output_test_filepath: ./data/prepared_data/test.parquet @@ -30,8 +30,8 @@ stages: outs: - path: data/prepared_data/ hash: md5 - md5: f9ef7ad073b43b249b43faa75c62fe07.dir - size: 21115444 + md5: 951ad046d4fca2b977a314f9520e8235.dir + size: 28249626 nfiles: 2 build_model: cmd: python 2_build_model.py @@ -42,8 +42,8 @@ stages: size: 5181 - path: data/prepared_data hash: md5 - md5: f9ef7ad073b43b249b43faa75c62fe07.dir - size: 21115444 + md5: 951ad046d4fca2b977a314f9520e8235.dir + size: 28249626 nfiles: 2 params: configs/build_model.yaml: @@ -58,21 +58,22 @@ stages: AutogluonAutoML: output_filepath: ./data/model/autogluonmodel/ problem_type: regression - eval_metric: mean_absolute_error + eval_metric: mean_squared_error time_limit: 1000 presets: medium_quality excluded_model_types: - KNN + - RF outs: - path: data/model/ hash: md5 - md5: d073af40ba5c7c2d9b8064665062f51e.dir - size: 363710367 + md5: 6cedf54b03ce278149c67ed118ed0c59.dir + size: 408028954 nfiles: 20 - path: metrics/fit_metrics.json hash: md5 - md5: dcd9ea03a2771077e1bd14018bb7fd18 - size: 183 + md5: 6352c43a896bedc387ac6fb0e1c6e09f + size: 184 generate_predictions: cmd: python 3_generate_predictions.py deps: @@ -82,13 +83,13 @@ stages: size: 4720 - path: data/model hash: md5 - md5: d073af40ba5c7c2d9b8064665062f51e.dir - size: 363710367 + md5: 6cedf54b03ce278149c67ed118ed0c59.dir + size: 408028954 nfiles: 20 - path: data/prepared_data hash: md5 - md5: f9ef7ad073b43b249b43faa75c62fe07.dir - size: 21115444 + md5: 951ad046d4fca2b977a314f9520e8235.dir + size: 28249626 nfiles: 2 params: configs/settings.yaml: @@ -100,8 +101,8 @@ stages: outs: - path: data/predictions/ hash: md5 - md5: a2ecfae1e418fe9cb9fe044c148bbb37.dir - size: 381538 + md5: 5f5b251bb260c366b73b9481b8ffec4c.dir + size: 374624 nfiles: 1 generate_metrics: cmd: python 4_generate_metrics.py @@ -112,13 +113,13 @@ stages: size: 4487 - path: data/predictions hash: md5 - md5: a2ecfae1e418fe9cb9fe044c148bbb37.dir - size: 381538 + md5: 5f5b251bb260c366b73b9481b8ffec4c.dir + size: 374624 nfiles: 1 - path: data/prepared_data hash: md5 - md5: f9ef7ad073b43b249b43faa75c62fe07.dir - size: 21115444 + md5: 951ad046d4fca2b977a314f9520e8235.dir + size: 28249626 nfiles: 2 params: configs/settings.yaml: @@ -128,8 +129,8 @@ stages: outs: - path: metrics/metrics.json hash: md5 - md5: ec02774fd01243fa4706189c60087ccf - size: 182 + md5: 1268ce542c45da1f38286aa941f21c1e + size: 183 startup_cleanup: cmd: python 0_startup_cleanup.py deps: