test model with 1 percent o change records

modules/ml-pipeline/src/pipeline/configs/build_model.yaml

@@ -13,7 +13,7 @@ default:
       output_filepath: ./data/model/allmodels/
       problem_type: regression
       eval_metric: mean_squared_error #mean_absolute_error
-      time_limit: 600
+      time_limit: 400
       presets: medium_quality
       excluded_model_types: ['KNN', 'RF']
       infer_limit: 0.05

modules/ml-pipeline/src/pipeline/configs/settings.yaml

@@ -22,7 +22,8 @@ default:
     # data_filepath: s3://retrofit-data-dev/sap_change_model/floor_area_clean_test.parquet
     # data_filepath: s3://retrofit-data-dev/sap_change_model/dataset_without_differencing.parquet
     # data_filepath: s3://retrofit-data-dev/sap_change_model/dataset_test.parquet
-    data_filepath: s3://retrofit-data-dev/sap_change_model/dataset.parquet
+    # data_filepath: s3://retrofit-data-dev/sap_change_model/dataset.parquet
+    data_filepath: s3://retrofit-datalake-dev/dataset_with0.parquet
     train_proportion: 0.9
     output_train_filepath: ./data/prepared_data/train.parquet
     output_test_filepath: ./data/prepared_data/test.parquet

modules/ml-pipeline/src/pipeline/dvc.lock

@@ -20,7 +20,7 @@ stages:
         default.feature_processor.feature_processor_config.subsample_seed: 0
         default.feature_processor.feature_processor_config.target: sap_ending
         default.feature_processor.feature_processor_type: dataframe
-        default.prepare_data.data_filepath: s3://retrofit-data-dev/sap_change_model/dataset.parquet
+        default.prepare_data.data_filepath: s3://retrofit-datalake-dev/dataset_with0.parquet
         default.prepare_data.input_dataclient_type: aws-s3
         default.prepare_data.output_dataclient_type: local
         default.prepare_data.output_test_filepath: ./data/prepared_data/test.parquet
@@ -29,8 +29,8 @@ stages:
     outs:
     - path: data/prepared_data/
       hash: md5
-      md5: d047420c632d91203199b9a93b6b0134.dir
-      size: 39476967
+      md5: 1b1f7467e4abc12e6febbf2a84756914.dir
+      size: 39780684
       nfiles: 2
   build_model:
     cmd: python 2_build_model.py
@@ -41,8 +41,8 @@ stages:
       size: 4149
     - path: data/prepared_data
       hash: md5
-      md5: d047420c632d91203199b9a93b6b0134.dir
-      size: 39476967
+      md5: 1b1f7467e4abc12e6febbf2a84756914.dir
+      size: 39780684
       nfiles: 2
     params:
       configs/build_model.yaml:
@@ -58,7 +58,7 @@ stages:
               output_filepath: ./data/model/allmodels/
               problem_type: regression
               eval_metric: mean_squared_error
-              time_limit: 600
+              time_limit: 400
               presets: medium_quality
               excluded_model_types:
               - KNN
@@ -68,13 +68,13 @@ stages:
     outs:
     - path: data/model/
       hash: md5
-      md5: 0ad794c5498acfcc79893a371b29be62.dir
-      size: 372199625
+      md5: c83b4cf0c51bd433bfb38307e978ed39.dir
+      size: 344485548
       nfiles: 24
     - path: metrics/fit_metrics.json
       hash: md5
-      md5: 534fa836074bdd9795b5879f0c479681
-      size: 225
+      md5: 3105f9cf71b69b5b0f5675b2c169273c
+      size: 223
   generate_predictions:
     cmd: python 3_generate_predictions.py
     deps:
@@ -84,13 +84,13 @@ stages:
       size: 2464
     - path: data/model
       hash: md5
-      md5: 0ad794c5498acfcc79893a371b29be62.dir
-      size: 372199625
+      md5: c83b4cf0c51bd433bfb38307e978ed39.dir
+      size: 344485548
       nfiles: 24
     - path: data/prepared_data
       hash: md5
-      md5: d047420c632d91203199b9a93b6b0134.dir
-      size: 39476967
+      md5: 1b1f7467e4abc12e6febbf2a84756914.dir
+      size: 39780684
       nfiles: 2
     params:
       configs/settings.yaml:
@@ -102,8 +102,8 @@ stages:
     outs:
     - path: data/predictions/
       hash: md5
-      md5: 25ac7334855d5eacc5fd9e2879900f33.dir
-      size: 367393
+      md5: f914cf31400e228ee6e1386155b68e7c.dir
+      size: 369783
       nfiles: 1
   generate_metrics:
     cmd: python 4_generate_metrics.py
@@ -114,13 +114,13 @@ stages:
       size: 3484
     - path: data/predictions
       hash: md5
-      md5: 25ac7334855d5eacc5fd9e2879900f33.dir
-      size: 367393
+      md5: f914cf31400e228ee6e1386155b68e7c.dir
+      size: 369783
       nfiles: 1
     - path: data/prepared_data
       hash: md5
-      md5: d047420c632d91203199b9a93b6b0134.dir
-      size: 39476967
+      md5: 1b1f7467e4abc12e6febbf2a84756914.dir
+      size: 39780684
       nfiles: 2
     params:
       configs/settings.yaml:
@@ -130,8 +130,8 @@ stages:
     outs:
     - path: metrics/metrics.json
       hash: md5
-      md5: a6fa095b4cc44e6dd7828708f8cca18b
-      size: 222
+      md5: c23b7f0628473bf42eef126167e8928e
+      size: 224
   startup_cleanup:
     cmd: python 0_startup_cleanup.py
     deps: