From 66f54a92e25251d5dd3bfc73edfce26420130fa0 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Thu, 18 Jan 2024 00:14:20 +0000 Subject: [PATCH] train new 600 second model with new data --- .../src/pipeline/configs/build_model.yaml | 2 +- .../configs/feature_processor_logic.py | 14 ++--- .../pipeline/configs/post_prediction_logic.py | 4 +- .../src/pipeline/configs/settings.yaml | 8 +-- modules/ml-pipeline/src/pipeline/dvc.lock | 60 +++++++++---------- .../version_control/requirements.txt | 8 +-- 6 files changed, 48 insertions(+), 48 deletions(-) diff --git a/modules/ml-pipeline/src/pipeline/configs/build_model.yaml b/modules/ml-pipeline/src/pipeline/configs/build_model.yaml index 4c72487..9c97ef0 100644 --- a/modules/ml-pipeline/src/pipeline/configs/build_model.yaml +++ b/modules/ml-pipeline/src/pipeline/configs/build_model.yaml @@ -13,7 +13,7 @@ default: output_filepath: ./data/model/allmodels/ problem_type: regression eval_metric: mean_squared_error #mean_absolute_error - time_limit: 400 + time_limit: 600 presets: medium_quality excluded_model_types: ['KNN', 'RF'] infer_limit: 0.05 diff --git a/modules/ml-pipeline/src/pipeline/configs/feature_processor_logic.py b/modules/ml-pipeline/src/pipeline/configs/feature_processor_logic.py index 78c29a9..1094862 100644 --- a/modules/ml-pipeline/src/pipeline/configs/feature_processor_logic.py +++ b/modules/ml-pipeline/src/pipeline/configs/feature_processor_logic.py @@ -9,22 +9,22 @@ Business Logic dict + functions def remove_starting_columns(df): keep_column_index = [ - False if col_name.endswith("_STARTING") else True + False if col_name.endswith("_starting") else True for col_name in list(df.columns) ] keep_columns = df.columns[keep_column_index].to_list() - keep_columns.append("SAP_STARTING") + keep_columns.append("sap_starting") df = df[keep_columns] return df def keep_negative_heat_change(df): - df = df[df["HEAT_DEMAND_CHANGE"] < 0] + df = df[df["heat_demand_change"] < 0] return df def keep_negative_carbon_change(df): - df = df[df["CARBON_CHANGE"] < 0] + df = df[df["carbon_change"] < 0] return df @@ -34,7 +34,7 @@ def remove_unreasonable_habitable_rooms(df): Assumption is that proportion of floor area to habitable rooms should be at least 6.5m2 """ minimum_room_size_index = ( - df["TOTAL_FLOOR_AREA_ENDING"] / df["NUMBER_HABITABLE_ROOMS"] >= 6.5 + df["total_floor_area_ending"] / df["number_habitable_rooms"] >= 6.5 ) df = df[minimum_room_size_index] return df @@ -43,14 +43,14 @@ def remove_unreasonable_habitable_rooms(df): def remove_top_1_percent_heat_demand(df): # threshold_value = df.describe(percentiles=[0.99])['HEAT_DEMAND_STARTING']['99%'] threshold_value = 860 - df = df[df["HEAT_DEMAND_STARTING"] < threshold_value] + df = df[df["heat_demand_starting"] < threshold_value] return df def remove_top_1_percent_carbon(df): # threshold_value = df.describe(percentiles=[0.99])['CARBON_STARTING']['99%'] threshold_value = 18 - df = df[df["CARBON_STARTING"] < threshold_value] + df = df[df["carbon_starting"] < threshold_value] return df diff --git a/modules/ml-pipeline/src/pipeline/configs/post_prediction_logic.py b/modules/ml-pipeline/src/pipeline/configs/post_prediction_logic.py index 83389b0..69e9575 100644 --- a/modules/ml-pipeline/src/pipeline/configs/post_prediction_logic.py +++ b/modules/ml-pipeline/src/pipeline/configs/post_prediction_logic.py @@ -13,10 +13,10 @@ def clip_predictions_to_minimum_value( predictions_df = pd.concat([data, predictions], axis=1) # We expect all prediction to be atleast one point improvement replace_index = ( - predictions_df["predictions"] > predictions_df["HEAT_DEMAND_STARTING"] - 1 + predictions_df["predictions"] > predictions_df["heat_demand_starting"] - 1 ) predictions_df.loc[replace_index, "predictions"] = ( - predictions_df.loc[replace_index, "HEAT_DEMAND_STARTING"] - minimum_value + predictions_df.loc[replace_index, "heat_demand_starting"] - minimum_value ) predictions_new = predictions_df["predictions"] diff --git a/modules/ml-pipeline/src/pipeline/configs/settings.yaml b/modules/ml-pipeline/src/pipeline/configs/settings.yaml index 6dd7e2b..09792cf 100644 --- a/modules/ml-pipeline/src/pipeline/configs/settings.yaml +++ b/modules/ml-pipeline/src/pipeline/configs/settings.yaml @@ -21,7 +21,7 @@ default: # data_filepath: s3://retrofit-data-dev/sap_change_model/dataset_with_differencing.parquet # data_filepath: s3://retrofit-data-dev/sap_change_model/floor_area_clean_test.parquet # data_filepath: s3://retrofit-data-dev/sap_change_model/dataset_without_differencing.parquet - data_filepath: s3://retrofit-data-dev/sap_change_model/dataset_test.parquet + data_filepath: s3://retrofit-data-dev/sap_change_model/dataset.parquet train_proportion: 0.9 output_train_filepath: ./data/prepared_data/train.parquet output_test_filepath: ./data/prepared_data/test.parquet @@ -31,9 +31,9 @@ default: feature_processor_config: subsample_amount: null subsample_seed: 0 - target: HEAT_DEMAND_ENDING - identifier_columns: ["UPRN"] - drop_columns: ["HEAT_DEMAND_CHANGE", "CARBON_CHANGE", "RDSAP_CHANGE", "SAP_ENDING", "CARBON_ENDING"] + target: heat_demand_ending + identifier_columns: ["uprn"] + drop_columns: ["heat_demand_change", "carbon_change", "rdsap_change", "sap_ending", "carbon_ending"] # retain_features: ["SAP_STARTING", "TOTAL_FLOOR_AREA_DIFF"] retain_features: null diff --git a/modules/ml-pipeline/src/pipeline/dvc.lock b/modules/ml-pipeline/src/pipeline/dvc.lock index 0011db6..c1bb042 100644 --- a/modules/ml-pipeline/src/pipeline/dvc.lock +++ b/modules/ml-pipeline/src/pipeline/dvc.lock @@ -10,17 +10,17 @@ stages: params: configs/settings.yaml: default.feature_processor.feature_processor_config.drop_columns: - - HEAT_DEMAND_CHANGE - - CARBON_CHANGE - - RDSAP_CHANGE - - SAP_ENDING - - CARBON_ENDING + - heat_demand_change + - carbon_change + - rdsap_change + - sap_ending + - carbon_ending default.feature_processor.feature_processor_config.retain_features: default.feature_processor.feature_processor_config.subsample_amount: default.feature_processor.feature_processor_config.subsample_seed: 0 - default.feature_processor.feature_processor_config.target: HEAT_DEMAND_ENDING + default.feature_processor.feature_processor_config.target: heat_demand_ending default.feature_processor.feature_processor_type: dataframe - default.prepare_data.data_filepath: s3://retrofit-data-dev/sap_change_model/dataset_test.parquet + default.prepare_data.data_filepath: s3://retrofit-data-dev/sap_change_model/dataset.parquet default.prepare_data.input_dataclient_type: aws-s3 default.prepare_data.output_dataclient_type: local default.prepare_data.output_test_filepath: ./data/prepared_data/test.parquet @@ -29,8 +29,8 @@ stages: outs: - path: data/prepared_data/ hash: md5 - md5: f235f38714fefcf6e4927ae95ba912c3.dir - size: 30774760 + md5: 613ddd198a29002e6e05a2d60275d924.dir + size: 32746979 nfiles: 2 build_model: cmd: python 2_build_model.py @@ -41,8 +41,8 @@ stages: size: 4149 - path: data/prepared_data hash: md5 - md5: f235f38714fefcf6e4927ae95ba912c3.dir - size: 30774760 + md5: 613ddd198a29002e6e05a2d60275d924.dir + size: 32746979 nfiles: 2 params: configs/build_model.yaml: @@ -58,7 +58,7 @@ stages: output_filepath: ./data/model/allmodels/ problem_type: regression eval_metric: mean_squared_error - time_limit: 400 + time_limit: 600 presets: medium_quality excluded_model_types: - KNN @@ -68,13 +68,13 @@ stages: outs: - path: data/model/ hash: md5 - md5: a868845999b46e0272dc27f5cb5bc618.dir - size: 310555147 - nfiles: 24 + md5: 837a42a0655862229620495c645d5fed.dir + size: 342382387 + nfiles: 26 - path: metrics/fit_metrics.json hash: md5 - md5: 809f27735c77cbcb62866b96018eedea - size: 216 + md5: f8a394b86c33dc1b3ce97abed803c8f1 + size: 220 generate_predictions: cmd: python 3_generate_predictions.py deps: @@ -84,13 +84,13 @@ stages: size: 2464 - path: data/model hash: md5 - md5: a868845999b46e0272dc27f5cb5bc618.dir - size: 310555147 - nfiles: 24 + md5: 837a42a0655862229620495c645d5fed.dir + size: 342382387 + nfiles: 26 - path: data/prepared_data hash: md5 - md5: f235f38714fefcf6e4927ae95ba912c3.dir - size: 30774760 + md5: 613ddd198a29002e6e05a2d60275d924.dir + size: 32746979 nfiles: 2 params: configs/settings.yaml: @@ -102,8 +102,8 @@ stages: outs: - path: data/predictions/ hash: md5 - md5: 2098fe82304751025e427f2cc241a2ff.dir - size: 295849 + md5: 75f8326e99eb9e1032728208229ec37b.dir + size: 314002 nfiles: 1 generate_metrics: cmd: python 4_generate_metrics.py @@ -114,13 +114,13 @@ stages: size: 3448 - path: data/predictions hash: md5 - md5: 2098fe82304751025e427f2cc241a2ff.dir - size: 295849 + md5: 75f8326e99eb9e1032728208229ec37b.dir + size: 314002 nfiles: 1 - path: data/prepared_data hash: md5 - md5: f235f38714fefcf6e4927ae95ba912c3.dir - size: 30774760 + md5: 613ddd198a29002e6e05a2d60275d924.dir + size: 32746979 nfiles: 2 params: configs/settings.yaml: @@ -130,8 +130,8 @@ stages: outs: - path: metrics/metrics.json hash: md5 - md5: aa671878e1bd8c6a8d4b5f9788c817c4 - size: 219 + md5: 269e89593f5e7ceb507c31dac2c2dd35 + size: 220 startup_cleanup: cmd: python 0_startup_cleanup.py deps: diff --git a/modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt b/modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt index 91cb005..a2b9531 100644 --- a/modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt +++ b/modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt @@ -1,4 +1,4 @@ -dvc==3.18.0 -dvc-s3==2.23.0 -gto==1.0.4 -pyOpenSSL==23.2.0 +dvc==3.36.0 +dvc-s3==3.0.1 +gto==1.6.1 +pyOpenSSL==23.3.0