From 2221283de40931958dae784d3f2ac671ed7bdf20 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Fri, 16 Feb 2024 16:43:23 +0000 Subject: [PATCH 01/11] try the scenario cml --- .github/workflows/MLPipelinePullRequest.yml | 4 + .../src/pipeline/5_generate_scenarios.py | 125 ++++++++++++++++++ modules/ml-pipeline/src/pipeline/config.py | 1 + .../src/pipeline/configs/build_model.yaml | 2 +- .../src/pipeline/configs/scenarios.yaml | 9 ++ .../src/pipeline/core/DataClient.py | 10 +- modules/ml-pipeline/src/pipeline/dvc.lock | 68 ++++++---- modules/ml-pipeline/src/pipeline/dvc.yaml | 11 ++ .../src/pipeline/metrics/.gitignore | 1 + 9 files changed, 205 insertions(+), 26 deletions(-) create mode 100644 modules/ml-pipeline/src/pipeline/5_generate_scenarios.py create mode 100644 modules/ml-pipeline/src/pipeline/configs/scenarios.yaml diff --git a/.github/workflows/MLPipelinePullRequest.yml b/.github/workflows/MLPipelinePullRequest.yml index cbc379d..ceb6800 100644 --- a/.github/workflows/MLPipelinePullRequest.yml +++ b/.github/workflows/MLPipelinePullRequest.yml @@ -98,6 +98,10 @@ jobs: git fetch --depth=1 origin ${TARGET_BRANCH}:${TARGET_BRANCH} dvc metrics diff --md --all ${TARGET_BRANCH} >> report.md + echo "## Scenario metrics" > report.md + + cat metrics/scenarios/scenario_table.md >> report.md + cml comment create report.md # echo "## Residuals plot from model" >> report.md diff --git a/modules/ml-pipeline/src/pipeline/5_generate_scenarios.py b/modules/ml-pipeline/src/pipeline/5_generate_scenarios.py new file mode 100644 index 0000000..28bcb9d --- /dev/null +++ b/modules/ml-pipeline/src/pipeline/5_generate_scenarios.py @@ -0,0 +1,125 @@ +""" +Fourth part of the pipeline: +After the model is built and metrics are generated, +we want to test this model against known scenarios +""" + +import os +import pandas as pd +from core.interface.InterfaceModels import MLModel +from core.interface.InterfaceDataClient import DataClient +from configs.post_prediction_logic import post_prediction_logic +from core.DataClient import dataclient_factory +from core.MLModels import model_factory +from core.Logger import logger +from config import settings + +logger.info(f"--- Initiate Parameters ---") + +RUNTIME_ENVIRONMENT = os.environ.get("RUNTIME_ENVIRONMENT", "local") + +client_params = settings.client +prepare_data_params = settings.prepare_data +build_model_params = settings.build_model +generate_predictions_params = settings.generate_predictions +generate_metrics_params = settings.generate_metrics +feature_process_params = settings.feature_processor +scenarios_params = settings.scenarios + +model_filepath = build_model_params["model_save_filepath"] +target = feature_process_params["feature_processor_config"]["target"] +scenario_data_filepaths = scenarios_params["scenario_data_filepaths"] +predictions_column_name = generate_predictions_params["predictions_column_name"] +output_filepath = scenarios_params["output_filepath"] + +logger.info(f"--- Initiate MLModel ---") + +model = model_factory(build_model_params["model_type"]) + +logger.info(f"--- Initiate DataClient ---") + +# Use data client for input and output, as we use dvc to cache later to the cloud +input_dataclient_type = scenarios_params["input_dataclient_type"] +input_dataclient = dataclient_factory( + dataclient_type=input_dataclient_type, + dataclient_config=client_params[input_dataclient_type], +) + +output_dataclient_type = scenarios_params["output_dataclient_type"] +output_dataclient = dataclient_factory( + dataclient_type=output_dataclient_type, + dataclient_config=client_params[output_dataclient_type], +) + + +def generate_scenario_predictions( + input_dataclient: DataClient, + output_dataclient: DataClient, + model: MLModel, + model_filepath: str, + scenario_data_filepaths: list, + predictions_column_name: str, + output_filepath: str, +): + """ + Given the new model, we generate prediction for expected scenarios + """ + + logger.info("--- Loading Scenario Data ---") + + scenario_data = pd.DataFrame() + + # Can have multiple scenario data files + for scenario_data_filepath in scenario_data_filepaths: + scenario_data = pd.concat( + [ + scenario_data, + input_dataclient.load_data(scenario_data_filepath, load_config=None), + ] + ) + + logger.info("--- Loading Model ---") + + model.load_model(model_filepath) + + logger.info("--- Generating Predictions ---") + + predictions = model.predict( + data=scenario_data, post_prediction_logic=post_prediction_logic + ) + + logger.info("--- Generate Scenario Predicted Impact ---") + + predictions_df = pd.DataFrame(predictions) + predictions_df.columns = [predictions_column_name] + + scenario_data = pd.concat([scenario_data, predictions_df], axis=1) + scenario_data["predicted_impact"] = abs( + scenario_data[predictions_column_name] - scenario_data["sap_starting"] + ) + + logger.info("--- Save prediction into metrics ---") + + output_df = scenario_data[["uprn", "id", "impact", "predicted_impact"]] + + output_dataclient.save_data( + obj=output_df, location=output_filepath, save_config=None + ) + + +if __name__ == "__main__": + logger.info(f"--- {__file__} - Start! ---") + + logger.info(f"--- Generate Scenario Predictions ---") + + generate_scenario_predictions( + input_dataclient=input_dataclient, + output_dataclient=output_dataclient, + model=model, + model_filepath=model_filepath, + scenario_data_filepaths=scenario_data_filepaths, + predictions_column_name=predictions_column_name, + output_filepath=output_filepath, + ) + + logger.info(f"--- {__file__} - Complete! ---") diff --git a/modules/ml-pipeline/src/pipeline/config.py b/modules/ml-pipeline/src/pipeline/config.py index 7a7366b..bac430c 100644 --- a/modules/ml-pipeline/src/pipeline/config.py +++ b/modules/ml-pipeline/src/pipeline/config.py @@ -7,6 +7,7 @@ settings = Dynaconf( "./configs/settings.yaml", "./configs/build_model.yaml", "./configs/analysis.yaml", + "./configs/scenarios.yaml", ], ) diff --git a/modules/ml-pipeline/src/pipeline/configs/build_model.yaml b/modules/ml-pipeline/src/pipeline/configs/build_model.yaml index 1acea2a..add3da1 100644 --- a/modules/ml-pipeline/src/pipeline/configs/build_model.yaml +++ b/modules/ml-pipeline/src/pipeline/configs/build_model.yaml @@ -14,7 +14,7 @@ default: output_filepath: ./data/model/allmodels/ problem_type: regression eval_metric: mean_squared_error #mean_absolute_error - time_limit: 4000 + time_limit: 60 presets: medium_quality excluded_model_types: ['RF', 'NN_TORCH', 'KNN', 'XT', 'CAT', 'FASTAI'] infer_limit: 0.05 diff --git a/modules/ml-pipeline/src/pipeline/configs/scenarios.yaml b/modules/ml-pipeline/src/pipeline/configs/scenarios.yaml new file mode 100644 index 0000000..29b6672 --- /dev/null +++ b/modules/ml-pipeline/src/pipeline/configs/scenarios.yaml @@ -0,0 +1,9 @@ +default: + scenarios: + input_dataclient_type: aws-s3 + output_dataclient_type: local + scenario_data_filepaths: + [ + s3://retrofit-data-dev/scenario_data/recommendations_scoring_data.parquet, + ] + output_filepath: ./metrics/scenario_table.md diff --git a/modules/ml-pipeline/src/pipeline/core/DataClient.py b/modules/ml-pipeline/src/pipeline/core/DataClient.py index 53f4072..b38ca32 100644 --- a/modules/ml-pipeline/src/pipeline/core/DataClient.py +++ b/modules/ml-pipeline/src/pipeline/core/DataClient.py @@ -245,7 +245,8 @@ class LocalClient: save_methods = { ".parquet": self._save_parquet, - ".json": self._save_json + ".json": self._save_json, + ".md": self._save_md, # "": _save_directory(**save_config), # ADD MORE save_methods HERE } @@ -294,3 +295,10 @@ class LocalClient: # Write the contents of the buffer to the local file with open(location, "wb") as f: f.write(buffer.getvalue()) + + def _save_md(self, obj: pd.DataFrame, location: str, save_config: dict): + """ + Save object as markdown + """ + + obj.to_markdown(location, **save_config) diff --git a/modules/ml-pipeline/src/pipeline/dvc.lock b/modules/ml-pipeline/src/pipeline/dvc.lock index 2f513d4..5959200 100644 --- a/modules/ml-pipeline/src/pipeline/dvc.lock +++ b/modules/ml-pipeline/src/pipeline/dvc.lock @@ -31,8 +31,8 @@ stages: outs: - path: data/prepared_data/ hash: md5 - md5: 8f0f5481075094460ab852ace2fa9b7a.dir - size: 43692138 + md5: 86d085385f7e170d951e95d5e9d0f0bc.dir + size: 43684784 nfiles: 2 build_model: cmd: python 2_build_model.py @@ -43,8 +43,8 @@ stages: size: 4820 - path: data/prepared_data hash: md5 - md5: 8f0f5481075094460ab852ace2fa9b7a.dir - size: 43692138 + md5: 86d085385f7e170d951e95d5e9d0f0bc.dir + size: 43684784 nfiles: 2 params: configs/build_model.yaml: @@ -61,7 +61,7 @@ stages: output_filepath: ./data/model/allmodels/ problem_type: regression eval_metric: mean_squared_error - time_limit: 4000 + time_limit: 60 presets: medium_quality excluded_model_types: - RF @@ -75,17 +75,17 @@ stages: outs: - path: data/fit_predictions/ hash: md5 - md5: e2a05a84a14d35516a6cda8e0a1e963c.dir - size: 3681005 + md5: 69cbcceee3e360e0040a7c45ed72ef7f.dir + size: 3674358 nfiles: 1 - path: data/model/ hash: md5 - md5: 7b0382d001ed2bd7aec5c8112f69d129.dir - size: 793365790 - nfiles: 30 + md5: 09757210fdbaa9ad216a84285cf1cbf2.dir + size: 353975267 + nfiles: 21 - path: metrics/fit_metrics.json hash: md5 - md5: bcfd8d3bd3af858fa3dc26433bc8cd9e + md5: 69be95e8d60eb7cef41ec1e69fa9d2ce size: 224 generate_predictions: cmd: python 3_generate_predictions.py @@ -96,13 +96,13 @@ stages: size: 2464 - path: data/model hash: md5 - md5: 7b0382d001ed2bd7aec5c8112f69d129.dir - size: 793365790 - nfiles: 30 + md5: 09757210fdbaa9ad216a84285cf1cbf2.dir + size: 353975267 + nfiles: 21 - path: data/prepared_data hash: md5 - md5: 8f0f5481075094460ab852ace2fa9b7a.dir - size: 43692138 + md5: 86d085385f7e170d951e95d5e9d0f0bc.dir + size: 43684784 nfiles: 2 params: configs/settings.yaml: @@ -114,8 +114,8 @@ stages: outs: - path: data/predictions/ hash: md5 - md5: 90b5275b5d9829a42573ade3f5a025d2.dir - size: 648526 + md5: 2a0421436d59d95e52a51571c34e0ce9.dir + size: 647012 nfiles: 1 generate_metrics: cmd: python 4_generate_metrics.py @@ -126,13 +126,13 @@ stages: size: 3484 - path: data/predictions hash: md5 - md5: 90b5275b5d9829a42573ade3f5a025d2.dir - size: 648526 + md5: 2a0421436d59d95e52a51571c34e0ce9.dir + size: 647012 nfiles: 1 - path: data/prepared_data hash: md5 - md5: 8f0f5481075094460ab852ace2fa9b7a.dir - size: 43692138 + md5: 86d085385f7e170d951e95d5e9d0f0bc.dir + size: 43684784 nfiles: 2 params: configs/settings.yaml: @@ -142,8 +142,8 @@ stages: outs: - path: metrics/metrics.json hash: md5 - md5: be48389ba2755e6c18e41243aaa9bb81 - size: 226 + md5: 83698142cedb9fb4df5ab82f408690a2 + size: 222 startup_cleanup: cmd: python 0_startup_cleanup.py deps: @@ -155,3 +155,23 @@ stages: configs/settings.yaml: default.startup_cleanup.artefacts: ./data default.startup_cleanup.metrics: ./metrics + generate_scenerio_metrics: + cmd: python 5_generate_scenarios.py + deps: + - path: 5_generate_scenarios.py + hash: md5 + md5: 30f80ffeb6ee50c5f7b82943a4dc7702 + size: 4014 + params: + configs/scenarios.yaml: + default.scenarios: + input_dataclient_type: aws-s3 + output_dataclient_type: local + scenario_data_filepaths: + - s3://retrofit-data-dev/scenario_data/recommendations_scoring_data.parquet + output_filepath: ./metrics/scenario_table.md + outs: + - path: metrics/scenario_table.md + hash: md5 + md5: 36b1b26224ebbbfd5b2bbb15ae173247 + size: 1648 diff --git a/modules/ml-pipeline/src/pipeline/dvc.yaml b/modules/ml-pipeline/src/pipeline/dvc.yaml index 58889cc..b513184 100644 --- a/modules/ml-pipeline/src/pipeline/dvc.yaml +++ b/modules/ml-pipeline/src/pipeline/dvc.yaml @@ -71,6 +71,17 @@ stages: outs: - metrics/metrics.json always_changed: true + generate_scenerio_metrics: + cmd: python 5_generate_scenarios.py + deps: + - 5_generate_scenarios.py + params: + - configs/scenarios.yaml: + - default.scenarios + outs: + - metrics/scenario_table.md + always_changed: true metrics: - metrics/metrics.json - metrics/fit_metrics.json + - metrics/scenario_table.md diff --git a/modules/ml-pipeline/src/pipeline/metrics/.gitignore b/modules/ml-pipeline/src/pipeline/metrics/.gitignore index e6fbc8d..189c2ee 100644 --- a/modules/ml-pipeline/src/pipeline/metrics/.gitignore +++ b/modules/ml-pipeline/src/pipeline/metrics/.gitignore @@ -1,2 +1,3 @@ /fit_metrics.json /metrics.json +/scenario_table.md From 49e66411ce272fe3ed297f173a742d26d5551384 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Fri, 16 Feb 2024 16:51:43 +0000 Subject: [PATCH 02/11] test this version --- modules/ml-pipeline/src/pipeline/dvc.lock | 44 +++++++++++------------ modules/ml-pipeline/src/pipeline/dvc.yaml | 1 - 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/modules/ml-pipeline/src/pipeline/dvc.lock b/modules/ml-pipeline/src/pipeline/dvc.lock index 20e33ef..c872404 100644 --- a/modules/ml-pipeline/src/pipeline/dvc.lock +++ b/modules/ml-pipeline/src/pipeline/dvc.lock @@ -31,8 +31,8 @@ stages: outs: - path: data/prepared_data/ hash: md5 - md5: 86d085385f7e170d951e95d5e9d0f0bc.dir - size: 43684784 + md5: 174752a2b228f7af687fe91de77ca0b8.dir + size: 42622503 nfiles: 2 build_model: cmd: python 2_build_model.py @@ -43,8 +43,8 @@ stages: size: 4820 - path: data/prepared_data hash: md5 - md5: 86d085385f7e170d951e95d5e9d0f0bc.dir - size: 43684784 + md5: 174752a2b228f7af687fe91de77ca0b8.dir + size: 42622503 nfiles: 2 params: configs/build_model.yaml: @@ -75,17 +75,17 @@ stages: outs: - path: data/fit_predictions/ hash: md5 - md5: 69cbcceee3e360e0040a7c45ed72ef7f.dir - size: 3674358 + md5: a7e32ced2c7ca88a1e80ed0c2135388d.dir + size: 3675177 nfiles: 1 - path: data/model/ hash: md5 - md5: 09757210fdbaa9ad216a84285cf1cbf2.dir - size: 353975267 + md5: 6d81c99ee00e03bba69db468161dfe19.dir + size: 335451645 nfiles: 21 - path: metrics/fit_metrics.json hash: md5 - md5: 69be95e8d60eb7cef41ec1e69fa9d2ce + md5: 296fd7785e867da96eec96683384c444 size: 224 generate_predictions: cmd: python 3_generate_predictions.py @@ -96,13 +96,13 @@ stages: size: 2464 - path: data/model hash: md5 - md5: 09757210fdbaa9ad216a84285cf1cbf2.dir - size: 353975267 + md5: 6d81c99ee00e03bba69db468161dfe19.dir + size: 335451645 nfiles: 21 - path: data/prepared_data hash: md5 - md5: 86d085385f7e170d951e95d5e9d0f0bc.dir - size: 43684784 + md5: 174752a2b228f7af687fe91de77ca0b8.dir + size: 42622503 nfiles: 2 params: configs/settings.yaml: @@ -114,8 +114,8 @@ stages: outs: - path: data/predictions/ hash: md5 - md5: 2a0421436d59d95e52a51571c34e0ce9.dir - size: 647012 + md5: 3fd770fe0f8064cfc30c2b68575f9e7f.dir + size: 647505 nfiles: 1 generate_metrics: cmd: python 4_generate_metrics.py @@ -126,13 +126,13 @@ stages: size: 3484 - path: data/predictions hash: md5 - md5: 2a0421436d59d95e52a51571c34e0ce9.dir - size: 647012 + md5: 3fd770fe0f8064cfc30c2b68575f9e7f.dir + size: 647505 nfiles: 1 - path: data/prepared_data hash: md5 - md5: 86d085385f7e170d951e95d5e9d0f0bc.dir - size: 43684784 + md5: 174752a2b228f7af687fe91de77ca0b8.dir + size: 42622503 nfiles: 2 params: configs/settings.yaml: @@ -142,8 +142,8 @@ stages: outs: - path: metrics/metrics.json hash: md5 - md5: 83698142cedb9fb4df5ab82f408690a2 - size: 222 + md5: fa4972e309c6e278d986f305543b3084 + size: 223 startup_cleanup: cmd: python 0_startup_cleanup.py deps: @@ -173,5 +173,5 @@ stages: outs: - path: metrics/scenario_table.md hash: md5 - md5: 36b1b26224ebbbfd5b2bbb15ae173247 + md5: 634d39623623a82ce8554a38d3fb82b0 size: 1648 diff --git a/modules/ml-pipeline/src/pipeline/dvc.yaml b/modules/ml-pipeline/src/pipeline/dvc.yaml index b513184..5ce35ce 100644 --- a/modules/ml-pipeline/src/pipeline/dvc.yaml +++ b/modules/ml-pipeline/src/pipeline/dvc.yaml @@ -84,4 +84,3 @@ stages: metrics: - metrics/metrics.json - metrics/fit_metrics.json - - metrics/scenario_table.md From fe430c432651da8d46c3f385534908558f21c182 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Fri, 16 Feb 2024 16:54:18 +0000 Subject: [PATCH 03/11] test this version --- .github/workflows/MLPipelinePullRequest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/MLPipelinePullRequest.yml b/.github/workflows/MLPipelinePullRequest.yml index ceb6800..962132c 100644 --- a/.github/workflows/MLPipelinePullRequest.yml +++ b/.github/workflows/MLPipelinePullRequest.yml @@ -100,7 +100,7 @@ jobs: echo "## Scenario metrics" > report.md - cat metrics/scenarios/scenario_table.md >> report.md + cat metrics/scenario_table.md >> report.md cml comment create report.md From 81e7c2a4bd954532a70638a284dc189f149e5dfb Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Fri, 16 Feb 2024 16:57:37 +0000 Subject: [PATCH 04/11] test this version --- .github/workflows/MLPipelinePullRequest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/MLPipelinePullRequest.yml b/.github/workflows/MLPipelinePullRequest.yml index 962132c..493aef9 100644 --- a/.github/workflows/MLPipelinePullRequest.yml +++ b/.github/workflows/MLPipelinePullRequest.yml @@ -98,7 +98,7 @@ jobs: git fetch --depth=1 origin ${TARGET_BRANCH}:${TARGET_BRANCH} dvc metrics diff --md --all ${TARGET_BRANCH} >> report.md - echo "## Scenario metrics" > report.md + echo "## Scenario metrics" >> report.md cat metrics/scenario_table.md >> report.md From cec3cc60e7fd8cc0739182421e2334b3a6d340fa Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Sat, 17 Feb 2024 16:26:49 +0000 Subject: [PATCH 05/11] test less features --- .../src/pipeline/configs/build_model.yaml | 2 +- .../src/pipeline/configs/settings.yaml | 33 +++++- modules/ml-pipeline/src/pipeline/dvc.lock | 111 ++++++++++++++---- 3 files changed, 116 insertions(+), 30 deletions(-) diff --git a/modules/ml-pipeline/src/pipeline/configs/build_model.yaml b/modules/ml-pipeline/src/pipeline/configs/build_model.yaml index 66981bf..fcec7f7 100644 --- a/modules/ml-pipeline/src/pipeline/configs/build_model.yaml +++ b/modules/ml-pipeline/src/pipeline/configs/build_model.yaml @@ -14,7 +14,7 @@ default: output_filepath: ./data/model/allmodels/ problem_type: regression eval_metric: mean_squared_error #mean_absolute_error - time_limit: 60 + time_limit: 4000 presets: medium_quality excluded_model_types: ['RF', 'FASTAI', 'CAT', 'NN_TORCH', 'KNN', 'XT'] infer_limit: 0.05 diff --git a/modules/ml-pipeline/src/pipeline/configs/settings.yaml b/modules/ml-pipeline/src/pipeline/configs/settings.yaml index 19b0a5b..dc28a9a 100644 --- a/modules/ml-pipeline/src/pipeline/configs/settings.yaml +++ b/modules/ml-pipeline/src/pipeline/configs/settings.yaml @@ -24,7 +24,7 @@ default: # data_filepath: s3://retrofit-data-dev/sap_change_model/dataset_test.parquet data_filepath: s3://retrofit-data-dev/sap_change_model/dataset.parquet # data_filepath: s3://retrofit-datalake-dev/dataset_with0perm_all.parquet - train_proportion: 1 + train_proportion: 0.95 output_train_filepath: ./data/prepared_data/train.parquet output_test_filepath: ./data/prepared_data/test.parquet @@ -36,8 +36,35 @@ default: target: sap_ending identifier_columns: ["uprn"] drop_columns: ["heat_demand_change", "carbon_change", "rdsap_change", "heat_demand_ending", "carbon_ending", "days_to_starting", "days_to_ending"] - # retain_features: ["SAP_STARTING", "TOTAL_FLOOR_AREA_DIFF"] - retain_features: null + retain_features: ['uprn', 'sap_starting', 'hot_water_energy_eff_ending', + 'mainheat_energy_eff_ending', 'constituency', 'roof_energy_eff_ending', + 'walls_energy_eff_ending', 'secondheat_description_ending', + 'property_type', 'mainheatc_energy_eff_ending', 'built_form', + 'walls_insulation_thickness_ending', 'potential_energy_efficiency', + 'transaction_type_ending', 'mainheat_energy_eff_starting', + 'floor_thermal_transmittance_ending', 'hot_water_energy_eff_starting', + 'low_energy_lighting_ending', 'heat_demand_starting', + 'photo_supply_ending', 'carbon_starting', + 'walls_thermal_transmittance_ending', 'fuel_type_ending', + 'roof_insulation_thickness_ending', 'transaction_type_starting', + 'total_floor_area_ending', 'number_open_fireplaces_ending', + 'roof_insulation_thickness', 'windows_energy_eff_ending', + 'walls_insulation_thickness', 'floor_height_ending', + 'secondheat_description_starting', 'floor_thermal_transmittance', + 'mainheatc_energy_eff_starting', 'extension_count_ending', + 'has_air_source_heat_pump_ending', 'walls_energy_eff_starting', + 'charging_system_ending', 'construction_age_band', 'glazed_type_ending', + 'roof_thermal_transmittance_ending', + 'floor_insulation_thickness_ending', 'has_mains_gas_ending', + 'estimated_perimeter_starting', 'energy_consumption_potential', + 'environment_impact_potential', 'roof_energy_eff_starting', + 'another_property_below', 'heater_type_ending', + 'walls_thermal_transmittance', 'total_floor_area_starting', + 'multi_glaze_proportion_ending', 'is_suspended', + 'floor_height_starting', 'lighting_energy_eff_ending', + 'energy_tariff_ending', 'fixed_lighting_outlets_count', + 'low_energy_lighting_starting', 'mechanical_ventilation_ending'] + # retain_features: null generate_predictions: input_dataclient_type: local diff --git a/modules/ml-pipeline/src/pipeline/dvc.lock b/modules/ml-pipeline/src/pipeline/dvc.lock index c872404..05844e3 100644 --- a/modules/ml-pipeline/src/pipeline/dvc.lock +++ b/modules/ml-pipeline/src/pipeline/dvc.lock @@ -18,6 +18,65 @@ stages: - days_to_starting - days_to_ending default.feature_processor.feature_processor_config.retain_features: + - uprn + - sap_starting + - hot_water_energy_eff_ending + - mainheat_energy_eff_ending + - constituency + - roof_energy_eff_ending + - walls_energy_eff_ending + - secondheat_description_ending + - property_type + - mainheatc_energy_eff_ending + - built_form + - walls_insulation_thickness_ending + - potential_energy_efficiency + - transaction_type_ending + - mainheat_energy_eff_starting + - floor_thermal_transmittance_ending + - hot_water_energy_eff_starting + - low_energy_lighting_ending + - heat_demand_starting + - photo_supply_ending + - carbon_starting + - walls_thermal_transmittance_ending + - fuel_type_ending + - roof_insulation_thickness_ending + - transaction_type_starting + - total_floor_area_ending + - number_open_fireplaces_ending + - roof_insulation_thickness + - windows_energy_eff_ending + - walls_insulation_thickness + - floor_height_ending + - secondheat_description_starting + - floor_thermal_transmittance + - mainheatc_energy_eff_starting + - extension_count_ending + - has_air_source_heat_pump_ending + - walls_energy_eff_starting + - charging_system_ending + - construction_age_band + - glazed_type_ending + - roof_thermal_transmittance_ending + - floor_insulation_thickness_ending + - has_mains_gas_ending + - estimated_perimeter_starting + - energy_consumption_potential + - environment_impact_potential + - roof_energy_eff_starting + - another_property_below + - heater_type_ending + - walls_thermal_transmittance + - total_floor_area_starting + - multi_glaze_proportion_ending + - is_suspended + - floor_height_starting + - lighting_energy_eff_ending + - energy_tariff_ending + - fixed_lighting_outlets_count + - low_energy_lighting_starting + - mechanical_ventilation_ending default.feature_processor.feature_processor_config.subsample_amount: default.feature_processor.feature_processor_config.subsample_seed: 0 default.feature_processor.feature_processor_config.target: sap_ending @@ -27,12 +86,12 @@ stages: default.prepare_data.output_dataclient_type: local default.prepare_data.output_test_filepath: ./data/prepared_data/test.parquet default.prepare_data.output_train_filepath: ./data/prepared_data/train.parquet - default.prepare_data.train_proportion: 1 + default.prepare_data.train_proportion: 0.95 outs: - path: data/prepared_data/ hash: md5 - md5: 174752a2b228f7af687fe91de77ca0b8.dir - size: 42622503 + md5: 59f8ea78ec225f5a05de451c6145e2d5.dir + size: 34059502 nfiles: 2 build_model: cmd: python 2_build_model.py @@ -43,8 +102,8 @@ stages: size: 4820 - path: data/prepared_data hash: md5 - md5: 174752a2b228f7af687fe91de77ca0b8.dir - size: 42622503 + md5: 59f8ea78ec225f5a05de451c6145e2d5.dir + size: 34059502 nfiles: 2 params: configs/build_model.yaml: @@ -61,7 +120,7 @@ stages: output_filepath: ./data/model/allmodels/ problem_type: regression eval_metric: mean_squared_error - time_limit: 60 + time_limit: 4000 presets: medium_quality excluded_model_types: - RF @@ -75,18 +134,18 @@ stages: outs: - path: data/fit_predictions/ hash: md5 - md5: a7e32ced2c7ca88a1e80ed0c2135388d.dir - size: 3675177 + md5: bb74626ff3d33581efe750955cdff860.dir + size: 3539589 nfiles: 1 - path: data/model/ hash: md5 - md5: 6d81c99ee00e03bba69db468161dfe19.dir - size: 335451645 - nfiles: 21 + md5: e100d4dcccc1c7d30367b0ca0672e3af.dir + size: 654714285 + nfiles: 31 - path: metrics/fit_metrics.json hash: md5 - md5: 296fd7785e867da96eec96683384c444 - size: 224 + md5: d074f5aa588d3405be65a9684f192465 + size: 226 generate_predictions: cmd: python 3_generate_predictions.py deps: @@ -96,13 +155,13 @@ stages: size: 2464 - path: data/model hash: md5 - md5: 6d81c99ee00e03bba69db468161dfe19.dir - size: 335451645 - nfiles: 21 + md5: e100d4dcccc1c7d30367b0ca0672e3af.dir + size: 654714285 + nfiles: 31 - path: data/prepared_data hash: md5 - md5: 174752a2b228f7af687fe91de77ca0b8.dir - size: 42622503 + md5: 59f8ea78ec225f5a05de451c6145e2d5.dir + size: 34059502 nfiles: 2 params: configs/settings.yaml: @@ -114,8 +173,8 @@ stages: outs: - path: data/predictions/ hash: md5 - md5: 3fd770fe0f8064cfc30c2b68575f9e7f.dir - size: 647505 + md5: 36e26c509176caae6290f75ad486810d.dir + size: 232044 nfiles: 1 generate_metrics: cmd: python 4_generate_metrics.py @@ -126,13 +185,13 @@ stages: size: 3484 - path: data/predictions hash: md5 - md5: 3fd770fe0f8064cfc30c2b68575f9e7f.dir - size: 647505 + md5: 36e26c509176caae6290f75ad486810d.dir + size: 232044 nfiles: 1 - path: data/prepared_data hash: md5 - md5: 174752a2b228f7af687fe91de77ca0b8.dir - size: 42622503 + md5: 59f8ea78ec225f5a05de451c6145e2d5.dir + size: 34059502 nfiles: 2 params: configs/settings.yaml: @@ -142,7 +201,7 @@ stages: outs: - path: metrics/metrics.json hash: md5 - md5: fa4972e309c6e278d986f305543b3084 + md5: 7b71931c5857358ca2603889de6abb3a size: 223 startup_cleanup: cmd: python 0_startup_cleanup.py @@ -173,5 +232,5 @@ stages: outs: - path: metrics/scenario_table.md hash: md5 - md5: 634d39623623a82ce8554a38d3fb82b0 + md5: 72db7530c9ca42470ee8bd1a1e7b52b4 size: 1648 From d5f40a8eb294924e0525904d6ee1864999d77c23 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Sat, 17 Feb 2024 21:17:34 +0000 Subject: [PATCH 06/11] only ending --- .../src/pipeline/configs/settings.yaml | 29 ++++---- modules/ml-pipeline/src/pipeline/dvc.lock | 67 +++++++------------ 2 files changed, 36 insertions(+), 60 deletions(-) diff --git a/modules/ml-pipeline/src/pipeline/configs/settings.yaml b/modules/ml-pipeline/src/pipeline/configs/settings.yaml index dc28a9a..a85d3ab 100644 --- a/modules/ml-pipeline/src/pipeline/configs/settings.yaml +++ b/modules/ml-pipeline/src/pipeline/configs/settings.yaml @@ -24,7 +24,7 @@ default: # data_filepath: s3://retrofit-data-dev/sap_change_model/dataset_test.parquet data_filepath: s3://retrofit-data-dev/sap_change_model/dataset.parquet # data_filepath: s3://retrofit-datalake-dev/dataset_with0perm_all.parquet - train_proportion: 0.95 + train_proportion: 0.98 output_train_filepath: ./data/prepared_data/train.parquet output_test_filepath: ./data/prepared_data/test.parquet @@ -41,29 +41,24 @@ default: 'walls_energy_eff_ending', 'secondheat_description_ending', 'property_type', 'mainheatc_energy_eff_ending', 'built_form', 'walls_insulation_thickness_ending', 'potential_energy_efficiency', - 'transaction_type_ending', 'mainheat_energy_eff_starting', - 'floor_thermal_transmittance_ending', 'hot_water_energy_eff_starting', + 'transaction_type_ending', + 'floor_thermal_transmittance_ending', 'low_energy_lighting_ending', 'heat_demand_starting', 'photo_supply_ending', 'carbon_starting', - 'walls_thermal_transmittance_ending', 'fuel_type_ending', - 'roof_insulation_thickness_ending', 'transaction_type_starting', + 'walls_thermal_transmittance_ending', + 'roof_insulation_thickness_ending', 'total_floor_area_ending', 'number_open_fireplaces_ending', - 'roof_insulation_thickness', 'windows_energy_eff_ending', - 'walls_insulation_thickness', 'floor_height_ending', - 'secondheat_description_starting', 'floor_thermal_transmittance', - 'mainheatc_energy_eff_starting', 'extension_count_ending', - 'has_air_source_heat_pump_ending', 'walls_energy_eff_starting', + 'windows_energy_eff_ending', + 'floor_height_ending', + 'extension_count_ending', + 'has_air_source_heat_pump_ending', 'charging_system_ending', 'construction_age_band', 'glazed_type_ending', 'roof_thermal_transmittance_ending', 'floor_insulation_thickness_ending', 'has_mains_gas_ending', 'estimated_perimeter_starting', 'energy_consumption_potential', - 'environment_impact_potential', 'roof_energy_eff_starting', - 'another_property_below', 'heater_type_ending', - 'walls_thermal_transmittance', 'total_floor_area_starting', - 'multi_glaze_proportion_ending', 'is_suspended', - 'floor_height_starting', 'lighting_energy_eff_ending', - 'energy_tariff_ending', 'fixed_lighting_outlets_count', - 'low_energy_lighting_starting', 'mechanical_ventilation_ending'] + 'environment_impact_potential', 'heater_type_ending', + 'multi_glaze_proportion_ending', + 'lighting_energy_eff_ending', 'fixed_lighting_outlets_count'] # retain_features: null generate_predictions: diff --git a/modules/ml-pipeline/src/pipeline/dvc.lock b/modules/ml-pipeline/src/pipeline/dvc.lock index 05844e3..71a9c44 100644 --- a/modules/ml-pipeline/src/pipeline/dvc.lock +++ b/modules/ml-pipeline/src/pipeline/dvc.lock @@ -32,29 +32,19 @@ stages: - walls_insulation_thickness_ending - potential_energy_efficiency - transaction_type_ending - - mainheat_energy_eff_starting - floor_thermal_transmittance_ending - - hot_water_energy_eff_starting - low_energy_lighting_ending - heat_demand_starting - photo_supply_ending - carbon_starting - walls_thermal_transmittance_ending - - fuel_type_ending - roof_insulation_thickness_ending - - transaction_type_starting - total_floor_area_ending - number_open_fireplaces_ending - - roof_insulation_thickness - windows_energy_eff_ending - - walls_insulation_thickness - floor_height_ending - - secondheat_description_starting - - floor_thermal_transmittance - - mainheatc_energy_eff_starting - extension_count_ending - has_air_source_heat_pump_ending - - walls_energy_eff_starting - charging_system_ending - construction_age_band - glazed_type_ending @@ -64,19 +54,10 @@ stages: - estimated_perimeter_starting - energy_consumption_potential - environment_impact_potential - - roof_energy_eff_starting - - another_property_below - heater_type_ending - - walls_thermal_transmittance - - total_floor_area_starting - multi_glaze_proportion_ending - - is_suspended - - floor_height_starting - lighting_energy_eff_ending - - energy_tariff_ending - fixed_lighting_outlets_count - - low_energy_lighting_starting - - mechanical_ventilation_ending default.feature_processor.feature_processor_config.subsample_amount: default.feature_processor.feature_processor_config.subsample_seed: 0 default.feature_processor.feature_processor_config.target: sap_ending @@ -86,12 +67,12 @@ stages: default.prepare_data.output_dataclient_type: local default.prepare_data.output_test_filepath: ./data/prepared_data/test.parquet default.prepare_data.output_train_filepath: ./data/prepared_data/train.parquet - default.prepare_data.train_proportion: 0.95 + default.prepare_data.train_proportion: 0.98 outs: - path: data/prepared_data/ hash: md5 - md5: 59f8ea78ec225f5a05de451c6145e2d5.dir - size: 34059502 + md5: 544427230544c2cc526334e246db4845.dir + size: 26132493 nfiles: 2 build_model: cmd: python 2_build_model.py @@ -102,8 +83,8 @@ stages: size: 4820 - path: data/prepared_data hash: md5 - md5: 59f8ea78ec225f5a05de451c6145e2d5.dir - size: 34059502 + md5: 544427230544c2cc526334e246db4845.dir + size: 26132493 nfiles: 2 params: configs/build_model.yaml: @@ -134,18 +115,18 @@ stages: outs: - path: data/fit_predictions/ hash: md5 - md5: bb74626ff3d33581efe750955cdff860.dir - size: 3539589 + md5: 8f9e2059782dd55d3ecdad54b4551f6a.dir + size: 3630849 nfiles: 1 - path: data/model/ hash: md5 - md5: e100d4dcccc1c7d30367b0ca0672e3af.dir - size: 654714285 + md5: e031eb3c3fdb63917aabfea745b56ac6.dir + size: 618445494 nfiles: 31 - path: metrics/fit_metrics.json hash: md5 - md5: d074f5aa588d3405be65a9684f192465 - size: 226 + md5: e68009f5b66230b3ee4cd2ffc9a2d697 + size: 222 generate_predictions: cmd: python 3_generate_predictions.py deps: @@ -155,13 +136,13 @@ stages: size: 2464 - path: data/model hash: md5 - md5: e100d4dcccc1c7d30367b0ca0672e3af.dir - size: 654714285 + md5: e031eb3c3fdb63917aabfea745b56ac6.dir + size: 618445494 nfiles: 31 - path: data/prepared_data hash: md5 - md5: 59f8ea78ec225f5a05de451c6145e2d5.dir - size: 34059502 + md5: 544427230544c2cc526334e246db4845.dir + size: 26132493 nfiles: 2 params: configs/settings.yaml: @@ -173,8 +154,8 @@ stages: outs: - path: data/predictions/ hash: md5 - md5: 36e26c509176caae6290f75ad486810d.dir - size: 232044 + md5: 1c14c9ac9711f5d33a60890e3ca72454.dir + size: 90361 nfiles: 1 generate_metrics: cmd: python 4_generate_metrics.py @@ -185,13 +166,13 @@ stages: size: 3484 - path: data/predictions hash: md5 - md5: 36e26c509176caae6290f75ad486810d.dir - size: 232044 + md5: 1c14c9ac9711f5d33a60890e3ca72454.dir + size: 90361 nfiles: 1 - path: data/prepared_data hash: md5 - md5: 59f8ea78ec225f5a05de451c6145e2d5.dir - size: 34059502 + md5: 544427230544c2cc526334e246db4845.dir + size: 26132493 nfiles: 2 params: configs/settings.yaml: @@ -201,8 +182,8 @@ stages: outs: - path: metrics/metrics.json hash: md5 - md5: 7b71931c5857358ca2603889de6abb3a - size: 223 + md5: 98e59ea9569522a8665c4e6c1bea7473 + size: 222 startup_cleanup: cmd: python 0_startup_cleanup.py deps: @@ -232,5 +213,5 @@ stages: outs: - path: metrics/scenario_table.md hash: md5 - md5: 72db7530c9ca42470ee8bd1a1e7b52b4 + md5: 3ee1966a06c1e5b9c37797597be94797 size: 1648 From ad2c4d60197a708a3565356c121deb9067476f7e Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Thu, 21 Mar 2024 14:41:58 +0000 Subject: [PATCH 07/11] upgrade autogluon --- .../src/pipeline/configs/build_model.yaml | 5 +- .../src/pipeline/configs/settings.yaml | 2 +- .../ml-pipeline/src/pipeline/core/MLModels.py | 4 +- modules/ml-pipeline/src/pipeline/dvc.lock | 56 ++++++++++--------- .../predictions/requirements-dev.txt | 4 +- .../requirements/predictions/requirements.txt | 4 +- .../training/requirements-dev.txt | 7 ++- .../requirements/training/requirements.txt | 4 +- 8 files changed, 46 insertions(+), 40 deletions(-) diff --git a/modules/ml-pipeline/src/pipeline/configs/build_model.yaml b/modules/ml-pipeline/src/pipeline/configs/build_model.yaml index fcec7f7..6fbf094 100644 --- a/modules/ml-pipeline/src/pipeline/configs/build_model.yaml +++ b/modules/ml-pipeline/src/pipeline/configs/build_model.yaml @@ -14,8 +14,9 @@ default: output_filepath: ./data/model/allmodels/ problem_type: regression eval_metric: mean_squared_error #mean_absolute_error - time_limit: 4000 - presets: medium_quality + time_limit: 1800 + presets: good_quality excluded_model_types: ['RF', 'FASTAI', 'CAT', 'NN_TORCH', 'KNN', 'XT'] infer_limit: 0.05 infer_limit_batch_size: 10000 + ag_args_ensemble: {'num_folds_parallel': 2} diff --git a/modules/ml-pipeline/src/pipeline/configs/settings.yaml b/modules/ml-pipeline/src/pipeline/configs/settings.yaml index 19b0a5b..4757d91 100644 --- a/modules/ml-pipeline/src/pipeline/configs/settings.yaml +++ b/modules/ml-pipeline/src/pipeline/configs/settings.yaml @@ -24,7 +24,7 @@ default: # data_filepath: s3://retrofit-data-dev/sap_change_model/dataset_test.parquet data_filepath: s3://retrofit-data-dev/sap_change_model/dataset.parquet # data_filepath: s3://retrofit-datalake-dev/dataset_with0perm_all.parquet - train_proportion: 1 + train_proportion: 0.9 output_train_filepath: ./data/prepared_data/train.parquet output_test_filepath: ./data/prepared_data/test.parquet diff --git a/modules/ml-pipeline/src/pipeline/core/MLModels.py b/modules/ml-pipeline/src/pipeline/core/MLModels.py index 4fc572a..257261d 100644 --- a/modules/ml-pipeline/src/pipeline/core/MLModels.py +++ b/modules/ml-pipeline/src/pipeline/core/MLModels.py @@ -25,7 +25,7 @@ def model_factory(model_type: str) -> MLModel: models = { "SKLearnLinearRegression": SKLearnLinearRegression(), "SKLearnSVMRegression": SKLearnSVMRegression(), - "AutogluonAutoML": AutogluonAutoML() + "AutogluonAutoML": AutogluonAutoML(), # ADD OTHER MODELS HERE } @@ -151,6 +151,7 @@ class AutogluonAutoML: "excluded_model_types", "infer_limit", "infer_limit_batch_size", + "ag_args_ensemble", ] def load_model(self, path: Union[Path, str]) -> None: @@ -207,6 +208,7 @@ class AutogluonAutoML: excluded_model_types=model_hyperparameters["excluded_model_types"], infer_limit=model_hyperparameters["infer_limit"], infer_limit_batch_size=model_hyperparameters["infer_limit_batch_size"], + ag_args_ensemble=model_hyperparameters["ag_args_ensemble"], ) def predict( diff --git a/modules/ml-pipeline/src/pipeline/dvc.lock b/modules/ml-pipeline/src/pipeline/dvc.lock index 826e654..530a3c8 100644 --- a/modules/ml-pipeline/src/pipeline/dvc.lock +++ b/modules/ml-pipeline/src/pipeline/dvc.lock @@ -27,12 +27,12 @@ stages: default.prepare_data.output_dataclient_type: local default.prepare_data.output_test_filepath: ./data/prepared_data/test.parquet default.prepare_data.output_train_filepath: ./data/prepared_data/train.parquet - default.prepare_data.train_proportion: 1 + default.prepare_data.train_proportion: 0.9 outs: - path: data/prepared_data/ hash: md5 - md5: 3c77fa10cd1cd503eb4d2540394629f6.dir - size: 42626894 + md5: 3d1144848fce4ce50f6abfaec5235552.dir + size: 46392840 nfiles: 2 build_model: cmd: python 2_build_model.py @@ -43,8 +43,8 @@ stages: size: 4820 - path: data/prepared_data hash: md5 - md5: 3c77fa10cd1cd503eb4d2540394629f6.dir - size: 42626894 + md5: 3d1144848fce4ce50f6abfaec5235552.dir + size: 46392840 nfiles: 2 params: configs/build_model.yaml: @@ -61,8 +61,8 @@ stages: output_filepath: ./data/model/allmodels/ problem_type: regression eval_metric: mean_squared_error - time_limit: 4000 - presets: medium_quality + time_limit: 1800 + presets: good_quality excluded_model_types: - RF - FASTAI @@ -72,21 +72,23 @@ stages: - XT infer_limit: 0.05 infer_limit_batch_size: 10000 + ag_args_ensemble: + num_folds_parallel: 2 outs: - path: data/fit_predictions/ hash: md5 - md5: e0a11ac6e4adf69d6180c0217c639a0e.dir - size: 3680908 + md5: 346b6611afbf2070e038bf945249a86e.dir + size: 3384302 nfiles: 1 - path: data/model/ hash: md5 - md5: bdaaf823857f9dc7b6ee2d4b88927cc1.dir - size: 805896324 - nfiles: 31 + md5: 8e37f21728cd092660bafa8c32dc109f.dir + size: 423840922 + nfiles: 118 - path: metrics/fit_metrics.json hash: md5 - md5: 0ed5b1141bbb8bc3156e7c056b29f3cd - size: 225 + md5: d63e1a8d31503055835ac35149554e41 + size: 223 generate_predictions: cmd: python 3_generate_predictions.py deps: @@ -96,13 +98,13 @@ stages: size: 2464 - path: data/model hash: md5 - md5: bdaaf823857f9dc7b6ee2d4b88927cc1.dir - size: 805896324 - nfiles: 31 + md5: 8e37f21728cd092660bafa8c32dc109f.dir + size: 423840922 + nfiles: 118 - path: data/prepared_data hash: md5 - md5: 3c77fa10cd1cd503eb4d2540394629f6.dir - size: 42626894 + md5: 3d1144848fce4ce50f6abfaec5235552.dir + size: 46392840 nfiles: 2 params: configs/settings.yaml: @@ -114,8 +116,8 @@ stages: outs: - path: data/predictions/ hash: md5 - md5: 38707d16ae1e2330cc03f524db9cdd60.dir - size: 648730 + md5: d148baf508140353d62c16d6ab0fb6b7.dir + size: 469224 nfiles: 1 generate_metrics: cmd: python 4_generate_metrics.py @@ -126,13 +128,13 @@ stages: size: 3484 - path: data/predictions hash: md5 - md5: 38707d16ae1e2330cc03f524db9cdd60.dir - size: 648730 + md5: d148baf508140353d62c16d6ab0fb6b7.dir + size: 469224 nfiles: 1 - path: data/prepared_data hash: md5 - md5: 3c77fa10cd1cd503eb4d2540394629f6.dir - size: 42626894 + md5: 3d1144848fce4ce50f6abfaec5235552.dir + size: 46392840 nfiles: 2 params: configs/settings.yaml: @@ -142,8 +144,8 @@ stages: outs: - path: metrics/metrics.json hash: md5 - md5: 145e7ac84ab4a4407b23695a632b4d91 - size: 226 + md5: 196232f94b563ac525cf65ee5cc6d639 + size: 222 startup_cleanup: cmd: python 0_startup_cleanup.py deps: diff --git a/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements-dev.txt b/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements-dev.txt index 0d259fb..258981d 100644 --- a/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements-dev.txt +++ b/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements-dev.txt @@ -1,7 +1,7 @@ joblib==1.3.2 boto3==1.28.17 -pandas==1.5.3 -autogluon==0.8.2 +pandas==2.1.4 +autogluon==1.0.0 dynaconf==3.2.0 pyarrow==13.0.0 pre-commit==3.3.3 diff --git a/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements.txt b/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements.txt index afad9be..2ab48e9 100644 --- a/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements.txt +++ b/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements.txt @@ -1,7 +1,7 @@ joblib==1.3.2 boto3==1.28.17 -pandas==1.5.3 -autogluon==0.8.2 +pandas==2.1.4 +autogluon==1.0.0 dynaconf==3.2.0 pyarrow==13.0.0 PyYAML==6.0.1 diff --git a/modules/ml-pipeline/src/pipeline/requirements/training/requirements-dev.txt b/modules/ml-pipeline/src/pipeline/requirements/training/requirements-dev.txt index d8c5907..2024d84 100644 --- a/modules/ml-pipeline/src/pipeline/requirements/training/requirements-dev.txt +++ b/modules/ml-pipeline/src/pipeline/requirements/training/requirements-dev.txt @@ -1,9 +1,10 @@ joblib==1.3.2 boto3==1.28.17 -pandas==1.5.3 -autogluon==0.8.2 +pandas==2.1.4 +autogluon==1.0.0 +ray==2.6.3 dynaconf==3.2.0 -alibi==0.9.4 +alibi==0.9.5 shap==0.42.1 pyarrow==13.0.0 pre-commit==3.3.3 diff --git a/modules/ml-pipeline/src/pipeline/requirements/training/requirements.txt b/modules/ml-pipeline/src/pipeline/requirements/training/requirements.txt index bbdc2fa..84452a3 100644 --- a/modules/ml-pipeline/src/pipeline/requirements/training/requirements.txt +++ b/modules/ml-pipeline/src/pipeline/requirements/training/requirements.txt @@ -1,4 +1,4 @@ boto3==1.28.41 -pandas==1.5.3 -autogluon==0.8.2 +pandas==2.1.4 +autogluon==1.0.0 dynaconf==3.2.0 From 8a9b5877b53c6dbff2d4f45e28fcb40a80b443d6 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 26 Mar 2024 22:30:50 +0000 Subject: [PATCH 08/11] medium model with scenario and upgraded autogluon --- .github/workflows/MLPipelinePullRequest.yml | 4 + .../src/pipeline/5_generate_scenarios.py | 125 ++++++++++++++++++ modules/ml-pipeline/src/pipeline/config.py | 1 + .../src/pipeline/configs/build_model.yaml | 4 +- .../src/pipeline/configs/scenarios.yaml | 8 ++ .../src/pipeline/configs/settings.yaml | 37 ++++-- .../src/pipeline/core/DataClient.py | 10 +- modules/ml-pipeline/src/pipeline/dvc.lock | 96 +++++++++----- modules/ml-pipeline/src/pipeline/dvc.yaml | 10 ++ .../src/pipeline/metrics/.gitignore | 1 + 10 files changed, 250 insertions(+), 46 deletions(-) create mode 100644 modules/ml-pipeline/src/pipeline/5_generate_scenarios.py create mode 100644 modules/ml-pipeline/src/pipeline/configs/scenarios.yaml diff --git a/.github/workflows/MLPipelinePullRequest.yml b/.github/workflows/MLPipelinePullRequest.yml index cbc379d..493aef9 100644 --- a/.github/workflows/MLPipelinePullRequest.yml +++ b/.github/workflows/MLPipelinePullRequest.yml @@ -98,6 +98,10 @@ jobs: git fetch --depth=1 origin ${TARGET_BRANCH}:${TARGET_BRANCH} dvc metrics diff --md --all ${TARGET_BRANCH} >> report.md + echo "## Scenario metrics" >> report.md + + cat metrics/scenario_table.md >> report.md + cml comment create report.md # echo "## Residuals plot from model" >> report.md diff --git a/modules/ml-pipeline/src/pipeline/5_generate_scenarios.py b/modules/ml-pipeline/src/pipeline/5_generate_scenarios.py new file mode 100644 index 0000000..28bcb9d --- /dev/null +++ b/modules/ml-pipeline/src/pipeline/5_generate_scenarios.py @@ -0,0 +1,125 @@ +""" +Fourth part of the pipeline: +After the model is built and metrics are generated, +we want to test this model against known scenarios +""" + +import os +import pandas as pd +from core.interface.InterfaceModels import MLModel +from core.interface.InterfaceDataClient import DataClient +from configs.post_prediction_logic import post_prediction_logic +from core.DataClient import dataclient_factory +from core.MLModels import model_factory +from core.Logger import logger +from config import settings + +logger.info(f"--- Initiate Parameters ---") + +RUNTIME_ENVIRONMENT = os.environ.get("RUNTIME_ENVIRONMENT", "local") + +client_params = settings.client +prepare_data_params = settings.prepare_data +build_model_params = settings.build_model +generate_predictions_params = settings.generate_predictions +generate_metrics_params = settings.generate_metrics +feature_process_params = settings.feature_processor +scenarios_params = settings.scenarios + +model_filepath = build_model_params["model_save_filepath"] +target = feature_process_params["feature_processor_config"]["target"] +scenario_data_filepaths = scenarios_params["scenario_data_filepaths"] +predictions_column_name = generate_predictions_params["predictions_column_name"] +output_filepath = scenarios_params["output_filepath"] + +logger.info(f"--- Initiate MLModel ---") + +model = model_factory(build_model_params["model_type"]) + +logger.info(f"--- Initiate DataClient ---") + +# Use data client for input and output, as we use dvc to cache later to the cloud +input_dataclient_type = scenarios_params["input_dataclient_type"] +input_dataclient = dataclient_factory( + dataclient_type=input_dataclient_type, + dataclient_config=client_params[input_dataclient_type], +) + +output_dataclient_type = scenarios_params["output_dataclient_type"] +output_dataclient = dataclient_factory( + dataclient_type=output_dataclient_type, + dataclient_config=client_params[output_dataclient_type], +) + + +def generate_scenario_predictions( + input_dataclient: DataClient, + output_dataclient: DataClient, + model: MLModel, + model_filepath: str, + scenario_data_filepaths: list, + predictions_column_name: str, + output_filepath: str, +): + """ + Given the new model, we generate prediction for expected scenarios + """ + + logger.info("--- Loading Scenario Data ---") + + scenario_data = pd.DataFrame() + + # Can have multiple scenario data files + for scenario_data_filepath in scenario_data_filepaths: + scenario_data = pd.concat( + [ + scenario_data, + input_dataclient.load_data(scenario_data_filepath, load_config=None), + ] + ) + + logger.info("--- Loading Model ---") + + model.load_model(model_filepath) + + logger.info("--- Generating Predictions ---") + + predictions = model.predict( + data=scenario_data, post_prediction_logic=post_prediction_logic + ) + + logger.info("--- Generate Scenario Predicted Impact ---") + + predictions_df = pd.DataFrame(predictions) + predictions_df.columns = [predictions_column_name] + + scenario_data = pd.concat([scenario_data, predictions_df], axis=1) + scenario_data["predicted_impact"] = abs( + scenario_data[predictions_column_name] - scenario_data["sap_starting"] + ) + + logger.info("--- Save prediction into metrics ---") + + output_df = scenario_data[["uprn", "id", "impact", "predicted_impact"]] + + output_dataclient.save_data( + obj=output_df, location=output_filepath, save_config=None + ) + + +if __name__ == "__main__": + logger.info(f"--- {__file__} - Start! ---") + + logger.info(f"--- Generate Scenario Predictions ---") + + generate_scenario_predictions( + input_dataclient=input_dataclient, + output_dataclient=output_dataclient, + model=model, + model_filepath=model_filepath, + scenario_data_filepaths=scenario_data_filepaths, + predictions_column_name=predictions_column_name, + output_filepath=output_filepath, + ) + + logger.info(f"--- {__file__} - Complete! ---") diff --git a/modules/ml-pipeline/src/pipeline/config.py b/modules/ml-pipeline/src/pipeline/config.py index 7a7366b..bac430c 100644 --- a/modules/ml-pipeline/src/pipeline/config.py +++ b/modules/ml-pipeline/src/pipeline/config.py @@ -7,6 +7,7 @@ settings = Dynaconf( "./configs/settings.yaml", "./configs/build_model.yaml", "./configs/analysis.yaml", + "./configs/scenarios.yaml", ], ) diff --git a/modules/ml-pipeline/src/pipeline/configs/build_model.yaml b/modules/ml-pipeline/src/pipeline/configs/build_model.yaml index 6fbf094..a36bfbc 100644 --- a/modules/ml-pipeline/src/pipeline/configs/build_model.yaml +++ b/modules/ml-pipeline/src/pipeline/configs/build_model.yaml @@ -15,8 +15,8 @@ default: problem_type: regression eval_metric: mean_squared_error #mean_absolute_error time_limit: 1800 - presets: good_quality - excluded_model_types: ['RF', 'FASTAI', 'CAT', 'NN_TORCH', 'KNN', 'XT'] + presets: medium_quality + excluded_model_types: ['RF', 'CAT', 'NN_TORCH', 'KNN', 'XT'] infer_limit: 0.05 infer_limit_batch_size: 10000 ag_args_ensemble: {'num_folds_parallel': 2} diff --git a/modules/ml-pipeline/src/pipeline/configs/scenarios.yaml b/modules/ml-pipeline/src/pipeline/configs/scenarios.yaml new file mode 100644 index 0000000..e76336a --- /dev/null +++ b/modules/ml-pipeline/src/pipeline/configs/scenarios.yaml @@ -0,0 +1,8 @@ +default: + scenarios: + input_dataclient_type: aws-s3 + output_dataclient_type: local + scenario_data_filepaths: + # - s3://retrofit-data-dev/scenario_data/22-03-2024-19-20-09/recommendations_scoring_data.parquet + - s3://retrofit-data-dev/scenario_data/24-03-2024-20-23-25/recommendations_scoring_data.parquet + output_filepath: ./metrics/scenario_table.md diff --git a/modules/ml-pipeline/src/pipeline/configs/settings.yaml b/modules/ml-pipeline/src/pipeline/configs/settings.yaml index 4757d91..f42b2be 100644 --- a/modules/ml-pipeline/src/pipeline/configs/settings.yaml +++ b/modules/ml-pipeline/src/pipeline/configs/settings.yaml @@ -18,12 +18,7 @@ default: prepare_data: input_dataclient_type: aws-s3 output_dataclient_type: local - # data_filepath: s3://retrofit-data-dev/sap_change_model/dataset_with_differencing.parquet - # data_filepath: s3://retrofit-data-dev/sap_change_model/floor_area_clean_test.parquet - # data_filepath: s3://retrofit-data-dev/sap_change_model/dataset_without_differencing.parquet - # data_filepath: s3://retrofit-data-dev/sap_change_model/dataset_test.parquet - data_filepath: s3://retrofit-data-dev/sap_change_model/dataset.parquet - # data_filepath: s3://retrofit-datalake-dev/dataset_with0perm_all.parquet + data_filepath: s3://retrofit-data-dev/sap_change_model/2024-03-22-18-56-53/dataset_rooms.parquet train_proportion: 0.9 output_train_filepath: ./data/prepared_data/train.parquet output_test_filepath: ./data/prepared_data/test.parquet @@ -35,9 +30,35 @@ default: subsample_seed: 0 target: sap_ending identifier_columns: ["uprn"] - drop_columns: ["heat_demand_change", "carbon_change", "rdsap_change", "heat_demand_ending", "carbon_ending", "days_to_starting", "days_to_ending"] - # retain_features: ["SAP_STARTING", "TOTAL_FLOOR_AREA_DIFF"] + # drop_columns: ["heat_demand_change", "carbon_change", "rdsap_change", "heat_demand_ending", "carbon_ending", "days_to_starting", "days_to_ending"] + drop_columns: [ + "heat_demand_change", "carbon_change", "rdsap_change", "heat_demand_ending", "carbon_ending", "days_to_starting", "days_to_ending", + 'number_habitable_rooms_starting', 'number_habitable_rooms_ending', 'number_heated_rooms_starting', 'number_heated_rooms_ending', + 'number_habitable_rooms', 'number_heated_rooms'] retain_features: null + # retain_features: ['uprn', 'sap_starting', 'hot_water_energy_eff_ending', + # 'mainheat_energy_eff_ending', 'constituency', 'roof_energy_eff_ending', + # 'walls_energy_eff_ending', 'secondheat_description_ending', + # 'property_type', 'mainheatc_energy_eff_ending', 'built_form', + # 'walls_insulation_thickness_ending', 'potential_energy_efficiency', + # 'transaction_type_ending', + # 'floor_thermal_transmittance_ending', + # 'low_energy_lighting_ending', 'heat_demand_starting', + # 'photo_supply_ending', 'carbon_starting', + # 'walls_thermal_transmittance_ending', + # 'roof_insulation_thickness_ending', + # 'total_floor_area_ending', 'number_open_fireplaces_ending', + # 'windows_energy_eff_ending', + # 'floor_height_ending', + # 'extension_count_ending', + # 'has_air_source_heat_pump_ending', + # 'charging_system_ending', 'construction_age_band', 'glazed_type_ending', + # 'roof_thermal_transmittance_ending', + # 'floor_insulation_thickness_ending', 'has_mains_gas_ending', + # 'estimated_perimeter_starting', 'energy_consumption_potential', + # 'environment_impact_potential', 'heater_type_ending', + # 'multi_glaze_proportion_ending', + # 'lighting_energy_eff_ending', 'fixed_lighting_outlets_count'] generate_predictions: input_dataclient_type: local diff --git a/modules/ml-pipeline/src/pipeline/core/DataClient.py b/modules/ml-pipeline/src/pipeline/core/DataClient.py index 53f4072..b38ca32 100644 --- a/modules/ml-pipeline/src/pipeline/core/DataClient.py +++ b/modules/ml-pipeline/src/pipeline/core/DataClient.py @@ -245,7 +245,8 @@ class LocalClient: save_methods = { ".parquet": self._save_parquet, - ".json": self._save_json + ".json": self._save_json, + ".md": self._save_md, # "": _save_directory(**save_config), # ADD MORE save_methods HERE } @@ -294,3 +295,10 @@ class LocalClient: # Write the contents of the buffer to the local file with open(location, "wb") as f: f.write(buffer.getvalue()) + + def _save_md(self, obj: pd.DataFrame, location: str, save_config: dict): + """ + Save object as markdown + """ + + obj.to_markdown(location, **save_config) diff --git a/modules/ml-pipeline/src/pipeline/dvc.lock b/modules/ml-pipeline/src/pipeline/dvc.lock index 530a3c8..fcc035b 100644 --- a/modules/ml-pipeline/src/pipeline/dvc.lock +++ b/modules/ml-pipeline/src/pipeline/dvc.lock @@ -1,5 +1,16 @@ schema: '2.0' stages: + startup_cleanup: + cmd: python 0_startup_cleanup.py + deps: + - path: 0_startup_cleanup.py + hash: md5 + md5: b1b12f6b6393fbf8b83d23684df0a3d4 + size: 1220 + params: + configs/settings.yaml: + default.startup_cleanup.artefacts: ./data + default.startup_cleanup.metrics: ./metrics prepare_data: cmd: python 1_prepare_data.py deps: @@ -17,12 +28,19 @@ stages: - carbon_ending - days_to_starting - days_to_ending + - number_habitable_rooms_starting + - number_habitable_rooms_ending + - number_heated_rooms_starting + - number_heated_rooms_ending + - number_habitable_rooms + - number_heated_rooms default.feature_processor.feature_processor_config.retain_features: default.feature_processor.feature_processor_config.subsample_amount: default.feature_processor.feature_processor_config.subsample_seed: 0 default.feature_processor.feature_processor_config.target: sap_ending default.feature_processor.feature_processor_type: dataframe - default.prepare_data.data_filepath: s3://retrofit-data-dev/sap_change_model/dataset.parquet + default.prepare_data.data_filepath: + s3://retrofit-data-dev/sap_change_model/2024-03-22-18-56-53/dataset_rooms.parquet default.prepare_data.input_dataclient_type: aws-s3 default.prepare_data.output_dataclient_type: local default.prepare_data.output_test_filepath: ./data/prepared_data/test.parquet @@ -31,8 +49,8 @@ stages: outs: - path: data/prepared_data/ hash: md5 - md5: 3d1144848fce4ce50f6abfaec5235552.dir - size: 46392840 + md5: efa416abea618ae6220a0c3d597603cf.dir + size: 44750997 nfiles: 2 build_model: cmd: python 2_build_model.py @@ -43,8 +61,8 @@ stages: size: 4820 - path: data/prepared_data hash: md5 - md5: 3d1144848fce4ce50f6abfaec5235552.dir - size: 46392840 + md5: efa416abea618ae6220a0c3d597603cf.dir + size: 44750997 nfiles: 2 params: configs/build_model.yaml: @@ -62,10 +80,9 @@ stages: problem_type: regression eval_metric: mean_squared_error time_limit: 1800 - presets: good_quality + presets: medium_quality excluded_model_types: - RF - - FASTAI - CAT - NN_TORCH - KNN @@ -77,18 +94,18 @@ stages: outs: - path: data/fit_predictions/ hash: md5 - md5: 346b6611afbf2070e038bf945249a86e.dir - size: 3384302 + md5: de46250d454c4d713ab580b10ff3fd31.dir + size: 3349318 nfiles: 1 - path: data/model/ hash: md5 - md5: 8e37f21728cd092660bafa8c32dc109f.dir - size: 423840922 - nfiles: 118 + md5: 18bd7a93ece75a65d3a950b7dfdab4fb.dir + size: 735951861 + nfiles: 35 - path: metrics/fit_metrics.json hash: md5 - md5: d63e1a8d31503055835ac35149554e41 - size: 223 + md5: 8a952a5e884c268e6059357a627b9251 + size: 224 generate_predictions: cmd: python 3_generate_predictions.py deps: @@ -98,13 +115,13 @@ stages: size: 2464 - path: data/model hash: md5 - md5: 8e37f21728cd092660bafa8c32dc109f.dir - size: 423840922 - nfiles: 118 + md5: 18bd7a93ece75a65d3a950b7dfdab4fb.dir + size: 735951861 + nfiles: 35 - path: data/prepared_data hash: md5 - md5: 3d1144848fce4ce50f6abfaec5235552.dir - size: 46392840 + md5: efa416abea618ae6220a0c3d597603cf.dir + size: 44750997 nfiles: 2 params: configs/settings.yaml: @@ -116,8 +133,8 @@ stages: outs: - path: data/predictions/ hash: md5 - md5: d148baf508140353d62c16d6ab0fb6b7.dir - size: 469224 + md5: 07ef721a0dc94a52e3ba7a70ac45b8ff.dir + size: 463563 nfiles: 1 generate_metrics: cmd: python 4_generate_metrics.py @@ -128,13 +145,13 @@ stages: size: 3484 - path: data/predictions hash: md5 - md5: d148baf508140353d62c16d6ab0fb6b7.dir - size: 469224 + md5: 07ef721a0dc94a52e3ba7a70ac45b8ff.dir + size: 463563 nfiles: 1 - path: data/prepared_data hash: md5 - md5: 3d1144848fce4ce50f6abfaec5235552.dir - size: 46392840 + md5: efa416abea618ae6220a0c3d597603cf.dir + size: 44750997 nfiles: 2 params: configs/settings.yaml: @@ -144,16 +161,25 @@ stages: outs: - path: metrics/metrics.json hash: md5 - md5: 196232f94b563ac525cf65ee5cc6d639 - size: 222 - startup_cleanup: - cmd: python 0_startup_cleanup.py + md5: 9f863f47799d42c101eba3b03a179455 + size: 224 + generate_scenerio_metrics: + cmd: python 5_generate_scenarios.py deps: - - path: 0_startup_cleanup.py + - path: 5_generate_scenarios.py hash: md5 - md5: b1b12f6b6393fbf8b83d23684df0a3d4 - size: 1220 + md5: 30f80ffeb6ee50c5f7b82943a4dc7702 + size: 4014 params: - configs/settings.yaml: - default.startup_cleanup.artefacts: ./data - default.startup_cleanup.metrics: ./metrics + configs/scenarios.yaml: + default.scenarios: + input_dataclient_type: aws-s3 + output_dataclient_type: local + scenario_data_filepaths: + - s3://retrofit-data-dev/scenario_data/24-03-2024-20-23-25/recommendations_scoring_data.parquet + output_filepath: ./metrics/scenario_table.md + outs: + - path: metrics/scenario_table.md + hash: md5 + md5: 54856c66fca8b2ebd1fa4dea2d25734a + size: 2133 diff --git a/modules/ml-pipeline/src/pipeline/dvc.yaml b/modules/ml-pipeline/src/pipeline/dvc.yaml index 58889cc..5ce35ce 100644 --- a/modules/ml-pipeline/src/pipeline/dvc.yaml +++ b/modules/ml-pipeline/src/pipeline/dvc.yaml @@ -71,6 +71,16 @@ stages: outs: - metrics/metrics.json always_changed: true + generate_scenerio_metrics: + cmd: python 5_generate_scenarios.py + deps: + - 5_generate_scenarios.py + params: + - configs/scenarios.yaml: + - default.scenarios + outs: + - metrics/scenario_table.md + always_changed: true metrics: - metrics/metrics.json - metrics/fit_metrics.json diff --git a/modules/ml-pipeline/src/pipeline/metrics/.gitignore b/modules/ml-pipeline/src/pipeline/metrics/.gitignore index e6fbc8d..189c2ee 100644 --- a/modules/ml-pipeline/src/pipeline/metrics/.gitignore +++ b/modules/ml-pipeline/src/pipeline/metrics/.gitignore @@ -1,2 +1,3 @@ /fit_metrics.json /metrics.json +/scenario_table.md From 9b6aeae0da24418aca6eb38a3f71731fddac0f1e Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 26 Mar 2024 22:32:44 +0000 Subject: [PATCH 09/11] medium model with scenario and upgraded autogluon --- modules/ml-pipeline/src/pipeline/dvc.lock | 84 ----------------------- 1 file changed, 84 deletions(-) diff --git a/modules/ml-pipeline/src/pipeline/dvc.lock b/modules/ml-pipeline/src/pipeline/dvc.lock index b3c5814..d6bce15 100644 --- a/modules/ml-pipeline/src/pipeline/dvc.lock +++ b/modules/ml-pipeline/src/pipeline/dvc.lock @@ -85,21 +85,12 @@ stages: default.prepare_data.output_dataclient_type: local default.prepare_data.output_test_filepath: ./data/prepared_data/test.parquet default.prepare_data.output_train_filepath: ./data/prepared_data/train.parquet -<<<<<<< HEAD default.prepare_data.train_proportion: 0.9 outs: - path: data/prepared_data/ hash: md5 md5: efa416abea618ae6220a0c3d597603cf.dir size: 44750997 -======= - default.prepare_data.train_proportion: 0.98 - outs: - - path: data/prepared_data/ - hash: md5 - md5: 544427230544c2cc526334e246db4845.dir - size: 26132493 ->>>>>>> d5f40a8eb294924e0525904d6ee1864999d77c23 nfiles: 2 build_model: cmd: python 2_build_model.py @@ -110,13 +101,8 @@ stages: size: 4820 - path: data/prepared_data hash: md5 -<<<<<<< HEAD md5: efa416abea618ae6220a0c3d597603cf.dir size: 44750997 -======= - md5: 544427230544c2cc526334e246db4845.dir - size: 26132493 ->>>>>>> d5f40a8eb294924e0525904d6ee1864999d77c23 nfiles: 2 params: configs/build_model.yaml: @@ -148,7 +134,6 @@ stages: outs: - path: data/fit_predictions/ hash: md5 -<<<<<<< HEAD md5: de46250d454c4d713ab580b10ff3fd31.dir size: 3349318 nfiles: 1 @@ -161,20 +146,6 @@ stages: hash: md5 md5: 8a952a5e884c268e6059357a627b9251 size: 224 -======= - md5: 8f9e2059782dd55d3ecdad54b4551f6a.dir - size: 3630849 - nfiles: 1 - - path: data/model/ - hash: md5 - md5: e031eb3c3fdb63917aabfea745b56ac6.dir - size: 618445494 - nfiles: 31 - - path: metrics/fit_metrics.json - hash: md5 - md5: e68009f5b66230b3ee4cd2ffc9a2d697 - size: 222 ->>>>>>> d5f40a8eb294924e0525904d6ee1864999d77c23 generate_predictions: cmd: python 3_generate_predictions.py deps: @@ -184,7 +155,6 @@ stages: size: 2464 - path: data/model hash: md5 -<<<<<<< HEAD md5: 18bd7a93ece75a65d3a950b7dfdab4fb.dir size: 735951861 nfiles: 35 @@ -192,15 +162,6 @@ stages: hash: md5 md5: efa416abea618ae6220a0c3d597603cf.dir size: 44750997 -======= - md5: e031eb3c3fdb63917aabfea745b56ac6.dir - size: 618445494 - nfiles: 31 - - path: data/prepared_data - hash: md5 - md5: 544427230544c2cc526334e246db4845.dir - size: 26132493 ->>>>>>> d5f40a8eb294924e0525904d6ee1864999d77c23 nfiles: 2 params: configs/settings.yaml: @@ -212,13 +173,8 @@ stages: outs: - path: data/predictions/ hash: md5 -<<<<<<< HEAD md5: 07ef721a0dc94a52e3ba7a70ac45b8ff.dir size: 463563 -======= - md5: 1c14c9ac9711f5d33a60890e3ca72454.dir - size: 90361 ->>>>>>> d5f40a8eb294924e0525904d6ee1864999d77c23 nfiles: 1 generate_metrics: cmd: python 4_generate_metrics.py @@ -229,7 +185,6 @@ stages: size: 3484 - path: data/predictions hash: md5 -<<<<<<< HEAD md5: 07ef721a0dc94a52e3ba7a70ac45b8ff.dir size: 463563 nfiles: 1 @@ -237,15 +192,6 @@ stages: hash: md5 md5: efa416abea618ae6220a0c3d597603cf.dir size: 44750997 -======= - md5: 1c14c9ac9711f5d33a60890e3ca72454.dir - size: 90361 - nfiles: 1 - - path: data/prepared_data - hash: md5 - md5: 544427230544c2cc526334e246db4845.dir - size: 26132493 ->>>>>>> d5f40a8eb294924e0525904d6ee1864999d77c23 nfiles: 2 params: configs/settings.yaml: @@ -255,55 +201,25 @@ stages: outs: - path: metrics/metrics.json hash: md5 -<<<<<<< HEAD md5: 9f863f47799d42c101eba3b03a179455 size: 224 generate_scenerio_metrics: cmd: python 5_generate_scenarios.py -======= - md5: 98e59ea9569522a8665c4e6c1bea7473 - size: 222 - startup_cleanup: - cmd: python 0_startup_cleanup.py ->>>>>>> d5f40a8eb294924e0525904d6ee1864999d77c23 deps: - path: 5_generate_scenarios.py hash: md5 md5: 30f80ffeb6ee50c5f7b82943a4dc7702 size: 4014 params: -<<<<<<< HEAD -======= - configs/settings.yaml: - default.startup_cleanup.artefacts: ./data - default.startup_cleanup.metrics: ./metrics - generate_scenerio_metrics: - cmd: python 5_generate_scenarios.py - deps: - - path: 5_generate_scenarios.py - hash: md5 - md5: 30f80ffeb6ee50c5f7b82943a4dc7702 - size: 4014 - params: ->>>>>>> d5f40a8eb294924e0525904d6ee1864999d77c23 configs/scenarios.yaml: default.scenarios: input_dataclient_type: aws-s3 output_dataclient_type: local scenario_data_filepaths: -<<<<<<< HEAD - s3://retrofit-data-dev/scenario_data/24-03-2024-20-23-25/recommendations_scoring_data.parquet -======= - - s3://retrofit-data-dev/scenario_data/recommendations_scoring_data.parquet ->>>>>>> d5f40a8eb294924e0525904d6ee1864999d77c23 output_filepath: ./metrics/scenario_table.md outs: - path: metrics/scenario_table.md hash: md5 -<<<<<<< HEAD md5: 54856c66fca8b2ebd1fa4dea2d25734a size: 2133 -======= - md5: 3ee1966a06c1e5b9c37797597be94797 - size: 1648 ->>>>>>> d5f40a8eb294924e0525904d6ee1864999d77c23 From c3985e2104d9acfa112ad4b0247a47755c552e97 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Wed, 27 Mar 2024 12:22:58 +0000 Subject: [PATCH 10/11] add metrics for scenarios --- .github/workflows/MLPipelinePullRequest.yml | 6 +++- .../src/pipeline/5_generate_scenarios.py | 33 ++++++++++++++++--- .../src/pipeline/configs/scenarios.yaml | 6 ++-- modules/ml-pipeline/src/pipeline/dvc.lock | 15 ++++++--- modules/ml-pipeline/src/pipeline/dvc.yaml | 1 + .../src/pipeline/metrics/.gitignore | 1 + 6 files changed, 50 insertions(+), 12 deletions(-) diff --git a/.github/workflows/MLPipelinePullRequest.yml b/.github/workflows/MLPipelinePullRequest.yml index 493aef9..8e59cc8 100644 --- a/.github/workflows/MLPipelinePullRequest.yml +++ b/.github/workflows/MLPipelinePullRequest.yml @@ -98,10 +98,14 @@ jobs: git fetch --depth=1 origin ${TARGET_BRANCH}:${TARGET_BRANCH} dvc metrics diff --md --all ${TARGET_BRANCH} >> report.md - echo "## Scenario metrics" >> report.md + echo "## Scenario comparison" >> report.md cat metrics/scenario_table.md >> report.md + echo "## Scenario metrics" >> report.md + + cat metrics/scenario_metrics.md >> report.md + cml comment create report.md # echo "## Residuals plot from model" >> report.md diff --git a/modules/ml-pipeline/src/pipeline/5_generate_scenarios.py b/modules/ml-pipeline/src/pipeline/5_generate_scenarios.py index 28bcb9d..9d2fa68 100644 --- a/modules/ml-pipeline/src/pipeline/5_generate_scenarios.py +++ b/modules/ml-pipeline/src/pipeline/5_generate_scenarios.py @@ -8,9 +8,11 @@ import os import pandas as pd from core.interface.InterfaceModels import MLModel from core.interface.InterfaceDataClient import DataClient +from core.interface.InterfaceMetrics import MLMetrics from configs.post_prediction_logic import post_prediction_logic from core.DataClient import dataclient_factory from core.MLModels import model_factory +from core.MLMetrics import metrics_factory from core.Logger import logger from config import settings @@ -30,7 +32,8 @@ model_filepath = build_model_params["model_save_filepath"] target = feature_process_params["feature_processor_config"]["target"] scenario_data_filepaths = scenarios_params["scenario_data_filepaths"] predictions_column_name = generate_predictions_params["predictions_column_name"] -output_filepath = scenarios_params["output_filepath"] +comparison_output_filepath = scenarios_params["comparison_output_filepath"] +metrics_output_filepath = scenarios_params["metrics_output_filepath"] logger.info(f"--- Initiate MLModel ---") @@ -51,15 +54,21 @@ output_dataclient = dataclient_factory( dataclient_config=client_params[output_dataclient_type], ) +logger.info(f"--- Initiate MLMetrics ---") + +metrics = metrics_factory(generate_metrics_params["metrics_type"]) + def generate_scenario_predictions( input_dataclient: DataClient, output_dataclient: DataClient, model: MLModel, + metrics: MLMetrics, model_filepath: str, scenario_data_filepaths: list, predictions_column_name: str, - output_filepath: str, + comparison_output_filepath: str, + metrics_output_filepath: str, ): """ Given the new model, we generate prediction for expected scenarios @@ -98,16 +107,30 @@ def generate_scenario_predictions( scenario_data[predictions_column_name] - scenario_data["sap_starting"] ) + logger.info("--- Generate Metrics ---") + + metrics_dict = metrics.generate_metrics( + scenario_data["impact"], scenario_data["predicted_impact"] + ) + + metrics_df = pd.DataFrame(metrics_dict, index=[0]).T.reset_index() + metrics_df.columns = ["metric", "value"] + logger.info("--- Save prediction into metrics ---") output_df = scenario_data[["uprn", "id", "impact", "predicted_impact"]] output_dataclient.save_data( - obj=output_df, location=output_filepath, save_config=None + obj=output_df, location=comparison_output_filepath, save_config=None + ) + + output_dataclient.save_data( + obj=metrics_df, location=metrics_output_filepath, save_config=None ) if __name__ == "__main__": + logger.info(f"--- {__file__} - Start! ---") logger.info(f"--- Generate Scenario Predictions ---") @@ -116,10 +139,12 @@ if __name__ == "__main__": input_dataclient=input_dataclient, output_dataclient=output_dataclient, model=model, + metrics=metrics, model_filepath=model_filepath, scenario_data_filepaths=scenario_data_filepaths, predictions_column_name=predictions_column_name, - output_filepath=output_filepath, + comparison_output_filepath=comparison_output_filepath, + metrics_output_filepath=metrics_output_filepath, ) logger.info(f"--- {__file__} - Complete! ---") diff --git a/modules/ml-pipeline/src/pipeline/configs/scenarios.yaml b/modules/ml-pipeline/src/pipeline/configs/scenarios.yaml index e76336a..2df0cb6 100644 --- a/modules/ml-pipeline/src/pipeline/configs/scenarios.yaml +++ b/modules/ml-pipeline/src/pipeline/configs/scenarios.yaml @@ -4,5 +4,7 @@ default: output_dataclient_type: local scenario_data_filepaths: # - s3://retrofit-data-dev/scenario_data/22-03-2024-19-20-09/recommendations_scoring_data.parquet - - s3://retrofit-data-dev/scenario_data/24-03-2024-20-23-25/recommendations_scoring_data.parquet - output_filepath: ./metrics/scenario_table.md + # - s3://retrofit-data-dev/scenario_data/24-03-2024-20-23-25/recommendations_scoring_data.parquet + - s3://retrofit-data-dev/scenario_data/27-03-2024-11-38-15/recommendations_scoring_data.parquet + comparison_output_filepath: ./metrics/scenario_table.md + metrics_output_filepath: ./metrics/scenario_metrics.md diff --git a/modules/ml-pipeline/src/pipeline/dvc.lock b/modules/ml-pipeline/src/pipeline/dvc.lock index d6bce15..104dc83 100644 --- a/modules/ml-pipeline/src/pipeline/dvc.lock +++ b/modules/ml-pipeline/src/pipeline/dvc.lock @@ -208,18 +208,23 @@ stages: deps: - path: 5_generate_scenarios.py hash: md5 - md5: 30f80ffeb6ee50c5f7b82943a4dc7702 - size: 4014 + md5: a18f6c6ae2082f038df47386cf3e418e + size: 4896 params: configs/scenarios.yaml: default.scenarios: input_dataclient_type: aws-s3 output_dataclient_type: local scenario_data_filepaths: - - s3://retrofit-data-dev/scenario_data/24-03-2024-20-23-25/recommendations_scoring_data.parquet - output_filepath: ./metrics/scenario_table.md + - s3://retrofit-data-dev/scenario_data/27-03-2024-11-38-15/recommendations_scoring_data.parquet + comparison_output_filepath: ./metrics/scenario_table.md + metrics_output_filepath: ./metrics/scenario_metrics.md outs: + - path: metrics/scenario_metrics.md + hash: md5 + md5: 64e7db945ff655ae03c20c9845f19106 + size: 363 - path: metrics/scenario_table.md hash: md5 - md5: 54856c66fca8b2ebd1fa4dea2d25734a + md5: d4f8afe07b774374aeaa48f1b7b8a5fc size: 2133 diff --git a/modules/ml-pipeline/src/pipeline/dvc.yaml b/modules/ml-pipeline/src/pipeline/dvc.yaml index 5ce35ce..6026a83 100644 --- a/modules/ml-pipeline/src/pipeline/dvc.yaml +++ b/modules/ml-pipeline/src/pipeline/dvc.yaml @@ -80,6 +80,7 @@ stages: - default.scenarios outs: - metrics/scenario_table.md + - metrics/scenario_metrics.md always_changed: true metrics: - metrics/metrics.json diff --git a/modules/ml-pipeline/src/pipeline/metrics/.gitignore b/modules/ml-pipeline/src/pipeline/metrics/.gitignore index 189c2ee..6427764 100644 --- a/modules/ml-pipeline/src/pipeline/metrics/.gitignore +++ b/modules/ml-pipeline/src/pipeline/metrics/.gitignore @@ -1,3 +1,4 @@ /fit_metrics.json /metrics.json /scenario_table.md +/scenario_metrics.md From 1bb1f8d61fb2f157290e014113589b69383798cd Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Wed, 27 Mar 2024 12:30:31 +0000 Subject: [PATCH 11/11] add metrics for scenarios --- .github/workflows/MLPipelinePullRequest.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/MLPipelinePullRequest.yml b/.github/workflows/MLPipelinePullRequest.yml index 8e59cc8..451b0a8 100644 --- a/.github/workflows/MLPipelinePullRequest.yml +++ b/.github/workflows/MLPipelinePullRequest.yml @@ -102,6 +102,8 @@ jobs: cat metrics/scenario_table.md >> report.md + echo "" >> report.md + echo "## Scenario metrics" >> report.md cat metrics/scenario_metrics.md >> report.md