diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..84abbe6 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +modules/ml-pipeline/src/pipeline/data/predictions +modules/ml-pipeline/src/pipeline/data/fit_predictions +modules/ml-pipeline/src/pipeline/data/prepared_data +modules/ml-pipeline/src/pipeline/data/model/allmodels +modules/ml-pipeline/src/pipeline/metrics +modules/ml-pipeline/src/pipeline/__pycache__ +modules/ml-pipeline/src/pipeline/.dvc +modules/ml-pipeline/src/pipeline/analysis +modules/ml-pipeline/src/pipeline/metrics diff --git a/deployment/.dockerignore b/deployment/.dockerignore index e01cbd5..c4103de 100644 --- a/deployment/.dockerignore +++ b/deployment/.dockerignore @@ -1,4 +1,9 @@ -modules/ml-pipeline/src/pipeline/data/predictions* -modules/ml-pipeline/src/pipeline/data/prepared_data* -modules/ml-pipeline/src/pipeline/data/model/allmodels* -modules/ml-pipeline/src/pipeline/metrics* +modules/ml-pipeline/src/pipeline/data/predictions +modules/ml-pipeline/src/pipeline/data/fit_predictions +modules/ml-pipeline/src/pipeline/data/prepared_data +modules/ml-pipeline/src/pipeline/data/model/allmodels +modules/ml-pipeline/src/pipeline/metrics +modules/ml-pipeline/src/__pycache__ +modules/ml-pipeline/src/.dvc +modules/ml-pipeline/src/analysis +modules/ml-pipeline/src/metrics diff --git a/modules/ml-pipeline/src/.dockerignore b/modules/ml-pipeline/src/.dockerignore index 14f71d7..bf48a5e 100644 --- a/modules/ml-pipeline/src/.dockerignore +++ b/modules/ml-pipeline/src/.dockerignore @@ -1,4 +1,8 @@ -pipeline/data/predictions* -pipeline/data/prepared_data/train.parquet* -pipeline/data/model/allmodels* -pipeline/metrics* +pipeline/data/predictions +pipeline/data/fit_predictions +pipeline/data/prepared_data/train.parquet +pipeline/data/fit_predictions +pipeline/data/model/allmodels +pipeline/metrics +pipeline/.dvc +pipeline/analysis diff --git a/modules/ml-pipeline/src/Prediction.Dockerfile b/modules/ml-pipeline/src/Prediction.Dockerfile index a6fc539..e0a292c 100644 --- a/modules/ml-pipeline/src/Prediction.Dockerfile +++ b/modules/ml-pipeline/src/Prediction.Dockerfile @@ -1,7 +1,7 @@ # Dockerfile that can be used to test loading a model to generate a prediction (part of CI/CD flow) FROM python:3.10.12-slim -RUN apt-get update && apt-get install -y libgomp1 +RUN apt-get update && apt-get install -y libgomp1 gcc python3-dev COPY pipeline/requirements/predictions/requirements.txt requirements.txt diff --git a/modules/ml-pipeline/src/pipeline/5_generate_scenarios.py b/modules/ml-pipeline/src/pipeline/5_generate_scenarios.py index 9d2fa68..6debe32 100644 --- a/modules/ml-pipeline/src/pipeline/5_generate_scenarios.py +++ b/modules/ml-pipeline/src/pipeline/5_generate_scenarios.py @@ -78,6 +78,18 @@ def generate_scenario_predictions( scenario_data = pd.DataFrame() + # If we have no scenario data, we can save empty dataframes + if scenario_data_filepaths is None: + logger.info("No scenario data filepaths provided") + output_dataclient.save_data( + obj=scenario_data, location=comparison_output_filepath, save_config=None + ) + + output_dataclient.save_data( + obj=scenario_data, location=metrics_output_filepath, save_config=None + ) + return + # Can have multiple scenario data files for scenario_data_filepath in scenario_data_filepaths: scenario_data = pd.concat( diff --git a/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements-dev.txt b/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements-dev.txt index 734419a..4dc4c36 100644 --- a/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements-dev.txt +++ b/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements-dev.txt @@ -1,7 +1,7 @@ joblib==1.3.2 boto3==1.28.17 pandas==2.1.4 -autogluon==1.0.0 +autogluon.tabular[all]==1.0.0 dynaconf==3.2.1 pyarrow==13.0.0 pre-commit==3.3.3 diff --git a/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements.txt b/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements.txt index 937b000..35bdb05 100644 --- a/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements.txt +++ b/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements.txt @@ -1,7 +1,7 @@ joblib==1.3.2 boto3==1.28.17 pandas==2.1.4 -autogluon==1.0.0 +autogluon.tabular[all]==1.0.0 dynaconf==3.2.1 pyarrow==13.0.0 PyYAML==6.0.1 diff --git a/modules/ml-pipeline/src/pipeline/requirements/training/requirements-dev.txt b/modules/ml-pipeline/src/pipeline/requirements/training/requirements-dev.txt index fe06a4d..93a042e 100644 --- a/modules/ml-pipeline/src/pipeline/requirements/training/requirements-dev.txt +++ b/modules/ml-pipeline/src/pipeline/requirements/training/requirements-dev.txt @@ -1,7 +1,7 @@ joblib==1.3.2 boto3==1.28.17 pandas==2.1.4 -autogluon==1.0.0 +autogluon.tabular[all]==1.0.0 ray==2.6.3 dynaconf==3.2.1 alibi==0.9.5 diff --git a/modules/ml-pipeline/src/pipeline/requirements/training/requirements.txt b/modules/ml-pipeline/src/pipeline/requirements/training/requirements.txt index a5bccd3..edeb764 100644 --- a/modules/ml-pipeline/src/pipeline/requirements/training/requirements.txt +++ b/modules/ml-pipeline/src/pipeline/requirements/training/requirements.txt @@ -1,4 +1,4 @@ boto3==1.28.41 pandas==2.1.4 -autogluon==1.0.0 +autogluon.tabular[all]==1.0.0 dynaconf==3.2.1