diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 34a9b78..f602a99 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,3 +10,6 @@ repos: rev: 22.10.0 hooks: - id: black +- repo: local + name: DVC - Push to experiment to remote location (experiments) + hooks: bash -c "ls" diff --git a/modules/ml-pipeline/Makefile b/modules/ml-pipeline/Makefile index fa3411e..c9b0ddf 100644 --- a/modules/ml-pipeline/Makefile +++ b/modules/ml-pipeline/Makefile @@ -11,7 +11,7 @@ dev-pyenv: pyenv install ${PYTHON_VERSION} || echo "Python version already installed" pyenv global ${PYTHON_VERSION} python3 -m venv .dev_env - . .dev_env/bin/activate && pip install --upgrade pip && pip install -r src/pipeline/training/requirements/requirements-dev.txt && pre-commit install + . .dev_env/bin/activate && pip install --upgrade pip && pip install -r src/pipeline/src/requirements/training/requirements-dev.txt && pre-commit install echo "TO ACTIVATE ENVIRONMENT, USE THE FOLLOWING COMMAND" echo "source .dev_env/bin/activate" diff --git a/modules/ml-pipeline/README.MD b/modules/ml-pipeline/README.MD index cf9316a..7dae279 100644 --- a/modules/ml-pipeline/README.MD +++ b/modules/ml-pipeline/README.MD @@ -19,3 +19,19 @@ To enable the virtual envrionemnt created in vscode: - Copy the path of the '.dev_env' folder into there. - When you select a kernel, clcik through create environment and refresh - The virutal environment should be there + +To use the docker environment for coding in VSCODE: +- Open the "pipeline" folder +- Open with remote container +- Select the Dockerfile +- Add the Git extension (for dvc) + +For running experiment, everything will be cached but the workflow will be: +- `dvc repro` to regenerate the current experiement +- Change parameters if needed +- Use `dvc exp run` +- Cachec the results by using `dvc push -r REMOTE_NAME` +- Repeat as needed +- When happy with results, use `dvc exp apply EXPERIMENT_NAME` +- Use `dvc pull` +- Commit code \ No newline at end of file diff --git a/modules/ml-pipeline/src/pipeline/Dockerfile b/modules/ml-pipeline/src/pipeline/Development.Dockerfile similarity index 71% rename from modules/ml-pipeline/src/pipeline/Dockerfile rename to modules/ml-pipeline/src/pipeline/Development.Dockerfile index 224be6b..043d516 100644 --- a/modules/ml-pipeline/src/pipeline/Dockerfile +++ b/modules/ml-pipeline/src/pipeline/Development.Dockerfile @@ -1,6 +1,7 @@ -# Dockerfile that grabs the current dvc hashed model +# Dockerfile that can be used for development FROM python:3.10.12-slim +COPY ~/.ssh ~/.ssh COPY training/requirements/requirements-dev.txt requirements.txt RUN pip install --upgrade pip diff --git a/modules/ml-pipeline/src/pipeline/inference/README.MD b/modules/ml-pipeline/src/pipeline/inference/README.MD deleted file mode 100644 index ab8d00c..0000000 --- a/modules/ml-pipeline/src/pipeline/inference/README.MD +++ /dev/null @@ -1,5 +0,0 @@ -# Inference - -This folder contains the inference codebase to: -- Load a model -- Generate a prediction diff --git a/modules/ml-pipeline/src/pipeline/main_training.py b/modules/ml-pipeline/src/pipeline/main_training.py deleted file mode 100644 index dada39d..0000000 --- a/modules/ml-pipeline/src/pipeline/main_training.py +++ /dev/null @@ -1,3 +0,0 @@ -""" -Pipeline that stitches all steps together -""" diff --git a/modules/ml-pipeline/src/pipeline/training/.dvc/.gitignore b/modules/ml-pipeline/src/pipeline/src/.dvc/.gitignore similarity index 100% rename from modules/ml-pipeline/src/pipeline/training/.dvc/.gitignore rename to modules/ml-pipeline/src/pipeline/src/.dvc/.gitignore diff --git a/modules/ml-pipeline/src/pipeline/training/.dvc/config b/modules/ml-pipeline/src/pipeline/src/.dvc/config similarity index 100% rename from modules/ml-pipeline/src/pipeline/training/.dvc/config rename to modules/ml-pipeline/src/pipeline/src/.dvc/config diff --git a/modules/ml-pipeline/src/pipeline/training/.dvcignore b/modules/ml-pipeline/src/pipeline/src/.dvcignore similarity index 100% rename from modules/ml-pipeline/src/pipeline/training/.dvcignore rename to modules/ml-pipeline/src/pipeline/src/.dvcignore diff --git a/modules/ml-pipeline/src/pipeline/training/README.md b/modules/ml-pipeline/src/pipeline/src/README.md similarity index 100% rename from modules/ml-pipeline/src/pipeline/training/README.md rename to modules/ml-pipeline/src/pipeline/src/README.md diff --git a/modules/ml-pipeline/src/pipeline/training/__init__.py b/modules/ml-pipeline/src/pipeline/src/__init__.py similarity index 100% rename from modules/ml-pipeline/src/pipeline/training/__init__.py rename to modules/ml-pipeline/src/pipeline/src/__init__.py diff --git a/modules/ml-pipeline/src/pipeline/training/build_model.py b/modules/ml-pipeline/src/pipeline/src/build_model.py similarity index 100% rename from modules/ml-pipeline/src/pipeline/training/build_model.py rename to modules/ml-pipeline/src/pipeline/src/build_model.py diff --git a/modules/ml-pipeline/src/pipeline/training/configs/build_model.yaml b/modules/ml-pipeline/src/pipeline/src/configs/build_model.yaml similarity index 100% rename from modules/ml-pipeline/src/pipeline/training/configs/build_model.yaml rename to modules/ml-pipeline/src/pipeline/src/configs/build_model.yaml diff --git a/modules/ml-pipeline/src/pipeline/training/configs/configs.py b/modules/ml-pipeline/src/pipeline/src/configs/configs.py similarity index 100% rename from modules/ml-pipeline/src/pipeline/training/configs/configs.py rename to modules/ml-pipeline/src/pipeline/src/configs/configs.py diff --git a/modules/ml-pipeline/src/pipeline/training/configs/generate_metrics.yaml b/modules/ml-pipeline/src/pipeline/src/configs/generate_metrics.yaml similarity index 100% rename from modules/ml-pipeline/src/pipeline/training/configs/generate_metrics.yaml rename to modules/ml-pipeline/src/pipeline/src/configs/generate_metrics.yaml diff --git a/modules/ml-pipeline/src/pipeline/training/configs/generate_predictions.yaml b/modules/ml-pipeline/src/pipeline/src/configs/generate_predictions.yaml similarity index 100% rename from modules/ml-pipeline/src/pipeline/training/configs/generate_predictions.yaml rename to modules/ml-pipeline/src/pipeline/src/configs/generate_predictions.yaml diff --git a/modules/ml-pipeline/src/pipeline/training/configs/prepare_data.yaml b/modules/ml-pipeline/src/pipeline/src/configs/prepare_data.yaml similarity index 100% rename from modules/ml-pipeline/src/pipeline/training/configs/prepare_data.yaml rename to modules/ml-pipeline/src/pipeline/src/configs/prepare_data.yaml diff --git a/modules/ml-pipeline/src/pipeline/training/core/DataClient.py b/modules/ml-pipeline/src/pipeline/src/core/DataClient.py similarity index 100% rename from modules/ml-pipeline/src/pipeline/training/core/DataClient.py rename to modules/ml-pipeline/src/pipeline/src/core/DataClient.py diff --git a/modules/ml-pipeline/src/pipeline/training/core/Logger.py b/modules/ml-pipeline/src/pipeline/src/core/Logger.py similarity index 100% rename from modules/ml-pipeline/src/pipeline/training/core/Logger.py rename to modules/ml-pipeline/src/pipeline/src/core/Logger.py diff --git a/modules/ml-pipeline/src/pipeline/training/core/MLMetrics.py b/modules/ml-pipeline/src/pipeline/src/core/MLMetrics.py similarity index 100% rename from modules/ml-pipeline/src/pipeline/training/core/MLMetrics.py rename to modules/ml-pipeline/src/pipeline/src/core/MLMetrics.py diff --git a/modules/ml-pipeline/src/pipeline/training/core/MLModels.py b/modules/ml-pipeline/src/pipeline/src/core/MLModels.py similarity index 100% rename from modules/ml-pipeline/src/pipeline/training/core/MLModels.py rename to modules/ml-pipeline/src/pipeline/src/core/MLModels.py diff --git a/modules/ml-pipeline/src/pipeline/training/core/__init__.py b/modules/ml-pipeline/src/pipeline/src/core/__init__.py similarity index 100% rename from modules/ml-pipeline/src/pipeline/training/core/__init__.py rename to modules/ml-pipeline/src/pipeline/src/core/__init__.py diff --git a/modules/ml-pipeline/src/pipeline/training/core/interface/InterfaceDataClient.py b/modules/ml-pipeline/src/pipeline/src/core/interface/InterfaceDataClient.py similarity index 100% rename from modules/ml-pipeline/src/pipeline/training/core/interface/InterfaceDataClient.py rename to modules/ml-pipeline/src/pipeline/src/core/interface/InterfaceDataClient.py diff --git a/modules/ml-pipeline/src/pipeline/training/core/interface/InterfaceMetrics.py b/modules/ml-pipeline/src/pipeline/src/core/interface/InterfaceMetrics.py similarity index 100% rename from modules/ml-pipeline/src/pipeline/training/core/interface/InterfaceMetrics.py rename to modules/ml-pipeline/src/pipeline/src/core/interface/InterfaceMetrics.py diff --git a/modules/ml-pipeline/src/pipeline/training/core/interface/InterfaceModels.py b/modules/ml-pipeline/src/pipeline/src/core/interface/InterfaceModels.py similarity index 100% rename from modules/ml-pipeline/src/pipeline/training/core/interface/InterfaceModels.py rename to modules/ml-pipeline/src/pipeline/src/core/interface/InterfaceModels.py diff --git a/modules/ml-pipeline/src/pipeline/training/core/interface/__init__.py b/modules/ml-pipeline/src/pipeline/src/core/interface/__init__.py similarity index 100% rename from modules/ml-pipeline/src/pipeline/training/core/interface/__init__.py rename to modules/ml-pipeline/src/pipeline/src/core/interface/__init__.py diff --git a/modules/ml-pipeline/src/pipeline/training/data/.gitignore b/modules/ml-pipeline/src/pipeline/src/data/.gitignore similarity index 100% rename from modules/ml-pipeline/src/pipeline/training/data/.gitignore rename to modules/ml-pipeline/src/pipeline/src/data/.gitignore diff --git a/modules/ml-pipeline/src/pipeline/training/dvc.lock b/modules/ml-pipeline/src/pipeline/src/dvc.lock similarity index 100% rename from modules/ml-pipeline/src/pipeline/training/dvc.lock rename to modules/ml-pipeline/src/pipeline/src/dvc.lock diff --git a/modules/ml-pipeline/src/pipeline/training/dvc.yaml b/modules/ml-pipeline/src/pipeline/src/dvc.yaml similarity index 100% rename from modules/ml-pipeline/src/pipeline/training/dvc.yaml rename to modules/ml-pipeline/src/pipeline/src/dvc.yaml diff --git a/modules/ml-pipeline/src/pipeline/training/generate_metrics.py b/modules/ml-pipeline/src/pipeline/src/generate_metrics.py similarity index 100% rename from modules/ml-pipeline/src/pipeline/training/generate_metrics.py rename to modules/ml-pipeline/src/pipeline/src/generate_metrics.py diff --git a/modules/ml-pipeline/src/pipeline/training/generate_predictions.py b/modules/ml-pipeline/src/pipeline/src/generate_predictions.py similarity index 100% rename from modules/ml-pipeline/src/pipeline/training/generate_predictions.py rename to modules/ml-pipeline/src/pipeline/src/generate_predictions.py diff --git a/modules/ml-pipeline/src/pipeline/training/metrics/.gitignore b/modules/ml-pipeline/src/pipeline/src/metrics/.gitignore similarity index 100% rename from modules/ml-pipeline/src/pipeline/training/metrics/.gitignore rename to modules/ml-pipeline/src/pipeline/src/metrics/.gitignore diff --git a/modules/ml-pipeline/src/pipeline/training/prepare_data.py b/modules/ml-pipeline/src/pipeline/src/prepare_data.py similarity index 100% rename from modules/ml-pipeline/src/pipeline/training/prepare_data.py rename to modules/ml-pipeline/src/pipeline/src/prepare_data.py diff --git a/modules/ml-pipeline/src/pipeline/training/requirements/requirements-dev.txt b/modules/ml-pipeline/src/pipeline/src/requirements/training/requirements-dev.txt similarity index 100% rename from modules/ml-pipeline/src/pipeline/training/requirements/requirements-dev.txt rename to modules/ml-pipeline/src/pipeline/src/requirements/training/requirements-dev.txt diff --git a/modules/ml-pipeline/src/pipeline/training/requirements/requirements.txt b/modules/ml-pipeline/src/pipeline/src/requirements/training/requirements.txt similarity index 100% rename from modules/ml-pipeline/src/pipeline/training/requirements/requirements.txt rename to modules/ml-pipeline/src/pipeline/src/requirements/training/requirements.txt