From 9491200e44ba7c6583894440ad9848f7cb836616 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Sat, 9 Sep 2023 09:52:37 +0000 Subject: [PATCH] add dvc and gto --- modules/ml-pipeline/.dvc/.gitignore | 3 +++ modules/ml-pipeline/.dvc/config | 0 modules/ml-pipeline/.dvcignore | 3 +++ modules/ml-pipeline/.gto | 1 + modules/ml-pipeline/Makefile | 20 ++++++++++++++++++++ modules/ml-pipeline/docs/README.md | 4 ++++ modules/ml-pipeline/tests/README.md | 3 +++ 7 files changed, 34 insertions(+) create mode 100644 modules/ml-pipeline/.dvc/.gitignore create mode 100644 modules/ml-pipeline/.dvc/config create mode 100644 modules/ml-pipeline/.dvcignore create mode 100644 modules/ml-pipeline/.gto create mode 100644 modules/ml-pipeline/Makefile create mode 100644 modules/ml-pipeline/docs/README.md create mode 100644 modules/ml-pipeline/tests/README.md diff --git a/modules/ml-pipeline/.dvc/.gitignore b/modules/ml-pipeline/.dvc/.gitignore new file mode 100644 index 0000000..528f30c --- /dev/null +++ b/modules/ml-pipeline/.dvc/.gitignore @@ -0,0 +1,3 @@ +/config.local +/tmp +/cache diff --git a/modules/ml-pipeline/.dvc/config b/modules/ml-pipeline/.dvc/config new file mode 100644 index 0000000..e69de29 diff --git a/modules/ml-pipeline/.dvcignore b/modules/ml-pipeline/.dvcignore new file mode 100644 index 0000000..5197305 --- /dev/null +++ b/modules/ml-pipeline/.dvcignore @@ -0,0 +1,3 @@ +# Add patterns of files dvc should ignore, which could improve +# the performance. Learn more at +# https://dvc.org/doc/user-guide/dvcignore diff --git a/modules/ml-pipeline/.gto b/modules/ml-pipeline/.gto new file mode 100644 index 0000000..339c197 --- /dev/null +++ b/modules/ml-pipeline/.gto @@ -0,0 +1 @@ +stages = ["dev", "staging", "prod"] diff --git a/modules/ml-pipeline/Makefile b/modules/ml-pipeline/Makefile new file mode 100644 index 0000000..576b954 --- /dev/null +++ b/modules/ml-pipeline/Makefile @@ -0,0 +1,20 @@ +export PYENV_ROOT=$(HOME)/.pyenv +export PATH := $(PYENV_ROOT)/bin:$(PATH) +PYTHON_VERSION ?= 3.10.12 + +.PHONY: init +init: dev-pyenv dvc-init + +.PHONY: dev-pyenv +dev-pyenv: + curl https://pyenv.run | bash || echo "Pyenv - Already installed" + pyenv install ${PYTHON_VERSION} || echo "Python version already installed" + pyenv global ${PYTHON_VERSION} + python3 -m venv .dev_env + . .dev_env/bin/activate && pip install --upgrade pip && pip install -r src/training/requirements/requirements-dev.txt && pre-commit install + echo "TO ACTIVATE ENVIRONMENT, USE THE FOLLOWING COMMAND" + echo "source .dev_env/bin/activate" + +.PHONY: dvc-init +dvc-init: + . .dev_env/bin/activate && dvc init --subdir diff --git a/modules/ml-pipeline/docs/README.md b/modules/ml-pipeline/docs/README.md new file mode 100644 index 0000000..cde8373 --- /dev/null +++ b/modules/ml-pipeline/docs/README.md @@ -0,0 +1,4 @@ +# Docs generation + +This will be done via a sphinx pre-commit hook +Only file to adjust is probably the conf.py file if needed diff --git a/modules/ml-pipeline/tests/README.md b/modules/ml-pipeline/tests/README.md new file mode 100644 index 0000000..4478b48 --- /dev/null +++ b/modules/ml-pipeline/tests/README.md @@ -0,0 +1,3 @@ +# Tests + +This is the testing folder for pipeline components and end to end testing