changed format

.pre-commit-config.yaml

@@ -10,3 +10,6 @@ repos:
     rev: 22.10.0
     hooks:
     -   id: black
+-   repo: local
+    name: DVC - Push to experiment to remote location (experiments)
+    hooks: bash -c "ls"

modules/ml-pipeline/Makefile

@@ -11,7 +11,7 @@ dev-pyenv:
 	pyenv install ${PYTHON_VERSION} || echo "Python version already installed"
 	pyenv global ${PYTHON_VERSION}
 	python3 -m venv .dev_env
-	. .dev_env/bin/activate && pip install --upgrade pip && pip install -r src/pipeline/training/requirements/requirements-dev.txt && pre-commit install
+	. .dev_env/bin/activate && pip install --upgrade pip && pip install -r src/pipeline/src/requirements/training/requirements-dev.txt && pre-commit install
 	echo "TO ACTIVATE ENVIRONMENT, USE THE FOLLOWING COMMAND"
 	echo "source .dev_env/bin/activate"
 

modules/ml-pipeline/README.MD

@@ -19,3 +19,19 @@ To enable the virtual envrionemnt created in vscode:
 - Copy the path of the '.dev_env' folder into there.
 - When you select a kernel, clcik through create environment and refresh
 - The virutal environment should be there
+
+To use the docker environment for coding in VSCODE:
+- Open the "pipeline" folder
+- Open with remote container
+- Select the Dockerfile
+- Add the Git extension (for dvc)
+
+For running experiment, everything will be cached but the workflow will be:
+- `dvc repro` to regenerate the current experiement 
+- Change parameters if needed
+- Use `dvc exp run`
+- Cachec the results by using `dvc push -r REMOTE_NAME`
+- Repeat as needed
+- When happy with results, use `dvc exp apply EXPERIMENT_NAME`
+- Use `dvc pull` 
+- Commit code

modules/ml-pipeline/src/pipeline/Development.Dockerfile

@@ -1,6 +1,7 @@
-# Dockerfile that grabs the current dvc hashed model
+# Dockerfile that can be used for development
 FROM python:3.10.12-slim
 
+COPY ~/.ssh ~/.ssh
 COPY training/requirements/requirements-dev.txt requirements.txt
 
 RUN pip install --upgrade pip

modules/ml-pipeline/src/pipeline/inference/README.MD

@@ -1,5 +0,0 @@
-# Inference
-
-This folder contains the inference codebase to:
-- Load a model
-- Generate a prediction

modules/ml-pipeline/src/pipeline/main_training.py

@@ -1,3 +0,0 @@
-"""
-Pipeline that stitches all steps together
-"""