diff --git a/.github/workflows/MLPipelinePullRequest.yml b/.github/workflows/MLPipelinePullRequest.yml index 588e239..186c382 100644 --- a/.github/workflows/MLPipelinePullRequest.yml +++ b/.github/workflows/MLPipelinePullRequest.yml @@ -23,8 +23,7 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} run: | pip install --upgrade pip - pip install dvc dvc_s3 - # pip install -r src/requirements/predictions/requirements-dev.txt + pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt - name: Retrieve artifacts (dvc.lock) env: @@ -50,13 +49,9 @@ jobs: steps: - uses: actions/checkout@v3 - name: Install packages to retrieve artifacts - env: - AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} run: | pip install --upgrade pip - pip install dvc dvc_s3 - # pip install -r src/requirements/predictions/requirements-dev.txt + pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt - name: Retrieve artifacts (dvc.lock) env: @@ -70,6 +65,8 @@ jobs: - uses: iterative/setup-cml@v1 - name: Generate report env: + AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | cd modules/ml-pipeline/src/pipeline/src diff --git a/modules/ml-pipeline/Makefile b/modules/ml-pipeline/Makefile index c9b0ddf..45a16f5 100644 --- a/modules/ml-pipeline/Makefile +++ b/modules/ml-pipeline/Makefile @@ -11,7 +11,7 @@ dev-pyenv: pyenv install ${PYTHON_VERSION} || echo "Python version already installed" pyenv global ${PYTHON_VERSION} python3 -m venv .dev_env - . .dev_env/bin/activate && pip install --upgrade pip && pip install -r src/pipeline/src/requirements/training/requirements-dev.txt && pre-commit install + . .dev_env/bin/activate && pip install --upgrade pip && pip install -r src/pipeline/src/requirements/training/requirements-dev.txt && pip install -r src/pipeline/src/requirements/version_control/requirements.txt && pre-commit install echo "TO ACTIVATE ENVIRONMENT, USE THE FOLLOWING COMMAND" echo "source .dev_env/bin/activate" diff --git a/modules/ml-pipeline/src/pipeline/src/requirements/predictions/requirements-dev.txt b/modules/ml-pipeline/src/pipeline/src/requirements/predictions/requirements-dev.txt index 28f6af4..5aac406 100644 --- a/modules/ml-pipeline/src/pipeline/src/requirements/predictions/requirements-dev.txt +++ b/modules/ml-pipeline/src/pipeline/src/requirements/predictions/requirements-dev.txt @@ -1,9 +1,6 @@ joblib==1.3.2 boto3==1.28.17 pandas==1.5.3 -dvc==3.18.0 -dvc-s3==2.23.0 -gto==1.0.4 scikit-learn==1.3.0 pyarrow==13.0.0 pre-commit==3.3.3 diff --git a/modules/ml-pipeline/src/pipeline/src/requirements/training/requirements-dev.txt b/modules/ml-pipeline/src/pipeline/src/requirements/training/requirements-dev.txt index 28f6af4..5aac406 100644 --- a/modules/ml-pipeline/src/pipeline/src/requirements/training/requirements-dev.txt +++ b/modules/ml-pipeline/src/pipeline/src/requirements/training/requirements-dev.txt @@ -1,9 +1,6 @@ joblib==1.3.2 boto3==1.28.17 pandas==1.5.3 -dvc==3.18.0 -dvc-s3==2.23.0 -gto==1.0.4 scikit-learn==1.3.0 pyarrow==13.0.0 pre-commit==3.3.3 diff --git a/modules/ml-pipeline/src/pipeline/src/requirements/training/requirements.txt b/modules/ml-pipeline/src/pipeline/src/requirements/training/requirements.txt index 8c146e8..196dfe7 100644 --- a/modules/ml-pipeline/src/pipeline/src/requirements/training/requirements.txt +++ b/modules/ml-pipeline/src/pipeline/src/requirements/training/requirements.txt @@ -1,5 +1,3 @@ boto3==1.28.41 pandas==1.5.3 -dvc==3.18.0 -gto==1.0.4 scikit-learn==1.3.0 diff --git a/modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt b/modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt new file mode 100644 index 0000000..8459d38 --- /dev/null +++ b/modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt @@ -0,0 +1,3 @@ +dvc==3.18.0 +dvc-s3==2.23.0 +gto==1.0.4