diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index b3879e8..6ea18c5 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -42,36 +42,36 @@ jobs: run: | echo "Hello Patch" - # Promote-Model-To-Dev: - # if: github.event.pull_request.merged == true - # runs-on: ubuntu-latest + Promote-Model-To-Dev: + if: github.event.pull_request.merged == true + runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v3 - # - name: Install packages to retrieve artifacts - # run: | - # pip install --upgrade pip - # pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt + steps: + - uses: actions/checkout@v3 + - name: Install packages to retrieve artifacts + run: | + pip install --upgrade pip + pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt - # - name: Retrieve artifacts (dvc.lock) - # env: - # AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} - # run: | - # cd modules/ml-pipeline/src/pipeline/src - # dvc pull -r experiments + - name: Retrieve artifacts (dvc.lock) + env: + AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} + run: | + cd modules/ml-pipeline/src/pipeline/src + dvc pull -r experiments - # - name: Push artifacts to Dev - # env: - # AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} - # run: | - # cd modules/ml-pipeline/src/pipeline/src - # dvc push -r dev + - name: Push artifacts to Dev + env: + AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} + run: | + cd modules/ml-pipeline/src/pipeline/src + dvc push -r dev Register-New-Model-Dev: - # if: github.event.pull_request.merged == true + if: github.event.pull_request.merged == true runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -104,49 +104,49 @@ jobs: # gto register test --repo https://github.com/Hestia-Homes/ML/ # echo "chicken" >> test.md - gto -v register ${REGISTER_MODEL_NAME} + # gto -v register ${REGISTER_MODEL_NAME} # gto assign regression --stage dev # gto show - # Register-Prediction-Image-Dev: - # needs: Promote-Model-To-Dev - # # needs: [Promote-Model-To-Dev, Register-New-Model-Dev] WILL ADD BACK ONCE REGISTER WORKS - # runs-on: ubuntu-latest + Register-Prediction-Image-Dev: + needs: Promote-Model-To-Dev + # needs: [Promote-Model-To-Dev, Register-New-Model-Dev] WILL ADD BACK ONCE REGISTER WORKS + runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v3 - # - name: Install packages to retrieve artifacts - # env: - # AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} - # run: | - # pip install --upgrade pip - # pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt + steps: + - uses: actions/checkout@v3 + - name: Install packages to retrieve artifacts + env: + AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} + run: | + pip install --upgrade pip + pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt - # - name: Retrieve artifacts (dvc.lock) - # env: - # AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} - # run: | - # cd modules/ml-pipeline/src/pipeline/src - # dvc pull -r dev + - name: Retrieve artifacts (dvc.lock) + env: + AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} + run: | + cd modules/ml-pipeline/src/pipeline/src + dvc pull -r dev - # - name: Build Prediction docker image (TODO - NEED LAMBDA IMAGE, need to add version from gto registry) - # run: | - # cd modules/ml-pipeline/src/pipeline/ - # REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}') - # docker build . --file Prediction.Dockerfile --tag ${REGISTER_MODEL_NAME} + - name: Build Prediction docker image (TODO - NEED LAMBDA IMAGE, need to add version from gto registry) + run: | + cd modules/ml-pipeline/src/pipeline/ + REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}') + docker build . --file Prediction.Dockerfile --tag ${REGISTER_MODEL_NAME} - # - name: ECR Login - Dev - # env: - # AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} - # run: | - # echo "LOGIN TO ECR" + - name: ECR Login - Dev + env: + AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} + run: | + echo "LOGIN TO ECR" - # - name: Push Prediction image to ECR - Dev - # env: - # AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} - # run: | - # echo "PUSH TO ECR" + - name: Push Prediction image to ECR - Dev + env: + AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} + run: | + echo "PUSH TO ECR" diff --git a/.github/workflows/MLPipelinePullRequest.yml b/.github/workflows/MLPipelinePullRequest.yml index a5b4d5a..52325e0 100644 --- a/.github/workflows/MLPipelinePullRequest.yml +++ b/.github/workflows/MLPipelinePullRequest.yml @@ -23,75 +23,75 @@ jobs: echo "Please choose one of these tags: 'major', 'major', 'patch'" exit(1) - # Verify-Model: + Verify-Model: - # runs-on: ubuntu-latest + runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v3 - # - name: Install packages to retrieve artifacts - # env: - # AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} - # run: | - # pip install --upgrade pip - # pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt + steps: + - uses: actions/checkout@v3 + - name: Install packages to retrieve artifacts + env: + AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} + run: | + pip install --upgrade pip + pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt - # - name: Retrieve artifacts (dvc.lock) - # env: - # AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} - # run: | - # cd modules/ml-pipeline/src/pipeline/src - # dvc pull -r experiments + - name: Retrieve artifacts (dvc.lock) + env: + AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} + run: | + cd modules/ml-pipeline/src/pipeline/src + dvc pull -r experiments - # - name: Build Prediction docker Image - # run: | - # cd modules/ml-pipeline/src/pipeline/ - # docker build . --file Prediction.Dockerfile --tag prediction_test + - name: Build Prediction docker Image + run: | + cd modules/ml-pipeline/src/pipeline/ + docker build . --file Prediction.Dockerfile --tag prediction_test - # - name: Run Prediction docker container - # run: | - # docker run prediction_test + - name: Run Prediction docker container + run: | + docker run prediction_test - # Trigger-CML: + Trigger-CML: - # runs-on: ubuntu-latest + runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v3 - # - name: Install packages to retrieve artifacts - # run: | - # pip install --upgrade pip - # pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt + steps: + - uses: actions/checkout@v3 + - name: Install packages to retrieve artifacts + run: | + pip install --upgrade pip + pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt - # - name: Retrieve artifacts (dvc.lock) - # env: - # AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} - # run: | - # cd modules/ml-pipeline/src/pipeline/src - # dvc pull -r experiments + - name: Retrieve artifacts (dvc.lock) + env: + AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} + run: | + cd modules/ml-pipeline/src/pipeline/src + dvc pull -r experiments - # - uses: actions/setup-python@v4 - # - uses: iterative/setup-cml@v1 - # - name: Generate report - # env: - # AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} - # REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # run: | - # cd modules/ml-pipeline/src/pipeline/src - # echo "## Model metrics" > report.md + - uses: actions/setup-python@v4 + - uses: iterative/setup-cml@v1 + - name: Generate report + env: + AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} + REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + cd modules/ml-pipeline/src/pipeline/src + echo "## Model metrics" > report.md - # # Compare metrics to master - # git fetch --depth=1 origin master:master - # dvc metrics diff --md --all master >> report.md + # Compare metrics to master + git fetch --depth=1 origin master:master + dvc metrics diff --md --all master >> report.md - # cml comment create report.md + cml comment create report.md - # # echo "## Residuals plot from model" >> report.md - # # metrics_location=$(find . -maxdepth 10 -name "residuals.png") - # # echo $metrics_location - # # cd $metric_location - # # echo "![](./residuals.png)" >> report.md + # echo "## Residuals plot from model" >> report.md + # metrics_location=$(find . -maxdepth 10 -name "residuals.png") + # echo $metrics_location + # cd $metric_location + # echo "![](./residuals.png)" >> report.md