From eb41a8d1a0ad248c7f56203400e8e2422b3642d4 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 12 Sep 2023 23:08:38 +0100 Subject: [PATCH] removed register step for now --- .github/workflows/MLPipelinePostMerge.yml | 189 ++++++++++------------ 1 file changed, 85 insertions(+), 104 deletions(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index 9ed3fd0..b43007d 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -1,58 +1,93 @@ name: Register the model for the given pipeline branch -on: - push: - branches: - - "model-**" - # on: -# pull_request: -# types: -# - closed +# push: # branches: -# - "master" +# - "model-**" + +on: + pull_request: + types: + - closed + branches: + - "master" permissions: write-all jobs: - # Promote-Model-To-Dev: + Promote-Model-To-Dev: + if: github.event.pull_request.merged == true + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Install packages to retrieve artifacts + run: | + pip install --upgrade pip + pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt + + - name: Retrieve artifacts (dvc.lock) + env: + AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} + run: | + cd modules/ml-pipeline/src/pipeline/src + dvc pull -r experiments + + - name: Push artifacts to Dev + env: + AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} + run: | + cd modules/ml-pipeline/src/pipeline/src + dvc push -r dev + + + # Register-New-Model-Dev: # if: github.event.pull_request.merged == true # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v3 - # - name: Install packages to retrieve artifacts + # - uses: actions/checkout@v2 + # with: + # fetch-depth: 0 + + # - name: Install packages to register model + # env: + # AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} + # AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} # run: | # pip install --upgrade pip # pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt - # - name: Retrieve artifacts (dvc.lock) + # - name: Register Model # env: # AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} # AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} # run: | - # cd modules/ml-pipeline/src/pipeline/src - # dvc pull -r experiments - # - name: Push artifacts to Dev - # env: - # AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} - # run: | - # cd modules/ml-pipeline/src/pipeline/src - # dvc push -r dev + # REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}') + # # REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}') + # git config user.name "Github-Bot" + # git config user.email "Github-Bot@no-reply.com" - Register-New-Model-Dev: - # if: github.event.pull_request.merged == true + # # gto register test --repo https://github.com/Hestia-Homes/ML/ + # # echo "chicken" >> test.md + + # git checkout master + # gto register ${REGISTER_MODEL_NAME} + # gto assign regression --stage dev + # gto show + + Register-Prediction-Image-Dev: + needs: Promote-Model-To-Dev + # needs: [Promote-Model-To-Dev, Register-New-Model-Dev] WILL ADD BACK ONCE REGISTER WORKS runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Install packages to register model + steps: + - uses: actions/checkout@v3 + - name: Install packages to retrieve artifacts env: AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} @@ -60,84 +95,30 @@ jobs: pip install --upgrade pip pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt - - name: Register Model + - name: Retrieve artifacts (dvc.lock) env: AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} run: | + cd modules/ml-pipeline/src/pipeline/src + dvc pull -r dev - # REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}') - REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}') + - name: Build Prediction docker image (TODO - NEED LAMBDA IMAGE, need to add version from gto registry) + run: | + cd modules/ml-pipeline/src/pipeline/ + REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}') + docker build . --file Prediction.Dockerfile --tag ${REGISTER_MODEL_NAME} - git config user.name "Github-Bot" - git config user.email "Github-Bot@no-reply.com" + - name: ECR Login - Dev + env: + AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} + run: | + echo "LOGIN TO ECR" - gto register ${REGISTER_MODEL_NAME} - - git branch -r - - # gto register test --repo https://github.com/Hestia-Homes/ML/ - # echo "chicken" >> test.md - - # git fetch --prune - # git pull origin master - # git add . - # git commit -m "test" - - # git push - - # git branch - # git tag model@v0.0.1 - # git push origin model@v0.0.1 - # git checkout master - - git remote show origin - # git status - # git push origin regression@v0.0.1 - git checkout master - gto register ${REGISTER_MODEL_NAME} - # gto assign regression --stage dev - # gto show - - # Register-Prediction-Image-Dev: - # needs: Promote-Model-To-Dev - # # needs: [Promote-Model-To-Dev, Register-New-Model-Dev] WILL ADD BACK ONCE REGISTER WORKS - # runs-on: ubuntu-latest - - # steps: - # - uses: actions/checkout@v3 - # - name: Install packages to retrieve artifacts - # env: - # AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} - # run: | - # pip install --upgrade pip - # pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt - - # - name: Retrieve artifacts (dvc.lock) - # env: - # AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} - # run: | - # cd modules/ml-pipeline/src/pipeline/src - # dvc pull -r dev - - # - name: Build Prediction docker image (TODO - NEED LAMBDA IMAGE, need to add version from gto registry) - # run: | - # cd modules/ml-pipeline/src/pipeline/ - # REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}') - # docker build . --file Prediction.Dockerfile --tag ${REGISTER_MODEL_NAME} - - # - name: ECR Login - Dev - # env: - # AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} - # run: | - # echo "LOGIN TO ECR" - - # - name: Push Prediction image to ECR - Dev - # env: - # AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} - # run: | - # echo "PUSH TO ECR" + - name: Push Prediction image to ECR - Dev + env: + AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} + run: | + echo "PUSH TO ECR"