From a4dffe527a6fb2e4a53729d32036f5e11fdc9aae Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Sun, 26 May 2024 09:47:08 +0100 Subject: [PATCH 1/4] add testing script --- .github/workflows/MLPipelineTESTING.yml | 238 ++++++++++++++++++++++++ 1 file changed, 238 insertions(+) create mode 100644 .github/workflows/MLPipelineTESTING.yml diff --git a/.github/workflows/MLPipelineTESTING.yml b/.github/workflows/MLPipelineTESTING.yml new file mode 100644 index 0000000..92c1792 --- /dev/null +++ b/.github/workflows/MLPipelineTESTING.yml @@ -0,0 +1,238 @@ +name: Register the model for the given pipeline branch (TESTING) + +on: + push: + branches: + - "sap-dev-gto" + +# on: +# pull_request: +# types: +# - closed +# branches: +# - "sap-dev" +# - "heat-dev" +# - "carbon-dev" + +permissions: write-all + +jobs: + Register-Major-Model-Dev: + # if: ${{ (github.event.pull_request.merged == true) && (contains(github.event.pull_request.labels.*.name, 'major')) }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install packages to register model + run: | + pip install --upgrade pip + pip install -r modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt + + - name: Register Model + run: | + # REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}') + REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}') + + git config user.name "Github-Bot" + git config user.email "Github-Bot@no-reply.com" + + latest_version=$(gto show ${REGISTER_MODEL_NAME}@latest --ref | awk -F"@v" '{print $2}') || false + if [ -z "${latest_version}" ]; then + increment_version="1.0.0" + else + increment_version=$(echo ${latest_version} | awk 'BEGIN { + FS="\\." # Set the field separator to a period + OFS="." # Set the output field separator to a period + } + { + major = $1 + 1 # Increment the major version + print major, "0", "0" # Print the new version + }') + fi + + new_tag=${REGISTER_MODEL_NAME}@v${increment_version} + + # git tag -a ${new_tag} -m "Registering new Major Version" + # git push origin ${new_tag} + + # gto show --json > MODEL_REGISTRY.md + # git add . + # git commit -m "Update Registry" + # git push + + Register-Minor-Model-Dev: + # if: ${{ (github.event.pull_request.merged == true) && (contains(github.event.pull_request.labels.*.name, 'minor')) }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install packages to register model + run: | + pip install --upgrade pip + pip install -r modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt + + - name: Register Model + run: | + # REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}') + REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}') + + git config user.name "Github-Bot" + git config user.email "Github-Bot@no-reply.com" + + latest_version=$(gto show ${REGISTER_MODEL_NAME}@latest --ref | awk -F"@v" '{print $2}') + if [ -z "${latest_version}" ]; then + increment_version="0.1.0" + else + increment_version=$(echo ${latest_version} | awk 'BEGIN { + FS="\\." # Set the field separator to a period + OFS="." # Set the output field separator to a period + } + { + minor = $2 + 1 # Increment the minor version + print $1, minor, "0" # Print the new version + }') + fi + + new_tag=${REGISTER_MODEL_NAME}@v${increment_version} + + # git tag -a ${new_tag} -m "Registering new Minor Version" + # git push origin ${new_tag} + + # gto show --json > MODEL_REGISTRY.md + # git add . + # git commit -m "Update Registry" + # git push + + Register-Patch-Model-Dev: + # if: ${{ (github.event.pull_request.merged == true) && (contains(github.event.pull_request.labels.*.name, 'patch')) }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install packages to register model + run: | + pip install --upgrade pip + pip install -r modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt + + - name: Register Model + run: | + # REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}') + REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}') + + git config user.name "Github-Bot" + git config user.email "Github-Bot@no-reply.com" + + latest_version=$(gto show ${REGISTER_MODEL_NAME}@latest --ref | awk -F"@v" '{print $2}') + if [ -z "${latest_version}" ]; then + increment_version="0.0.1" + else + increment_version=$(echo ${latest_version} | awk 'BEGIN { + FS="\\." # Set the field separator to a period + OFS="." # Set the output field separator to a period + } + { + patch = $3 + 1 # Increment the patch version + print $1, $2, patch # Print the new version + }') + fi + + new_tag=${REGISTER_MODEL_NAME}@v${increment_version} + + # git tag -a ${new_tag} -m "Registering new Patch Version" + # git push origin ${new_tag} + + # gto show --json > MODEL_REGISTRY.md + # git add . + # git commit -m "Update Registry" + # git push + + Promote-Artefacts-To-Dev: + if: github.event.pull_request.merged == true + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Install packages to retrieve artifacts + run: | + pip install --upgrade pip + pip install -r modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt + + - name: Retrieve artifacts (dvc.lock) + env: + AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} + run: | + cd modules/ml-pipeline/src/pipeline + dvc pull -r experiments + + - name: Push artifacts to Dev + env: + AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} + run: | + cd modules/ml-pipeline/src/pipeline + dvc push -r dev + + Register-New-Model-Dev: + needs: [Register-Major-Model-Dev, Register-Minor-Model-Dev, Register-Patch-Model-Dev] + if: | + always() && + (needs.Register-Major-Model-Dev.result == 'success' || needs.Register-Major-Model-Dev.result == 'skipped') && + (needs.Register-Minor-Model-Dev.result == 'success' || needs.Register-Minor-Model-Dev.result == 'skipped') && + (needs.Register-Patch-Model-Dev.result == 'success' || needs.Register-Patch-Model-Dev.result == 'skipped') + + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Install packages to register model + run: | + pip install --upgrade pip + pip install -r modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt + + - name: Register Model + env: + TARGET_BRANCH: ${{ github.base_ref }} + run: | + + REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}') + # REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}') + + git config user.name "Github-Bot" + git config user.email "Github-Bot@no-reply.com" + + latest_dev_version=$(gto history ${REGISTER_MODEL_NAME} --asc --plain | awk '{print $NF}' | awk '/dev/' | awk 'END {print}') + if [ -z "${latest_dev_version}" ]; then + increment_version="1" + else + increment_version=$(echo ${latest_dev_version} | awk '{print $NF}' | awk -F"#" '{print $3}' | awk '{$1++; print}') + fi + + new_tag=${REGISTER_MODEL_NAME}#dev#${increment_version} + latest_version=$(gto show ${REGISTER_MODEL_NAME}@latest --ref | awk -F"@" '{print $2}') + + echo ${new_tag} + + commit_hash=$(gto history ${REGISTER_MODEL_NAME} --asc --plain | awk "/${latest_version}/" | awk '{print $(NF-1)}') + git checkout ${commit_hash} + + # git pull #Get new model registry md file changes + git tag -a ${new_tag} -m "Assigning stage dev to artifact ${REGISTER_MODEL_NAME} version ${latest_version}" + git push origin ${new_tag} + + git checkout ${TARGET_BRANCH} + git fetch --all + git pull + + gto show --json > MODEL_REGISTRY.md + git add . + git commit -m "Update Registry" + git push origin ${TARGET_BRANCH} From 0768ace94787e547c637ac6b6e4ae395db2edf52 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Sun, 26 May 2024 09:50:39 +0100 Subject: [PATCH 2/4] add testing script --- .github/workflows/MLPipelineTESTING.yml | 176 ++++++++++++------------ 1 file changed, 89 insertions(+), 87 deletions(-) diff --git a/.github/workflows/MLPipelineTESTING.yml b/.github/workflows/MLPipelineTESTING.yml index 92c1792..f2a200a 100644 --- a/.github/workflows/MLPipelineTESTING.yml +++ b/.github/workflows/MLPipelineTESTING.yml @@ -38,21 +38,23 @@ jobs: git config user.name "Github-Bot" git config user.email "Github-Bot@no-reply.com" - latest_version=$(gto show ${REGISTER_MODEL_NAME}@latest --ref | awk -F"@v" '{print $2}') || false - if [ -z "${latest_version}" ]; then - increment_version="1.0.0" - else - increment_version=$(echo ${latest_version} | awk 'BEGIN { - FS="\\." # Set the field separator to a period - OFS="." # Set the output field separator to a period - } - { - major = $1 + 1 # Increment the major version - print major, "0", "0" # Print the new version - }') - fi + gto show - new_tag=${REGISTER_MODEL_NAME}@v${increment_version} + # latest_version=$(gto show ${REGISTER_MODEL_NAME}@latest --ref | awk -F"@v" '{print $2}') || false + # if [ -z "${latest_version}" ]; then + # increment_version="1.0.0" + # else + # increment_version=$(echo ${latest_version} | awk 'BEGIN { + # FS="\\." # Set the field separator to a period + # OFS="." # Set the output field separator to a period + # } + # { + # major = $1 + 1 # Increment the major version + # print major, "0", "0" # Print the new version + # }') + # fi + + # new_tag=${REGISTER_MODEL_NAME}@v${increment_version} # git tag -a ${new_tag} -m "Registering new Major Version" # git push origin ${new_tag} @@ -83,21 +85,21 @@ jobs: git config user.name "Github-Bot" git config user.email "Github-Bot@no-reply.com" - latest_version=$(gto show ${REGISTER_MODEL_NAME}@latest --ref | awk -F"@v" '{print $2}') - if [ -z "${latest_version}" ]; then - increment_version="0.1.0" - else - increment_version=$(echo ${latest_version} | awk 'BEGIN { - FS="\\." # Set the field separator to a period - OFS="." # Set the output field separator to a period - } - { - minor = $2 + 1 # Increment the minor version - print $1, minor, "0" # Print the new version - }') - fi + # latest_version=$(gto show ${REGISTER_MODEL_NAME}@latest --ref | awk -F"@v" '{print $2}') + # if [ -z "${latest_version}" ]; then + # increment_version="0.1.0" + # else + # increment_version=$(echo ${latest_version} | awk 'BEGIN { + # FS="\\." # Set the field separator to a period + # OFS="." # Set the output field separator to a period + # } + # { + # minor = $2 + 1 # Increment the minor version + # print $1, minor, "0" # Print the new version + # }') + # fi - new_tag=${REGISTER_MODEL_NAME}@v${increment_version} + # new_tag=${REGISTER_MODEL_NAME}@v${increment_version} # git tag -a ${new_tag} -m "Registering new Minor Version" # git push origin ${new_tag} @@ -128,21 +130,21 @@ jobs: git config user.name "Github-Bot" git config user.email "Github-Bot@no-reply.com" - latest_version=$(gto show ${REGISTER_MODEL_NAME}@latest --ref | awk -F"@v" '{print $2}') - if [ -z "${latest_version}" ]; then - increment_version="0.0.1" - else - increment_version=$(echo ${latest_version} | awk 'BEGIN { - FS="\\." # Set the field separator to a period - OFS="." # Set the output field separator to a period - } - { - patch = $3 + 1 # Increment the patch version - print $1, $2, patch # Print the new version - }') - fi + # latest_version=$(gto show ${REGISTER_MODEL_NAME}@latest --ref | awk -F"@v" '{print $2}') + # if [ -z "${latest_version}" ]; then + # increment_version="0.0.1" + # else + # increment_version=$(echo ${latest_version} | awk 'BEGIN { + # FS="\\." # Set the field separator to a period + # OFS="." # Set the output field separator to a period + # } + # { + # patch = $3 + 1 # Increment the patch version + # print $1, $2, patch # Print the new version + # }') + # fi - new_tag=${REGISTER_MODEL_NAME}@v${increment_version} + # new_tag=${REGISTER_MODEL_NAME}@v${increment_version} # git tag -a ${new_tag} -m "Registering new Patch Version" # git push origin ${new_tag} @@ -179,60 +181,60 @@ jobs: cd modules/ml-pipeline/src/pipeline dvc push -r dev - Register-New-Model-Dev: - needs: [Register-Major-Model-Dev, Register-Minor-Model-Dev, Register-Patch-Model-Dev] - if: | - always() && - (needs.Register-Major-Model-Dev.result == 'success' || needs.Register-Major-Model-Dev.result == 'skipped') && - (needs.Register-Minor-Model-Dev.result == 'success' || needs.Register-Minor-Model-Dev.result == 'skipped') && - (needs.Register-Patch-Model-Dev.result == 'success' || needs.Register-Patch-Model-Dev.result == 'skipped') + # Register-New-Model-Dev: + # needs: [Register-Major-Model-Dev, Register-Minor-Model-Dev, Register-Patch-Model-Dev] + # if: | + # always() && + # (needs.Register-Major-Model-Dev.result == 'success' || needs.Register-Major-Model-Dev.result == 'skipped') && + # (needs.Register-Minor-Model-Dev.result == 'success' || needs.Register-Minor-Model-Dev.result == 'skipped') && + # (needs.Register-Patch-Model-Dev.result == 'success' || needs.Register-Patch-Model-Dev.result == 'skipped') - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v3 + # with: + # fetch-depth: 0 - - name: Install packages to register model - run: | - pip install --upgrade pip - pip install -r modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt + # - name: Install packages to register model + # run: | + # pip install --upgrade pip + # pip install -r modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt - - name: Register Model - env: - TARGET_BRANCH: ${{ github.base_ref }} - run: | + # - name: Register Model + # env: + # TARGET_BRANCH: ${{ github.base_ref }} + # run: | - REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}') - # REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}') + # REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}') + # # REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}') - git config user.name "Github-Bot" - git config user.email "Github-Bot@no-reply.com" + # git config user.name "Github-Bot" + # git config user.email "Github-Bot@no-reply.com" - latest_dev_version=$(gto history ${REGISTER_MODEL_NAME} --asc --plain | awk '{print $NF}' | awk '/dev/' | awk 'END {print}') - if [ -z "${latest_dev_version}" ]; then - increment_version="1" - else - increment_version=$(echo ${latest_dev_version} | awk '{print $NF}' | awk -F"#" '{print $3}' | awk '{$1++; print}') - fi + # latest_dev_version=$(gto history ${REGISTER_MODEL_NAME} --asc --plain | awk '{print $NF}' | awk '/dev/' | awk 'END {print}') + # if [ -z "${latest_dev_version}" ]; then + # increment_version="1" + # else + # increment_version=$(echo ${latest_dev_version} | awk '{print $NF}' | awk -F"#" '{print $3}' | awk '{$1++; print}') + # fi - new_tag=${REGISTER_MODEL_NAME}#dev#${increment_version} - latest_version=$(gto show ${REGISTER_MODEL_NAME}@latest --ref | awk -F"@" '{print $2}') + # new_tag=${REGISTER_MODEL_NAME}#dev#${increment_version} + # latest_version=$(gto show ${REGISTER_MODEL_NAME}@latest --ref | awk -F"@" '{print $2}') - echo ${new_tag} + # echo ${new_tag} - commit_hash=$(gto history ${REGISTER_MODEL_NAME} --asc --plain | awk "/${latest_version}/" | awk '{print $(NF-1)}') - git checkout ${commit_hash} + # commit_hash=$(gto history ${REGISTER_MODEL_NAME} --asc --plain | awk "/${latest_version}/" | awk '{print $(NF-1)}') + # git checkout ${commit_hash} - # git pull #Get new model registry md file changes - git tag -a ${new_tag} -m "Assigning stage dev to artifact ${REGISTER_MODEL_NAME} version ${latest_version}" - git push origin ${new_tag} + # # git pull #Get new model registry md file changes + # git tag -a ${new_tag} -m "Assigning stage dev to artifact ${REGISTER_MODEL_NAME} version ${latest_version}" + # git push origin ${new_tag} - git checkout ${TARGET_BRANCH} - git fetch --all - git pull + # git checkout ${TARGET_BRANCH} + # git fetch --all + # git pull - gto show --json > MODEL_REGISTRY.md - git add . - git commit -m "Update Registry" - git push origin ${TARGET_BRANCH} + # gto show --json > MODEL_REGISTRY.md + # git add . + # git commit -m "Update Registry" + # git push origin ${TARGET_BRANCH} From 9e23eae27af4c22a5f52280c718b1551a6db31ca Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Sun, 26 May 2024 09:54:22 +0100 Subject: [PATCH 3/4] add testing script --- .../pipeline/requirements/version_control/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt b/modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt index a2b9531..173550d 100644 --- a/modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt +++ b/modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt @@ -1,4 +1,4 @@ -dvc==3.36.0 -dvc-s3==3.0.1 -gto==1.6.1 +dvc==3.51.0 +dvc-s3==3.2.0 +gto==1.7.1 pyOpenSSL==23.3.0 From e0954b52bce8088ec2b1550d2a58fd40de454b87 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Sun, 26 May 2024 09:56:05 +0100 Subject: [PATCH 4/4] Upgrade dvc packages to fix pygit2 error --- .github/workflows/MLPipelineTESTING.yml | 240 ------------------------ 1 file changed, 240 deletions(-) delete mode 100644 .github/workflows/MLPipelineTESTING.yml diff --git a/.github/workflows/MLPipelineTESTING.yml b/.github/workflows/MLPipelineTESTING.yml deleted file mode 100644 index f2a200a..0000000 --- a/.github/workflows/MLPipelineTESTING.yml +++ /dev/null @@ -1,240 +0,0 @@ -name: Register the model for the given pipeline branch (TESTING) - -on: - push: - branches: - - "sap-dev-gto" - -# on: -# pull_request: -# types: -# - closed -# branches: -# - "sap-dev" -# - "heat-dev" -# - "carbon-dev" - -permissions: write-all - -jobs: - Register-Major-Model-Dev: - # if: ${{ (github.event.pull_request.merged == true) && (contains(github.event.pull_request.labels.*.name, 'major')) }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Install packages to register model - run: | - pip install --upgrade pip - pip install -r modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt - - - name: Register Model - run: | - # REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}') - REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}') - - git config user.name "Github-Bot" - git config user.email "Github-Bot@no-reply.com" - - gto show - - # latest_version=$(gto show ${REGISTER_MODEL_NAME}@latest --ref | awk -F"@v" '{print $2}') || false - # if [ -z "${latest_version}" ]; then - # increment_version="1.0.0" - # else - # increment_version=$(echo ${latest_version} | awk 'BEGIN { - # FS="\\." # Set the field separator to a period - # OFS="." # Set the output field separator to a period - # } - # { - # major = $1 + 1 # Increment the major version - # print major, "0", "0" # Print the new version - # }') - # fi - - # new_tag=${REGISTER_MODEL_NAME}@v${increment_version} - - # git tag -a ${new_tag} -m "Registering new Major Version" - # git push origin ${new_tag} - - # gto show --json > MODEL_REGISTRY.md - # git add . - # git commit -m "Update Registry" - # git push - - Register-Minor-Model-Dev: - # if: ${{ (github.event.pull_request.merged == true) && (contains(github.event.pull_request.labels.*.name, 'minor')) }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Install packages to register model - run: | - pip install --upgrade pip - pip install -r modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt - - - name: Register Model - run: | - # REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}') - REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}') - - git config user.name "Github-Bot" - git config user.email "Github-Bot@no-reply.com" - - # latest_version=$(gto show ${REGISTER_MODEL_NAME}@latest --ref | awk -F"@v" '{print $2}') - # if [ -z "${latest_version}" ]; then - # increment_version="0.1.0" - # else - # increment_version=$(echo ${latest_version} | awk 'BEGIN { - # FS="\\." # Set the field separator to a period - # OFS="." # Set the output field separator to a period - # } - # { - # minor = $2 + 1 # Increment the minor version - # print $1, minor, "0" # Print the new version - # }') - # fi - - # new_tag=${REGISTER_MODEL_NAME}@v${increment_version} - - # git tag -a ${new_tag} -m "Registering new Minor Version" - # git push origin ${new_tag} - - # gto show --json > MODEL_REGISTRY.md - # git add . - # git commit -m "Update Registry" - # git push - - Register-Patch-Model-Dev: - # if: ${{ (github.event.pull_request.merged == true) && (contains(github.event.pull_request.labels.*.name, 'patch')) }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Install packages to register model - run: | - pip install --upgrade pip - pip install -r modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt - - - name: Register Model - run: | - # REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}') - REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}') - - git config user.name "Github-Bot" - git config user.email "Github-Bot@no-reply.com" - - # latest_version=$(gto show ${REGISTER_MODEL_NAME}@latest --ref | awk -F"@v" '{print $2}') - # if [ -z "${latest_version}" ]; then - # increment_version="0.0.1" - # else - # increment_version=$(echo ${latest_version} | awk 'BEGIN { - # FS="\\." # Set the field separator to a period - # OFS="." # Set the output field separator to a period - # } - # { - # patch = $3 + 1 # Increment the patch version - # print $1, $2, patch # Print the new version - # }') - # fi - - # new_tag=${REGISTER_MODEL_NAME}@v${increment_version} - - # git tag -a ${new_tag} -m "Registering new Patch Version" - # git push origin ${new_tag} - - # gto show --json > MODEL_REGISTRY.md - # git add . - # git commit -m "Update Registry" - # git push - - Promote-Artefacts-To-Dev: - if: github.event.pull_request.merged == true - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - name: Install packages to retrieve artifacts - run: | - pip install --upgrade pip - pip install -r modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt - - - name: Retrieve artifacts (dvc.lock) - env: - AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} - run: | - cd modules/ml-pipeline/src/pipeline - dvc pull -r experiments - - - name: Push artifacts to Dev - env: - AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} - run: | - cd modules/ml-pipeline/src/pipeline - dvc push -r dev - - # Register-New-Model-Dev: - # needs: [Register-Major-Model-Dev, Register-Minor-Model-Dev, Register-Patch-Model-Dev] - # if: | - # always() && - # (needs.Register-Major-Model-Dev.result == 'success' || needs.Register-Major-Model-Dev.result == 'skipped') && - # (needs.Register-Minor-Model-Dev.result == 'success' || needs.Register-Minor-Model-Dev.result == 'skipped') && - # (needs.Register-Patch-Model-Dev.result == 'success' || needs.Register-Patch-Model-Dev.result == 'skipped') - - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v3 - # with: - # fetch-depth: 0 - - # - name: Install packages to register model - # run: | - # pip install --upgrade pip - # pip install -r modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt - - # - name: Register Model - # env: - # TARGET_BRANCH: ${{ github.base_ref }} - # run: | - - # REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}') - # # REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}') - - # git config user.name "Github-Bot" - # git config user.email "Github-Bot@no-reply.com" - - # latest_dev_version=$(gto history ${REGISTER_MODEL_NAME} --asc --plain | awk '{print $NF}' | awk '/dev/' | awk 'END {print}') - # if [ -z "${latest_dev_version}" ]; then - # increment_version="1" - # else - # increment_version=$(echo ${latest_dev_version} | awk '{print $NF}' | awk -F"#" '{print $3}' | awk '{$1++; print}') - # fi - - # new_tag=${REGISTER_MODEL_NAME}#dev#${increment_version} - # latest_version=$(gto show ${REGISTER_MODEL_NAME}@latest --ref | awk -F"@" '{print $2}') - - # echo ${new_tag} - - # commit_hash=$(gto history ${REGISTER_MODEL_NAME} --asc --plain | awk "/${latest_version}/" | awk '{print $(NF-1)}') - # git checkout ${commit_hash} - - # # git pull #Get new model registry md file changes - # git tag -a ${new_tag} -m "Assigning stage dev to artifact ${REGISTER_MODEL_NAME} version ${latest_version}" - # git push origin ${new_tag} - - # git checkout ${TARGET_BRANCH} - # git fetch --all - # git pull - - # gto show --json > MODEL_REGISTRY.md - # git add . - # git commit -m "Update Registry" - # git push origin ${TARGET_BRANCH}