From 0768ace94787e547c637ac6b6e4ae395db2edf52 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Sun, 26 May 2024 09:50:39 +0100 Subject: [PATCH] add testing script --- .github/workflows/MLPipelineTESTING.yml | 176 ++++++++++++------------ 1 file changed, 89 insertions(+), 87 deletions(-) diff --git a/.github/workflows/MLPipelineTESTING.yml b/.github/workflows/MLPipelineTESTING.yml index 92c1792..f2a200a 100644 --- a/.github/workflows/MLPipelineTESTING.yml +++ b/.github/workflows/MLPipelineTESTING.yml @@ -38,21 +38,23 @@ jobs: git config user.name "Github-Bot" git config user.email "Github-Bot@no-reply.com" - latest_version=$(gto show ${REGISTER_MODEL_NAME}@latest --ref | awk -F"@v" '{print $2}') || false - if [ -z "${latest_version}" ]; then - increment_version="1.0.0" - else - increment_version=$(echo ${latest_version} | awk 'BEGIN { - FS="\\." # Set the field separator to a period - OFS="." # Set the output field separator to a period - } - { - major = $1 + 1 # Increment the major version - print major, "0", "0" # Print the new version - }') - fi + gto show - new_tag=${REGISTER_MODEL_NAME}@v${increment_version} + # latest_version=$(gto show ${REGISTER_MODEL_NAME}@latest --ref | awk -F"@v" '{print $2}') || false + # if [ -z "${latest_version}" ]; then + # increment_version="1.0.0" + # else + # increment_version=$(echo ${latest_version} | awk 'BEGIN { + # FS="\\." # Set the field separator to a period + # OFS="." # Set the output field separator to a period + # } + # { + # major = $1 + 1 # Increment the major version + # print major, "0", "0" # Print the new version + # }') + # fi + + # new_tag=${REGISTER_MODEL_NAME}@v${increment_version} # git tag -a ${new_tag} -m "Registering new Major Version" # git push origin ${new_tag} @@ -83,21 +85,21 @@ jobs: git config user.name "Github-Bot" git config user.email "Github-Bot@no-reply.com" - latest_version=$(gto show ${REGISTER_MODEL_NAME}@latest --ref | awk -F"@v" '{print $2}') - if [ -z "${latest_version}" ]; then - increment_version="0.1.0" - else - increment_version=$(echo ${latest_version} | awk 'BEGIN { - FS="\\." # Set the field separator to a period - OFS="." # Set the output field separator to a period - } - { - minor = $2 + 1 # Increment the minor version - print $1, minor, "0" # Print the new version - }') - fi + # latest_version=$(gto show ${REGISTER_MODEL_NAME}@latest --ref | awk -F"@v" '{print $2}') + # if [ -z "${latest_version}" ]; then + # increment_version="0.1.0" + # else + # increment_version=$(echo ${latest_version} | awk 'BEGIN { + # FS="\\." # Set the field separator to a period + # OFS="." # Set the output field separator to a period + # } + # { + # minor = $2 + 1 # Increment the minor version + # print $1, minor, "0" # Print the new version + # }') + # fi - new_tag=${REGISTER_MODEL_NAME}@v${increment_version} + # new_tag=${REGISTER_MODEL_NAME}@v${increment_version} # git tag -a ${new_tag} -m "Registering new Minor Version" # git push origin ${new_tag} @@ -128,21 +130,21 @@ jobs: git config user.name "Github-Bot" git config user.email "Github-Bot@no-reply.com" - latest_version=$(gto show ${REGISTER_MODEL_NAME}@latest --ref | awk -F"@v" '{print $2}') - if [ -z "${latest_version}" ]; then - increment_version="0.0.1" - else - increment_version=$(echo ${latest_version} | awk 'BEGIN { - FS="\\." # Set the field separator to a period - OFS="." # Set the output field separator to a period - } - { - patch = $3 + 1 # Increment the patch version - print $1, $2, patch # Print the new version - }') - fi + # latest_version=$(gto show ${REGISTER_MODEL_NAME}@latest --ref | awk -F"@v" '{print $2}') + # if [ -z "${latest_version}" ]; then + # increment_version="0.0.1" + # else + # increment_version=$(echo ${latest_version} | awk 'BEGIN { + # FS="\\." # Set the field separator to a period + # OFS="." # Set the output field separator to a period + # } + # { + # patch = $3 + 1 # Increment the patch version + # print $1, $2, patch # Print the new version + # }') + # fi - new_tag=${REGISTER_MODEL_NAME}@v${increment_version} + # new_tag=${REGISTER_MODEL_NAME}@v${increment_version} # git tag -a ${new_tag} -m "Registering new Patch Version" # git push origin ${new_tag} @@ -179,60 +181,60 @@ jobs: cd modules/ml-pipeline/src/pipeline dvc push -r dev - Register-New-Model-Dev: - needs: [Register-Major-Model-Dev, Register-Minor-Model-Dev, Register-Patch-Model-Dev] - if: | - always() && - (needs.Register-Major-Model-Dev.result == 'success' || needs.Register-Major-Model-Dev.result == 'skipped') && - (needs.Register-Minor-Model-Dev.result == 'success' || needs.Register-Minor-Model-Dev.result == 'skipped') && - (needs.Register-Patch-Model-Dev.result == 'success' || needs.Register-Patch-Model-Dev.result == 'skipped') + # Register-New-Model-Dev: + # needs: [Register-Major-Model-Dev, Register-Minor-Model-Dev, Register-Patch-Model-Dev] + # if: | + # always() && + # (needs.Register-Major-Model-Dev.result == 'success' || needs.Register-Major-Model-Dev.result == 'skipped') && + # (needs.Register-Minor-Model-Dev.result == 'success' || needs.Register-Minor-Model-Dev.result == 'skipped') && + # (needs.Register-Patch-Model-Dev.result == 'success' || needs.Register-Patch-Model-Dev.result == 'skipped') - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v3 + # with: + # fetch-depth: 0 - - name: Install packages to register model - run: | - pip install --upgrade pip - pip install -r modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt + # - name: Install packages to register model + # run: | + # pip install --upgrade pip + # pip install -r modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt - - name: Register Model - env: - TARGET_BRANCH: ${{ github.base_ref }} - run: | + # - name: Register Model + # env: + # TARGET_BRANCH: ${{ github.base_ref }} + # run: | - REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}') - # REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}') + # REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}') + # # REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}') - git config user.name "Github-Bot" - git config user.email "Github-Bot@no-reply.com" + # git config user.name "Github-Bot" + # git config user.email "Github-Bot@no-reply.com" - latest_dev_version=$(gto history ${REGISTER_MODEL_NAME} --asc --plain | awk '{print $NF}' | awk '/dev/' | awk 'END {print}') - if [ -z "${latest_dev_version}" ]; then - increment_version="1" - else - increment_version=$(echo ${latest_dev_version} | awk '{print $NF}' | awk -F"#" '{print $3}' | awk '{$1++; print}') - fi + # latest_dev_version=$(gto history ${REGISTER_MODEL_NAME} --asc --plain | awk '{print $NF}' | awk '/dev/' | awk 'END {print}') + # if [ -z "${latest_dev_version}" ]; then + # increment_version="1" + # else + # increment_version=$(echo ${latest_dev_version} | awk '{print $NF}' | awk -F"#" '{print $3}' | awk '{$1++; print}') + # fi - new_tag=${REGISTER_MODEL_NAME}#dev#${increment_version} - latest_version=$(gto show ${REGISTER_MODEL_NAME}@latest --ref | awk -F"@" '{print $2}') + # new_tag=${REGISTER_MODEL_NAME}#dev#${increment_version} + # latest_version=$(gto show ${REGISTER_MODEL_NAME}@latest --ref | awk -F"@" '{print $2}') - echo ${new_tag} + # echo ${new_tag} - commit_hash=$(gto history ${REGISTER_MODEL_NAME} --asc --plain | awk "/${latest_version}/" | awk '{print $(NF-1)}') - git checkout ${commit_hash} + # commit_hash=$(gto history ${REGISTER_MODEL_NAME} --asc --plain | awk "/${latest_version}/" | awk '{print $(NF-1)}') + # git checkout ${commit_hash} - # git pull #Get new model registry md file changes - git tag -a ${new_tag} -m "Assigning stage dev to artifact ${REGISTER_MODEL_NAME} version ${latest_version}" - git push origin ${new_tag} + # # git pull #Get new model registry md file changes + # git tag -a ${new_tag} -m "Assigning stage dev to artifact ${REGISTER_MODEL_NAME} version ${latest_version}" + # git push origin ${new_tag} - git checkout ${TARGET_BRANCH} - git fetch --all - git pull + # git checkout ${TARGET_BRANCH} + # git fetch --all + # git pull - gto show --json > MODEL_REGISTRY.md - git add . - git commit -m "Update Registry" - git push origin ${TARGET_BRANCH} + # gto show --json > MODEL_REGISTRY.md + # git add . + # git commit -m "Update Registry" + # git push origin ${TARGET_BRANCH}