name: Register the model for the given pipeline branch # on: # push: # branches: # - "model-**" on: pull_request: types: - closed branches: - "sap-dev" - "heat-dev" - "carbon-dev" permissions: write-all jobs: Register-Major-Model-Dev: if: ${{ (github.event.pull_request.merged == true) && (contains(github.event.pull_request.labels.*.name, 'major')) }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Install packages to register model run: | pip install --upgrade pip pip install -r modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt - name: Register Model run: | REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}') # REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}') git config user.name "Github-Bot" git config user.email "Github-Bot@no-reply.com" latest_version=$(gto show ${REGISTER_MODEL_NAME}@latest --ref | awk -F"@v" '{print $2}') || false if [ -z "${latest_version}" ]; then increment_version="1.0.0" else increment_version=$(echo ${latest_version} | awk 'BEGIN { FS="\\." # Set the field separator to a period OFS="." # Set the output field separator to a period } { major = $1 + 1 # Increment the major version print major, "0", "0" # Print the new version }') fi new_tag=${REGISTER_MODEL_NAME}@v${increment_version} git tag -a ${new_tag} -m "Registering new Major Version" git push origin ${new_tag} gto show --json > MODEL_REGISTRY.md git add . git commit -m "Update Registry" git push Register-Minor-Model-Dev: if: ${{ (github.event.pull_request.merged == true) && (contains(github.event.pull_request.labels.*.name, 'minor')) }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Install packages to register model run: | pip install --upgrade pip pip install -r modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt - name: Register Model run: | REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}') # REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}') git config user.name "Github-Bot" git config user.email "Github-Bot@no-reply.com" latest_version=$(gto show ${REGISTER_MODEL_NAME}@latest --ref | awk -F"@v" '{print $2}') if [ -z "${latest_version}" ]; then increment_version="0.1.0" else increment_version=$(echo ${latest_version} | awk 'BEGIN { FS="\\." # Set the field separator to a period OFS="." # Set the output field separator to a period } { minor = $2 + 1 # Increment the minor version print $1, minor, "0" # Print the new version }') fi new_tag=${REGISTER_MODEL_NAME}@v${increment_version} git tag -a ${new_tag} -m "Registering new Minor Version" git push origin ${new_tag} gto show --json > MODEL_REGISTRY.md git add . git commit -m "Update Registry" git push Register-Patch-Model-Dev: if: ${{ (github.event.pull_request.merged == true) && (contains(github.event.pull_request.labels.*.name, 'patch')) }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Install packages to register model run: | pip install --upgrade pip pip install -r modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt - name: Register Model run: | REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}') # REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}') git config user.name "Github-Bot" git config user.email "Github-Bot@no-reply.com" latest_version=$(gto show ${REGISTER_MODEL_NAME}@latest --ref | awk -F"@v" '{print $2}') if [ -z "${latest_version}" ]; then increment_version="0.0.1" else increment_version=$(echo ${latest_version} | awk 'BEGIN { FS="\\." # Set the field separator to a period OFS="." # Set the output field separator to a period } { patch = $3 + 1 # Increment the patch version print $1, $2, patch # Print the new version }') fi new_tag=${REGISTER_MODEL_NAME}@v${increment_version} git tag -a ${new_tag} -m "Registering new Patch Version" git push origin ${new_tag} gto show --json > MODEL_REGISTRY.md git add . git commit -m "Update Registry" git push Promote-Artefacts-To-Dev: if: github.event.pull_request.merged == true runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - name: Install packages to retrieve artifacts run: | pip install --upgrade pip pip install -r modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt - name: Retrieve artifacts (dvc.lock) env: AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} run: | cd modules/ml-pipeline/src/pipeline dvc pull -r experiments - name: Push artifacts to Dev env: AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} run: | cd modules/ml-pipeline/src/pipeline dvc push -r dev Register-New-Model-Dev: needs: [Register-Major-Model-Dev, Register-Minor-Model-Dev, Register-Patch-Model-Dev] if: | always() && (needs.Register-Major-Model-Dev.result == 'success' || needs.Register-Major-Model-Dev.result == 'skipped') && (needs.Register-Minor-Model-Dev.result == 'success' || needs.Register-Minor-Model-Dev.result == 'skipped') && (needs.Register-Patch-Model-Dev.result == 'success' || needs.Register-Patch-Model-Dev.result == 'skipped') runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Install packages to register model run: | pip install --upgrade pip pip install -r modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt - name: Register Model env: TARGET_BRANCH: ${{ github.base_ref }} run: | REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}') # REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}') git config user.name "Github-Bot" git config user.email "Github-Bot@no-reply.com" latest_dev_version=$(gto history ${REGISTER_MODEL_NAME} --asc --plain | awk '{print $NF}' | awk '/dev/' | awk 'END {print}') if [ -z "${latest_dev_version}" ]; then increment_version="1" else increment_version=$(echo ${latest_dev_version} | awk '{print $NF}' | awk -F"#" '{print $3}' | awk '{$1++; print}') fi new_tag=${REGISTER_MODEL_NAME}#dev#${increment_version} latest_version=$(gto show ${REGISTER_MODEL_NAME}@latest --ref | awk -F"@" '{print $2}') echo ${new_tag} commit_hash=$(gto history ${REGISTER_MODEL_NAME} --asc --plain | awk "/${latest_version}/" | awk '{print $(NF-1)}') git checkout ${commit_hash} # git pull #Get new model registry md file changes git tag -a ${new_tag} -m "Assigning stage dev to artifact ${REGISTER_MODEL_NAME} version ${latest_version}" git push origin ${new_tag} git checkout ${TARGET_BRANCH} git fetch --all git pull gto show --json > MODEL_REGISTRY.md git add . git commit -m "Update Registry" git push origin ${TARGET_BRANCH}