mirror of
https://github.com/Hestia-Homes/ML.git
synced 2026-06-08 11:17:25 +00:00
244 lines
8.5 KiB
YAML
244 lines
8.5 KiB
YAML
name: Register the model for the given pipeline branch
|
|
|
|
# on:
|
|
# push:
|
|
# branches:
|
|
# - "model-**"
|
|
|
|
on:
|
|
pull_request:
|
|
types:
|
|
- closed
|
|
branches:
|
|
- "sap-dev"
|
|
- "heat-dev"
|
|
- "carbon-dev"
|
|
- "sap_starting-dev"
|
|
|
|
permissions: write-all
|
|
|
|
jobs:
|
|
Register-Major-Model-Dev:
|
|
if: ${{ (github.event.pull_request.merged == true) && (contains(github.event.pull_request.labels.*.name, 'major')) }}
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: Install packages to register model
|
|
run: |
|
|
pip install --upgrade pip
|
|
pip install -r modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt
|
|
|
|
- name: Register Model
|
|
run: |
|
|
REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}')
|
|
# REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}')
|
|
|
|
git config user.name "Github-Bot"
|
|
git config user.email "Github-Bot@no-reply.com"
|
|
|
|
latest_version=$(gto show ${REGISTER_MODEL_NAME}@latest --ref | awk -F"@v" '{print $2}') || false
|
|
if [ -z "${latest_version}" ]; then
|
|
increment_version="1.0.0"
|
|
else
|
|
increment_version=$(echo ${latest_version} | awk 'BEGIN {
|
|
FS="\\." # Set the field separator to a period
|
|
OFS="." # Set the output field separator to a period
|
|
}
|
|
{
|
|
major = $1 + 1 # Increment the major version
|
|
print major, "0", "0" # Print the new version
|
|
}')
|
|
fi
|
|
|
|
new_tag=${REGISTER_MODEL_NAME}@v${increment_version}
|
|
|
|
git tag -a ${new_tag} -m "Registering new Major Version"
|
|
git push origin ${new_tag}
|
|
|
|
gto show --json > MODEL_REGISTRY.md
|
|
git add .
|
|
git commit -m "Update Registry"
|
|
git push
|
|
|
|
Register-Minor-Model-Dev:
|
|
if: ${{ (github.event.pull_request.merged == true) && (contains(github.event.pull_request.labels.*.name, 'minor')) }}
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: Install packages to register model
|
|
run: |
|
|
pip install --upgrade pip
|
|
pip install -r modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt
|
|
|
|
- name: Register Model
|
|
run: |
|
|
REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}')
|
|
# REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}')
|
|
|
|
git config user.name "Github-Bot"
|
|
git config user.email "Github-Bot@no-reply.com"
|
|
|
|
latest_version=$(gto show ${REGISTER_MODEL_NAME}@latest --ref | awk -F"@v" '{print $2}')
|
|
if [ -z "${latest_version}" ]; then
|
|
increment_version="0.1.0"
|
|
else
|
|
increment_version=$(echo ${latest_version} | awk 'BEGIN {
|
|
FS="\\." # Set the field separator to a period
|
|
OFS="." # Set the output field separator to a period
|
|
}
|
|
{
|
|
minor = $2 + 1 # Increment the minor version
|
|
print $1, minor, "0" # Print the new version
|
|
}')
|
|
fi
|
|
|
|
new_tag=${REGISTER_MODEL_NAME}@v${increment_version}
|
|
|
|
git tag -a ${new_tag} -m "Registering new Minor Version"
|
|
git push origin ${new_tag}
|
|
|
|
gto show --json > MODEL_REGISTRY.md
|
|
git add .
|
|
git commit -m "Update Registry"
|
|
git push
|
|
|
|
Register-Patch-Model-Dev:
|
|
if: ${{ (github.event.pull_request.merged == true) && (contains(github.event.pull_request.labels.*.name, 'patch')) }}
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: Install packages to register model
|
|
run: |
|
|
pip install --upgrade pip
|
|
pip install -r modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt
|
|
|
|
- name: Register Model
|
|
run: |
|
|
REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}')
|
|
# REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}')
|
|
|
|
git config user.name "Github-Bot"
|
|
git config user.email "Github-Bot@no-reply.com"
|
|
|
|
latest_version=$(gto show ${REGISTER_MODEL_NAME}@latest --ref | awk -F"@v" '{print $2}')
|
|
if [ -z "${latest_version}" ]; then
|
|
increment_version="0.0.1"
|
|
else
|
|
increment_version=$(echo ${latest_version} | awk 'BEGIN {
|
|
FS="\\." # Set the field separator to a period
|
|
OFS="." # Set the output field separator to a period
|
|
}
|
|
{
|
|
patch = $3 + 1 # Increment the patch version
|
|
print $1, $2, patch # Print the new version
|
|
}')
|
|
fi
|
|
|
|
new_tag=${REGISTER_MODEL_NAME}@v${increment_version}
|
|
|
|
git tag -a ${new_tag} -m "Registering new Patch Version"
|
|
git push origin ${new_tag}
|
|
|
|
gto show --json > MODEL_REGISTRY.md
|
|
git add .
|
|
git commit -m "Update Registry"
|
|
git push
|
|
|
|
Promote-Artefacts-To-Dev:
|
|
if: github.event.pull_request.merged == true
|
|
runs-on: ubuntu-latest
|
|
|
|
steps:
|
|
- uses: actions/checkout@v3
|
|
- name: Install packages to retrieve artifacts
|
|
run: |
|
|
pip install --upgrade pip
|
|
pip install -r modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt
|
|
|
|
- name: Retrieve artifacts (dvc.lock)
|
|
env:
|
|
AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }}
|
|
AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }}
|
|
run: |
|
|
cd modules/ml-pipeline/src/pipeline
|
|
dvc pull -r experiments
|
|
|
|
- name: Push artifacts to Dev
|
|
env:
|
|
AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }}
|
|
AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }}
|
|
run: |
|
|
cd modules/ml-pipeline/src/pipeline
|
|
dvc push -r dev
|
|
|
|
Register-New-Model-Dev:
|
|
needs:
|
|
[
|
|
Register-Major-Model-Dev,
|
|
Register-Minor-Model-Dev,
|
|
Register-Patch-Model-Dev,
|
|
]
|
|
if: |
|
|
always() &&
|
|
(needs.Register-Major-Model-Dev.result == 'success' || needs.Register-Major-Model-Dev.result == 'skipped') &&
|
|
(needs.Register-Minor-Model-Dev.result == 'success' || needs.Register-Minor-Model-Dev.result == 'skipped') &&
|
|
(needs.Register-Patch-Model-Dev.result == 'success' || needs.Register-Patch-Model-Dev.result == 'skipped')
|
|
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v3
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: Install packages to register model
|
|
run: |
|
|
pip install --upgrade pip
|
|
pip install -r modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt
|
|
|
|
- name: Register Model
|
|
env:
|
|
TARGET_BRANCH: ${{ github.base_ref }}
|
|
run: |
|
|
|
|
REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}')
|
|
# REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}')
|
|
|
|
git config user.name "Github-Bot"
|
|
git config user.email "Github-Bot@no-reply.com"
|
|
|
|
latest_dev_version=$(gto history ${REGISTER_MODEL_NAME} --asc --plain | awk '{print $NF}' | awk '/dev/' | awk 'END {print}')
|
|
if [ -z "${latest_dev_version}" ]; then
|
|
increment_version="1"
|
|
else
|
|
increment_version=$(echo ${latest_dev_version} | awk '{print $NF}' | awk -F"#" '{print $3}' | awk '{$1++; print}')
|
|
fi
|
|
|
|
new_tag=${REGISTER_MODEL_NAME}#dev#${increment_version}
|
|
latest_version=$(gto show ${REGISTER_MODEL_NAME}@latest --ref | awk -F"@" '{print $2}')
|
|
|
|
echo ${new_tag}
|
|
|
|
commit_hash=$(gto history ${REGISTER_MODEL_NAME} --asc --plain | awk "/${latest_version}/" | awk '{print $(NF-1)}')
|
|
git checkout ${commit_hash}
|
|
|
|
# git pull #Get new model registry md file changes
|
|
git tag -a ${new_tag} -m "Assigning stage dev to artifact ${REGISTER_MODEL_NAME} version ${latest_version}"
|
|
git push origin ${new_tag}
|
|
|
|
git checkout ${TARGET_BRANCH}
|
|
git fetch --all
|
|
git pull
|
|
|
|
gto show --json > MODEL_REGISTRY.md
|
|
git add .
|
|
git commit -m "Update Registry"
|
|
git push origin ${TARGET_BRANCH}
|