mirror of
https://github.com/Hestia-Homes/ML.git
synced 2026-06-08 11:17:25 +00:00
cleaned up workflow
This commit is contained in:
Michael Duong
parent
a576b3e024
commit
ddcce2b56d
3 changed files with 116 additions and 56 deletions
153
.github/workflows/MLPipelinePostMerge.yml
vendored
153
.github/workflows/MLPipelinePostMerge.yml
vendored
|
|
@ -15,34 +15,88 @@ on:
|
|||
permissions: write-all
|
||||
|
||||
jobs:
|
||||
Use-Major-Label-on-Merge:
|
||||
Register-Major-Model-Dev:
|
||||
if: ${{ (github.event.pull_request.merged == true) && (contains(github.event.pull_request.labels.*.name, 'major')) }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Test echoo
|
||||
run: |
|
||||
echo "Hello Major"
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
Use-Minor-Label-on-Merge:
|
||||
- name: Install packages to register model
|
||||
run: |
|
||||
pip install --upgrade pip
|
||||
pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt
|
||||
|
||||
- name: Register Model
|
||||
run: |
|
||||
REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}')
|
||||
|
||||
git config user.name "Github-Bot"
|
||||
git config user.email "Github-Bot@no-reply.com"
|
||||
|
||||
# gto register test --repo https://github.com/Hestia-Homes/ML/
|
||||
# echo "chicken" >> test.md
|
||||
|
||||
# gto register ${REGISTER_MODEL_NAME} --bump-major
|
||||
# gto assign regression --stage dev
|
||||
# gto show >> Model_Register.md
|
||||
|
||||
Register-Minor-Model-Dev:
|
||||
if: ${{ (github.event.pull_request.merged == true) && (contains(github.event.pull_request.labels.*.name, 'minor')) }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Test echoo
|
||||
run: |
|
||||
echo "Hello Minor"
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
Use-Patch-Label-on-Merge:
|
||||
- name: Install packages to register model
|
||||
run: |
|
||||
pip install --upgrade pip
|
||||
pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt
|
||||
|
||||
- name: Register Model
|
||||
run: |
|
||||
REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}')
|
||||
|
||||
git config user.name "Github-Bot"
|
||||
git config user.email "Github-Bot@no-reply.com"
|
||||
|
||||
# gto register test --repo https://github.com/Hestia-Homes/ML/
|
||||
# echo "chicken" >> test.md
|
||||
|
||||
# gto register ${REGISTER_MODEL_NAME} --bump-minor
|
||||
# gto assign regression --stage dev
|
||||
# gto show >> Model_Register.md
|
||||
|
||||
Register-Patch-Model-Dev:
|
||||
if: ${{ (github.event.pull_request.merged == true) && (contains(github.event.pull_request.labels.*.name, 'patch')) }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Test echoo
|
||||
run: |
|
||||
echo "Hello Patch"
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
Promote-Model-To-Dev:
|
||||
- name: Install packages to register model
|
||||
run: |
|
||||
pip install --upgrade pip
|
||||
pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt
|
||||
|
||||
- name: Register Model
|
||||
run: |
|
||||
REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}')
|
||||
|
||||
git config user.name "Github-Bot"
|
||||
git config user.email "Github-Bot@no-reply.com"
|
||||
|
||||
# gto register test --repo https://github.com/Hestia-Homes/ML/
|
||||
# echo "chicken" >> test.md
|
||||
|
||||
# gto register ${REGISTER_MODEL_NAME} --bump-major
|
||||
# gto assign regression --stage dev
|
||||
# gto show >> Model_Register.md
|
||||
|
||||
Promote-Artefacts-To-Dev:
|
||||
if: github.event.pull_request.merged == true
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
|
|
@ -70,55 +124,52 @@ jobs:
|
|||
dvc push -r dev
|
||||
|
||||
|
||||
Register-New-Model-Dev:
|
||||
if: github.event.pull_request.merged == true
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
# Register-New-Model-Dev:
|
||||
# if: github.event.pull_request.merged == true
|
||||
# runs-on: ubuntu-latest
|
||||
# steps:
|
||||
# - uses: actions/checkout@v4
|
||||
# with:
|
||||
# fetch-depth: 0
|
||||
|
||||
- name: Install packages to register model
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }}
|
||||
run: |
|
||||
pip install --upgrade pip
|
||||
pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt
|
||||
# - name: Install packages to register model
|
||||
# env:
|
||||
# AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }}
|
||||
# AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }}
|
||||
# run: |
|
||||
# pip install --upgrade pip
|
||||
# pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt
|
||||
|
||||
- name: Register Model
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }}
|
||||
run: |
|
||||
# - name: Register Model
|
||||
# env:
|
||||
# AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }}
|
||||
# AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }}
|
||||
# run: |
|
||||
|
||||
# REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}')
|
||||
REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}')
|
||||
# # REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}')
|
||||
# REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}')
|
||||
|
||||
git config user.name "Github-Bot"
|
||||
git config user.email "Github-Bot@no-reply.com"
|
||||
# git config user.name "Github-Bot"
|
||||
# git config user.email "Github-Bot@no-reply.com"
|
||||
|
||||
# git tag model@v0.0.1
|
||||
# git push origin model@v0.0.1
|
||||
# # git tag model@v0.0.1
|
||||
# # git push origin model@v0.0.1
|
||||
|
||||
# gto register test --repo https://github.com/Hestia-Homes/ML/
|
||||
# echo "chicken" >> test.md
|
||||
# # gto register test --repo https://github.com/Hestia-Homes/ML/
|
||||
# # echo "chicken" >> test.md
|
||||
|
||||
# gto -v register ${REGISTER_MODEL_NAME}
|
||||
# gto assign regression --stage dev
|
||||
# gto show
|
||||
# # gto -v register ${REGISTER_MODEL_NAME}
|
||||
# # gto assign regression --stage dev
|
||||
# # gto show
|
||||
|
||||
Register-Prediction-Image-Dev:
|
||||
needs: Promote-Model-To-Dev
|
||||
# needs: [Promote-Model-To-Dev, Register-New-Model-Dev] WILL ADD BACK ONCE REGISTER WORKS
|
||||
needs: Promote-Artefacts-To-Dev
|
||||
# needs: [Promote-Artefacts-To-Dev, Register-New-Model-Dev] WILL ADD BACK ONCE REGISTER WORKS
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Install packages to retrieve artifacts
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }}
|
||||
run: |
|
||||
pip install --upgrade pip
|
||||
pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt
|
||||
|
|
|
|||
2
.github/workflows/MLPipelinePullRequest.yml
vendored
2
.github/workflows/MLPipelinePullRequest.yml
vendored
|
|
@ -13,7 +13,7 @@ permissions: write-all
|
|||
|
||||
jobs:
|
||||
|
||||
No-Label:
|
||||
Check-Label:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: yogevbd/enforce-label-action@2.1.0
|
||||
|
|
|
|||
|
|
@ -15,6 +15,15 @@ def run_cleanup(artefacts_directory: str, metrics_directory: str) -> None:
|
|||
"""
|
||||
Remove the directory where artefacts are stored
|
||||
"""
|
||||
|
||||
logger.info("---------------------")
|
||||
logger.info(f"--- Run Clean up ---")
|
||||
logger.info("---------------------")
|
||||
|
||||
logger.info("-------------------------")
|
||||
logger.info(f"--- Delete artefacts ---")
|
||||
logger.info("-------------------------")
|
||||
|
||||
artefact_directory_path = Path(artefacts_directory)
|
||||
|
||||
if artefact_directory_path.exists():
|
||||
|
|
@ -22,6 +31,10 @@ def run_cleanup(artefacts_directory: str, metrics_directory: str) -> None:
|
|||
logger.info(f"Removing the directory: {artefacts_directory}")
|
||||
shutil.rmtree(artefact_directory_path)
|
||||
|
||||
logger.info("-----------------------")
|
||||
logger.info(f"--- Delete metrics ---")
|
||||
logger.info("-----------------------")
|
||||
|
||||
metrics_directory_path = Path(metrics_directory)
|
||||
|
||||
if metrics_directory_path.exists():
|
||||
|
|
@ -36,10 +49,6 @@ if __name__ == "__main__":
|
|||
logger.info(f"--- {__file__} - Start! ---")
|
||||
logger.info("----------------------------")
|
||||
|
||||
logger.info("---------------------")
|
||||
logger.info(f"--- Run Clean up ---")
|
||||
logger.info("---------------------")
|
||||
|
||||
run_cleanup(
|
||||
artefacts_directory=startup_cleanup_params["artefacts"],
|
||||
metrics_directory=startup_cleanup_params["metrics"],
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue