From ddcce2b56d58a28b2edf641db24919b79c4f7d0d Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Thu, 14 Sep 2023 23:28:00 +0100 Subject: [PATCH] cleaned up workflow --- .github/workflows/MLPipelinePostMerge.yml | 153 ++++++++++++------ .github/workflows/MLPipelinePullRequest.yml | 2 +- .../src/pipeline/src/startup_cleanup.py | 17 +- 3 files changed, 116 insertions(+), 56 deletions(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index 6ea18c5..19c6ce1 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -15,34 +15,88 @@ on: permissions: write-all jobs: - Use-Major-Label-on-Merge: + Register-Major-Model-Dev: if: ${{ (github.event.pull_request.merged == true) && (contains(github.event.pull_request.labels.*.name, 'major')) }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Test echoo - run: | - echo "Hello Major" + - uses: actions/checkout@v4 + with: + fetch-depth: 0 - Use-Minor-Label-on-Merge: + - name: Install packages to register model + run: | + pip install --upgrade pip + pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt + + - name: Register Model + run: | + REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}') + + git config user.name "Github-Bot" + git config user.email "Github-Bot@no-reply.com" + + # gto register test --repo https://github.com/Hestia-Homes/ML/ + # echo "chicken" >> test.md + + # gto register ${REGISTER_MODEL_NAME} --bump-major + # gto assign regression --stage dev + # gto show >> Model_Register.md + + Register-Minor-Model-Dev: if: ${{ (github.event.pull_request.merged == true) && (contains(github.event.pull_request.labels.*.name, 'minor')) }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Test echoo - run: | - echo "Hello Minor" + - uses: actions/checkout@v4 + with: + fetch-depth: 0 - Use-Patch-Label-on-Merge: + - name: Install packages to register model + run: | + pip install --upgrade pip + pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt + + - name: Register Model + run: | + REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}') + + git config user.name "Github-Bot" + git config user.email "Github-Bot@no-reply.com" + + # gto register test --repo https://github.com/Hestia-Homes/ML/ + # echo "chicken" >> test.md + + # gto register ${REGISTER_MODEL_NAME} --bump-minor + # gto assign regression --stage dev + # gto show >> Model_Register.md + + Register-Patch-Model-Dev: if: ${{ (github.event.pull_request.merged == true) && (contains(github.event.pull_request.labels.*.name, 'patch')) }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Test echoo - run: | - echo "Hello Patch" + - uses: actions/checkout@v4 + with: + fetch-depth: 0 - Promote-Model-To-Dev: + - name: Install packages to register model + run: | + pip install --upgrade pip + pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt + + - name: Register Model + run: | + REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}') + + git config user.name "Github-Bot" + git config user.email "Github-Bot@no-reply.com" + + # gto register test --repo https://github.com/Hestia-Homes/ML/ + # echo "chicken" >> test.md + + # gto register ${REGISTER_MODEL_NAME} --bump-major + # gto assign regression --stage dev + # gto show >> Model_Register.md + + Promote-Artefacts-To-Dev: if: github.event.pull_request.merged == true runs-on: ubuntu-latest @@ -70,55 +124,52 @@ jobs: dvc push -r dev - Register-New-Model-Dev: - if: github.event.pull_request.merged == true - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 + # Register-New-Model-Dev: + # if: github.event.pull_request.merged == true + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + # with: + # fetch-depth: 0 - - name: Install packages to register model - env: - AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} - run: | - pip install --upgrade pip - pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt + # - name: Install packages to register model + # env: + # AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} + # AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} + # run: | + # pip install --upgrade pip + # pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt - - name: Register Model - env: - AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} - run: | + # - name: Register Model + # env: + # AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} + # AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} + # run: | - # REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}') - REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}') + # # REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}') + # REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}') - git config user.name "Github-Bot" - git config user.email "Github-Bot@no-reply.com" + # git config user.name "Github-Bot" + # git config user.email "Github-Bot@no-reply.com" - # git tag model@v0.0.1 - # git push origin model@v0.0.1 + # # git tag model@v0.0.1 + # # git push origin model@v0.0.1 - # gto register test --repo https://github.com/Hestia-Homes/ML/ - # echo "chicken" >> test.md + # # gto register test --repo https://github.com/Hestia-Homes/ML/ + # # echo "chicken" >> test.md - # gto -v register ${REGISTER_MODEL_NAME} - # gto assign regression --stage dev - # gto show + # # gto -v register ${REGISTER_MODEL_NAME} + # # gto assign regression --stage dev + # # gto show Register-Prediction-Image-Dev: - needs: Promote-Model-To-Dev - # needs: [Promote-Model-To-Dev, Register-New-Model-Dev] WILL ADD BACK ONCE REGISTER WORKS + needs: Promote-Artefacts-To-Dev + # needs: [Promote-Artefacts-To-Dev, Register-New-Model-Dev] WILL ADD BACK ONCE REGISTER WORKS runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - name: Install packages to retrieve artifacts - env: - AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} run: | pip install --upgrade pip pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt diff --git a/.github/workflows/MLPipelinePullRequest.yml b/.github/workflows/MLPipelinePullRequest.yml index 9ac5602..e1ebd5d 100644 --- a/.github/workflows/MLPipelinePullRequest.yml +++ b/.github/workflows/MLPipelinePullRequest.yml @@ -13,7 +13,7 @@ permissions: write-all jobs: - No-Label: + Check-Label: runs-on: ubuntu-latest steps: - uses: yogevbd/enforce-label-action@2.1.0 diff --git a/modules/ml-pipeline/src/pipeline/src/startup_cleanup.py b/modules/ml-pipeline/src/pipeline/src/startup_cleanup.py index d30308c..af63291 100644 --- a/modules/ml-pipeline/src/pipeline/src/startup_cleanup.py +++ b/modules/ml-pipeline/src/pipeline/src/startup_cleanup.py @@ -15,6 +15,15 @@ def run_cleanup(artefacts_directory: str, metrics_directory: str) -> None: """ Remove the directory where artefacts are stored """ + + logger.info("---------------------") + logger.info(f"--- Run Clean up ---") + logger.info("---------------------") + + logger.info("-------------------------") + logger.info(f"--- Delete artefacts ---") + logger.info("-------------------------") + artefact_directory_path = Path(artefacts_directory) if artefact_directory_path.exists(): @@ -22,6 +31,10 @@ def run_cleanup(artefacts_directory: str, metrics_directory: str) -> None: logger.info(f"Removing the directory: {artefacts_directory}") shutil.rmtree(artefact_directory_path) + logger.info("-----------------------") + logger.info(f"--- Delete metrics ---") + logger.info("-----------------------") + metrics_directory_path = Path(metrics_directory) if metrics_directory_path.exists(): @@ -36,10 +49,6 @@ if __name__ == "__main__": logger.info(f"--- {__file__} - Start! ---") logger.info("----------------------------") - logger.info("---------------------") - logger.info(f"--- Run Clean up ---") - logger.info("---------------------") - run_cleanup( artefacts_directory=startup_cleanup_params["artefacts"], metrics_directory=startup_cleanup_params["metrics"],