cleaned up workflow

.github/workflows/MLPipelinePostMerge.yml

@@ -15,34 +15,88 @@ on:
 permissions: write-all
 
 jobs:
-  Use-Major-Label-on-Merge:
+  Register-Major-Model-Dev:
     if: ${{ (github.event.pull_request.merged == true) && (contains(github.event.pull_request.labels.*.name, 'major')) }}
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
-    - name: Test echoo
+      with:
-      run: |
+        fetch-depth: 0
-        echo "Hello Major"
 
-  Use-Minor-Label-on-Merge:
+    - name: Install packages to register model
+      run: |
+        pip install --upgrade pip
+        pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt
+
+    - name: Register Model
+      run: |
+        REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}')
+
+        git config user.name "Github-Bot"
+        git config user.email "Github-Bot@no-reply.com"
+
+        # gto register test --repo https://github.com/Hestia-Homes/ML/
+        # echo "chicken" >> test.md
+
+        # gto register ${REGISTER_MODEL_NAME} --bump-major
+        # gto assign regression --stage dev
+        # gto show >> Model_Register.md
+
+  Register-Minor-Model-Dev:
     if: ${{ (github.event.pull_request.merged == true) && (contains(github.event.pull_request.labels.*.name, 'minor')) }}
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
-    - name: Test echoo
+      with:
-      run: |
+        fetch-depth: 0
-        echo "Hello Minor"
 
-  Use-Patch-Label-on-Merge:
+    - name: Install packages to register model
+      run: |
+        pip install --upgrade pip
+        pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt
+
+    - name: Register Model
+      run: |
+        REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}')
+
+        git config user.name "Github-Bot"
+        git config user.email "Github-Bot@no-reply.com"
+
+        # gto register test --repo https://github.com/Hestia-Homes/ML/
+        # echo "chicken" >> test.md
+
+        # gto register ${REGISTER_MODEL_NAME} --bump-minor
+        # gto assign regression --stage dev
+        # gto show >> Model_Register.md
+
+  Register-Patch-Model-Dev:
     if: ${{ (github.event.pull_request.merged == true) && (contains(github.event.pull_request.labels.*.name, 'patch')) }}
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
-    - name: Test echoo
+      with:
-      run: |
+        fetch-depth: 0
-        echo "Hello Patch"
 
-  Promote-Model-To-Dev:
+    - name: Install packages to register model
+      run: |
+        pip install --upgrade pip
+        pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt
+
+    - name: Register Model
+      run: |
+        REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}')
+
+        git config user.name "Github-Bot"
+        git config user.email "Github-Bot@no-reply.com"
+
+        # gto register test --repo https://github.com/Hestia-Homes/ML/
+        # echo "chicken" >> test.md
+
+        # gto register ${REGISTER_MODEL_NAME} --bump-major
+        # gto assign regression --stage dev
+        # gto show >> Model_Register.md
+
+  Promote-Artefacts-To-Dev:
     if: github.event.pull_request.merged == true
     runs-on: ubuntu-latest
 
@@ -70,55 +124,52 @@ jobs:
         dvc push -r dev
 
 
-  Register-New-Model-Dev:
+  # Register-New-Model-Dev:
-    if: github.event.pull_request.merged == true
+  #   if: github.event.pull_request.merged == true
-    runs-on: ubuntu-latest
+  #   runs-on: ubuntu-latest
-    steps:
+  #   steps:
-    - uses: actions/checkout@v4
+  #   - uses: actions/checkout@v4
-      with:
+  #     with:
-        fetch-depth: 0
+  #       fetch-depth: 0
 
-    - name: Install packages to register model
+  #   - name: Install packages to register model
-      env:
+  #     env:
-        AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }}
+  #       AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }}
+  #       AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }}
-      run: |
+  #     run: |
-        pip install --upgrade pip
+  #       pip install --upgrade pip
-        pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt
+  #       pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt
 
-    - name: Register Model
+  #   - name: Register Model
-      env:
+  #     env:
-        AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }}
+  #       AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }}
+  #       AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }}
-      run: |
+  #     run: |
 
-        # REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}')
+  #       # REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}')
-        REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}')
+  #       REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}')
 
-        git config user.name "Github-Bot"
+  #       git config user.name "Github-Bot"
-        git config user.email "Github-Bot@no-reply.com"
+  #       git config user.email "Github-Bot@no-reply.com"
 
-        # git tag model@v0.0.1
+  #       # git tag model@v0.0.1
-        # git push origin model@v0.0.1
+  #       # git push origin model@v0.0.1
 
-        # gto register test --repo https://github.com/Hestia-Homes/ML/
+  #       # gto register test --repo https://github.com/Hestia-Homes/ML/
-        # echo "chicken" >> test.md
+  #       # echo "chicken" >> test.md
 
-        # gto -v register ${REGISTER_MODEL_NAME}
+  #       # gto -v register ${REGISTER_MODEL_NAME}
-        # gto assign regression --stage dev
+  #       # gto assign regression --stage dev
-        # gto show
+  #       # gto show
 
   Register-Prediction-Image-Dev:
-    needs: Promote-Model-To-Dev
+    needs: Promote-Artefacts-To-Dev
-    # needs: [Promote-Model-To-Dev, Register-New-Model-Dev] WILL ADD BACK ONCE REGISTER WORKS
+    # needs: [Promote-Artefacts-To-Dev, Register-New-Model-Dev] WILL ADD BACK ONCE REGISTER WORKS
     runs-on: ubuntu-latest
 
     steps:
     - uses: actions/checkout@v3
     - name: Install packages to retrieve artifacts
-      env:
-        AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }}
       run: |
         pip install --upgrade pip
         pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt

.github/workflows/MLPipelinePullRequest.yml

@@ -13,7 +13,7 @@ permissions: write-all
 
 jobs:
 
-  No-Label:
+  Check-Label:
     runs-on: ubuntu-latest
     steps:
     - uses: yogevbd/enforce-label-action@2.1.0

modules/ml-pipeline/src/pipeline/src/startup_cleanup.py

@@ -15,6 +15,15 @@ def run_cleanup(artefacts_directory: str, metrics_directory: str) -> None:
     """
     Remove the directory where artefacts are stored
     """
+
+    logger.info("---------------------")
+    logger.info(f"--- Run Clean up ---")
+    logger.info("---------------------")
+
+    logger.info("-------------------------")
+    logger.info(f"--- Delete artefacts ---")
+    logger.info("-------------------------")
+
     artefact_directory_path = Path(artefacts_directory)
 
     if artefact_directory_path.exists():
@@ -22,6 +31,10 @@ def run_cleanup(artefacts_directory: str, metrics_directory: str) -> None:
         logger.info(f"Removing the directory: {artefacts_directory}")
         shutil.rmtree(artefact_directory_path)
 
+    logger.info("-----------------------")
+    logger.info(f"--- Delete metrics ---")
+    logger.info("-----------------------")
+
     metrics_directory_path = Path(metrics_directory)
 
     if metrics_directory_path.exists():
@@ -36,10 +49,6 @@ if __name__ == "__main__":
     logger.info(f"--- {__file__} - Start! ---")
     logger.info("----------------------------")
 
-    logger.info("---------------------")
-    logger.info(f"--- Run Clean up ---")
-    logger.info("---------------------")
-
     run_cleanup(
         artefacts_directory=startup_cleanup_params["artefacts"],
         metrics_directory=startup_cleanup_params["metrics"],