From 90bf4ae2c10f233a58931ab864bbffa5d8085aa0 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 12 Sep 2023 15:52:22 +0100 Subject: [PATCH 01/41] test debug --- .github/workflows/MLPipelinePostMerge.yml | 141 +++++++++--------- modules/ml-pipeline/.gitignore | 2 + .../ml-pipeline/src/pipeline/src/.DS_Store | Bin 6148 -> 6148 bytes 3 files changed, 74 insertions(+), 69 deletions(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index a83a6b4..3eaea50 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -1,47 +1,47 @@ name: Register the model for the given pipeline branch -# on: -# push: -# branches: -# - "model-**" - on: - pull_request: - types: - - closed + push: branches: - - "master" + - "model-**" + +# on: +# pull_request: +# types: +# - closed +# branches: +# - "master" permissions: write-all jobs: - Promote-Model-To-Dev: - if: github.event.pull_request.merged == true - runs-on: ubuntu-latest + # Promote-Model-To-Dev: + # if: github.event.pull_request.merged == true + # runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Install packages to retrieve artifacts - run: | - pip install --upgrade pip - pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt + # steps: + # - uses: actions/checkout@v3 + # - name: Install packages to retrieve artifacts + # run: | + # pip install --upgrade pip + # pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt - - name: Retrieve artifacts (dvc.lock) - env: - AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} - run: | - cd modules/ml-pipeline/src/pipeline/src - dvc pull -r experiments + # - name: Retrieve artifacts (dvc.lock) + # env: + # AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} + # AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} + # run: | + # cd modules/ml-pipeline/src/pipeline/src + # dvc pull -r experiments - - name: Push artifacts to Dev - env: - AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} - run: | - cd modules/ml-pipeline/src/pipeline/src - dvc push -r dev + # - name: Push artifacts to Dev + # env: + # AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} + # AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} + # run: | + # cd modules/ml-pipeline/src/pipeline/src + # dvc push -r dev Register-New-Model-Dev: @@ -62,6 +62,7 @@ jobs: - name: Register Model env: + ACTIONS_STEP_DEBUG: true REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -69,50 +70,52 @@ jobs: # REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}') git config user.name "Github-Bot" git config user.email "Github-Bot@no-reply.com" + git branch + git tag gto register ${REGISTER_MODEL_NAME} gto assign ${REGISTER_MODEL_NAME} --stage dev gto show - Register-Prediction-Image-Dev: - needs: Promote-Model-To-Dev - # needs: [Promote-Model-To-Dev, Register-New-Model-Dev] WILL ADD BACK ONCE REGISTER WORKS - runs-on: ubuntu-latest + # Register-Prediction-Image-Dev: + # needs: Promote-Model-To-Dev + # # needs: [Promote-Model-To-Dev, Register-New-Model-Dev] WILL ADD BACK ONCE REGISTER WORKS + # runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Install packages to retrieve artifacts - env: - AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} - run: | - pip install --upgrade pip - pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt + # steps: + # - uses: actions/checkout@v3 + # - name: Install packages to retrieve artifacts + # env: + # AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} + # AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} + # run: | + # pip install --upgrade pip + # pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt - - name: Retrieve artifacts (dvc.lock) - env: - AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} - run: | - cd modules/ml-pipeline/src/pipeline/src - dvc pull -r dev + # - name: Retrieve artifacts (dvc.lock) + # env: + # AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} + # AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} + # run: | + # cd modules/ml-pipeline/src/pipeline/src + # dvc pull -r dev - - name: Build Prediction docker image (TODO - NEED LAMBDA IMAGE, need to add version from gto registry) - run: | - cd modules/ml-pipeline/src/pipeline/ - REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}') - docker build . --file Prediction.Dockerfile --tag ${REGISTER_MODEL_NAME} + # - name: Build Prediction docker image (TODO - NEED LAMBDA IMAGE, need to add version from gto registry) + # run: | + # cd modules/ml-pipeline/src/pipeline/ + # REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}') + # docker build . --file Prediction.Dockerfile --tag ${REGISTER_MODEL_NAME} - - name: ECR Login - Dev - env: - AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} - run: | - echo "LOGIN TO ECR" + # - name: ECR Login - Dev + # env: + # AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} + # AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} + # run: | + # echo "LOGIN TO ECR" - - name: Push Prediction image to ECR - Dev - env: - AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} - run: | - echo "PUSH TO ECR" + # - name: Push Prediction image to ECR - Dev + # env: + # AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} + # AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} + # run: | + # echo "PUSH TO ECR" diff --git a/modules/ml-pipeline/.gitignore b/modules/ml-pipeline/.gitignore index cd87845..63900db 100644 --- a/modules/ml-pipeline/.gitignore +++ b/modules/ml-pipeline/.gitignore @@ -1,2 +1,4 @@ .dev_env/ __pycache__/ +.DS_Store +.vscode/ diff --git a/modules/ml-pipeline/src/pipeline/src/.DS_Store b/modules/ml-pipeline/src/pipeline/src/.DS_Store index 4dff51ac089c8d5519e55cb38af42fcbc1594bf7..5008ddfcf53c02e82d7eee2e57c38e5672ef89f6 100644 GIT binary patch delta 70 zcmZoMXfc=|#>AjHu~2NHo+1YW5HK<@2yA9#Vq@DZz_f~SGdl-A2T%b}B`mF;Q%yo}wrd0|Nsi1A_oVQh9MfQcivnkiTPM;_`Zs1Pen7Ln1>7 zLn5+dQbA5;afyM!HAW_87FITP4t5T1j@aOg{PN(E#FEltr^KRY5HBP@KPL&sPD~2R zOf8QW5OL1WD@n}EODzIx$V^EEDv1ft%uC5Hcgio#ODP8H4bIF0%W!aVaK;NrR99P= z8|o;S8k^VZC{$Y-8R#gO7@O7Ba&m|&>strKXXoVR<#z!c3!f Date: Tue, 12 Sep 2023 15:52:54 +0100 Subject: [PATCH 02/41] test debug --- .github/workflows/MLPipelinePostMerge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index 3eaea50..a5e3ef2 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -45,7 +45,7 @@ jobs: Register-New-Model-Dev: - if: github.event.pull_request.merged == true + # if: github.event.pull_request.merged == true runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 From 9837f865501838465fad2ddd32db3df7a6c896da Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 12 Sep 2023 15:54:09 +0100 Subject: [PATCH 03/41] test debug --- .github/workflows/MLPipelinePostMerge.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index a5e3ef2..740c64e 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -66,8 +66,8 @@ jobs: REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}') - # REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}') + # REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}') + REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}') git config user.name "Github-Bot" git config user.email "Github-Bot@no-reply.com" git branch From 16828cd8707e844c6483f93531ec262d7f637f7a Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 12 Sep 2023 15:55:29 +0100 Subject: [PATCH 04/41] test debug --- .github/workflows/MLPipelinePostMerge.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index 740c64e..4ee56b5 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -72,6 +72,7 @@ jobs: git config user.email "Github-Bot@no-reply.com" git branch git tag + git remote show origin gto register ${REGISTER_MODEL_NAME} gto assign ${REGISTER_MODEL_NAME} --stage dev From 1e0fd4f5727397ffa82a6ffc9766e8b43653497d Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 12 Sep 2023 15:59:21 +0100 Subject: [PATCH 05/41] test debug --- .github/workflows/MLPipelinePostMerge.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index 4ee56b5..035ffcd 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -60,6 +60,9 @@ jobs: pip install --upgrade pip pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt + - uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: Register Model env: ACTIONS_STEP_DEBUG: true From 605ec6978197ce7133ac97a2a1703c46e56f8699 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 12 Sep 2023 16:03:19 +0100 Subject: [PATCH 06/41] test debug --- .github/workflows/MLPipelinePostMerge.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index 035ffcd..7370060 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -73,6 +73,14 @@ jobs: REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}') git config user.name "Github-Bot" git config user.email "Github-Bot@no-reply.com" + + echo "chicken" >> test.md + + git add . + git commit -m "test" + + git push + git branch git tag git remote show origin From 3d693cb73540f4f3440b1695efaffe4d9b9bd47e Mon Sep 17 00:00:00 2001 From: Github-Bot Date: Tue, 12 Sep 2023 15:04:16 +0000 Subject: [PATCH 07/41] test --- test.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 test.md diff --git a/test.md b/test.md new file mode 100644 index 0000000..4768c08 --- /dev/null +++ b/test.md @@ -0,0 +1 @@ +chicken From a0168a113e75fc45195f6974618b1b0c334fb7ee Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 12 Sep 2023 16:07:57 +0100 Subject: [PATCH 08/41] test debug --- .github/workflows/MLPipelinePostMerge.yml | 13 +++++-------- test.md | 1 - 2 files changed, 5 insertions(+), 9 deletions(-) delete mode 100644 test.md diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index 7370060..9f3e100 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -60,9 +60,6 @@ jobs: pip install --upgrade pip pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - name: Register Model env: ACTIONS_STEP_DEBUG: true @@ -73,13 +70,13 @@ jobs: REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}') git config user.name "Github-Bot" git config user.email "Github-Bot@no-reply.com" + git config --local remote.origin.push master:master + # echo "chicken" >> test.md - echo "chicken" >> test.md + # git add . + # git commit -m "test" - git add . - git commit -m "test" - - git push + # git push git branch git tag diff --git a/test.md b/test.md deleted file mode 100644 index 4768c08..0000000 --- a/test.md +++ /dev/null @@ -1 +0,0 @@ -chicken From bf781465550d762726a45886fdb923f09df7fe8c Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 12 Sep 2023 16:12:47 +0100 Subject: [PATCH 09/41] test debug --- .github/workflows/MLPipelinePostMerge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index 9f3e100..3875598 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -72,7 +72,7 @@ jobs: git config user.email "Github-Bot@no-reply.com" git config --local remote.origin.push master:master # echo "chicken" >> test.md - + git pull origin master # git add . # git commit -m "test" From e2de1d0ef192a2c8c04a045978d797eeed4f93a6 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 12 Sep 2023 16:14:28 +0100 Subject: [PATCH 10/41] test debug --- .github/workflows/MLPipelinePostMerge.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index 3875598..7f16ecf 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -72,7 +72,9 @@ jobs: git config user.email "Github-Bot@no-reply.com" git config --local remote.origin.push master:master # echo "chicken" >> test.md - git pull origin master + + git fetch --prune + # git pull origin master # git add . # git commit -m "test" From 95f0ddad11ee07ac7436a73218031e9e6de66e6b Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 12 Sep 2023 16:15:48 +0100 Subject: [PATCH 11/41] test debug --- .github/workflows/MLPipelinePostMerge.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index 7f16ecf..dec05fd 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -73,7 +73,8 @@ jobs: git config --local remote.origin.push master:master # echo "chicken" >> test.md - git fetch --prune + + # git fetch --prune # git pull origin master # git add . # git commit -m "test" @@ -83,6 +84,7 @@ jobs: git branch git tag git remote show origin + git status gto register ${REGISTER_MODEL_NAME} gto assign ${REGISTER_MODEL_NAME} --stage dev From 1165d042ceb8b7a5a555266111a936642c22b7b6 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 12 Sep 2023 16:19:09 +0100 Subject: [PATCH 12/41] test debug --- .github/workflows/MLPipelinePostMerge.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index dec05fd..0b5f616 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -82,7 +82,9 @@ jobs: # git push git branch - git tag + git tag model@v0.0.1 + git push origin model@v0.0.1 + git remote show origin git status From 13b10b3946219b758177020b2f33ca5e69662e31 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 12 Sep 2023 16:23:55 +0100 Subject: [PATCH 13/41] test debug --- .github/workflows/MLPipelinePostMerge.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index 0b5f616..21b4366 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -81,14 +81,14 @@ jobs: # git push - git branch - git tag model@v0.0.1 - git push origin model@v0.0.1 + # git branch + # git tag model@v0.0.1 + # git push origin model@v0.0.1 git remote show origin git status - gto register ${REGISTER_MODEL_NAME} + gto register ${REGISTER_MODEL_NAME} --repo https://github.com/Hestia-Homes/ML gto assign ${REGISTER_MODEL_NAME} --stage dev gto show From 23d54ef6af662b57302c1288f20dcf3091eec4a3 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 12 Sep 2023 16:26:12 +0100 Subject: [PATCH 14/41] test debug --- .github/workflows/MLPipelinePostMerge.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index 21b4366..2ac2f74 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -64,6 +64,7 @@ jobs: env: ACTIONS_STEP_DEBUG: true REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} + token: ${{ secrets.PAT }} run: | # REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}') From e12a68afcb9fc3e0b9012174da530e47f65f5060 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 12 Sep 2023 16:30:11 +0100 Subject: [PATCH 15/41] test debug --- .github/workflows/MLPipelinePostMerge.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index 2ac2f74..eb4e9eb 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -48,7 +48,7 @@ jobs: # if: github.event.pull_request.merged == true runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v3 with: fetch-depth: 0 @@ -63,7 +63,7 @@ jobs: - name: Register Model env: ACTIONS_STEP_DEBUG: true - REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} token: ${{ secrets.PAT }} run: | From 06190116a2f2111cfc54dce746a4c2c0add94239 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 12 Sep 2023 16:31:43 +0100 Subject: [PATCH 16/41] test debug --- .github/workflows/MLPipelinePostMerge.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index eb4e9eb..264a4f4 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -74,7 +74,6 @@ jobs: git config --local remote.origin.push master:master # echo "chicken" >> test.md - # git fetch --prune # git pull origin master # git add . @@ -89,7 +88,7 @@ jobs: git remote show origin git status - gto register ${REGISTER_MODEL_NAME} --repo https://github.com/Hestia-Homes/ML + gto register ${REGISTER_MODEL_NAME} --repo remote gto assign ${REGISTER_MODEL_NAME} --stage dev gto show From b09096cbfb4b57bbab48f45d2ed436264a3cca8a Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 12 Sep 2023 16:32:59 +0100 Subject: [PATCH 17/41] test debug --- .github/workflows/MLPipelinePostMerge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index 264a4f4..ca7b95a 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -88,7 +88,7 @@ jobs: git remote show origin git status - gto register ${REGISTER_MODEL_NAME} --repo remote + gto register ${REGISTER_MODEL_NAME} --repo . gto assign ${REGISTER_MODEL_NAME} --stage dev gto show From 6af855fd739979253f9b828df2d865f18fd59222 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 12 Sep 2023 16:37:51 +0100 Subject: [PATCH 18/41] test debug --- .github/workflows/MLPipelinePostMerge.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index ca7b95a..763acdb 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -85,8 +85,9 @@ jobs: # git tag model@v0.0.1 # git push origin model@v0.0.1 - git remote show origin - git status + # git remote show origin + # git status + git fetch --all gto register ${REGISTER_MODEL_NAME} --repo . gto assign ${REGISTER_MODEL_NAME} --stage dev From c25bbac1ef6271d5a8da30556d6dab79d8b859e0 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 12 Sep 2023 16:44:44 +0100 Subject: [PATCH 19/41] test debug --- .github/workflows/MLPipelinePostMerge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index 763acdb..ed10aa0 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -87,7 +87,7 @@ jobs: # git remote show origin # git status - git fetch --all + git push origin regression@v0.0.1 gto register ${REGISTER_MODEL_NAME} --repo . gto assign ${REGISTER_MODEL_NAME} --stage dev From f5ee080d044767724d14121991c678154c9e13df Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 12 Sep 2023 16:59:02 +0100 Subject: [PATCH 20/41] test debug --- .github/workflows/MLPipelinePostMerge.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index ed10aa0..dd548ac 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -61,10 +61,10 @@ jobs: pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt - name: Register Model - env: - ACTIONS_STEP_DEBUG: true - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - token: ${{ secrets.PAT }} + # env: + # ACTIONS_STEP_DEBUG: true + # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # token: ${{ secrets.PAT }} run: | # REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}') @@ -87,9 +87,9 @@ jobs: # git remote show origin # git status - git push origin regression@v0.0.1 + # git push origin regression@v0.0.1 - gto register ${REGISTER_MODEL_NAME} --repo . + gto register ${REGISTER_MODEL_NAME} --repo https://github.com/Hestia-Homes/ML gto assign ${REGISTER_MODEL_NAME} --stage dev gto show From 3ed2aef412d05871688dba264f22de66172e084d Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 12 Sep 2023 17:03:11 +0100 Subject: [PATCH 21/41] test debug --- .github/workflows/MLPipelinePostMerge.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index dd548ac..072e0ba 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -84,6 +84,7 @@ jobs: # git branch # git tag model@v0.0.1 # git push origin model@v0.0.1 + git checkout master # git remote show origin # git status From a3b8de7b69555e4e22f1ca18201d6cd62d991459 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 12 Sep 2023 17:04:31 +0100 Subject: [PATCH 22/41] test debug --- .github/workflows/MLPipelinePostMerge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index 072e0ba..57a2c71 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -90,7 +90,7 @@ jobs: # git status # git push origin regression@v0.0.1 - gto register ${REGISTER_MODEL_NAME} --repo https://github.com/Hestia-Homes/ML + gto register ${REGISTER_MODEL_NAME} gto assign ${REGISTER_MODEL_NAME} --stage dev gto show From 8b86449e3fc346749b99ccad763f6ff6898ea1cd Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 12 Sep 2023 17:09:47 +0100 Subject: [PATCH 23/41] test debug --- .github/workflows/MLPipelinePostMerge.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index 57a2c71..aa406b0 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -71,7 +71,6 @@ jobs: REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}') git config user.name "Github-Bot" git config user.email "Github-Bot@no-reply.com" - git config --local remote.origin.push master:master # echo "chicken" >> test.md # git fetch --prune @@ -84,13 +83,13 @@ jobs: # git branch # git tag model@v0.0.1 # git push origin model@v0.0.1 - git checkout master + # git checkout master # git remote show origin # git status # git push origin regression@v0.0.1 - gto register ${REGISTER_MODEL_NAME} + gto register ${REGISTER_MODEL_NAME} --push gto assign ${REGISTER_MODEL_NAME} --stage dev gto show From feceb70f4ae6fc4165a7cff27ada6ef0faea11f5 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 12 Sep 2023 17:11:29 +0100 Subject: [PATCH 24/41] test debug --- .github/workflows/MLPipelinePostMerge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index aa406b0..5743800 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -89,7 +89,7 @@ jobs: # git status # git push origin regression@v0.0.1 - gto register ${REGISTER_MODEL_NAME} --push + gto register ${REGISTER_MODEL_NAME} --force gto assign ${REGISTER_MODEL_NAME} --stage dev gto show From 4454d78c6b87aee417c0a7ce7c2505308ffb922b Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 12 Sep 2023 17:12:45 +0100 Subject: [PATCH 25/41] test debug --- .github/workflows/MLPipelinePostMerge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index 5743800..9dbe2a4 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -81,7 +81,7 @@ jobs: # git push # git branch - # git tag model@v0.0.1 + git tag model@v0.0.1 # git push origin model@v0.0.1 # git checkout master From 7564ab6bbfe0dcabfbf528741c5907576049b314 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 12 Sep 2023 17:14:12 +0100 Subject: [PATCH 26/41] test debug --- .github/workflows/MLPipelinePostMerge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index 9dbe2a4..224d1fc 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -81,7 +81,7 @@ jobs: # git push # git branch - git tag model@v0.0.1 + git tag model@v0.0.2 # git push origin model@v0.0.1 # git checkout master From e7fb51e62a726e2754f18ebf75e03f6ba844fd18 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 12 Sep 2023 17:15:44 +0100 Subject: [PATCH 27/41] test debug --- .github/workflows/MLPipelinePostMerge.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index 224d1fc..45c61bb 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -81,8 +81,8 @@ jobs: # git push # git branch - git tag model@v0.0.2 - # git push origin model@v0.0.1 + git tag model@v0.0.3 + git push origin model@v0.0.3 # git checkout master # git remote show origin From dd072c52818ef5af3d4bc7b97d2d59f01a24a505 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 12 Sep 2023 17:21:55 +0100 Subject: [PATCH 28/41] test debug --- .github/workflows/MLPipelinePostMerge.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index 45c61bb..e188236 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -81,8 +81,8 @@ jobs: # git push # git branch - git tag model@v0.0.3 - git push origin model@v0.0.3 + git tag model@v0.0.1 + git push origin model@v0.0.1 # git checkout master # git remote show origin From d671de76e010f816017bf42c68df78d146ec2be0 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 12 Sep 2023 18:35:39 +0100 Subject: [PATCH 29/41] test basic command --- .github/workflows/MLPipelinePostMerge.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index e188236..76e2ad5 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -69,6 +69,8 @@ jobs: # REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}') REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}') + gto register test + git config user.name "Github-Bot" git config user.email "Github-Bot@no-reply.com" # echo "chicken" >> test.md @@ -81,17 +83,17 @@ jobs: # git push # git branch - git tag model@v0.0.1 - git push origin model@v0.0.1 + # git tag model@v0.0.1 + # git push origin model@v0.0.1 # git checkout master # git remote show origin # git status # git push origin regression@v0.0.1 - gto register ${REGISTER_MODEL_NAME} --force - gto assign ${REGISTER_MODEL_NAME} --stage dev - gto show + # gto register ${REGISTER_MODEL_NAME} --force + # gto assign ${REGISTER_MODEL_NAME} --stage dev + # gto show # Register-Prediction-Image-Dev: # needs: Promote-Model-To-Dev From a2bf6b937a84f4268415dda2f9280e64ac08aeed Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 12 Sep 2023 18:37:16 +0100 Subject: [PATCH 30/41] test basic command --- .github/workflows/MLPipelinePostMerge.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index 76e2ad5..b0edfdf 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -71,8 +71,13 @@ jobs: REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}') gto register test - git config user.name "Github-Bot" - git config user.email "Github-Bot@no-reply.com" + git config user.name "Michael Duong" + git config user.email "michael123ster@gmail.com" + + # git config user.name "Github-Bot" + # git config user.email "Github-Bot@no-reply.com" + + gto register test # echo "chicken" >> test.md # git fetch --prune From 8006aff7cc4ff4fcb7911dddec5505914cf02b78 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 12 Sep 2023 18:38:21 +0100 Subject: [PATCH 31/41] test basic command --- .github/workflows/MLPipelinePostMerge.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index b0edfdf..9f9fd97 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -69,7 +69,6 @@ jobs: # REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}') REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}') - gto register test git config user.name "Michael Duong" git config user.email "michael123ster@gmail.com" From 0dc27801a56cc839dcc4fa850aea49efa2c243cb Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 12 Sep 2023 18:40:13 +0100 Subject: [PATCH 32/41] test basic command --- .github/workflows/MLPipelinePostMerge.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index 9f9fd97..a9b0d95 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -70,13 +70,10 @@ jobs: # REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}') REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}') - git config user.name "Michael Duong" - git config user.email "michael123ster@gmail.com" + git config user.name "Github-Bot" + git config user.email "Github-Bot@no-reply.com" - # git config user.name "Github-Bot" - # git config user.email "Github-Bot@no-reply.com" - - gto register test + gto register test --force --push # echo "chicken" >> test.md # git fetch --prune From 4f8c3df646e515294d999e5dd34e1b729cfbc7f0 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 12 Sep 2023 18:57:10 +0100 Subject: [PATCH 33/41] try ssh key --- .github/workflows/MLPipelinePostMerge.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index a9b0d95..e3824cd 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -61,7 +61,8 @@ jobs: pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt - name: Register Model - # env: + env: + ACCESS_TOKEN: $ {{ secrets.SSH_TOKEN }} # ACTIONS_STEP_DEBUG: true # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # token: ${{ secrets.PAT }} @@ -73,7 +74,7 @@ jobs: git config user.name "Github-Bot" git config user.email "Github-Bot@no-reply.com" - gto register test --force --push + gto register test --repo https://github.com/Hestia-Homes/ML/ # echo "chicken" >> test.md # git fetch --prune From e6a66ee4d8dfe0fc707b22afba0ce49bda513569 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 12 Sep 2023 22:30:46 +0100 Subject: [PATCH 34/41] Look at refspec --- .github/workflows/MLPipelinePostMerge.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index e3824cd..fd9be80 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -74,7 +74,9 @@ jobs: git config user.name "Github-Bot" git config user.email "Github-Bot@no-reply.com" - gto register test --repo https://github.com/Hestia-Homes/ML/ + git branch -r + + # gto register test --repo https://github.com/Hestia-Homes/ML/ # echo "chicken" >> test.md # git fetch --prune From 96902960e0cac21eba264353ce07be3f2692a757 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 12 Sep 2023 22:33:10 +0100 Subject: [PATCH 35/41] try to assign after register --- .github/workflows/MLPipelinePostMerge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index fd9be80..f80450d 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -96,7 +96,7 @@ jobs: # git push origin regression@v0.0.1 # gto register ${REGISTER_MODEL_NAME} --force - # gto assign ${REGISTER_MODEL_NAME} --stage dev + gto assign regression --stage dev # gto show # Register-Prediction-Image-Dev: From 6c899bbb09d46a313f4e918724ca9d64f9181ebb Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 12 Sep 2023 22:37:48 +0100 Subject: [PATCH 36/41] look at the remote origin --- .github/workflows/MLPipelinePostMerge.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index f80450d..96cbd53 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -61,11 +61,6 @@ jobs: pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt - name: Register Model - env: - ACCESS_TOKEN: $ {{ secrets.SSH_TOKEN }} - # ACTIONS_STEP_DEBUG: true - # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # token: ${{ secrets.PAT }} run: | # REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}') @@ -91,12 +86,12 @@ jobs: # git push origin model@v0.0.1 # git checkout master - # git remote show origin + git remote show origin # git status # git push origin regression@v0.0.1 # gto register ${REGISTER_MODEL_NAME} --force - gto assign regression --stage dev + # gto assign regression --stage dev # gto show # Register-Prediction-Image-Dev: From 74aca512ced6bc525e35f410e8db5956d8c7fc52 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 12 Sep 2023 22:39:15 +0100 Subject: [PATCH 37/41] look at the remote origin - try v2 --- .github/workflows/MLPipelinePostMerge.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index 96cbd53..a85680d 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -48,7 +48,7 @@ jobs: # if: github.event.pull_request.merged == true runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v2 with: fetch-depth: 0 @@ -90,7 +90,7 @@ jobs: # git status # git push origin regression@v0.0.1 - # gto register ${REGISTER_MODEL_NAME} --force + gto register ${REGISTER_MODEL_NAME} # gto assign regression --stage dev # gto show From 402332f0d5aefdd5122ac39e7613a08afdd9376b Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 12 Sep 2023 22:41:43 +0100 Subject: [PATCH 38/41] look at the remote origin - try v2 --- .github/workflows/MLPipelinePostMerge.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index a85680d..cf7e925 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -89,8 +89,8 @@ jobs: git remote show origin # git status # git push origin regression@v0.0.1 - - gto register ${REGISTER_MODEL_NAME} + git checkout master + # gto register ${REGISTER_MODEL_NAME} # gto assign regression --stage dev # gto show From 5bedb72a5aa1a5279f4caf6e19cc73b91c63f9ad Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 12 Sep 2023 22:42:54 +0100 Subject: [PATCH 39/41] look at the remote origin - try v2 --- .github/workflows/MLPipelinePostMerge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index cf7e925..b1d665d 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -90,7 +90,7 @@ jobs: # git status # git push origin regression@v0.0.1 git checkout master - # gto register ${REGISTER_MODEL_NAME} + gto register ${REGISTER_MODEL_NAME} # gto assign regression --stage dev # gto show From d25c4409ebb3c438fabce0e5563e5fd136666b91 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 12 Sep 2023 22:54:28 +0100 Subject: [PATCH 40/41] add credentials --- .github/workflows/MLPipelinePostMerge.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index b1d665d..9ed3fd0 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -61,6 +61,9 @@ jobs: pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt - name: Register Model + env: + AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} run: | # REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}') @@ -69,6 +72,8 @@ jobs: git config user.name "Github-Bot" git config user.email "Github-Bot@no-reply.com" + gto register ${REGISTER_MODEL_NAME} + git branch -r # gto register test --repo https://github.com/Hestia-Homes/ML/ From eb41a8d1a0ad248c7f56203400e8e2422b3642d4 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 12 Sep 2023 23:08:38 +0100 Subject: [PATCH 41/41] removed register step for now --- .github/workflows/MLPipelinePostMerge.yml | 189 ++++++++++------------ 1 file changed, 85 insertions(+), 104 deletions(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index 9ed3fd0..b43007d 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -1,58 +1,93 @@ name: Register the model for the given pipeline branch -on: - push: - branches: - - "model-**" - # on: -# pull_request: -# types: -# - closed +# push: # branches: -# - "master" +# - "model-**" + +on: + pull_request: + types: + - closed + branches: + - "master" permissions: write-all jobs: - # Promote-Model-To-Dev: + Promote-Model-To-Dev: + if: github.event.pull_request.merged == true + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Install packages to retrieve artifacts + run: | + pip install --upgrade pip + pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt + + - name: Retrieve artifacts (dvc.lock) + env: + AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} + run: | + cd modules/ml-pipeline/src/pipeline/src + dvc pull -r experiments + + - name: Push artifacts to Dev + env: + AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} + run: | + cd modules/ml-pipeline/src/pipeline/src + dvc push -r dev + + + # Register-New-Model-Dev: # if: github.event.pull_request.merged == true # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v3 - # - name: Install packages to retrieve artifacts + # - uses: actions/checkout@v2 + # with: + # fetch-depth: 0 + + # - name: Install packages to register model + # env: + # AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} + # AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} # run: | # pip install --upgrade pip # pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt - # - name: Retrieve artifacts (dvc.lock) + # - name: Register Model # env: # AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} # AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} # run: | - # cd modules/ml-pipeline/src/pipeline/src - # dvc pull -r experiments - # - name: Push artifacts to Dev - # env: - # AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} - # run: | - # cd modules/ml-pipeline/src/pipeline/src - # dvc push -r dev + # REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}') + # # REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}') + # git config user.name "Github-Bot" + # git config user.email "Github-Bot@no-reply.com" - Register-New-Model-Dev: - # if: github.event.pull_request.merged == true + # # gto register test --repo https://github.com/Hestia-Homes/ML/ + # # echo "chicken" >> test.md + + # git checkout master + # gto register ${REGISTER_MODEL_NAME} + # gto assign regression --stage dev + # gto show + + Register-Prediction-Image-Dev: + needs: Promote-Model-To-Dev + # needs: [Promote-Model-To-Dev, Register-New-Model-Dev] WILL ADD BACK ONCE REGISTER WORKS runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Install packages to register model + steps: + - uses: actions/checkout@v3 + - name: Install packages to retrieve artifacts env: AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} @@ -60,84 +95,30 @@ jobs: pip install --upgrade pip pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt - - name: Register Model + - name: Retrieve artifacts (dvc.lock) env: AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} run: | + cd modules/ml-pipeline/src/pipeline/src + dvc pull -r dev - # REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}') - REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}') + - name: Build Prediction docker image (TODO - NEED LAMBDA IMAGE, need to add version from gto registry) + run: | + cd modules/ml-pipeline/src/pipeline/ + REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}') + docker build . --file Prediction.Dockerfile --tag ${REGISTER_MODEL_NAME} - git config user.name "Github-Bot" - git config user.email "Github-Bot@no-reply.com" + - name: ECR Login - Dev + env: + AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} + run: | + echo "LOGIN TO ECR" - gto register ${REGISTER_MODEL_NAME} - - git branch -r - - # gto register test --repo https://github.com/Hestia-Homes/ML/ - # echo "chicken" >> test.md - - # git fetch --prune - # git pull origin master - # git add . - # git commit -m "test" - - # git push - - # git branch - # git tag model@v0.0.1 - # git push origin model@v0.0.1 - # git checkout master - - git remote show origin - # git status - # git push origin regression@v0.0.1 - git checkout master - gto register ${REGISTER_MODEL_NAME} - # gto assign regression --stage dev - # gto show - - # Register-Prediction-Image-Dev: - # needs: Promote-Model-To-Dev - # # needs: [Promote-Model-To-Dev, Register-New-Model-Dev] WILL ADD BACK ONCE REGISTER WORKS - # runs-on: ubuntu-latest - - # steps: - # - uses: actions/checkout@v3 - # - name: Install packages to retrieve artifacts - # env: - # AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} - # run: | - # pip install --upgrade pip - # pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt - - # - name: Retrieve artifacts (dvc.lock) - # env: - # AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} - # run: | - # cd modules/ml-pipeline/src/pipeline/src - # dvc pull -r dev - - # - name: Build Prediction docker image (TODO - NEED LAMBDA IMAGE, need to add version from gto registry) - # run: | - # cd modules/ml-pipeline/src/pipeline/ - # REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}') - # docker build . --file Prediction.Dockerfile --tag ${REGISTER_MODEL_NAME} - - # - name: ECR Login - Dev - # env: - # AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} - # run: | - # echo "LOGIN TO ECR" - - # - name: Push Prediction image to ECR - Dev - # env: - # AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} - # run: | - # echo "PUSH TO ECR" + - name: Push Prediction image to ECR - Dev + env: + AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} + run: | + echo "PUSH TO ECR"