From b2e836c232dde948482dfc991abd0fa49031d7e1 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Sun, 10 Sep 2023 20:34:34 +0100 Subject: [PATCH 01/19] test gto step --- .github/workflows/MLPipelinePostMerge.yml | 37 +++++++++++++++++++++ .github/workflows/MLPipelinePullRequest.yml | 8 ++--- 2 files changed, 41 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/MLPipelinePostMerge.yml diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml new file mode 100644 index 0000000..7787ea8 --- /dev/null +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -0,0 +1,37 @@ +name: Register the model for the given pipeline branch + +on: + push: + branches: + - "model-**" + +# on: +# pull_request: +# types: +# - closed +# branches: +# - "master" + +permissions: write-all + +jobs: + if_merged: + if: github.event.pull_request.merged == true + runs_on: ubuntu-latest + steps: + # - uses: actions/checkout@v3 + # - name: Install packages to register model + # env: + # AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} + # AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} + # run: | + # pip install --upgrade pip + # pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt + + # - name: Register Model + # run: | + + # dvc pull -r experiments + - name: echo branch + run: | + github.ref_name diff --git a/.github/workflows/MLPipelinePullRequest.yml b/.github/workflows/MLPipelinePullRequest.yml index 186c382..614a984 100644 --- a/.github/workflows/MLPipelinePullRequest.yml +++ b/.github/workflows/MLPipelinePullRequest.yml @@ -1,9 +1,9 @@ name: Build and test model using a dummy prediction pipeline on: - push: - branches: - - "model-**" + # push: + # branches: + # - "model-**" pull_request: branches: [ "master" ] @@ -74,7 +74,7 @@ jobs: # Compare metrics to master git fetch --depth=1 origin master:master - dvc metrics diff --md master >> report.md + dvc metrics diff --md --all master >> report.md cml comment create report.md From 06f82dd3180119a5cd2e8ce49672d3894b5df115 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Sun, 10 Sep 2023 20:35:37 +0100 Subject: [PATCH 02/19] test gto step --- .github/workflows/MLPipelinePostMerge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index 7787ea8..e0a7174 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -17,7 +17,7 @@ permissions: write-all jobs: if_merged: if: github.event.pull_request.merged == true - runs_on: ubuntu-latest + runs-on: ubuntu-latest steps: # - uses: actions/checkout@v3 # - name: Install packages to register model From 9a72ac3a79aeacfc75b87b8ad5ad3daa91eea476 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Sun, 10 Sep 2023 20:36:04 +0100 Subject: [PATCH 03/19] test gto step --- .github/workflows/MLPipelinePostMerge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index e0a7174..3e9292b 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -16,7 +16,7 @@ permissions: write-all jobs: if_merged: - if: github.event.pull_request.merged == true + # if: github.event.pull_request.merged == true runs-on: ubuntu-latest steps: # - uses: actions/checkout@v3 From 9878b08263e983e63c584b222146ce2c855a2305 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Sun, 10 Sep 2023 20:36:52 +0100 Subject: [PATCH 04/19] test gto step --- .github/workflows/MLPipelinePostMerge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index 3e9292b..f3aff41 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -34,4 +34,4 @@ jobs: # dvc pull -r experiments - name: echo branch run: | - github.ref_name + echo ${github.ref_name} From efc52639bacdd27bf4d27c83f6cf7dd17a7dab84 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Sun, 10 Sep 2023 20:37:48 +0100 Subject: [PATCH 05/19] test gto step --- .github/workflows/MLPipelinePostMerge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index f3aff41..79c7d02 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -34,4 +34,4 @@ jobs: # dvc pull -r experiments - name: echo branch run: | - echo ${github.ref_name} + echo ${{github.ref_name}} From 1dbce68e83783bc99bc5ab7d706875200dd153f0 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Sun, 10 Sep 2023 20:38:59 +0100 Subject: [PATCH 06/19] test gto step --- .github/workflows/MLPipelinePostMerge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index 79c7d02..bc0ca29 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -34,4 +34,4 @@ jobs: # dvc pull -r experiments - name: echo branch run: | - echo ${{github.ref_name}} + echo ${{github.head_ref}} From a94a9385a80aa91b7d67af3e3a972accdf18ed93 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Sun, 10 Sep 2023 20:43:26 +0100 Subject: [PATCH 07/19] test gto step --- .github/workflows/MLPipelinePostMerge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index bc0ca29..3d4d27b 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -34,4 +34,4 @@ jobs: # dvc pull -r experiments - name: echo branch run: | - echo ${{github.head_ref}} + register_name=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}') From faddfbf7cad712b593ab4808a1dc0f761348a9af Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Sun, 10 Sep 2023 20:45:51 +0100 Subject: [PATCH 08/19] test gto step --- .github/workflows/MLPipelinePostMerge.yml | 29 +++++++++++------------ 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index 3d4d27b..ded8c15 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -19,19 +19,18 @@ jobs: # if: github.event.pull_request.merged == true runs-on: ubuntu-latest steps: - # - uses: actions/checkout@v3 - # - name: Install packages to register model - # env: - # AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} - # run: | - # pip install --upgrade pip - # pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt - - # - name: Register Model - # run: | - - # dvc pull -r experiments - - name: echo branch + - uses: actions/checkout@v3 + - name: Install packages to register model + env: + AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }} run: | - register_name=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}') + pip install --upgrade pip + pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt + + - name: Register Model + run: | + REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}') + gto register ${REGISTER_MODEL_NAME} + gto assign ${REGISTER_MODEL_NAME} --stage dev + gto show From 6bdf34082b751fbb050eebcde7152e14b3a00a3b Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Sun, 10 Sep 2023 20:47:47 +0100 Subject: [PATCH 09/19] test gto step --- .github/workflows/MLPipelinePostMerge.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index ded8c15..005009a 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -31,6 +31,9 @@ jobs: - name: Register Model run: | REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}') + git config user.name "Github-Bot" + git config user.email "Github-Bot" + gto register ${REGISTER_MODEL_NAME} gto assign ${REGISTER_MODEL_NAME} --stage dev gto show From d718e15627ac1214856ab5a63ef203be07e557fa Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Sun, 10 Sep 2023 20:50:40 +0100 Subject: [PATCH 10/19] test gto step --- .github/workflows/MLPipelinePostMerge.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index 005009a..e5595e9 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -33,6 +33,7 @@ jobs: REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}') git config user.name "Github-Bot" git config user.email "Github-Bot" + git fetch gto register ${REGISTER_MODEL_NAME} gto assign ${REGISTER_MODEL_NAME} --stage dev From 62ea7675c747cad3ced04cbe789f52f64a90413c Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Sun, 10 Sep 2023 20:54:14 +0100 Subject: [PATCH 11/19] test gto step --- .github/workflows/MLPipelinePostMerge.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index e5595e9..4331639 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -34,6 +34,7 @@ jobs: git config user.name "Github-Bot" git config user.email "Github-Bot" git fetch + git reset --hard origin/main gto register ${REGISTER_MODEL_NAME} gto assign ${REGISTER_MODEL_NAME} --stage dev From 86cb4740628a1ff5b4bd9f5f0cfaaaa85fe0a453 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Sun, 10 Sep 2023 20:55:10 +0100 Subject: [PATCH 12/19] test gto step --- .github/workflows/MLPipelinePostMerge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index 4331639..737ae41 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -34,7 +34,7 @@ jobs: git config user.name "Github-Bot" git config user.email "Github-Bot" git fetch - git reset --hard origin/main + git reset --hard origin/master gto register ${REGISTER_MODEL_NAME} gto assign ${REGISTER_MODEL_NAME} --stage dev From 0afb645f30aec81557c04ed279e66bffca896e52 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Sun, 10 Sep 2023 21:00:45 +0100 Subject: [PATCH 13/19] test gto step --- .github/workflows/MLPipelinePostMerge.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index 737ae41..5517430 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -20,6 +20,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + with: + fetch-depth: '0' - name: Install packages to register model env: AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} From 35e287b6642d40edd0396408ab6c660206f98ee1 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Sun, 10 Sep 2023 21:04:21 +0100 Subject: [PATCH 14/19] test gto step --- .github/workflows/MLPipelinePostMerge.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index 5517430..a555630 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -35,8 +35,6 @@ jobs: REGISTER_MODEL_NAME=$(echo ${{github.ref_name}} | awk -F"-" '{print $1}') git config user.name "Github-Bot" git config user.email "Github-Bot" - git fetch - git reset --hard origin/master gto register ${REGISTER_MODEL_NAME} gto assign ${REGISTER_MODEL_NAME} --stage dev From 15c623b5aa496056dcb348988c218087d87f44cd Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Sun, 10 Sep 2023 21:05:30 +0100 Subject: [PATCH 15/19] test gto step --- .github/workflows/MLPipelinePostMerge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index a555630..77f892f 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -21,7 +21,7 @@ jobs: steps: - uses: actions/checkout@v3 with: - fetch-depth: '0' + fetch-depth: '5' - name: Install packages to register model env: AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} From 95536a0830635fb538e75e2c438635071f67dc4c Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Mon, 11 Sep 2023 08:52:49 +0100 Subject: [PATCH 16/19] test gto step --- .github/workflows/MLPipelinePostMerge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index 77f892f..a555630 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -21,7 +21,7 @@ jobs: steps: - uses: actions/checkout@v3 with: - fetch-depth: '5' + fetch-depth: '0' - name: Install packages to register model env: AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} From 59fa11a6523a8269652cf1fdc53fbf12a8a9b490 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Mon, 11 Sep 2023 08:55:04 +0100 Subject: [PATCH 17/19] test gto step --- .github/workflows/MLPipelinePostMerge.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index a555630..558e800 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -21,7 +21,8 @@ jobs: steps: - uses: actions/checkout@v3 with: - fetch-depth: '0' + fetch-depth: 0 + - name: Install packages to register model env: AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }} From 4107dd7642ec7d625512e2000da3b1ab47a9df13 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Mon, 11 Sep 2023 08:57:49 +0100 Subject: [PATCH 18/19] test gto step --- .github/workflows/MLPipelinePostMerge.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index 558e800..28d651d 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -37,6 +37,8 @@ jobs: git config user.name "Github-Bot" git config user.email "Github-Bot" + git fetch origin + gto register ${REGISTER_MODEL_NAME} gto assign ${REGISTER_MODEL_NAME} --stage dev gto show From c871c0d610371e28d6b2f75283a146643e77d454 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Mon, 11 Sep 2023 09:02:15 +0100 Subject: [PATCH 19/19] test gto step - with merge to master --- .github/workflows/MLPipelinePostMerge.yml | 24 +++++++++++------------ 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index 28d651d..660acd0 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -1,22 +1,22 @@ name: Register the model for the given pipeline branch -on: - push: - branches: - - "model-**" - # on: -# pull_request: -# types: -# - closed +# push: # branches: -# - "master" +# - "model-**" + +on: + pull_request: + types: + - closed + branches: + - "master" permissions: write-all jobs: - if_merged: - # if: github.event.pull_request.merged == true + Register-New-Model: + if: github.event.pull_request.merged == true runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -37,8 +37,6 @@ jobs: git config user.name "Github-Bot" git config user.email "Github-Bot" - git fetch origin - gto register ${REGISTER_MODEL_NAME} gto assign ${REGISTER_MODEL_NAME} --stage dev gto show