mirror of
https://github.com/Hestia-Homes/ML.git
synced 2026-06-08 11:17:25 +00:00
change workflow
This commit is contained in:
Michael Duong
parent
40bb9d6f0a
commit
a74c1975b2
4 changed files with 29 additions and 57 deletions
|
|
@ -16,19 +16,19 @@ jobs:
|
|||
- name: Install packages to retrieve artifacts
|
||||
run: |
|
||||
pip install --upgrade pip
|
||||
pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt
|
||||
pip install -r modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt
|
||||
|
||||
- name: Retrieve artifacts (dvc.lock)
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }}
|
||||
run: |
|
||||
cd modules/ml-pipeline/src/pipeline/src
|
||||
cd modules/ml-pipeline/src/pipeline
|
||||
dvc pull -r dev
|
||||
|
||||
- name: Build Prediction docker image (TODO - NEED LAMBDA IMAGE, need to add version from gto registry)
|
||||
run: |
|
||||
cd modules/ml-pipeline/src/pipeline/
|
||||
cd modules/ml-pipeline/src/
|
||||
REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}')
|
||||
docker build . --file Prediction.Dockerfile --tag ${REGISTER_MODEL_NAME}
|
||||
|
||||
54
.github/workflows/MLPipelinePostMerge.yml
vendored
54
.github/workflows/MLPipelinePostMerge.yml
vendored
|
|
@ -26,7 +26,7 @@ jobs:
|
|||
- name: Install packages to register model
|
||||
run: |
|
||||
pip install --upgrade pip
|
||||
pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt
|
||||
pip install -r modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt
|
||||
|
||||
- name: Register Model
|
||||
run: |
|
||||
|
|
@ -64,7 +64,7 @@ jobs:
|
|||
- name: Install packages to register model
|
||||
run: |
|
||||
pip install --upgrade pip
|
||||
pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt
|
||||
pip install -r modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt
|
||||
|
||||
- name: Register Model
|
||||
run: |
|
||||
|
|
@ -102,7 +102,7 @@ jobs:
|
|||
- name: Install packages to register model
|
||||
run: |
|
||||
pip install --upgrade pip
|
||||
pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt
|
||||
pip install -r modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt
|
||||
|
||||
- name: Register Model
|
||||
run: |
|
||||
|
|
@ -138,14 +138,14 @@ jobs:
|
|||
- name: Install packages to retrieve artifacts
|
||||
run: |
|
||||
pip install --upgrade pip
|
||||
pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt
|
||||
pip install -r modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt
|
||||
|
||||
- name: Retrieve artifacts (dvc.lock)
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }}
|
||||
run: |
|
||||
cd modules/ml-pipeline/src/pipeline/src
|
||||
cd modules/ml-pipeline/src/pipeline
|
||||
dvc pull -r experiments
|
||||
|
||||
- name: Push artifacts to Dev
|
||||
|
|
@ -153,7 +153,7 @@ jobs:
|
|||
AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }}
|
||||
run: |
|
||||
cd modules/ml-pipeline/src/pipeline/src
|
||||
cd modules/ml-pipeline/src/pipeline
|
||||
dvc push -r dev
|
||||
|
||||
Register-New-Model-Dev:
|
||||
|
|
@ -173,7 +173,7 @@ jobs:
|
|||
- name: Install packages to register model
|
||||
run: |
|
||||
pip install --upgrade pip
|
||||
pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt
|
||||
pip install -r modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt
|
||||
|
||||
- name: Register Model
|
||||
run: |
|
||||
|
|
@ -211,43 +211,3 @@ jobs:
|
|||
git add .
|
||||
git commit -m "Update Registry"
|
||||
git push origin master
|
||||
|
||||
|
||||
Register-Prediction-Image-Dev:
|
||||
needs: [Promote-Artefacts-To-Dev, Register-New-Model-Dev]
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Install packages to retrieve artifacts
|
||||
run: |
|
||||
pip install --upgrade pip
|
||||
pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt
|
||||
|
||||
- name: Retrieve artifacts (dvc.lock)
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }}
|
||||
run: |
|
||||
cd modules/ml-pipeline/src/pipeline/src
|
||||
dvc pull -r dev
|
||||
|
||||
- name: Build Prediction docker image (TODO - NEED LAMBDA IMAGE, need to add version from gto registry)
|
||||
run: |
|
||||
cd modules/ml-pipeline/src/pipeline/
|
||||
REGISTER_MODEL_NAME=$(echo ${{ github.event.pull_request.head.ref }} | awk -F"-" '{print $1}')
|
||||
docker build . --file Prediction.Dockerfile --tag ${REGISTER_MODEL_NAME}
|
||||
|
||||
- name: ECR Login - Dev
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }}
|
||||
run: |
|
||||
echo "LOGIN TO ECR"
|
||||
|
||||
- name: Push Prediction image to ECR - Dev
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }}
|
||||
run: |
|
||||
echo "PUSH TO ECR"
|
||||
|
|
|
|||
12
.github/workflows/MLPipelinePullRequest.yml
vendored
12
.github/workflows/MLPipelinePullRequest.yml
vendored
|
|
@ -44,19 +44,19 @@ jobs:
|
|||
AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }}
|
||||
run: |
|
||||
pip install --upgrade pip
|
||||
pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt
|
||||
pip install -r modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt
|
||||
|
||||
- name: Retrieve artifacts (dvc.lock)
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }}
|
||||
run: |
|
||||
cd modules/ml-pipeline/src/pipeline/src
|
||||
cd modules/ml-pipeline/src/pipeline
|
||||
dvc pull -r experiments
|
||||
|
||||
- name: Build Prediction docker Image
|
||||
run: |
|
||||
cd modules/ml-pipeline/src/pipeline/
|
||||
cd modules/ml-pipeline/src/
|
||||
docker build . --file Prediction.Dockerfile --tag prediction_test
|
||||
|
||||
- name: Run Prediction docker container
|
||||
|
|
@ -72,14 +72,14 @@ jobs:
|
|||
- name: Install packages to retrieve artifacts
|
||||
run: |
|
||||
pip install --upgrade pip
|
||||
pip install -r modules/ml-pipeline/src/pipeline/src/requirements/version_control/requirements.txt
|
||||
pip install -r modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt
|
||||
|
||||
- name: Retrieve artifacts (dvc.lock)
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }}
|
||||
run: |
|
||||
cd modules/ml-pipeline/src/pipeline/src
|
||||
cd modules/ml-pipeline/src/pipeline
|
||||
dvc pull -r experiments
|
||||
|
||||
- uses: actions/setup-python@v4
|
||||
|
|
@ -90,7 +90,7 @@ jobs:
|
|||
AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }}
|
||||
REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
cd modules/ml-pipeline/src/pipeline/src
|
||||
cd modules/ml-pipeline/src/pipeline
|
||||
echo "## Model metrics" > report.md
|
||||
|
||||
# Compare metrics to master
|
||||
|
|
|
|||
14
README.md
14
README.md
|
|
@ -3,9 +3,21 @@
|
|||
Creating a ML-toolkit that can be reused:
|
||||
|
||||
- ML pipeline:
|
||||
- A dummy pipeline that has data version control, experiment
|
||||
- A generic pipeline that has data version control, experiment
|
||||
tracking and a model registry
|
||||
|
||||
- ML monitoring:
|
||||
- A bolt-on service that can implement model monitoring
|
||||
|
||||
There are multiple protected branches which adapt the generic pipeline to produce different models:
|
||||
- sap_change-**
|
||||
- heat_change-**
|
||||
- carbon_change-**
|
||||
|
||||
These branches will differ by the configuration files that define the data used and the outputs of the ML-pipeline
|
||||
- There can be different additional logic for each branch but the pipeline will be the same.
|
||||
|
||||
# Deployment
|
||||
|
||||
TBD
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue