name: Sap Change Model Deploy on: push: branches: [ sap-dev, sap-prod, heat-dev, heat-prod, carbon-dev, carbon-prod, sap_baseline-dev, sap_baseline-prod, ] jobs: deploy: runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v3 - name: Clean Up Disk Space if: runner.os == 'Linux' run: | # Space usage before cleanup df -h / # Remove unused tool caches (comment any required ones with #) sudo rm -rf /usr/lib/jvm sudo rm -rf /usr/local/.ghcup sudo rm -rf /usr/local/lib/android sudo rm -rf /usr/local/share/powershell sudo rm -rf /usr/share/dotnet sudo rm -rf /usr/share/swift sudo rm -rf "$AGENT_TOOLSDIRECTORY" # Verify gains df -h / - name: Set up Python uses: actions/setup-python@v2 with: python-version: 3.10.12 - name: Install Serverless and plugins run: | npm install -g serverless@^3.38.0 npm install -g serverless-domain-manager@^7.3.8 - name: Install DVC run: | pip install --upgrade pip pip install boto3 pip install -r modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt # Set up all of the secrets required for the deployment - name: set secret prefix which is used across multiple steps id: secret_prefix run: | # Convert branch name to uppercase and replace hyphens with underscores echo "::set-output name=secret_prefix::$(echo "${{ github.ref_name }}" | tr 'a-z-' 'A-Z_')" - name: Set domain name id: set_domain run: echo "::set-output name=domain::${{ secrets[format('{0}_DOMAIN_NAME', steps.secret_prefix.outputs.secret_prefix)] }}" - name: Set ECR credentials id: set_ecr_credentials run: | # Fetch the secret using the secret prefix echo "::set-output name=ecr_uri::${{ secrets[format('{0}_ECR_URI', steps.secret_prefix.outputs.secret_prefix)] }}" - name: Set S3 buckets id: set_s3_buckets run: | # Fetch the secret using the secret prefix echo "::set-output name=data_bucket::${{ secrets[format('{0}_DATA_BUCKET', steps.secret_prefix.outputs.secret_prefix)] }}" echo "::set-output name=predictions_bucket::${{ secrets[format('{0}_PREDICTIONS_BUCKET', steps.secret_prefix.outputs.secret_prefix)] }}" - name: Set stack_name id: set_stack_name run: | # Take branch prefix and add "model" for stack name # stack_name=$( echo ${{ github.ref_name }} | awk -F"-" '{print $1}' | sed 's/$/model/g') stack_name=$(echo ${{ github.ref_name }} | awk -F"-" '{gsub("_", "", $1); print $1}' | sed 's/$/model/g') if [ -z "${stack_name}" ]; then echo "::set-output name=stack_name::" else echo "::set-output name=stack_name::${stack_name}" fi - name: Set runtime_environment id: set_runtime_environment run: | # Extract the suffix after the hyphen from the branch name runtime_environment=$(echo "${{ github.ref_name }}" | awk -F'-' '{print $NF}') echo "::set-output name=runtime_environment::$runtime_environment" - name: AWS credentials for dev if: ${{ steps.set_runtime_environment.outputs.runtime_environment }} == 'dev' uses: aws-actions/configure-aws-credentials@v1 with: aws-access-key-id: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} aws-region: eu-west-2 - name: AWS credentials for prod if: ${{ steps.set_runtime_environment.outputs.runtime_environment }} == 'prod' uses: aws-actions/configure-aws-credentials@v1 with: aws-access-key-id: ${{ secrets.PROD_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.PROD_AWS_SECRET_ACCESS_KEY }} aws-region: eu-west-2 - name: DVC Pull run: | cd modules/ml-pipeline/src/pipeline dvc pull -r ${{ steps.set_runtime_environment.outputs.runtime_environment }} cd data/model ls rm -r allmodels - name: Setup Docker uses: docker/setup-buildx-action@v1 - name: Login to ECR run: | aws ecr get-login-password --region eu-west-2 | docker login --username AWS --password-stdin ${{ steps.set_ecr_credentials.outputs.ecr_uri }} # Building and pushing Docker image with caching - name: Build and push Docker image uses: docker/build-push-action@v3 with: context: . file: ./deployment/Dockerfile.prediction.lambda push: true tags: ${{ steps.set_ecr_credentials.outputs.ecr_uri }}:${{ github.sha }} cache-from: type=gha cache-to: type=gha,mode=max platforms: linux/amd64 provenance: false build-args: | RUNTIME_ENVIRONMENT=${{ steps.set_runtime_environment.outputs.runtime_environment }} - name: Deploy to AWS Lambda via Serverless env: RUNTIME_ENVIRONMENT: ${{ steps.set_runtime_environment.outputs.runtime_environment }} PREDICTIONS_BUCKET: ${{ steps.set_s3_buckets.outputs.predictions_bucket }} DATA_BUCKET: ${{ steps.set_s3_buckets.outputs.data_bucket }} DOMAIN_NAME: ${{ steps.set_domain.outputs.domain }} ECR_URI: ${{ steps.set_ecr_credentials.outputs.ecr_uri }} GITHUB_SHA: ${{ github.sha }} STACK_NAME: ${{ steps.set_stack_name.outputs.stack_name }} run: | # Deploy to AWS Lambda via Serverless cd deployment sls deploy --config serverless.yml --stage ${{ steps.set_runtime_environment.outputs.runtime_environment }} --verbose