From ad9faa8b9def47134d0272d75b09f31530c3e0f0 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 1 Sep 2023 18:23:17 +0100 Subject: [PATCH 1/3] attempting to use v3 of docker/build-push-action@v3 --- .github/workflows/deploy_sap_model_lambda.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy_sap_model_lambda.yml b/.github/workflows/deploy_sap_model_lambda.yml index 0f72e3e3..c06feeaf 100644 --- a/.github/workflows/deploy_sap_model_lambda.yml +++ b/.github/workflows/deploy_sap_model_lambda.yml @@ -56,7 +56,7 @@ jobs: # Building and pushing Docker image with caching - name: Build and push Docker image - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v3 with: context: ./model_data/simulation_system file: ./model_data/simulation_system/Dockerfiles/Dockerfile.prediction.lambda @@ -65,6 +65,7 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max platform: linux/amd64 + provenance: false - name: Deploy to AWS Lambda via Serverless env: From 15233849dbcdcaae56a27674f3b02b9efb8cca97 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 1 Sep 2023 18:41:20 +0100 Subject: [PATCH 2/3] test adding logging --- .../simulation_system/handlers/predictions_app.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/model_data/simulation_system/handlers/predictions_app.py b/model_data/simulation_system/handlers/predictions_app.py index 2ecdd522..b09e2d67 100644 --- a/model_data/simulation_system/handlers/predictions_app.py +++ b/model_data/simulation_system/handlers/predictions_app.py @@ -1,7 +1,11 @@ import os -import urllib.parse +import logging + from predictions import prediction +logger = logging.getLogger() +logger.setLevel(logging.INFO) + RUNTIME_ENVIRONMENT = os.environ.get("RUNTIME_ENVIRONMENT", "dev") @@ -10,6 +14,8 @@ def handler(event, context): Take in event and trigger the prediction pipeline """ + logger.info("received event: " + str(event)) + # Assuming a file in a bucket landing for now? # Assuming we have a model to use @@ -30,8 +36,8 @@ def handler(event, context): return storage_filepath except (Exception, KeyError, ValueError) as e: - print("Prediction failed") - print(e) + logger.info("Prediction failed") + logger.info(e) if __name__ == "__main__": From 4540ed82d1f4577469f630cfb35160177d4205bb Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 1 Sep 2023 19:13:28 +0100 Subject: [PATCH 3/3] Give permission to lambda to read from data bucket --- .github/workflows/deploy_sap_model_lambda.yml | 1 + model_data/simulation_system/handlers/predictions_app.py | 9 ++++----- sapmodel.serverless.yml | 7 +++++-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/deploy_sap_model_lambda.yml b/.github/workflows/deploy_sap_model_lambda.yml index c06feeaf..fb4b8dde 100644 --- a/.github/workflows/deploy_sap_model_lambda.yml +++ b/.github/workflows/deploy_sap_model_lambda.yml @@ -72,6 +72,7 @@ jobs: RUNTIME_ENVIRONMENT: ${{ github.ref_name }} MODEL_DIRECTORY_BUCKET: 'retrofit-model-directory-${{ github.ref_name }}' PREDICTIONS_BUCKET: 'retrofit-sap-predictions-${{ github.ref_name }}' + DATA_BUCKET: 'retrofit-data-${{ github.ref_name }}' DOMAIN_NAME: ${{ steps.set_domain.outputs.domain }} ECR_URI: ${{ steps.set_ecr_credentials.outputs.ecr_uri }} GITHUB_SHA: ${{ github.sha }} diff --git a/model_data/simulation_system/handlers/predictions_app.py b/model_data/simulation_system/handlers/predictions_app.py index b09e2d67..ea764276 100644 --- a/model_data/simulation_system/handlers/predictions_app.py +++ b/model_data/simulation_system/handlers/predictions_app.py @@ -19,11 +19,10 @@ def handler(event, context): # Assuming a file in a bucket landing for now? # Assuming we have a model to use - payload = event["body"] - data_path = payload["file_location"] - property_id = payload["property_id"] - portfolio_id = payload["portfolio_id"] - created_at = payload["created_at"] + data_path = event["file_location"] + property_id = event["property_id"] + portfolio_id = event["portfolio_id"] + created_at = event["created_at"] try: # We could fix the model path but for the moment, we just take the best model path based on the registry diff --git a/sapmodel.serverless.yml b/sapmodel.serverless.yml index a37e2df5..28a21527 100644 --- a/sapmodel.serverless.yml +++ b/sapmodel.serverless.yml @@ -8,6 +8,7 @@ provider: RUNTIME_ENVIRONMENT: ${env:RUNTIME_ENVIRONMENT} MODEL_DIRECTORY_BUCKET: ${env:MODEL_DIRECTORY_BUCKET} PREDICTIONS_BUCKET: ${env:PREDICTIONS_BUCKET} + DATA_BUCKET: ${env:DATA_BUCKET} DOMAIN_NAME: ${env:DOMAIN_NAME} ECR_URI: ${env:ECR_URI} GITHUB_SHA: ${env:GITHUB_SHA} @@ -52,7 +53,7 @@ resources: PolicyDocument: Version: '2012-10-17' Statement: - # Allow reading from MODEL_DIRECTORY_BUCKET + # Allow reading from MODEL_DIRECTORY_BUCKET and DATA_BUCKET - Effect: Allow Action: - s3:GetObject @@ -60,6 +61,8 @@ resources: Resource: - arn:aws:s3:::${env:MODEL_DIRECTORY_BUCKET} - arn:aws:s3:::${env:MODEL_DIRECTORY_BUCKET}/* + - arn:aws:s3:::${env:DATA_BUCKET} + - arn:aws:s3:::${env:DATA_BUCKET}/* # Allow reading and writing to PREDICTIONS_BUCKET - Effect: Allow Action: @@ -68,4 +71,4 @@ resources: - s3:ListBucket Resource: - arn:aws:s3:::${env:PREDICTIONS_BUCKET} - - arn:aws:s3:::${env:PREDICTIONS_BUCKET}/* \ No newline at end of file + - arn:aws:s3:::${env:PREDICTIONS_BUCKET}/*