refactoring serverless script with sqs queue

backend/lambda/README.md

@@ -0,0 +1,8 @@
+# Model Engine Lambda
+
+This repository contains the code for the Model Engine Lambda, which is responsible for serving machine learning models
+in a serverless environment. The Lambda function is designed to handle requests for model inference and return the
+results to the client.
+
+This service consumes an SQS queue and is triggered by messages sent to the queue. The Lambda function processes the
+messages, performs model inference, and sends the results back to the client.

backend/lambda/handler.py

@@ -0,0 +1,13 @@
+import json
+
+
+def handler(event, context):
+    """
+    This is an ascynchonous lambda handler that will run the model engine
+    :param event:
+    :param context:
+    :return:
+    """
+    for record in event["Records"]:
+        body = json.loads(record["body"])
+        asyncio.run(model_engine(body))

serverless.yml

@@ -1,8 +1,9 @@
-service: fastapi-lambda
+service: retrofit-platform
 
 provider:
   name: aws
   region: eu-west-2
+  runtime: python3.11
   architecture: x86_64
   environment:
     API_KEY: ${env:API_KEY}
@@ -23,62 +24,109 @@ provider:
     SAP_PREDICTIONS_BUCKET: ${env:SAP_PREDICTIONS_BUCKET}
     CARBON_PREDICTIONS_BUCKET: ${env:CARBON_PREDICTIONS_BUCKET}
     HEAT_PREDICTIONS_BUCKET: ${env:HEAT_PREDICTIONS_BUCKET}
-    #    LIGHTING_COST_PREDICTIONS_BUCKET: ${env:LIGHTING_COST_PREDICTIONS_BUCKET}
-    #    HEATING_COST_PREDICTIONS_BUCKET: ${env:HEATING_COST_PREDICTIONS_BUCKET}
-    #    HOT_WATER_COST_PREDICTIONS_BUCKET: ${env:HOT_WATER_COST_PREDICTIONS_BUCKET}
     HEATING_KWH_PREDICTIONS_BUCKET: ${env:HEATING_KWH_PREDICTIONS_BUCKET}
     HOTWATER_KWH_PREDICTIONS_BUCKET: ${env:HOTWATER_KWH_PREDICTIONS_BUCKET}
     ENERGY_ASSESSMENTS_BUCKET: ${env:ENERGY_ASSESSMENTS_BUCKET}
     GOOGLE_SOLAR_API_KEY: ${env:GOOGLE_SOLAR_API_KEY}
-  # Give lambda access to read from the bucket
+    ENGINE_SQS_URL:
-  iam:
+      Ref: EngineQueue
-    role:
-      name: fastapi_backend_${env:PLAN_TRIGGER_BUCKET}_access
-      statements:
-        - Effect: Allow
-          Action:
-            - s3:GetObject
-            - s3:ListBucket
-          Resource:
-            - arn:aws:s3:::${env:PLAN_TRIGGER_BUCKET}
-            - arn:aws:s3:::${env:PLAN_TRIGGER_BUCKET}/*
-        - Effect: Allow
-          Action:
-            - s3:*
-          Resource:
-            - arn:aws:s3:::${env:PREDICTIONS_BUCKET}
-            - arn:aws:s3:::${env:PREDICTIONS_BUCKET}/*
-            - arn:aws:s3:::${env:DATA_BUCKET}
-            - arn:aws:s3:::${env:DATA_BUCKET}/*
-            - arn:aws:s3:::${env:ENERGY_ASSESSMENTS_BUCKET}
-            - arn:aws:s3:::${env:ENERGY_ASSESSMENTS_BUCKET}/*
-            - arn:aws:s3:::${env:SAP_PREDICTIONS_BUCKET}
-            - arn:aws:s3:::${env:SAP_PREDICTIONS_BUCKET}/*
-            - arn:aws:s3:::${env:CARBON_PREDICTIONS_BUCKET}
-            - arn:aws:s3:::${env:CARBON_PREDICTIONS_BUCKET}/*
-            - arn:aws:s3:::${env:HEAT_PREDICTIONS_BUCKET}
-            - arn:aws:s3:::${env:HEAT_PREDICTIONS_BUCKET}/*
-            - arn:aws:s3:::${env:HEATING_KWH_PREDICTIONS_BUCKET}
-            - arn:aws:s3:::${env:HEATING_KWH_PREDICTIONS_BUCKET}/*
-            - arn:aws:s3:::${env:HOTWATER_KWH_PREDICTIONS_BUCKET}
-            - arn:aws:s3:::${env:HOTWATER_KWH_PREDICTIONS_BUCKET}/*
-
 
 plugins:
+  - serverless-python-requirements
   - serverless-domain-manager
 
 custom:
+  pythonRequirements:
+    fileName: backend/requirements/requirements.txt
+    dockerizePip: true
   customDomain:
     domainName: api.${self:provider.environment.DOMAIN_NAME}
     createRoute53Record: true
     certificateArn: ${ssm:/ssl_certificate_arn}
 
 functions:
-  app:
+
-    image:
+  fastapi-backend:
-      uri: ${env:ECR_URI}:${env:GITHUB_SHA}
+    handler: backend.app.main.handler
+    timeout: 30
+    memorySize: 512
     events:
       - http:
           path: /{proxy+}
           method: ANY
-    timeout: 900 # Max timeout to 15 mins for engine runs
+    iamRoleStatements:
+      - Effect: Allow
+        Action:
+          - sqs:SendMessage
+        Resource:
+          - Fn::GetAtt: [ EngineQueue, Arn ]
+      - Effect: Allow
+        Action:
+          - s3:GetObject
+          - s3:ListBucket
+        Resource:
+          - arn:aws:s3:::${env:PLAN_TRIGGER_BUCKET}
+          - arn:aws:s3:::${env:PLAN_TRIGGER_BUCKET}/*
+      - Effect: Allow
+        Action:
+          - s3:GetObject
+        Resource:
+          - arn:aws:s3:::${env:DATA_BUCKET}/*
+          - arn:aws:s3:::${env:ENERGY_ASSESSMENTS_BUCKET}/*
+          - arn:aws:s3:::${env:SAP_PREDICTIONS_BUCKET}/*
+          - arn:aws:s3:::${env:CARBON_PREDICTIONS_BUCKET}/*
+          - arn:aws:s3:::${env:HEAT_PREDICTIONS_BUCKET}/*
+          - arn:aws:s3:::${env:HEATING_KWH_PREDICTIONS_BUCKET}/*
+          - arn:aws:s3:::${env:HOTWATER_KWH_PREDICTIONS_BUCKET}/*
+
+  model-engine-lambda:
+    image:
+      uri: ${env:ECR_URI}:${env:GITHUB_SHA}
+    timeout: 900
+    memorySize: 2048
+    events:
+      - sqs:
+          arn: arn:aws:sqs:${self:provider.region}:${aws:accountId}:model-engine-queue
+          batchSize: 1
+    iamRoleStatements:
+      - Effect: Allow
+        Action:
+          - sqs:ReceiveMessage
+          - sqs:DeleteMessage
+          - sqs:GetQueueAttributes
+        Resource:
+          - Fn::GetAtt: [ EngineQueue, Arn ]
+      - Effect: Allow
+        Action:
+          - s3:GetObject
+          - s3:ListBucket
+        Resource:
+          - arn:aws:s3:::${env:PLAN_TRIGGER_BUCKET}
+          - arn:aws:s3:::${env:PLAN_TRIGGER_BUCKET}/*
+      - Effect: Allow
+        Action:
+          - s3:*
+        Resource:
+          - arn:aws:s3:::${env:PREDICTIONS_BUCKET}
+          - arn:aws:s3:::${env:PREDICTIONS_BUCKET}/*
+          - arn:aws:s3:::${env:DATA_BUCKET}
+          - arn:aws:s3:::${env:DATA_BUCKET}/*
+          - arn:aws:s3:::${env:ENERGY_ASSESSMENTS_BUCKET}
+          - arn:aws:s3:::${env:ENERGY_ASSESSMENTS_BUCKET}/*
+          - arn:aws:s3:::${env:SAP_PREDICTIONS_BUCKET}
+          - arn:aws:s3:::${env:SAP_PREDICTIONS_BUCKET}/*
+          - arn:aws:s3:::${env:CARBON_PREDICTIONS_BUCKET}
+          - arn:aws:s3:::${env:CARBON_PREDICTIONS_BUCKET}/*
+          - arn:aws:s3:::${env:HEAT_PREDICTIONS_BUCKET}
+          - arn:aws:s3:::${env:HEAT_PREDICTIONS_BUCKET}/*
+          - arn:aws:s3:::${env:HEATING_KWH_PREDICTIONS_BUCKET}
+          - arn:aws:s3:::${env:HEATING_KWH_PREDICTIONS_BUCKET}/*
+          - arn:aws:s3:::${env:HOTWATER_KWH_PREDICTIONS_BUCKET}
+          - arn:aws:s3:::${env:HOTWATER_KWH_PREDICTIONS_BUCKET}/*
+
+resources:
+  Resources:
+    EngineQueue:
+      Type: AWS::SQS::Queue
+      Properties:
+        QueueName: model-engine-queue