Merge pull request #395 from Hestia-Homes/remote-assessment-api

Re-structuring the backend to allow async api calls, where fastapi just facilitates the requests and triggers the jobs via sqs

.github/workflows/deploy_fastapi_backend.yml

@@ -84,9 +84,9 @@ jobs:
       #        run: |
       #          docker buildx create --use
 
-      - name: Build Docker Image
+      - name: Build Docker Image For Engine
         run: |
-          docker build -t fastapi-lambda-image:${{ github.sha }} -f backend/docker/lambda.Dockerfile . --load
+          docker build -t fastapi-lambda-image:${{ github.sha }} -f backend/docker/engine.Dockerfile . --load
 
       - name: Login to ECR
         run: |

backend/app/config.py

@@ -9,6 +9,7 @@ class Settings(BaseSettings):
     ENVIRONMENT: str
     DATA_BUCKET: str
     PLAN_TRIGGER_BUCKET: str
+    ENGINE_SQS_URL: str
 
     # Third parties
     EPC_AUTH_TOKEN: str

backend/docker/Dockerfile

@@ -15,7 +15,7 @@ RUN #apt-get update && apt-get install -y netcat-openbsd
 COPY ./backend/requirements/requirements.txt ./backend/requirements/requirements.txt
 RUN pip install --upgrade pip
 # Install and clean up temp caches
-RUN pip install -r backend/requirements/requirements.txt && rm -rf /root/.cache
+RUN pip install -r backend/requirements/lambda.requirements.txt && rm -rf /root/.cache
 
 # Since we are not using a base AWS image, there is some additional setup required. We need to set up the runtime
 # interface client

backend/docker/lambda.Dockerfile

@@ -1,53 +0,0 @@
-# Pull base image
-FROM python:3.11.10-slim-bullseye as build-image
-
-# Set environment variables
-ENV PYTHONDONTWRITEBYTECODE 1
-ENV PYTHONUNBUFFERED 1
-
-# Set work directory to the root of your project
-WORKDIR var/task/Model
-
-# Install system dependencies
-#RUN apt-get update && apt-get install -y netcat-openbsd
-
-# Install python dependencies
-COPY ./backend/requirements/requirements.txt ./backend/requirements/requirements.txt
-# Install and clean up temp caches
-RUN pip install --upgrade pip \
-    && pip install -r backend/requirements/requirements.txt && rm -rf /root/.cache
-
-# Since we are not using a base AWS image, there is some additional setup required. We need to set up the runtime
-# interface client
-# https://docs.aws.amazon.com/lambda/latest/dg/python-image.html#python-image-clients
-# Additionally install the AWS Lambda RIC
-RUN pip install awslambdaric
-
-# Second stage: "runtime-image"
-FROM python:3.11.10-slim-bullseye
-
-# Create the extensions directory to avoid warnings with RIE
-RUN mkdir -p /opt/extensions
-
-# Set work directory to the root of your project
-WORKDIR /var/task/Model
-
-# Copy the python dependencies from the build-image
-COPY --from=build-image /usr/local/lib/python3.11/site-packages/ /usr/local/lib/python3.11/site-packages/
-
-# Copy project files
-COPY ./backend/ ./backend
-COPY ./recommendations/ ./recommendations
-COPY ./utils/ ./utils/
-COPY ./etl/epc/ ./etl/epc/
-COPY ./etl/epc_clean/ ./etl/epc_clean/
-COPY ./etl/bill_savings/ ./etl/bill_savings/
-COPY ./etl/spatial/ ./etl/spatial/
-COPY ./BaseUtility.py ./BaseUtility.py
-COPY ./datatypes/ ./datatypes/
-COPY ./etl/find_my_epc/ ./etl/find_my_epc/
-
-# Set the ENTRYPOINT to the AWS Lambda RIC and CMD to your function handler
-ENTRYPOINT [ "/usr/local/bin/python", "-m", "awslambdaric" ]
-# Define the handler location
-CMD ["backend.app.main.handler"]

backend/requirements/dev.txt

@@ -1,4 +0,0 @@
-pytest
-mock
-pytest-cov
-pytest-mock

backend/requirements/requirements.txt

@@ -1,33 +0,0 @@
-# Pandas and numpy
-numpy==2.1.2
-pandas==2.2.3
-pytz==2024.2
-six==1.16.0
-# tqdm
-tqdm==4.66.5
-# fastapi
-fastapi==0.115.2
-sqlalchemy==2.0.36
-pydantic-settings==2.6.0
-psycopg2-binary==2.9.10
-python-jose==3.3.0
-cryptography==43.0.3
-mangum==0.19.0
-# AWS
-boto3==1.35.44
-# ML, Data Science
-usaddress==0.5.11
-epc-api-python==1.0.2
-fuzzywuzzy==0.18.0
-python-Levenshtein==0.26.0
-textblob==0.18.0.post0
-msgpack==1.1.0
-scikit-learn==1.5.2
-cffi==1.15.1
-mip==1.15.0
-# Data
-pyarrow==17.0.0
-fastparquet==2024.5.0
-aiohttp==3.10.10
-# find my epc
-beautifulsoup4

etl/non_intrusive_surveys/photos/README.md

@@ -7,7 +7,7 @@ This folder contains photos taken during non-intrusive surveys. Photos are store
 Install the required packages by running the following command:
 
 ```bash
-pip install -r requirements.txt
+pip install -r lambda.requirements.txt
 ```
 
 ## Usage

serverless.yml

@@ -1,8 +1,9 @@
-service: fastapi-lambda
+service: retrofit-platform
 
 provider:
   name: aws
   region: eu-west-2
+  runtime: python3.11
   architecture: x86_64
   environment:
     API_KEY: ${env:API_KEY}
@@ -23,62 +24,109 @@ provider:
     SAP_PREDICTIONS_BUCKET: ${env:SAP_PREDICTIONS_BUCKET}
     CARBON_PREDICTIONS_BUCKET: ${env:CARBON_PREDICTIONS_BUCKET}
     HEAT_PREDICTIONS_BUCKET: ${env:HEAT_PREDICTIONS_BUCKET}
-    #    LIGHTING_COST_PREDICTIONS_BUCKET: ${env:LIGHTING_COST_PREDICTIONS_BUCKET}
-    #    HEATING_COST_PREDICTIONS_BUCKET: ${env:HEATING_COST_PREDICTIONS_BUCKET}
-    #    HOT_WATER_COST_PREDICTIONS_BUCKET: ${env:HOT_WATER_COST_PREDICTIONS_BUCKET}
     HEATING_KWH_PREDICTIONS_BUCKET: ${env:HEATING_KWH_PREDICTIONS_BUCKET}
     HOTWATER_KWH_PREDICTIONS_BUCKET: ${env:HOTWATER_KWH_PREDICTIONS_BUCKET}
     ENERGY_ASSESSMENTS_BUCKET: ${env:ENERGY_ASSESSMENTS_BUCKET}
     GOOGLE_SOLAR_API_KEY: ${env:GOOGLE_SOLAR_API_KEY}
-  # Give lambda access to read from the bucket
-  iam:
-    role:
-      name: fastapi_backend_${env:PLAN_TRIGGER_BUCKET}_access
-      statements:
-        - Effect: Allow
-          Action:
-            - s3:GetObject
-            - s3:ListBucket
-          Resource:
-            - arn:aws:s3:::${env:PLAN_TRIGGER_BUCKET}
-            - arn:aws:s3:::${env:PLAN_TRIGGER_BUCKET}/*
-        - Effect: Allow
-          Action:
-            - s3:*
-          Resource:
-            - arn:aws:s3:::${env:PREDICTIONS_BUCKET}
-            - arn:aws:s3:::${env:PREDICTIONS_BUCKET}/*
-            - arn:aws:s3:::${env:DATA_BUCKET}
-            - arn:aws:s3:::${env:DATA_BUCKET}/*
-            - arn:aws:s3:::${env:ENERGY_ASSESSMENTS_BUCKET}
-            - arn:aws:s3:::${env:ENERGY_ASSESSMENTS_BUCKET}/*
-            - arn:aws:s3:::${env:SAP_PREDICTIONS_BUCKET}
-            - arn:aws:s3:::${env:SAP_PREDICTIONS_BUCKET}/*
-            - arn:aws:s3:::${env:CARBON_PREDICTIONS_BUCKET}
-            - arn:aws:s3:::${env:CARBON_PREDICTIONS_BUCKET}/*
-            - arn:aws:s3:::${env:HEAT_PREDICTIONS_BUCKET}
-            - arn:aws:s3:::${env:HEAT_PREDICTIONS_BUCKET}/*
-            - arn:aws:s3:::${env:HEATING_KWH_PREDICTIONS_BUCKET}
-            - arn:aws:s3:::${env:HEATING_KWH_PREDICTIONS_BUCKET}/*
-            - arn:aws:s3:::${env:HOTWATER_KWH_PREDICTIONS_BUCKET}
-            - arn:aws:s3:::${env:HOTWATER_KWH_PREDICTIONS_BUCKET}/*
-
+    ENGINE_SQS_URL:
+      Ref: EngineQueue
 
 plugins:
+  - serverless-python-requirements
   - serverless-domain-manager
 
 custom:
+  pythonRequirements:
+    fileName: backend/app/requirements/requirements.txt
+    dockerizePip: true
   customDomain:
     domainName: api.${self:provider.environment.DOMAIN_NAME}
     createRoute53Record: true
     certificateArn: ${ssm:/ssl_certificate_arn}
 
 functions:
-  app:
-    image:
-      uri: ${env:ECR_URI}:${env:GITHUB_SHA}
+
+  fastapi-backend:
+    handler: backend.app.main.handler
+    timeout: 30
+    memorySize: 512
     events:
       - http:
           path: /{proxy+}
           method: ANY
-    timeout: 900 # Max timeout to 15 mins for engine runs
+    iamRoleStatements:
+      - Effect: Allow
+        Action:
+          - sqs:SendMessage
+        Resource:
+          - Fn::GetAtt: [ EngineQueue, Arn ]
+      - Effect: Allow
+        Action:
+          - s3:GetObject
+          - s3:ListBucket
+        Resource:
+          - arn:aws:s3:::${env:PLAN_TRIGGER_BUCKET}
+          - arn:aws:s3:::${env:PLAN_TRIGGER_BUCKET}/*
+      - Effect: Allow
+        Action:
+          - s3:GetObject
+        Resource:
+          - arn:aws:s3:::${env:DATA_BUCKET}/*
+          - arn:aws:s3:::${env:ENERGY_ASSESSMENTS_BUCKET}/*
+          - arn:aws:s3:::${env:SAP_PREDICTIONS_BUCKET}/*
+          - arn:aws:s3:::${env:CARBON_PREDICTIONS_BUCKET}/*
+          - arn:aws:s3:::${env:HEAT_PREDICTIONS_BUCKET}/*
+          - arn:aws:s3:::${env:HEATING_KWH_PREDICTIONS_BUCKET}/*
+          - arn:aws:s3:::${env:HOTWATER_KWH_PREDICTIONS_BUCKET}/*
+
+  model-engine-lambda:
+    image:
+      uri: ${env:ECR_URI}:${env:GITHUB_SHA}
+    timeout: 900
+    memorySize: 2048
+    events:
+      - sqs:
+          arn: arn:aws:sqs:${self:provider.region}:${aws:accountId}:model-engine-queue
+          batchSize: 1
+    iamRoleStatements:
+      - Effect: Allow
+        Action:
+          - sqs:ReceiveMessage
+          - sqs:DeleteMessage
+          - sqs:GetQueueAttributes
+        Resource:
+          - Fn::GetAtt: [ EngineQueue, Arn ]
+      - Effect: Allow
+        Action:
+          - s3:GetObject
+          - s3:ListBucket
+        Resource:
+          - arn:aws:s3:::${env:PLAN_TRIGGER_BUCKET}
+          - arn:aws:s3:::${env:PLAN_TRIGGER_BUCKET}/*
+      - Effect: Allow
+        Action:
+          - s3:*
+        Resource:
+          - arn:aws:s3:::${env:PREDICTIONS_BUCKET}
+          - arn:aws:s3:::${env:PREDICTIONS_BUCKET}/*
+          - arn:aws:s3:::${env:DATA_BUCKET}
+          - arn:aws:s3:::${env:DATA_BUCKET}/*
+          - arn:aws:s3:::${env:ENERGY_ASSESSMENTS_BUCKET}
+          - arn:aws:s3:::${env:ENERGY_ASSESSMENTS_BUCKET}/*
+          - arn:aws:s3:::${env:SAP_PREDICTIONS_BUCKET}
+          - arn:aws:s3:::${env:SAP_PREDICTIONS_BUCKET}/*
+          - arn:aws:s3:::${env:CARBON_PREDICTIONS_BUCKET}
+          - arn:aws:s3:::${env:CARBON_PREDICTIONS_BUCKET}/*
+          - arn:aws:s3:::${env:HEAT_PREDICTIONS_BUCKET}
+          - arn:aws:s3:::${env:HEAT_PREDICTIONS_BUCKET}/*
+          - arn:aws:s3:::${env:HEATING_KWH_PREDICTIONS_BUCKET}
+          - arn:aws:s3:::${env:HEATING_KWH_PREDICTIONS_BUCKET}/*
+          - arn:aws:s3:::${env:HOTWATER_KWH_PREDICTIONS_BUCKET}
+          - arn:aws:s3:::${env:HOTWATER_KWH_PREDICTIONS_BUCKET}/*
+
+resources:
+  Resources:
+    EngineQueue:
+      Type: AWS::SQS::Queue
+      Properties:
+        QueueName: model-engine-queue