restructuring app to split out engine code

.github/workflows/deploy_fastapi_backend.yml

@@ -84,9 +84,9 @@ jobs:
       #        run: |
       #          docker buildx create --use
 
-      - name: Build Docker Image
+      - name: Build Docker Image For Engine
         run: |
-          docker build -t fastapi-lambda-image:${{ github.sha }} -f backend/docker/lambda.Dockerfile . --load
+          docker build -t fastapi-lambda-image:${{ github.sha }} -f backend/docker/engine.Dockerfile . --load
 
       - name: Login to ECR
         run: |

backend/app/config.py

@@ -9,6 +9,7 @@ class Settings(BaseSettings):
     ENVIRONMENT: str
     DATA_BUCKET: str
     PLAN_TRIGGER_BUCKET: str
+    ENGINE_SQS_URL: str
 
     # Third parties
     EPC_AUTH_TOKEN: str

backend/app/plan/router.py

@@ -1,7 +1,8 @@
-import asyncio
+import boto3
 from fastapi import APIRouter, Depends, BackgroundTasks
 from backend.app.dependencies import validate_token
 from backend.app.plan.schemas import PlanTriggerRequest
+from backend.app.config import get_settings
 from utils.logger import setup_logger
 
 logger = setup_logger()
@@ -13,18 +14,33 @@ router = APIRouter(
     responses={404: {"description": "Not found"}}
 )
 
+sqs_client = boto3.client("sqs")
+
 
 @router.post("/trigger", status_code=202)
 async def trigger_plan_entrypoint(body: PlanTriggerRequest, background_tasks: BackgroundTasks):
     """
-    This function is the entrypoint for the plan trigger API. It will handle the request and call the
+    Entry point for triggering the plan engine via SQS.
-    trigger_plan function.
-    :param body: The request body
-    :param background_tasks: The background tasks
-    :return:
     """
     logger.info("API triggered with body: %s", body)
-    # Kick off the async background task
+
-    # TODO: Trigger SQS job
+    settings = get_settings()
+
+    # Serialize the PlanTriggerRequest into JSON
+    try:
+        message_body = body.model_dump_json()
+    except Exception as e:
+        logger.error("Failed to serialize request body: %s", e)
+        return {"message": "Invalid request"}, 400
+
+    try:
+        response = sqs_client.send_message(
+            QueueUrl=settings.ENGINE_SQS_URL,
+            MessageBody=message_body
+        )
+        logger.info(f"SQS message sent. Message ID: {response.get('MessageId')}")
+    except Exception as e:
+        logger.error("Failed to send SQS message: %s", e)
+        return {"message": "Failed to trigger engine"}, 500
 
     return {"message": "Plan job accepted"}

backend/docker/Dockerfile

@@ -15,7 +15,7 @@ RUN #apt-get update && apt-get install -y netcat-openbsd
 COPY ./backend/requirements/requirements.txt ./backend/requirements/requirements.txt
 RUN pip install --upgrade pip
 # Install and clean up temp caches
-RUN pip install -r backend/requirements/requirements.txt && rm -rf /root/.cache
+RUN pip install -r backend/requirements/lambda.requirements.txt && rm -rf /root/.cache
 
 # Since we are not using a base AWS image, there is some additional setup required. We need to set up the runtime
 # interface client

backend/docker/lambda.Dockerfile

@@ -1,53 +0,0 @@
-# Pull base image
-FROM python:3.11.10-slim-bullseye as build-image
-
-# Set environment variables
-ENV PYTHONDONTWRITEBYTECODE 1
-ENV PYTHONUNBUFFERED 1
-
-# Set work directory to the root of your project
-WORKDIR var/task/Model
-
-# Install system dependencies
-#RUN apt-get update && apt-get install -y netcat-openbsd
-
-# Install python dependencies
-COPY ./backend/requirements/requirements.txt ./backend/requirements/requirements.txt
-# Install and clean up temp caches
-RUN pip install --upgrade pip \
-    && pip install -r backend/requirements/requirements.txt && rm -rf /root/.cache
-
-# Since we are not using a base AWS image, there is some additional setup required. We need to set up the runtime
-# interface client
-# https://docs.aws.amazon.com/lambda/latest/dg/python-image.html#python-image-clients
-# Additionally install the AWS Lambda RIC
-RUN pip install awslambdaric
-
-# Second stage: "runtime-image"
-FROM python:3.11.10-slim-bullseye
-
-# Create the extensions directory to avoid warnings with RIE
-RUN mkdir -p /opt/extensions
-
-# Set work directory to the root of your project
-WORKDIR /var/task/Model
-
-# Copy the python dependencies from the build-image
-COPY --from=build-image /usr/local/lib/python3.11/site-packages/ /usr/local/lib/python3.11/site-packages/
-
-# Copy project files
-COPY ./backend/ ./backend
-COPY ./recommendations/ ./recommendations
-COPY ./utils/ ./utils/
-COPY ./etl/epc/ ./etl/epc/
-COPY ./etl/epc_clean/ ./etl/epc_clean/
-COPY ./etl/bill_savings/ ./etl/bill_savings/
-COPY ./etl/spatial/ ./etl/spatial/
-COPY ./BaseUtility.py ./BaseUtility.py
-COPY ./datatypes/ ./datatypes/
-COPY ./etl/find_my_epc/ ./etl/find_my_epc/
-
-# Set the ENTRYPOINT to the AWS Lambda RIC and CMD to your function handler
-ENTRYPOINT [ "/usr/local/bin/python", "-m", "awslambdaric" ]
-# Define the handler location
-CMD ["backend.app.main.handler"]

backend/lambda/README.md

@@ -1,8 +0,0 @@
-# Model Engine Lambda
-
-This repository contains the code for the Model Engine Lambda, which is responsible for serving machine learning models
-in a serverless environment. The Lambda function is designed to handle requests for model inference and return the
-results to the client.
-
-This service consumes an SQS queue and is triggered by messages sent to the queue. The Lambda function processes the
-messages, performs model inference, and sends the results back to the client.

backend/lambda/handler.py

@@ -1,13 +0,0 @@
-import json
-
-
-def handler(event, context):
-    """
-    This is an ascynchonous lambda handler that will run the model engine
-    :param event:
-    :param context:
-    :return:
-    """
-    for record in event["Records"]:
-        body = json.loads(record["body"])
-        asyncio.run(model_engine(body))

backend/requirements/dev.txt

@@ -1,4 +0,0 @@
-pytest
-mock
-pytest-cov
-pytest-mock

backend/requirements/requirements.txt

@@ -1,33 +0,0 @@
-# Pandas and numpy
-numpy==2.1.2
-pandas==2.2.3
-pytz==2024.2
-six==1.16.0
-# tqdm
-tqdm==4.66.5
-# fastapi
-fastapi==0.115.2
-sqlalchemy==2.0.36
-pydantic-settings==2.6.0
-psycopg2-binary==2.9.10
-python-jose==3.3.0
-cryptography==43.0.3
-mangum==0.19.0
-# AWS
-boto3==1.35.44
-# ML, Data Science
-usaddress==0.5.11
-epc-api-python==1.0.2
-fuzzywuzzy==0.18.0
-python-Levenshtein==0.26.0
-textblob==0.18.0.post0
-msgpack==1.1.0
-scikit-learn==1.5.2
-cffi==1.15.1
-mip==1.15.0
-# Data
-pyarrow==17.0.0
-fastparquet==2024.5.0
-aiohttp==3.10.10
-# find my epc
-beautifulsoup4

etl/non_intrusive_surveys/photos/README.md

@@ -7,7 +7,7 @@ This folder contains photos taken during non-intrusive surveys. Photos are store
 Install the required packages by running the following command:
 
 ```bash
-pip install -r requirements.txt
+pip install -r lambda.requirements.txt
 ```
 
 ## Usage

serverless.yml

@@ -37,7 +37,7 @@ plugins:
 
 custom:
   pythonRequirements:
-    fileName: backend/requirements/requirements.txt
+    fileName: backend/app/requirements/requirements.txt
     dockerizePip: true
   customDomain:
     domainName: api.${self:provider.environment.DOMAIN_NAME}