From 0618fc05fda91fac52f61d2e6020e2fb162f9126 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 16 Apr 2025 17:10:13 +0100 Subject: [PATCH] restructuring app to split out engine code --- .github/workflows/deploy_fastapi_backend.yml | 4 +- backend/app/config.py | 1 + backend/app/plan/router.py | 32 +++++++++--- backend/docker/Dockerfile | 2 +- backend/docker/lambda.Dockerfile | 53 -------------------- backend/lambda/README.md | 8 --- backend/lambda/handler.py | 13 ----- backend/requirements/dev.txt | 4 -- backend/requirements/requirements.txt | 33 ------------ etl/non_intrusive_surveys/photos/README.md | 2 +- serverless.yml | 2 +- 11 files changed, 30 insertions(+), 124 deletions(-) delete mode 100644 backend/docker/lambda.Dockerfile delete mode 100644 backend/lambda/README.md delete mode 100644 backend/lambda/handler.py delete mode 100644 backend/requirements/dev.txt delete mode 100644 backend/requirements/requirements.txt diff --git a/.github/workflows/deploy_fastapi_backend.yml b/.github/workflows/deploy_fastapi_backend.yml index 4fc11b19..81c7367d 100644 --- a/.github/workflows/deploy_fastapi_backend.yml +++ b/.github/workflows/deploy_fastapi_backend.yml @@ -84,9 +84,9 @@ jobs: # run: | # docker buildx create --use - - name: Build Docker Image + - name: Build Docker Image For Engine run: | - docker build -t fastapi-lambda-image:${{ github.sha }} -f backend/docker/lambda.Dockerfile . --load + docker build -t fastapi-lambda-image:${{ github.sha }} -f backend/docker/engine.Dockerfile . --load - name: Login to ECR run: | diff --git a/backend/app/config.py b/backend/app/config.py index 3ab3d79d..b53d5223 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -9,6 +9,7 @@ class Settings(BaseSettings): ENVIRONMENT: str DATA_BUCKET: str PLAN_TRIGGER_BUCKET: str + ENGINE_SQS_URL: str # Third parties EPC_AUTH_TOKEN: str diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py index a7351534..405e292f 100644 --- a/backend/app/plan/router.py +++ b/backend/app/plan/router.py @@ -1,7 +1,8 @@ -import asyncio +import boto3 from fastapi import APIRouter, Depends, BackgroundTasks from backend.app.dependencies import validate_token from backend.app.plan.schemas import PlanTriggerRequest +from backend.app.config import get_settings from utils.logger import setup_logger logger = setup_logger() @@ -13,18 +14,33 @@ router = APIRouter( responses={404: {"description": "Not found"}} ) +sqs_client = boto3.client("sqs") + @router.post("/trigger", status_code=202) async def trigger_plan_entrypoint(body: PlanTriggerRequest, background_tasks: BackgroundTasks): """ - This function is the entrypoint for the plan trigger API. It will handle the request and call the - trigger_plan function. - :param body: The request body - :param background_tasks: The background tasks - :return: + Entry point for triggering the plan engine via SQS. """ logger.info("API triggered with body: %s", body) - # Kick off the async background task - # TODO: Trigger SQS job + + settings = get_settings() + + # Serialize the PlanTriggerRequest into JSON + try: + message_body = body.model_dump_json() + except Exception as e: + logger.error("Failed to serialize request body: %s", e) + return {"message": "Invalid request"}, 400 + + try: + response = sqs_client.send_message( + QueueUrl=settings.ENGINE_SQS_URL, + MessageBody=message_body + ) + logger.info(f"SQS message sent. Message ID: {response.get('MessageId')}") + except Exception as e: + logger.error("Failed to send SQS message: %s", e) + return {"message": "Failed to trigger engine"}, 500 return {"message": "Plan job accepted"} diff --git a/backend/docker/Dockerfile b/backend/docker/Dockerfile index 759d316c..9c00b9ea 100644 --- a/backend/docker/Dockerfile +++ b/backend/docker/Dockerfile @@ -15,7 +15,7 @@ RUN #apt-get update && apt-get install -y netcat-openbsd COPY ./backend/requirements/requirements.txt ./backend/requirements/requirements.txt RUN pip install --upgrade pip # Install and clean up temp caches -RUN pip install -r backend/requirements/requirements.txt && rm -rf /root/.cache +RUN pip install -r backend/requirements/lambda.requirements.txt && rm -rf /root/.cache # Since we are not using a base AWS image, there is some additional setup required. We need to set up the runtime # interface client diff --git a/backend/docker/lambda.Dockerfile b/backend/docker/lambda.Dockerfile deleted file mode 100644 index 810d14c2..00000000 --- a/backend/docker/lambda.Dockerfile +++ /dev/null @@ -1,53 +0,0 @@ -# Pull base image -FROM python:3.11.10-slim-bullseye as build-image - -# Set environment variables -ENV PYTHONDONTWRITEBYTECODE 1 -ENV PYTHONUNBUFFERED 1 - -# Set work directory to the root of your project -WORKDIR var/task/Model - -# Install system dependencies -#RUN apt-get update && apt-get install -y netcat-openbsd - -# Install python dependencies -COPY ./backend/requirements/requirements.txt ./backend/requirements/requirements.txt -# Install and clean up temp caches -RUN pip install --upgrade pip \ - && pip install -r backend/requirements/requirements.txt && rm -rf /root/.cache - -# Since we are not using a base AWS image, there is some additional setup required. We need to set up the runtime -# interface client -# https://docs.aws.amazon.com/lambda/latest/dg/python-image.html#python-image-clients -# Additionally install the AWS Lambda RIC -RUN pip install awslambdaric - -# Second stage: "runtime-image" -FROM python:3.11.10-slim-bullseye - -# Create the extensions directory to avoid warnings with RIE -RUN mkdir -p /opt/extensions - -# Set work directory to the root of your project -WORKDIR /var/task/Model - -# Copy the python dependencies from the build-image -COPY --from=build-image /usr/local/lib/python3.11/site-packages/ /usr/local/lib/python3.11/site-packages/ - -# Copy project files -COPY ./backend/ ./backend -COPY ./recommendations/ ./recommendations -COPY ./utils/ ./utils/ -COPY ./etl/epc/ ./etl/epc/ -COPY ./etl/epc_clean/ ./etl/epc_clean/ -COPY ./etl/bill_savings/ ./etl/bill_savings/ -COPY ./etl/spatial/ ./etl/spatial/ -COPY ./BaseUtility.py ./BaseUtility.py -COPY ./datatypes/ ./datatypes/ -COPY ./etl/find_my_epc/ ./etl/find_my_epc/ - -# Set the ENTRYPOINT to the AWS Lambda RIC and CMD to your function handler -ENTRYPOINT [ "/usr/local/bin/python", "-m", "awslambdaric" ] -# Define the handler location -CMD ["backend.app.main.handler"] \ No newline at end of file diff --git a/backend/lambda/README.md b/backend/lambda/README.md deleted file mode 100644 index b5c42f11..00000000 --- a/backend/lambda/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# Model Engine Lambda - -This repository contains the code for the Model Engine Lambda, which is responsible for serving machine learning models -in a serverless environment. The Lambda function is designed to handle requests for model inference and return the -results to the client. - -This service consumes an SQS queue and is triggered by messages sent to the queue. The Lambda function processes the -messages, performs model inference, and sends the results back to the client. \ No newline at end of file diff --git a/backend/lambda/handler.py b/backend/lambda/handler.py deleted file mode 100644 index 44d62d8f..00000000 --- a/backend/lambda/handler.py +++ /dev/null @@ -1,13 +0,0 @@ -import json - - -def handler(event, context): - """ - This is an ascynchonous lambda handler that will run the model engine - :param event: - :param context: - :return: - """ - for record in event["Records"]: - body = json.loads(record["body"]) - asyncio.run(model_engine(body)) diff --git a/backend/requirements/dev.txt b/backend/requirements/dev.txt deleted file mode 100644 index a466954c..00000000 --- a/backend/requirements/dev.txt +++ /dev/null @@ -1,4 +0,0 @@ -pytest -mock -pytest-cov -pytest-mock diff --git a/backend/requirements/requirements.txt b/backend/requirements/requirements.txt deleted file mode 100644 index 577776be..00000000 --- a/backend/requirements/requirements.txt +++ /dev/null @@ -1,33 +0,0 @@ -# Pandas and numpy -numpy==2.1.2 -pandas==2.2.3 -pytz==2024.2 -six==1.16.0 -# tqdm -tqdm==4.66.5 -# fastapi -fastapi==0.115.2 -sqlalchemy==2.0.36 -pydantic-settings==2.6.0 -psycopg2-binary==2.9.10 -python-jose==3.3.0 -cryptography==43.0.3 -mangum==0.19.0 -# AWS -boto3==1.35.44 -# ML, Data Science -usaddress==0.5.11 -epc-api-python==1.0.2 -fuzzywuzzy==0.18.0 -python-Levenshtein==0.26.0 -textblob==0.18.0.post0 -msgpack==1.1.0 -scikit-learn==1.5.2 -cffi==1.15.1 -mip==1.15.0 -# Data -pyarrow==17.0.0 -fastparquet==2024.5.0 -aiohttp==3.10.10 -# find my epc -beautifulsoup4 diff --git a/etl/non_intrusive_surveys/photos/README.md b/etl/non_intrusive_surveys/photos/README.md index a58603b4..914d4173 100644 --- a/etl/non_intrusive_surveys/photos/README.md +++ b/etl/non_intrusive_surveys/photos/README.md @@ -7,7 +7,7 @@ This folder contains photos taken during non-intrusive surveys. Photos are store Install the required packages by running the following command: ```bash -pip install -r requirements.txt +pip install -r lambda.requirements.txt ``` ## Usage diff --git a/serverless.yml b/serverless.yml index 116c2e29..455ae48e 100644 --- a/serverless.yml +++ b/serverless.yml @@ -37,7 +37,7 @@ plugins: custom: pythonRequirements: - fileName: backend/requirements/requirements.txt + fileName: backend/app/requirements/requirements.txt dockerizePip: true customDomain: domainName: api.${self:provider.environment.DOMAIN_NAME}