Merge pull request #141 from Hestia-Homes/main

Updating fastapi lambda to use docker image as base

.dockerignore

@@ -1 +1,13 @@
 model_data/local_data/*
+backend/tests/*
+backend/node_modules/*
+recommendations/tests/*
+model_data/tests/*
+infrastructure/*
+data_collection/*
+node_modules/*
+conservation_areas/*
+open_uprn/*
+land_registry/*
+pytest.ini
+*/README.md

.github/workflows/deploy_fastapi_backend.yml

@@ -59,16 +59,26 @@ jobs:
           echo "::set-output name=db_port::${{ secrets[format('{0}_DB_PORT', github.ref_name)] }}"
           echo "::set-output name=db_name::${{ secrets[format('{0}_DB_NAME', github.ref_name)] }}"
 
-      #      - name: Build Lambda Layer
-      #        run: |
-      #          cd backend
-      #          pip install -r requirements/lambda.txt -t python
-      #          zip -r layer.zip python
-      #
-      #      - name: Publish Lambda Layer
-      #        run: |
-      #          LAYER_ARN=$(aws lambda publish-layer-version --layer-name LambdaDependenciesLayer --zip-file fileb://backend/layer.zip | jq -r '.LayerVersionArn')
-      #          aws ssm put-parameter --name "/${{ github.ref_name }}/LambdaDependenciesLayerArn" --value "$LAYER_ARN" --type String --overwrite
+      - name: Set ECR credentials
+        id: set_ecr_credentials
+        run: |
+          echo "::set-output name=ecr_uri::${{ secrets[format('{0}_ECR_URI', github.ref_name)] }}"
+
+      - name: Setup Docker
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build Docker Image
+        run: |
+          docker build -t fastapi-lambda-image:${{ github.sha }} -f backend/docker/lambda.Dockerfile .
+
+      - name: Login to ECR
+        run: |
+          aws ecr get-login-password --region eu-west-2 | docker login --username AWS --password-stdin ${{ steps.set_ecr_credentials.outputs.ecr_uri }}
+
+      - name: Tag and Push Docker Image to ECR
+        run: |
+          docker tag fastapi-lambda-image:${{ github.sha }} ${{ steps.set_ecr_credentials.outputs.ecr_uri }}:${{ github.sha }}
+          docker push ${{ steps.set_ecr_credentials.outputs.ecr_uri }}:${{ github.sha }}
 
       - name: Deploy to AWS Lambda via Serverless
         env:
@@ -81,9 +91,10 @@ jobs:
           DB_HOST: ${{ steps.set_db_credentials.outputs.db_host }}
           DB_PORT: ${{ steps.set_db_credentials.outputs.db_port }}
           DB_NAME: ${{ steps.set_db_credentials.outputs.db_name }}
+          ECR_URI: ${{ steps.set_ecr_credentials.outputs.ecr_uri }}
         run: |
           # Fetch database credentials from AWS Secrets Manager
-          SECRET_VALUE=$(aws secretsmanager get-secret-value --secret-id dev/assessment_model/db_credentials --query SecretString)
+          SECRET_VALUE=$(aws secretsmanager get-secret-value --secret-id ${{ github.ref_name }}/assessment_model/db_credentials --query SecretString)
           DB_USERNAME=$(echo "$SECRET_VALUE" | jq -r '. | fromjson | .db_assessment_model_username')
           DB_PASSWORD=$(echo "$SECRET_VALUE" | jq -r '. | fromjson | .db_assessment_model_password')
 

README.md

@@ -10,6 +10,11 @@ part of a larger application
 
 # Folders
 
+### backend/
+
+This folder contains the code for the fastapi backend service, which provides an interface to
+much of the functionality in this repository, for the frontend
+
 ### model_data/
 
 This folder contains related to the reading and preparation of

backend/README.md

@@ -59,6 +59,26 @@ FastAPI automatically generates interactive API documentation for your applicati
 server and visit <yourlocalurl>/docs in your browser. Alternatively, you can go to
 <yourlocalurl>/redoc to view the documentation in the ReDoc format.
 
+## Building the backend docker image locally
+
+To build the backend docker image locally, run the following command from the root of the project directory:
+
+```commandline
+docker build -t fastapi-lambda-image:latest -f backend/docker/lambda.Dockerfile .
+```
+
+To check the size of the resulting image, run the following command:
+
+```bash
+docker images | grep fastapi-lambda-image
+```
+
+To run a shell inside the Docker container to inspect its contents, run:
+
+```commandline
+docker run -it fastapi-lambda-image:latest /bin/bash
+```
+
 ## Testing
 
 To run tests, run the following command from the root of the project directory:

backend/docker/lambda.Dockerfile

@@ -9,4 +9,25 @@ ENV PYTHONUNBUFFERED 1
 WORKDIR /Model
 
 # Install system dependencies
-RUN apt-get update && apt-get install -y netcat-openbsd
+RUN apt-get update && apt-get install -y netcat-openbsd
+
+# Install python dependencies
+COPY ./backend/requirements/base.txt ./backend/requirements/base.txt
+RUN pip install --upgrade pip
+# Install and clean up temp caches
+RUN pip install -r backend/requirements/base.txt && rm -rf /root/.cache
+
+# Copy project files
+COPY ./backend/ ./backend
+COPY ./recommendations/ ./recommendations
+COPY ./model_data/BaseUtility.py ./model_data/BaseUtility.py
+COPY ./model_data/config.py ./model_data/config.py
+COPY ./model_data/optimiser/ ./model_data/optimiser/
+COPY ./model_data/__init__.py ./model_data/__init__.py
+COPY ./model_data/EpcClean.py ./model_data/EpcClean.py
+COPY ./model_data/utils.py ./model_data/utils.py
+COPY ./model_data/epc_attributes/ ./model_data/epc_attributes/
+COPY ./datatypes/ ./datatypes/
+
+# Define the handler location
+CMD ["backend.app.main.handler"]

infrastructure/terraform/main.tf

@@ -98,4 +98,10 @@ module "route53" {
   providers = {
     aws.aws_use1 = aws.aws_use1
   }
-}
+}
+
+# Create an ECR repository for storage of the lambda's docker images
+module "ecr" {
+  source = "./modules/ecr"
+  environment = var.stage
+}

infrastructure/terraform/modules/ecr/main.tf

@@ -0,0 +1,29 @@
+resource "aws_ecr_repository" "my_repository" {
+  name                 = "fastapi-repository-${var.environment}"
+  image_tag_mutability = "MUTABLE"        # Allows overwriting image tags, change to IMMUTABLE if you want to prevent overwriting
+
+  image_scanning_configuration {
+    scan_on_push = true
+  }
+}
+
+resource "aws_ecr_lifecycle_policy" "my_repository_policy" {
+  repository = aws_ecr_repository.my_repository.name
+
+  policy = jsonencode({
+    rules = [
+      {
+        rulePriority = 1
+        description  = "Retain only the last 10 images"
+        selection = {
+          tagStatus   = "any"
+          countType   = "imageCountMoreThan"
+          countNumber = 10
+        }
+        action = {
+          type = "expire"
+        }
+      }
+    ]
+  })
+}

infrastructure/terraform/modules/ecr/outputs.tf

@@ -0,0 +1,4 @@
+output "ecr_repository_name" {
+  description = "Name of the EPR repo in AWS"
+  value       = aws_ecr_repository.my_repository.name
+}

infrastructure/terraform/modules/ecr/variables.tf

@@ -0,0 +1,4 @@
+variable "environment" {
+  description = "The environment for the ECR repository (dev or prod)"
+  type        = string
+}

serverless.yml

@@ -2,7 +2,6 @@ service: fastapi-lambda
 
 provider:
   name: aws
-  runtime: python3.10
   region: eu-west-2
   architecture: x86_64
   environment:
@@ -17,6 +16,7 @@ provider:
     DB_USERNAME: ${env:DB_USERNAME}
     DB_PASSWORD: ${env:DB_PASSWORD}
     DB_PORT: ${env:DB_PORT}
+    ECR_URI: ${env:ECR_URI}
   # Give lambda access to read from the bucket
   iam:
     role:
@@ -31,46 +31,11 @@ provider:
             - arn:aws:s3:::${env:PLAN_TRIGGER_BUCKET}/*
 
 
-package:
-  individually: true
-  patterns:
-    - 'backend/**'
-    - '!backend/tests/**'
-    - 'recommendations/**'
-    - '!recommendations/tests/**'
-    # Exclude all of model_data but then re-include the files we need
-    - '!model_data/**'
-    - 'model_data/BaseUtility.py'
-    - 'model_data/config.py'
-    - 'model_data/optimiser/**'
-    - 'model_data/__init__.py'
-    - 'model_data/EpcClean.py'
-    - 'model_data/utils.py'
-    - 'model_data/epc_attributes/**'
-    - 'datatypes/**'
-    - '!infrastructure/**'
-    - '!data_collection/**'
-    - '!node_modules/**'
-    - '!conservation_areas/**'
-    - '!open_uprn/**'
-    - '!land_registry/**'
-    - '!pytest.ini'
-    - '**/README.md'
-
-
 plugins:
   - serverless-python-requirements
   - serverless-domain-manager
 
 custom:
-  pythonRequirements:
-    dockerizePip: true
-    dockerFile: backend/docker/lambda.Dockerfile
-    useDocker: true
-    dockerSsh: true
-    fileName: backend/requirements/base.txt
-    dockerBuildCmdExtraArgs:
-      - '--progress=plain'
   customDomain:
     domainName: api.${self:provider.environment.DOMAIN_NAME}
     createRoute53Record: true
@@ -78,7 +43,8 @@ custom:
 
 functions:
   app:
-    handler: backend.app.main.handler
+    image:
+      uri: ${env:ECR_URI}:latest
     events:
       - http:
           path: /{proxy+}