diff --git a/.dockerignore b/.dockerignore index f5c7b106..28718547 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1,13 @@ model_data/local_data/* +backend/tests/* +backend/node_modules/* +recommendations/tests/* +model_data/tests/* +infrastructure/* +data_collection/* +node_modules/* +conservation_areas/* +open_uprn/* +land_registry/* +pytest.ini +*/README.md diff --git a/.github/workflows/deploy_fastapi_backend.yml b/.github/workflows/deploy_fastapi_backend.yml index b8c533f9..8028d428 100644 --- a/.github/workflows/deploy_fastapi_backend.yml +++ b/.github/workflows/deploy_fastapi_backend.yml @@ -59,16 +59,26 @@ jobs: echo "::set-output name=db_port::${{ secrets[format('{0}_DB_PORT', github.ref_name)] }}" echo "::set-output name=db_name::${{ secrets[format('{0}_DB_NAME', github.ref_name)] }}" - # - name: Build Lambda Layer - # run: | - # cd backend - # pip install -r requirements/lambda.txt -t python - # zip -r layer.zip python - # - # - name: Publish Lambda Layer - # run: | - # LAYER_ARN=$(aws lambda publish-layer-version --layer-name LambdaDependenciesLayer --zip-file fileb://backend/layer.zip | jq -r '.LayerVersionArn') - # aws ssm put-parameter --name "/${{ github.ref_name }}/LambdaDependenciesLayerArn" --value "$LAYER_ARN" --type String --overwrite + - name: Set ECR credentials + id: set_ecr_credentials + run: | + echo "::set-output name=ecr_uri::${{ secrets[format('{0}_ECR_URI', github.ref_name)] }}" + + - name: Setup Docker + uses: docker/setup-buildx-action@v1 + + - name: Build Docker Image + run: | + docker build -t fastapi-lambda-image:${{ github.sha }} -f backend/docker/lambda.Dockerfile . + + - name: Login to ECR + run: | + aws ecr get-login-password --region eu-west-2 | docker login --username AWS --password-stdin ${{ steps.set_ecr_credentials.outputs.ecr_uri }} + + - name: Tag and Push Docker Image to ECR + run: | + docker tag fastapi-lambda-image:${{ github.sha }} ${{ steps.set_ecr_credentials.outputs.ecr_uri }}:${{ github.sha }} + docker push ${{ steps.set_ecr_credentials.outputs.ecr_uri }}:${{ github.sha }} - name: Deploy to AWS Lambda via Serverless env: @@ -81,9 +91,10 @@ jobs: DB_HOST: ${{ steps.set_db_credentials.outputs.db_host }} DB_PORT: ${{ steps.set_db_credentials.outputs.db_port }} DB_NAME: ${{ steps.set_db_credentials.outputs.db_name }} + ECR_URI: ${{ steps.set_ecr_credentials.outputs.ecr_uri }} run: | # Fetch database credentials from AWS Secrets Manager - SECRET_VALUE=$(aws secretsmanager get-secret-value --secret-id dev/assessment_model/db_credentials --query SecretString) + SECRET_VALUE=$(aws secretsmanager get-secret-value --secret-id ${{ github.ref_name }}/assessment_model/db_credentials --query SecretString) DB_USERNAME=$(echo "$SECRET_VALUE" | jq -r '. | fromjson | .db_assessment_model_username') DB_PASSWORD=$(echo "$SECRET_VALUE" | jq -r '. | fromjson | .db_assessment_model_password') diff --git a/README.md b/README.md index 63cd9ac8..df36cfe7 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,11 @@ part of a larger application # Folders +### backend/ + +This folder contains the code for the fastapi backend service, which provides an interface to +much of the functionality in this repository, for the frontend + ### model_data/ This folder contains related to the reading and preparation of diff --git a/backend/README.md b/backend/README.md index 2817fd8f..70739265 100644 --- a/backend/README.md +++ b/backend/README.md @@ -59,6 +59,26 @@ FastAPI automatically generates interactive API documentation for your applicati server and visit /docs in your browser. Alternatively, you can go to /redoc to view the documentation in the ReDoc format. +## Building the backend docker image locally + +To build the backend docker image locally, run the following command from the root of the project directory: + +```commandline +docker build -t fastapi-lambda-image:latest -f backend/docker/lambda.Dockerfile . +``` + +To check the size of the resulting image, run the following command: + +```bash +docker images | grep fastapi-lambda-image +``` + +To run a shell inside the Docker container to inspect its contents, run: + +```commandline +docker run -it fastapi-lambda-image:latest /bin/bash +``` + ## Testing To run tests, run the following command from the root of the project directory: diff --git a/backend/docker/lambda.Dockerfile b/backend/docker/lambda.Dockerfile index 40a1e927..9e16c21a 100644 --- a/backend/docker/lambda.Dockerfile +++ b/backend/docker/lambda.Dockerfile @@ -9,4 +9,25 @@ ENV PYTHONUNBUFFERED 1 WORKDIR /Model # Install system dependencies -RUN apt-get update && apt-get install -y netcat-openbsd \ No newline at end of file +RUN apt-get update && apt-get install -y netcat-openbsd + +# Install python dependencies +COPY ./backend/requirements/base.txt ./backend/requirements/base.txt +RUN pip install --upgrade pip +# Install and clean up temp caches +RUN pip install -r backend/requirements/base.txt && rm -rf /root/.cache + +# Copy project files +COPY ./backend/ ./backend +COPY ./recommendations/ ./recommendations +COPY ./model_data/BaseUtility.py ./model_data/BaseUtility.py +COPY ./model_data/config.py ./model_data/config.py +COPY ./model_data/optimiser/ ./model_data/optimiser/ +COPY ./model_data/__init__.py ./model_data/__init__.py +COPY ./model_data/EpcClean.py ./model_data/EpcClean.py +COPY ./model_data/utils.py ./model_data/utils.py +COPY ./model_data/epc_attributes/ ./model_data/epc_attributes/ +COPY ./datatypes/ ./datatypes/ + +# Define the handler location +CMD ["backend.app.main.handler"] diff --git a/infrastructure/terraform/main.tf b/infrastructure/terraform/main.tf index f48da21f..95fa5e06 100644 --- a/infrastructure/terraform/main.tf +++ b/infrastructure/terraform/main.tf @@ -98,4 +98,10 @@ module "route53" { providers = { aws.aws_use1 = aws.aws_use1 } -} \ No newline at end of file +} + +# Create an ECR repository for storage of the lambda's docker images +module "ecr" { + source = "./modules/ecr" + environment = var.stage +} diff --git a/infrastructure/terraform/modules/ecr/main.tf b/infrastructure/terraform/modules/ecr/main.tf new file mode 100644 index 00000000..5a30c3cf --- /dev/null +++ b/infrastructure/terraform/modules/ecr/main.tf @@ -0,0 +1,29 @@ +resource "aws_ecr_repository" "my_repository" { + name = "fastapi-repository-${var.environment}" + image_tag_mutability = "MUTABLE" # Allows overwriting image tags, change to IMMUTABLE if you want to prevent overwriting + + image_scanning_configuration { + scan_on_push = true + } +} + +resource "aws_ecr_lifecycle_policy" "my_repository_policy" { + repository = aws_ecr_repository.my_repository.name + + policy = jsonencode({ + rules = [ + { + rulePriority = 1 + description = "Retain only the last 10 images" + selection = { + tagStatus = "any" + countType = "imageCountMoreThan" + countNumber = 10 + } + action = { + type = "expire" + } + } + ] + }) +} \ No newline at end of file diff --git a/infrastructure/terraform/modules/ecr/outputs.tf b/infrastructure/terraform/modules/ecr/outputs.tf new file mode 100644 index 00000000..53839718 --- /dev/null +++ b/infrastructure/terraform/modules/ecr/outputs.tf @@ -0,0 +1,4 @@ +output "ecr_repository_name" { + description = "Name of the EPR repo in AWS" + value = aws_ecr_repository.my_repository.name +} \ No newline at end of file diff --git a/infrastructure/terraform/modules/ecr/variables.tf b/infrastructure/terraform/modules/ecr/variables.tf new file mode 100644 index 00000000..108bb626 --- /dev/null +++ b/infrastructure/terraform/modules/ecr/variables.tf @@ -0,0 +1,4 @@ +variable "environment" { + description = "The environment for the ECR repository (dev or prod)" + type = string +} \ No newline at end of file diff --git a/serverless.yml b/serverless.yml index afd76c81..28896ad3 100644 --- a/serverless.yml +++ b/serverless.yml @@ -2,7 +2,6 @@ service: fastapi-lambda provider: name: aws - runtime: python3.10 region: eu-west-2 architecture: x86_64 environment: @@ -17,6 +16,7 @@ provider: DB_USERNAME: ${env:DB_USERNAME} DB_PASSWORD: ${env:DB_PASSWORD} DB_PORT: ${env:DB_PORT} + ECR_URI: ${env:ECR_URI} # Give lambda access to read from the bucket iam: role: @@ -31,46 +31,11 @@ provider: - arn:aws:s3:::${env:PLAN_TRIGGER_BUCKET}/* -package: - individually: true - patterns: - - 'backend/**' - - '!backend/tests/**' - - 'recommendations/**' - - '!recommendations/tests/**' - # Exclude all of model_data but then re-include the files we need - - '!model_data/**' - - 'model_data/BaseUtility.py' - - 'model_data/config.py' - - 'model_data/optimiser/**' - - 'model_data/__init__.py' - - 'model_data/EpcClean.py' - - 'model_data/utils.py' - - 'model_data/epc_attributes/**' - - 'datatypes/**' - - '!infrastructure/**' - - '!data_collection/**' - - '!node_modules/**' - - '!conservation_areas/**' - - '!open_uprn/**' - - '!land_registry/**' - - '!pytest.ini' - - '**/README.md' - - plugins: - serverless-python-requirements - serverless-domain-manager custom: - pythonRequirements: - dockerizePip: true - dockerFile: backend/docker/lambda.Dockerfile - useDocker: true - dockerSsh: true - fileName: backend/requirements/base.txt - dockerBuildCmdExtraArgs: - - '--progress=plain' customDomain: domainName: api.${self:provider.environment.DOMAIN_NAME} createRoute53Record: true @@ -78,7 +43,8 @@ custom: functions: app: - handler: backend.app.main.handler + image: + uri: ${env:ECR_URI}:latest events: - http: path: /{proxy+}