test deployment

This commit is contained in:
Jun-te Kim 2026-02-03 11:12:25 +00:00
parent 5cf3644cf4
commit ef3e808c9c
37 changed files with 736 additions and 113 deletions

79
.github/workflows/_build_image.yml vendored Normal file
View file

@ -0,0 +1,79 @@
name: Build Docker image
on:
workflow_call:
inputs:
ecr_repo:
description: "ECR repository name"
required: true
type: string
aws_region:
description: "AWS region"
required: true
type: string
dockerfile_path:
description: "Path to Dockerfile"
required: true
type: string
build_context:
description: "Docker build context directory"
required: false
default: "."
type: string
outputs:
image_digest:
description: "Pushed image digest"
value: ${{ jobs.build.outputs.image_digest }}
secrets:
AWS_ACCESS_KEY_ID:
required: true
AWS_SECRET_ACCESS_KEY:
required: true
AWS_ACCOUNT_ID:
required: true
jobs:
build:
runs-on: ubuntu-latest
outputs:
image_digest: ${{ steps.digest.outputs.image_digest }}
steps:
- uses: actions/checkout@v4
- uses: aws-actions/configure-aws-credentials@v4
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ inputs.aws_region }}
- uses: aws-actions/amazon-ecr-login@v2
- name: Build & push image
run: |
IMAGE_TAG=${GITHUB_SHA}
IMAGE_URI=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ inputs.aws_region }}.amazonaws.com/${{ inputs.ecr_repo }}:${IMAGE_TAG}
docker build \
-f ${{ inputs.dockerfile_path }} \
-t $IMAGE_URI \
${{ inputs.build_context }}
docker push $IMAGE_URI
- name: Resolve image digest
id: digest
run: |
DIGEST=$(aws ecr describe-images \
--repository-name ${{ inputs.ecr_repo }} \
--image-ids imageTag=${GITHUB_SHA} \
--query 'imageDetails[0].imageDigest' \
--output text)
echo "image_digest=$DIGEST" >> $GITHUB_OUTPUT

68
.github/workflows/_deploy_lambda.yml vendored Normal file
View file

@ -0,0 +1,68 @@
name: Deploy Lambda (Terraform)
on:
workflow_call:
inputs:
lambda_name:
required: true
type: string
lambda_path:
required: true
type: string
stage:
required: true
type: string
aws_region:
required: true
type: string
image_digest:
required: true
type: string
secrets:
AWS_ACCESS_KEY_ID:
required: true
AWS_SECRET_ACCESS_KEY:
required: true
AWS_ACCOUNT_ID:
required: true
jobs:
deploy:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: aws-actions/configure-aws-credentials@v4
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ inputs.aws_region }}
- uses: hashicorp/setup-terraform@v3
- name: Terraform Init
working-directory: ${{ inputs.lambda_path }}
run: terraform init -reconfigure
- name: Terraform Workspace
working-directory: ${{ inputs.lambda_path }}
run: |
terraform workspace select ${{ inputs.stage }} \
|| terraform workspace new ${{ inputs.stage }}
- name: Terraform Plan
working-directory: ${{ inputs.lambda_path }}
run: |
terraform plan \
-var="stage=${{ inputs.stage }}" \
-var="image_digest=${{ inputs.image_digest }}"
# - name: Terraform Apply
# working-directory: ${{ inputs.lambda_path }}
# run: |
# terraform apply \
# -auto-approve \
# -var="stage=${{ inputs.stage }}" \
# -var="image_digest=${{ inputs.image_digest }}"

View file

@ -1,13 +1,23 @@
name: Deploy terraform stack
name: Deploy infrastructure
on:
push:
branches:
- "**"
env:
AWS_REGION: eu-west-2
# Temporary until we have more environemnts. You'll just need export STAGE dynamically in the future
STAGE: dev
jobs:
deploy_shared_terraform_stack:
# ============================================================
# 1⃣ Shared Terraform (plan only for now)
# ============================================================
shared_terraform:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
@ -15,48 +25,60 @@ jobs:
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v4
with:
aws-access-key-id: "${{ secrets.DEV_AWS_ACCESS_KEY_ID }}"
aws-secret-access-key: "${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }}"
aws-region: eu-west-2
# This will need to be changed to env imports when we have different env to dynamically allocate prod, staging etc
aws-access-key-id: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ env.AWS_REGION }}
- name: Setup Terraform
uses: hashicorp/setup-terraform@v3
- name: Terraform Init
working-directory: ./infrastructure/terraform/shared/
shell: bash
- name: Terraform Init (shared)
working-directory: infrastructure/terraform/shared
run: terraform init -reconfigure
- name: Terraform Workspace
working-directory: ./infrastructure/terraform/shared/
shell: bash
run: terraform workspace select dev || terraform workspace new dev
- name: Terraform Workspace (shared)
working-directory: infrastructure/terraform/shared
run: |
terraform workspace select ${STAGE} \
|| terraform workspace new ${STAGE}
- name: Terraform Plan (shared)
working-directory: ./infrastructure/terraform/shared/
shell: bash
run: terraform plan -var-file=dev.tfvars -out=tfplan
working-directory: infrastructure/terraform/shared
run: terraform plan -var-file=${STAGE}.tfvars
# - name: Terraform Apply
# working-directory: ./infrastructure/terraform/shared
# shell: bash
# run: terraform apply -auto-approve tfplan
# # apply shared dev
# - name: Terraform Apply (shared)
# run: |
# cd infrastructure/terraform/shared
# terraform apply -auto-approve -var-file=dev.tfvars
# working-directory: infrastructure/terraform/shared
# run: terraform apply -auto-approve -var-file=${STAGE}.tfvars
# - name: Build & push image
# run: |
# IMAGE_TAG=address2uprn-${GITHUB_SHA}
# IMAGE_URI=${AWS_ACCOUNT_ID}.dkr.ecr.eu-west-2.amazonaws.com/lambda-shared-dev:${IMAGE_TAG}
# docker build -t $IMAGE_URI .
# docker push $IMAGE_URI
# echo "IMAGE_URI=$IMAGE_URI" >> $GITHUB_ENV
# # ============================================================
# # 2⃣ Build Docker image (tag = GitHub SHA, digest resolved)
# # ============================================================
# image:
# uses: ./.github/workflows/_build_docker_image.yml
# with:
# ecr_repo: address2uprn-dev
# aws_region: ${{ env.AWS_REGION }}
# dockerfile_path: backend/address2UPRN/Dockerfile
# build_context: backend/address2UPRN
# secrets:
# AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }}
# AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }}
# AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }}
# # ============================================================
# # 3⃣ Deploy Lambda (Terraform, immutable digest)
# # ============================================================
# deploy_lambda:
# needs: image
# uses: ./.github/workflows/_deploy_lambda.yml
# with:
# lambda_name: address2uprn
# lambda_path: infrastructure/terraform/lambda/address2uprn
# stage: ${{ env.STAGE }}
# aws_region: ${{ env.AWS_REGION }}
# image_digest: ${{ needs.image.outputs.image_digest }}
# secrets:
# AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }}
# AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }}
# AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }}

View file

@ -0,0 +1,7 @@
### Checklist for a new lambda
- [ ] Copy cp -r lambda/_template lambda/<name>
- [ ] Set `state_bucket_name`
- [ ] Add ECR repo in shared/main.tf
- [ ] Add shared output for repo name/url
- [ ] Push to GitHub (CI will deploy)

View file

@ -0,0 +1,21 @@
data "terraform_remote_state" "shared" {
backend = "s3"
config = {
bucket = "assessment-model-terraform-state"
key = "terraform.tfstate"
region = "eu-west-2"
}
}
module "lambda" {
source = "../modules/lambda_with_sqs"
name = "REPLACE_ME"
stage = var.stage
image_uri = "${data.terraform_remote_state.shared.outputs.REPLACE_ME_repository_url}@${var.image_digest}"
environment = {
STAGE = var.stage
}
}

View file

@ -0,0 +1,20 @@
terraform {
required_providers {
aws = {
source = "hashicorp/aws"
version = "~> 4.16"
}
}
backend "s3" {
bucket = var.state_bucket_name
key = "terraform.tfstate"
region = "eu-west-2"
}
required_version = ">= 1.2.0"
}
provider "aws" {
region = var.region
}

View file

@ -0,0 +1,17 @@
variable "region" {
type = string
default = "eu-west-2"
}
variable "stage" {
type = string
}
variable "image_digest" {
type = string
}
variable "state_bucket_name" {
type = string
description = "S3 bucket name used for this lambda's Terraform state"
}

View file

@ -0,0 +1,88 @@
############################################
# Read shared state to get outputs
############################################
data "terraform_remote_state" "shared" {
backend = "s3"
config = {
bucket = "assessment-model-terraform-state"
key = "terraform.tfstate"
region = "eu-west-2"
}
}
############################################
# IAM role
############################################
module "role" {
source = "../../modules/lambda_execution_role"
name = "address2uprn-lambda-${var.stage}"
}
############################################
# SQS queue
############################################
module "queue" {
source = "../../modules/sqs_queue"
name = "address2uprn-queue-${var.stage}"
}
############################################
# Lambda (image-based)
############################################
module "lambda" {
source = "../../modules/lambda_service"
name = "address2uprn-${var.stage}"
role_arn = module.role.role_arn
image_uri = "${data.terraform_remote_state.shared.outputs.address2uprn_repository_url}@${var.image_digest}"
timeout = 60
memory_size = 1024
environment = {
STAGE = var.stage
LOG_LEVEL = "info"
}
}
############################################
# SQS Lambda trigger
############################################
module "sqs_trigger" {
source = "../../modules/lambda_sqs_trigger"
lambda_arn = module.lambda.lambda_arn
lambda_role_name = module.role.role_name
queue_arn = module.queue.queue_arn
}
############################################
# Read shared state to get outputs
############################################
data "terraform_remote_state" "shared" {
backend = "s3"
config = {
bucket = "assessment-model-terraform-state"
key = "terraform.tfstate"
region = "eu-west-2"
}
}
############################################
# Address2UPRN Lambda (via reusable module)
############################################
module "address2uprn" {
source = "../modules/lambda_with_sqs"
name = "address2uprn"
stage = var.stage
image_uri = "${data.terraform_remote_state.shared.outputs.address2uprn_repository_url}@${var.image_digest}"
environment = {
STAGE = var.stage
LOG_LEVEL = "info"
}
}

View file

@ -0,0 +1,20 @@
terraform {
required_providers {
aws = {
source = "hashicorp/aws"
version = "~> 4.16"
}
}
backend "s3" {
bucket = "address2uprn-terraform-state"
key = "terraform.tfstate"
region = "eu-west-2"
}
required_version = ">= 1.2.0"
}
provider "aws" {
region = var.region
}

View file

@ -0,0 +1,13 @@
variable "region" {
type = string
default = "eu-west-2"
}
variable "stage" {
type = string
}
variable "image_digest" {
type = string
description = "sha256 image digest from CI"
}

View file

@ -0,0 +1,44 @@
############################################
# IAM role
############################################
module "role" {
source = "../../../modules/lambda_execution_role"
name = "${var.name}-lambda-${var.stage}"
}
############################################
# SQS queue + DLQ
############################################
module "queue" {
source = "../../../modules/sqs_queue"
name = "${var.name}-queue-${var.stage}"
}
############################################
# Lambda
############################################
module "lambda" {
source = "../../../modules/lambda_service"
name = "${var.name}-${var.stage}"
role_arn = module.role.role_arn
image_uri = var.image_uri
timeout = var.timeout
memory_size = var.memory_size
environment = var.environment
}
############################################
# SQS Lambda trigger
############################################
module "sqs_trigger" {
source = "../../../modules/lambda_sqs_trigger"
lambda_arn = module.lambda.lambda_arn
lambda_role_name = module.role.role_name
queue_arn = module.queue.queue_arn
batch_size = var.batch_size
}

View file

@ -0,0 +1,11 @@
output "lambda_arn" {
value = module.lambda.lambda_arn
}
output "queue_arn" {
value = module.queue.queue_arn
}
output "queue_url" {
value = module.queue.queue_url
}

View file

@ -0,0 +1,36 @@
variable "name" {
type = string
}
variable "stage" {
type = string
}
variable "image_uri" {
type = string
}
variable "region" {
type = string
default = "eu-west-2"
}
variable "timeout" {
type = number
default = 60
}
variable "memory_size" {
type = number
default = 1024
}
variable "environment" {
type = map(string)
default = {}
}
variable "batch_size" {
type = number
default = 10
}

View file

@ -0,0 +1,30 @@
resource "aws_ecr_repository" "this" {
name = "${var.name}-${var.stage}"
image_tag_mutability = "MUTABLE"
image_scanning_configuration {
scan_on_push = true
}
}
resource "aws_ecr_lifecycle_policy" "this" {
repository = aws_ecr_repository.this.name
policy = jsonencode({
rules = [
{
rulePriority = 1
description = "Expire old images"
selection = {
tagStatus = "any"
countType = "imageCountMoreThan"
countNumber = var.retain_count
}
action = {
type = "expire"
}
}
]
})
}

View file

@ -0,0 +1,11 @@
output "repository_name" {
value = aws_ecr_repository.this.name
}
output "repository_url" {
value = aws_ecr_repository.this.repository_url
}
output "repository_arn" {
value = aws_ecr_repository.this.arn
}

View file

@ -0,0 +1,15 @@
variable "name" {
description = "Base name of the repository (without stage)"
type = string
}
variable "stage" {
description = "Deployment stage (e.g. dev, prod)"
type = string
}
variable "retain_count" {
description = "Number of images to retain"
type = number
default = 20
}

View file

@ -1,3 +1,8 @@
# This ECR module is used in Khalim's default code, Junte tried changing it
# but decided to priotise delivariables as sales projects are coming soon
# one day we can unify ECR policies together but Junte decided to seperate
# the continaer lambda as it runs slighly differently
resource "aws_ecr_repository" "my_repository" {
name = "${var.ecr_name}"
image_tag_mutability = "MUTABLE"

View file

@ -0,0 +1,37 @@
data "aws_iam_policy_document" "assume" {
statement {
effect = "Allow"
principals {
type = "Service"
identifiers = ["lambda.amazonaws.com"]
}
actions = ["sts:AssumeRole"]
}
}
resource "aws_iam_role" "this" {
name = var.name
assume_role_policy = data.aws_iam_policy_document.assume.json
}
resource "aws_iam_role_policy_attachment" "basic_logs" {
role = aws_iam_role.this.name
policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
}
resource "aws_iam_role_policy" "ecr_pull" {
role = aws_iam_role.this.name
policy = jsonencode({
Version = "2012-10-17"
Statement = [{
Effect = "Allow"
Action = [
"ecr:GetAuthorizationToken",
"ecr:BatchGetImage",
"ecr:GetDownloadUrlForLayer"
]
Resource = "*"
}]
})
}

View file

@ -0,0 +1,7 @@
output "role_arn" {
value = aws_iam_role.this.arn
}
output "role_name" {
value = aws_iam_role.this.name
}

View file

@ -0,0 +1,15 @@
resource "aws_lambda_function" "this" {
function_name = var.name
role = var.role_arn
package_type = "Image"
image_uri = var.image_uri
timeout = var.timeout
memory_size = var.memory_size
publish = true
environment {
variables = var.environment
}
}

View file

@ -0,0 +1,3 @@
output "lambda_arn" {
value = aws_lambda_function.this.arn
}

View file

@ -0,0 +1,18 @@
variable "name" { type = string }
variable "role_arn" { type = string }
variable "image_uri" { type = string }
variable "timeout" {
type = number
default = 30
}
variable "memory_size" {
type = number
default = 512
}
variable "environment" {
type = map(string)
default = {}
}

View file

@ -0,0 +1,23 @@
resource "aws_lambda_event_source_mapping" "this" {
event_source_arn = var.queue_arn
function_name = var.lambda_arn
batch_size = var.batch_size
enabled = true
}
resource "aws_iam_role_policy" "allow_sqs" {
role = var.lambda_role_name
policy = jsonencode({
Version = "2012-10-17"
Statement = [{
Effect = "Allow"
Action = [
"sqs:ReceiveMessage",
"sqs:DeleteMessage",
"sqs:GetQueueAttributes"
]
Resource = var.queue_arn
}]
})
}

View file

@ -0,0 +1,8 @@
variable "lambda_arn" { type = string }
variable "lambda_role_name" { type = string }
variable "queue_arn" { type = string }
variable "batch_size" {
type = number
default = 10
}

View file

@ -1,23 +0,0 @@
resource "aws_sqs_queue" "this" {
name = "${var.name}-queue"
tags = var.tags
}
resource "aws_lambda_function" "this" {
function_name = var.name
role = var.lambda_role_arn
package_type = "Image"
image_uri = var.image_uri
timeout = var.timeout
tags = var.tags
}
resource "aws_lambda_event_source_mapping" "this" {
event_source_arn = aws_sqs_queue.this.arn
function_name = aws_lambda_function.this.arn
batch_size = var.sqs_batch_size
}

View file

@ -1,15 +0,0 @@
output "lambda_name" {
value = aws_lambda_function.this.function_name
}
output "lambda_arn" {
value = aws_lambda_function.this.arn
}
output "sqs_queue_url" {
value = aws_sqs_queue.this.url
}
output "sqs_queue_arn" {
value = aws_sqs_queue.this.arn
}

View file

@ -1,32 +0,0 @@
variable "name" {
description = "Base name for lambda and related resources"
type = string
}
variable "image_uri" {
description = "ECR image URI with tag"
type = string
}
variable "lambda_role_arn" {
description = "IAM role ARN for Lambda execution"
type = string
}
variable "timeout" {
description = "Lambda timeout in seconds"
type = number
default = 10
}
variable "sqs_batch_size" {
description = "Number of SQS messages per batch"
type = number
default = 1
}
variable "tags" {
description = "Tags to apply to resources"
type = map(string)
default = {}
}

View file

@ -0,0 +1,12 @@
resource "aws_sqs_queue" "dlq" {
name = "${var.name}-dlq"
}
resource "aws_sqs_queue" "this" {
name = var.name
redrive_policy = jsonencode({
deadLetterTargetArn = aws_sqs_queue.dlq.arn
maxReceiveCount = var.max_receive_count
})
}

View file

@ -0,0 +1,7 @@
output "queue_arn" {
value = aws_sqs_queue.this.arn
}
output "queue_url" {
value = aws_sqs_queue.this.url
}

View file

@ -0,0 +1,6 @@
variable "name" { type = string }
variable "max_receive_count" {
type = number
default = 5
}

View file

@ -0,0 +1,30 @@
resource "aws_s3_bucket" "this" {
bucket = var.bucket_name
}
resource "aws_s3_bucket_versioning" "this" {
bucket = aws_s3_bucket.this.id
versioning_configuration {
status = "Enabled"
}
}
resource "aws_s3_bucket_server_side_encryption_configuration" "this" {
bucket = aws_s3_bucket.this.id
rule {
apply_server_side_encryption_by_default {
sse_algorithm = "AES256"
}
}
}
resource "aws_s3_bucket_public_access_block" "this" {
bucket = aws_s3_bucket.this.id
block_public_acls = true
block_public_policy = true
ignore_public_acls = true
restrict_public_buckets = true
}

View file

@ -0,0 +1,7 @@
output "bucket_name" {
value = aws_s3_bucket.this.bucket
}
output "bucket_arn" {
value = aws_s3_bucket.this.arn
}

View file

@ -0,0 +1,3 @@
variable "bucket_name" {
type = string
}

View file

@ -8,7 +8,6 @@ terraform {
backend "s3" {
bucket = "assessment-model-terraform-state"
region = "eu-west-2"
# profile = "DevAdmin"
key = "terraform.tfstate"
}
@ -290,11 +289,23 @@ output "ses_dns_records" {
value = module.ses.dns_records
}
################################################
# Address2UPRN Lambda ECR
################################################
module "address2uprn_state_bucket" {
source = "../modules/tf_state_bucket"
bucket_name = "address2uprn-terraform-state"
}
################################################
# One ECR to rule all the lambdas
################################################
module "lambda_shared_ecr" {
source = "../modules/ecr"
ecr_name = "lambda-shared-${var.stage}"
output "address2uprn_state_bucket_name" {
value = module.address2uprn_state_bucket.bucket_name
}
module "address2uprn_registry" {
source = "../modules/container_registry"
name = "address2uprn-${var.stage}"
}
output "address2uprn_repository_url" {
value = module.address2uprn_registry.repository_url
}

View file

@ -3,7 +3,6 @@ variable stage {
type = string
}
variable "region" {
description = "AWS region"
type = string