diff --git a/.github/workflows/push_docker_image_to_ecr.yml b/.github/workflows/push_docker_image_to_ecr.yml index 6b016e2..178e646 100644 --- a/.github/workflows/push_docker_image_to_ecr.yml +++ b/.github/workflows/push_docker_image_to_ecr.yml @@ -6,11 +6,12 @@ on: env: AWS_REGION: eu-west-2 - ECR_REPOSITORY: lambda_example jobs: - build-and-push-to-elastic-container-registry: + build-and-push-to-ecr-for-lambda-example: runs-on: ubuntu-latest + env: + ECR_REPOSITORY: lambda_example permissions: id-token: write @@ -47,3 +48,42 @@ jobs: echo "Pushing Docker image to ECR..." docker push $IMAGE_URI + build-and-push-to-ecr-for-extractor-and-loader-example: + runs-on: ubuntu-latest + env: + ECR_REPOSITORY: extractor_and_loader + + permissions: + id-token: write + contents: read + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + # as of 14/07/2025 it'll be using user:Junte's keys + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_REGION }} + + - name: Log in to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Build, tag, and push Docker image to ECR + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + IMAGE_TAG: latest + run: | + IMAGE_URI=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} + echo "pwd" + pwd + ls -la + echo "Building Docker image..." + docker build -t $IMAGE_URI -f deployment/extractor_and_loader/Dockerfile . + + echo "Pushing Docker image to ECR..." + docker push $IMAGE_URI \ No newline at end of file diff --git a/deployment/extractor_and_loader/Dockerfile b/deployment/extractor_and_loader/Dockerfile new file mode 100644 index 0000000..c1caf03 --- /dev/null +++ b/deployment/extractor_and_loader/Dockerfile @@ -0,0 +1,8 @@ +# AWS Lambda python pacakge +FROM public.ecr.aws/lambda/python:3.11 + +# Copy function code +COPY deployment/lambda_example/app.py ./ + +# Set the CMD to your handler +CMD ["app.handler"] \ No newline at end of file diff --git a/deployment/extractor_and_loader/app.py b/deployment/extractor_and_loader/app.py new file mode 100644 index 0000000..a520464 --- /dev/null +++ b/deployment/extractor_and_loader/app.py @@ -0,0 +1,26 @@ +""" +A quick example of lambda working a function in python +""" + +def handler(event, context): + try: + s3_uri = event.get("file_location") + if not s3_uri: + return { + "statusCode": 400, + "body": "Missing 'file_location' in event" + } + + print(f"s3 uri is {s3_uri}") + + return { + "statusCode": 200, + "body": f"s3 uri {s3_uri}" + } + + except Exception as e: + print(f"❌ Error: {e}") + return { + "statusCode": 500, + "body": str(e) + } \ No newline at end of file diff --git a/deployment/extractor_and_loader_lambda.tf b/deployment/extractor_and_loader_lambda.tf new file mode 100644 index 0000000..1d74b3f --- /dev/null +++ b/deployment/extractor_and_loader_lambda.tf @@ -0,0 +1,85 @@ +# SQS queue for extractor_and_loader +resource "aws_sqs_queue" "extractor_and_loader_queue" { + name = "extractor-loader-queue" +} + +# ECR repo +resource "aws_ecr_repository" "extractor_and_loader" { + name = "extractor_and_loader" +} + +# IAM policy specific to this Lambda +resource "aws_iam_policy" "extractor_loader_policy" { + name = "extractor-loader-policy" + + policy = jsonencode({ + Version = "2012-10-17", + Statement = [ + { + Effect = "Allow", + Action = [ + "sqs:ReceiveMessage", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes" + ], + Resource = aws_sqs_queue.extractor_and_loader_queue.arn + }, + { + Effect = "Allow", + Action = [ + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage", + "ecr:BatchCheckLayerAvailability" + ], + Resource = aws_ecr_repository.extractor_and_loader.arn + }, + { + Effect = "Allow", + Action = ["ecr:GetAuthorizationToken"], + Resource = "*" + } + ] + }) +} + +resource "aws_iam_role_policy_attachment" "extractor_loader_policy_attach" { + role = aws_iam_role.lambda_exec_role.name + policy_arn = aws_iam_policy.extractor_loader_policy.arn +} + +# Lambda function +resource "aws_lambda_function" "extractor_and_loader" { + function_name = "extractor-and-loader" + role = aws_iam_role.lambda_exec_role.arn + package_type = "Image" + image_uri = "${aws_ecr_repository.extractor_and_loader.repository_url}:latest" + timeout = 10 +} + +# SQS trigger +resource "aws_lambda_event_source_mapping" "extractor_and_loader_trigger" { + event_source_arn = aws_sqs_queue.extractor_and_loader_queue.arn + function_name = aws_lambda_function.extractor_and_loader.arn + batch_size = 1 +} + +# ECR policy to allow Lambda access +resource "aws_ecr_repository_policy" "extractor_loader_ecr_access" { + repository = aws_ecr_repository.extractor_and_loader.name + + policy = jsonencode({ + Version = "2008-10-17", + Statement = [{ + Sid = "AllowLambdaPull", + Effect = "Allow", + Principal = { + Service = "lambda.amazonaws.com" + }, + Action = [ + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage", + "ecr:BatchCheckLayerAvailability" + ] + }] + }) +} diff --git a/deployment/lambda_example.tf b/deployment/lambda_example.tf deleted file mode 100644 index 0748af4..0000000 --- a/deployment/lambda_example.tf +++ /dev/null @@ -1,121 +0,0 @@ -# This is an example file to setup a lamda function with a sqs and cloudwatch. -# Please us this as a template for future lambda. -# Be sure to push the image you are using to ECR or it won't deploy properly - -# Create an SQS queue that will trigger the Lambda -resource "aws_sqs_queue" "my_queue" { - name = "my-lambda-queue" -} - -# Create an ECR repository to store the Docker image for the Lambda function -resource "aws_ecr_repository" "lambda_repo" { - name = "lambda_example" -} - -# IAM role that the Lambda function will assume -resource "aws_iam_role" "lambda_exec_role" { - name = "lambda-exec-role" - - assume_role_policy = jsonencode({ - Version = "2012-10-17", - Statement = [ - { - Action = "sts:AssumeRole", - Effect = "Allow", - Principal = { - Service = "lambda.amazonaws.com" - } - } - ] - }) -} - -# Attach AWS-managed policy for basic Lambda execution (CloudWatch logging) -resource "aws_iam_role_policy_attachment" "lambda_basic_execution" { - role = aws_iam_role.lambda_exec_role.name - policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" -} - -# Custom policy: SQS access + ECR image pull permissions -resource "aws_iam_policy" "lambda_custom_policy" { - name = "lambda-sqs-ecr-policy" - - policy = jsonencode({ - Version = "2012-10-17", - Statement = [ - # Allow Lambda to read from SQS - { - Effect = "Allow", - Action = [ - "sqs:ReceiveMessage", - "sqs:DeleteMessage", - "sqs:GetQueueAttributes" - ], - Resource = aws_sqs_queue.my_queue.arn - }, - # Allow Lambda to pull images from ECR - { - Effect = "Allow", - Action = [ - "ecr:GetDownloadUrlForLayer", - "ecr:BatchGetImage", - "ecr:BatchCheckLayerAvailability" - ], - Resource = aws_ecr_repository.lambda_repo.arn - }, - # Needed to authenticate to ECR (pulling the image) - { - Effect = "Allow", - Action = [ - "ecr:GetAuthorizationToken" - ], - Resource = "*" - } - ] - }) -} - -# Attach the custom policy to the Lambda role -resource "aws_iam_role_policy_attachment" "lambda_custom_policy_attach" { - role = aws_iam_role.lambda_exec_role.name - policy_arn = aws_iam_policy.lambda_custom_policy.arn -} - -# Define the Lambda function using a Docker image from ECR -resource "aws_lambda_function" "lambda_docker" { - function_name = "docker-hello-world-python-example" - role = aws_iam_role.lambda_exec_role.arn - package_type = "Image" - image_uri = "${aws_ecr_repository.lambda_repo.repository_url}:latest" - timeout = 10 -} - -# Connect the SQS queue to the Lambda so it gets triggered by incoming messages -resource "aws_lambda_event_source_mapping" "sqs_trigger" { - event_source_arn = aws_sqs_queue.my_queue.arn - function_name = aws_lambda_function.lambda_docker.arn - batch_size = 1 -} - - -resource "aws_ecr_repository_policy" "lambda_ecr_access" { - repository = aws_ecr_repository.lambda_repo.name - - policy = jsonencode({ - Version = "2008-10-17", - Statement = [ - { - Sid = "AllowLambdaPull", - Effect = "Allow", - Principal = { - Service = "lambda.amazonaws.com" - }, - Action = [ - "ecr:GetDownloadUrlForLayer", - "ecr:BatchGetImage", - "ecr:BatchCheckLayerAvailability" - ] - } - ] - }) -} \ No newline at end of file diff --git a/deployment/lambda_example_and_config.tf b/deployment/lambda_example_and_config.tf new file mode 100644 index 0000000..b020e2d --- /dev/null +++ b/deployment/lambda_example_and_config.tf @@ -0,0 +1,106 @@ +# IAM role for both Lambdas (can be shared) +resource "aws_iam_role" "lambda_exec_role" { + name = "lambda-exec-role" + + assume_role_policy = jsonencode({ + Version = "2012-10-17", + Statement = [{ + Action = "sts:AssumeRole", + Effect = "Allow", + Principal = { + Service = "lambda.amazonaws.com" + } + }] + }) +} + +resource "aws_iam_role_policy_attachment" "lambda_basic_execution" { + role = aws_iam_role.lambda_exec_role.name + policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" +} + +# SQS queue for lambda_example +resource "aws_sqs_queue" "lambda_example_queue" { + name = "lambda-example-queue" +} + +# ECR repo for lambda_example +resource "aws_ecr_repository" "lambda_example" { + name = "lambda_example" +} + +# Custom IAM policy specific to lambda_example +resource "aws_iam_policy" "lambda_example_policy" { + name = "lambda-example-policy" + + policy = jsonencode({ + Version = "2012-10-17", + Statement = [ + { + Effect = "Allow", + Action = [ + "sqs:ReceiveMessage", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes" + ], + Resource = aws_sqs_queue.lambda_example_queue.arn + }, + { + Effect = "Allow", + Action = [ + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage", + "ecr:BatchCheckLayerAvailability" + ], + Resource = aws_ecr_repository.lambda_example.arn + }, + { + Effect = "Allow", + Action = ["ecr:GetAuthorizationToken"], + Resource = "*" + } + ] + }) +} + +resource "aws_iam_role_policy_attachment" "lambda_example_policy_attach" { + role = aws_iam_role.lambda_exec_role.name + policy_arn = aws_iam_policy.lambda_example_policy.arn +} + +# Lambda function +resource "aws_lambda_function" "lambda_example" { + function_name = "lambda-example" + role = aws_iam_role.lambda_exec_role.arn + package_type = "Image" + image_uri = "${aws_ecr_repository.lambda_example.repository_url}:latest" + timeout = 10 +} + +# SQS trigger +resource "aws_lambda_event_source_mapping" "lambda_example_trigger" { + event_source_arn = aws_sqs_queue.lambda_example_queue.arn + function_name = aws_lambda_function.lambda_example.arn + batch_size = 1 +} + +# ECR policy to allow Lambda access +resource "aws_ecr_repository_policy" "lambda_example_ecr_access" { + repository = aws_ecr_repository.lambda_example.name + + policy = jsonencode({ + Version = "2008-10-17", + Statement = [{ + Sid = "AllowLambdaPull", + Effect = "Allow", + Principal = { + Service = "lambda.amazonaws.com" + }, + Action = [ + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage", + "ecr:BatchCheckLayerAvailability" + ] + }] + }) +}