push images

.github/workflows/push_docker_image_to_ecr.yml

@@ -6,11 +6,12 @@ on:
 
 env:
   AWS_REGION: eu-west-2
-  ECR_REPOSITORY: lambda_example
 
 jobs:
-  build-and-push-to-elastic-container-registry:
+  build-and-push-to-ecr-for-lambda-example:
     runs-on: ubuntu-latest
+    env:
+      ECR_REPOSITORY: lambda_example
 
     permissions:
       id-token: write
@@ -47,3 +48,42 @@ jobs:
         echo "Pushing Docker image to ECR..."
         docker push $IMAGE_URI
 
+  build-and-push-to-ecr-for-extractor-and-loader-example:
+    runs-on: ubuntu-latest
+    env:
+      ECR_REPOSITORY: extractor_and_loader
+
+    permissions:
+      id-token: write
+      contents: read
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: AWS credentials
+      uses: aws-actions/configure-aws-credentials@v4
+      with:
+        # as of 14/07/2025 it'll be using user:Junte's keys
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws-region: ${{ secrets.AWS_REGION }}
+
+    - name: Log in to Amazon ECR
+      id: login-ecr
+      uses: aws-actions/amazon-ecr-login@v2
+
+    - name: Build, tag, and push Docker image to ECR
+      env:
+        ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+        IMAGE_TAG: latest
+      run: |
+        IMAGE_URI=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }}
+        echo "pwd"
+        pwd
+        ls -la
+        echo "Building Docker image..."
+        docker build -t $IMAGE_URI -f deployment/extractor_and_loader/Dockerfile .
+
+        echo "Pushing Docker image to ECR..."
+        docker push $IMAGE_URI

deployment/extractor_and_loader/Dockerfile

@@ -0,0 +1,8 @@
+# AWS Lambda python pacakge
+FROM public.ecr.aws/lambda/python:3.11
+
+# Copy function code
+COPY deployment/lambda_example/app.py ./
+
+# Set the CMD to your handler
+CMD ["app.handler"]

deployment/extractor_and_loader/app.py

@@ -0,0 +1,26 @@
+"""
+A quick example of lambda working a function in python
+"""
+
+def handler(event, context):
+    try:
+        s3_uri = event.get("file_location")
+        if not s3_uri:
+            return {
+                "statusCode": 400,
+                "body": "Missing 'file_location' in event"
+            }
+
+        print(f"s3 uri is {s3_uri}")
+
+        return {
+            "statusCode": 200,
+            "body": f"s3 uri {s3_uri}"
+        }
+
+    except Exception as e:
+        print(f"❌ Error: {e}")
+        return {
+            "statusCode": 500,
+            "body": str(e)
+        }

deployment/extractor_and_loader_lambda.tf

@@ -0,0 +1,85 @@
+# SQS queue for extractor_and_loader
+resource "aws_sqs_queue" "extractor_and_loader_queue" {
+  name = "extractor-loader-queue"
+}
+
+# ECR repo
+resource "aws_ecr_repository" "extractor_and_loader" {
+  name = "extractor_and_loader"
+}
+
+# IAM policy specific to this Lambda
+resource "aws_iam_policy" "extractor_loader_policy" {
+  name = "extractor-loader-policy"
+
+  policy = jsonencode({
+    Version = "2012-10-17",
+    Statement = [
+      {
+        Effect = "Allow",
+        Action = [
+          "sqs:ReceiveMessage",
+          "sqs:DeleteMessage",
+          "sqs:GetQueueAttributes"
+        ],
+        Resource = aws_sqs_queue.extractor_and_loader_queue.arn
+      },
+      {
+        Effect = "Allow",
+        Action = [
+          "ecr:GetDownloadUrlForLayer",
+          "ecr:BatchGetImage",
+          "ecr:BatchCheckLayerAvailability"
+        ],
+        Resource = aws_ecr_repository.extractor_and_loader.arn
+      },
+      {
+        Effect = "Allow",
+        Action = ["ecr:GetAuthorizationToken"],
+        Resource = "*"
+      }
+    ]
+  })
+}
+
+resource "aws_iam_role_policy_attachment" "extractor_loader_policy_attach" {
+  role       = aws_iam_role.lambda_exec_role.name
+  policy_arn = aws_iam_policy.extractor_loader_policy.arn
+}
+
+# Lambda function
+resource "aws_lambda_function" "extractor_and_loader" {
+  function_name = "extractor-and-loader"
+  role          = aws_iam_role.lambda_exec_role.arn
+  package_type  = "Image"
+  image_uri     = "${aws_ecr_repository.extractor_and_loader.repository_url}:latest"
+  timeout       = 10
+}
+
+# SQS trigger
+resource "aws_lambda_event_source_mapping" "extractor_and_loader_trigger" {
+  event_source_arn = aws_sqs_queue.extractor_and_loader_queue.arn
+  function_name    = aws_lambda_function.extractor_and_loader.arn
+  batch_size       = 1
+}
+
+# ECR policy to allow Lambda access
+resource "aws_ecr_repository_policy" "extractor_loader_ecr_access" {
+  repository = aws_ecr_repository.extractor_and_loader.name
+
+  policy = jsonencode({
+    Version = "2008-10-17",
+    Statement = [{
+      Sid       = "AllowLambdaPull",
+      Effect    = "Allow",
+      Principal = {
+        Service = "lambda.amazonaws.com"
+      },
+      Action = [
+        "ecr:GetDownloadUrlForLayer",
+        "ecr:BatchGetImage",
+        "ecr:BatchCheckLayerAvailability"
+      ]
+    }]
+  })
+}

deployment/lambda_example.tf

@@ -1,121 +0,0 @@
-# This is an example file to setup a lamda function with a sqs and cloudwatch.
-# Please us this as a template for future lambda.
-# Be sure to push the image you are using to ECR or it won't deploy properly
-
-# Create an SQS queue that will trigger the Lambda
-resource "aws_sqs_queue" "my_queue" {
-  name = "my-lambda-queue"
-}
-
-# Create an ECR repository to store the Docker image for the Lambda function
-resource "aws_ecr_repository" "lambda_repo" {
-  name = "lambda_example"
-}
-
-# IAM role that the Lambda function will assume
-resource "aws_iam_role" "lambda_exec_role" {
-  name = "lambda-exec-role"
-
-  assume_role_policy = jsonencode({
-    Version = "2012-10-17",
-    Statement = [
-      {
-        Action = "sts:AssumeRole",
-        Effect = "Allow",
-        Principal = {
-          Service = "lambda.amazonaws.com"
-        }
-      }
-    ]
-  })
-}
-
-# Attach AWS-managed policy for basic Lambda execution (CloudWatch logging)
-resource "aws_iam_role_policy_attachment" "lambda_basic_execution" {
-  role       = aws_iam_role.lambda_exec_role.name
-  policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
-}
-
-# Custom policy: SQS access + ECR image pull permissions
-resource "aws_iam_policy" "lambda_custom_policy" {
-  name = "lambda-sqs-ecr-policy"
-
-  policy = jsonencode({
-    Version = "2012-10-17",
-    Statement = [
-      # Allow Lambda to read from SQS
-      {
-        Effect = "Allow",
-        Action = [
-          "sqs:ReceiveMessage",
-          "sqs:DeleteMessage",
-          "sqs:GetQueueAttributes"
-        ],
-        Resource = aws_sqs_queue.my_queue.arn
-      },
-      # Allow Lambda to pull images from ECR
-      {
-        Effect = "Allow",
-        Action = [
-          "ecr:GetDownloadUrlForLayer",
-          "ecr:BatchGetImage",
-          "ecr:BatchCheckLayerAvailability"
-        ],
-        Resource = aws_ecr_repository.lambda_repo.arn
-      },
-      # Needed to authenticate to ECR (pulling the image)
-      {
-        Effect = "Allow",
-        Action = [
-          "ecr:GetAuthorizationToken"
-        ],
-        Resource = "*"
-      }
-    ]
-  })
-}
-
-# Attach the custom policy to the Lambda role
-resource "aws_iam_role_policy_attachment" "lambda_custom_policy_attach" {
-  role       = aws_iam_role.lambda_exec_role.name
-  policy_arn = aws_iam_policy.lambda_custom_policy.arn
-}
-
-# Define the Lambda function using a Docker image from ECR
-resource "aws_lambda_function" "lambda_docker" {
-  function_name = "docker-hello-world-python-example"
-  role          = aws_iam_role.lambda_exec_role.arn
-  package_type  = "Image"
-  image_uri     = "${aws_ecr_repository.lambda_repo.repository_url}:latest"
-  timeout       = 10
-}
-
-# Connect the SQS queue to the Lambda so it gets triggered by incoming messages
-resource "aws_lambda_event_source_mapping" "sqs_trigger" {
-  event_source_arn = aws_sqs_queue.my_queue.arn
-  function_name    = aws_lambda_function.lambda_docker.arn
-  batch_size       = 1
-}
-
-
-resource "aws_ecr_repository_policy" "lambda_ecr_access" {
-  repository = aws_ecr_repository.lambda_repo.name
-
-  policy = jsonencode({
-    Version = "2008-10-17",
-    Statement = [
-      {
-        Sid       = "AllowLambdaPull",
-        Effect    = "Allow",
-        Principal = {
-          Service = "lambda.amazonaws.com"
-        },
-        Action = [
-          "ecr:GetDownloadUrlForLayer",
-          "ecr:BatchGetImage",
-          "ecr:BatchCheckLayerAvailability"
-        ]
-      }
-    ]
-  })
-}

deployment/lambda_example_and_config.tf

@@ -0,0 +1,106 @@
+# IAM role for both Lambdas (can be shared)
+resource "aws_iam_role" "lambda_exec_role" {
+  name = "lambda-exec-role"
+
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17",
+    Statement = [{
+      Action = "sts:AssumeRole",
+      Effect = "Allow",
+      Principal = {
+        Service = "lambda.amazonaws.com"
+      }
+    }]
+  })
+}
+
+resource "aws_iam_role_policy_attachment" "lambda_basic_execution" {
+  role       = aws_iam_role.lambda_exec_role.name
+  policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
+}
+
+# SQS queue for lambda_example
+resource "aws_sqs_queue" "lambda_example_queue" {
+  name = "lambda-example-queue"
+}
+
+# ECR repo for lambda_example
+resource "aws_ecr_repository" "lambda_example" {
+  name = "lambda_example"
+}
+
+# Custom IAM policy specific to lambda_example
+resource "aws_iam_policy" "lambda_example_policy" {
+  name = "lambda-example-policy"
+
+  policy = jsonencode({
+    Version = "2012-10-17",
+    Statement = [
+      {
+        Effect = "Allow",
+        Action = [
+          "sqs:ReceiveMessage",
+          "sqs:DeleteMessage",
+          "sqs:GetQueueAttributes"
+        ],
+        Resource = aws_sqs_queue.lambda_example_queue.arn
+      },
+      {
+        Effect = "Allow",
+        Action = [
+          "ecr:GetDownloadUrlForLayer",
+          "ecr:BatchGetImage",
+          "ecr:BatchCheckLayerAvailability"
+        ],
+        Resource = aws_ecr_repository.lambda_example.arn
+      },
+      {
+        Effect = "Allow",
+        Action = ["ecr:GetAuthorizationToken"],
+        Resource = "*"
+      }
+    ]
+  })
+}
+
+resource "aws_iam_role_policy_attachment" "lambda_example_policy_attach" {
+  role       = aws_iam_role.lambda_exec_role.name
+  policy_arn = aws_iam_policy.lambda_example_policy.arn
+}
+
+# Lambda function
+resource "aws_lambda_function" "lambda_example" {
+  function_name = "lambda-example"
+  role          = aws_iam_role.lambda_exec_role.arn
+  package_type  = "Image"
+  image_uri     = "${aws_ecr_repository.lambda_example.repository_url}:latest"
+  timeout       = 10
+}
+
+# SQS trigger
+resource "aws_lambda_event_source_mapping" "lambda_example_trigger" {
+  event_source_arn = aws_sqs_queue.lambda_example_queue.arn
+  function_name    = aws_lambda_function.lambda_example.arn
+  batch_size       = 1
+}
+
+# ECR policy to allow Lambda access
+resource "aws_ecr_repository_policy" "lambda_example_ecr_access" {
+  repository = aws_ecr_repository.lambda_example.name
+
+  policy = jsonencode({
+    Version = "2008-10-17",
+    Statement = [{
+      Sid       = "AllowLambdaPull",
+      Effect    = "Allow",
+      Principal = {
+        Service = "lambda.amazonaws.com"
+      },
+      Action = [
+        "ecr:GetDownloadUrlForLayer",
+        "ecr:BatchGetImage",
+        "ecr:BatchCheckLayerAvailability"
+      ]
+    }]
+  })
+}