diff --git a/.github/workflows/push_docker_image_to_ecr.yml b/.github/workflows/push_docker_image_to_ecr.yml index cf96382..d96507f 100644 --- a/.github/workflows/push_docker_image_to_ecr.yml +++ b/.github/workflows/push_docker_image_to_ecr.yml @@ -48,42 +48,42 @@ jobs: echo "Pushing Docker image to ECR..." docker push $IMAGE_URI - # build-and-push-to-ecr-for-extractor-and-loader-example: - # runs-on: ubuntu-latest - # env: - # ECR_REPOSITORY: extractor_and_loader + build-and-push-to-ecr-for-extractor-and-loader-example: + runs-on: ubuntu-latest + env: + ECR_REPOSITORY: extractor_and_loader - # permissions: - # id-token: write - # contents: read + permissions: + id-token: write + contents: read - # steps: - # - name: Checkout code - # uses: actions/checkout@v4 + steps: + - name: Checkout code + uses: actions/checkout@v4 - # - name: AWS credentials - # uses: aws-actions/configure-aws-credentials@v4 - # with: - # # as of 14/07/2025 it'll be using user:Junte's keys - # aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - # aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # aws-region: ${{ secrets.AWS_REGION }} + - name: AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + # as of 14/07/2025 it'll be using user:Junte's keys + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_REGION }} - # - name: Log in to Amazon ECR - # id: login-ecr - # uses: aws-actions/amazon-ecr-login@v2 + - name: Log in to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 - # - name: Build, tag, and push Docker image to ECR - # env: - # ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - # IMAGE_TAG: latest5 - # run: | - # IMAGE_URI=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} - # echo "pwd" - # pwd - # ls -la - # echo "Building Docker image..." - # docker build -t $IMAGE_URI -f deployment/extractor_and_loader/Dockerfile . + - name: Build, tag, and push Docker image to ECR + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + IMAGE_TAG: latest + run: | + IMAGE_URI=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} + echo "pwd" + pwd + ls -la + echo "Building Docker image..." + docker build -t $IMAGE_URI -f deployment/lambda/extractor_and_loader/docker/Dockerfile . - # echo "Pushing Docker image to ECR..." - # docker push $IMAGE_URI \ No newline at end of file + echo "Pushing Docker image to ECR..." + docker push $IMAGE_URI \ No newline at end of file diff --git a/deployment/lambda/extractor_and_loader/docker/Dockerfile b/deployment/lambda/extractor_and_loader/docker/Dockerfile index 609981a..cdd1f8a 100644 --- a/deployment/lambda/extractor_and_loader/docker/Dockerfile +++ b/deployment/lambda/extractor_and_loader/docker/Dockerfile @@ -19,7 +19,7 @@ RUN poetry config virtualenvs.create false \ && poetry install --only main --no-interaction --no-ansi # Copy app code -COPY deployment/extractor_and_loader/app.py ./ +COPY deployment/lambda/extractor_and_loader/docker/app.py ./ # Set Lambda handler CMD ["app.handler"] \ No newline at end of file diff --git a/deployment/lambda/extractor_and_loader/extractor_and_loader_lambda.tf b/deployment/lambda/extractor_and_loader/extractor_and_loader_lambda.tf index eb0aed9..71ff4d1 100644 --- a/deployment/lambda/extractor_and_loader/extractor_and_loader_lambda.tf +++ b/deployment/lambda/extractor_and_loader/extractor_and_loader_lambda.tf @@ -1,12 +1,18 @@ +# Reference existing IAM role +data "aws_iam_role" "lambda_exec_role" { + name = "lambda-exec-role" +} + +# Reference existing ECR repository +data "aws_ecr_repository" "extractor_and_loader" { + name = "extractor_and_loader" +} + # SQS queue for extractor_and_loader resource "aws_sqs_queue" "extractor_and_loader_queue" { name = "extractor-loader-queue" } -# ECR repo -resource "aws_ecr_repository" "extractor_and_loader" { - name = "extractor_and_loader" -} # IAM policy specific to this Lambda resource "aws_iam_policy" "extractor_loader_policy" { @@ -31,7 +37,7 @@ resource "aws_iam_policy" "extractor_loader_policy" { "ecr:BatchGetImage", "ecr:BatchCheckLayerAvailability" ], - Resource = aws_ecr_repository.extractor_and_loader.arn + Resource = data.aws_ecr_repository.lambda_example.arn }, { Effect = "Allow", @@ -43,7 +49,7 @@ resource "aws_iam_policy" "extractor_loader_policy" { } resource "aws_iam_role_policy_attachment" "extractor_loader_policy_attach" { - role = aws_iam_role.lambda_exec_role.name + role = data.aws_iam_role.lambda_exec_role.name policy_arn = aws_iam_policy.extractor_loader_policy.arn } @@ -52,7 +58,7 @@ resource "aws_lambda_function" "extractor_and_loader" { function_name = "extractor-and-loader" role = aws_iam_role.lambda_exec_role.arn package_type = "Image" - image_uri = "${aws_ecr_repository.extractor_and_loader.repository_url}:latest5" + mage_uri = "${data.aws_ecr_repository.extractor_and_loader.repository_url}:latest" timeout = 30 } @@ -63,27 +69,6 @@ resource "aws_lambda_event_source_mapping" "extractor_and_loader_trigger" { batch_size = 1 } -# ECR policy to allow Lambda access -resource "aws_ecr_repository_policy" "extractor_loader_ecr_access" { - repository = aws_ecr_repository.extractor_and_loader.name - - policy = jsonencode({ - Version = "2008-10-17", - Statement = [{ - Sid = "AllowLambdaPull", - Effect = "Allow", - Principal = { - Service = "lambda.amazonaws.com" - }, - Action = [ - "ecr:GetDownloadUrlForLayer", - "ecr:BatchGetImage", - "ecr:BatchCheckLayerAvailability" - ] - }] - }) -} - # TODO: Seperate lambda jobs from ecr creation. This is because we need to # Create the ecr, then push images, then lambda jobs can be made \ No newline at end of file