From dd49a9e5975d3cc6ec34bc46b918b0c12271197a Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 14 Jul 2025 10:59:32 +0000 Subject: [PATCH 01/16] lambda tf --- deployment/lambda.tf | 53 +++++++++++++++++++++++++++++++++++++ deployment/lamda/Dockerfile | 8 ++++++ deployment/lamda/app.py | 10 +++++++ 3 files changed, 71 insertions(+) create mode 100644 deployment/lambda.tf create mode 100644 deployment/lamda/Dockerfile create mode 100644 deployment/lamda/app.py diff --git a/deployment/lambda.tf b/deployment/lambda.tf new file mode 100644 index 0000000..6903652 --- /dev/null +++ b/deployment/lambda.tf @@ -0,0 +1,53 @@ +provider "aws" { + region = "us-east-1" # Change if needed +} + +resource "aws_sqs_queue" "my_queue" { + name = "my-lambda-queue" +} + +resource "aws_iam_role" "lambda_exec_role" { + name = "lambda-exec-role" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = "sts:AssumeRole" + Effect = "Allow" + Principal = { + Service = "lambda.amazonaws.com" + } + } + ] + }) +} + +resource "aws_iam_role_policy_attachment" "lambda_basic_execution" { + role = aws_iam_role.lambda_exec_role.name + policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" +} + +resource "aws_iam_role_policy_attachment" "sqs_access" { + role = aws_iam_role.lambda_exec_role.name + policy_arn = "arn:aws:iam::aws:policy/AWSLambdaSQSQueueExecutionRole" +} + +resource "aws_ecr_repository" "lambda_repo" { + name = "lambda-hello-world" +} + +resource "aws_lambda_function" "lambda_docker" { + function_name = "docker-hello-world" + role = aws_iam_role.lambda_exec_role.arn + package_type = "Image" + image_uri = "${aws_ecr_repository.lambda_repo.repository_url}:latest" + + timeout = 10 +} + +resource "aws_lambda_event_source_mapping" "sqs_trigger" { + event_source_arn = aws_sqs_queue.my_queue.arn + function_name = aws_lambda_function.lambda_docker.arn + batch_size = 1 +} diff --git a/deployment/lamda/Dockerfile b/deployment/lamda/Dockerfile new file mode 100644 index 0000000..4b57200 --- /dev/null +++ b/deployment/lamda/Dockerfile @@ -0,0 +1,8 @@ +# AWS Lambda python pacakge +FROM public.ecr.aws/lambda/python:3.11 + +# Copy function code +COPY app.py ./ + +# Set the CMD to your handler +CMD ["app.handler"] \ No newline at end of file diff --git a/deployment/lamda/app.py b/deployment/lamda/app.py new file mode 100644 index 0000000..696c654 --- /dev/null +++ b/deployment/lamda/app.py @@ -0,0 +1,10 @@ +""" +A quick example of lambda working a function in python +""" + +def handler(event, context): + print("Hello from Python function. This shold be running from a dockerfile env and executed on a aws lambda!") + return { + 'statusCode': 200, + 'body': 'Hello World' + } \ No newline at end of file From 04a7c68b5bf03e552ef09c4a9c5f358855c86830 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 14 Jul 2025 12:34:14 +0000 Subject: [PATCH 02/16] lamda addition --- .devcontainer/devcontainer.json | 1 + deployment/lambda.tf | 12 ++++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 445d2e6..d050974 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -7,6 +7,7 @@ "postStartCommand": "bash .devcontainer/post-install.sh", "mounts": [ "source=${localEnv:HOME},target=/workspaces/home,type=bind", + // Make sure you aws credentials are saved at ~/.aws "source=${localEnv:HOME}/.aws/,target=/home/vscode/.aws/,type=bind" ], "customizations": { diff --git a/deployment/lambda.tf b/deployment/lambda.tf index 6903652..dc24055 100644 --- a/deployment/lambda.tf +++ b/deployment/lambda.tf @@ -1,11 +1,9 @@ -provider "aws" { - region = "us-east-1" # Change if needed -} - +# Create an SQS queue that will trigger the Lambda resource "aws_sqs_queue" "my_queue" { name = "my-lambda-queue" } +# IAM role that the Lambda function will assume to get permissions resource "aws_iam_role" "lambda_exec_role" { name = "lambda-exec-role" @@ -23,20 +21,25 @@ resource "aws_iam_role" "lambda_exec_role" { }) } +# Attach the basic execution policy (writes logs to CloudWatch) to the Lambda role resource "aws_iam_role_policy_attachment" "lambda_basic_execution" { role = aws_iam_role.lambda_exec_role.name policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" } +# Give Lambda permission to poll and process SQS messages resource "aws_iam_role_policy_attachment" "sqs_access" { role = aws_iam_role.lambda_exec_role.name policy_arn = "arn:aws:iam::aws:policy/AWSLambdaSQSQueueExecutionRole" } + +# Create an ECR repository to store the Docker image for the Lambda function resource "aws_ecr_repository" "lambda_repo" { name = "lambda-hello-world" } +# Define the Lambda function using a Docker image from ECR resource "aws_lambda_function" "lambda_docker" { function_name = "docker-hello-world" role = aws_iam_role.lambda_exec_role.arn @@ -46,6 +49,7 @@ resource "aws_lambda_function" "lambda_docker" { timeout = 10 } +# Connect the SQS queue to the Lambda so it gets triggered by incoming messages resource "aws_lambda_event_source_mapping" "sqs_trigger" { event_source_arn = aws_sqs_queue.my_queue.arn function_name = aws_lambda_function.lambda_docker.arn From c3a24085be87df1b374d4619cd687d005c239fce Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 14 Jul 2025 13:41:58 +0000 Subject: [PATCH 03/16] epr etl example --- etl/epr_etl_example.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 etl/epr_etl_example.py diff --git a/etl/epr_etl_example.py b/etl/epr_etl_example.py new file mode 100644 index 0000000..e69de29 From 549d7265cb3bac4ef258d270bdee99e17ffdd05f Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 14 Jul 2025 15:00:32 +0000 Subject: [PATCH 04/16] need to upload an image otherwise nothing will work --- .../workflows/push_docker_image_to_ecr.yml | 43 +++++++++ deployment/lambda.tf | 90 +++++++++++++++---- etl/epr_etl_example.py | 8 ++ etl/fileReader/pdfReaderToText.py | 3 +- 4 files changed, 128 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/push_docker_image_to_ecr.yml diff --git a/.github/workflows/push_docker_image_to_ecr.yml b/.github/workflows/push_docker_image_to_ecr.yml new file mode 100644 index 0000000..394f7da --- /dev/null +++ b/.github/workflows/push_docker_image_to_ecr.yml @@ -0,0 +1,43 @@ +name: Build and Push Docker to ECR + +on: + push: + branches: [feature/energy_report_etl, main] + +env: + AWS_REGION: eu-west-2 + ECR_REPOSITORY: survey_extractor + +jobs: + deploy: + runs-on: ubuntu-latest + + permissions: + id-token: write + contents: read + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam:::role/ + aws-region: ${{ env.AWS_REGION }} + + - name: Log in to Amazon ECR + uses: aws-actions/amazon-ecr-login@v2 + + - name: Build, tag, and push Docker image to ECR + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + IMAGE_TAG: latest + run: | + IMAGE_URI=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} + + echo "Building Docker image..." + docker build -t $IMAGE_URI . + + echo "Pushing Docker image to ECR..." + docker push $IMAGE_URI diff --git a/deployment/lambda.tf b/deployment/lambda.tf index dc24055..4b56763 100644 --- a/deployment/lambda.tf +++ b/deployment/lambda.tf @@ -3,16 +3,21 @@ resource "aws_sqs_queue" "my_queue" { name = "my-lambda-queue" } -# IAM role that the Lambda function will assume to get permissions +# Create an ECR repository to store the Docker image for the Lambda function +resource "aws_ecr_repository" "lambda_repo" { + name = "survey_extractor" +} + +# IAM role that the Lambda function will assume resource "aws_iam_role" "lambda_exec_role" { name = "lambda-exec-role" assume_role_policy = jsonencode({ - Version = "2012-10-17" + Version = "2012-10-17", Statement = [ { - Action = "sts:AssumeRole" - Effect = "Allow" + Action = "sts:AssumeRole", + Effect = "Allow", Principal = { Service = "lambda.amazonaws.com" } @@ -21,22 +26,55 @@ resource "aws_iam_role" "lambda_exec_role" { }) } -# Attach the basic execution policy (writes logs to CloudWatch) to the Lambda role +# Attach AWS-managed policy for basic Lambda execution (CloudWatch logging) resource "aws_iam_role_policy_attachment" "lambda_basic_execution" { role = aws_iam_role.lambda_exec_role.name policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" } -# Give Lambda permission to poll and process SQS messages -resource "aws_iam_role_policy_attachment" "sqs_access" { - role = aws_iam_role.lambda_exec_role.name - policy_arn = "arn:aws:iam::aws:policy/AWSLambdaSQSQueueExecutionRole" +# Custom policy: SQS access + ECR image pull permissions +resource "aws_iam_policy" "lambda_custom_policy" { + name = "lambda-sqs-ecr-policy" + + policy = jsonencode({ + Version = "2012-10-17", + Statement = [ + # Allow Lambda to read from SQS + { + Effect = "Allow", + Action = [ + "sqs:ReceiveMessage", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes" + ], + Resource = aws_sqs_queue.my_queue.arn + }, + # Allow Lambda to pull images from ECR + { + Effect = "Allow", + Action = [ + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage", + "ecr:BatchCheckLayerAvailability" + ], + Resource = aws_ecr_repository.lambda_repo.arn + }, + # Needed to authenticate to ECR (pulling the image) + { + Effect = "Allow", + Action = [ + "ecr:GetAuthorizationToken" + ], + Resource = "*" + } + ] + }) } - -# Create an ECR repository to store the Docker image for the Lambda function -resource "aws_ecr_repository" "lambda_repo" { - name = "lambda-hello-world" +# Attach the custom policy to the Lambda role +resource "aws_iam_role_policy_attachment" "lambda_custom_policy_attach" { + role = aws_iam_role.lambda_exec_role.name + policy_arn = aws_iam_policy.lambda_custom_policy.arn } # Define the Lambda function using a Docker image from ECR @@ -45,8 +83,7 @@ resource "aws_lambda_function" "lambda_docker" { role = aws_iam_role.lambda_exec_role.arn package_type = "Image" image_uri = "${aws_ecr_repository.lambda_repo.repository_url}:latest" - - timeout = 10 + timeout = 10 } # Connect the SQS queue to the Lambda so it gets triggered by incoming messages @@ -55,3 +92,26 @@ resource "aws_lambda_event_source_mapping" "sqs_trigger" { function_name = aws_lambda_function.lambda_docker.arn batch_size = 1 } + + +resource "aws_ecr_repository_policy" "lambda_ecr_access" { + repository = aws_ecr_repository.lambda_repo.name + + policy = jsonencode({ + Version = "2008-10-17", + Statement = [ + { + Sid = "AllowLambdaPull", + Effect = "Allow", + Principal = { + Service = "lambda.amazonaws.com" + }, + Action = [ + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage", + "ecr:BatchCheckLayerAvailability" + ] + } + ] + }) +} \ No newline at end of file diff --git a/etl/epr_etl_example.py b/etl/epr_etl_example.py index e69de29..bd7601e 100644 --- a/etl/epr_etl_example.py +++ b/etl/epr_etl_example.py @@ -0,0 +1,8 @@ +from etl.surveyedData.surveryedData import surveyedDataProcessor + +files = [ + "/tmp/sharepoint/Sandwell/SANDWELL-001/26 Willow close B64 6EG/Content (13).pdf", +] + +from sqlalchemy.dialects.postgresql import UUID +sdp = surveyedDataProcessor("fake address", files) diff --git a/etl/fileReader/pdfReaderToText.py b/etl/fileReader/pdfReaderToText.py index bc9643f..531c316 100644 --- a/etl/fileReader/pdfReaderToText.py +++ b/etl/fileReader/pdfReaderToText.py @@ -3,6 +3,7 @@ import logging import pymupdf from etl.fileReader.sitenotes import QuidosSiteNotesExtractor, CSR, WarmHomesConditionReport, ECOConditionReport, RDSAPEnergyReport from etl.fileReader.reportType import ReportType +from pprint import pprint class pdfReaderToText(): @@ -24,7 +25,7 @@ class pdfReaderToText(): self.all_text += text self.text_list = self.all_text.split('\n') - print(self.text_list) + pprint(self.text_list) def get_list_of_text(self): return self.text_list From 73318c493bacc847c2bb1d974b6d7e5ed612c265 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 14 Jul 2025 15:14:49 +0000 Subject: [PATCH 05/16] see if login works --- .github/workflows/push_docker_image_to_ecr.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/push_docker_image_to_ecr.yml b/.github/workflows/push_docker_image_to_ecr.yml index 394f7da..cfbf6b9 100644 --- a/.github/workflows/push_docker_image_to_ecr.yml +++ b/.github/workflows/push_docker_image_to_ecr.yml @@ -9,7 +9,7 @@ env: ECR_REPOSITORY: survey_extractor jobs: - deploy: + build-and-push-to-elastic-container-registry: runs-on: ubuntu-latest permissions: @@ -20,13 +20,16 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - - name: Configure AWS credentials + - name: AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: - role-to-assume: arn:aws:iam:::role/ - aws-region: ${{ env.AWS_REGION }} + # as of 14/07/2025 it'll be using user:Junte's keys + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_REGION }} - name: Log in to Amazon ECR + id: login-ecr uses: aws-actions/amazon-ecr-login@v2 - name: Build, tag, and push Docker image to ECR From c2d3608a1ac4433928dd982f5d64ab165b05febe Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 14 Jul 2025 15:16:38 +0000 Subject: [PATCH 06/16] show me pwd --- .github/workflows/push_docker_image_to_ecr.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/push_docker_image_to_ecr.yml b/.github/workflows/push_docker_image_to_ecr.yml index cfbf6b9..9ae984d 100644 --- a/.github/workflows/push_docker_image_to_ecr.yml +++ b/.github/workflows/push_docker_image_to_ecr.yml @@ -38,9 +38,11 @@ jobs: IMAGE_TAG: latest run: | IMAGE_URI=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} - + echo "pwd" + pwd + echo "Building Docker image..." - docker build -t $IMAGE_URI . + docker build -t $IMAGE_URI -f . echo "Pushing Docker image to ECR..." docker push $IMAGE_URI From 16bd105f86a3927f5a562ad868f919ca8abbc4c0 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 14 Jul 2025 15:17:31 +0000 Subject: [PATCH 07/16] show me pwd --- .github/workflows/push_docker_image_to_ecr.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/push_docker_image_to_ecr.yml b/.github/workflows/push_docker_image_to_ecr.yml index 9ae984d..9838854 100644 --- a/.github/workflows/push_docker_image_to_ecr.yml +++ b/.github/workflows/push_docker_image_to_ecr.yml @@ -40,9 +40,9 @@ jobs: IMAGE_URI=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} echo "pwd" pwd - + echo "Building Docker image..." - docker build -t $IMAGE_URI -f . + docker build -t $IMAGE_URI -f deployment/lambda/Dockerfile . echo "Pushing Docker image to ECR..." docker push $IMAGE_URI From 6554418e6fcc3e5a896a254e4d7420c295834100 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 14 Jul 2025 15:18:43 +0000 Subject: [PATCH 08/16] lambda spelling --- .github/workflows/push_docker_image_to_ecr.yml | 2 +- deployment/{lamda => lambda}/Dockerfile | 0 deployment/{lamda => lambda}/app.py | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename deployment/{lamda => lambda}/Dockerfile (100%) rename deployment/{lamda => lambda}/app.py (100%) diff --git a/.github/workflows/push_docker_image_to_ecr.yml b/.github/workflows/push_docker_image_to_ecr.yml index 9838854..0e0d558 100644 --- a/.github/workflows/push_docker_image_to_ecr.yml +++ b/.github/workflows/push_docker_image_to_ecr.yml @@ -40,7 +40,7 @@ jobs: IMAGE_URI=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} echo "pwd" pwd - + ls -la echo "Building Docker image..." docker build -t $IMAGE_URI -f deployment/lambda/Dockerfile . diff --git a/deployment/lamda/Dockerfile b/deployment/lambda/Dockerfile similarity index 100% rename from deployment/lamda/Dockerfile rename to deployment/lambda/Dockerfile diff --git a/deployment/lamda/app.py b/deployment/lambda/app.py similarity index 100% rename from deployment/lamda/app.py rename to deployment/lambda/app.py From 8b1b0b8979edd7eee79d68165a453728417b2e5b Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 14 Jul 2025 15:20:16 +0000 Subject: [PATCH 09/16] lambda spelling --- deployment/lambda/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/lambda/Dockerfile b/deployment/lambda/Dockerfile index 4b57200..b27e4ff 100644 --- a/deployment/lambda/Dockerfile +++ b/deployment/lambda/Dockerfile @@ -2,7 +2,7 @@ FROM public.ecr.aws/lambda/python:3.11 # Copy function code -COPY app.py ./ +COPY deployment/lambda/app.py ./ # Set the CMD to your handler CMD ["app.handler"] \ No newline at end of file From 37d8a885315b0dfabbe33a0673fc3063ca54080f Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 14 Jul 2025 15:22:57 +0000 Subject: [PATCH 10/16] lambda spelling --- .github/workflows/push_docker_image_to_ecr.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/push_docker_image_to_ecr.yml b/.github/workflows/push_docker_image_to_ecr.yml index 0e0d558..53a640f 100644 --- a/.github/workflows/push_docker_image_to_ecr.yml +++ b/.github/workflows/push_docker_image_to_ecr.yml @@ -46,3 +46,4 @@ jobs: echo "Pushing Docker image to ECR..." docker push $IMAGE_URI + From 616b6dde9bcea23e5d6e9c4bc934b87d5b6e45a4 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 14 Jul 2025 15:29:18 +0000 Subject: [PATCH 11/16] lambda spelling --- deployment/lambda/app.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deployment/lambda/app.py b/deployment/lambda/app.py index 696c654..05e2693 100644 --- a/deployment/lambda/app.py +++ b/deployment/lambda/app.py @@ -7,4 +7,5 @@ def handler(event, context): return { 'statusCode': 200, 'body': 'Hello World' - } \ No newline at end of file + } + From c220f35252c1de6e3b251452b0080378d7a042cf Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 15 Jul 2025 12:24:14 +0000 Subject: [PATCH 12/16] example of lambda function --- .github/workflows/push_docker_image_to_ecr.yml | 4 ++-- deployment/lambda/Dockerfile | 8 -------- deployment/lambda/app.py | 11 ----------- deployment/{lambda.tf => lambda_example.tf} | 8 ++++++-- 4 files changed, 8 insertions(+), 23 deletions(-) delete mode 100644 deployment/lambda/Dockerfile delete mode 100644 deployment/lambda/app.py rename deployment/{lambda.tf => lambda_example.tf} (91%) diff --git a/.github/workflows/push_docker_image_to_ecr.yml b/.github/workflows/push_docker_image_to_ecr.yml index 53a640f..e6722cd 100644 --- a/.github/workflows/push_docker_image_to_ecr.yml +++ b/.github/workflows/push_docker_image_to_ecr.yml @@ -1,4 +1,4 @@ -name: Build and Push Docker to ECR +name: Build and Push Docker Image to ECR lambda example on: push: @@ -6,7 +6,7 @@ on: env: AWS_REGION: eu-west-2 - ECR_REPOSITORY: survey_extractor + ECR_REPOSITORY: lambda_example jobs: build-and-push-to-elastic-container-registry: diff --git a/deployment/lambda/Dockerfile b/deployment/lambda/Dockerfile deleted file mode 100644 index b27e4ff..0000000 --- a/deployment/lambda/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -# AWS Lambda python pacakge -FROM public.ecr.aws/lambda/python:3.11 - -# Copy function code -COPY deployment/lambda/app.py ./ - -# Set the CMD to your handler -CMD ["app.handler"] \ No newline at end of file diff --git a/deployment/lambda/app.py b/deployment/lambda/app.py deleted file mode 100644 index 05e2693..0000000 --- a/deployment/lambda/app.py +++ /dev/null @@ -1,11 +0,0 @@ -""" -A quick example of lambda working a function in python -""" - -def handler(event, context): - print("Hello from Python function. This shold be running from a dockerfile env and executed on a aws lambda!") - return { - 'statusCode': 200, - 'body': 'Hello World' - } - diff --git a/deployment/lambda.tf b/deployment/lambda_example.tf similarity index 91% rename from deployment/lambda.tf rename to deployment/lambda_example.tf index 4b56763..0748af4 100644 --- a/deployment/lambda.tf +++ b/deployment/lambda_example.tf @@ -1,3 +1,7 @@ +# This is an example file to setup a lamda function with a sqs and cloudwatch. +# Please us this as a template for future lambda. +# Be sure to push the image you are using to ECR or it won't deploy properly + # Create an SQS queue that will trigger the Lambda resource "aws_sqs_queue" "my_queue" { name = "my-lambda-queue" @@ -5,7 +9,7 @@ resource "aws_sqs_queue" "my_queue" { # Create an ECR repository to store the Docker image for the Lambda function resource "aws_ecr_repository" "lambda_repo" { - name = "survey_extractor" + name = "lambda_example" } # IAM role that the Lambda function will assume @@ -79,7 +83,7 @@ resource "aws_iam_role_policy_attachment" "lambda_custom_policy_attach" { # Define the Lambda function using a Docker image from ECR resource "aws_lambda_function" "lambda_docker" { - function_name = "docker-hello-world" + function_name = "docker-hello-world-python-example" role = aws_iam_role.lambda_exec_role.arn package_type = "Image" image_uri = "${aws_ecr_repository.lambda_repo.repository_url}:latest" From e354c3d4f764142650c3cc8901c5b3ffab0bdaf5 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 15 Jul 2025 12:25:25 +0000 Subject: [PATCH 13/16] example of lambda function --- .github/workflows/push_docker_image_to_ecr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push_docker_image_to_ecr.yml b/.github/workflows/push_docker_image_to_ecr.yml index e6722cd..7bc8040 100644 --- a/.github/workflows/push_docker_image_to_ecr.yml +++ b/.github/workflows/push_docker_image_to_ecr.yml @@ -42,7 +42,7 @@ jobs: pwd ls -la echo "Building Docker image..." - docker build -t $IMAGE_URI -f deployment/lambda/Dockerfile . + docker build -t $IMAGE_URI -f deployment/lambda_example/Dockerfile . echo "Pushing Docker image to ECR..." docker push $IMAGE_URI From 83ffa4234e640a67ba145ba6b29aa3e401d73607 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 15 Jul 2025 12:27:23 +0000 Subject: [PATCH 14/16] add lambda exaple files --- deployment/lambda_example/Dockerfile | 8 ++++++++ deployment/lambda_example/app.py | 11 +++++++++++ 2 files changed, 19 insertions(+) create mode 100644 deployment/lambda_example/Dockerfile create mode 100644 deployment/lambda_example/app.py diff --git a/deployment/lambda_example/Dockerfile b/deployment/lambda_example/Dockerfile new file mode 100644 index 0000000..c1caf03 --- /dev/null +++ b/deployment/lambda_example/Dockerfile @@ -0,0 +1,8 @@ +# AWS Lambda python pacakge +FROM public.ecr.aws/lambda/python:3.11 + +# Copy function code +COPY deployment/lambda_example/app.py ./ + +# Set the CMD to your handler +CMD ["app.handler"] \ No newline at end of file diff --git a/deployment/lambda_example/app.py b/deployment/lambda_example/app.py new file mode 100644 index 0000000..05e2693 --- /dev/null +++ b/deployment/lambda_example/app.py @@ -0,0 +1,11 @@ +""" +A quick example of lambda working a function in python +""" + +def handler(event, context): + print("Hello from Python function. This shold be running from a dockerfile env and executed on a aws lambda!") + return { + 'statusCode': 200, + 'body': 'Hello World' + } + From 0811339557a0ee572e55e520477eb7901fb97dbe Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 15 Jul 2025 15:15:42 +0000 Subject: [PATCH 15/16] summary report and epr with data is now identified --- etl/epr_etl_example.py | 1 + etl/fileReader/pdfReaderToText.py | 23 +++++++++++++++++------ etl/fileReader/reportType.py | 4 ++-- etl/fileReader/sitenotes.py | 25 +++++++++++++++++-------- etl/hubSpotClient/types.py | 7 +++++-- etl/surveyedData/surveryedData.py | 10 +++++++--- 6 files changed, 49 insertions(+), 21 deletions(-) diff --git a/etl/epr_etl_example.py b/etl/epr_etl_example.py index bd7601e..866c75a 100644 --- a/etl/epr_etl_example.py +++ b/etl/epr_etl_example.py @@ -2,6 +2,7 @@ from etl.surveyedData.surveryedData import surveyedDataProcessor files = [ "/tmp/sharepoint/Sandwell/SANDWELL-001/26 Willow close B64 6EG/Content (13).pdf", + "/tmp/sharepoint/Livewest/Livewest-001/12 Birch End/Summary Information 12 Birch End.pdf" ] from sqlalchemy.dialects.postgresql import UUID diff --git a/etl/fileReader/pdfReaderToText.py b/etl/fileReader/pdfReaderToText.py index 531c316..9668eed 100644 --- a/etl/fileReader/pdfReaderToText.py +++ b/etl/fileReader/pdfReaderToText.py @@ -1,7 +1,15 @@ from etl.utils.logger import Logger import logging import pymupdf -from etl.fileReader.sitenotes import QuidosSiteNotesExtractor, CSR, WarmHomesConditionReport, ECOConditionReport, RDSAPEnergyReport +from etl.fileReader.sitenotes import ( + QuidosSiteNotesExtractor, + CSR, + WarmHomesConditionReport, + ECOConditionReport, + EnergyPerformanceReportWithData, + EnergyPerformanceReportSummaryInformation + +) from etl.fileReader.reportType import ReportType from pprint import pprint @@ -46,8 +54,10 @@ class pdfReaderToText(): self.type = ReportType.WARM_HOMES_CONDITION_REPORT elif "Domna NEW PAS 2035 ECO Condition Report".lower() in self.text_list[0].lower(): self.type = ReportType.ECO_CONDITION_REPORT - elif "ENERGY REPORT".lower() == self.text_list[0].lower(): - self.type = ReportType.RDSAP_ENERGY_REPORT + elif "ENERGY REPORT".lower() == self.text_list[0].lower() and "Data inputs" in self.text_list: + self.type = ReportType.ENERGY_PERFORMANCE_REPORT_WITH_DATA + elif "Summary Information".lower() == self.text_list[0].lower(): + self.type = ReportType.ENERGY_PERFORMANCE_REPORT_SUMMARY_INFORMATION else: pass return self.type @@ -63,6 +73,7 @@ class pdfReaderToText(): return WarmHomesConditionReport(self.text_list) elif self.type == ReportType.ECO_CONDITION_REPORT: return ECOConditionReport(self.text_list) - elif self.type == ReportType.RDSAP_ENERGY_REPORT: - return RDSAPEnergyReport(self.text_list) - \ No newline at end of file + elif self.type == ReportType.ENERGY_PERFORMANCE_REPORT_WITH_DATA: + return EnergyPerformanceReportWithData(self.text_list) + elif self.type == ReportType.ENERGY_PERFORMANCE_REPORT_SUMMARY_INFORMATION: + return EnergyPerformanceReportSummaryInformation(self.text_list) \ No newline at end of file diff --git a/etl/fileReader/reportType.py b/etl/fileReader/reportType.py index 07ac12e..9fce283 100644 --- a/etl/fileReader/reportType.py +++ b/etl/fileReader/reportType.py @@ -4,12 +4,12 @@ from enum import Enum class ReportType(Enum): QUIDOS_PRESITE_NOTE = "quidos_presite_note" CHARTED_SURVEYOR_REPORT = "charted_surveyor_report" - ENERGY_PERFORMANCE_REPORT = "energy_performance_report" U_VALUE_CALCULATOR_REPORT = "u_value_calculator_report" OVERWRITING_U_VALUE_DECLARATION_FORM = "overwriting_u_value_declaration_form" ECO_CONDITION_REPORT = "osmosis_condition_pas_2035_report" WARM_HOMES_CONDITION_REPORT = "warm_homes_condition_pas_2035_report" - RDSAP_ENERGY_REPORT = "rdsap_energy_report" + ENERGY_PERFORMANCE_REPORT_WITH_DATA = "energy_performance_report_with_data" + ENERGY_PERFORMANCE_REPORT_SUMMARY_INFORMATION = "energy_performance_report_summary_information" LIG_XML = "lodgement_xml_needed_for_lodgement_to_like_trademark" RDSAP_XML = "reduce_xml_needed_to_generate_full_sap_xml" FULLSAP_XML = "full_xml_needed_for_co_ordination" diff --git a/etl/fileReader/sitenotes.py b/etl/fileReader/sitenotes.py index f46726c..1df8747 100644 --- a/etl/fileReader/sitenotes.py +++ b/etl/fileReader/sitenotes.py @@ -88,14 +88,7 @@ class CSR(SiteNotesExtractor): type=dict_.get('detailed_description_of_existing_cavity_wall_insulation_', "") ) if dict_ is not None else None -class RDSAPEnergyReport(SiteNotesExtractor): - def __init__(self, data_list): - super().__init__(data_list) - self.type = ReportType.RDSAP_ENERGY_REPORT - self.master_obj = self.setup_energy_report() - def setup_energy_report(self): - pass class ECOConditionReport(SiteNotesExtractor): def __init__(self, data_list): @@ -1597,4 +1590,20 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): main_gas_avalible=True if dict_.get("main_gas_available", "NO").upper() == "YES" else False, ) - \ No newline at end of file +class EnergyPerformanceReportWithData(SiteNotesExtractor): + def __init__(self, data_list): + super().__init__(data_list) + self.type = ReportType.ENERGY_PERFORMANCE_REPORT_WITH_DATA + self.master_obj = self.setup() + + def setup(self): + pass + +class EnergyPerformanceReportSummaryInformation(SiteNotesExtractor): + def __init__(self, data_list): + super().__init__(data_list) + self.type = ReportType.ENERGY_PERFORMANCE_REPORT_SUMMARY_INFORMATION + self.master_obj = self.setup() + + def setup(self): + pass \ No newline at end of file diff --git a/etl/hubSpotClient/types.py b/etl/hubSpotClient/types.py index dd97011..e004243 100644 --- a/etl/hubSpotClient/types.py +++ b/etl/hubSpotClient/types.py @@ -80,8 +80,8 @@ class SubmissionInfoFromDeal(BaseModel): if sdp.condition_report is None: missing_items.append("Condition Report") - if sdp.energy_report is None: - missing_items.append("Energy Report PDF") + if sdp.epr_summary_information is None: + missing_items.append("EPR Energy report with data is missing") if sdp.rd_sap_xml is None: missing_items.append("RDSAP XML") @@ -89,6 +89,9 @@ class SubmissionInfoFromDeal(BaseModel): if sdp.lig_sap_xml is None: missing_items.append("LIG SAP XML") + if sdp.epr_summary_information is None: + missing_items.append("EPR Summary information is missing") + if missing_items: raise ValueError(f"Missing required items: {', '.join(missing_items)}") diff --git a/etl/surveyedData/surveryedData.py b/etl/surveyedData/surveryedData.py index 485d617..f75b112 100644 --- a/etl/surveyedData/surveryedData.py +++ b/etl/surveyedData/surveryedData.py @@ -41,7 +41,8 @@ class surveyedDataProcessor(): self.csr = None self.condition_report = None self.hubspot_deal_id = None - self.energy_report = None + self.epr_with_data = None + self.epr_summary_information = None self.full_sap_xml = None self.lig_sap_xml = None self.rd_sap_xml = None @@ -64,8 +65,11 @@ class surveyedDataProcessor(): self.condition_report = pdf.get_reader() elif pdf.type == ReportType.ECO_CONDITION_REPORT: self.condition_report = pdf.get_reader() - elif pdf.type == ReportType.RDSAP_ENERGY_REPORT: - self.energy_report = pdf.get_reader() + elif pdf.type == ReportType.ENERGY_PERFORMANCE_REPORT_WITH_DATA: + self.epr_with_data = pdf.get_reader() + elif pdf.type == ReportType.ENERGY_PERFORMANCE_REPORT_SUMMARY_INFORMATION: + self.epr_summary_information = pdf.get_reader() + elif file.lower().endswith('.xml'): xml = xmlReader(file) if xml: From b3f9929554aca2740c07e6860a1f6de6086e73ef Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 15 Jul 2025 15:21:31 +0000 Subject: [PATCH 16/16] modified script to re run existing scripts --- etl/hubspot_surveyed_needs_sign_off.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/etl/hubspot_surveyed_needs_sign_off.py b/etl/hubspot_surveyed_needs_sign_off.py index a8eeabc..7794d07 100644 --- a/etl/hubspot_surveyed_needs_sign_off.py +++ b/etl/hubspot_surveyed_needs_sign_off.py @@ -16,10 +16,17 @@ from etl.hubSpotClient.hubspot import DealStage, HubSpotClient os.environ["DATABASE_URL"] = "postgresql://postgres:makingwarmhomes@db:5432/postgres" hubspotClient = HubSpotClient() -deals = hubspotClient.get_deals_from_deal_stage(DealStage.SURVEYED_COMPLETE_NEEDS_SIGN_OFF) + +# files missing from assessor column +deals = hubspotClient.get_deals_from_deal_stage(DealStage.NEEDS_ADDITIONAL_INFORMATION_FROM_ASSESSOR) for deal in deals: hubspotClient.move_deals_to_different_stage([deal.deal_id], DealStage.SURVEYED_COMPLETED_SIGNED_OFF.value) -# TODO load when we are at 'ready to co-ordination' - script! \ No newline at end of file +# Survyed_complete +deals = hubspotClient.get_deals_from_deal_stage(DealStage.SURVEYED_COMPLETE_NEEDS_SIGN_OFF) + + +for deal in deals: + hubspotClient.move_deals_to_different_stage([deal.deal_id], DealStage.SURVEYED_COMPLETED_SIGNED_OFF.value)