Merge pull request #67 from Hestia-Homes/feature/elmhurst_epr

Feature/elmhurst epr

.devcontainer/devcontainer.json

@@ -7,6 +7,7 @@
   "postStartCommand": "bash .devcontainer/post-install.sh",
   "mounts": [
     "source=${localEnv:HOME},target=/workspaces/home,type=bind",
+    // Make sure you aws credentials are saved at ~/.aws
     "source=${localEnv:HOME}/.aws/,target=/home/vscode/.aws/,type=bind"
   ],
   "customizations": {

.github/workflows/push_docker_image_to_ecr.yml

@@ -0,0 +1,49 @@
+name: Build and Push Docker Image to ECR lambda example
+
+on:
+  push:
+    branches: [feature/energy_report_etl, main] 
+
+env:
+  AWS_REGION: eu-west-2
+  ECR_REPOSITORY: lambda_example
+
+jobs:
+  build-and-push-to-elastic-container-registry:
+    runs-on: ubuntu-latest
+
+    permissions:
+      id-token: write
+      contents: read
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: AWS credentials
+      uses: aws-actions/configure-aws-credentials@v4
+      with:
+        # as of 14/07/2025 it'll be using user:Junte's keys
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws-region: ${{ secrets.AWS_REGION }}
+
+    - name: Log in to Amazon ECR
+      id: login-ecr
+      uses: aws-actions/amazon-ecr-login@v2
+
+    - name: Build, tag, and push Docker image to ECR
+      env:
+        ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+        IMAGE_TAG: latest
+      run: |
+        IMAGE_URI=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }}
+        echo "pwd"
+        pwd
+        ls -la
+        echo "Building Docker image..."
+        docker build -t $IMAGE_URI -f deployment/lambda_example/Dockerfile .
+
+        echo "Pushing Docker image to ECR..."
+        docker push $IMAGE_URI
+

deployment/lambda_example.tf

@@ -0,0 +1,121 @@
+# This is an example file to setup a lamda function with a sqs and cloudwatch.
+# Please us this as a template for future lambda.
+# Be sure to push the image you are using to ECR or it won't deploy properly
+
+# Create an SQS queue that will trigger the Lambda
+resource "aws_sqs_queue" "my_queue" {
+  name = "my-lambda-queue"
+}
+
+# Create an ECR repository to store the Docker image for the Lambda function
+resource "aws_ecr_repository" "lambda_repo" {
+  name = "lambda_example"
+}
+
+# IAM role that the Lambda function will assume
+resource "aws_iam_role" "lambda_exec_role" {
+  name = "lambda-exec-role"
+
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17",
+    Statement = [
+      {
+        Action = "sts:AssumeRole",
+        Effect = "Allow",
+        Principal = {
+          Service = "lambda.amazonaws.com"
+        }
+      }
+    ]
+  })
+}
+
+# Attach AWS-managed policy for basic Lambda execution (CloudWatch logging)
+resource "aws_iam_role_policy_attachment" "lambda_basic_execution" {
+  role       = aws_iam_role.lambda_exec_role.name
+  policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
+}
+
+# Custom policy: SQS access + ECR image pull permissions
+resource "aws_iam_policy" "lambda_custom_policy" {
+  name = "lambda-sqs-ecr-policy"
+
+  policy = jsonencode({
+    Version = "2012-10-17",
+    Statement = [
+      # Allow Lambda to read from SQS
+      {
+        Effect = "Allow",
+        Action = [
+          "sqs:ReceiveMessage",
+          "sqs:DeleteMessage",
+          "sqs:GetQueueAttributes"
+        ],
+        Resource = aws_sqs_queue.my_queue.arn
+      },
+      # Allow Lambda to pull images from ECR
+      {
+        Effect = "Allow",
+        Action = [
+          "ecr:GetDownloadUrlForLayer",
+          "ecr:BatchGetImage",
+          "ecr:BatchCheckLayerAvailability"
+        ],
+        Resource = aws_ecr_repository.lambda_repo.arn
+      },
+      # Needed to authenticate to ECR (pulling the image)
+      {
+        Effect = "Allow",
+        Action = [
+          "ecr:GetAuthorizationToken"
+        ],
+        Resource = "*"
+      }
+    ]
+  })
+}
+
+# Attach the custom policy to the Lambda role
+resource "aws_iam_role_policy_attachment" "lambda_custom_policy_attach" {
+  role       = aws_iam_role.lambda_exec_role.name
+  policy_arn = aws_iam_policy.lambda_custom_policy.arn
+}
+
+# Define the Lambda function using a Docker image from ECR
+resource "aws_lambda_function" "lambda_docker" {
+  function_name = "docker-hello-world-python-example"
+  role          = aws_iam_role.lambda_exec_role.arn
+  package_type  = "Image"
+  image_uri     = "${aws_ecr_repository.lambda_repo.repository_url}:latest"
+  timeout       = 10
+}
+
+# Connect the SQS queue to the Lambda so it gets triggered by incoming messages
+resource "aws_lambda_event_source_mapping" "sqs_trigger" {
+  event_source_arn = aws_sqs_queue.my_queue.arn
+  function_name    = aws_lambda_function.lambda_docker.arn
+  batch_size       = 1
+}
+
+
+resource "aws_ecr_repository_policy" "lambda_ecr_access" {
+  repository = aws_ecr_repository.lambda_repo.name
+
+  policy = jsonencode({
+    Version = "2008-10-17",
+    Statement = [
+      {
+        Sid       = "AllowLambdaPull",
+        Effect    = "Allow",
+        Principal = {
+          Service = "lambda.amazonaws.com"
+        },
+        Action = [
+          "ecr:GetDownloadUrlForLayer",
+          "ecr:BatchGetImage",
+          "ecr:BatchCheckLayerAvailability"
+        ]
+      }
+    ]
+  })
+}

deployment/lambda_example/Dockerfile

@@ -0,0 +1,8 @@
+# AWS Lambda python pacakge
+FROM public.ecr.aws/lambda/python:3.11
+
+# Copy function code
+COPY deployment/lambda_example/app.py ./
+
+# Set the CMD to your handler
+CMD ["app.handler"]

deployment/lambda_example/app.py

@@ -0,0 +1,11 @@
+"""
+A quick example of lambda working a function in python
+"""
+
+def handler(event, context):
+    print("Hello from Python function. This shold be running from a dockerfile env and executed on a aws lambda!")
+    return {
+        'statusCode': 200,
+        'body': 'Hello World'
+    }
+

etl/epr_etl_example.py

@@ -0,0 +1,9 @@
+from etl.surveyedData.surveryedData import surveyedDataProcessor
+
+files  = [
+    "/tmp/sharepoint/Sandwell/SANDWELL-001/26 Willow close B64 6EG/Content (13).pdf",
+    "/tmp/sharepoint/Livewest/Livewest-001/12 Birch End/Summary Information 12 Birch End.pdf"
+]
+
+from sqlalchemy.dialects.postgresql import UUID
+sdp = surveyedDataProcessor("fake address", files)

etl/fileReader/pdfReaderToText.py

@@ -1,8 +1,17 @@
 from etl.utils.logger import Logger
 import logging
 import pymupdf
-from etl.fileReader.sitenotes import QuidosSiteNotesExtractor, CSR, WarmHomesConditionReport, ECOConditionReport, RDSAPEnergyReport
+from etl.fileReader.sitenotes import (
+    QuidosSiteNotesExtractor,
+    CSR,
+    WarmHomesConditionReport,
+    ECOConditionReport,
+    EnergyPerformanceReportWithData,
+    EnergyPerformanceReportSummaryInformation
+    
+)
 from etl.fileReader.reportType import ReportType
+from pprint import pprint
 
 class pdfReaderToText():
 
@@ -24,7 +33,7 @@ class pdfReaderToText():
             self.all_text += text
 
         self.text_list = self.all_text.split('\n')
-        print(self.text_list)
+        pprint(self.text_list)
         
     def get_list_of_text(self):
         return self.text_list
@@ -45,8 +54,10 @@ class pdfReaderToText():
                 self.type = ReportType.WARM_HOMES_CONDITION_REPORT
             elif "Domna NEW PAS 2035 ECO Condition Report".lower() in self.text_list[0].lower():
                 self.type = ReportType.ECO_CONDITION_REPORT
-            elif "ENERGY REPORT".lower() == self.text_list[0].lower():
-                self.type = ReportType.RDSAP_ENERGY_REPORT
+            elif "ENERGY REPORT".lower() == self.text_list[0].lower() and "Data inputs" in self.text_list:
+                self.type = ReportType.ENERGY_PERFORMANCE_REPORT_WITH_DATA
+            elif "Summary Information".lower() == self.text_list[0].lower():
+                self.type = ReportType.ENERGY_PERFORMANCE_REPORT_SUMMARY_INFORMATION
             else:
                 pass
             return self.type
@@ -62,6 +73,7 @@ class pdfReaderToText():
             return WarmHomesConditionReport(self.text_list)
         elif self.type == ReportType.ECO_CONDITION_REPORT:
             return ECOConditionReport(self.text_list)
-        elif self.type == ReportType.RDSAP_ENERGY_REPORT:
-            return RDSAPEnergyReport(self.text_list)
-        
+        elif self.type == ReportType.ENERGY_PERFORMANCE_REPORT_WITH_DATA:
+            return EnergyPerformanceReportWithData(self.text_list)
+        elif self.type == ReportType.ENERGY_PERFORMANCE_REPORT_SUMMARY_INFORMATION:
+            return EnergyPerformanceReportSummaryInformation(self.text_list)

etl/fileReader/reportType.py

@@ -4,12 +4,12 @@ from enum import Enum
 class ReportType(Enum):
     QUIDOS_PRESITE_NOTE = "quidos_presite_note"
     CHARTED_SURVEYOR_REPORT = "charted_surveyor_report"
-    ENERGY_PERFORMANCE_REPORT = "energy_performance_report"
     U_VALUE_CALCULATOR_REPORT = "u_value_calculator_report"
     OVERWRITING_U_VALUE_DECLARATION_FORM = "overwriting_u_value_declaration_form"
     ECO_CONDITION_REPORT = "osmosis_condition_pas_2035_report"
     WARM_HOMES_CONDITION_REPORT = "warm_homes_condition_pas_2035_report"
-    RDSAP_ENERGY_REPORT = "rdsap_energy_report"
+    ENERGY_PERFORMANCE_REPORT_WITH_DATA = "energy_performance_report_with_data"
+    ENERGY_PERFORMANCE_REPORT_SUMMARY_INFORMATION = "energy_performance_report_summary_information"
     LIG_XML = "lodgement_xml_needed_for_lodgement_to_like_trademark"
     RDSAP_XML = "reduce_xml_needed_to_generate_full_sap_xml"
     FULLSAP_XML = "full_xml_needed_for_co_ordination"

etl/fileReader/sitenotes.py

@@ -88,14 +88,7 @@ class CSR(SiteNotesExtractor):
                 type=dict_.get('detailed_description_of_existing_cavity_wall_insulation_', "")
             ) if dict_ is not None else None
 
-class RDSAPEnergyReport(SiteNotesExtractor):
-    def __init__(self, data_list):
-        super().__init__(data_list)
-        self.type = ReportType.RDSAP_ENERGY_REPORT
-        self.master_obj = self.setup_energy_report()
 
-    def setup_energy_report(self):
-        pass
 
 class ECOConditionReport(SiteNotesExtractor):
     def __init__(self, data_list):
@@ -1597,4 +1590,20 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor):
             main_gas_avalible=True if dict_.get("main_gas_available", "NO").upper() == "YES" else False,
         )
 
-    
+class EnergyPerformanceReportWithData(SiteNotesExtractor):
+    def __init__(self, data_list):
+        super().__init__(data_list)
+        self.type = ReportType.ENERGY_PERFORMANCE_REPORT_WITH_DATA
+        self.master_obj = self.setup()
+
+    def setup(self):
+        pass
+
+class EnergyPerformanceReportSummaryInformation(SiteNotesExtractor):
+    def __init__(self, data_list):
+        super().__init__(data_list)
+        self.type = ReportType.ENERGY_PERFORMANCE_REPORT_SUMMARY_INFORMATION
+        self.master_obj = self.setup()
+
+    def setup(self):
+        pass

etl/hubSpotClient/types.py

@@ -80,8 +80,8 @@ class SubmissionInfoFromDeal(BaseModel):
             if sdp.condition_report is None:
                 missing_items.append("Condition Report")
 
-            if sdp.energy_report is None:
-                missing_items.append("Energy Report PDF")
+            if sdp.epr_summary_information is None:
+                missing_items.append("EPR Energy report with data is missing")
 
             if sdp.rd_sap_xml is None:
                 missing_items.append("RDSAP XML")
@@ -89,6 +89,9 @@ class SubmissionInfoFromDeal(BaseModel):
             if sdp.lig_sap_xml is None:
                 missing_items.append("LIG SAP XML")
 
+            if sdp.epr_summary_information is None:
+                missing_items.append("EPR Summary information is missing")
+
             if missing_items:
                 raise ValueError(f"Missing required items: {', '.join(missing_items)}")
 

etl/hubspot_surveyed_needs_sign_off.py

@@ -16,10 +16,17 @@ from etl.hubSpotClient.hubspot import DealStage, HubSpotClient
 os.environ["DATABASE_URL"] = "postgresql://postgres:makingwarmhomes@db:5432/postgres"
 
 hubspotClient = HubSpotClient()
-deals = hubspotClient.get_deals_from_deal_stage(DealStage.SURVEYED_COMPLETE_NEEDS_SIGN_OFF)
+
+# files missing from assessor column
+deals = hubspotClient.get_deals_from_deal_stage(DealStage.NEEDS_ADDITIONAL_INFORMATION_FROM_ASSESSOR)
 
 
 for deal in deals:
     hubspotClient.move_deals_to_different_stage([deal.deal_id], DealStage.SURVEYED_COMPLETED_SIGNED_OFF.value)
 
-# TODO load when we are at 'ready to co-ordination' - script!
+# Survyed_complete
+deals = hubspotClient.get_deals_from_deal_stage(DealStage.SURVEYED_COMPLETE_NEEDS_SIGN_OFF)
+
+
+for deal in deals:
+    hubspotClient.move_deals_to_different_stage([deal.deal_id], DealStage.SURVEYED_COMPLETED_SIGNED_OFF.value)

etl/surveyedData/surveryedData.py

@@ -41,7 +41,8 @@ class surveyedDataProcessor():
         self.csr = None
         self.condition_report = None
         self.hubspot_deal_id = None
-        self.energy_report = None
+        self.epr_with_data = None
+        self.epr_summary_information = None
         self.full_sap_xml = None
         self.lig_sap_xml = None
         self.rd_sap_xml = None
@@ -64,8 +65,11 @@ class surveyedDataProcessor():
                         self.condition_report = pdf.get_reader()
                     elif pdf.type == ReportType.ECO_CONDITION_REPORT:
                         self.condition_report = pdf.get_reader()
-                    elif pdf.type == ReportType.RDSAP_ENERGY_REPORT:
-                        self.energy_report = pdf.get_reader()
+                    elif pdf.type == ReportType.ENERGY_PERFORMANCE_REPORT_WITH_DATA:
+                        self.epr_with_data = pdf.get_reader()
+                    elif pdf.type == ReportType.ENERGY_PERFORMANCE_REPORT_SUMMARY_INFORMATION:
+                        self.epr_summary_information = pdf.get_reader()
+
             elif file.lower().endswith('.xml'):
                 xml = xmlReader(file)
                 if xml: