diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 445d2e6..d050974 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -7,6 +7,7 @@ "postStartCommand": "bash .devcontainer/post-install.sh", "mounts": [ "source=${localEnv:HOME},target=/workspaces/home,type=bind", + // Make sure you aws credentials are saved at ~/.aws "source=${localEnv:HOME}/.aws/,target=/home/vscode/.aws/,type=bind" ], "customizations": { diff --git a/.github/workflows/push_docker_image_to_ecr.yml b/.github/workflows/push_docker_image_to_ecr.yml new file mode 100644 index 0000000..7bc8040 --- /dev/null +++ b/.github/workflows/push_docker_image_to_ecr.yml @@ -0,0 +1,49 @@ +name: Build and Push Docker Image to ECR lambda example + +on: + push: + branches: [feature/energy_report_etl, main] + +env: + AWS_REGION: eu-west-2 + ECR_REPOSITORY: lambda_example + +jobs: + build-and-push-to-elastic-container-registry: + runs-on: ubuntu-latest + + permissions: + id-token: write + contents: read + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + # as of 14/07/2025 it'll be using user:Junte's keys + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_REGION }} + + - name: Log in to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Build, tag, and push Docker image to ECR + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + IMAGE_TAG: latest + run: | + IMAGE_URI=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} + echo "pwd" + pwd + ls -la + echo "Building Docker image..." + docker build -t $IMAGE_URI -f deployment/lambda_example/Dockerfile . + + echo "Pushing Docker image to ECR..." + docker push $IMAGE_URI + diff --git a/deployment/lambda_example.tf b/deployment/lambda_example.tf new file mode 100644 index 0000000..0748af4 --- /dev/null +++ b/deployment/lambda_example.tf @@ -0,0 +1,121 @@ +# This is an example file to setup a lamda function with a sqs and cloudwatch. +# Please us this as a template for future lambda. +# Be sure to push the image you are using to ECR or it won't deploy properly + +# Create an SQS queue that will trigger the Lambda +resource "aws_sqs_queue" "my_queue" { + name = "my-lambda-queue" +} + +# Create an ECR repository to store the Docker image for the Lambda function +resource "aws_ecr_repository" "lambda_repo" { + name = "lambda_example" +} + +# IAM role that the Lambda function will assume +resource "aws_iam_role" "lambda_exec_role" { + name = "lambda-exec-role" + + assume_role_policy = jsonencode({ + Version = "2012-10-17", + Statement = [ + { + Action = "sts:AssumeRole", + Effect = "Allow", + Principal = { + Service = "lambda.amazonaws.com" + } + } + ] + }) +} + +# Attach AWS-managed policy for basic Lambda execution (CloudWatch logging) +resource "aws_iam_role_policy_attachment" "lambda_basic_execution" { + role = aws_iam_role.lambda_exec_role.name + policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" +} + +# Custom policy: SQS access + ECR image pull permissions +resource "aws_iam_policy" "lambda_custom_policy" { + name = "lambda-sqs-ecr-policy" + + policy = jsonencode({ + Version = "2012-10-17", + Statement = [ + # Allow Lambda to read from SQS + { + Effect = "Allow", + Action = [ + "sqs:ReceiveMessage", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes" + ], + Resource = aws_sqs_queue.my_queue.arn + }, + # Allow Lambda to pull images from ECR + { + Effect = "Allow", + Action = [ + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage", + "ecr:BatchCheckLayerAvailability" + ], + Resource = aws_ecr_repository.lambda_repo.arn + }, + # Needed to authenticate to ECR (pulling the image) + { + Effect = "Allow", + Action = [ + "ecr:GetAuthorizationToken" + ], + Resource = "*" + } + ] + }) +} + +# Attach the custom policy to the Lambda role +resource "aws_iam_role_policy_attachment" "lambda_custom_policy_attach" { + role = aws_iam_role.lambda_exec_role.name + policy_arn = aws_iam_policy.lambda_custom_policy.arn +} + +# Define the Lambda function using a Docker image from ECR +resource "aws_lambda_function" "lambda_docker" { + function_name = "docker-hello-world-python-example" + role = aws_iam_role.lambda_exec_role.arn + package_type = "Image" + image_uri = "${aws_ecr_repository.lambda_repo.repository_url}:latest" + timeout = 10 +} + +# Connect the SQS queue to the Lambda so it gets triggered by incoming messages +resource "aws_lambda_event_source_mapping" "sqs_trigger" { + event_source_arn = aws_sqs_queue.my_queue.arn + function_name = aws_lambda_function.lambda_docker.arn + batch_size = 1 +} + + +resource "aws_ecr_repository_policy" "lambda_ecr_access" { + repository = aws_ecr_repository.lambda_repo.name + + policy = jsonencode({ + Version = "2008-10-17", + Statement = [ + { + Sid = "AllowLambdaPull", + Effect = "Allow", + Principal = { + Service = "lambda.amazonaws.com" + }, + Action = [ + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage", + "ecr:BatchCheckLayerAvailability" + ] + } + ] + }) +} \ No newline at end of file diff --git a/deployment/lambda_example/Dockerfile b/deployment/lambda_example/Dockerfile new file mode 100644 index 0000000..c1caf03 --- /dev/null +++ b/deployment/lambda_example/Dockerfile @@ -0,0 +1,8 @@ +# AWS Lambda python pacakge +FROM public.ecr.aws/lambda/python:3.11 + +# Copy function code +COPY deployment/lambda_example/app.py ./ + +# Set the CMD to your handler +CMD ["app.handler"] \ No newline at end of file diff --git a/deployment/lambda_example/app.py b/deployment/lambda_example/app.py new file mode 100644 index 0000000..05e2693 --- /dev/null +++ b/deployment/lambda_example/app.py @@ -0,0 +1,11 @@ +""" +A quick example of lambda working a function in python +""" + +def handler(event, context): + print("Hello from Python function. This shold be running from a dockerfile env and executed on a aws lambda!") + return { + 'statusCode': 200, + 'body': 'Hello World' + } + diff --git a/etl/epr_etl_example.py b/etl/epr_etl_example.py new file mode 100644 index 0000000..866c75a --- /dev/null +++ b/etl/epr_etl_example.py @@ -0,0 +1,9 @@ +from etl.surveyedData.surveryedData import surveyedDataProcessor + +files = [ + "/tmp/sharepoint/Sandwell/SANDWELL-001/26 Willow close B64 6EG/Content (13).pdf", + "/tmp/sharepoint/Livewest/Livewest-001/12 Birch End/Summary Information 12 Birch End.pdf" +] + +from sqlalchemy.dialects.postgresql import UUID +sdp = surveyedDataProcessor("fake address", files) diff --git a/etl/fileReader/pdfReaderToText.py b/etl/fileReader/pdfReaderToText.py index bc9643f..9668eed 100644 --- a/etl/fileReader/pdfReaderToText.py +++ b/etl/fileReader/pdfReaderToText.py @@ -1,8 +1,17 @@ from etl.utils.logger import Logger import logging import pymupdf -from etl.fileReader.sitenotes import QuidosSiteNotesExtractor, CSR, WarmHomesConditionReport, ECOConditionReport, RDSAPEnergyReport +from etl.fileReader.sitenotes import ( + QuidosSiteNotesExtractor, + CSR, + WarmHomesConditionReport, + ECOConditionReport, + EnergyPerformanceReportWithData, + EnergyPerformanceReportSummaryInformation + +) from etl.fileReader.reportType import ReportType +from pprint import pprint class pdfReaderToText(): @@ -24,7 +33,7 @@ class pdfReaderToText(): self.all_text += text self.text_list = self.all_text.split('\n') - print(self.text_list) + pprint(self.text_list) def get_list_of_text(self): return self.text_list @@ -45,8 +54,10 @@ class pdfReaderToText(): self.type = ReportType.WARM_HOMES_CONDITION_REPORT elif "Domna NEW PAS 2035 ECO Condition Report".lower() in self.text_list[0].lower(): self.type = ReportType.ECO_CONDITION_REPORT - elif "ENERGY REPORT".lower() == self.text_list[0].lower(): - self.type = ReportType.RDSAP_ENERGY_REPORT + elif "ENERGY REPORT".lower() == self.text_list[0].lower() and "Data inputs" in self.text_list: + self.type = ReportType.ENERGY_PERFORMANCE_REPORT_WITH_DATA + elif "Summary Information".lower() == self.text_list[0].lower(): + self.type = ReportType.ENERGY_PERFORMANCE_REPORT_SUMMARY_INFORMATION else: pass return self.type @@ -62,6 +73,7 @@ class pdfReaderToText(): return WarmHomesConditionReport(self.text_list) elif self.type == ReportType.ECO_CONDITION_REPORT: return ECOConditionReport(self.text_list) - elif self.type == ReportType.RDSAP_ENERGY_REPORT: - return RDSAPEnergyReport(self.text_list) - \ No newline at end of file + elif self.type == ReportType.ENERGY_PERFORMANCE_REPORT_WITH_DATA: + return EnergyPerformanceReportWithData(self.text_list) + elif self.type == ReportType.ENERGY_PERFORMANCE_REPORT_SUMMARY_INFORMATION: + return EnergyPerformanceReportSummaryInformation(self.text_list) \ No newline at end of file diff --git a/etl/fileReader/reportType.py b/etl/fileReader/reportType.py index 07ac12e..9fce283 100644 --- a/etl/fileReader/reportType.py +++ b/etl/fileReader/reportType.py @@ -4,12 +4,12 @@ from enum import Enum class ReportType(Enum): QUIDOS_PRESITE_NOTE = "quidos_presite_note" CHARTED_SURVEYOR_REPORT = "charted_surveyor_report" - ENERGY_PERFORMANCE_REPORT = "energy_performance_report" U_VALUE_CALCULATOR_REPORT = "u_value_calculator_report" OVERWRITING_U_VALUE_DECLARATION_FORM = "overwriting_u_value_declaration_form" ECO_CONDITION_REPORT = "osmosis_condition_pas_2035_report" WARM_HOMES_CONDITION_REPORT = "warm_homes_condition_pas_2035_report" - RDSAP_ENERGY_REPORT = "rdsap_energy_report" + ENERGY_PERFORMANCE_REPORT_WITH_DATA = "energy_performance_report_with_data" + ENERGY_PERFORMANCE_REPORT_SUMMARY_INFORMATION = "energy_performance_report_summary_information" LIG_XML = "lodgement_xml_needed_for_lodgement_to_like_trademark" RDSAP_XML = "reduce_xml_needed_to_generate_full_sap_xml" FULLSAP_XML = "full_xml_needed_for_co_ordination" diff --git a/etl/fileReader/sitenotes.py b/etl/fileReader/sitenotes.py index f46726c..1df8747 100644 --- a/etl/fileReader/sitenotes.py +++ b/etl/fileReader/sitenotes.py @@ -88,14 +88,7 @@ class CSR(SiteNotesExtractor): type=dict_.get('detailed_description_of_existing_cavity_wall_insulation_', "") ) if dict_ is not None else None -class RDSAPEnergyReport(SiteNotesExtractor): - def __init__(self, data_list): - super().__init__(data_list) - self.type = ReportType.RDSAP_ENERGY_REPORT - self.master_obj = self.setup_energy_report() - def setup_energy_report(self): - pass class ECOConditionReport(SiteNotesExtractor): def __init__(self, data_list): @@ -1597,4 +1590,20 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): main_gas_avalible=True if dict_.get("main_gas_available", "NO").upper() == "YES" else False, ) - \ No newline at end of file +class EnergyPerformanceReportWithData(SiteNotesExtractor): + def __init__(self, data_list): + super().__init__(data_list) + self.type = ReportType.ENERGY_PERFORMANCE_REPORT_WITH_DATA + self.master_obj = self.setup() + + def setup(self): + pass + +class EnergyPerformanceReportSummaryInformation(SiteNotesExtractor): + def __init__(self, data_list): + super().__init__(data_list) + self.type = ReportType.ENERGY_PERFORMANCE_REPORT_SUMMARY_INFORMATION + self.master_obj = self.setup() + + def setup(self): + pass \ No newline at end of file diff --git a/etl/hubSpotClient/types.py b/etl/hubSpotClient/types.py index dd97011..e004243 100644 --- a/etl/hubSpotClient/types.py +++ b/etl/hubSpotClient/types.py @@ -80,8 +80,8 @@ class SubmissionInfoFromDeal(BaseModel): if sdp.condition_report is None: missing_items.append("Condition Report") - if sdp.energy_report is None: - missing_items.append("Energy Report PDF") + if sdp.epr_summary_information is None: + missing_items.append("EPR Energy report with data is missing") if sdp.rd_sap_xml is None: missing_items.append("RDSAP XML") @@ -89,6 +89,9 @@ class SubmissionInfoFromDeal(BaseModel): if sdp.lig_sap_xml is None: missing_items.append("LIG SAP XML") + if sdp.epr_summary_information is None: + missing_items.append("EPR Summary information is missing") + if missing_items: raise ValueError(f"Missing required items: {', '.join(missing_items)}") diff --git a/etl/hubspot_surveyed_needs_sign_off.py b/etl/hubspot_surveyed_needs_sign_off.py index a8eeabc..7794d07 100644 --- a/etl/hubspot_surveyed_needs_sign_off.py +++ b/etl/hubspot_surveyed_needs_sign_off.py @@ -16,10 +16,17 @@ from etl.hubSpotClient.hubspot import DealStage, HubSpotClient os.environ["DATABASE_URL"] = "postgresql://postgres:makingwarmhomes@db:5432/postgres" hubspotClient = HubSpotClient() -deals = hubspotClient.get_deals_from_deal_stage(DealStage.SURVEYED_COMPLETE_NEEDS_SIGN_OFF) + +# files missing from assessor column +deals = hubspotClient.get_deals_from_deal_stage(DealStage.NEEDS_ADDITIONAL_INFORMATION_FROM_ASSESSOR) for deal in deals: hubspotClient.move_deals_to_different_stage([deal.deal_id], DealStage.SURVEYED_COMPLETED_SIGNED_OFF.value) -# TODO load when we are at 'ready to co-ordination' - script! \ No newline at end of file +# Survyed_complete +deals = hubspotClient.get_deals_from_deal_stage(DealStage.SURVEYED_COMPLETE_NEEDS_SIGN_OFF) + + +for deal in deals: + hubspotClient.move_deals_to_different_stage([deal.deal_id], DealStage.SURVEYED_COMPLETED_SIGNED_OFF.value) diff --git a/etl/surveyedData/surveryedData.py b/etl/surveyedData/surveryedData.py index 485d617..f75b112 100644 --- a/etl/surveyedData/surveryedData.py +++ b/etl/surveyedData/surveryedData.py @@ -41,7 +41,8 @@ class surveyedDataProcessor(): self.csr = None self.condition_report = None self.hubspot_deal_id = None - self.energy_report = None + self.epr_with_data = None + self.epr_summary_information = None self.full_sap_xml = None self.lig_sap_xml = None self.rd_sap_xml = None @@ -64,8 +65,11 @@ class surveyedDataProcessor(): self.condition_report = pdf.get_reader() elif pdf.type == ReportType.ECO_CONDITION_REPORT: self.condition_report = pdf.get_reader() - elif pdf.type == ReportType.RDSAP_ENERGY_REPORT: - self.energy_report = pdf.get_reader() + elif pdf.type == ReportType.ENERGY_PERFORMANCE_REPORT_WITH_DATA: + self.epr_with_data = pdf.get_reader() + elif pdf.type == ReportType.ENERGY_PERFORMANCE_REPORT_SUMMARY_INFORMATION: + self.epr_summary_information = pdf.get_reader() + elif file.lower().endswith('.xml'): xml = xmlReader(file) if xml: