diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 45e6e3c..7674c1d 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -2,13 +2,17 @@ FROM library/python:3.12-bullseye ARG USER=vscode ARG DEBIAN_FRONTEND=noninteractive -RUN apt update \ - && apt install -y --no-install-recommends sudo \ + +# Install system dependencies in a single layer +RUN apt update && apt install -y --no-install-recommends \ + sudo jq vim \ && apt autoremove -y \ - && rm -rf /var/lib/apt/lists/* \ - && useradd -m -s /usr/bin/bash ${USER} \ + && rm -rf /var/lib/apt/lists/* + +# Create the user and grant sudo privileges +RUN useradd -m -s /usr/bin/bash ${USER} \ && echo "${USER} ALL=(ALL) NOPASSWD: ALL" >/etc/sudoers.d/${USER} \ && chmod 0440 /etc/sudoers.d/${USER} # Install Poetry -RUN pip install poetry \ No newline at end of file +RUN pip install --no-cache-dir poetry diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index cbeb010..a99fc9c 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -3,12 +3,17 @@ "build": { "dockerfile": "Dockerfile" }, "remoteUser": "vscode", "postStartCommand": "bash .devcontainer/post-install.sh", + "mounts": [ + "source=${localEnv:HOME},target=/workspaces/home,type=bind" + ], "customizations": { "vscode": { "extensions": [ "ms-python.python", "ms-azuretools.vscode-docker", - "ms-toolsai.jupyter" + "ms-toolsai.jupyter", + "mechatroner.rainbow-csv", + "ms-toolsai.datawrangler" ] } } diff --git a/.gitignore b/.gitignore index 8b32ca3..16a5ff1 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,6 @@ __pycache__/ *.py[cod] data/ .env -*ipynb \ No newline at end of file +*ipynb +env.py +etl/survery_data.csv \ No newline at end of file diff --git a/etl/jjc_invoice.py b/etl/jjc_invoice.py new file mode 100644 index 0000000..2db0916 --- /dev/null +++ b/etl/jjc_invoice.py @@ -0,0 +1,570 @@ +from etl.scraper.scraper import SharePointScraper, SharePointInstaller, WEEK_COMMENCING +from pprint import pformat +from etl.pdfReader.pdfReaderToText import pdfReaderToText +from etl.surveyedData.surveryedData import surveyedDataProcessor +import pandas as pd +import math +from etl.temp.submissionReader import SubmissionReader +import os +from fuzzywuzzy import fuzz + +file_paths = [{'119 CUTNOOK LANE, M44 6LU': ['/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/FOR HOUSING/119 CUTNOOK LANE, M44 6LU/CSR - 119 Cutnook.pdf', + '/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/FOR HOUSING/119 CUTNOOK LANE, M44 6LU/EPR.pdf', + '/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/FOR HOUSING/119 CUTNOOK LANE, M44 6LU/INPUTS.pdf', + '/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/FOR HOUSING/119 CUTNOOK LANE, M44 6LU/UB.pdf', + '/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/FOR HOUSING/119 CUTNOOK LANE, M44 6LU/XML.xml']}, + {'22 LINNET DRIVE, M44 6LW': ['/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/FOR HOUSING/22 LINNET DRIVE, M44 6LW/CSR - 22 Linnet.pdf', + '/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/FOR HOUSING/22 LINNET DRIVE, M44 6LW/EPR.pdf', + '/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/FOR HOUSING/22 LINNET DRIVE, M44 6LW/INPUTS.pdf', + '/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/FOR HOUSING/22 LINNET DRIVE, M44 6LW/XML.xml']}, + {'50 PARKSTONE ROAD, M44 6LN': ['/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/FOR HOUSING/50 PARKSTONE ROAD, M44 6LN/CSR - 50 Parkstone.pdf', + '/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/FOR HOUSING/50 PARKSTONE ROAD, M44 6LN/EPR.pdf', + '/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/FOR HOUSING/50 PARKSTONE ROAD, M44 6LN/INPUTS.pdf', + '/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/FOR HOUSING/50 PARKSTONE ROAD, M44 6LN/XML.xml']}, + {'89 PARKSTONE ROAD, M44 6LJ': ['/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/FOR HOUSING/89 PARKSTONE ROAD, M44 6LJ/CSR - 89 Parkstone.pdf', + '/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/FOR HOUSING/89 PARKSTONE ROAD, M44 6LJ/EPR.pdf', + '/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/FOR HOUSING/89 PARKSTONE ROAD, M44 6LJ/INPUTS.pdf', + '/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/FOR HOUSING/89 PARKSTONE ROAD, M44 6LJ/XML.xml']}, + {'153 HEATHCOTE STREET, ST3 1AD': ['/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/UNITAS/153 HEATHCOTE STREET, ST3 1AD/CSR - 153 Heathcote.pdf', + '/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/UNITAS/153 HEATHCOTE STREET, ST3 1AD/EPR.pdf', + '/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/UNITAS/153 HEATHCOTE STREET, ST3 1AD/INPUTS.pdf', + '/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/UNITAS/153 HEATHCOTE STREET, ST3 1AD/XML.xml']}, + {'16 IONA PLACE, ST3 2DY': ['/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/UNITAS/16 IONA PLACE, ST3 2DY/EPR.pdf', + '/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/UNITAS/16 IONA PLACE, ST3 2DY/INPUTS.pdf', + '/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/UNITAS/16 IONA PLACE, ST3 2DY/XML.xml']}, + {'24 SEEDFIELDS ROAD, ST3 2BZ': ['/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/UNITAS/24 SEEDFIELDS ROAD, ST3 2BZ/CSR - 24 Seedfields.pdf', + '/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/UNITAS/24 SEEDFIELDS ROAD, ST3 2BZ/EPR.pdf', + '/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/UNITAS/24 SEEDFIELDS ROAD, ST3 2BZ/INPUTS.pdf', + '/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/UNITAS/24 SEEDFIELDS ROAD, ST3 2BZ/XML.xml']}, + {'27 ROUNDWAY, ST3 2BH': ['/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/UNITAS/27 ROUNDWAY, ST3 2BH/CSR - 27 Roundway.pdf', + '/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/UNITAS/27 ROUNDWAY, ST3 2BH/EPR.pdf', + '/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/UNITAS/27 ROUNDWAY, ST3 2BH/INPUTS.pdf', + '/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/UNITAS/27 ROUNDWAY, ST3 2BH/XML.xml']}, + {'30 ANGLESEY DRIVE, ST3 2SS': ['/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/UNITAS/30 ANGLESEY DRIVE, ST3 2SS/CSR - 30 Anglesey.pdf', + '/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/UNITAS/30 ANGLESEY DRIVE, ST3 2SS/EPR.pdf', + '/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/UNITAS/30 ANGLESEY DRIVE, ST3 2SS/INPUTS.pdf', + '/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/UNITAS/30 ANGLESEY DRIVE, ST3 2SS/XML.xml']}, + {'37 MEAFORD DRIVE, ST3 2BY': ['/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/UNITAS/37 MEAFORD DRIVE, ST3 2BY/CSR - 37 Meaford.pdf', + '/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/UNITAS/37 MEAFORD DRIVE, ST3 2BY/EPR.pdf', + '/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/UNITAS/37 MEAFORD DRIVE, ST3 2BY/INPUTS.pdf', + '/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/UNITAS/37 MEAFORD DRIVE, ST3 2BY/UB.pdf', + '/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/UNITAS/37 MEAFORD DRIVE, ST3 2BY/XML.xml']}, + {'50 LOUISE DRIVE, ST3 2DT': ['/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/UNITAS/50 LOUISE DRIVE, ST3 2DT/CSR - 50 Louise.pdf', + '/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/UNITAS/50 LOUISE DRIVE, ST3 2DT/EPR.pdf', + '/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/UNITAS/50 LOUISE DRIVE, ST3 2DT/INPUTS.pdf', + '/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/UNITAS/50 LOUISE DRIVE, ST3 2DT/UB.pdf', + '/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/UNITAS/50 LOUISE DRIVE, ST3 2DT/XML.xml']}, + {'90 MEAFORD DRIVE, ST3 2BB': ['/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/UNITAS/90 MEAFORD DRIVE, ST3 2BB/CSR - 90 Meaford.pdf', + '/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/UNITAS/90 MEAFORD DRIVE, ST3 2BB/EPR.pdf', + '/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/UNITAS/90 MEAFORD DRIVE, ST3 2BB/INPUTS.pdf', + '/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/UNITAS/90 MEAFORD DRIVE, ST3 2BB/XML.xml']}, + {'99 MEAFORD DRIVE, ST3 2BG': ['/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/UNITAS/99 MEAFORD DRIVE, ST3 2BG/CSR - 99 Meaford.pdf', + '/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/UNITAS/99 MEAFORD DRIVE, ST3 2BG/EPR.pdf', + '/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/UNITAS/99 MEAFORD DRIVE, ST3 2BG/INPUTS.pdf', + '/tmp/sharepoint/Andy Rumfitt/W.C. 10.03.2025/UNITAS/99 MEAFORD DRIVE, ST3 2BG/XML.xml']}, + {'1 Till Walk': ['/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/1 Till Walk/1 Till Walk PRE EPR PDF.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/1 Till Walk/1 Till Walk PRE EPR RDSAP XML.xml', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/1 Till Walk/1 Till Walk PRE EPR SITE NOTES.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/1 Till Walk/1 Till Walk ST3 5DF.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/1 Till Walk/Dec - 1 Till Walk ST3 5DF.pdf']}, + {'10 Lomond walk': ['/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/10 Lomond walk/10 Lomond Walk PRE EPR PDF.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/10 Lomond walk/10 Lomond Walk PRE EPR RDSAP XML.xml', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/10 Lomond walk/10 Lomond Walk PRE EPR SITE NOTES.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/10 Lomond walk/10 Lomond Walk ST3 3HW.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/10 Lomond walk/Dec - 10 Lomond Walk ST3 3HW.pdf']}, + {'10 Pembridge road': ['/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/10 Pembridge road/10 Pembridge Road PRE EPR PDF.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/10 Pembridge road/10 Pembridge Road PRE EPR RDSAP XML.xml', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/10 Pembridge road/10 Pembridge Road PRE EPR SITE NOTES.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/10 Pembridge road/10 Pembridge Road ST3 3BX.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/10 Pembridge road/Dec - 10 Pembridge Road ST3 3BX.pdf']}, + {'10A Pembridge road': ['/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/10A Pembridge road/10A Pembridge Road PRE EPR PDF.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/10A Pembridge road/10A Pembridge Road PRE EPR RDSAP XML.xml', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/10A Pembridge road/10A Pembridge Road PRE EPR SITE NOTES.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/10A Pembridge road/10A Pembridge Road ST3 3BX.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/10A Pembridge road/Dec - 10A Pembridge Road ST3 3BX.pdf']}, + {'12 Lomond walk': ['/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/12 Lomond walk/12 Lomond Walk PRE EPR RDSAP XML.xml', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/12 Lomond walk/12 Lomond Walk PRE ERP PDF.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/12 Lomond walk/12 Lomond Walk PRE ERP SITE NOTES.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/12 Lomond walk/12 Lomond Walk ST3 3HW.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/12 Lomond walk/Dec - 12 Lomond Walk ST3 3HW.pdf']}, + {'12A Pembridge road': ['/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/12A Pembridge road/12A Pembridge Road PRE EPR PDF.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/12A Pembridge road/12A Pembridge Road PRE EPR RDSAP XML.xml', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/12A Pembridge road/12A Pembridge Road PRE EPR SITE NOTES.pdf']}, + {'15 Lomond walk': ['/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/15 Lomond walk/15 Lomond Walk PRE EPR PDF.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/15 Lomond walk/15 Lomond Walk PRE EPR RDSAP XML.xml', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/15 Lomond walk/15 Lomond Walk PRE EPR SITE NOTES.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/15 Lomond walk/15 Lomond Walk ST3 3HW.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/15 Lomond walk/Dec - 15 Lomond Walk ST3 3HW.pdf']}, + {'18 Bilston street': ['/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/18 Bilston street/18 Bilston Street PRE EPR PDF.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/18 Bilston street/18 Bilston Street PRE EPR RDSAP XML.xml', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/18 Bilston street/18 Bilston Street PRE EPR SITE NOTES.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/18 Bilston street/18 Bilston Street ST4 5DA.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/18 Bilston street/Dec - 18 Bilston Street ST4 5DA.pdf']}, + {'18 Lomond walk': ['/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/18 Lomond walk/18 Lomond Walk PRE EPR PDF.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/18 Lomond walk/18 Lomond Walk PRE EPR RDSAP XML.xml', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/18 Lomond walk/18 Lomond Walk PRE EPR SITE NOTES.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/18 Lomond walk/18 Lomond Walk ST3 3HW.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/18 Lomond walk/Dec - 18 Lomond Walk ST3 3HW.pdf']}, + {'19 sedgley walk': ['/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/19 sedgley walk/19 Sedgley Walk PRE EPR PDF.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/19 sedgley walk/19 Sedgley Walk PRE EPR RDSAP XML.xml', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/19 sedgley walk/19 Sedgley Walk PRE EPR SITE NOTES.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/19 sedgley walk/19 Sedgley Walk ST3 1RY.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/19 sedgley walk/Dec - 19 Sedgley Walk ST3 1RY.pdf']}, + {'2 Gregson close': ['/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/2 Gregson close/2 Gregson close PRE EPR PDF.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/2 Gregson close/2 Gregson close PRE EPR RDSAP XML.xml', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/2 Gregson close/2 Gregson close PRE EPR SITE NOTES.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/2 Gregson close/2 Gregson close ST3 2SJ.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/2 Gregson close/Dec - 2 Gregson close ST3 2SJ.pdf']}, + {'2 Lomond walk': ['/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/2 Lomond walk/2 Lomond Walk PRE EPR PDF.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/2 Lomond walk/2 Lomond Walk PRE EPR RDSAP XML.xml', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/2 Lomond walk/2 Lomond Walk PRE EPR SITE NOTES.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/2 Lomond walk/2 Lomond Walk ST3 3HW.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/2 Lomond walk/Dec - 2 Lomond Walk ST3 3HW.pdf']}, + {'20 Lomond walk': ['/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/20 Lomond walk/20 Lomond Walk PRE ERR PDF.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/20 Lomond walk/20 Lomond Walk PRE ERR RDSAP XML.xml', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/20 Lomond walk/20 Lomond Walk PRE ERR SITE NOTES.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/20 Lomond walk/20 Lomond Walk ST3 3HW.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/20 Lomond walk/Dec - 20 Lomond Walk ST3 3HW.pdf']}, + {'3 Lomond walk': ['/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/3 Lomond walk/3 Lomond Walk PRE EPR PDF.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/3 Lomond walk/3 Lomond Walk PRE EPR RDSAP XML.xml', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/3 Lomond walk/3 Lomond Walk PRE EPR SITE NOTES.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/3 Lomond walk/3 Lomond Walk ST3 3HW.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/3 Lomond walk/Dec - 3 Lomond Walk ST3 3HW.pdf']}, + {'32 Anglesey drive': ['/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/32 Anglesey drive/32 Anglesey Drive PRER EPR PDF.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/32 Anglesey drive/32 Anglesey Drive PRER EPR RDSAP XML.xml', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/32 Anglesey drive/32 Anglesey Drive PRER EPR SITE NOTES.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/32 Anglesey drive/32 Anglesey Drive ST3 2SS.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/32 Anglesey drive/Dec - 32 Anglesey Drive ST3 2SS.pdf']}, + {'36 Woodville road': ['/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/36 Woodville road/36 Woodville Road PRE EPR PDF.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/36 Woodville road/36 Woodville Road PRE EPR RDSAP XML.xml', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/36 Woodville road/36 Woodville Road PRE EPR SITE NOTES.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/36 Woodville road/36 Woodville Road ST3 6BX.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/36 Woodville road/Dec - 36 Woodville Road ST3 6BX.pdf']}, + {'379 lightwood road': ['/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/379 lightwood road/379 Lightwood Road PRE EPR PDF.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/379 lightwood road/379 Lightwood Road PRE EPR RDSAP XML.xml', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/379 lightwood road/379 Lightwood Road PRE EPR SITE NOTES.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/379 lightwood road/379 Lightwood Road ST3 4JT.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/379 lightwood road/Dec - 379 Lightwood Road ST3 4JT.pdf']}, + {'387 Lightwood road': ['/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/387 Lightwood road/387 Lightwood Road PRE EPR PDF.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/387 Lightwood road/387 Lightwood Road PRE EPR RDSAP XML.xml', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/387 Lightwood road/387 Lightwood Road PRE EPR SITE NOTES.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/387 Lightwood road/387 Lightwood Road ST3 4JT.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/387 Lightwood road/Dec - 387 Lightwood Road ST3 4JT.pdf']}, + {'4 Pembridge road': ['/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/4 Pembridge road/4 Pembridge Road PRE EPR PDF.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/4 Pembridge road/4 Pembridge Road PRE EPR RDSAP XML.xml', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/4 Pembridge road/4 Pembridge Road PRE EPR SITE NOTES.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/4 Pembridge road/4 Pembridge Road ST3 3BX.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/4 Pembridge road/Dec - 4 Pembridge Road ST3 3BX.pdf']}, + {'49 bridle path': ['/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/49 bridle path/49 Bridle Path PRE EPR PDF.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/49 bridle path/49 Bridle Path PRE EPR RDSAP XML.xml', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/49 bridle path/49 Bridle Path PRE EPR SITE NOTES.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/49 bridle path/49 Bridle Path ST3 4SB.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/49 bridle path/Dec - 49 Bridle Path ST3 4SB.pdf']}, + {'5 Lomond walk': ['/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/5 Lomond walk/5 Lomond Walk PRE EPR PDF.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/5 Lomond walk/5 Lomond Walk PRE EPR RDSASP XML.xml', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/5 Lomond walk/5 Lomond Walk PRE EPR SITE NOTES.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/5 Lomond walk/5 Lomond Walk ST3 3HW.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/5 Lomond walk/Dec - 5 Lomond Walk ST3 3HW.pdf']}, + {'5 Poolside': ['/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/5 Poolside/5 Poolside PRE EPR PDF.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/5 Poolside/5 Poolside PRE EPR RDSAP XML.xml', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/5 Poolside/5 Poolside PRE EPR SITE NOTES.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/5 Poolside/5 Poolside ST3 3NY.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/5 Poolside/Dec - 5 Poolside ST3 3NY.pdf']}, + {'52 Bridle path': ['/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/52 Bridle path/52 Bridle Path PRE EPR PDF.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/52 Bridle path/52 Bridle Path PRE EPR RDSAP XML.xml', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/52 Bridle path/52 Bridle Path PRE EPR SITE NOTES.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/52 Bridle path/52 Bridle Path ST3 4SB.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/52 Bridle path/Dec - 52 Bridle Path ST3 4SB.pdf']}, + {'58 Wise street': ['/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/58 Wise street/58 Wise Street PRE EPR PDF.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/58 Wise street/58 Wise Street PRE EPR RDSAP XML.xml', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/58 Wise street/58 Wise Street PRE EPR SITE NOTES.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/58 Wise street/58 Wise Street ST3 4PQ.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/58 Wise street/Dec - 58 Wise Street ST3 4PQ.pdf']}, + {'7 ingleby road': ['/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/7 ingleby road/7 Ingleby Road PRE ERP PDF.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/7 ingleby road/7 Ingleby Road PRE ERP RDSAP XML.xml', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/7 ingleby road/7 Ingleby Road PRE ERP SITE NOTES.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/7 ingleby road/7 Ingleby Road ST3 3JY.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/7 ingleby road/Dec - 7 Ingleby Road ST3 3JY.pdf']}, + {'7 Lomond walk': ['/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/7 Lomond walk/7 Lomond Walk PRE EPR PDF.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/7 Lomond walk/7 Lomond Walk PRE EPR RDSAP XML.xml', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/7 Lomond walk/7 Lomond Walk PRE EPR SITE NOTES.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/7 Lomond walk/7 Lomond Walk ST3 3HW.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/7 Lomond walk/Dec - 7 Lomond Walk ST3 3HW.pdf']}, + {'7 Peel Street': ['/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/7 Peel Street/7 Peel Street PRE EPR PDF.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/7 Peel Street/7 Peel Street PRE EPR RDSAP XML.xml', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/7 Peel Street/7 Peel Street PRE EPR SITE NOTES.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/7 Peel Street/7 Peel Street ST3 4PE.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/7 Peel Street/Dec - 7 Peel Street ST3 4PE.pdf']}, + {'83 Wimbourne avenue': []}, + {'85 Wimbourne avenue': []}, + {'87 Wimbourne avenue': []}, + {'9 Farrington close': []}, + {'9 Lomond walk': ['/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/9 Lomond walk/9 Lomond Walk PRE EPR RDSAP XML.xml', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/9 Lomond walk/9 Lomond Walk PRE EPR SITE NOTES.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/9 Lomond walk/9 Lomond Walk ST3 3HW.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/9 Lomond walk/Dec - 9 Lomond Walk PRE EPR PDF.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/9 Lomond walk/Dec - 9 Lomond Walk ST3 3HW.pdf']}, + {'9 Melbourne street': ['/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/9 Melbourne street/9 Melbourne Street PRE EPR PDF.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/9 Melbourne street/9 Melbourne Street PRE EPR RDSAP XML.xml', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/9 Melbourne street/9 Melbourne Street PRE EPR SITE NOTES.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/9 Melbourne street/9 Melbourne Street ST3 5AA.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/9 Melbourne street/Dec - 9 Melbourne Street ST3 5AA.pdf']}, + {'91 Wimbourne avenue': ['/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/91 Wimbourne avenue/91 Wimbourne Avenue PRE EPR PDF.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/91 Wimbourne avenue/91 Wimbourne Avenue PRE EPR RDSAP XML.xml', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/91 Wimbourne avenue/91 Wimbourne Avenue PRE EPR SITE NOTES.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/91 Wimbourne avenue/91 Wimbourne Avenue ST3 3LS.pdf', + '/tmp/sharepoint/Ben Langslow/W.C. 10.03.2025/UNITAS/91 Wimbourne avenue/Dec - 91 Wimbourne Avenue ST3 3LS.pdf']}, + {'101 Edgefield Road ST3 1AF': []}, + {'103 Edgefield Road ST3 1AF': []}, + {'105 Edgefield Road ST3 1AF': []}, + {'111 Edgefield Road ST3 1AF': []}, + {'113 Edgefield Road ST3 1AF': []}, + {'114 Foley Road ST3 2LH': ['/tmp/sharepoint/Mev Malik/W.C. 10.03.2025/UNITAS/114 Foley Road ST3 2LH/5209-8505-3432-0097-5106 epr.pdf', + '/tmp/sharepoint/Mev Malik/W.C. 10.03.2025/UNITAS/114 Foley Road ST3 2LH/5209-8505-3432-0097-5106.pdf', + '/tmp/sharepoint/Mev Malik/W.C. 10.03.2025/UNITAS/114 Foley Road ST3 2LH/5209-8505-3432-0097-5106.xml']}, + {'114 Longley Road ST3 1AN': []}, + {'15 Sedgley Walk ST3 1RY': ['/tmp/sharepoint/Mev Malik/W.C. 10.03.2025/UNITAS/15 Sedgley Walk ST3 1RY/0976-0245-5835-8209-5804 epr.pdf', + '/tmp/sharepoint/Mev Malik/W.C. 10.03.2025/UNITAS/15 Sedgley Walk ST3 1RY/0976-0245-5835-8209-5804.pdf', + '/tmp/sharepoint/Mev Malik/W.C. 10.03.2025/UNITAS/15 Sedgley Walk ST3 1RY/0976-0245-5835-8209-5804.xml']}, + {'151 Edgefield Road ST3 1AF': []}, + {'155 Edgefield Road ST3 1AF': []}, + {'17 Conewood Place ST3 2NW': []}, + {'17 Sedgley Walk ST3 1RY': ['/tmp/sharepoint/Mev Malik/W.C. 10.03.2025/UNITAS/17 Sedgley Walk ST3 1RY/2080-2607-5453-9500-2595 epr.pdf', + '/tmp/sharepoint/Mev Malik/W.C. 10.03.2025/UNITAS/17 Sedgley Walk ST3 1RY/2080-2607-5453-9500-2595.pdf', + '/tmp/sharepoint/Mev Malik/W.C. 10.03.2025/UNITAS/17 Sedgley Walk ST3 1RY/2080-2607-5453-9500-2595.xml']}, + {'21 Conewood Place ST3 2NW': []}, + {'27 Conewood Place ST3 2NW': []}, + {'30 Caroline Street ST3 1DE': []}, + {'32 Caroline Street ST3 1DE': []}, + {'34 Caroline Street ST3 1DE': []}, + {'35 Heathcote Street ST3 1AA': []}, + {'4 Milverton Place ST3 2SH': []}, + {'47 Hurst Road ST3 2LT': []}, + {'48 Lundy Road ST3 2DZ': []}, + {'5 Conewood Place ST3 2NW': []}, + {'54 Kirkup Walk ST3 2RR': []}, + {'56 Heathcote Road ST3 2LX': []}, + {'59 Caroline Street ST3 1DE': []}, + {'96 Edgefield Road ST3 1AF': []}, + {'96 Longley Road ST3 1AN': []}, + {'97 Edgefield Road ST3 1AF': ['/tmp/sharepoint/Mev Malik/W.C. 10.03.2025/UNITAS/97 Edgefield Road ST3 1AF/CS reports 16032025.pdf']}, + {'112 Chiltern Drive WD3 8JZ': []}, + {'13 Hubbards Road WD3 5JL': []}, + {'13 Orchard Way WD3 8HZ': []}, + {'14 Orchard Way WD3 8HZ': []}, + {'15 Hubbards Road WD3 5JL': []}, + {'19 Orchard Way WD3 8HZ': []}, + {'20 Edinburgh Avenue WD3 8LB': []}, + {'21 Clitheroe Gardens WD19 6RP': []}, + {'26 Clitheroe Gardens WD19 6RP': []}, + {'29 The Queens Drive WD3 8LN': []}, + {'31 The Queens Drive WD3 8LN': []}, + {'34 Hubbards Road WD3 5JJ': []}, + {'36 Orchard Way WD3 8HZ': []}, + {'42 The Queens Drive WD3 8LT': []}, + {'45 Orchard Way WD3 8HZ': []}, + {'5 Hubbards Road WD3 5JJ': []}, + {'56 The Queens Drive WD3 8LT': []}, + {'7 Hubbards Road WD3 5JL': []}, + {'8 The Greenway WD3 8HU': []}, + {'80 The Queens Drive WD3 8LX': []}, + {'82 The Queens Drive WD3 8LX': []}, + {'9 Orchard Way WD3 8HZ': []}, + {'90 Chiltern Drive WD3 8JZ': []}, + {'96 Chiltern Drive WD3 8JZ': []}] + +def get_band(score): + bands = [ + ("HIGH A", 96, float("inf")), + ("LOW A", 92, 96), + ("HIGH B", 86, 92), + ("LOW B", 81, 86), + ("HIGH C", 74.5, 81), + ("LOW C", 69, 74.5), + ("HIGH D", 61.5, 69), + ("LOW D", 55, 61.5), + ("HIGH E", 46.5, 55), + ("LOW E", 39, 46.5), + ("HIGH F", 29.5, 39), + ("LOW F", 21, 29.5), + ("HIGH G", 10.5, 21), + ("LOW G", 1, 10.5), + ] + + for band, lower, upper in bands: + if lower <= score < upper: + return band + + return "Out of range" + +def work_out_total_floor_area(pre_site_note): + + total = 0 + def add_all_floors(floor_list): + total = 0 + for floor in floor_list: + total += floor.floor_area_m2 + + return total + + main = True if pre_site_note.property_description.no_of_main_property > 0 else False + ext1 = True if pre_site_note.property_description.no_of_extension_1 > 0 else False + ext2 = True if pre_site_note.property_description.no_of_extension_2 > 0 else False + ext3 = True if pre_site_note.property_description.no_of_extension_3 > 0 else False + ext4 = True if pre_site_note.property_description.no_of_extension_4 > 0 else False + + total += add_all_floors(pre_site_note.property_description.main_property.dimensions) if main is True else 0 + total += add_all_floors(pre_site_note.property_description.ex1_property.dimensions) if ext1 is True else 0 + total += add_all_floors(pre_site_note.property_description.ex2_property.dimensions) if ext2 is True else 0 + total += add_all_floors(pre_site_note.property_description.ex3_property.dimensions) if ext3 is True else 0 + total += add_all_floors(pre_site_note.property_description.ex4_proprerty.dimensions) if ext4 is True else 0 + + + floor_area = math.ceil(total) if total%1 >=0.5 else math.floor(total) + if 0 <= floor_area <= 72: + return '0-72m', floor_area + elif 73 <= floor_area <= 97: + return '73-97m', floor_area + elif 98 <= floor_area <= 199: + return '98-199m', floor_area + elif floor_area >= 200: + return 'over 200m', floor_area + else: + return 'Invalid floor area', floor_area + +def fuzzy_match_address(row, target_address, threshold=90): + return fuzz.ratio(row, target_address) >= threshold + +def rounding(x): + x = float(x) + return math.ceil(x) if x%1 >=0.5 else math.floor(x) + +def type_of_work(presap, postsap): + """ + *ECO4 (minimum movement)* + D to C + E to C + F to D + G to D + + G -> ABCD + F -> ABCD + E -> ABC + D -> ABC + + *GBIS - Cavity Wall Insulation ONLY* + + D-D + E-D + E-E + F-E + F-F + G-E + G-F + G-G + """ + eco4 = { + "G": ['A', 'B', 'C', 'D'], + "F": ['A', 'B', 'C', 'D'], + "E": ['A', 'B', 'C'], + "D": ['A', 'B', 'C'], + } + + gbis ={ + 'D': ['D'], + 'E': ['E', 'D'], + 'F': ['E', 'F'], + 'G': ['E', 'F', 'G'], + } + + if presap.upper() in eco4: + if postsap.upper() in eco4[presap.upper()]: + return "ECO4" + if presap.upper() in gbis: + if postsap.upper() in gbis[presap.upper()]: + return "GBIS" + + return None + +def get_trickle_vent(str): + if "vent" in str.lower(): + return 1 + else: + return 0 + +def get_sap_letter(str): + for char in str: + if char.isalpha(): + return char.upper() + +def get_sap_number(str): + num = "" + for char in str: + if char.isnumeric(): + num+= char + + return int(num) + +def get_no_of_wet_rooms(str): + words = ["WC", "BATHROOM", "KITCHEN"] + count = 0 + for word in words: + if word in str.upper(): + count += 1 + return count + +def get_insulation_type(str): + if "foam" in str.lower(): + return "FOAM" + else: + return "GENERAL" + + +def get_jjc_price_matrix(file="empty.csv"): + df = pd.read_csv(os.path.join(os.getcwd(), "ratecard", "jjcRateCards", file)) + + columns_to_check = { + "no extractors or ventilation required": {"Trickle Vent": 0, "wetrooms": 0}, + "Trickle Vents ONLY": {"Trickle Vent": 1, "wetrooms": 0}, + "1 wet room extractor required": {"Trickle Vent": 0, "wetrooms": 1}, + "2 wet room extractor required": {"Trickle Vent": 0, "wetrooms": 2}, + "3 wet room extractor required": {"Trickle Vent": 0, "wetrooms": 3}, + 'Trickle Vents + 1 wet room extractor': {"Trickle Vent": 1, "wetrooms": 1}, + 'Trickle Vents + 2 wet room extractor': {"Trickle Vent": 1, "wetrooms": 2}, + 'Trickle Vents + 3 wet room extractor': {"Trickle Vent": 1, "wetrooms": 3}, + } + + pricing_table = [] + for _, row in df.iterrows(): + for key, variables in columns_to_check.items(): + pricing_table.append( + { + "funding": row["Funding"], + "floor_area_group": row["Total Floor Area"][:-1], + **variables, + "price": row[key][1:] if row[key] != "Not viable" else None, + } + ) + pricing_table = pd.DataFrame(pricing_table) + return pricing_table + +def main(): + + price_empty = get_jjc_price_matrix() + price_foam = get_jjc_price_matrix("foam.csv") + price_general = get_jjc_price_matrix("general.csv") + + csv_file_path = os.path.join(os.getcwd(), "temp", "jjc_submissionData.csv") + submissionData = SubmissionReader(csv_file_path) + + total_price = [] + + + # jjc = SharePointScraper(SharePointInstaller.JJC) + # file_paths = jjc.download_file_for_each_address() + list_of_surveys = [] + for eachAddress in file_paths: + for address, files in eachAddress.items(): + list_of_surveys.append(surveyedDataProcessor(address, files)) + for survey in list_of_surveys: + if survey.pre_site_note: + floor_banding, total_floor_area = work_out_total_floor_area(survey.pre_site_note) + letter, number = survey.pre_site_note.survey_information.current_sap.split(" ") + pre_sap_score = number+letter + filtered_df = submissionData.df[submissionData.df["PRE INSTALL SAP SCORE"].apply(lambda x: x == pre_sap_score)] + if not filtered_df.empty: + filtered_df = filtered_df[filtered_df["FLOOR AREA"].apply(lambda x: rounding(x) == total_floor_area)] + if not filtered_df.empty: + postcode = survey.pre_site_note.survey_information.address.split(",")[-1].strip().replace(" ", "") + filtered_df = filtered_df[filtered_df["Post Code"].apply(lambda x: x.replace(" ", "").upper() == postcode.upper())] + if not filtered_df.empty: + house_no = survey.pre_site_note.survey_information.address.split(" ")[0].strip() + filtered_df = filtered_df[filtered_df["NO"].apply(lambda x: house_no in x)] + if len(filtered_df) > 1: + def get_num(x): + num = "" + for char in x: + if char.isnumeric(): + num += char + num = int(num) + return num + + filtered_df = filtered_df[filtered_df["NO"].apply(lambda x: get_num(house_no) == get_num(x))] + if not filtered_df.empty: + funding_type = type_of_work(letter.upper(), get_sap_letter(filtered_df["POST INSTALL SAP SCORE"].values[0])) + + data = { + "Address": survey.address, + "Surveyor's Name": survey.pre_site_note.assessor_information.name, + "floor_area_group" : floor_banding, + "wetrooms" : get_no_of_wet_rooms(filtered_df["Ventilation Requirements"].values[0]), + "Trickle Vent" : int(get_trickle_vent(filtered_df["Ventilation Requirements"].values[0])), + } + + insultation = None + merged_df = pd.DataFrame() + if survey.csr: + if survey.csr.insulation_info: + insultation = get_insulation_type(survey.csr.insulation_info.type) + + if funding_type == "GBIS": + data.update({"insulation": insultation}) + + if insultation is None: + data.update({"funding": funding_type.upper()}) + df = pd.DataFrame([data]) + merged_df = pd.merge(df, price_empty, on=['funding', 'Trickle Vent', 'floor_area_group', 'wetrooms'], how='left') + elif "GENERAL" in insultation.upper(): + data.update({"funding": funding_type.upper() + " Remedial"}) + df = pd.DataFrame([data]) + merged_df = pd.merge(df, price_general, on=['funding', 'Trickle Vent', 'floor_area_group', 'wetrooms'], how='left') + elif "FOAM" in insultation.upper(): + data.update({"funding": funding_type.upper() + " Remedial"}) + df = pd.DataFrame([data]) + merged_df = pd.merge(df, price_foam, on=['funding', 'Trickle Vent', 'floor_area_group', 'wetrooms'], how='left') + else: + raise RuntimeError(f"unknonw insulation in gbis: {insultation}") + elif funding_type == "ECO4": + data.update({"insulation": insultation}) + if insultation is None: + formatted_funding_type = f"{funding_type.upper()} - SAP {get_band(int(number))} to {get_band(get_sap_number(filtered_df["POST INSTALL SAP SCORE"].values[0]))}" + data.update({"funding": formatted_funding_type}) + df = pd.DataFrame([data]) + merged_df = pd.merge(df, price_empty, on=['funding', 'Trickle Vent', 'floor_area_group', 'wetrooms'], how='left') + elif "GENERAL" in insultation.upper(): + formatted_funding_type = f"REMEDIAL - {funding_type.upper()} - SAP {get_band(int(number))} to {get_band(get_sap_number(filtered_df["POST INSTALL SAP SCORE"].values[0]))}" + data.update({"funding": formatted_funding_type}) + df = pd.DataFrame([data]) + merged_df = pd.merge(df, price_general, on=['funding', 'Trickle Vent', 'floor_area_group', 'wetrooms'], how='left') + elif "FOAM" in insultation.upper(): + formatted_funding_type = f"REMEDIAL - {funding_type.upper()} - SAP {get_band(int(number))} to {get_band(get_sap_number(filtered_df["POST INSTALL SAP SCORE"].values[0]))}" + data.update({"funding": formatted_funding_type}) + df = pd.DataFrame([data]) + merged_df = pd.merge(df, price_foam, on=['funding', 'Trickle Vent', 'floor_area_group', 'wetrooms'], how='left') + else: + raise RuntimeError(f"unknonw insultation in eco4: {insultation}") + else: + raise RuntimeError(f"UNKNOWN FUNDING TYPE {funding_type}") + + if not merged_df.empty: + total_price.append(merged_df) + + final_df = pd.concat(total_price, ignore_index=True) + + + + # Save to an Excel file + final_df.to_excel("survey_data.xlsx", index=False) + final_df.to_csv("survery_data.csv", index=False) + + print(f"WEEK COMMENCING {WEEK_COMMENCING}") + print("Excel file 'survey_data.xlsx' created successfully!") + print(pformat(data)) + + +if __name__ == "__main__": + main() + + +# left join on the price after making a truth table of every variation and the price +# Make sure i catch one that don't match so i have list of address that didn't match in share point \ No newline at end of file diff --git a/etl/pdfReader/pdfReaderToText.py b/etl/pdfReader/pdfReaderToText.py index 9632296..5ea7b42 100644 --- a/etl/pdfReader/pdfReaderToText.py +++ b/etl/pdfReader/pdfReaderToText.py @@ -50,5 +50,5 @@ class pdfReaderToText(): if self.type.name == ReportType.QUIDOS_PRESITE_NOTE.name: return QuidosSiteNotesExtractor(self.text_list) elif self.type == ReportType.CHARTED_SURVEYOR_REPORT: - return CSR(self.text_list) + return CSR(self.text_list) \ No newline at end of file diff --git a/etl/pdfReader/sitenotes.py b/etl/pdfReader/sitenotes.py index 4dbcdad..3ed2f1a 100644 --- a/etl/pdfReader/sitenotes.py +++ b/etl/pdfReader/sitenotes.py @@ -1,25 +1,40 @@ from etl.pdfReader.reportType import ReportType -from transform.types import ( +from etl.transform.types import ( CompanyInfo, SurverySummaryInfo, AssessorInfo, PropertyDescription, PropertyDetail, Dimension, Walls, Roofs, Floors, Door, VentilationAndCooling, Lighting, WaterHeating, HotWaterCylinder, SolarWaterHeating, ShowerAndBaths, FlueGasHeatRecoverySystem, PhotovoltaicPanel, WindTurbine, OtherDetails, Windows, Heating, HeatingSystemControls, - HeatingType + HeatingType, Insulation ) from datetime import datetime class SiteNotesExtractor(): def __init__(self, data_list): self.raw_data = data_list - def get_x_occurance(self, lst, value, x=1): try: return [i for i, v in enumerate(lst) if v == value][x] except IndexError: return None # Return None if the value does not occur twice - + + def two_columns_processor(self, data, sub_titles_to_gather, avoid, indexAdd = 1): + def get_value(key): + try: + index = data.index(key) + value = data[index + indexAdd] + return None if value in avoid else value + except (ValueError, IndexError): + return None + dict_ = {} + for items in data: + if items in avoid: + continue + elif items in sub_titles_to_gather: + dict_.update({f"{items.lower().replace('-', '_').replace(' ','_')}":get_value(items)}) + return dict_ + def get_data_between(self, a, b): return self.raw_data[self.raw_data.index(a):self.raw_data.index(b)] @@ -27,6 +42,22 @@ class CSR(SiteNotesExtractor): def __init__(self, data_list): super().__init__(data_list) self.type = ReportType.CHARTED_SURVEYOR_REPORT + self.insulation_info = None + self.setup() + + def setup(self): + self.get_materials() + + def get_materials(self): + lst = self.get_data_between("Detailed description of existing Cavity Wall Insulation ", "Detailed description of Defects in existing Cavity Wall Insulation") + dict_ = self.two_columns_processor(lst, ["Detailed description of existing Cavity Wall Insulation "], ["Detailed description of Defects in existing Cavity Wall Insulation"]) + + self.insulation_info = Insulation( + type=dict_.get('detailed_description_of_existing_cavity_wall_insulation_', "") + ) if dict_ is not None else None + + + @@ -563,21 +594,6 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): u_value_w_m2_k=dict_.get("u_value_(w/m²k)", "") if dict_.get("u_value_(w/m²k), '')") is not None else "", ) - def two_columns_processor(self, data, sub_titles_to_gather, avoid, indexAdd = 1): - def get_value(key): - try: - index = data.index(key) - value = data[index + indexAdd] - return None if value in avoid else value - except (ValueError, IndexError): - return None - dict_ = {} - for items in data: - if items in avoid: - continue - elif items in sub_titles_to_gather: - dict_.update({f"{items.lower().replace('-', '_').replace(' ','_')}":get_value(items)}) - return dict_ def get_windows(self): data = self.get_data_between("11.0 Windows", "12.0 Ventilation & Cooling") diff --git a/etl/ratecard/jjcRateCards/empty.csv b/etl/ratecard/jjcRateCards/empty.csv new file mode 100644 index 0000000..963a63e --- /dev/null +++ b/etl/ratecard/jjcRateCards/empty.csv @@ -0,0 +1,21 @@ +Funding,Total Floor Area,ABS,no extractors or ventilation required,Trickle Vents ONLY,1 wet room extractor required,2 wet room extractor required,3 wet room extractor required,Trickle Vents + 1 wet room extractor,Trickle Vents + 2 wet room extractor,Trickle Vents + 3 wet room extractor +ECO4 - SAP HIGH D to LOW C,0-72m�,155,�750,�750,�500,�400,�300,�500,�400,�300 +ECO4 - SAP HIGH D to LOW C,73-97m�,169,�800,�800,�600,�500,�400,�600,�500,�400 +ECO4 - SAP HIGH D to LOW C,98-199m�,196.4,�825,�825,�700,�600,�500,�700,�600,�500 +ECO4 - SAP HIGH D to LOW C,over 200m�,350.1,"�1,500","�1,500","�1,500","�1,250","�1,000","�1,500","�1,250","�1,000" +ECO4 - SAP HIGH D to HIGH C,0-72m�,276.9,"�1,500","�1,500","�1,200","�1,000",�800,"�1,200","�1,000",�800 +ECO4 - SAP HIGH D to HIGH C,73-97m�,301.8,"�1,500","�1,500","�1,250","�1,000","�1,000","�1,250","�1,000","�1,000" +ECO4 - SAP HIGH D to HIGH C,98-199m�,350.8,"�1,500","�1,500","�1,500","�1,250","�1,000","�1,500","�1,250","�1,000" +ECO4 - SAP HIGH D to HIGH C,over 200m�,625.2,"�1,500","�1,500","�1,500","�1,500","�1,500","�1,500","�1,500","�1,500" +ECO4 - SAP LOW D to LOW C,0-72m�,299,"�1,500","�1,500","�1,500","�1,500","�1,500","�1,500","�1,500","�1,500" +ECO4 - SAP LOW D to LOW C,73-97m�,325.9,"�1,500","�1,500","�1,500","�1,500","�1,500","�1,500","�1,500","�1,500" +ECO4 - SAP LOW D to LOW C,98-199m�,378.8,"�1,500","�1,500","�1,500","�1,500","�1,500","�1,500","�1,500","�1,500" +ECO4 - SAP LOW D to LOW C,over 200m�,675.2,"�2,500","�2,500","�2,250","�2,000","�1,750","�2,250","�2,000","�1,750" +ECO4 - SAP LOW D to HIGH C,0-72m�,420.8,"�1,500","�1,500","�1,500",�150,"�1,500","�1,500","�1,500","�1,500" +ECO4 - SAP LOW D to HIGH C,73-97m�,458.7,"�1,500","�1,500","�1,500","�1,500","�1,500","�1,500","�1,500","�1,500" +ECO4 - SAP LOW D to HIGH C,98-199m�,533.2,"�1,750","�1,750","�1,750","�1,750","�1,750","�1,750","�1,750","�1,750" +ECO4 - SAP LOW D to HIGH C,over 200m�,950.3,"�2,500","�2,500","�2,250","�2,000","�1,750","�2,250","�2,000","�1,750" +GBIS,0-72m�,,�600,�600,�400,�200,Not viable,�400,�200,not viable +GBIS,73-97m�,,"�1,000","�1,000",�800,�600,�400,�800,�600,�400 +GBIS,98-199m�,,"�1,000","�1,000",�800,�700,�500,�800,�700,�500 +GBIS,over 200m�,,"�1,200","�1,200","�1,200","�1,000",�800,"�1,200","�1,000",�800 \ No newline at end of file diff --git a/etl/ratecard/jjcRateCards/foam.csv b/etl/ratecard/jjcRateCards/foam.csv new file mode 100644 index 0000000..5060c0f --- /dev/null +++ b/etl/ratecard/jjcRateCards/foam.csv @@ -0,0 +1,21 @@ +Funding,Total Floor Area,ABS,no extractors or ventilation required,Trickle Vents ONLY,1 wet room extractor required,2 wet room extractor required,3 wet room extractor required,Trickle Vents + 1 wet room extractor,Trickle Vents + 2 wet room extractor,Trickle Vents + 3 wet room extractor +REMEDIAL - ECO4 - SAP HIGH D to LOW C,0-72m�,155,�500.00,�500.00,�400.00,Not viable,Not viable,�400.00,Not viable,Not viable +REMEDIAL - ECO4 - SAP HIGH D to LOW C,73-97m�,169,�500.00,�500.00,�400.00,Not viable,Not viable,�401.00,Not viable,Not viable +REMEDIAL - ECO4 - SAP HIGH D to LOW C,98-199m�,196.4,�700.00,�700.00,�500.00,�400.00,�300.00,�500.00,�400.00,�300.00 +REMEDIAL - ECO4 - SAP HIGH D to LOW C,over 200m�,350.1,"�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00" +REMEDIAL - ECO4 - SAP HIGH D to HIGH C,0-72m�,276.9,"�1,250.00","�1,250.00","�1,250.00","�1,000.00",�800.00,"�1,250.00","�1,000.00",�800.00 +REMEDIAL - ECO4 - SAP HIGH D to HIGH C,73-97m�,301.8,"�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,100.00","�1,250.00","�1,250.00","�1,100.00" +REMEDIAL - ECO4 - SAP HIGH D to HIGH C,98-199m�,350.8,"�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00" +REMEDIAL - ECO4 - SAP HIGH D to HIGH C,over 200m�,625.2,"�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00" +REMEDIAL - ECO4 - SAP LOW D to LOW C,0-72m�,299,"�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00" +REMEDIAL - ECO4 - SAP LOW D to LOW C,73-97m�,325.9,"�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00" +REMEDIAL - ECO4 - SAP LOW D to LOW C,98-199m�,378.8,"�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00" +REMEDIAL - ECO4 - SAP LOW D to LOW C,over 200m�,675.2,"�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00" +REMEDIAL - ECO4 - SAP LOW D to HIGH C,0-72m�,420.8,"�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00" +REMEDIAL - ECO4 - SAP LOW D to HIGH C,73-97m�,458.7,"�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00" +REMEDIAL - ECO4 - SAP LOW D to HIGH C,98-199m�,533.2,"�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00" +REMEDIAL - ECO4 - SAP LOW D to HIGH C,over 200m�,950.3,"�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00","�1,250.00" +GBIS Remedial,0-72m�,,�375.00,�375.00,Not viable,Not viable,Not viable,Not viable,Not viable,Not viable +GBIS Remedial,73-97m�,,�500.00,�500.00,�400.00,Not viable,Not viable,�400.00,Not viable,Not viable +GBIS Remedial,98-199m�,,�500.00,�500.00,�400.00,Not viable,Not viable,�400.00,Not viable,Not viable +GBIS Remedial,over 200m�,,�500.00,�500.00,�400.00,Not viable,Not viable,�400.00,Not viable,Not viable \ No newline at end of file diff --git a/etl/ratecard/jjcRateCards/general.csv b/etl/ratecard/jjcRateCards/general.csv new file mode 100644 index 0000000..f1ec6c0 --- /dev/null +++ b/etl/ratecard/jjcRateCards/general.csv @@ -0,0 +1,21 @@ +Funding,Total Floor Area,ABS,no extractors or ventilation required,Trickle Vents ONLY,1 wet room extractor required,2 wet room extractor required,3 wet room extractor required,Trickle Vents + 1 wet room extractor,Trickle Vents + 2 wet room extractor,Trickle Vents + 3 wet room extractor +REMEDIAL - ECO4 - SAP HIGH D to LOW C,0-72m�,155,�750.00,�750.00,�525.00,�400.00,�300.00,�525.00,�400.00,�300.00 +REMEDIAL - ECO4 - SAP HIGH D to LOW C,73-97m�,169,�750.00,�750.00,�600.00,�500.00,�400.00,�600.00,�500.00,�400.00 +REMEDIAL - ECO4 - SAP HIGH D to LOW C,98-199m�,196.4,�750.00,�750.00,�600.00,�500.00,�400.00,�525.00,�500.00,�400.00 +REMEDIAL - ECO4 - SAP HIGH D to LOW C,over 200m�,350.1,"�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00" +REMEDIAL - ECO4 - SAP HIGH D to HIGH C,0-72m�,276.9,"�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00" +REMEDIAL - ECO4 - SAP HIGH D to HIGH C,73-97m�,301.8,"�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00" +REMEDIAL - ECO4 - SAP HIGH D to HIGH C,98-199m�,350.8,"�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00" +REMEDIAL - ECO4 - SAP HIGH D to HIGH C,over 200m�,625.2,"�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00" +REMEDIAL - ECO4 - SAP LOW D to LOW C,0-72m�,299,"�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00" +REMEDIAL - ECO4 - SAP LOW D to LOW C,73-97m�,325.9,"�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00" +REMEDIAL - ECO4 - SAP LOW D to LOW C,98-199m�,378.8,"�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00" +REMEDIAL - ECO4 - SAP LOW D to LOW C,over 200m�,675.2,"�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00" +REMEDIAL - ECO4 - SAP LOW D to HIGH C,0-72m�,420.8,"�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00" +REMEDIAL - ECO4 - SAP LOW D to HIGH C,73-97m�,458.7,"�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00" +REMEDIAL - ECO4 - SAP LOW D to HIGH C,98-199m�,533.2,"�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00" +REMEDIAL - ECO4 - SAP LOW D to HIGH C,over 200m�,950.3,"�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00","�1,500.00" +GBIS Remedial,0-72m�,,�500.00,�500.00,�400.00,Not viable,Not viable,�500.00,Not viable,Not viable +GBIS Remedial,73-97m�,,�500.00,�500.00,�500.00,�500.00,�400.00,�500.00,�500.00,�400.00 +GBIS Remedial,98-199m�,,�500.00,�500.00,�500.00,�500.00,�500.00,�500.00,�500.00,�500.00 +GBIS Remedial,over 200m�,,�500.00,�500.00,�500.00,�500.00,�500.00,�500.00,�500.00,�500.00 \ No newline at end of file diff --git a/etl/scis_invoice.py b/etl/scis_invoice.py index 0af6e9a..fc9d481 100644 --- a/etl/scis_invoice.py +++ b/etl/scis_invoice.py @@ -25,7 +25,8 @@ def main(): for survey in list_of_surveys: if survey.pre_site_note: - if survey.pre_site_note.property_description.main_property.wall.insulation.lower() == 'as built' and survey.pre_site_note.property_description.main_property.wall.construction.lower() == "cavity wall": + if survey.pre_site_note.property_description.main_property.wall.insulation.lower() == 'as built' \ + and survey.pre_site_note.property_description.main_property.wall.construction.lower() == "cavity wall": if survey.csr: data["Price"].append(500) data["Type of Work"].append("REMIDIAL CWI ONLY") diff --git a/etl/scraper/scraper.py b/etl/scraper/scraper.py index c2881c4..1a6a1fa 100644 --- a/etl/scraper/scraper.py +++ b/etl/scraper/scraper.py @@ -14,7 +14,7 @@ from datetime import datetime, timedelta def previous_monday(): today = datetime.today() last_monday = today - timedelta(days=today.weekday() + 7) # Go back to last week's Monday - # return "W.C. 03.03.2025" # Quick one off script for nic. Hopefully get 199 Do not merge!!!! + return "W.C. 10.03.2025" # Quick one off script for nic. Hopefully get 199 Do not merge!!!! return f"W.C. {last_monday.strftime('%d.%m.%Y')}" WEEK_COMMENCING = os.getenv("WEEK_COMMENCING", previous_monday()) diff --git a/etl/scraper/submission_csv.py b/etl/scraper/submission_csv.py new file mode 100644 index 0000000..b58c1da --- /dev/null +++ b/etl/scraper/submission_csv.py @@ -0,0 +1,5 @@ + +class submissionScraper(): + + def __init__(self): + pass \ No newline at end of file diff --git a/etl/sgec_invoice.py b/etl/sgec_invoice.py new file mode 100644 index 0000000..fc9d481 --- /dev/null +++ b/etl/sgec_invoice.py @@ -0,0 +1,55 @@ +from etl.scraper.scraper import SharePointScraper, SharePointInstaller, WEEK_COMMENCING +from pprint import pformat +from etl.pdfReader.pdfReaderToText import pdfReaderToText +from etl.surveyedData.surveryedData import surveyedDataProcessor +import pandas as pd + + + + +def main(): + data = { + "Address": [], + "Surveyor's Name": [], + "Type of Work": [], + "Price": [] + } + + south_coast_scraper = SharePointScraper(SharePointInstaller.SOUTH_COAST_INSULATION) + file_paths = south_coast_scraper.download_file_for_each_address() + list_of_surveys = [] + for eachAddress in file_paths: + for address, files in eachAddress.items(): + list_of_surveys.append(surveyedDataProcessor(address, files)) + + + for survey in list_of_surveys: + if survey.pre_site_note: + if survey.pre_site_note.property_description.main_property.wall.insulation.lower() == 'as built' \ + and survey.pre_site_note.property_description.main_property.wall.construction.lower() == "cavity wall": + if survey.csr: + data["Price"].append(500) + data["Type of Work"].append("REMIDIAL CWI ONLY") + else: + data["Price"].append(1000) + data["Type of Work"].append("CAVITY ONLY") + else: + # Solar + data["Price"].append(1608) + data["Type of Work"].append("SOLAR") + + data["Address"].append(survey.address) + data["Surveyor's Name"].append(survey.pre_site_note.assessor_information.name) + + df = pd.DataFrame(data) + + # Save to an Excel file + df.to_excel("survey_data.xlsx", index=False) + + print(f"WEEK COMMENCING {WEEK_COMMENCING}") + print("Excel file 'survey_data.xlsx' created successfully!") + print(pformat(data)) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/etl/temp/jjc_submissionData.csv b/etl/temp/jjc_submissionData.csv new file mode 100644 index 0000000..01d1fd9 --- /dev/null +++ b/etl/temp/jjc_submissionData.csv @@ -0,0 +1,222 @@ +SUBMISSION TYPE,Ventilation Requirements,PRE INSTALL SAP SCORE,POST INSTALL SAP SCORE,NO,Street Name,Post Code,FLOOR AREA +ECO4 - CAVITY ONLY,NONE,64D,71C,2, HICKMORE WALK,SW4 6EQ,105.1 +ECO4 - CAVITY ONLY,Trickle vents and kitchen extractor ,68D,72C,6, HICKMORE WALK,SW4 6EQ,96.2 +ECO4 - CAVITY ONLY,NONE,66D,72C,28, HICKMORE WALK,SW4 6EQ,89 +ECO4 - CAVITY ONLY,NONE,63D,70C,98, St. Alphonsus Road,SW4 7AT,79.18 +ECO4 - REMEDIAL CWI ONLY,Kitchen Extractor,66D,73C,140, St. Alphonsus Road,SW4 7BN,98.71 +ECO4 - CAVITY ONLY,Trickle Vents,66D,73C,132, St. Alphonsus Road,SW4 7BN,98.71 +ECO4 - CAVITY ONLY,Bathroom Extractor,64D,70C,141, St. Alphonsus Road,SW4 7BP,79.18 +ECO4 - REMEDIAL CWI ONLY,NONE,64D,71C,39, Haselrigge Road,SW4 7EN,107.67 +ECO4 - REMEDIAL CWI ONLY,Kitchen Extractor,66D,69C,43, Haselrigge Road,SW4 7EN,107.67 +ECO4 - REMEDIAL CWI ONLY,NONE,68D,70C,51, Haselrigge Road,SW4 7EN,78.21 +ECO4 - REMEDIAL CWI ONLY,NONE,64D,69C,24, Felmersham Close,SW4 7EU,120.16 +ECO4 - REMEDIAL CWI ONLY,Kitchen Extractor,65D,70C,26, Felmersham Close,SW4 7EU,107.67 +ECO4 - REMEDIAL CWI ONLY,NONE,64D,71C,16, Bowland Road,SW4 7LE,107.67 +ECO4 - REMEDIAL CWI ONLY,NONE,66D,70C,37, Britannia Close,SW4 7NN,120.16 +N/A,Kitchen Extractor,65D,,FLAT 1,3 Willis Road,CB1 2AQ,35.24 +N/A,Kitchen Extractor,57D,,FLAT 2,3 Willis Road,CB1 2AQ,35.65 +N/A,NONE,65D,,FLAT 3,3 Willis Road,CB1 2AQ,16.57 +N/A,Kitchen Extractor,72C,,FLAT4,3 Willis Road,CB1 2AQ,45.12 +N/A,Bathroom&Kitchen extractor,30F,,FLAT 5,3 Willis Road,CB1 2AQ,43.56 +ECO4 - REMEDIAL CWI ONLY,Trickle Vents,66D,72C,15,Brookfield Road,CB23 3RP,75.1 +ECO4 - REMEDIAL CWI ONLY,NONE,66D,70C,72,Ermine Street South,CB23 3QA,86.71 +ECO4 - REMEDIAL CWI ONLY,WC&Bathroom extractor,66D,70C,36,Ermine Street South,CB23 3QA,86.71 +ECO4 - REMEDIAL CWI ONLY,NONE,68D,73C,20,Barons Way,CB23 3QJ,93.22 +ECO4 - REMEDIAL CWI ONLY,WC Extractor,65D,70C,76,Ermine Street South,CB23 3QA,86.71 +ECO4 - REMEDIAL CWI ONLY,WC&Bathroom extractor,65D,70C,4,Brookfield Road,CB23 3RP,75.1 +ECO4 - REMEDIAL CWI ONLY,WC Extractor,66D,70C,50,Ermine Street South,CB23 3QA,86.71 +ECO4 - REMEDIAL CWI ONLY,NONE,68D,73C,23,Ridgeway,CB23 3RW,92.11 +ECO4 - REMEDIAL CWI ONLY,NONE,64D,69C,148,PEEL GREEN ROAD,M30 7BA,61 +ECO4 GBIS REMEDIAL,NONE,58D,65D,122,PEEL GREEN ROAD,M30 7AZ,61 +ECO4 GBIS REMEDIAL,NONE,63D,67D,130,PEEL GREEN ROAD,M30 7AZ,70 +ECO4 GBIS REMEDIAL,NONE,61D,65D,146,PEEL GREEN ROAD,M30 7BA,70 +ECO4 - REMEDIAL CWI ONLY,WC Extractor,63D,69C,56,PEEL GREEN ROAD,M30 7AY,90 +ECO4 - REMEDIAL CWI ONLY,WC&Bathroom extractor,68D,71C,62,PEEL GREEN ROAD,M30 7AY,90 +ECO4 - REMEDIAL CWI ONLY,Kitchen&WC extractor,67D,70C,17,MARSDEN STREET,M30 9QP,78 +ECO4 - REMEDIAL CWI ONLY,Kitchen Extractor,64D,69C,7,KEAN PLACE,M30 0PS,51 +ECO4 GBIS REMEDIAL,Bathroom Extractor,61D,65D,41,PEEL GREEN ROAD,M30 7AY,92 +ECO4 - REMEDIAL CWI ONLY,WC Extractor,67D,70C,65,PEEL GREEN ROAD,M30 7AY,85 +ECO4 GBIS REMEDIAL,NONE,62D,67D,1,Ermine Street South,CB23 3RD,87.73 +ECO4 GBIS REMEDIAL,Kitchen Extractor,55D,68D,21,Barons Way,CB23 3QJ,93.22 +ECO4 - REMEDIAL CWI ONLY,NONE,66D,71C,1,BULLSLAND GARDENS,WD3 5BH,61.25 +ECO4 - REMEDIAL CWI ONLY,NONE,67D,72C,2,BULLSLAND GARDENS,WD3 5BH,61.25 +ECO4 - REMEDIAL CWI ONLY,Bathroom Extractor,65D,70C,5,BULLSLAND GARDENS,WD3 5BH,61.25 +ECO4 - REMEDIAL CWI ONLY,NONE,65D,70C,6,BULLSLAND GARDENS,WD3 5BH,61.25 +ECO4 - REMEDIAL CWI ONLY,Bathroom&Kitchen extractor,65D,70C,7,BULLSLAND GARDENS,WD3 5BH,61.25 +ECO4 - REMEDIAL CWI ONLY,Bathroom Extractor,66D,71C,8,BULLSLAND GARDENS,WD3 5BH,61.25 +ECO4 - REMEDIAL CWI ONLY,NONE,66D,72C,10,BULLSLAND GARDENS,WD3 5BH,61.25 +ECO4 - REMEDIAL CWI ONLY,Bathroom&Kitchen extractor,66D,71C,11,BULLSLAND GARDENS,WD3 5BH,61.25 +ECO4 - REMEDIAL CWI ONLY,Bathroom Extractor,66D,71C,14,BULLSLAND GARDENS,WD3 5BH,61.25 +ECO4 - REMEDIAL CWI ONLY,NONE,66D,71C,15,BULLSLAND GARDENS,WD3 5BH,61.25 +ECO4 - REMEDIAL CWI ONLY,NONE,66D,72C,29,BULLSLAND GARDENS,WD3 5BH,61.25 +ECO4 - REMEDIAL CWI ONLY,NONE,66D,71C,30,BULLSLAND GARDENS,WD3 5BH,61.25 +ECO4 - REMEDIAL CWI ONLY,Trickle Vents and Extractors for the whole house,68D,73C,26,ST LAWRENCE CLOSE,WD5 0AU,56.76 +ECO4 - REMEDIAL CWI ONLY,Trickle Vents,65D,71C,29,ST LAWRENCE CLOSE,WD5 0AU,56.76 +ECO4 - REMEDIAL CWI ONLY,BATHROOM EXTRACTOR,67D,72C,5,TUDOR WAY,WD3 8JA,91 +ECO4 - REMEDIAL CWI ONLY,BATHROOM EXTRACTOR AND TRICKLE VENTS,67D,72C,14,TUDOR WAY,WD3 8JB,82 +ECO4 - REMEDIAL CWI ONLY,KITCHEN AND BATHROOM EXTRACTORS AND TRICKLE VENTS,66D,71C,21,TUDOR WAY,WD3 8JA,91 +ECO4 - REMEDIAL CWI ONLY,BATHROOM EXTRACTOR AND TRICKLE VENTS,64D,70C,26,TUDOR WAY,WD3 8JB,91 +ECO4 - REMEDIAL CWI ONLY,BATHROOM EXTRACTOR AND TRICKLE VENTS,66D,71C,33,TUDOR WAY,WD3 8JA,91 +ECO4 - REMEDIAL CWI ONLY,KITCHEN EXTRACTOR AND TRICKLE VENTS,67D,72C,42,TUDOR WAY,WD3 8JB,82 +ECO4 - REMEDIAL CWI ONLY,TRICKLE VENTS,64D,70C,56,TUDOR WAY,WD3 8JB,91 +ECO4 - REMEDIAL CWI ONLY,KITCHEN EXTRACTOR AND TRICKLE VENTS,66D,71C,59,TUDOR WAY,WD3 8JA,82 +ECO4 - REMEDIAL CWI ONLY,KITCHEN AND BATHROOM EXTRACTORS AND TRICKLE VENTS,66D,71C,12,CHILTERN DRIVE,WD3 8JS,78 +ECO4 - REMEDIAL CWI ONLY,KITCHEN EXTRACTOR ,67D,72C,14,CHILTERN DRIVE,WD3 8JS,84 +ECO4 - REMEDIAL CWI ONLY,KITCHEN EXTRACTOR AND TRICKLE VENTS,68D,72C,20,CHILTERN DRIVE,WD3 8JS,84 +ECO4 - REMEDIAL CWI ONLY,KITCHEN EXTRACTOR,67D,72C,25,CHILTERN DRIVE,WD3 8JX,90 +ECO4 - REMEDIAL CWI ONLY,NONE,66D,71C,35,CHILTERN DRIVE,WD3 8JX,80 +ECO4 - REMEDIAL CWI ONLY,KITCHEN AND BATHROOM EXTRACTORS AND TRICKLE VENTS,66D,71C,9,BERRY LANE,WD3 7HJ,82 +ECO4 - REMEDIAL CWI ONLY,KITCHEN AND BATHROOM EXTRACTORS AND TRICKLE VENTS,68D,72C,13,BERRY LANE,WD3 7HJ,82 +ECO4 - REMEDIAL CWI ONLY,TRICKLE VENTS,65D,69C,1,LONGCLIFFE PATH,WD19 7PH,34 +ECO4 GBIS,WC Extractor,66D,71C,21,MARLBOROUGH ROAD,M44 6HH,102 +ECO4 - REMEDIAL CWI ONLY,Bathroom Extractor,66D,70C,57,SILVER STREET,M44 6HT,66 +ECO4 - REMEDIAL CWI ONLY,Kitchen Extractor,66D,69C,5,DUNLIN DRIVE,M44 6PL,86 +ECO4 - REMEDIAL CWI ONLY,WC Extractor,67D,72C,71,SILVER STREET,M44 6JJ,96 +ECO4 GBIS REMEDIAL,WC Extractor,63D,67D,85,SILVER STREET,M44 6JJ,86 +ECO4 - REMEDIAL CWI ONLY,Kitchen Extractor,65D,70C,22,HAWK ROAD,M44 6LE,73 +ECO4 - REMEDIAL CWI ONLY,WC Extractor,66D,69C,16,HAWK ROAD,M44 6LE,86 +ECO4 GBIS REMEDIAL,Bathroom Extractor,61D,66D,1,HAWK ROAD,M44 6LE,73 +ECO4 GBIS,Bathroom&Kitchen extractor,66D,70C,52,PARKSTONE ROAD,M44 6LN,73 +ECO4 - REMEDIAL CWI ONLY,NONE,68D,71C,4,RAVEN DRIVE,M44 6LH,92 +ECO4 - REMEDIAL CWI ONLY,NONE,68D,71C,10,PEREGRINE DRIVE,M44 6PF,92 +ECO4 GBIS REMEDIAL,Bathroom Extractor,62D,65D,4,PEREGRINE DRIVE,M44 6PF,92 +ECO4 - REMEDIAL CWI ONLY,NONE,66D,70C,15,MORILLON ROAD,M44 6PN,72 +ECO4 GBIS - JJC <73m�,NONE,60D,66D,14,ABBEY DRIVE,M27 6AS,62 +ECO4 - REMEDIAL CWI ONLY,Kitchen Extractor,67D,71C,11,ST LAWRENCE CLOSE,WD5 0AU,45.97 +ECO4 - REMEDIAL CWI ONLY,Trickle Vents and Extractors for the whole house,68D,72C,12,ST LAWRENCE CLOSE,WD5 0AU,45.97 +ECO4 - REMEDIAL CWI ONLY,Trickle Vents,64D,70C,16,ST LAWRENCE CLOSE,WD5 0AU,56.76 +ECO4 - REMEDIAL CWI ONLY,Bathroom Extractor,62D,69C,17,ST LAWRENCE CLOSE,WD5 0AU,48.31 +ECO4 - REMEDIAL CWI ONLY,Trickle Vents,67D,72C,18,ST LAWRENCE CLOSE,WD5 0AU,56.76 +ECO4 - REMEDIAL CWI ONLY,Trickle Vents,65D,71C,19,ST LAWRENCE CLOSE,WD5 0AU,56.76 +ECO4 - REMEDIAL CWI ONLY,Trickle Vents and Extractors for the whole house,63D,70C,20,ST LAWRENCE CLOSE,WD5 0AU,48.31 +ECO4 - REMEDIAL CWI ONLY,Bathroom Extractor,66D,71C,21,ST LAWRENCE CLOSE,WD5 0AU,48.31 +ECO4 - REMEDIAL CWI ONLY,Trickle Vents and Extractors for the whole house,66D,72C,22,ST LAWRENCE CLOSE,WD5 0AU,48.31 +ECO4 - REMEDIAL CWI ONLY,Trickle Vents and Extractors for the whole house,62D,70C,23,ST LAWRENCE CLOSE,WD5 0AU,48.31 +ECO4 - REMEDIAL CWI ONLY,Trickle Vents and Extractors for the whole house,64D,71C,24,ST LAWRENCE CLOSE,WD5 0AU,48.31 +ECO4 - REMEDIAL CWI ONLY,Trickle Vents and Extractors for the whole house,67D,72C,25,ST LAWRENCE CLOSE,WD5 0AU,48.31 +ECO4 - REMEDIAL CWI ONLY,Trickle Vents and Extractors for the whole house,65D,71C,28,ST LAWRENCE CLOSE,WD5 0AU,56.76 +ECO4 - REMEDIAL CWI ONLY,Trickle Vents and Extractors for the whole house,64D,71C,30,ST LAWRENCE CLOSE,WD5 0AU,50.78 +ECO4 - REMEDIAL CWI ONLY,Trickle Vents and Extractors for the whole house,68D,72C,31,ST LAWRENCE CLOSE,WD5 0AU,50.78 +ECO4 - REMEDIAL CWI ONLY,Trickle Vents,68D,72C,32,ST LAWRENCE CLOSE,WD5 0AU,50.78 +ECO4 - REMEDIAL CWI ONLY,Trickle Vents and Extractors for the whole house,68D,72C,33,ST LAWRENCE CLOSE,WD5 0AU,50.78 +ECO4 - REMEDIAL CWI ONLY,NONE,65D,73C,103,THE QUEENS DRIVE,WD3 8LY,75.3 +ECO4 - REMEDIAL CWI ONLY,NONE,65D,73C,135,THE QUEENS DRIVE,WD3 8LY,75.3 +ECO4 - REMEDIAL CWI ONLY,Bathroom&Kitchen extractor,64D,71C,26,LONGCROFT ROAD,WD3 9TR,59.39 +ECO4 - REMEDIAL CWI ONLY,Kitchen Extractor,64D,71C,28,LONGCROFT ROAD,WD3 9TR,59.39 +ECO4 - REMEDIAL CWI ONLY,Kitchen Extractor,60D,69C,30,LONGCROFT ROAD,WD3 9TR,59.39 +ECO4 - REMEDIAL CWI ONLY,Kitchen Extractor,61D,70C,36,LONGCROFT ROAD,WD3 9TR,59.39 +ECO4 - REMEDIAL CWI ONLY,Trickle Vents,64D,70C,1,CLITHEROE GARDENS,WD19 6RP,35.68 +ECO4 - REMEDIAL CWI ONLY,Trickle Vents,64D,70C,3,CLITHEROE GARDENS,WD19 6RP,35.68 +ECO4 - REMEDIAL CWI ONLY,Bathroom Extractor,66D,70C,50,PARKSTONE ROAD,M44 6LN,73 +ECO4 - REMEDIAL CWI ONLY,NONE,66D,70C,119,CUTNOOK LANE,M44 6LU,72 +ECO4 GBIS REMEDIAL,NONE,62D,67D,89,PARKSTONE ROAD,M44 6LJ,68 +ECO4 GBIS REMEDIAL,WC&Bathroom extractor,61D,66D,22,LINNET DRIVE,M44 6LW,81 +ECO4 - REMEDIAL CWI ONLY,All wet rooms need extractors,68D,75C,153,HEATHCOTE STREET,ST3 1AD,74 +ECO4 GBIS REMEDIAL,NONE,61D,67D,37,MEAFORD DRIVE,ST3 2BY,83 +ECO4 GBIS REMEDIAL,NONE,61D,67D,24,SEEDFIELDS ROAD,ST3 2BZ,84 +ECO4 GBIS,WC&Bathroom extractor,61D,67D,16,IONA PLACE ,ST3 2DY,82 +ECO4 GBIS REMEDIAL,NONE,62D,68D,27,ROUNDWAY,ST3 2BH,83 +ECO4 GBIS REMEDIAL,Bathroom Extractor,62D,67D,50,LOUISE DRIVE,ST3 2DT,82 +ECO4 GBIS REMEDIAL,NONE,63D,66D,30,ANGLESEY DRIVE,ST3 2SS,49 +ECO4 - REMEDIAL CWI ONLY,NONE,67D,71C,99,MEAFORD DRIVE,ST3 2BG,83 +ECO4 - REMEDIAL CWI ONLY,WC Extractor,66D,71C,90,MEAFORD DRIVE,ST3 2BB,83 +ECO4 - REMEDIAL CWI ONLY,NONE,66D,72C,2, GREGSON CLOSE,ST3 2SJ,50.62 +ECO4 - REMEDIAL CWI ONLY,NONE,66D,71C,32, ANGLESEY DRIVE,ST3 2SS,50.62 +ECO4 - REMEDIAL CWI ONLY,Bathroom Extractor,64D,70C,4, PEMBRIDGE ROAD,ST3 3BX,41.68 +ECO4 - REMEDIAL CWI ONLY,NONE,60D,69C,12A,PEMBRIDGE ROAD,ST3 3BX,41.68 +ECO4 - REMEDIAL CWI ONLY,Bathroom Extractor,62D,69C,10, PEMBRIDGE ROAD,ST3 3BX,41.68 +ECO4 - REMEDIAL CWI ONLY,NONE,63D,69C,10A, PEMBRIDGE ROAD,ST3 3BX,41.68 +ECO4 - REMEDIAL CWI ONLY,NONE,66D,71C,9, LOMOND WALK,ST3 3HW,50.62 +ECO4 - REMEDIAL CWI ONLY,NONE,66D,71C,7, LOMOND WALK,ST3 3HW,50.62 +ECO4 - REMEDIAL CWI ONLY,NONE,66D,71C,5, LOMOND WALK,ST3 3HW,50.62 +ECO4 - REMEDIAL CWI ONLY,NONE,66D,71C,3, LOMOND WALK,ST3 3HW,50.62 +ECO4 - REMEDIAL CWI ONLY,Kitchen Extractor,66D,71C,2, LOMOND WALK,ST3 3HW,50.62 +ECO4 - REMEDIAL CWI ONLY,NONE,67D,72C,20, LOMOND WALK,ST3 3HW,50.62 +ECO4 - REMEDIAL CWI ONLY,NONE,66D,71C,18, LOMOND WALK,ST3 3HW,50.62 +ECO4 - REMEDIAL CWI ONLY,Kitchen Extractor,66D,71C,15, LOMOND WALK,ST3 3HW,50.62 +ECO4 - REMEDIAL CWI ONLY,NONE,66D,71C,12, LOMOND WALK,ST3 3HW,50.62 +ECO4 - REMEDIAL CWI ONLY,NONE,66D,71C,10, LOMOND WALK,ST3 3HW,50.62 +ECO4 - REMEDIAL CWI ONLY,Bathroom&Kitchen extractor,65D,71C,7, INGLEBY ROAD,ST3 3JY,49.45 +ECO4 - REMEDIAL CWI ONLY,NONE,66D,70C,91, WIMBORNE AVENUE,ST3 3LS,49.45 +ECO4 - REMEDIAL CWI ONLY,NONE,66D,71C,87, WIMBORNE AVENUE,ST3 3LS,49.45 +ECO4 - REMEDIAL CWI ONLY,Kitchen Extractor,67D,71C,85, WIMBORNE AVENUE,ST3 3LS,49.45 +ECO4 - REMEDIAL CWI ONLY,NONE,67D,71C,83, WIMBORNE AVENUE,ST3 3LS,49.45 +ECO4 - REMEDIAL CWI ONLY,NONE,65D,70C,5, POOLSIDE,ST3 3NY,50.62 +ECO4 - REMEDIAL CWI ONLY,NONE,61D,69C,387, LIGHTWOOD ROAD,ST3 4JT,58.7 +ECO4 - REMEDIAL CWI ONLY,Kitchen Extractor,63D,69C,379, LIGHTWOOD ROAD,ST3 4JT,42.68 +ECO4 - REMEDIAL CWI ONLY,NONE,66D,72C,7, PEEL STREET,ST3 4PE,50.62 +ECO4 - REMEDIAL CWI ONLY,NONE,65D,70C,58, WISE STREET,ST3 4PQ,41.88 +ECO4 - REMEDIAL CWI ONLY,NONE,64D,70C,52, BRIDLE PATH,ST3 4SB,41.88 +ECO4 - REMEDIAL CWI ONLY,NONE,63D,70C,49, BRIDLE PATH,ST3 4SB,50.62 +ECO4 - REMEDIAL CWI ONLY,NONE,65D,71C,9, MELBOURNE STREET,ST3 5AA,50.62 +ECO4 - REMEDIAL CWI ONLY,NONE,62D,69C,1,TILL WALK,ST3 5DF,41.88 +ECO4 - REMEDIAL CWI ONLY,NONE,68D,75C,36,Woodville road,ST3 6BX,73.92 +ECO4 - REMEDIAL CWI ONLY,NONE,64D,70C,18,Bilston street,ST4 5DA,41.88 +ECO4 - REMEDIAL CWI ONLY,NONE,65D,72C,19,Sedgley walk,ST3 1RY,50.62 +ECO4 - REMEDIAL CWI ONLY,NONE,65D,72C,9,Farrington close,ST6 8DQ,49.45 +ECO4 - REMEDIAL CWI ONLY,Bathroom Extractor,65D,70C,33,Oakfield,WD3 8LR,67.43 +ECO4 - REMEDIAL CWI ONLY,Bathroom Extractor,65D,70C,49,Oakfield,WD3 8NA,67.43 +ECO4 - REMEDIAL CWI ONLY,Kitchen Extractor,66D,71C,45,Oakfield,WD3 8NA,67.43 +ECO4 - REMEDIAL CWI ONLY,WC Extractor,67D,72C,5,Owens way,WD3 3PS,82.14 +ECO4 - REMEDIAL CWI ONLY,Kitchen Extractor,67D,72C,7,Owens way,WD3 3PS,82.14 +ECO4 - REMEDIAL CWI ONLY,NONE,60D,69C,5,Fyfield Road,SW9 7HT,98.36 +ECO4 - REMEDIAL CWI ONLY,NONE,68D,73C,17,Willington Road,SW9 9NA,79.2 +ECO4 - REMEDIAL CWI ONLY,NONE,68D,74C,19,Willington Road,SW9 9NA,82.33 +ECO4 - REMEDIAL CWI ONLY,NONE,68D,73C,27,Willington Road,SW9 9NA,82.33 +ECO4 - REMEDIAL CWI ONLY,NONE,67D,74C,33,Willington Road,SW9 9NA,82.33 +ECO4 - REMEDIAL CWI ONLY,NONE,68D,74C,41,Willington Road,SW9 9NA,82.33 +ECO4 - REMEDIAL CWI ONLY,NONE,66D,72C,5,Mallams Mews,SW9 7HR,98.36 +ECO4 - REMEDIAL CWI ONLY,Bathroom Extractor,67D,71C,35,HEATHCOTE STREET,ST3 1AA,38.99 +ECO4 - REMEDIAL CWI ONLY,NONE,66D,70C,96,EDGEFIELD ROAD,ST3 1AF,49.88 +ECO4 - REMEDIAL CWI ONLY,Kitchen Extractor,66D,70C,97,EDGEFIELD ROAD,ST3 1AG,38.99 +ECO4 - REMEDIAL CWI ONLY,Bathroom Extractor,68D,72C,155,EDGEFIELD ROAD,ST3 1AG,38.99 +ECO4 - REMEDIAL CWI ONLY,Kitchen Extractor,66D,70C,151, EDGEFIELD ROAD,ST3 1AG,38.99 +ECO4 - REMEDIAL CWI ONLY,NONE,66D,70C,113, EDGEFIELD ROAD,ST3 1AG,38.99 +ECO4 - REMEDIAL CWI ONLY,NONE,65D,69C,111, EDGEFIELD ROAD,ST3 1AG,38.99 +ECO4 - REMEDIAL CWI ONLY,Kitchen Extractor,67D,70C,105, EDGEFIELD ROAD,ST3 1AG,38.99 +ECO4 - REMEDIAL CWI ONLY,NONE,67D,70C,103, EDGEFIELD ROAD,ST3 1AG,38.99 +ECO4 - REMEDIAL CWI ONLY,Kitchen Extractor,67D,71C,101, EDGEFIELD ROAD,ST3 1AG,38.99 +ECO4 - REMEDIAL CWI ONLY,NONE,67D,71C,96,LONGLEY ROAD,ST3 1AN,38.99 +ECO4 - REMEDIAL CWI ONLY,NONE,68D,72C,114, LONGLEY ROAD,ST3 1AN,48.51 +ECO4 - REMEDIAL CWI ONLY,Kitchen Extractor,66D,70C,30,Caroline Street,ST3 1DE,44.18 +ECO4 - REMEDIAL CWI ONLY,NONE,65D,69C,32,Caroline Street,ST3 1DE,30.68 +ECO4 - REMEDIAL CWI ONLY,NONE,65D,69C,34,Caroline Street,ST3 1DE,30.68 +ECO4 - REMEDIAL CWI ONLY,NONE,68D,71C,5,Conewood Place,ST3 2NW,40.26 +ECO4 - REMEDIAL CWI ONLY,NONE,65D,70C,17,Conewood Place,ST3 2NW,57.09 +ECO4 - REMEDIAL CWI ONLY,NONE,67D,71C,21,Conewood Place,ST3 2NW,42.51 +ECO4 - REMEDIAL CWI ONLY,NONE,68D,71C,27,Conewood Place,ST3 2NW,42.5 +ECO4 - REMEDIAL CWI ONLY,NONE,68D,70C,54,Kirkup Walk,ST3 2SD,36.94 +ECO4 - REMEDIAL CWI ONLY,NONE,65D,69C,4,Milverton Place,ST3 2SH,37.96 +ECO4 - REMEDIAL CWI ONLY,NONE,68D,71C,48,Lundy Road,ST3 2DZ,38.22 +ECO4 - REMEDIAL CWI ONLY,NONE,68D,71C,56,Heathcote Road,ST3 2LX,41.94 +ECO4 - REMEDIAL CWI ONLY,NONE,66D,70C,47,Hurst Street,ST3 2LT,47.79 +ECO4 - REMEDIAL CWI ONLY,NONE,66D,70C,59,Caroline Street,ST3 1DE,49.44 +ECO4 - REMEDIAL CWI ONLY,NONE,67D,70C,15,Sedgley Walk,ST3 1RY,47.79 +ECO4 - REMEDIAL CWI ONLY,NONE,65D,69C,17,Sedgley Walk,ST31RY,47.79 +ECO4 - REMEDIAL CWI ONLY,NONE,66D,69C,144,Foley Road,ST 2LH,48.51 +ECO4 - CAVITY ONLY,Trickle Vents,68D,71C,13,Logepole clos ,M30 7GF,79.44 +ECO4 - CAVITY ONLY,NONE,68D,71C,21,Earlsdon crescent ,M38 9HF,80.2 +ECO4 - REMEDIAL CWI ONLY,Bathroom&Kitchen extractor,64D,70C,5,HUBBARDS ROAD,WD3 5JL,49.87 +ECO4 - REMEDIAL CWI ONLY,Bathroom&Kitchen extractor,67D,72C,7,HUBBARDS ROAD,WD3 5JL,49.87 +ECO4 - REMEDIAL CWI ONLY,Kitchen Extractor,65D,71C,13,HUBBARDS ROAD,WD3 5JL,49.87 +ECO4 - REMEDIAL CWI ONLY,Bathroom&Kitchen extractor,66D,72C,15,HUBBARDS ROAD,WD3 5JL,49.87 +ECO4 - REMEDIAL CWI ONLY,NONE,64D,72C,34,HUBBARDS ROAD,WD3 5JJ,81.86 +ECO4 - REMEDIAL CWI ONLY,Bathroom Extractor,65D,74C,8,THE GREENWAY,WD3 8HU,94.26 +ECO4 - REMEDIAL CWI ONLY,Trickle Vents and Extractors for the whole house,65D,74C,9,ORCHARD WAY,WD3 8HZ,94.26 +ECO4 - REMEDIAL CWI ONLY,Trickle Vents and Extractors for the whole house,64D,72C,13,ORCHARD WAY,WD3 8HZ,74.39 +ECO4 - REMEDIAL CWI ONLY,Trickle Vents and Extractors for the whole house,65D,73C,14,ORCHARD WAY,WD3 8HZ,73.1 +ECO4 - REMEDIAL CWI ONLY,Trickle Vents and Extractors for the whole house,67D,73C,19,ORCHARD WAY,WD3 8HZ,87.45 +ECO4 - REMEDIAL CWI ONLY,Trickle Vents,65D,73C,36,ORCHARD WAY,WD3 8HZ,73.1 +ECO4 - REMEDIAL CWI ONLY,Trickle Vents,65D,74C,45,ORCHARD WAY,WD3 8HZ,73.1 +ECO4 - REMEDIAL CWI ONLY,Kitchen Extractor,62D,71C,2O,EDINBURGH AVENUE,WD3 8LB,72.72 +ECO4 - REMEDIAL CWI ONLY,Bathroom Extractor,64D,73C,29,THE QUEENS DRIVE,WD3 8LN,98.36 +ECO4 - REMEDIAL CWI ONLY,Kitchen Extractor,65D,74C,31,THE QUEENS DRIVE,WD3 8LN,98.36 +ECO4 - REMEDIAL CWI ONLY,NONE,63D,72C,42,THE QUEENS DRIVE,WD3 8LT,78.9 +ECO4 - REMEDIAL CWI ONLY,Trickle Vents and Extractors for the whole house,66D,74C,56,THE QUEENS DRIVE,WD3 8LT,78.06 +ECO4 - REMEDIAL CWI ONLY,Trickle Vents,65D,72C,80,THE QUEENS DRIVE,WD3 8LX,78.06 +ECO4 - REMEDIAL CWI ONLY,NONE,65D,74C,82,THE QUEENS DRIVE,WD3 8LX,84.48 +ECO4 - REMEDIAL CWI ONLY,NONE,65D,73C,90,CHILTERN DRIVE,WD3 8JZ,74.8 +ECO4 - REMEDIAL CWI ONLY,Kitchen Extractor,66D,74C,96,CHILTERN DRIVE,WD3 8JZ,74.8 +ECO4 - REMEDIAL CWI ONLY,Kitchen Extractor,66D,73C,112,CHILTERN DRIVE,WD3 8JZ,74.8 +ECO4 - REMEDIAL CWI ONLY,Trickle Vents,64D,70C,21,CLITHEROE GARDENS,WD19 6RP,35.68 +ECO4 - REMEDIAL CWI ONLY,NONE,64D,70C,26,CLITHEROE GARDENS,WD19 6RP,35.68 \ No newline at end of file diff --git a/etl/temp/submissionReader.py b/etl/temp/submissionReader.py new file mode 100644 index 0000000..b672b86 --- /dev/null +++ b/etl/temp/submissionReader.py @@ -0,0 +1,11 @@ +import pandas as pd + + +class SubmissionReader(): + + def __init__(self, path): + """ + A quick and dirty csv reader to run in parrall with Lewis' Deemed score process + """ + self.path = path + self.df = pd.read_csv(self.path) \ No newline at end of file diff --git a/etl/transform/types.py b/etl/transform/types.py index de84fad..ebccc5f 100644 --- a/etl/transform/types.py +++ b/etl/transform/types.py @@ -150,6 +150,7 @@ class Windows(BaseModel): g_value: int class PropertyDetail(BaseModel): + # change this name to build parts age_band: str dimensions: List[Dimension] = [] wall: Optional[Walls] = None @@ -192,3 +193,6 @@ class PropertyDescription(BaseModel): mainHeating: Optional[Heating] mainHeating2: Optional[Heating] secondaryHeatingType: Optional[HeatingType] + +class Insulation(BaseModel): + type: str \ No newline at end of file diff --git a/jjc_invoice.sh b/jjc_invoice.sh new file mode 100644 index 0000000..da46058 --- /dev/null +++ b/jjc_invoice.sh @@ -0,0 +1,2 @@ +# Example of how to run python code in this environment +poetry run python etl/jjc_invoice.py \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index 273b7a4..335f979 100644 --- a/poetry.lock +++ b/poetry.lock @@ -437,6 +437,21 @@ files = [ [package.extras] tests = ["asttokens (>=2.1.0)", "coverage", "coverage-enable-subprocess", "ipython", "littleutils", "pytest", "rich ; python_version >= \"3.11\""] +[[package]] +name = "fuzzywuzzy" +version = "0.18.0" +description = "Fuzzy string matching in python" +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "fuzzywuzzy-0.18.0-py2.py3-none-any.whl", hash = "sha256:928244b28db720d1e0ee7587acf660ea49d7e4c632569cad4f1cd7e68a5f0993"}, + {file = "fuzzywuzzy-0.18.0.tar.gz", hash = "sha256:45016e92264780e58972dca1b3d939ac864b78437422beecebb3095f8efd00e8"}, +] + +[package.extras] +speedup = ["python-levenshtein (>=0.12)"] + [[package]] name = "idna" version = "3.10" @@ -1463,4 +1478,4 @@ files = [ [metadata] lock-version = "2.1" python-versions = ">=3.12" -content-hash = "7c7fb2198bf2cb04e0af34fa6769280fda46907a2024b8f4c188847962964631" +content-hash = "46fe0a96958fc7f51391f94b0c4ba7fd4e85c34dcdf900667f24279c441d4fc0" diff --git a/pyproject.toml b/pyproject.toml index 344a5ab..a4a0680 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ dependencies = [ "pandas (>=2.2.3,<3.0.0)", "pydantic[email] (>=2.10.6,<3.0.0)", "openpyxl (>=3.1.5,<4.0.0)", + "fuzzywuzzy (>=0.18.0,<0.19.0)", ] [tool.poetry]