From 4c62360d9c6b328d0f88cc444d9fb433f522a808 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 17 Mar 2025 21:51:26 +0000 Subject: [PATCH 1/8] get rid of reduntnat prints --- etl/scraper/scraper.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/etl/scraper/scraper.py b/etl/scraper/scraper.py index cfa5481..a75771c 100644 --- a/etl/scraper/scraper.py +++ b/etl/scraper/scraper.py @@ -225,8 +225,6 @@ class SharePointScraper(): else: self.surveyor_work_completed.update({name: 1}) break - print("trololol") - return self.surveyor_work_completed @ensure_housing_assosiation_is_loaded From 213e3741527f79fb8d43f1c40fe1b5cb1979b7a4 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 18 Mar 2025 11:18:51 +0000 Subject: [PATCH 2/8] try running 03.03.2025@ --- etl/scraper/scraper.py | 1 + 1 file changed, 1 insertion(+) diff --git a/etl/scraper/scraper.py b/etl/scraper/scraper.py index a75771c..d5ccbbe 100644 --- a/etl/scraper/scraper.py +++ b/etl/scraper/scraper.py @@ -14,6 +14,7 @@ from datetime import datetime, timedelta def previous_monday(): today = datetime.today() last_monday = today - timedelta(days=today.weekday() + 7) # Go back to last week's Monday + return "W.C. 03.03.2025" # Quick one off script for nic. Hopefully get 199 Do not merge!!!! return f"W.C. {last_monday.strftime('%d.%m.%Y')}" WEEK_COMMENCING = os.getenv("WEEK_COMMENCING", previous_monday()) From 575931522336b304a7d8be19bc4bd8cac8e10939 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 19 Mar 2025 09:14:26 +0000 Subject: [PATCH 3/8] added a new script --- etl/daily_script.py | 2 +- etl/scis_invoice.py | 9 +++++++++ scis_invoice.sh | 2 ++ 3 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 etl/scis_invoice.py create mode 100644 scis_invoice.sh diff --git a/etl/daily_script.py b/etl/daily_script.py index 7b1ab7e..b2c76d9 100644 --- a/etl/daily_script.py +++ b/etl/daily_script.py @@ -57,7 +57,7 @@ def main(): if b_names: logger.info("Baxter Kelly with wrong date format") logger.info(pformat(b_names)) - logger.info("-------------------------------------------") + logger.info("-") logger.info("------EACH PRE SITE NOTES SUBMISSIONS------") logger.info("-------------------------------------------") logger.info(f"For week commencing: {WEEK_COMMENCING}") diff --git a/etl/scis_invoice.py b/etl/scis_invoice.py new file mode 100644 index 0000000..605c86e --- /dev/null +++ b/etl/scis_invoice.py @@ -0,0 +1,9 @@ +def main(): + print("hello world") + + +if __name__ == "__main__": + """ + Hopefully a very small script to generate a CSV and any othe relevant information to work out the invoice for each work + """ + main() diff --git a/scis_invoice.sh b/scis_invoice.sh new file mode 100644 index 0000000..bf168a0 --- /dev/null +++ b/scis_invoice.sh @@ -0,0 +1,2 @@ +# Example of how to run python code in this environment +poetry run python etl/scis_invoice.py \ No newline at end of file From be208961d3c600f820ec4f18a545bb461ac32509 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 19 Mar 2025 11:48:44 +0000 Subject: [PATCH 4/8] save existing work --- etl/pdfReader/reportType.py | 4 +++- etl/scis_invoice.py | 20 ++++++++++++++++---- etl/scraper/scraper.py | 11 ++++++++--- 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/etl/pdfReader/reportType.py b/etl/pdfReader/reportType.py index 6433d79..90bd5f4 100644 --- a/etl/pdfReader/reportType.py +++ b/etl/pdfReader/reportType.py @@ -2,4 +2,6 @@ from enum import Enum class ReportType(Enum): - QUIDOS_PRESITE_NOTE = 1 \ No newline at end of file + QUIDOS_PRESITE_NOTE = 1 + CHARTED_SURVEYOR_REPORT = 2 + ENERGY_PERFORMANCE_REPORT = 3 \ No newline at end of file diff --git a/etl/scis_invoice.py b/etl/scis_invoice.py index 605c86e..5d478cc 100644 --- a/etl/scis_invoice.py +++ b/etl/scis_invoice.py @@ -1,9 +1,21 @@ +from etl.scraper.scraper import SharePointScraper, SharePointInstaller +from pprint import pformat + + +def get_type_of_file(path): + pass + def main(): - print("hello world") + south_coast_scraper = SharePointScraper(SharePointInstaller.SOUTH_COAST_INSULATION, development=True) + file_paths = south_coast_scraper.download_file_for_each_address() + + print(pformat(file_paths)) + + for eachAddress in file_paths: + print(eachAddress) + + # Download presite notes and add it to pdf reader class if __name__ == "__main__": - """ - Hopefully a very small script to generate a CSV and any othe relevant information to work out the invoice for each work - """ main() diff --git a/etl/scraper/scraper.py b/etl/scraper/scraper.py index d5ccbbe..4568a5a 100644 --- a/etl/scraper/scraper.py +++ b/etl/scraper/scraper.py @@ -230,6 +230,7 @@ class SharePointScraper(): @ensure_housing_assosiation_is_loaded def download_file_for_each_address(self): + paths = [] for name in self.surveyor_names: if WEEK_COMMENCING in self.surveyor_to_dates_folder[name]: for house_ass in self.surveyor_to_housing_assosications[name]: @@ -242,7 +243,7 @@ class SharePointScraper(): if 'file' not in address: # Only directories allAddress.append(address['name']) - + address_paths = {} for address in allAddress: path = f"/{name}/{WEEK_COMMENCING}/{house_ass}/{address}" files_to_download_sharepoint_info = self.get_folders_in_path(path) @@ -257,11 +258,15 @@ class SharePointScraper(): if any(file["name"].endswith(ext) for ext in avoid): continue file_names_to_download.update({file["name"]: file['@microsoft.graph.downloadUrl']}) - + each_file = [] for file_name, url in file_names_to_download.items(): self.logger.info(f"Downloading {file_name} from {url}") content = self.get_file_content(url) - self.create_temp_file(content, f"{name}/{WEEK_COMMENCING}/{house_ass}/{address}/{file_name}") + file_path = self.create_temp_file(content, f"{name}/{WEEK_COMMENCING}/{house_ass}/{address}/{file_name}") + each_file.append(file_path) + address_paths.update({address: each_file}) + paths.append(address_paths) + return paths def create_temp_file(self, content, path): # Ensure the path is under /tmp/ From 32fb397ca14cb3ca4912f2d39567d481ff8de5d6 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 19 Mar 2025 12:34:03 +0000 Subject: [PATCH 5/8] survery data processor --- etl/pdfReader/pdfReaderToText.py | 5 ++--- etl/scis_invoice.py | 8 +++----- etl/scraper/scraper.py | 4 ++-- etl/surveyedData/__init__.py | 0 etl/surveyedData/surveryedData.py | 6 ++++++ 5 files changed, 13 insertions(+), 10 deletions(-) create mode 100644 etl/surveyedData/__init__.py create mode 100644 etl/surveyedData/surveryedData.py diff --git a/etl/pdfReader/pdfReaderToText.py b/etl/pdfReader/pdfReaderToText.py index 16adee0..350b31a 100644 --- a/etl/pdfReader/pdfReaderToText.py +++ b/etl/pdfReader/pdfReaderToText.py @@ -31,10 +31,9 @@ class pdfReaderToText(): if len(self.text_list) > 1: if "Quidos Ltd using Argyle software BRE approved calculator".lower() in self.text_list[0].lower(): self.type = ReportType.QUIDOS_PRESITE_NOTE - return self.type else: - return None - # raise NotImplementedError("New type of file - please contact Jun-te Kim") + pass + return self.type def get_reader(self): self.get_file_type() diff --git a/etl/scis_invoice.py b/etl/scis_invoice.py index 5d478cc..dd91364 100644 --- a/etl/scis_invoice.py +++ b/etl/scis_invoice.py @@ -1,5 +1,7 @@ from etl.scraper.scraper import SharePointScraper, SharePointInstaller from pprint import pformat +from etl.pdfReader.pdfReaderToText import pdfReaderToText +from etl.surveyedData.surveryedData import surveyedDataProcessor def get_type_of_file(path): @@ -9,12 +11,8 @@ def main(): south_coast_scraper = SharePointScraper(SharePointInstaller.SOUTH_COAST_INSULATION, development=True) file_paths = south_coast_scraper.download_file_for_each_address() - print(pformat(file_paths)) - for eachAddress in file_paths: - print(eachAddress) - - # Download presite notes and add it to pdf reader class + survey = surveyedDataProcessor(eachAddress) if __name__ == "__main__": diff --git a/etl/scraper/scraper.py b/etl/scraper/scraper.py index 4568a5a..17ad03f 100644 --- a/etl/scraper/scraper.py +++ b/etl/scraper/scraper.py @@ -243,9 +243,9 @@ class SharePointScraper(): if 'file' not in address: # Only directories allAddress.append(address['name']) - address_paths = {} - for address in allAddress: + for i, address in enumerate(allAddress): path = f"/{name}/{WEEK_COMMENCING}/{house_ass}/{address}" + address_paths = {} files_to_download_sharepoint_info = self.get_folders_in_path(path) if 'value' not in files_to_download_sharepoint_info: raise RuntimeError("Failed to get files to download") diff --git a/etl/surveyedData/__init__.py b/etl/surveyedData/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/etl/surveyedData/surveryedData.py b/etl/surveyedData/surveryedData.py new file mode 100644 index 0000000..a47bfe0 --- /dev/null +++ b/etl/surveyedData/surveryedData.py @@ -0,0 +1,6 @@ +class surveyedDataProcessor(): + def __init__(self, address_to_files): + for key, value in address_to_files.items(): + self.address = key + self.files = value + print(f"Address is {self.address}, with all files at location {self.files}") From 9f9abe2280270fb23e76f2164c77e5ed60f63140 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 19 Mar 2025 14:20:26 +0000 Subject: [PATCH 6/8] time to find out if it works for all --- etl/pdfReader/pdfReaderToText.py | 18 ++++++++++++--- etl/pdfReader/reportType.py | 4 +++- etl/pdfReader/sitenotes.py | 7 +++++- etl/scis_invoice.py | 38 +++++++++++++++++++++++++++---- etl/scraper/scraper.py | 3 +++ etl/surveyedData/surveryedData.py | 27 ++++++++++++++++++---- poetry.lock | 29 ++++++++++++++++++++++- pyproject.toml | 1 + 8 files changed, 112 insertions(+), 15 deletions(-) diff --git a/etl/pdfReader/pdfReaderToText.py b/etl/pdfReader/pdfReaderToText.py index 350b31a..9632296 100644 --- a/etl/pdfReader/pdfReaderToText.py +++ b/etl/pdfReader/pdfReaderToText.py @@ -1,7 +1,7 @@ from etl.utils.logger import Logger import logging import pymupdf -from etl.pdfReader.sitenotes import QuidosSiteNotesExtractor +from etl.pdfReader.sitenotes import QuidosSiteNotesExtractor, CSR from etl.pdfReader.reportType import ReportType class pdfReaderToText(): @@ -13,6 +13,7 @@ class pdfReaderToText(): self.text_list = [] self.get_text_from_pdf_file() self.type = None + self.get_file_type() def get_text_from_pdf_file(self): self.logger.debug(f"Extrating text from {self.source_path}") @@ -31,6 +32,14 @@ class pdfReaderToText(): if len(self.text_list) > 1: if "Quidos Ltd using Argyle software BRE approved calculator".lower() in self.text_list[0].lower(): self.type = ReportType.QUIDOS_PRESITE_NOTE + elif "Wall pre - Masonry cavity wall-unĮlled".lower() in self.text_list[0].lower(): + self.type = ReportType.U_VALUE_CALCULATOR_REPORT + elif "Overwriting U-Values for EPRs for ECO4 and GBIS:" in self.text_list[1].lower(): + self.type = ReportType.OVERWRITING_U_VALUE_DECLARATION_FORM + elif "Energy Performance Report" in self.text_list: + self.type = ReportType.ENERGY_PERFORMANCE_REPORT + elif "Chartered Surveyor Report: Recommending Extraction of Defective Cavity Wall Insulation " in self.text_list: + self.type = ReportType.CHARTED_SURVEYOR_REPORT else: pass return self.type @@ -38,5 +47,8 @@ class pdfReaderToText(): def get_reader(self): self.get_file_type() - if self.type.name == ReportType.QUIDOS_SITE_NOTE.name: - return QuidosSiteNotesExtractor(self.text_list) \ No newline at end of file + if self.type.name == ReportType.QUIDOS_PRESITE_NOTE.name: + return QuidosSiteNotesExtractor(self.text_list) + elif self.type == ReportType.CHARTED_SURVEYOR_REPORT: + return CSR(self.text_list) + \ No newline at end of file diff --git a/etl/pdfReader/reportType.py b/etl/pdfReader/reportType.py index 90bd5f4..25c5b22 100644 --- a/etl/pdfReader/reportType.py +++ b/etl/pdfReader/reportType.py @@ -4,4 +4,6 @@ from enum import Enum class ReportType(Enum): QUIDOS_PRESITE_NOTE = 1 CHARTED_SURVEYOR_REPORT = 2 - ENERGY_PERFORMANCE_REPORT = 3 \ No newline at end of file + ENERGY_PERFORMANCE_REPORT = 3 + U_VALUE_CALCULATOR_REPORT = 4 + OVERWRITING_U_VALUE_DECLARATION_FORM = 5 \ No newline at end of file diff --git a/etl/pdfReader/sitenotes.py b/etl/pdfReader/sitenotes.py index 4161395..bbfcfb2 100644 --- a/etl/pdfReader/sitenotes.py +++ b/etl/pdfReader/sitenotes.py @@ -22,13 +22,18 @@ class SiteNotesExtractor(): def get_data_between(self, a, b): return self.raw_data[self.raw_data.index(a):self.raw_data.index(b)] + +class CSR(SiteNotesExtractor): + def __init__(self, data_list): + super().__init__(data_list) + self.type = ReportType.CHARTED_SURVEYOR_REPORT class QuidosSiteNotesExtractor(SiteNotesExtractor): def __init__(self, data_list): super().__init__(data_list) - self.type = ReportType.QUIDOS_SITE_NOTE + self.type = ReportType.QUIDOS_PRESITE_NOTE self.company_information = None self.survey_information = None self.property_description = None diff --git a/etl/scis_invoice.py b/etl/scis_invoice.py index dd91364..1b77b2e 100644 --- a/etl/scis_invoice.py +++ b/etl/scis_invoice.py @@ -2,17 +2,47 @@ from etl.scraper.scraper import SharePointScraper, SharePointInstaller from pprint import pformat from etl.pdfReader.pdfReaderToText import pdfReaderToText from etl.surveyedData.surveryedData import surveyedDataProcessor +import pandas as pd + -def get_type_of_file(path): - pass def main(): + data = { + "Address": [], + "Surveyor's Name": [], + "Type of Work": [], + "Price": [] + } + south_coast_scraper = SharePointScraper(SharePointInstaller.SOUTH_COAST_INSULATION, development=True) file_paths = south_coast_scraper.download_file_for_each_address() - + + list_of_surveys = [] for eachAddress in file_paths: - survey = surveyedDataProcessor(eachAddress) + for address, files in eachAddress.items(): + list_of_surveys.append(surveyedDataProcessor(address, files)) + break + + + for survey in list_of_surveys: + if survey.pre_site_note: + if survey.csr: + data["Price"].append(500) + data["Type of Work"].append("CAVITY ONLY") + else: + data["Price"].append(1000) + data["Type of Work"].append("REMIDIAL CWI ONLY") + + data["Address"].append(survey.address) + data["Surveyor's Name"].append(survey.pre_site) + + df = pd.DataFrame(data) + + # Save to an Excel file + df.to_excel("survey_data.xlsx", index=False) + + print("Excel file 'survey_data.xlsx' created successfully!") if __name__ == "__main__": diff --git a/etl/scraper/scraper.py b/etl/scraper/scraper.py index 17ad03f..72c4edc 100644 --- a/etl/scraper/scraper.py +++ b/etl/scraper/scraper.py @@ -58,6 +58,9 @@ class SharePointScraper(): self.surveyor_to_housing_assosications = {"Abdul Koddus":['Southern Housing']} self.surveyor_to_dates_folder = {'Abdul Koddus': ['W.C. 03.03.2025', 'W.C. 24.02.2025']} + self.surveyor_names = ['Carl Fitzgerald'] + self.surveyor_to_housing_assosications = {"Carl Fitzgerald":['ACIS']} + self.surveyor_to_dates_folder = {'Carl Fitzgerald': ['W.C. 03.03.2025']} diff --git a/etl/surveyedData/surveryedData.py b/etl/surveyedData/surveryedData.py index a47bfe0..338ee33 100644 --- a/etl/surveyedData/surveryedData.py +++ b/etl/surveyedData/surveryedData.py @@ -1,6 +1,23 @@ +from etl.pdfReader.pdfReaderToText import pdfReaderToText +from etl.pdfReader.reportType import ReportType + class surveyedDataProcessor(): - def __init__(self, address_to_files): - for key, value in address_to_files.items(): - self.address = key - self.files = value - print(f"Address is {self.address}, with all files at location {self.files}") + def __init__(self, address, files): + self.address = address + self.files = files + self.pre_site_note = None + self.csr = None + self.identify_files() + + + def identify_files(self): + for file in self.files: + pdf = pdfReaderToText(file) + print("Junte was here") + print(file) + print(pdf.text_list) + if pdf: + if pdf.type == ReportType.QUIDOS_PRESITE_NOTE: + self.pre_site_note = pdf.get_reader() + elif pdf.type == ReportType.CHARTED_SURVEYOR_REPORT: + self.csr = pdf.get_reader() diff --git a/poetry.lock b/poetry.lock index 90ea6c7..273b7a4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -410,6 +410,18 @@ files = [ dnspython = ">=2.0.0" idna = ">=2.0.0" +[[package]] +name = "et-xmlfile" +version = "2.0.0" +description = "An implementation of lxml.xmlfile for the standard library" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa"}, + {file = "et_xmlfile-2.0.0.tar.gz", hash = "sha256:dab3f4764309081ce75662649be815c4c9081e88f0837825f90fd28317d4da54"}, +] + [[package]] name = "executing" version = "2.2.0" @@ -697,6 +709,21 @@ files = [ {file = "numpy-2.2.3.tar.gz", hash = "sha256:dbdc15f0c81611925f382dfa97b3bd0bc2c1ce19d4fe50482cb0ddc12ba30020"}, ] +[[package]] +name = "openpyxl" +version = "3.1.5" +description = "A Python library to read/write Excel 2010 xlsx/xlsm files" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2"}, + {file = "openpyxl-3.1.5.tar.gz", hash = "sha256:cf0e3cf56142039133628b5acffe8ef0c12bc902d2aadd3e0fe5878dc08d1050"}, +] + +[package.dependencies] +et-xmlfile = "*" + [[package]] name = "packaging" version = "24.2" @@ -1436,4 +1463,4 @@ files = [ [metadata] lock-version = "2.1" python-versions = ">=3.12" -content-hash = "710051703d97e156a540ad08b0815338a4283146f6fca3c0ae89cc4e6dad459a" +content-hash = "7c7fb2198bf2cb04e0af34fa6769280fda46907a2024b8f4c188847962964631" diff --git a/pyproject.toml b/pyproject.toml index 4d3d01b..344a5ab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,7 @@ dependencies = [ "msal (>=1.31.1,<2.0.0)", "pandas (>=2.2.3,<3.0.0)", "pydantic[email] (>=2.10.6,<3.0.0)", + "openpyxl (>=3.1.5,<4.0.0)", ] [tool.poetry] From 62b8550e49ae2de5c30e1518b96df368871804bd Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 19 Mar 2025 14:22:22 +0000 Subject: [PATCH 7/8] new work flow --- .github/workflows/scis_invoice_calculator.yml | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/scis_invoice_calculator.yml diff --git a/.github/workflows/scis_invoice_calculator.yml b/.github/workflows/scis_invoice_calculator.yml new file mode 100644 index 0000000..49a5ae3 --- /dev/null +++ b/.github/workflows/scis_invoice_calculator.yml @@ -0,0 +1,33 @@ +on: + schedule: + - cron: '0 9 * * *' + workflow_dispatch: + +jobs: + scis_invoice_calculator: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.12' + + - name: Install dependencies + run: | + pip install poetry + poetry install --no-root + + - name: run script + run: | + bash scis_invoice.sh + env: + PYTHONPATH: ${{ github.workspace }} + SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID: ${{ secrets.SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID }} + JJC_SERVICE_SHAREPOINT_ID: ${{ secrets.JJC_SERVICE_SHAREPOINT_ID }} + BAXTER_KELLY_SERVICE_SHAREPOINT_ID: ${{ secrets.BAXTER_KELLY_SERVICE_SHAREPOINT_ID }} + SGEC_SERVICE_SHAREPOINT_ID: ${{ secrets.SGEC_SERVICE_SHAREPOINT_ID }} + SHAREPOINT_CLIENT_ID: ${{ secrets.SHAREPOINT_CLIENT_ID }} + SHAREPOINT_CLIENT_SECRET: ${{ secrets.SHAREPOINT_CLIENT_SECRET }} + SHAREPOINT_TENANT_ID: ${{ secrets.SHAREPOINT_TENANT_ID }} \ No newline at end of file From 6863fc98fd63b9e7aebf02afd68c59aedd36b9b2 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 19 Mar 2025 14:23:31 +0000 Subject: [PATCH 8/8] need to merge to develope to get script working with live --- etl/scraper/scraper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etl/scraper/scraper.py b/etl/scraper/scraper.py index 72c4edc..fdbc739 100644 --- a/etl/scraper/scraper.py +++ b/etl/scraper/scraper.py @@ -14,7 +14,7 @@ from datetime import datetime, timedelta def previous_monday(): today = datetime.today() last_monday = today - timedelta(days=today.weekday() + 7) # Go back to last week's Monday - return "W.C. 03.03.2025" # Quick one off script for nic. Hopefully get 199 Do not merge!!!! + # return "W.C. 03.03.2025" # Quick one off script for nic. Hopefully get 199 Do not merge!!!! return f"W.C. {last_monday.strftime('%d.%m.%Y')}" WEEK_COMMENCING = os.getenv("WEEK_COMMENCING", previous_monday())