From 4e6181e413b7f0333aeb765166cbc87f7f758e75 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 19 Mar 2025 14:51:50 +0000 Subject: [PATCH 1/4] sharepoint version --- etl/pdfReader/sitenotes.py | 10 +++++----- etl/scraper/scraper.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/etl/pdfReader/sitenotes.py b/etl/pdfReader/sitenotes.py index 72200e3..fd5efe9 100644 --- a/etl/pdfReader/sitenotes.py +++ b/etl/pdfReader/sitenotes.py @@ -710,8 +710,8 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): model_name=dict_.get("model_name", ""), model_qualifer=dict_.get("model_qualifier", ""), controls=HeatingSystemControls( - control_type=dict_.get("control_type", ""), - flue_type=dict_.get("flue_type",""), + control_type=dict_.get("control_type", "") if dict_.get("control_type", "") is not None else "", + flue_type=dict_.get("flue_type","") if dict_.get("flue_type", "") is not None else "", fan_assisted_flue=True if dict_.get("fan_assisted_flue", "NO").upper() == "YES" else False, heat_emitter_type=dict_.get("heat_emitter_type", "") if dict_.get("heat_emitter_type") is not None else "", electricity_meter_type=dict_.get("electricity_meter_type", ""), @@ -751,11 +751,11 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): model_name=dict_.get("model_name", ""), model_qualifer=dict_.get("model_qualifier", ""), controls=HeatingSystemControls( - control_type=dict_.get("control_type", ""), - flue_type=dict_.get("flue_type",""), + control_type=dict_.get("control_type", "") if dict_.get("control_type", "") is not None else "", + flue_type=dict_.get("flue_type","") if dict_.get("flue_type", "") is not None else "", fan_assisted_flue=True if dict_.get("fan_assisted_flue", "NO").upper() == "YES" else False, heat_emitter_type=dict_.get("heat_emitter_type", "") if dict_.get("heat_emitter_type") is not None else "", - electricity_meter_type=dict_.get("electricity_meter_type", ""), + electricity_meter_type=dict_.get("electricity_meter_type", "") if dict_.get("electricity_meter_type", "") is not None else "", mains_gas_available=True if dict_.get("mains_gas_available", "NO").upper() == "YES" else False, ) ) diff --git a/etl/scraper/scraper.py b/etl/scraper/scraper.py index fdbc739..72c4edc 100644 --- a/etl/scraper/scraper.py +++ b/etl/scraper/scraper.py @@ -14,7 +14,7 @@ from datetime import datetime, timedelta def previous_monday(): today = datetime.today() last_monday = today - timedelta(days=today.weekday() + 7) # Go back to last week's Monday - # return "W.C. 03.03.2025" # Quick one off script for nic. Hopefully get 199 Do not merge!!!! + return "W.C. 03.03.2025" # Quick one off script for nic. Hopefully get 199 Do not merge!!!! return f"W.C. {last_monday.strftime('%d.%m.%Y')}" WEEK_COMMENCING = os.getenv("WEEK_COMMENCING", previous_monday()) From 3ab438d56db46d01ade2be481156d4a6ecaf3a33 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 19 Mar 2025 14:52:21 +0000 Subject: [PATCH 2/4] sharepoint version --- etl/scis_invoice.py | 1 + 1 file changed, 1 insertion(+) diff --git a/etl/scis_invoice.py b/etl/scis_invoice.py index 13c0b7d..0f57572 100644 --- a/etl/scis_invoice.py +++ b/etl/scis_invoice.py @@ -43,6 +43,7 @@ def main(): df.to_excel("survey_data.xlsx", index=False) print("Excel file 'survey_data.xlsx' created successfully!") + print(pformat(data)) if __name__ == "__main__": From 9ffa8ade0985546e9d16d80925035bf0df0f3f8d Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 19 Mar 2025 15:29:11 +0000 Subject: [PATCH 3/4] script for non remidial --- etl/deemed_report_generator.py | 30 ----- etl/development.py | 219 ++++++++++++++++++++++++++++----- etl/scis_invoice.py | 14 +-- etl/scraper/scraper.py | 8 +- 4 files changed, 192 insertions(+), 79 deletions(-) delete mode 100644 etl/deemed_report_generator.py diff --git a/etl/deemed_report_generator.py b/etl/deemed_report_generator.py deleted file mode 100644 index fa76f75..0000000 --- a/etl/deemed_report_generator.py +++ /dev/null @@ -1,30 +0,0 @@ -import os -from pdfReader.pdfReaderToText import pdfReaderToText -from etl.scraper.scraper import SharePointScraper, SharePointInstaller -from pprint import pprint, pformat -import logging -from etl.utils.logger import Logger -from etl.validator.validator import DomnaSharePointValidator - -logger = Logger(name="main.py", level=logging.DEBUG).get_logger() -DATA_LOC_1 = "/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/10 Turnberry Close TN38 0WL/PRE SITE NOTES.pdf" -DATA_LOC_2 = "/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/16 Sunningdale Drive TN38 0WB/PRE SITE NOTES.pdf" - -pdfReader = pdfReaderToText(DATA_LOC_1) -doc2 = pdfReader.get_reader() -pdfReader2 = pdfReaderToText(DATA_LOC_2) -doc1 = pdfReader2.get_reader() -vars(doc1) - -def main(): - pass - # POC of downloading each file - south_coast_scraper = SharePointScraper(SharePointInstaller.SOUTH_COAST_INSULATION_SERVICE) - south_coast_scraper.download_file_for_each_address() - - # POC of pdf reader - - - -if __name__ == "__main__": - main() diff --git a/etl/development.py b/etl/development.py index 58bfd6b..e43d105 100644 --- a/etl/development.py +++ b/etl/development.py @@ -1,42 +1,195 @@ -import os -from pdfReader.pdfReaderToText import pdfReaderToText from etl.scraper.scraper import SharePointScraper, SharePointInstaller -from pprint import pprint, pformat -import logging -from etl.utils.logger import Logger -from etl.validator.validator import DomnaSharePointValidator - -logger = Logger(name="main.py", level=logging.DEBUG).get_logger() -DATA_LOC_1 = "/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/10 Turnberry Close TN38 0WL/PRE SITE NOTES.pdf" -DATA_LOC_2 = "/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/16 Sunningdale Drive TN38 0WB/PRE SITE NOTES.pdf" - -# Extract and trasform -pdfReader = pdfReaderToText(DATA_LOC_1) -doc2 = pdfReader.get_reader() -pdfReader2 = pdfReaderToText(DATA_LOC_2) -doc1 = pdfReader2.get_reader() -print(doc2.property_description.waterHeating) +from pprint import pformat +from etl.pdfReader.pdfReaderToText import pdfReaderToText +from etl.surveyedData.surveryedData import surveyedDataProcessor +import pandas as pd + -# Transform def main(): - # south_coast_scraper = SharePointScraper(SharePointInstaller.SOUTH_COAST_INSULATION_SERVICE, development = True) - # south_coast_scraper.download_file_for_each_address() - pass + data = { + "Address": [], + "Surveyor's Name": [], + "Type of Work": [], + "Price": [] + } + + #south_coast_scraper = SharePointScraper(SharePointInstaller.SOUTH_COAST_INSULATION, development=True) + #file_paths = south_coast_scraper.download_file_for_each_address() + + file_paths = [{'10 Turnberry Close TN38 0WL': ['/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/10 Turnberry Close TN38 0WL/10 Turnberry Close, TN38 0WL UValue.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/10 Turnberry Close TN38 0WL/Dec - 10 Turnberry Close, TN38 0WL.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/10 Turnberry Close TN38 0WL/PRE EPR.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/10 Turnberry Close TN38 0WL/PRE SAP.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/10 Turnberry Close TN38 0WL/PRE SITE NOTES.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/10 Turnberry Close TN38 0WL/PRE XML.xml']}, + {'15 Turnberry Close TN38 0WL': ['/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/15 Turnberry Close TN38 0WL/15 Turnberry Close, TN38 0WL UValue.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/15 Turnberry Close TN38 0WL/Dec - 15 Turnberry Close, TN38 0WL.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/15 Turnberry Close TN38 0WL/PRE EPR.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/15 Turnberry Close TN38 0WL/PRE SAP.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/15 Turnberry Close TN38 0WL/PRE SITE NOTES.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/15 Turnberry Close TN38 0WL/PRE XML.xml']}, + {'16 Sunningdale Drive TN38 0WB': ['/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/16 Sunningdale Drive TN38 0WB/16 Sunningdale Drive, TN38 0WD Uvalue.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/16 Sunningdale Drive TN38 0WB/Dec - 16 Sunningdale Drive, TN38 0WD.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/16 Sunningdale Drive TN38 0WB/PRE EPR.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/16 Sunningdale Drive TN38 0WB/PRE SAP.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/16 Sunningdale Drive TN38 0WB/PRE SITE NOTES.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/16 Sunningdale Drive TN38 0WB/PRE XML.xml']}, + {'17 Sunningdale Drive TN38 0WB': ['/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/17 Sunningdale Drive TN38 0WB/17 Sunningdale Drive, TN38 0WD Uvalue.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/17 Sunningdale Drive TN38 0WB/Dec - 17 Sunningdale Drive, TN38 0WD.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/17 Sunningdale Drive TN38 0WB/PRE EPR.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/17 Sunningdale Drive TN38 0WB/PRE SAP.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/17 Sunningdale Drive TN38 0WB/PRE SITE NOTES.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/17 Sunningdale Drive TN38 0WB/PRE XML.xml']}, + {'2 Badgers Court TN37 6JB': ['/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/2 Badgers Court TN37 6JB/2 Badgers Court, TN37 6JB UValue.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/2 Badgers Court TN37 6JB/Dec - 2 Badgers Court, TN37 6JB.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/2 Badgers Court TN37 6JB/PRE EPR.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/2 Badgers Court TN37 6JB/PRE SAP.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/2 Badgers Court TN37 6JB/PRE SITE SITES.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/2 Badgers Court TN37 6JB/PRE XML.xml', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/2 Badgers Court TN37 6JB/statement_10197841_70195407-2.pdf']}, + {'23 Sunningdale Drive TN38 0WD': ['/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/23 Sunningdale Drive TN38 0WD/23 Sunningdale Drive, TN38 0WD UValue.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/23 Sunningdale Drive TN38 0WD/Dec - 23 Sunningdale Drive, TN38 0WD.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/23 Sunningdale Drive TN38 0WD/PRE EPR.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/23 Sunningdale Drive TN38 0WD/PRE SAP.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/23 Sunningdale Drive TN38 0WD/PRE SITE NOTES.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/23 Sunningdale Drive TN38 0WD/PRE XML.xml']}, + {'3 Turnberry Close TN38 0WL': ['/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/3 Turnberry Close TN38 0WL/3 Turnberry Close, TN38 0WL UValue.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/3 Turnberry Close TN38 0WL/Dec - 3 Turnberry Close, TN38 0WL.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/3 Turnberry Close TN38 0WL/PRE EPR.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/3 Turnberry Close TN38 0WL/PRE SAP.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/3 Turnberry Close TN38 0WL/PRE SITE NOTES.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/3 Turnberry Close TN38 0WL/PRE XML.xml']}, + {'35 Sunningdale Drive TN38 0WD': ['/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/35 Sunningdale Drive TN38 0WD/35 Sunningdale Drive, TN38 0WD UValue.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/35 Sunningdale Drive TN38 0WD/Dec - 35 Sunningdale Drive, TN38 0WD.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/35 Sunningdale Drive TN38 0WD/PRE EPR.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/35 Sunningdale Drive TN38 0WD/PRE SAP.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/35 Sunningdale Drive TN38 0WD/PRE SITE NOTES.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/35 Sunningdale Drive TN38 0WD/PRE XML.xml']}, + {'43 Sunningdale Drive TN38 0WD': ['/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/43 Sunningdale Drive TN38 0WD/43 Sunningdale Drive, TN38 0WD UValue.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/43 Sunningdale Drive TN38 0WD/Dec - 43 Sunningdale Drive, TN38 0WD.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/43 Sunningdale Drive TN38 0WD/PRE EPR.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/43 Sunningdale Drive TN38 0WD/PRE SAP.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/43 Sunningdale Drive TN38 0WD/PRE SITE NOTES.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/43 Sunningdale Drive TN38 0WD/PRE XML.xml']}, + {'45 Sunningdale Drive TN38 0WD': ['/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/45 Sunningdale Drive TN38 0WD/45 Sunningdale Drive, TN38 0WD UValue.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/45 Sunningdale Drive TN38 0WD/Dec - 45 Sunningdale Drive, TN38 0WD.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/45 Sunningdale Drive TN38 0WD/PRE EPR.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/45 Sunningdale Drive TN38 0WD/PRE SAP.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/45 Sunningdale Drive TN38 0WD/PRE SITE NOTES.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/45 Sunningdale Drive TN38 0WD/PRE XML.xml']}, + {'7 Turnberry Close TN38 0WL': ['/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/7 Turnberry Close TN38 0WL/7 Turnberry Close, TN38 0WL UValue.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/7 Turnberry Close TN38 0WL/Dec - 7 Turnberry Close, TN38 0WL.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/7 Turnberry Close TN38 0WL/PRE EPR.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/7 Turnberry Close TN38 0WL/PRE SAP.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/7 Turnberry Close TN38 0WL/PRE SITE NOTES.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/7 Turnberry Close TN38 0WL/PRE XML.xml']}, + {'9 Turnberry Close TN38 0WL': ['/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/9 Turnberry Close TN38 0WL/9 Turnberry Close, TN38 0WL UValue.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/9 Turnberry Close TN38 0WL/Dec - 9 Turnberry Close, TN38 0WL.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/9 Turnberry Close TN38 0WL/invoice-225560.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/9 Turnberry Close TN38 0WL/PRE EPR.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/9 Turnberry Close TN38 0WL/PRE SAP.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/9 Turnberry Close TN38 0WL/PRE SITE NOTES.pdf', + '/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/9 Turnberry Close TN38 0WL/PRE XML.xml']}, + {'10 Chapel Lane': ['/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/10 Chapel Lane/10 Chapel Lane PRE EPR PDF.pdf', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/10 Chapel Lane/10 Chapel Lane PRE EPR RDSAP.xml', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/10 Chapel Lane/10 Chapel Lane PRE EPR SITE NOTES.pdf', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/10 Chapel Lane/10 Chapel.pdf']}, + {'10 South Drive': ['/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/10 South Drive/10 South Drive PRE EPR PDF.pdf', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/10 South Drive/10 South Drive PRE EPR RDSAP.xml', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/10 South Drive/10 South Drive PRE EPR SITE NOTES.pdf', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/10 South Drive/10 South.pdf']}, + {'11 Dorset Place': ['/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/11 Dorset Place/11 Dorset Place PRE EPR PDF.pdf', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/11 Dorset Place/11 Dorset Place PRE EPR RDSAP.xml', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/11 Dorset Place/11 Dorset Place PRE EPR SITE NOTES.pdf', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/11 Dorset Place/11 Dorset.pdf']}, + {'11 Swan Close': ['/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/11 Swan Close/11 Swan Close PRE EPR PDF.pdf', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/11 Swan Close/11 Swan Close PRE EPR RDSAP.xml', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/11 Swan Close/11 Swan Close PRE EPR SITE NOTES.pdf', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/11 Swan Close/11 Swan.pdf']}, + {'12 Ingham Road': ['/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/12 Ingham Road/12 Ingham Road PRE EPR PDF.pdf', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/12 Ingham Road/12 Ingham Road PRE EPR RDSAP.xml', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/12 Ingham Road/12 Ingham Road PRE EPR SITE NOTES.pdf', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/12 Ingham Road/12 Ingham.pdf']}, + {'13 Morris Close': ['/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/13 Morris Close/13 Morris Close PRE EPR PDF.pdf', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/13 Morris Close/13 Morris Close PRE EPR RDSAP.xml', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/13 Morris Close/13 Morris Close PRE EPR SITE NOTES.pdf', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/13 Morris Close/13 Morris.pdf']}, + {'13 Swan Close': ['/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/13 Swan Close/13 Swan Close PRE EPR PDF.pdf', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/13 Swan Close/13 Swan Close PRE EPR RDSA.xml', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/13 Swan Close/13 Swan.pdf']}, + {'14 Ingham Road': ['/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/14 Ingham Road/14 Ingham Road PRE EPR PDF.pdf', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/14 Ingham Road/14 Ingham Road PRE EPR RDSAP.xml', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/14 Ingham Road/14 Ingham Road PRE EPR SITE NTES.pdf', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/14 Ingham Road/14 Ingham.pdf']}, + {'16 Station Road': ['/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/16 Station Road/16 Station Road PRE EPR PDF.pdf', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/16 Station Road/16 Station Road PRE EPR RDSAP.xml', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/16 Station Road/16 Station Road PRE EPR SITE NOTES.pdf', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/16 Station Road/16 Station.pdf']}, + {'16 Swan Close': ['/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/16 Swan Close/16 Swan Close PRE EPR PDF.pdf', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/16 Swan Close/16 Swan Close PRE EPR RDSAP.xml', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/16 Swan Close/16 Swan Close PRE EPR SITE NOTES.pdf', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/16 Swan Close/16 Swan.pdf']}, + {'2 Deane Walk': ['/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/2 Deane Walk/2 Deane Walk PRE EPR PDF.pdf', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/2 Deane Walk/2 Deane Walk PRE EPR RDSAP.xml', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/2 Deane Walk/2 Deane Walk PRE EPR SITE NOTES.pdf', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/2 Deane Walk/2 Deane.pdf']}, + {'25 High Leas': ['/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/25 High Leas/25 High Leas PRE EPR PDF.pdf', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/25 High Leas/25 High Leas PRE EPR RDSAP.xml', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/25 High Leas/25 High Leas PRE EPR SITE NOTES.pdf', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/25 High Leas/25 High.pdf']}, + {'26 Fossdyke Gardens': ['/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/26 Fossdyke Gardens/26 Fossdyke Gardens PRE EPR PDF.pdf', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/26 Fossdyke Gardens/26 Fossdyke Gardens PRE EPR RDSAP.xml', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/26 Fossdyke Gardens/26 Fossdyke Gardens PRE EPR SITE NOTES.pdf', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/26 Fossdyke Gardens/26 Fossdyke.pdf']}, + {'28 Station Road': ['/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/28 Station Road/28 Station Road PRE EPR PDF.pdf', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/28 Station Road/28 Station Road PRE EPR RDSAP.xml', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/28 Station Road/28 Station Road PRE EPR SITE NOTES.pdf', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/28 Station Road/28 Station.pdf']}, + {'30 Saxilby Road': ['/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/30 Saxilby Road/30 Saxilby Road PRE EPR RDSAP.xml', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/30 Saxilby Road/30 Saxilby Road PRE EPR SITE NOTES.pdf', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/30 Saxilby Road/30 Saxilby.pdf']}, + {'5 Queensway': ['/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/5 Queensway/5 Queensway PRE EPR PDF.pdf', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/5 Queensway/5 Queensway PRE EPR RDSAP.xml', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/5 Queensway/5 Queensway PRE EPR SITE NOTES.pdf', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/5 Queensway/5-Queensway.pdf']}, + {'7 Queensway': ['/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/7 Queensway/7 Queensway PRE EPR PDF.pdf', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/7 Queensway/7 Queensway PRE EPR RDSAP.xml', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/7 Queensway/7 Queensway PRE EPR SITE NOTES.pdf', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/7 Queensway/7-Queensway.pdf']}, + {'9 Bridge Place': ['/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/9 Bridge Place/9 Bridge Place PRE EPR PDF.pdf', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/9 Bridge Place/9 Bridge Place PRE EPR RDSAP.xml', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/9 Bridge Place/9 Bridge Place PRE EPR SITE NOTES.pdf', + '/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/9 Bridge Place/9 Bridge.pdf']}] + + list_of_surveys = [] + for eachAddress in file_paths: + for address, files in eachAddress.items(): + list_of_surveys.append(surveyedDataProcessor(address, files)) + break + + + for survey in list_of_surveys: + if survey.pre_site_note: + if survey.csr: + data["Price"].append(500) + data["Type of Work"].append("REMIDIAL CWI ONLY") + else: + data["Price"].append(1000) + data["Type of Work"].append("CAVITY ONLY") + + data["Address"].append(survey.address) + data["Surveyor's Name"].append(survey.pre_site_note.assessor_information.name) + + df = pd.DataFrame(data) + + # Save to an Excel file + df.to_excel("survey_data.xlsx", index=False) + + print("Excel file 'survey_data.xlsx' created successfully!") + print(pformat(data)) if __name__ == "__main__": main() - - -# Current todo list - -# - [x] Finish off scraping for section that I need to finish -# - [in progress] Pydantic format for deemed report -# - [] Generate deemed report -> scopre of automations thats possible -# - [] Ensure lewis has sent me this week deemed score (invoice price for each job) -# - [] Ask lewis to put his rate card in sharepoint ( ask nick where the most logical place this will be) -# [] Ask kieran how some information is recieved from survery to submissions -# - [] Docker compose to an sql database in docker compose (2 hours, then work on sql) -# - [] Deploy via terraform to aws (1 day) \ No newline at end of file diff --git a/etl/scis_invoice.py b/etl/scis_invoice.py index 0f57572..b2602ea 100644 --- a/etl/scis_invoice.py +++ b/etl/scis_invoice.py @@ -1,4 +1,4 @@ -from etl.scraper.scraper import SharePointScraper, SharePointInstaller +from etl.scraper.scraper import SharePointScraper, SharePointInstaller, WEEK_COMMENCING from pprint import pformat from etl.pdfReader.pdfReaderToText import pdfReaderToText from etl.surveyedData.surveryedData import surveyedDataProcessor @@ -29,13 +29,13 @@ def main(): if survey.pre_site_note: if survey.csr: data["Price"].append(500) - data["Type of Work"].append("CAVITY ONLY") + data["Type of Work"].append("REMIDIAL CWI ONLY") else: data["Price"].append(1000) - data["Type of Work"].append("REMIDIAL CWI ONLY") + data["Type of Work"].append("CAVITY ONLY") data["Address"].append(survey.address) - data["Surveyor's Name"].append(survey.pre_site) + data["Surveyor's Name"].append(survey.pre_site_note.assessor_information.name) df = pd.DataFrame(data) @@ -43,8 +43,4 @@ def main(): df.to_excel("survey_data.xlsx", index=False) print("Excel file 'survey_data.xlsx' created successfully!") - print(pformat(data)) - - -if __name__ == "__main__": - main() + print(pformat(data)) \ No newline at end of file diff --git a/etl/scraper/scraper.py b/etl/scraper/scraper.py index 72c4edc..0c45e60 100644 --- a/etl/scraper/scraper.py +++ b/etl/scraper/scraper.py @@ -54,13 +54,7 @@ class SharePointScraper(): # Delete me for production if development: - self.surveyor_names = ['Abdul Koddus'] - self.surveyor_to_housing_assosications = {"Abdul Koddus":['Southern Housing']} - self.surveyor_to_dates_folder = {'Abdul Koddus': ['W.C. 03.03.2025', 'W.C. 24.02.2025']} - - self.surveyor_names = ['Carl Fitzgerald'] - self.surveyor_to_housing_assosications = {"Carl Fitzgerald":['ACIS']} - self.surveyor_to_dates_folder = {'Carl Fitzgerald': ['W.C. 03.03.2025']} + self.surveyor_names = ['Abdul Koddus','Carl Fitzgerald'] From d67155ea771b4e6ea7bc10539be04bf6e5284a9e Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 19 Mar 2025 15:31:55 +0000 Subject: [PATCH 4/4] scraper --- etl/scraper/scraper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etl/scraper/scraper.py b/etl/scraper/scraper.py index 0c45e60..c2881c4 100644 --- a/etl/scraper/scraper.py +++ b/etl/scraper/scraper.py @@ -14,7 +14,7 @@ from datetime import datetime, timedelta def previous_monday(): today = datetime.today() last_monday = today - timedelta(days=today.weekday() + 7) # Go back to last week's Monday - return "W.C. 03.03.2025" # Quick one off script for nic. Hopefully get 199 Do not merge!!!! + # return "W.C. 03.03.2025" # Quick one off script for nic. Hopefully get 199 Do not merge!!!! return f"W.C. {last_monday.strftime('%d.%m.%Y')}" WEEK_COMMENCING = os.getenv("WEEK_COMMENCING", previous_monday())