mirror of
https://github.com/Hestia-Homes/survey-extraction.git
synced 2026-07-27 23:35:02 +00:00
Merge pull request #20 from Hestia-Homes/feature/sharepoint_integration
Feature/sharepoint integration
This commit is contained in:
Jun-te Kim
GitHub
commit
dd6d93304f
5 changed files with 197 additions and 83 deletions
|
|
@ -1,30 +0,0 @@
|
|||
import os
|
||||
from pdfReader.pdfReaderToText import pdfReaderToText
|
||||
from etl.scraper.scraper import SharePointScraper, SharePointInstaller
|
||||
from pprint import pprint, pformat
|
||||
import logging
|
||||
from etl.utils.logger import Logger
|
||||
from etl.validator.validator import DomnaSharePointValidator
|
||||
|
||||
logger = Logger(name="main.py", level=logging.DEBUG).get_logger()
|
||||
DATA_LOC_1 = "/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/10 Turnberry Close TN38 0WL/PRE SITE NOTES.pdf"
|
||||
DATA_LOC_2 = "/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/16 Sunningdale Drive TN38 0WB/PRE SITE NOTES.pdf"
|
||||
|
||||
pdfReader = pdfReaderToText(DATA_LOC_1)
|
||||
doc2 = pdfReader.get_reader()
|
||||
pdfReader2 = pdfReaderToText(DATA_LOC_2)
|
||||
doc1 = pdfReader2.get_reader()
|
||||
vars(doc1)
|
||||
|
||||
def main():
|
||||
pass
|
||||
# POC of downloading each file
|
||||
south_coast_scraper = SharePointScraper(SharePointInstaller.SOUTH_COAST_INSULATION_SERVICE)
|
||||
south_coast_scraper.download_file_for_each_address()
|
||||
|
||||
# POC of pdf reader
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -1,42 +1,195 @@
|
|||
import os
|
||||
from pdfReader.pdfReaderToText import pdfReaderToText
|
||||
from etl.scraper.scraper import SharePointScraper, SharePointInstaller
|
||||
from pprint import pprint, pformat
|
||||
import logging
|
||||
from etl.utils.logger import Logger
|
||||
from etl.validator.validator import DomnaSharePointValidator
|
||||
|
||||
logger = Logger(name="main.py", level=logging.DEBUG).get_logger()
|
||||
DATA_LOC_1 = "/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/10 Turnberry Close TN38 0WL/PRE SITE NOTES.pdf"
|
||||
DATA_LOC_2 = "/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/16 Sunningdale Drive TN38 0WB/PRE SITE NOTES.pdf"
|
||||
|
||||
# Extract and trasform
|
||||
pdfReader = pdfReaderToText(DATA_LOC_1)
|
||||
doc2 = pdfReader.get_reader()
|
||||
pdfReader2 = pdfReaderToText(DATA_LOC_2)
|
||||
doc1 = pdfReader2.get_reader()
|
||||
print(doc2.property_description.waterHeating)
|
||||
from pprint import pformat
|
||||
from etl.pdfReader.pdfReaderToText import pdfReaderToText
|
||||
from etl.surveyedData.surveryedData import surveyedDataProcessor
|
||||
import pandas as pd
|
||||
|
||||
|
||||
|
||||
# Transform
|
||||
|
||||
def main():
|
||||
# south_coast_scraper = SharePointScraper(SharePointInstaller.SOUTH_COAST_INSULATION_SERVICE, development = True)
|
||||
# south_coast_scraper.download_file_for_each_address()
|
||||
pass
|
||||
data = {
|
||||
"Address": [],
|
||||
"Surveyor's Name": [],
|
||||
"Type of Work": [],
|
||||
"Price": []
|
||||
}
|
||||
|
||||
#south_coast_scraper = SharePointScraper(SharePointInstaller.SOUTH_COAST_INSULATION, development=True)
|
||||
#file_paths = south_coast_scraper.download_file_for_each_address()
|
||||
|
||||
file_paths = [{'10 Turnberry Close TN38 0WL': ['/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/10 Turnberry Close TN38 0WL/10 Turnberry Close, TN38 0WL UValue.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/10 Turnberry Close TN38 0WL/Dec - 10 Turnberry Close, TN38 0WL.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/10 Turnberry Close TN38 0WL/PRE EPR.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/10 Turnberry Close TN38 0WL/PRE SAP.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/10 Turnberry Close TN38 0WL/PRE SITE NOTES.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/10 Turnberry Close TN38 0WL/PRE XML.xml']},
|
||||
{'15 Turnberry Close TN38 0WL': ['/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/15 Turnberry Close TN38 0WL/15 Turnberry Close, TN38 0WL UValue.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/15 Turnberry Close TN38 0WL/Dec - 15 Turnberry Close, TN38 0WL.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/15 Turnberry Close TN38 0WL/PRE EPR.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/15 Turnberry Close TN38 0WL/PRE SAP.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/15 Turnberry Close TN38 0WL/PRE SITE NOTES.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/15 Turnberry Close TN38 0WL/PRE XML.xml']},
|
||||
{'16 Sunningdale Drive TN38 0WB': ['/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/16 Sunningdale Drive TN38 0WB/16 Sunningdale Drive, TN38 0WD Uvalue.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/16 Sunningdale Drive TN38 0WB/Dec - 16 Sunningdale Drive, TN38 0WD.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/16 Sunningdale Drive TN38 0WB/PRE EPR.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/16 Sunningdale Drive TN38 0WB/PRE SAP.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/16 Sunningdale Drive TN38 0WB/PRE SITE NOTES.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/16 Sunningdale Drive TN38 0WB/PRE XML.xml']},
|
||||
{'17 Sunningdale Drive TN38 0WB': ['/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/17 Sunningdale Drive TN38 0WB/17 Sunningdale Drive, TN38 0WD Uvalue.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/17 Sunningdale Drive TN38 0WB/Dec - 17 Sunningdale Drive, TN38 0WD.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/17 Sunningdale Drive TN38 0WB/PRE EPR.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/17 Sunningdale Drive TN38 0WB/PRE SAP.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/17 Sunningdale Drive TN38 0WB/PRE SITE NOTES.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/17 Sunningdale Drive TN38 0WB/PRE XML.xml']},
|
||||
{'2 Badgers Court TN37 6JB': ['/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/2 Badgers Court TN37 6JB/2 Badgers Court, TN37 6JB UValue.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/2 Badgers Court TN37 6JB/Dec - 2 Badgers Court, TN37 6JB.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/2 Badgers Court TN37 6JB/PRE EPR.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/2 Badgers Court TN37 6JB/PRE SAP.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/2 Badgers Court TN37 6JB/PRE SITE SITES.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/2 Badgers Court TN37 6JB/PRE XML.xml',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/2 Badgers Court TN37 6JB/statement_10197841_70195407-2.pdf']},
|
||||
{'23 Sunningdale Drive TN38 0WD': ['/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/23 Sunningdale Drive TN38 0WD/23 Sunningdale Drive, TN38 0WD UValue.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/23 Sunningdale Drive TN38 0WD/Dec - 23 Sunningdale Drive, TN38 0WD.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/23 Sunningdale Drive TN38 0WD/PRE EPR.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/23 Sunningdale Drive TN38 0WD/PRE SAP.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/23 Sunningdale Drive TN38 0WD/PRE SITE NOTES.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/23 Sunningdale Drive TN38 0WD/PRE XML.xml']},
|
||||
{'3 Turnberry Close TN38 0WL': ['/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/3 Turnberry Close TN38 0WL/3 Turnberry Close, TN38 0WL UValue.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/3 Turnberry Close TN38 0WL/Dec - 3 Turnberry Close, TN38 0WL.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/3 Turnberry Close TN38 0WL/PRE EPR.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/3 Turnberry Close TN38 0WL/PRE SAP.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/3 Turnberry Close TN38 0WL/PRE SITE NOTES.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/3 Turnberry Close TN38 0WL/PRE XML.xml']},
|
||||
{'35 Sunningdale Drive TN38 0WD': ['/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/35 Sunningdale Drive TN38 0WD/35 Sunningdale Drive, TN38 0WD UValue.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/35 Sunningdale Drive TN38 0WD/Dec - 35 Sunningdale Drive, TN38 0WD.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/35 Sunningdale Drive TN38 0WD/PRE EPR.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/35 Sunningdale Drive TN38 0WD/PRE SAP.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/35 Sunningdale Drive TN38 0WD/PRE SITE NOTES.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/35 Sunningdale Drive TN38 0WD/PRE XML.xml']},
|
||||
{'43 Sunningdale Drive TN38 0WD': ['/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/43 Sunningdale Drive TN38 0WD/43 Sunningdale Drive, TN38 0WD UValue.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/43 Sunningdale Drive TN38 0WD/Dec - 43 Sunningdale Drive, TN38 0WD.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/43 Sunningdale Drive TN38 0WD/PRE EPR.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/43 Sunningdale Drive TN38 0WD/PRE SAP.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/43 Sunningdale Drive TN38 0WD/PRE SITE NOTES.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/43 Sunningdale Drive TN38 0WD/PRE XML.xml']},
|
||||
{'45 Sunningdale Drive TN38 0WD': ['/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/45 Sunningdale Drive TN38 0WD/45 Sunningdale Drive, TN38 0WD UValue.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/45 Sunningdale Drive TN38 0WD/Dec - 45 Sunningdale Drive, TN38 0WD.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/45 Sunningdale Drive TN38 0WD/PRE EPR.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/45 Sunningdale Drive TN38 0WD/PRE SAP.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/45 Sunningdale Drive TN38 0WD/PRE SITE NOTES.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/45 Sunningdale Drive TN38 0WD/PRE XML.xml']},
|
||||
{'7 Turnberry Close TN38 0WL': ['/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/7 Turnberry Close TN38 0WL/7 Turnberry Close, TN38 0WL UValue.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/7 Turnberry Close TN38 0WL/Dec - 7 Turnberry Close, TN38 0WL.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/7 Turnberry Close TN38 0WL/PRE EPR.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/7 Turnberry Close TN38 0WL/PRE SAP.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/7 Turnberry Close TN38 0WL/PRE SITE NOTES.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/7 Turnberry Close TN38 0WL/PRE XML.xml']},
|
||||
{'9 Turnberry Close TN38 0WL': ['/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/9 Turnberry Close TN38 0WL/9 Turnberry Close, TN38 0WL UValue.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/9 Turnberry Close TN38 0WL/Dec - 9 Turnberry Close, TN38 0WL.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/9 Turnberry Close TN38 0WL/invoice-225560.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/9 Turnberry Close TN38 0WL/PRE EPR.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/9 Turnberry Close TN38 0WL/PRE SAP.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/9 Turnberry Close TN38 0WL/PRE SITE NOTES.pdf',
|
||||
'/tmp/sharepoint/Abdul Koddus/W.C. 03.03.2025/Southern Housing/9 Turnberry Close TN38 0WL/PRE XML.xml']},
|
||||
{'10 Chapel Lane': ['/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/10 Chapel Lane/10 Chapel Lane PRE EPR PDF.pdf',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/10 Chapel Lane/10 Chapel Lane PRE EPR RDSAP.xml',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/10 Chapel Lane/10 Chapel Lane PRE EPR SITE NOTES.pdf',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/10 Chapel Lane/10 Chapel.pdf']},
|
||||
{'10 South Drive': ['/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/10 South Drive/10 South Drive PRE EPR PDF.pdf',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/10 South Drive/10 South Drive PRE EPR RDSAP.xml',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/10 South Drive/10 South Drive PRE EPR SITE NOTES.pdf',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/10 South Drive/10 South.pdf']},
|
||||
{'11 Dorset Place': ['/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/11 Dorset Place/11 Dorset Place PRE EPR PDF.pdf',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/11 Dorset Place/11 Dorset Place PRE EPR RDSAP.xml',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/11 Dorset Place/11 Dorset Place PRE EPR SITE NOTES.pdf',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/11 Dorset Place/11 Dorset.pdf']},
|
||||
{'11 Swan Close': ['/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/11 Swan Close/11 Swan Close PRE EPR PDF.pdf',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/11 Swan Close/11 Swan Close PRE EPR RDSAP.xml',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/11 Swan Close/11 Swan Close PRE EPR SITE NOTES.pdf',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/11 Swan Close/11 Swan.pdf']},
|
||||
{'12 Ingham Road': ['/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/12 Ingham Road/12 Ingham Road PRE EPR PDF.pdf',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/12 Ingham Road/12 Ingham Road PRE EPR RDSAP.xml',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/12 Ingham Road/12 Ingham Road PRE EPR SITE NOTES.pdf',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/12 Ingham Road/12 Ingham.pdf']},
|
||||
{'13 Morris Close': ['/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/13 Morris Close/13 Morris Close PRE EPR PDF.pdf',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/13 Morris Close/13 Morris Close PRE EPR RDSAP.xml',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/13 Morris Close/13 Morris Close PRE EPR SITE NOTES.pdf',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/13 Morris Close/13 Morris.pdf']},
|
||||
{'13 Swan Close': ['/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/13 Swan Close/13 Swan Close PRE EPR PDF.pdf',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/13 Swan Close/13 Swan Close PRE EPR RDSA.xml',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/13 Swan Close/13 Swan.pdf']},
|
||||
{'14 Ingham Road': ['/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/14 Ingham Road/14 Ingham Road PRE EPR PDF.pdf',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/14 Ingham Road/14 Ingham Road PRE EPR RDSAP.xml',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/14 Ingham Road/14 Ingham Road PRE EPR SITE NTES.pdf',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/14 Ingham Road/14 Ingham.pdf']},
|
||||
{'16 Station Road': ['/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/16 Station Road/16 Station Road PRE EPR PDF.pdf',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/16 Station Road/16 Station Road PRE EPR RDSAP.xml',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/16 Station Road/16 Station Road PRE EPR SITE NOTES.pdf',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/16 Station Road/16 Station.pdf']},
|
||||
{'16 Swan Close': ['/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/16 Swan Close/16 Swan Close PRE EPR PDF.pdf',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/16 Swan Close/16 Swan Close PRE EPR RDSAP.xml',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/16 Swan Close/16 Swan Close PRE EPR SITE NOTES.pdf',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/16 Swan Close/16 Swan.pdf']},
|
||||
{'2 Deane Walk': ['/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/2 Deane Walk/2 Deane Walk PRE EPR PDF.pdf',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/2 Deane Walk/2 Deane Walk PRE EPR RDSAP.xml',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/2 Deane Walk/2 Deane Walk PRE EPR SITE NOTES.pdf',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/2 Deane Walk/2 Deane.pdf']},
|
||||
{'25 High Leas': ['/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/25 High Leas/25 High Leas PRE EPR PDF.pdf',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/25 High Leas/25 High Leas PRE EPR RDSAP.xml',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/25 High Leas/25 High Leas PRE EPR SITE NOTES.pdf',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/25 High Leas/25 High.pdf']},
|
||||
{'26 Fossdyke Gardens': ['/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/26 Fossdyke Gardens/26 Fossdyke Gardens PRE EPR PDF.pdf',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/26 Fossdyke Gardens/26 Fossdyke Gardens PRE EPR RDSAP.xml',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/26 Fossdyke Gardens/26 Fossdyke Gardens PRE EPR SITE NOTES.pdf',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/26 Fossdyke Gardens/26 Fossdyke.pdf']},
|
||||
{'28 Station Road': ['/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/28 Station Road/28 Station Road PRE EPR PDF.pdf',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/28 Station Road/28 Station Road PRE EPR RDSAP.xml',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/28 Station Road/28 Station Road PRE EPR SITE NOTES.pdf',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/28 Station Road/28 Station.pdf']},
|
||||
{'30 Saxilby Road': ['/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/30 Saxilby Road/30 Saxilby Road PRE EPR RDSAP.xml',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/30 Saxilby Road/30 Saxilby Road PRE EPR SITE NOTES.pdf',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/30 Saxilby Road/30 Saxilby.pdf']},
|
||||
{'5 Queensway': ['/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/5 Queensway/5 Queensway PRE EPR PDF.pdf',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/5 Queensway/5 Queensway PRE EPR RDSAP.xml',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/5 Queensway/5 Queensway PRE EPR SITE NOTES.pdf',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/5 Queensway/5-Queensway.pdf']},
|
||||
{'7 Queensway': ['/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/7 Queensway/7 Queensway PRE EPR PDF.pdf',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/7 Queensway/7 Queensway PRE EPR RDSAP.xml',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/7 Queensway/7 Queensway PRE EPR SITE NOTES.pdf',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/7 Queensway/7-Queensway.pdf']},
|
||||
{'9 Bridge Place': ['/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/9 Bridge Place/9 Bridge Place PRE EPR PDF.pdf',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/9 Bridge Place/9 Bridge Place PRE EPR RDSAP.xml',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/9 Bridge Place/9 Bridge Place PRE EPR SITE NOTES.pdf',
|
||||
'/tmp/sharepoint/Carl Fitzgerald/W.C. 03.03.2025/ACIS/9 Bridge Place/9 Bridge.pdf']}]
|
||||
|
||||
list_of_surveys = []
|
||||
for eachAddress in file_paths:
|
||||
for address, files in eachAddress.items():
|
||||
list_of_surveys.append(surveyedDataProcessor(address, files))
|
||||
break
|
||||
|
||||
|
||||
for survey in list_of_surveys:
|
||||
if survey.pre_site_note:
|
||||
if survey.csr:
|
||||
data["Price"].append(500)
|
||||
data["Type of Work"].append("REMIDIAL CWI ONLY")
|
||||
else:
|
||||
data["Price"].append(1000)
|
||||
data["Type of Work"].append("CAVITY ONLY")
|
||||
|
||||
data["Address"].append(survey.address)
|
||||
data["Surveyor's Name"].append(survey.pre_site_note.assessor_information.name)
|
||||
|
||||
df = pd.DataFrame(data)
|
||||
|
||||
# Save to an Excel file
|
||||
df.to_excel("survey_data.xlsx", index=False)
|
||||
|
||||
print("Excel file 'survey_data.xlsx' created successfully!")
|
||||
print(pformat(data))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
|
||||
# Current todo list
|
||||
|
||||
# - [x] Finish off scraping for section that I need to finish
|
||||
# - [in progress] Pydantic format for deemed report
|
||||
# - [] Generate deemed report -> scopre of automations thats possible
|
||||
# - [] Ensure lewis has sent me this week deemed score (invoice price for each job)
|
||||
# - [] Ask lewis to put his rate card in sharepoint ( ask nick where the most logical place this will be)
|
||||
# [] Ask kieran how some information is recieved from survery to submissions
|
||||
# - [] Docker compose to an sql database in docker compose (2 hours, then work on sql)
|
||||
# - [] Deploy via terraform to aws (1 day)
|
||||
|
|
@ -710,8 +710,8 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor):
|
|||
model_name=dict_.get("model_name", ""),
|
||||
model_qualifer=dict_.get("model_qualifier", ""),
|
||||
controls=HeatingSystemControls(
|
||||
control_type=dict_.get("control_type", ""),
|
||||
flue_type=dict_.get("flue_type",""),
|
||||
control_type=dict_.get("control_type", "") if dict_.get("control_type", "") is not None else "",
|
||||
flue_type=dict_.get("flue_type","") if dict_.get("flue_type", "") is not None else "",
|
||||
fan_assisted_flue=True if dict_.get("fan_assisted_flue", "NO").upper() == "YES" else False,
|
||||
heat_emitter_type=dict_.get("heat_emitter_type", "") if dict_.get("heat_emitter_type") is not None else "",
|
||||
electricity_meter_type=dict_.get("electricity_meter_type", ""),
|
||||
|
|
@ -751,11 +751,11 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor):
|
|||
model_name=dict_.get("model_name", ""),
|
||||
model_qualifer=dict_.get("model_qualifier", ""),
|
||||
controls=HeatingSystemControls(
|
||||
control_type=dict_.get("control_type", ""),
|
||||
flue_type=dict_.get("flue_type",""),
|
||||
control_type=dict_.get("control_type", "") if dict_.get("control_type", "") is not None else "",
|
||||
flue_type=dict_.get("flue_type","") if dict_.get("flue_type", "") is not None else "",
|
||||
fan_assisted_flue=True if dict_.get("fan_assisted_flue", "NO").upper() == "YES" else False,
|
||||
heat_emitter_type=dict_.get("heat_emitter_type", "") if dict_.get("heat_emitter_type") is not None else "",
|
||||
electricity_meter_type=dict_.get("electricity_meter_type", ""),
|
||||
electricity_meter_type=dict_.get("electricity_meter_type", "") if dict_.get("electricity_meter_type", "") is not None else "",
|
||||
mains_gas_available=True if dict_.get("mains_gas_available", "NO").upper() == "YES" else False,
|
||||
)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
from etl.scraper.scraper import SharePointScraper, SharePointInstaller
|
||||
from etl.scraper.scraper import SharePointScraper, SharePointInstaller, WEEK_COMMENCING
|
||||
from pprint import pformat
|
||||
from etl.pdfReader.pdfReaderToText import pdfReaderToText
|
||||
from etl.surveyedData.surveryedData import surveyedDataProcessor
|
||||
|
|
@ -29,13 +29,13 @@ def main():
|
|||
if survey.pre_site_note:
|
||||
if survey.csr:
|
||||
data["Price"].append(500)
|
||||
data["Type of Work"].append("CAVITY ONLY")
|
||||
data["Type of Work"].append("REMIDIAL CWI ONLY")
|
||||
else:
|
||||
data["Price"].append(1000)
|
||||
data["Type of Work"].append("REMIDIAL CWI ONLY")
|
||||
data["Type of Work"].append("CAVITY ONLY")
|
||||
|
||||
data["Address"].append(survey.address)
|
||||
data["Surveyor's Name"].append(survey.pre_site)
|
||||
data["Surveyor's Name"].append(survey.pre_site_note.assessor_information.name)
|
||||
|
||||
df = pd.DataFrame(data)
|
||||
|
||||
|
|
@ -43,7 +43,4 @@ def main():
|
|||
df.to_excel("survey_data.xlsx", index=False)
|
||||
|
||||
print("Excel file 'survey_data.xlsx' created successfully!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
print(pformat(data))
|
||||
|
|
@ -54,13 +54,7 @@ class SharePointScraper():
|
|||
|
||||
# Delete me for production
|
||||
if development:
|
||||
self.surveyor_names = ['Abdul Koddus']
|
||||
self.surveyor_to_housing_assosications = {"Abdul Koddus":['Southern Housing']}
|
||||
self.surveyor_to_dates_folder = {'Abdul Koddus': ['W.C. 03.03.2025', 'W.C. 24.02.2025']}
|
||||
|
||||
self.surveyor_names = ['Carl Fitzgerald']
|
||||
self.surveyor_to_housing_assosications = {"Carl Fitzgerald":['ACIS']}
|
||||
self.surveyor_to_dates_folder = {'Carl Fitzgerald': ['W.C. 03.03.2025']}
|
||||
self.surveyor_names = ['Abdul Koddus','Carl Fitzgerald']
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue