junte/survey-extraction
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/survey-extraction.git synced 2026-07-27 14:35:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge pull request #18 from Hestia-Homes/feature/sharepoint_integration

Browse source 
Feature/sharepoint integration
This commit is contained in:
Jun-te Kim 2025-03-19 14:23:47 +00:00 • committed by GitHub
commit f1440f72db
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 178 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
.github/workflows/scis_invoice_calculator.yml vendored Normal file
View file

@ -0,0 +1,33 @@
on:
schedule:
- cron: '0 9 * * *'
workflow_dispatch:
jobs:
scis_invoice_calculator:
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.12'
- name: Install dependencies
run: |
pip install poetry
poetry install --no-root
- name: run script
run: |
bash scis_invoice.sh
env:
PYTHONPATH: ${{ github.workspace }}
SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID: ${{ secrets.SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID }}
JJC_SERVICE_SHAREPOINT_ID: ${{ secrets.JJC_SERVICE_SHAREPOINT_ID }}
BAXTER_KELLY_SERVICE_SHAREPOINT_ID: ${{ secrets.BAXTER_KELLY_SERVICE_SHAREPOINT_ID }}
SGEC_SERVICE_SHAREPOINT_ID: ${{ secrets.SGEC_SERVICE_SHAREPOINT_ID }}
SHAREPOINT_CLIENT_ID: ${{ secrets.SHAREPOINT_CLIENT_ID }}
SHAREPOINT_CLIENT_SECRET: ${{ secrets.SHAREPOINT_CLIENT_SECRET }}
SHAREPOINT_TENANT_ID: ${{ secrets.SHAREPOINT_TENANT_ID }}

2
etl/daily_script.py
View file

@ -57,7 +57,7 @@ def main():
if b_names:
logger.info("Baxter Kelly with wrong date format")
logger.info(pformat(b_names))
logger.info("-------------------------------------------")
logger.info("-")
logger.info("------EACH PRE SITE NOTES SUBMISSIONS------")
logger.info("-------------------------------------------")
logger.info(f"For week commencing: {WEEK_COMMENCING}")

23
etl/pdfReader/pdfReaderToText.py
View file

@ -1,7 +1,7 @@
from etl.utils.logger import Logger
import logging
import pymupdf
from etl.pdfReader.sitenotes import QuidosSiteNotesExtractor
from etl.pdfReader.sitenotes import QuidosSiteNotesExtractor, CSR
from etl.pdfReader.reportType import ReportType
class pdfReaderToText():
@ -13,6 +13,7 @@ class pdfReaderToText():
self.text_list = []
self.get_text_from_pdf_file()
self.type = None
self.get_file_type()
def get_text_from_pdf_file(self):
self.logger.debug(f"Extrating text from {self.source_path}")
@ -31,13 +32,23 @@ class pdfReaderToText():
if len(self.text_list) > 1:
if "Quidos Ltd using Argyle software BRE approved calculator".lower() in self.text_list[0].lower():
self.type = ReportType.QUIDOS_PRESITE_NOTE
return self.type
elif "Wall pre - Masonry cavity wall-unĮlled".lower() in self.text_list[0].lower():
self.type = ReportType.U_VALUE_CALCULATOR_REPORT
elif "Overwriting U-Values for EPRs for ECO4 and GBIS:" in self.text_list[1].lower():
self.type = ReportType.OVERWRITING_U_VALUE_DECLARATION_FORM
elif "Energy Performance Report" in self.text_list:
self.type = ReportType.ENERGY_PERFORMANCE_REPORT
elif "Chartered Surveyor Report: Recommending Extraction of Defective Cavity Wall Insulation " in self.text_list:
self.type = ReportType.CHARTED_SURVEYOR_REPORT
else:
return None
# raise NotImplementedError("New type of file - please contact Jun-te Kim")
pass
return self.type
def get_reader(self):
self.get_file_type()
if self.type.name == ReportType.QUIDOS_SITE_NOTE.name:
return QuidosSiteNotesExtractor(self.text_list)
if self.type.name == ReportType.QUIDOS_PRESITE_NOTE.name:
return QuidosSiteNotesExtractor(self.text_list)
elif self.type == ReportType.CHARTED_SURVEYOR_REPORT:
return CSR(self.text_list)

6
etl/pdfReader/reportType.py
View file

@ -2,4 +2,8 @@ from enum import Enum
class ReportType(Enum):
QUIDOS_PRESITE_NOTE = 1
QUIDOS_PRESITE_NOTE = 1
CHARTED_SURVEYOR_REPORT = 2
ENERGY_PERFORMANCE_REPORT = 3
U_VALUE_CALCULATOR_REPORT = 4
OVERWRITING_U_VALUE_DECLARATION_FORM = 5

7
etl/pdfReader/sitenotes.py
View file

@ -22,13 +22,18 @@ class SiteNotesExtractor():
def get_data_between(self, a, b):
return self.raw_data[self.raw_data.index(a):self.raw_data.index(b)]
class CSR(SiteNotesExtractor):
def __init__(self, data_list):
super().__init__(data_list)
self.type = ReportType.CHARTED_SURVEYOR_REPORT
class QuidosSiteNotesExtractor(SiteNotesExtractor):
def __init__(self, data_list):
super().__init__(data_list)
self.type = ReportType.QUIDOS_SITE_NOTE
self.type = ReportType.QUIDOS_PRESITE_NOTE
self.company_information = None
self.survey_information = None
self.property_description = None

49
etl/scis_invoice.py Normal file
View file

@ -0,0 +1,49 @@
from etl.scraper.scraper import SharePointScraper, SharePointInstaller
from pprint import pformat
from etl.pdfReader.pdfReaderToText import pdfReaderToText
from etl.surveyedData.surveryedData import surveyedDataProcessor
import pandas as pd
def main():
data = {
"Address": [],
"Surveyor's Name": [],
"Type of Work": [],
"Price": []
}
south_coast_scraper = SharePointScraper(SharePointInstaller.SOUTH_COAST_INSULATION, development=True)
file_paths = south_coast_scraper.download_file_for_each_address()
list_of_surveys = []
for eachAddress in file_paths:
for address, files in eachAddress.items():
list_of_surveys.append(surveyedDataProcessor(address, files))
break
for survey in list_of_surveys:
if survey.pre_site_note:
if survey.csr:
data["Price"].append(500)
data["Type of Work"].append("CAVITY ONLY")
else:
data["Price"].append(1000)
data["Type of Work"].append("REMIDIAL CWI ONLY")
data["Address"].append(survey.address)
data["Surveyor's Name"].append(survey.pre_site)
df = pd.DataFrame(data)
# Save to an Excel file
df.to_excel("survey_data.xlsx", index=False)
print("Excel file 'survey_data.xlsx' created successfully!")
if __name__ == "__main__":
main()

19
etl/scraper/scraper.py
View file

@ -14,6 +14,7 @@ from datetime import datetime, timedelta
def previous_monday():
today = datetime.today()
last_monday = today - timedelta(days=today.weekday() + 7) # Go back to last week's Monday
# return "W.C. 03.03.2025" # Quick one off script for nic. Hopefully get 199 Do not merge!!!!
return f"W.C. {last_monday.strftime('%d.%m.%Y')}"
WEEK_COMMENCING = os.getenv("WEEK_COMMENCING", previous_monday())
@ -57,6 +58,9 @@ class SharePointScraper():
self.surveyor_to_housing_assosications = {"Abdul Koddus":['Southern Housing']}
self.surveyor_to_dates_folder = {'Abdul Koddus': ['W.C. 03.03.2025', 'W.C. 24.02.2025']}
self.surveyor_names = ['Carl Fitzgerald']
self.surveyor_to_housing_assosications = {"Carl Fitzgerald":['ACIS']}
self.surveyor_to_dates_folder = {'Carl Fitzgerald': ['W.C. 03.03.2025']}
@ -225,12 +229,11 @@ class SharePointScraper():
else:
self.surveyor_work_completed.update({name: 1})
break
print("trololol")
return self.surveyor_work_completed
@ensure_housing_assosiation_is_loaded
def download_file_for_each_address(self):
paths = []
for name in self.surveyor_names:
if WEEK_COMMENCING in self.surveyor_to_dates_folder[name]:
for house_ass in self.surveyor_to_housing_assosications[name]:
@ -243,9 +246,9 @@ class SharePointScraper():
if 'file' not in address:
# Only directories
allAddress.append(address['name'])
for address in allAddress:
for i, address in enumerate(allAddress):
path = f"/{name}/{WEEK_COMMENCING}/{house_ass}/{address}"
address_paths = {}
files_to_download_sharepoint_info = self.get_folders_in_path(path)
if 'value' not in files_to_download_sharepoint_info:
raise RuntimeError("Failed to get files to download")
@ -258,11 +261,15 @@ class SharePointScraper():
if any(file["name"].endswith(ext) for ext in avoid):
continue
file_names_to_download.update({file["name"]: file['@microsoft.graph.downloadUrl']})
each_file = []
for file_name, url in file_names_to_download.items():
self.logger.info(f"Downloading {file_name} from {url}")
content = self.get_file_content(url)
self.create_temp_file(content, f"{name}/{WEEK_COMMENCING}/{house_ass}/{address}/{file_name}")
file_path = self.create_temp_file(content, f"{name}/{WEEK_COMMENCING}/{house_ass}/{address}/{file_name}")
each_file.append(file_path)
address_paths.update({address: each_file})
paths.append(address_paths)
return paths
def create_temp_file(self, content, path):
# Ensure the path is under /tmp/

0
etl/surveyedData/__init__.py Normal file
View file

23
etl/surveyedData/surveryedData.py Normal file
View file

@ -0,0 +1,23 @@
from etl.pdfReader.pdfReaderToText import pdfReaderToText
from etl.pdfReader.reportType import ReportType
class surveyedDataProcessor():
def __init__(self, address, files):
self.address = address
self.files = files
self.pre_site_note = None
self.csr = None
self.identify_files()
def identify_files(self):
for file in self.files:
pdf = pdfReaderToText(file)
print("Junte was here")
print(file)
print(pdf.text_list)
if pdf:
if pdf.type == ReportType.QUIDOS_PRESITE_NOTE:
self.pre_site_note = pdf.get_reader()
elif pdf.type == ReportType.CHARTED_SURVEYOR_REPORT:
self.csr = pdf.get_reader()

29
poetry.lock generated
View file

@ -410,6 +410,18 @@ files = [
dnspython = ">=2.0.0"
idna = ">=2.0.0"
[[package]]
name = "et-xmlfile"
version = "2.0.0"
description = "An implementation of lxml.xmlfile for the standard library"
optional = false
python-versions = ">=3.8"
groups = ["main"]
files = [
{file = "et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa"},
{file = "et_xmlfile-2.0.0.tar.gz", hash = "sha256:dab3f4764309081ce75662649be815c4c9081e88f0837825f90fd28317d4da54"},
]
[[package]]
name = "executing"
version = "2.2.0"
@ -697,6 +709,21 @@ files = [
{file = "numpy-2.2.3.tar.gz", hash = "sha256:dbdc15f0c81611925f382dfa97b3bd0bc2c1ce19d4fe50482cb0ddc12ba30020"},
]
[[package]]
name = "openpyxl"
version = "3.1.5"
description = "A Python library to read/write Excel 2010 xlsx/xlsm files"
optional = false
python-versions = ">=3.8"
groups = ["main"]
files = [
{file = "openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2"},
{file = "openpyxl-3.1.5.tar.gz", hash = "sha256:cf0e3cf56142039133628b5acffe8ef0c12bc902d2aadd3e0fe5878dc08d1050"},
]
[package.dependencies]
et-xmlfile = "*"
[[package]]
name = "packaging"
version = "24.2"
@ -1436,4 +1463,4 @@ files = [
[metadata]
lock-version = "2.1"
python-versions = ">=3.12"
content-hash = "710051703d97e156a540ad08b0815338a4283146f6fca3c0ae89cc4e6dad459a"
content-hash = "7c7fb2198bf2cb04e0af34fa6769280fda46907a2024b8f4c188847962964631"

1
pyproject.toml
View file

@ -12,6 +12,7 @@ dependencies = [
"msal (>=1.31.1,<2.0.0)",
"pandas (>=2.2.3,<3.0.0)",
"pydantic[email] (>=2.10.6,<3.0.0)",
"openpyxl (>=3.1.5,<4.0.0)",
]
[tool.poetry]

2
scis_invoice.sh Normal file
View file

@ -0,0 +1,2 @@
# Example of how to run python code in this environment
poetry run python etl/scis_invoice.py