junte/survey-extraction
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/survey-extraction.git synced 2026-06-30 13:10:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

nick code

Browse source
This commit is contained in:
Jun-te Kim 2025-09-01 22:13:21 +00:00
parent 1026d90a3a
commit 27a6227311
2 changed files with 42 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
etl/MonthEndUploader.py
View file

@ -22,3 +22,21 @@ def upload_to_month_end_folder(file_name_on_sp, local_file_path, add_to_path):
print("Uploading to sharepoint...")
sharepoint.upload_file(local_file_path, sharepoint_path, file_name_on_sp)
print(f"Finished upload of {local_file_path} to sharepoint. It's found under {sharepoint_path}/{file_name_on_sp}")
def upload_to_nick_folder(file_name_on_sp, local_file_path, add_to_path=None):
sharepoint = SharePointScraper(SharePointInstaller.OSMOSIS_WAVE_2)
parent_folder = "General/Junte Kim/For Nick"
today = datetime.today()
formatted_date = today.strftime("%Y-%m-%d%H-%M-S")
sharepoint.create_dir(formatted_date, parent_folder)
sharepoint_path = parent_folder + "/" + formatted_date
if add_to_path:
sharepoint.create_dir(add_to_path, sharepoint_path)
sharepoint_path += "/" + add_to_path
print("Uploading to sharepoint...")
sharepoint.upload_file(local_file_path, sharepoint_path, file_name_on_sp)
print(f"Finished upload of {local_file_path} to sharepoint. It's found under {sharepoint_path}/{file_name_on_sp}")

28
etl/sharepoint_installer_data_gatherer.py
View file

@ -5,6 +5,7 @@ os.environ["SHAREPOINT_TENANT_ID"] = "10d5af8b-2cfd-4882-9ccd-b96e4812dacf"
from etl.scraper.scraper import SharePointScraper, SharePointInstaller
from etl.fileReader.pdfReaderToText import pdfReaderToText
import pandas as pd
from MonthEndUploader import upload_to_nick_folder
sharepoint = SharePointScraper(SharePointInstaller.NEW_JJC)
file_paths = sharepoint.download_file_for_each_address()
@ -55,11 +56,30 @@ for eachaddress in all_address_to_work_on:
for file in files:
pdf = pdfReaderToText(file)
if "Summary Information".lower() == pdf.text_list[0].lower():
potential_sap_rating = f"{pdf.text_list[pdf.text_list.index("Emissions (t/year):")-1]}"
potential_sap_rating = extract_rating(potential_sap_rating)
row = {"address": addressName, "potential sap rating": potential_sap_rating}
current_sap_rating = pdf.text_list[pdf.text_list.index("Current SAP rating:") + 1]
house_no = pdf.text_list[pdf.text_list.index("House No:") + 1]
street = pdf.text_list[pdf.text_list.index("Street:") + 1]
post_code = pdf.text_list[pdf.text_list.index("Postcode:") + 1]
address = f"{house_no} {street.title()}"
floor_area = pdf.text_list[pdf.text_list.index("Lowest Floor:") + 1]
fuel_bill = pdf.text_list[pdf.text_list.index("Fuel Bill:") + 1]
row = {
"address": address,
"postcode": post_code,
"current sap rating": current_sap_rating,
"floor_area ": floor_area,
"fuel_bill ": fuel_bill,
}
final_data.append(row)
pd.DataFrame(final_data)
df = pd.DataFrame(final_data)
file_name = "installer.xlsx"
df.to_excel(file_name, index=False)
# Get local path
file_path = os.path.abspath(file_name)
upload_to_nick_folder(file_name, file_path)
print(f"File saved at: {file_path}")