mirror of
https://github.com/Hestia-Homes/survey-extraction.git
synced 2026-07-22 08:48:39 +00:00
81 lines
No EOL
2.7 KiB
Python
81 lines
No EOL
2.7 KiB
Python
import os
|
|
os.environ["SHAREPOINT_CLIENT_ID"] = "895e3b77-b1d7-43ec-b18f-dcfe07cdfeaf"
|
|
os.environ["SHAREPOINT_CLIENT_SECRET"] = "SOf8Q~-is4wdQiqvEEm9FlJQRAY9ELGaj5Qz-a6E"
|
|
os.environ["SHAREPOINT_TENANT_ID"] = "c3f7519c-2719-4547-af04-6da6cbfd8f8f"
|
|
os.environ["SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID"] = "b5a51507-9427-4ee0-b03e-90ec7681e2d3"
|
|
os.environ["JJC_SERVICE_SHAREPOINT_ID"] = "7fdd0485-bbf3-4b29-b30f-98c81c2a6284"
|
|
from etl.scraper.scraper import SharePointScraper, SharePointInstaller, WEEK_COMMENCING
|
|
import pandas as pd
|
|
import hashlib
|
|
|
|
def calculate_sha256(bytes_io):
|
|
bytes_io.seek(0) # Make sure we're at the start
|
|
data = bytes_io.read()
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
south_coast_scraper = SharePointScraper(SharePointInstaller.JJC)
|
|
|
|
|
|
folders = south_coast_scraper.get_folders_in_path('/')
|
|
|
|
|
|
list_of_file_names = []
|
|
for folder in folders['value']:
|
|
if "Khalim" in folder["name"]:
|
|
continue
|
|
elif ".Training" in folder["name"]:
|
|
continue
|
|
if 'file' not in folder:
|
|
list_of_file_names.append("/" + folder["name"])
|
|
|
|
list_of_dates = []
|
|
for folder in list_of_file_names:
|
|
dates = south_coast_scraper.get_folders_in_path(folder)
|
|
for date in dates['value']:
|
|
if 'file' not in date:
|
|
list_of_dates.append(folder + "/" + date["name"])
|
|
|
|
print(list_of_dates)
|
|
|
|
list_of_housing_associations = []
|
|
for folder in list_of_dates:
|
|
house_ass = south_coast_scraper.get_folders_in_path(folder)
|
|
for house in house_ass['value']:
|
|
if 'file' not in house:
|
|
list_of_housing_associations.append(folder + "/" + house["name"])
|
|
|
|
list_of_address = []
|
|
|
|
for folder in list_of_housing_associations:
|
|
address = south_coast_scraper.get_folders_in_path(folder)
|
|
for add in address['value']:
|
|
if 'file' not in add:
|
|
list_of_address.append(folder + "/" + add['name'])
|
|
|
|
list_of_pictures = []
|
|
|
|
for folder in list_of_address:
|
|
pictures = south_coast_scraper.get_folders_in_path(folder)
|
|
for pic in pictures['value']:
|
|
if 'file' not in pic:
|
|
list_of_pictures.append(folder + "/" + pic['name'])
|
|
|
|
print(list_of_pictures)
|
|
|
|
final_list = []
|
|
for files in list_of_pictures:
|
|
content = south_coast_scraper.get_folders_in_path(files)
|
|
for file in content['value']:
|
|
if 'file' in file:
|
|
url = file['@microsoft.graph.downloadUrl']
|
|
print(f"Downloading {files}/{file['name']}")
|
|
sha256 = calculate_sha256(south_coast_scraper.get_file_content(url))
|
|
final_list.append({
|
|
"Directories": files,
|
|
"Photo Name": file['name'],
|
|
"sha256": sha256,
|
|
})
|
|
|
|
final_df = pd.DataFrame(final_list)
|
|
|
|
final_df.to_csv("jjc.csv") |