junte/survey-extraction
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/survey-extraction.git synced 2026-06-30 13:10:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

save osmosis automation for khalim

Browse source
This commit is contained in:
Jun-te Kim 2025-04-07 13:29:35 +00:00
parent d26383ee25
commit 3bd76ad8bb
3 changed files with 80 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

BIN
etl/osmosis_data/asset_list.xlsx Normal file
View file

Binary file not shown.

71
etl/osmosis_monday_to_sharepoint_automation.py
View file

@ -1,16 +1,79 @@
import os
# osmsis keys
# os.environ["SHAREPOINT_CLIENT_ID"] = "6832a4c5-fb8c-4082-a746-4f51e1020f0d"
# os.environ["SHAREPOINT_CLIENT_SECRET"] = "xpC8Q~Frww48SM1V-D8lGy5iOY7P_cJ7FF3jgarQ"
# os.environ["SHAREPOINT_TENANT_ID"] = "10d5af8b-2cfd-4882-9ccd-b96e4812dacf"
# warmfront keys
os.environ["SHAREPOINT_CLIENT_ID"] = "895e3b77-b1d7-43ec-b18f-dcfe07cdfeaf"
os.environ["SHAREPOINT_CLIENT_SECRET"] = "SOf8Q~-is4wdQiqvEEm9FlJQRAY9ELGaj5Qz-a6E"
os.environ["SHAREPOINT_TENANT_ID"] = "c3f7519c-2719-4547-af04-6da6cbfd8f8f"
os.environ["SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID"] = "b5a51507-9427-4ee0-b03e-90ec7681e2d3"
os.environ["JJC_SERVICE_SHAREPOINT_ID"] = "7fdd0485-bbf3-4b29-b30f-98c81c2a6284"
from etl.scraper.scraper import SharePointInstaller
from etl.scraper.scraper import SharePointScraper
import pandas as pd
jjc = SharePointScraper(SharePointInstaller.JJC)
osmosis = SharePointScraper(SharePointInstaller.BAXTER_KELLY)
parent_folder = "Jun-te Kim Automation Test"
osmosis.create_file(parent_folder, "/")
asset_list = pd.read_excel("osmosis_data/asset_list.xlsx", sheet_name="2502 accent housing")
new_asset_list = []
# Create asset list and location
for index, address in asset_list.iterrows():
webUrl = osmosis.create_file(address['Name'], parent_folder)
first_folder = "1. Retrofit Assessment"
osmosis.create_file(first_folder, parent_folder + f"/{address['Name']}")
osmosis.create_file("A. Assessment", parent_folder + f"/{address['Name']}/{first_folder}")
osmosis.create_file("B. Air Tightness Tests", parent_folder + f"/{address['Name']}/{first_folder}")
second_folder = "2. RC Mid-Term Plan"
osmosis.create_file(second_folder, parent_folder + f"/{address['Name']}")
osmosis.create_file("SAP", parent_folder + f"/{address['Name']}/{second_folder}")
third_folder = "3. Retrofit Design"
osmosis.create_file(third_folder, parent_folder + f"/{address['Name']}")
osmosis.create_file('ss', parent_folder + f"/{address['Name']}/{third_folder}")
fourth_folder = "4. Post EPC"
osmosis.create_file(fourth_folder, parent_folder + f"/{address['Name']}")
osmosis.create_file("POST EPC Photos", parent_folder + f"/{address['Name']}/{fourth_folder}")
fifth_folder = "5. Trustmark Lodgement"
osmosis.create_file(fifth_folder, parent_folder + f"/{address['Name']}")
osmosis.create_file("1. Works", parent_folder + f"/{address['Name']}/{fifth_folder}")
osmosis.create_file("Doors", parent_folder + f"/{address['Name']}/{fifth_folder}/1. Works")
osmosis.create_file("Photos", parent_folder + f"/{address['Name']}/{fifth_folder}/1. Works/Doors")
osmosis.create_file("Survey", parent_folder + f"/{address['Name']}/{fifth_folder}/1. Works/Doors")
osmosis.create_file("Loft Insulation", parent_folder + f"/{address['Name']}/{fifth_folder}/1. Works")
osmosis.create_file("Ventilation", parent_folder + f"/{address['Name']}/{fifth_folder}/1. Works")
osmosis.create_file("Windows", parent_folder + f"/{address['Name']}/{fifth_folder}/1. Works")
osmosis.create_file("Photos", parent_folder + f"/{address['Name']}/{fifth_folder}/1. Works/Windows")
osmosis.create_file("Survey", parent_folder + f"/{address['Name']}/{fifth_folder}/1. Works/Windows")
osmosis.create_file("2. Required Documents", parent_folder + f"/{address['Name']}/{fifth_folder}")
osmosis.create_file("3. Additional Documents", parent_folder + f"/{address['Name']}/{fifth_folder}")
asset_data = {
"Name": address['Name'],
"Sharepoint": f"Sharepoint Link - {webUrl}",
}
new_asset_list.append(asset_data)
# Osmosist File strucutre
df = pd.DataFrame(new_asset_list)
df.to_csv("output.csv", index=False)
print(jjc.does_folder_exists_at("junte was here", "Tom Miller (JJ CRUMP)"))
data = jjc.create_file("junte was here", "Tom Miller (JJ CRUMP)")
data["webUrl"]

20
etl/scraper/scraper.py
View file

@ -24,8 +24,9 @@ class SharePointInstaller(Enum):
SOUTH_COAST_INSULATION = os.getenv("SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID", None)
JJC = os.getenv("JJC_SERVICE_SHAREPOINT_ID", None)
SGEC = os.getenv("SGEC_SERVICE_SHAREPOINT_ID", None)
BAXTER_KELLY = os.getenv("BAXTER_KELLY_SERVICE_SHAREPOINT_ID", None)
BAXTER_KELLY = os.getenv("BAXTER_KELLY_SERVICE_SHAREPOINT_ID", "6f930bf3-572d-4f91-b1ae-ec536fa319e2")
DOMNA = os.getenv("DOMNA_SHAREPOINT_ID", "8ab64924-ccde-4b56-b0dc-4e11596446e4")
OSMOSIS = os.getenv("OSMOSIS_SHAREPOINT_ID", "350a3b48-8311-4506-8abb-69bafc280d6f")
class SharePointScraper():
"""
@ -107,12 +108,12 @@ class SharePointScraper():
def does_folder_exists_at(self, file_name, file_path):
for folders in self.get_folders_in_path(file_path)['value']:
if file_name.upper() in folders["name"].upper():
return True
folders = self.get_folders_in_path(file_path)
if 'value' in folders:
for folder in folders['value']:
if file_name.upper() in folder["name"].upper():
return True
return False
def create_file(self, file_name, at_path="/"):
@ -122,8 +123,13 @@ class SharePointScraper():
client_secret=self.sharepoint_client_secret,
site_id=self.sharepoint_drive.value,
)
if self.does_folder_exists_at(file_name, at_path) is False:
return sharepoint_client.create_folder(file_name, at_path)
return sharepoint_client.create_folder(file_name, at_path)['webUrl']
else:
for folders in self.get_folders_in_path(at_path)['value']:
if file_name.upper() in folders["name"].upper():
return folders["webUrl"]