diff --git a/etl/osmosis_data/asset_list.xlsx b/etl/osmosis_data/asset_list.xlsx new file mode 100644 index 0000000..965de92 Binary files /dev/null and b/etl/osmosis_data/asset_list.xlsx differ diff --git a/etl/osmosis_monday_to_sharepoint_automation.py b/etl/osmosis_monday_to_sharepoint_automation.py index 70294d7..f9d2c25 100644 --- a/etl/osmosis_monday_to_sharepoint_automation.py +++ b/etl/osmosis_monday_to_sharepoint_automation.py @@ -1,16 +1,79 @@ import os +# osmsis keys +# os.environ["SHAREPOINT_CLIENT_ID"] = "6832a4c5-fb8c-4082-a746-4f51e1020f0d" +# os.environ["SHAREPOINT_CLIENT_SECRET"] = "xpC8Q~Frww48SM1V-D8lGy5iOY7P_cJ7FF3jgarQ" +# os.environ["SHAREPOINT_TENANT_ID"] = "10d5af8b-2cfd-4882-9ccd-b96e4812dacf" + +# warmfront keys os.environ["SHAREPOINT_CLIENT_ID"] = "895e3b77-b1d7-43ec-b18f-dcfe07cdfeaf" os.environ["SHAREPOINT_CLIENT_SECRET"] = "SOf8Q~-is4wdQiqvEEm9FlJQRAY9ELGaj5Qz-a6E" os.environ["SHAREPOINT_TENANT_ID"] = "c3f7519c-2719-4547-af04-6da6cbfd8f8f" +os.environ["SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID"] = "b5a51507-9427-4ee0-b03e-90ec7681e2d3" os.environ["JJC_SERVICE_SHAREPOINT_ID"] = "7fdd0485-bbf3-4b29-b30f-98c81c2a6284" from etl.scraper.scraper import SharePointInstaller from etl.scraper.scraper import SharePointScraper +import pandas as pd -jjc = SharePointScraper(SharePointInstaller.JJC) +osmosis = SharePointScraper(SharePointInstaller.BAXTER_KELLY) +parent_folder = "Jun-te Kim Automation Test" +osmosis.create_file(parent_folder, "/") + +asset_list = pd.read_excel("osmosis_data/asset_list.xlsx", sheet_name="2502 accent housing") + + +new_asset_list = [] + +# Create asset list and location +for index, address in asset_list.iterrows(): + webUrl = osmosis.create_file(address['Name'], parent_folder) + + first_folder = "1. Retrofit Assessment" + osmosis.create_file(first_folder, parent_folder + f"/{address['Name']}") + osmosis.create_file("A. Assessment", parent_folder + f"/{address['Name']}/{first_folder}") + osmosis.create_file("B. Air Tightness Tests", parent_folder + f"/{address['Name']}/{first_folder}") + + second_folder = "2. RC Mid-Term Plan" + osmosis.create_file(second_folder, parent_folder + f"/{address['Name']}") + osmosis.create_file("SAP", parent_folder + f"/{address['Name']}/{second_folder}") + + third_folder = "3. Retrofit Design" + osmosis.create_file(third_folder, parent_folder + f"/{address['Name']}") + osmosis.create_file('ss', parent_folder + f"/{address['Name']}/{third_folder}") + + fourth_folder = "4. Post EPC" + osmosis.create_file(fourth_folder, parent_folder + f"/{address['Name']}") + osmosis.create_file("POST EPC Photos", parent_folder + f"/{address['Name']}/{fourth_folder}") + + fifth_folder = "5. Trustmark Lodgement" + osmosis.create_file(fifth_folder, parent_folder + f"/{address['Name']}") + osmosis.create_file("1. Works", parent_folder + f"/{address['Name']}/{fifth_folder}") + osmosis.create_file("Doors", parent_folder + f"/{address['Name']}/{fifth_folder}/1. Works") + osmosis.create_file("Photos", parent_folder + f"/{address['Name']}/{fifth_folder}/1. Works/Doors") + osmosis.create_file("Survey", parent_folder + f"/{address['Name']}/{fifth_folder}/1. Works/Doors") + + osmosis.create_file("Loft Insulation", parent_folder + f"/{address['Name']}/{fifth_folder}/1. Works") + osmosis.create_file("Ventilation", parent_folder + f"/{address['Name']}/{fifth_folder}/1. Works") + osmosis.create_file("Windows", parent_folder + f"/{address['Name']}/{fifth_folder}/1. Works") + osmosis.create_file("Photos", parent_folder + f"/{address['Name']}/{fifth_folder}/1. Works/Windows") + osmosis.create_file("Survey", parent_folder + f"/{address['Name']}/{fifth_folder}/1. Works/Windows") + + osmosis.create_file("2. Required Documents", parent_folder + f"/{address['Name']}/{fifth_folder}") + osmosis.create_file("3. Additional Documents", parent_folder + f"/{address['Name']}/{fifth_folder}") + + asset_data = { + "Name": address['Name'], + "Sharepoint": f"Sharepoint Link - {webUrl}", + } + + new_asset_list.append(asset_data) + + +# Osmosist File strucutre + + +df = pd.DataFrame(new_asset_list) +df.to_csv("output.csv", index=False) -print(jjc.does_folder_exists_at("junte was here", "Tom Miller (JJ CRUMP)")) -data = jjc.create_file("junte was here", "Tom Miller (JJ CRUMP)") -data["webUrl"] \ No newline at end of file diff --git a/etl/scraper/scraper.py b/etl/scraper/scraper.py index 8651042..30917ff 100644 --- a/etl/scraper/scraper.py +++ b/etl/scraper/scraper.py @@ -24,8 +24,9 @@ class SharePointInstaller(Enum): SOUTH_COAST_INSULATION = os.getenv("SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID", None) JJC = os.getenv("JJC_SERVICE_SHAREPOINT_ID", None) SGEC = os.getenv("SGEC_SERVICE_SHAREPOINT_ID", None) - BAXTER_KELLY = os.getenv("BAXTER_KELLY_SERVICE_SHAREPOINT_ID", None) + BAXTER_KELLY = os.getenv("BAXTER_KELLY_SERVICE_SHAREPOINT_ID", "6f930bf3-572d-4f91-b1ae-ec536fa319e2") DOMNA = os.getenv("DOMNA_SHAREPOINT_ID", "8ab64924-ccde-4b56-b0dc-4e11596446e4") + OSMOSIS = os.getenv("OSMOSIS_SHAREPOINT_ID", "350a3b48-8311-4506-8abb-69bafc280d6f") class SharePointScraper(): """ @@ -107,12 +108,12 @@ class SharePointScraper(): def does_folder_exists_at(self, file_name, file_path): - for folders in self.get_folders_in_path(file_path)['value']: - if file_name.upper() in folders["name"].upper(): - return True - + folders = self.get_folders_in_path(file_path) + if 'value' in folders: + for folder in folders['value']: + if file_name.upper() in folder["name"].upper(): + return True return False - def create_file(self, file_name, at_path="/"): @@ -122,8 +123,13 @@ class SharePointScraper(): client_secret=self.sharepoint_client_secret, site_id=self.sharepoint_drive.value, ) + if self.does_folder_exists_at(file_name, at_path) is False: - return sharepoint_client.create_folder(file_name, at_path) + return sharepoint_client.create_folder(file_name, at_path)['webUrl'] + else: + for folders in self.get_folders_in_path(at_path)['value']: + if file_name.upper() in folders["name"].upper(): + return folders["webUrl"]