diff --git a/etl/osmosis_monday_to_sharepoint_automation.py b/etl/osmosis_monday_to_sharepoint_automation.py index 7647672..28c0f0c 100644 --- a/etl/osmosis_monday_to_sharepoint_automation.py +++ b/etl/osmosis_monday_to_sharepoint_automation.py @@ -48,7 +48,7 @@ def retry(max_attempts: int = 3, base_delay: float = 1.0): # Folder Creation Logic (Wrapped with retry) # -------------------------------------------------- -@retry(max_attempts=3) +@retry(max_attempts=5) def process_asset(address: pd.Series): folder_name = f"{address['Name']} {address['Postcode']}" print(f"\nšŸ“ Processing {folder_name}") @@ -97,7 +97,7 @@ def process_asset(address: pd.Series): results = [] failed_rows = [] -MAX_WORKERS = 5 +MAX_WORKERS = 10 with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor: futures = [ @@ -121,26 +121,7 @@ print(f"āŒ Failed: {len(failed_rows)}") if failed_rows: pd.DataFrame(failed_rows).to_excel("failed_rows.xlsx", index=False) print("šŸ“„ Saved failed rows to failed_rows.xlsx") -# # Run this is you just want to get url -# def just_url(asset_list): -# new_asset_list = [] -# for index, address in tqdm(asset_list.iterrows()): -# folder_name = address['Name'] + " " + address['Postcode'] -# webUrl = osmosis.create_dir(folder_name, parent_folder) -# asset_data = { -# "Name": address['Name'], -# "Postcode": address['Postcode'], -# "Sharepoint": webUrl, -# } -# print(asset_data) - -# new_asset_list.append(asset_data) -# return new_asset_list - -# # new_asset_list = just_url(asset_list=asset_list) - -# df = pd.DataFrame(new_asset_list) -# df.to_csv("output.csv", index=False) - - - +if results: + results_df = pd.DataFrame(results) + results_df.to_csv("sharepoint_results.csv", index=False) + print("šŸ“„ Saved successful results to sharepoint_results.csv")