diff --git a/.github/workflows/months_end.yml b/.github/workflows/months_end.yml index 32ed301..11274e5 100644 --- a/.github/workflows/months_end.yml +++ b/.github/workflows/months_end.yml @@ -3,8 +3,9 @@ name: Months End on: schedule: - cron: '0 7 * * 1' # Every Monday at 07:00 UTC - - cron: '0 7 29 * *' # On the 29th of every month at 07:00 UTC + - cron: '0 7 23 * *' # On the 23th of every month at 07:00 UTC - cron: '0 7 26 * *' # On the 26th of every month at 07:00 UTC + - cron: '0 7 29 * *' # On the 29th of every month at 07:00 UTC workflow_dispatch: jobs: diff --git a/etl/MonthEndUploader.py b/etl/MonthEndUploader.py index c3f4f65..943286d 100644 --- a/etl/MonthEndUploader.py +++ b/etl/MonthEndUploader.py @@ -1,13 +1,15 @@ import os -os.environ["SHAREPOINT_CLIENT_ID"] = "6832a4c5-fb8c-4082-a746-4f51e1020f0d" -os.environ["SHAREPOINT_CLIENT_SECRET"] = "xpC8Q~Frww48SM1V-D8lGy5iOY7P_cJ7FF3jgarQ" -os.environ["SHAREPOINT_TENANT_ID"] = "10d5af8b-2cfd-4882-9ccd-b96e4812dacf" +os.environ["SHAREPOINT_CLIENT_ID"] = "0e28c4f9-3e77-4571-8d63-df1857f4266a" +os.environ["SHAREPOINT_CLIENT_SECRET"] = "2s48Q~t8.pI-~rbtQaSCFcYY97Z3LiMYhuo0GaOb" +os.environ["SHAREPOINT_TENANT_ID"] = "6f080c63-8a66-4bbc-9d72-b85d5df30555" + from etl.scraper.scraper import SharePointScraper, SharePointInstaller from datetime import datetime def upload_to_month_end_folder(file_name_on_sp, local_file_path, add_to_path): - sharepoint = SharePointScraper(SharePointInstaller.OSMOSIS_WAVE_2) + sharepoint = SharePointScraper(SharePointInstaller.OSMOSIS_ACD) + sharepoint.get_folders_in_path("/") parent_folder = "General/Junte Kim/month end" today = datetime.today() @@ -32,8 +34,7 @@ def upload_to_month_end_folder(file_name_on_sp, local_file_path, add_to_path): def upload_to_nick_folder(file_name_on_sp, local_file_path, add_to_path=None): - sharepoint = SharePointScraper(SharePointInstaller.OSMOSIS_WAVE_2) - + sharepoint = SharePointScraper(SharePointInstaller.OSMOSIS_ACD) parent_folder = "General/Junte Kim/For Nick" today = datetime.today() formatted_date = today.strftime("%Y-%m-%d%H-%M-S") diff --git a/etl/month_end_automation_wave_2_layout.py b/etl/month_end_automation_wave_2_layout.py index 1fb6dd1..1dec182 100644 --- a/etl/month_end_automation_wave_2_layout.py +++ b/etl/month_end_automation_wave_2_layout.py @@ -6,7 +6,7 @@ from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset from pprint import pprint import pandas as pd import json -from MonthEndUploader import upload_to_month_end_folder +from etl.MonthEndUploader import upload_to_month_end_folder import os diff --git a/etl/month_end_automation_wave_2_no_10.py b/etl/month_end_automation_wave_2_no_10.py index 0ca72eb..16f4819 100644 --- a/etl/month_end_automation_wave_2_no_10.py +++ b/etl/month_end_automation_wave_2_no_10.py @@ -6,7 +6,7 @@ from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset from pprint import pprint import pandas as pd import json -from MonthEndUploader import upload_to_month_end_folder +from etl.MonthEndUploader import upload_to_month_end_folder monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) diff --git a/etl/month_end_automation_wave_2_no_11.py b/etl/month_end_automation_wave_2_no_11.py index a5a7f0a..253678e 100644 --- a/etl/month_end_automation_wave_2_no_11.py +++ b/etl/month_end_automation_wave_2_no_11.py @@ -6,7 +6,7 @@ from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset from pprint import pprint import pandas as pd import json -from MonthEndUploader import upload_to_month_end_folder +from etl.MonthEndUploader import upload_to_month_end_folder monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) diff --git a/etl/month_end_automation_wave_2_no_12.py b/etl/month_end_automation_wave_2_no_12.py index e8c5462..1adcdbb 100644 --- a/etl/month_end_automation_wave_2_no_12.py +++ b/etl/month_end_automation_wave_2_no_12.py @@ -6,7 +6,7 @@ from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset from pprint import pprint import pandas as pd import json -from MonthEndUploader import upload_to_month_end_folder +from etl.MonthEndUploader import upload_to_month_end_folder monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) diff --git a/etl/month_end_automation_wave_2_no_13.py b/etl/month_end_automation_wave_2_no_13.py index 53535d3..ed2fdaf 100644 --- a/etl/month_end_automation_wave_2_no_13.py +++ b/etl/month_end_automation_wave_2_no_13.py @@ -6,7 +6,7 @@ from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset from pprint import pprint import pandas as pd import json -from MonthEndUploader import upload_to_month_end_folder +from etl.MonthEndUploader import upload_to_month_end_folder monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) diff --git a/etl/month_end_automation_wave_2_no_14.py b/etl/month_end_automation_wave_2_no_14.py index 150dcdd..80fa7aa 100644 --- a/etl/month_end_automation_wave_2_no_14.py +++ b/etl/month_end_automation_wave_2_no_14.py @@ -6,7 +6,7 @@ from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset from pprint import pprint import pandas as pd import json -from MonthEndUploader import upload_to_month_end_folder +from etl.MonthEndUploader import upload_to_month_end_folder monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) diff --git a/etl/month_end_automation_wave_2_no_15.py b/etl/month_end_automation_wave_2_no_15.py index e5e5d55..50338f9 100644 --- a/etl/month_end_automation_wave_2_no_15.py +++ b/etl/month_end_automation_wave_2_no_15.py @@ -6,7 +6,7 @@ from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset from pprint import pprint import pandas as pd import json -from MonthEndUploader import upload_to_month_end_folder +from etl.MonthEndUploader import upload_to_month_end_folder monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) diff --git a/etl/month_end_automation_wave_2_no_16.py b/etl/month_end_automation_wave_2_no_16.py index 70b5175..e116c72 100644 --- a/etl/month_end_automation_wave_2_no_16.py +++ b/etl/month_end_automation_wave_2_no_16.py @@ -6,7 +6,7 @@ from pprint import pprint import pandas as pd import json import os -from MonthEndUploader import upload_to_month_end_folder +from etl.MonthEndUploader import upload_to_month_end_folder monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) diff --git a/etl/month_end_automation_wave_2_no_3.py b/etl/month_end_automation_wave_2_no_3.py index b3b82a4..424834c 100644 --- a/etl/month_end_automation_wave_2_no_3.py +++ b/etl/month_end_automation_wave_2_no_3.py @@ -2,7 +2,7 @@ import os from tqdm import tqdm from monday import MondayClient -from MonthEndUploader import upload_to_month_end_folder +from etl.MonthEndUploader import upload_to_month_end_folder from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids from pprint import pprint import pandas as pd diff --git a/etl/month_end_automation_wave_2_no_4.py b/etl/month_end_automation_wave_2_no_4.py index f376330..dd45c07 100644 --- a/etl/month_end_automation_wave_2_no_4.py +++ b/etl/month_end_automation_wave_2_no_4.py @@ -6,7 +6,7 @@ from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset from pprint import pprint import pandas as pd import json -from MonthEndUploader import upload_to_month_end_folder +from etl.MonthEndUploader import upload_to_month_end_folder monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) diff --git a/etl/month_end_automation_wave_2_no_5.py b/etl/month_end_automation_wave_2_no_5.py index da06633..75612eb 100644 --- a/etl/month_end_automation_wave_2_no_5.py +++ b/etl/month_end_automation_wave_2_no_5.py @@ -6,7 +6,7 @@ from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset from pprint import pprint import pandas as pd import json -from MonthEndUploader import upload_to_month_end_folder +from etl.MonthEndUploader import upload_to_month_end_folder monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) diff --git a/etl/month_end_automation_wave_2_no_6.py b/etl/month_end_automation_wave_2_no_6.py index 064e676..3a1169b 100644 --- a/etl/month_end_automation_wave_2_no_6.py +++ b/etl/month_end_automation_wave_2_no_6.py @@ -2,7 +2,7 @@ import os from tqdm import tqdm from monday import MondayClient -from MonthEndUploader import upload_to_month_end_folder +from etl.MonthEndUploader import upload_to_month_end_folder from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids from pprint import pprint import pandas as pd diff --git a/etl/month_end_automation_wave_2_no_7.py b/etl/month_end_automation_wave_2_no_7.py index 023ee94..a8e8c10 100644 --- a/etl/month_end_automation_wave_2_no_7.py +++ b/etl/month_end_automation_wave_2_no_7.py @@ -6,7 +6,7 @@ from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset from pprint import pprint import pandas as pd import json -from MonthEndUploader import upload_to_month_end_folder +from etl.MonthEndUploader import upload_to_month_end_folder monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) diff --git a/etl/month_end_automation_wave_2_no_8.py b/etl/month_end_automation_wave_2_no_8.py index 7c2dd56..1d5784b 100644 --- a/etl/month_end_automation_wave_2_no_8.py +++ b/etl/month_end_automation_wave_2_no_8.py @@ -6,7 +6,7 @@ from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset from pprint import pprint import pandas as pd import json -from MonthEndUploader import upload_to_month_end_folder +from etl.MonthEndUploader import upload_to_month_end_folder monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) diff --git a/etl/month_end_automation_wave_2_no_9.py b/etl/month_end_automation_wave_2_no_9.py index ad01a92..f28a27d 100644 --- a/etl/month_end_automation_wave_2_no_9.py +++ b/etl/month_end_automation_wave_2_no_9.py @@ -6,7 +6,7 @@ from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset from pprint import pprint import pandas as pd import json -from MonthEndUploader import upload_to_month_end_folder +from etl.MonthEndUploader import upload_to_month_end_folder monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) diff --git a/etl/month_end_automation_wave_3_layout.py b/etl/month_end_automation_wave_3_layout.py index bfa2004..4db67a5 100644 --- a/etl/month_end_automation_wave_3_layout.py +++ b/etl/month_end_automation_wave_3_layout.py @@ -7,7 +7,7 @@ from pprint import pprint import pandas as pd import json import os -from MonthEndUploader import upload_to_month_end_folder +from etl.MonthEndUploader import upload_to_month_end_folder from enum import Enum diff --git a/etl/month_end_automation_wave_accent_housing.py b/etl/month_end_automation_wave_accent_housing.py index ad1994b..318c208 100644 --- a/etl/month_end_automation_wave_accent_housing.py +++ b/etl/month_end_automation_wave_accent_housing.py @@ -6,7 +6,7 @@ from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset from pprint import pprint import pandas as pd import json -from MonthEndUploader import upload_to_month_end_folder +from etl.MonthEndUploader import upload_to_month_end_folder monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) diff --git a/etl/osmosis_complaince_address_to_files.py b/etl/osmosis_complaince_address_to_files.py index c48e4eb..ceee811 100644 --- a/etl/osmosis_complaince_address_to_files.py +++ b/etl/osmosis_complaince_address_to_files.py @@ -17,10 +17,10 @@ monday = MondayClient(monday_key) # osmsis keys -os.environ["SHAREPOINT_CLIENT_ID"] = "6832a4c5-fb8c-4082-a746-4f51e1020f0d" -os.environ["SHAREPOINT_CLIENT_SECRET"] = "xpC8Q~Frww48SM1V-D8lGy5iOY7P_cJ7FF3jgarQ" -os.environ["SHAREPOINT_TENANT_ID"] = "10d5af8b-2cfd-4882-9ccd-b96e4812dacf" -osmosis = SharePointScraper(SharePointInstaller.OSMOSIS_WAVE_2) +os.environ["SHAREPOINT_CLIENT_ID"] = "0e28c4f9-3e77-4571-8d63-df1857f4266a" +os.environ["SHAREPOINT_CLIENT_SECRET"] = "7935eb53-31e1-4eb4-bcbc-9c5ec408d4b0" +os.environ["SHAREPOINT_TENANT_ID"] = "6f080c63-8a66-4bbc-9d72-b85d5df30555" +osmosis = SharePointScraper(SharePointInstaller.OSMOSIS_ACD) parent_folder = "/Osmosis ACD/Osmosis ACD Projects/" # Change this per installer diff --git a/etl/scraper/scraper.py b/etl/scraper/scraper.py index 58f9e09..2fc5545 100644 --- a/etl/scraper/scraper.py +++ b/etl/scraper/scraper.py @@ -20,15 +20,16 @@ WEEK_COMMENCING = os.getenv("WEEK_COMMENCING", previous_monday()) class SharePointInstaller(Enum): # https//{tenant}.sharepoint.com/sites/{site}/_api/site/id - SOUTH_COAST_INSULATION = os.getenv("SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID", "b5a51507-9427-4ee0-b03e-90ec7681e2d3") - JJC = os.getenv("JJC_SERVICE_SHAREPOINT_ID", "7fdd0485-bbf3-4b29-b30f-98c81c2a6284") - SGEC = os.getenv("SGEC_SERVICE_SHAREPOINT_ID", "52018e5c-3215-4fe4-a4e3-bbf0d0aa7cd9") - BAXTER_KELLY = os.getenv("BAXTER_KELLY_SERVICE_SHAREPOINT_ID", "6f930bf3-572d-4f91-b1ae-ec536fa319e2") + # SOUTH_COAST_INSULATION = os.getenv("SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID", "b5a51507-9427-4ee0-b03e-90ec7681e2d3") + # JJC = os.getenv("JJC_SERVICE_SHAREPOINT_ID", "7fdd0485-bbf3-4b29-b30f-98c81c2a6284") + # SGEC = os.getenv("SGEC_SERVICE_SHAREPOINT_ID", "52018e5c-3215-4fe4-a4e3-bbf0d0aa7cd9") + # BAXTER_KELLY = os.getenv("BAXTER_KELLY_SERVICE_SHAREPOINT_ID", "6f930bf3-572d-4f91-b1ae-ec536fa319e2") DOMNA = os.getenv("DOMNA_SHAREPOINT_ID", "8ab64924-ccde-4b56-b0dc-4e11596446e4") - OSMOSIS_WAVE_3 = os.getenv("OSMOSIS_SHAREPOINT_ID", "350a3b48-8311-4506-8abb-69bafc280d6f") - OSMOSIS_WAVE_2 = os.getenv("OSMOSIS_SHAREPOINT_ID", "bc925a9a-ad0b-4de9-9a3c-e61014cc7489") - WARMFRONT = os.getenv("WARMFRONT_SHARPOINT_ID", "bea71c30-d366-454c-a484-ae4d6fd95bc4") - NEW_JJC = os.getenv("NEW JJC", "10d96eba-b4f9-4e30-804f-05a8b60507b0") + # OSMOSIS_WAVE_3 = os.getenv("OSMOSIS_SHAREPOINT_ID", "350a3b48-8311-4506-8abb-69bafc280d6f") + # OSMOSIS_WAVE_2 = os.getenv("OSMOSIS_SHAREPOINT_ID", "bc925a9a-ad0b-4de9-9a3c-e61014cc7489") + # WARMFRONT = os.getenv("WARMFRONT_SHARPOINT_ID", "bea71c30-d366-454c-a484-ae4d6fd95bc4") + # NEW_JJC = os.getenv("NEW JJC", "10d96eba-b4f9-4e30-804f-05a8b60507b0") + OSMOSIS_ACD = os.getenv("OSMOSIS_ACD_SHAREPOINT_ID", "931c4361-681b-44e4-86f6-1a54aba3ae8a") class SharePointScraper(): """