junte/survey-extraction
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/survey-extraction.git synced 2026-06-08 11:17:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

new api keys

Browse source
This commit is contained in:
Jun-te Kim 2025-10-16 20:26:41 +00:00
parent a5499fb784
commit a11cb417a8
21 changed files with 39 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.github/workflows/months_end.yml vendored
View file

@ -3,8 +3,9 @@ name: Months End
on:
schedule:
- cron: '0 7 * * 1' # Every Monday at 07:00 UTC
- cron: '0 7 29 * *' # On the 29th of every month at 07:00 UTC
- cron: '0 7 23 * *' # On the 23th of every month at 07:00 UTC
- cron: '0 7 26 * *' # On the 26th of every month at 07:00 UTC
- cron: '0 7 29 * *' # On the 29th of every month at 07:00 UTC
workflow_dispatch:
jobs:

13
etl/MonthEndUploader.py
View file

@ -1,13 +1,15 @@
import os
os.environ["SHAREPOINT_CLIENT_ID"] = "6832a4c5-fb8c-4082-a746-4f51e1020f0d"
os.environ["SHAREPOINT_CLIENT_SECRET"] = "xpC8Q~Frww48SM1V-D8lGy5iOY7P_cJ7FF3jgarQ"
os.environ["SHAREPOINT_TENANT_ID"] = "10d5af8b-2cfd-4882-9ccd-b96e4812dacf"
os.environ["SHAREPOINT_CLIENT_ID"] = "0e28c4f9-3e77-4571-8d63-df1857f4266a"
os.environ["SHAREPOINT_CLIENT_SECRET"] = "2s48Q~t8.pI-~rbtQaSCFcYY97Z3LiMYhuo0GaOb"
os.environ["SHAREPOINT_TENANT_ID"] = "6f080c63-8a66-4bbc-9d72-b85d5df30555"
from etl.scraper.scraper import SharePointScraper, SharePointInstaller
from datetime import datetime
def upload_to_month_end_folder(file_name_on_sp, local_file_path, add_to_path):
sharepoint = SharePointScraper(SharePointInstaller.OSMOSIS_WAVE_2)
sharepoint = SharePointScraper(SharePointInstaller.OSMOSIS_ACD)
sharepoint.get_folders_in_path("/")
parent_folder = "General/Junte Kim/month end"
today = datetime.today()
@ -32,8 +34,7 @@ def upload_to_month_end_folder(file_name_on_sp, local_file_path, add_to_path):
def upload_to_nick_folder(file_name_on_sp, local_file_path, add_to_path=None):
sharepoint = SharePointScraper(SharePointInstaller.OSMOSIS_WAVE_2)
sharepoint = SharePointScraper(SharePointInstaller.OSMOSIS_ACD)
parent_folder = "General/Junte Kim/For Nick"
today = datetime.today()
formatted_date = today.strftime("%Y-%m-%d%H-%M-S")

2
etl/month_end_automation_wave_2_layout.py
View file

@ -6,7 +6,7 @@ from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset
from pprint import pprint
import pandas as pd
import json
from MonthEndUploader import upload_to_month_end_folder
from etl.MonthEndUploader import upload_to_month_end_folder
import os

2
etl/month_end_automation_wave_2_no_10.py
View file

@ -6,7 +6,7 @@ from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset
from pprint import pprint
import pandas as pd
import json
from MonthEndUploader import upload_to_month_end_folder
from etl.MonthEndUploader import upload_to_month_end_folder
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
monday = MondayClient(monday_key)

2
etl/month_end_automation_wave_2_no_11.py
View file

@ -6,7 +6,7 @@ from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset
from pprint import pprint
import pandas as pd
import json
from MonthEndUploader import upload_to_month_end_folder
from etl.MonthEndUploader import upload_to_month_end_folder
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
monday = MondayClient(monday_key)

2
etl/month_end_automation_wave_2_no_12.py
View file

@ -6,7 +6,7 @@ from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset
from pprint import pprint
import pandas as pd
import json
from MonthEndUploader import upload_to_month_end_folder
from etl.MonthEndUploader import upload_to_month_end_folder
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
monday = MondayClient(monday_key)

2
etl/month_end_automation_wave_2_no_13.py
View file

@ -6,7 +6,7 @@ from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset
from pprint import pprint
import pandas as pd
import json
from MonthEndUploader import upload_to_month_end_folder
from etl.MonthEndUploader import upload_to_month_end_folder
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
monday = MondayClient(monday_key)

2
etl/month_end_automation_wave_2_no_14.py
View file

@ -6,7 +6,7 @@ from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset
from pprint import pprint
import pandas as pd
import json
from MonthEndUploader import upload_to_month_end_folder
from etl.MonthEndUploader import upload_to_month_end_folder
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
monday = MondayClient(monday_key)

2
etl/month_end_automation_wave_2_no_15.py
View file

@ -6,7 +6,7 @@ from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset
from pprint import pprint
import pandas as pd
import json
from MonthEndUploader import upload_to_month_end_folder
from etl.MonthEndUploader import upload_to_month_end_folder
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
monday = MondayClient(monday_key)

2
etl/month_end_automation_wave_2_no_16.py
View file

@ -6,7 +6,7 @@ from pprint import pprint
import pandas as pd
import json
import os
from MonthEndUploader import upload_to_month_end_folder
from etl.MonthEndUploader import upload_to_month_end_folder
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
monday = MondayClient(monday_key)

2
etl/month_end_automation_wave_2_no_3.py
View file

@ -2,7 +2,7 @@
import os
from tqdm import tqdm
from monday import MondayClient
from MonthEndUploader import upload_to_month_end_folder
from etl.MonthEndUploader import upload_to_month_end_folder
from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
from pprint import pprint
import pandas as pd

2
etl/month_end_automation_wave_2_no_4.py
View file

@ -6,7 +6,7 @@ from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset
from pprint import pprint
import pandas as pd
import json
from MonthEndUploader import upload_to_month_end_folder
from etl.MonthEndUploader import upload_to_month_end_folder
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
monday = MondayClient(monday_key)

2
etl/month_end_automation_wave_2_no_5.py
View file

@ -6,7 +6,7 @@ from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset
from pprint import pprint
import pandas as pd
import json
from MonthEndUploader import upload_to_month_end_folder
from etl.MonthEndUploader import upload_to_month_end_folder
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
monday = MondayClient(monday_key)

2
etl/month_end_automation_wave_2_no_6.py
View file

@ -2,7 +2,7 @@
import os
from tqdm import tqdm
from monday import MondayClient
from MonthEndUploader import upload_to_month_end_folder
from etl.MonthEndUploader import upload_to_month_end_folder
from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
from pprint import pprint
import pandas as pd

2
etl/month_end_automation_wave_2_no_7.py
View file

@ -6,7 +6,7 @@ from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset
from pprint import pprint
import pandas as pd
import json
from MonthEndUploader import upload_to_month_end_folder
from etl.MonthEndUploader import upload_to_month_end_folder
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
monday = MondayClient(monday_key)

2
etl/month_end_automation_wave_2_no_8.py
View file

@ -6,7 +6,7 @@ from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset
from pprint import pprint
import pandas as pd
import json
from MonthEndUploader import upload_to_month_end_folder
from etl.MonthEndUploader import upload_to_month_end_folder
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
monday = MondayClient(monday_key)

2
etl/month_end_automation_wave_2_no_9.py
View file

@ -6,7 +6,7 @@ from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset
from pprint import pprint
import pandas as pd
import json
from MonthEndUploader import upload_to_month_end_folder
from etl.MonthEndUploader import upload_to_month_end_folder
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
monday = MondayClient(monday_key)

2
etl/month_end_automation_wave_3_layout.py
View file

@ -7,7 +7,7 @@ from pprint import pprint
import pandas as pd
import json
import os
from MonthEndUploader import upload_to_month_end_folder
from etl.MonthEndUploader import upload_to_month_end_folder
from enum import Enum

2
etl/month_end_automation_wave_accent_housing.py
View file

@ -6,7 +6,7 @@ from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset
from pprint import pprint
import pandas as pd
import json
from MonthEndUploader import upload_to_month_end_folder
from etl.MonthEndUploader import upload_to_month_end_folder
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
monday = MondayClient(monday_key)

8
etl/osmosis_complaince_address_to_files.py
View file

@ -17,10 +17,10 @@ monday = MondayClient(monday_key)
# osmsis keys
os.environ["SHAREPOINT_CLIENT_ID"] = "6832a4c5-fb8c-4082-a746-4f51e1020f0d"
os.environ["SHAREPOINT_CLIENT_SECRET"] = "xpC8Q~Frww48SM1V-D8lGy5iOY7P_cJ7FF3jgarQ"
os.environ["SHAREPOINT_TENANT_ID"] = "10d5af8b-2cfd-4882-9ccd-b96e4812dacf"
osmosis = SharePointScraper(SharePointInstaller.OSMOSIS_WAVE_2)
os.environ["SHAREPOINT_CLIENT_ID"] = "0e28c4f9-3e77-4571-8d63-df1857f4266a"
os.environ["SHAREPOINT_CLIENT_SECRET"] = "7935eb53-31e1-4eb4-bcbc-9c5ec408d4b0"
os.environ["SHAREPOINT_TENANT_ID"] = "6f080c63-8a66-4bbc-9d72-b85d5df30555"
osmosis = SharePointScraper(SharePointInstaller.OSMOSIS_ACD)
parent_folder = "/Osmosis ACD/Osmosis ACD Projects/"
# Change this per installer

17
etl/scraper/scraper.py
View file

@ -20,15 +20,16 @@ WEEK_COMMENCING = os.getenv("WEEK_COMMENCING", previous_monday())
class SharePointInstaller(Enum):
# https//{tenant}.sharepoint.com/sites/{site}/_api/site/id
SOUTH_COAST_INSULATION = os.getenv("SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID", "b5a51507-9427-4ee0-b03e-90ec7681e2d3")
JJC = os.getenv("JJC_SERVICE_SHAREPOINT_ID", "7fdd0485-bbf3-4b29-b30f-98c81c2a6284")
SGEC = os.getenv("SGEC_SERVICE_SHAREPOINT_ID", "52018e5c-3215-4fe4-a4e3-bbf0d0aa7cd9")
BAXTER_KELLY = os.getenv("BAXTER_KELLY_SERVICE_SHAREPOINT_ID", "6f930bf3-572d-4f91-b1ae-ec536fa319e2")
# SOUTH_COAST_INSULATION = os.getenv("SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID", "b5a51507-9427-4ee0-b03e-90ec7681e2d3")
# JJC = os.getenv("JJC_SERVICE_SHAREPOINT_ID", "7fdd0485-bbf3-4b29-b30f-98c81c2a6284")
# SGEC = os.getenv("SGEC_SERVICE_SHAREPOINT_ID", "52018e5c-3215-4fe4-a4e3-bbf0d0aa7cd9")
# BAXTER_KELLY = os.getenv("BAXTER_KELLY_SERVICE_SHAREPOINT_ID", "6f930bf3-572d-4f91-b1ae-ec536fa319e2")
DOMNA = os.getenv("DOMNA_SHAREPOINT_ID", "8ab64924-ccde-4b56-b0dc-4e11596446e4")
OSMOSIS_WAVE_3 = os.getenv("OSMOSIS_SHAREPOINT_ID", "350a3b48-8311-4506-8abb-69bafc280d6f")
OSMOSIS_WAVE_2 = os.getenv("OSMOSIS_SHAREPOINT_ID", "bc925a9a-ad0b-4de9-9a3c-e61014cc7489")
WARMFRONT = os.getenv("WARMFRONT_SHARPOINT_ID", "bea71c30-d366-454c-a484-ae4d6fd95bc4")
NEW_JJC = os.getenv("NEW JJC", "10d96eba-b4f9-4e30-804f-05a8b60507b0")
# OSMOSIS_WAVE_3 = os.getenv("OSMOSIS_SHAREPOINT_ID", "350a3b48-8311-4506-8abb-69bafc280d6f")
# OSMOSIS_WAVE_2 = os.getenv("OSMOSIS_SHAREPOINT_ID", "bc925a9a-ad0b-4de9-9a3c-e61014cc7489")
# WARMFRONT = os.getenv("WARMFRONT_SHARPOINT_ID", "bea71c30-d366-454c-a484-ae4d6fd95bc4")
# NEW_JJC = os.getenv("NEW JJC", "10d96eba-b4f9-4e30-804f-05a8b60507b0")
OSMOSIS_ACD = os.getenv("OSMOSIS_ACD_SHAREPOINT_ID", "931c4361-681b-44e4-86f6-1a54aba3ae8a")
class SharePointScraper():
"""