junte/survey-extraction
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/survey-extraction.git synced 2026-06-08 11:17:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

month end

Browse source
This commit is contained in:
Jun-te Kim 2025-08-29 12:03:15 +00:00
parent 7bfabf9e5c
commit f3abeeb1eb
5 changed files with 65 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.github/workflows/months_end.yml vendored
View file

@ -39,5 +39,9 @@ jobs:
poetry run python etl/month_end_automation_wave_2_no_11.py
poetry run python etl/month_end_automation_wave_2_no_12.py
poetry run python etl/month_end_automation_wave_2_no_13.py
poetry run python etl/month_end_automation_wave_2_no_14.py
poetry run python etl/month_end_automation_wave_2_no_15.py
poetry run python etl/month_end_automation_wave_2_no_16.py
poetry run python etl/month_end_automation_wave_accent_housing.py
env:
PYTHONPATH: ${{ github.workspace }}

16
etl/month_end_automation_wave_2_no_14.py
View file

@ -1,10 +1,12 @@
# Wave 2's month end automation
import os
from tqdm import tqdm
from monday import MondayClient
from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
from pprint import pprint
import pandas as pd
import json
from MonthEndUploader import upload_to_month_end_folder
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
monday = MondayClient(monday_key)
@ -202,5 +204,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left")
import datetime
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')
# Upload to sharepoint
attribute = ['address', 'client', 'job_type', 'rate']
combined_with_rates[attribute].to_excel(f'Decent Homes Stonewater - Operations_{timestamp}.xlsx', index=False)
master_folder_name = "Decent Homes Stonewater - Operations"
file_name = f"{master_folder_name}_{timestamp}.xlsx"
combined_with_rates[attribute].to_excel(file_name, index=False)
file_path = os.path.abspath(file_name)
upload_to_month_end_folder(file_name, file_path, master_folder_name)
invoice_name = "rate_card.xlsx"
file_path = os.path.abspath(invoice_name)
rate_card_df.to_excel(invoice_name, index=False)
upload_to_month_end_folder(invoice_name, file_path, master_folder_name)

16
etl/month_end_automation_wave_2_no_15.py
View file

@ -1,10 +1,12 @@
# Wave 2's month end automation
import os
from tqdm import tqdm
from monday import MondayClient
from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
from pprint import pprint
import pandas as pd
import json
from MonthEndUploader import upload_to_month_end_folder
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
monday = MondayClient(monday_key)
@ -210,5 +212,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left")
import datetime
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')
# Upload to sharepoint
attribute = ['address', 'client', 'job_type', 'rate']
combined_with_rates[attribute].to_excel(f'WCHG SHDF 2.1 Mansard {timestamp}.xlsx', index=False)
master_folder_name = "WCHG SHDF 2.1 Mansard"
file_name = f"{master_folder_name}_{timestamp}.xlsx"
combined_with_rates[attribute].to_excel(file_name, index=False)
file_path = os.path.abspath(file_name)
upload_to_month_end_folder(file_name, file_path, master_folder_name)
invoice_name = "rate_card.xlsx"
file_path = os.path.abspath(invoice_name)
rate_card_df.to_excel(invoice_name, index=False)
upload_to_month_end_folder(invoice_name, file_path, master_folder_name)

16
etl/month_end_automation_wave_2_no_16.py
View file

@ -5,6 +5,8 @@ from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset
from pprint import pprint
import pandas as pd
import json
import os
from MonthEndUploader import upload_to_month_end_folder
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
monday = MondayClient(monday_key)
@ -204,5 +206,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left")
import datetime
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')
# Upload to sharepoint
attribute = ['address', 'client', 'job_type', 'rate']
combined_with_rates[attribute].to_excel(f'NCHA SHDF Wave 3 On Hold_{timestamp}.xlsx', index=False)
master_folder_name = "NCHA SHDF Wave 3 On Hold"
file_name = f"{master_folder_name}_{timestamp}.xlsx"
combined_with_rates[attribute].to_excel(file_name, index=False)
file_path = os.path.abspath(file_name)
upload_to_month_end_folder(file_name, file_path, master_folder_name)
invoice_name = "rate_card.xlsx"
file_path = os.path.abspath(invoice_name)
rate_card_df.to_excel(invoice_name, index=False)
upload_to_month_end_folder(invoice_name, file_path, master_folder_name)

19
etl/month_end_automation_wave_accent_housing.py
View file

@ -1,11 +1,12 @@
# Wave 3's month end automation
import os
from tqdm import tqdm
from monday import MondayClient
from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
from pprint import pprint
import pandas as pd
import json
from MonthEndUploader import upload_to_month_end_folder
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
monday = MondayClient(monday_key)
@ -88,7 +89,7 @@ if not second_payment.empty:
full_cost = get_df(df, "mtp invoicing status", ["(v1) full cost mtp to invoice (no previous modelling)"], "full cost mtp")
if not full_cost.empty:
filtered_dfs(full_cost)
filtered_dfs.append(full_cost)
final_df = pd.concat(filtered_dfs).reset_index(drop=True)
@ -100,5 +101,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left")
import datetime
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')
# Upload to sharepoint
attribute = ['address', 'client', 'job_type', 'rate']
combined_with_rates[attribute].to_excel(f'2502 Accent housing {timestamp}.xlsx', index=False)
master_folder_name = "2501 Accent housing"
file_name = f"{master_folder_name}_{timestamp}.xlsx"
combined_with_rates[attribute].to_excel(file_name, index=False)
file_path = os.path.abspath(file_name)
upload_to_month_end_folder(file_name, file_path, master_folder_name)
invoice_name = "rate_card.xlsx"
file_path = os.path.abspath(invoice_name)
rate_card_df.to_excel(invoice_name, index=False)
upload_to_month_end_folder(invoice_name, file_path, master_folder_name)