diff --git a/.github/workflows/months_end.yml b/.github/workflows/months_end.yml index 758045b..80955ad 100644 --- a/.github/workflows/months_end.yml +++ b/.github/workflows/months_end.yml @@ -39,5 +39,9 @@ jobs: poetry run python etl/month_end_automation_wave_2_no_11.py poetry run python etl/month_end_automation_wave_2_no_12.py poetry run python etl/month_end_automation_wave_2_no_13.py + poetry run python etl/month_end_automation_wave_2_no_14.py + poetry run python etl/month_end_automation_wave_2_no_15.py + poetry run python etl/month_end_automation_wave_2_no_16.py + poetry run python etl/month_end_automation_wave_accent_housing.py env: PYTHONPATH: ${{ github.workspace }} \ No newline at end of file diff --git a/etl/month_end_automation_wave_2_no_14.py b/etl/month_end_automation_wave_2_no_14.py index 27cadd0..1099185 100644 --- a/etl/month_end_automation_wave_2_no_14.py +++ b/etl/month_end_automation_wave_2_no_14.py @@ -1,10 +1,12 @@ # Wave 2's month end automation +import os from tqdm import tqdm from monday import MondayClient from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids from pprint import pprint import pandas as pd import json +from MonthEndUploader import upload_to_month_end_folder monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) @@ -202,5 +204,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') +# Upload to sharepoint attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'Decent Homes Stonewater - Operations_{timestamp}.xlsx', index=False) +master_folder_name = "Decent Homes Stonewater - Operations" +file_name = f"{master_folder_name}_{timestamp}.xlsx" +combined_with_rates[attribute].to_excel(file_name, index=False) + +file_path = os.path.abspath(file_name) + +upload_to_month_end_folder(file_name, file_path, master_folder_name) + +invoice_name = "rate_card.xlsx" +file_path = os.path.abspath(invoice_name) +rate_card_df.to_excel(invoice_name, index=False) +upload_to_month_end_folder(invoice_name, file_path, master_folder_name) diff --git a/etl/month_end_automation_wave_2_no_15.py b/etl/month_end_automation_wave_2_no_15.py index 8644db9..1b10f25 100644 --- a/etl/month_end_automation_wave_2_no_15.py +++ b/etl/month_end_automation_wave_2_no_15.py @@ -1,10 +1,12 @@ # Wave 2's month end automation +import os from tqdm import tqdm from monday import MondayClient from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids from pprint import pprint import pandas as pd import json +from MonthEndUploader import upload_to_month_end_folder monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) @@ -210,5 +212,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') +# Upload to sharepoint attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'WCHG SHDF 2.1 Mansard {timestamp}.xlsx', index=False) +master_folder_name = "WCHG SHDF 2.1 Mansard" +file_name = f"{master_folder_name}_{timestamp}.xlsx" +combined_with_rates[attribute].to_excel(file_name, index=False) + +file_path = os.path.abspath(file_name) + +upload_to_month_end_folder(file_name, file_path, master_folder_name) + +invoice_name = "rate_card.xlsx" +file_path = os.path.abspath(invoice_name) +rate_card_df.to_excel(invoice_name, index=False) +upload_to_month_end_folder(invoice_name, file_path, master_folder_name) diff --git a/etl/month_end_automation_wave_2_no_16.py b/etl/month_end_automation_wave_2_no_16.py index 0af8bfe..664ebff 100644 --- a/etl/month_end_automation_wave_2_no_16.py +++ b/etl/month_end_automation_wave_2_no_16.py @@ -5,6 +5,8 @@ from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset from pprint import pprint import pandas as pd import json +import os +from MonthEndUploader import upload_to_month_end_folder monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) @@ -204,5 +206,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') +# Upload to sharepoint attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'NCHA SHDF Wave 3 On Hold_{timestamp}.xlsx', index=False) \ No newline at end of file +master_folder_name = "NCHA SHDF Wave 3 On Hold" +file_name = f"{master_folder_name}_{timestamp}.xlsx" +combined_with_rates[attribute].to_excel(file_name, index=False) + +file_path = os.path.abspath(file_name) + +upload_to_month_end_folder(file_name, file_path, master_folder_name) + +invoice_name = "rate_card.xlsx" +file_path = os.path.abspath(invoice_name) +rate_card_df.to_excel(invoice_name, index=False) +upload_to_month_end_folder(invoice_name, file_path, master_folder_name) diff --git a/etl/month_end_automation_wave_accent_housing.py b/etl/month_end_automation_wave_accent_housing.py index e02cdd9..b13d155 100644 --- a/etl/month_end_automation_wave_accent_housing.py +++ b/etl/month_end_automation_wave_accent_housing.py @@ -1,11 +1,12 @@ # Wave 3's month end automation - +import os from tqdm import tqdm from monday import MondayClient from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids from pprint import pprint import pandas as pd import json +from MonthEndUploader import upload_to_month_end_folder monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) @@ -88,7 +89,7 @@ if not second_payment.empty: full_cost = get_df(df, "mtp invoicing status", ["(v1) full cost mtp to invoice (no previous modelling)"], "full cost mtp") if not full_cost.empty: - filtered_dfs(full_cost) + filtered_dfs.append(full_cost) final_df = pd.concat(filtered_dfs).reset_index(drop=True) @@ -100,5 +101,17 @@ combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') +# Upload to sharepoint attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'2502 Accent housing {timestamp}.xlsx', index=False) +master_folder_name = "2501 Accent housing" +file_name = f"{master_folder_name}_{timestamp}.xlsx" +combined_with_rates[attribute].to_excel(file_name, index=False) + +file_path = os.path.abspath(file_name) + +upload_to_month_end_folder(file_name, file_path, master_folder_name) + +invoice_name = "rate_card.xlsx" +file_path = os.path.abspath(invoice_name) +rate_card_df.to_excel(invoice_name, index=False) +upload_to_month_end_folder(invoice_name, file_path, master_folder_name)