From 2057b5cd67f7909d5bc6fb7441dc9ad53bf74eff Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 23 Oct 2025 13:44:15 +0000 Subject: [PATCH] month end --- .github/workflows/months_end.yml | 2 -- etl/month_end_automation_wave_2_no_7.py | 33 +++++++++++------------ etl/month_end_automation_wave_3_layout.py | 20 +++++++------- 3 files changed, 26 insertions(+), 29 deletions(-) diff --git a/.github/workflows/months_end.yml b/.github/workflows/months_end.yml index 11274e5..86cc0ed 100644 --- a/.github/workflows/months_end.yml +++ b/.github/workflows/months_end.yml @@ -37,8 +37,6 @@ jobs: poetry run python etl/month_end_automation_wave_2_no_7.py # Done poetry run python etl/month_end_automation_wave_2_no_8.py # Done poetry run python etl/month_end_automation_wave_2_no_11.py # Done - poetry run python etl/month_end_automation_wave_2_no_12.py # Done - poetry run python etl/month_end_automation_wave_2_no_14.py # Done poetry run python etl/month_end_automation_wave_2_no_15.py # Done poetry run python etl/month_end_automation_wave_accent_housing.py # Done poetry run python etl/month_end_automation_wave_3_layout.py # Done diff --git a/etl/month_end_automation_wave_2_no_7.py b/etl/month_end_automation_wave_2_no_7.py index a8e8c10..31b4f31 100644 --- a/etl/month_end_automation_wave_2_no_7.py +++ b/etl/month_end_automation_wave_2_no_7.py @@ -252,7 +252,7 @@ other_df = final_df[~final_df.index.isin(net_zero_df.index) & ~final_df.index.is combined_with_rates_net_zero_df = net_zero_df.merge(rate_card_df_net_zero, on="job_type", how="left") combined_with_rates_sp_plus = sp_plus_df.merge(rate_card_df_sp_plus, on="job_type", how="left") -combined_with_rates_other_from_home_group = other_df.merge(rate_card_df_error_message, on="job_type", how="left") +combined_with_rates_other_from_home_group = other_df.merge(rate_card_df_net_zero, on="job_type", how="left") import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') @@ -264,33 +264,32 @@ combined_with_rates_other_from_home_group[attribute].to_excel(f'HomeGroup Wave U master_folder_name = "Home Group" -# Wave 2SP+ +# --- Combine Unsure + Net Zero --- +combined_net_zero_with_unsure = pd.concat([ + combined_with_rates_net_zero_df, + combined_with_rates_other_from_home_group +], ignore_index=True) + +# --- Wave 2SP+ --- file_name = f"{master_folder_name}_Wave 2SP+_{timestamp}.xlsx" combined_with_rates_sp_plus[attribute].to_excel(file_name, index=False) file_path = os.path.abspath(file_name) upload_to_month_end_folder(file_name, file_path, master_folder_name) + invoice_name = "wave_2sp_plus_rate_card.xlsx" file_path = os.path.abspath(invoice_name) rate_card_df_sp_plus.to_excel(invoice_name, index=False) upload_to_month_end_folder(invoice_name, file_path, master_folder_name) -# Net Zero -file_name = f"{master_folder_name}_NetZero_{timestamp}.xlsx" -combined_with_rates_net_zero_df[attribute].to_excel(file_name, index=False) +# --- Net Zero (now includes Unsure) --- +file_name = f"{master_folder_name}_NetZero_plus_Unsure_{timestamp}.xlsx" +combined_net_zero_with_unsure[attribute].to_excel(file_name, index=False) file_path = os.path.abspath(file_name) upload_to_month_end_folder(file_name, file_path, master_folder_name) -invoice_name = "net_zero_rate_card.xlsx" -file_path = os.path.abspath(invoice_name) -rate_card_df_net_zero.to_excel(invoice_name, index=False) -upload_to_month_end_folder(invoice_name, file_path, master_folder_name) -# Unsure -file_name = f"{master_folder_name}_unsure_who_to_bill_{timestamp}.xlsx" -combined_with_rates_other_from_home_group[attribute].to_excel(file_name, index=False) -file_path = os.path.abspath(file_name) -upload_to_month_end_folder(file_name, file_path, master_folder_name) -invoice_name = "unsure_who_to_bill_rate_card.xlsx" +invoice_name = "net_zero_plus_unsure_rate_card.xlsx" file_path = os.path.abspath(invoice_name) -rate_card_df_error_message.to_excel(invoice_name, index=False) -upload_to_month_end_folder(invoice_name, file_path, master_folder_name) +combined_rate_card = pd.concat([rate_card_df_net_zero, rate_card_df_error_message], ignore_index=True) +combined_rate_card.to_excel(invoice_name, index=False) +upload_to_month_end_folder(invoice_name, file_path, master_folder_name) \ No newline at end of file diff --git a/etl/month_end_automation_wave_3_layout.py b/etl/month_end_automation_wave_3_layout.py index e698961..aac0657 100644 --- a/etl/month_end_automation_wave_3_layout.py +++ b/etl/month_end_automation_wave_3_layout.py @@ -26,12 +26,12 @@ class BoardID(Enum): board_ids = [ "9349630181", # WCHG Walkups-Operations - "8830772914", # "L&Q London" - "9601691730", # Cardo Wales & West - Wave 3 - "9660895490", # Northumberland County SHDF Wave 3 - "9641491000", # Watford Warm Homes - "9671463094", # Seddon - "9929454382", # NCHA SHDF Wave 3 Main Operation Board + # "8830772914", # "L&Q London" + # "9601691730", # Cardo Wales & West - Wave 3 + # "9660895490", # Northumberland County SHDF Wave 3 + # "9641491000", # Watford Warm Homes + # "9671463094", # Seddon + # "9929454382", # NCHA SHDF Wave 3 Main Operation Board ] empty = "Rate card info missing" @@ -63,10 +63,10 @@ rate_card_data_ncha_wave_3 = { "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling", "design type not specified" ], "rate": [ - empty, empty, empty, empty, empty, - empty, empty, empty, empty, empty, - empty, empty, empty, empty, empty, - empty, empty, empty, empty, empty, check_with_andres + 259, 125, 280, 125, 125, + 650, 415, 195, 175, 135, + 120, "60 - check if EPC or EPR", "150 - check if EPC or EPR", 125, 60, + 45, 45, 45, empty, empty, check_with_andres ] } rate_cards.update({"9929454382": pd.DataFrame(rate_card_data_ncha_wave_3)})