From a751b8d3613aadb7c0be101414935a4ac23cd4c5 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 29 Jul 2025 13:13:45 +0000 Subject: [PATCH] month end --- etl/month_end_automation_wave_2_no_10.py | 63 +++++++++++++----- etl/month_end_automation_wave_2_no_5.py | 83 +++++++++++++++++------- etl/month_end_automation_wave_2_no_6.py | 61 ++++++++++++----- etl/month_end_automation_wave_2_no_9.py | 51 +++++++++++---- 4 files changed, 189 insertions(+), 69 deletions(-) diff --git a/etl/month_end_automation_wave_2_no_10.py b/etl/month_end_automation_wave_2_no_10.py index 29e5845..acba26b 100644 --- a/etl/month_end_automation_wave_2_no_10.py +++ b/etl/month_end_automation_wave_2_no_10.py @@ -11,6 +11,24 @@ monday = MondayClient(monday_key) # NCHA Derbyshire Dales (DDDCC) SHDF board_ids = ["6947307148"] +rate_card_data = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + 259, 125, 280, 125, 125, + 650, 415, 195, 175, 135, + 120, "Post EPR Please Marianne", 85, 125, 60, + 25, 25, 25 + ] +} + +rate_card_df = pd.DataFrame(rate_card_data) + + for board in tqdm(board_ids): board_data = monday.boards.fetch_boards_by_id(board) columns = board_data["data"]["boards"][0]["columns"] @@ -86,7 +104,7 @@ filtered_dfs.append(att) # V1 Coordination v1 = get_df(df, "coordination status".lower(), [ "rc complete", -], "V1 Coordination") +], "Coordination Stage 1 v1") filtered_dfs.append(v1) # V2 Coordination @@ -98,15 +116,15 @@ filtered_dfs.append(v1) # # filtered_dfs.append(v3) # Coordination stage 2 Please complete -# cors2 = df[ -# df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) -# ] -# cors2["joby_type"] = "Coordination Stage 2" -# filtered_dfs.append(cors2) +cors2 = df[ + df["rc stg. 2"].str.lower().isin(["to invoice"]) +] +cors2["joby_type"] = "Coordination Stage 2" +filtered_dfs.append(cors2) -# Design stage 1 -design1 = get_df(df, "design upload to sharepoint", ["done"], "Design") -filtered_dfs.append(design1) +# # Design stage 1 +# design1 = get_df(df, "design upload to sharepoint", ["done"], "Design") +# filtered_dfs.append(design1) # Design revision # design2 = get_df(df, "design revision invoice", [ @@ -123,7 +141,7 @@ filtered_dfs.append(lodg1) # Full Lodgement Phase -full_lodgement = get_df(df, "full lodgement invoice status".lower(), ["to invoice"], "Full Lodgement") +full_lodgement = get_df(df, "full lodgement invoice status".lower(), ["to invoice"], "Full lodgement phase 2") filtered_dfs.append(full_lodgement) # POST EPC @@ -131,12 +149,12 @@ post_epc = get_df(df, "lodged epc", ["complete"], "POST EPC") filtered_dfs.append(post_epc) -# # POST EPR -# post_epr = df[ -# df["post-epc status"].str.lower().isin(["post epr completed"]) -# ].copy() -# post_epr["job_type"] = "POST ATT" -# filtered_dfs.append(post_epr) +# POST EPR +post_epr = df[ + df["lodged epc"].str.lower().isin(["post epr completed"]) +].copy() +post_epr["job_type"] = "Post EPR" +filtered_dfs.append(post_epr) # Post ATT post_att = get_df(df, "post att", ["done"], "POST ATT") @@ -176,4 +194,15 @@ filtered_dfs.append(att_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) -final_df[['address', 'client', 'job_type']] \ No newline at end of file +final_df[['address', 'client', 'job_type']] + +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_excel(f'NCHA Derbyshire Dales (DDDCC) SHDF_{timestamp}.xlsx', index=False) diff --git a/etl/month_end_automation_wave_2_no_5.py b/etl/month_end_automation_wave_2_no_5.py index 5ed035b..5d40b55 100644 --- a/etl/month_end_automation_wave_2_no_5.py +++ b/etl/month_end_automation_wave_2_no_5.py @@ -12,6 +12,23 @@ monday = MondayClient(monday_key) # ECO 4 NCHA Almshouses Operations board_ids = ["9136254638"] +rate_card_data = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + 259, 125, 280, 125, 125, + 650, 195, 175, 135, + 120, "Post EPR Please Marianne", 85, 125, 60, + 25, 25, 25 + ] +} + +rate_card_df = pd.DataFrame(rate_card_data) + for board in tqdm(board_ids): board_data = monday.boards.fetch_boards_by_id(board) @@ -87,39 +104,50 @@ filtered_dfs.append(att) v1 = df[ df["coordination status"].str.lower().isin(["ioe/mtp complete"]) ].copy() -v1["job_type"] = "V1 Coordination" +v1["job_type"] = "Coordination Stage 1 v1" filtered_dfs.append(v1) # V2 Coordination # _ = df[df["mtp v2 invoiced"].str.lower().isin(['done', 'needs to be invoiced'])].copy() -# _["job_type"] = "V2 Coordination" +# _["job_type"] = "Coordination Stage 1 v2 remodel" # filtered_dfs.append(_) # V3 Coordination # v3 = df[df["v3 rc status"].str.lower().isin(['uploaded'])].copy() -# v3["job_type"] = "V3 Coordination" +# v3["job_type"] = "Coordination Stage 1 v3 remodel" # filtered_dfs.append(_) # Coordination stage 2 Please complete -# cors2 = df[ -# df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) -# ] -# cors2["joby_type"] = "Coordination Stage 2" -# filtered_dfs.append(cors2) +cors2 = df[ + df["rc stage 2"].str.lower().isin(["to invoice",]) +] +cors2["joby_type"] = "Coordination Stage 2" +filtered_dfs.append(cors2) # Design stage 1 design1 = df[ df["retrofit design status"].str.lower().isin(["to invoice"]) ].copy() -design1["job_type"] = "Design" +design1 = design1[design1["design type"].str.lower().isin(["archetype"])].copy() +design1["job_type"] = "Design Archetype" filtered_dfs.append(design1) # Design stage 2 -# design2 = df[ -# df[""].str.lower().isin(["to invoice"]) -# ].copy() -# design2["job_type"] = "Design Revision" -# filtered_dfs.append(design2) +design2 = df[ + df["retrofit design status"].str.lower().isin(["to invoice"]) +].copy() +design2 = design2[design2["design type"].str.lower().isin(["repetitive"])].copy() +design2["job_type"] = "Design Repetitive" +filtered_dfs.append(design2) + +# Design revision +design2 = df[ + df["retrofit design status"].str.lower().isin(["to invoice"]) +].copy() +design2 = design2[design2["design revision"].str.lower().isin(["A", "B", "C"])].copy() +design2["job_type"] = "Design Repetitive" +filtered_dfs.append(design2) + # Lodgement Phase 1 # lodg1 = df[ @@ -132,7 +160,7 @@ filtered_dfs.append(design1) _ = df[ df["trustmark lodgement"].str.lower().isin(["done"]) ].copy() -_["job_type"] = "Full Lodgement" +_["job_type"] = "Full lodgement phase 2" filtered_dfs.append(_) # POST EPC @@ -143,12 +171,12 @@ post_epc["job_type"] = "POST EPC" filtered_dfs.append(post_epc) -# # POST EPR -# post_epr = df[ -# df["post-epc status"].str.lower().isin(["post epr completed"]) -# ].copy() -# post_epr["job_type"] = "POST ATT" -# filtered_dfs.append(post_epr) +# POST EPR +post_epr = df[ + df["post epc status"].str.lower().isin(["post epr completed"]) +].copy() +post_epr["job_type"] = "POST ATT" +filtered_dfs.append(post_epr) # Post ATT post_att = df[ @@ -177,7 +205,7 @@ filtered_dfs.append(ra_ns) # ATT NO Show att_ns = df[ (df["pre att no show evidence"].fillna(-9999) != df["pre att no show invoice"].fillna(-9999)) & - (df["pre att no show evidence"] ! = 0) + (df["pre att no show evidence"] != 0) ].copy() att_ns["job_type"] = "ATT NO SHOW" filtered_dfs.append(att_ns) @@ -193,4 +221,13 @@ filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) -final_df[['address', 'client', 'job_type']] \ No newline at end of file +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_excel(f'ECO 4 NCHA Almshouses Operations_{timestamp}.xlsx', index=False) diff --git a/etl/month_end_automation_wave_2_no_6.py b/etl/month_end_automation_wave_2_no_6.py index 1b149ae..69e9e53 100644 --- a/etl/month_end_automation_wave_2_no_6.py +++ b/etl/month_end_automation_wave_2_no_6.py @@ -12,6 +12,22 @@ monday = MondayClient(monday_key) #ECO 4 Wates Operations board_ids = ["9520779048"] +rate_card_data = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + 259, 125, 280, 125, 125, + 650, 195, 175, 135, + 120, "Post EPR Please Marianne", 85, 125, 60, + 25, 25, 25 + ] +} + +rate_card_df = pd.DataFrame(rate_card_data) for board in tqdm(board_ids): board_data = monday.boards.fetch_boards_by_id(board) @@ -89,25 +105,25 @@ v1 = get_df(df, "Coordination Status IOE/MTP".lower(), [ "(V1) IOE/MTP Complete".lower(), "(V2) IOE/MTP Complete".lower(), "(V3) IOE/MTP Complete".lower(), -], "V1 Coordination") +], "Coordination Stage 1 v1") filtered_dfs.append(v1) # V2 Coordination # _ = df[df["mtp v2 invoiced"].str.lower().isin(['done', 'needs to be invoiced'])].copy() -# _["job_type"] = "V2 Coordination" +# _["job_type"] = "Coordination Stage 1 v2 remodel" # filtered_dfs.append(_) # V3 Coordination # v3 = df[df["v3 rc status"].str.lower().isin(['uploaded'])].copy() -# v3["job_type"] = "V3 Coordination" +# v3["job_type"] = "Coordination Stage 1 v3 remodel" # filtered_dfs.append(_) # Coordination stage 2 Please complete -# cors2 = df[ -# df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) -# ] -# cors2["joby_type"] = "Coordination Stage 2" -# filtered_dfs.append(cors2) +cors2 = df[ + df["rc stage 2 status"].str.lower().isin(["to invoice"]) +] +cors2["joby_type"] = "Coordination Stage 2" +filtered_dfs.append(cors2) # Design stage 1 design1 = get_df(df, "retrofit design status", ["completed"], "Design") @@ -119,7 +135,7 @@ design2 = get_df(df, "design revision invoice", [ "Rev. B to invoice".lower(), "Rev. C to invoice".lower(), "Rev. D to invoice".lower(), -], "Design Revision") +], "Design Repetitive") filtered_dfs.append(design2) # Lodgement Phase 1 @@ -130,7 +146,7 @@ filtered_dfs.append(design2) # filtered_dfs.append(lodg1) # Full Lodgement Phase -full_lodgement = get_df(df, "full lodgement", ["completed"], "Full Lodgement") +full_lodgement = get_df(df, "full lodgement", ["completed"], "Full lodgement phase 2") filtered_dfs.append(full_lodgement) # POST EPC @@ -138,12 +154,12 @@ post_epc = get_df(df, "post epc & evaluation status", ["uploaded"], "POST EPC") filtered_dfs.append(post_epc) -# # POST EPR -# post_epr = df[ -# df["post-epc status"].str.lower().isin(["post epr completed"]) -# ].copy() -# post_epr["job_type"] = "POST ATT" -# filtered_dfs.append(post_epr) +# POST EPR +post_epr = df[ + df["post epc & evaluation status"].str.lower().isin(["post epr completed"]) +].copy() +post_epr["job_type"] = "POST ATT" +filtered_dfs.append(post_epr) # Post ATT post_att = get_df(df, "post att status", ["uploaded"], "POST ATT") @@ -151,7 +167,7 @@ filtered_dfs.append(post_att) # Retrofit Evaluation -retro = get_df(df, "retrofit evaluation", ["done"], "Retrofit Evaluation") +retro = get_df(df, "post epc & evaluation status", ["uploaded"], "Retrofit Evaluation") filtered_dfs.append(retro) # RA NO Show @@ -179,4 +195,13 @@ filtered_dfs.append(retro) final_df = pd.concat(filtered_dfs).reset_index(drop=True) -final_df[['address', 'client', 'job_type']] \ No newline at end of file +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_excel(f'ECO 4 Wates Operations_{timestamp}.xlsx', index=False) diff --git a/etl/month_end_automation_wave_2_no_9.py b/etl/month_end_automation_wave_2_no_9.py index a24b6e7..4946193 100644 --- a/etl/month_end_automation_wave_2_no_9.py +++ b/etl/month_end_automation_wave_2_no_9.py @@ -11,6 +11,23 @@ monday = MondayClient(monday_key) # NCHA Almshouses board_ids = ["5423364294"] +rate_card_data = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + 259, 125, 280, 125, 125, + 650, 415, 195, 175, 135, + 120, "Post EPR Please Marianne", 85, 125, 60, + 25, 25, 25 + ] +} + +rate_card_df = pd.DataFrame(rate_card_data) + for board in tqdm(board_ids): board_data = monday.boards.fetch_boards_by_id(board) @@ -87,23 +104,24 @@ filtered_dfs.append(att) # V1 Coordination v1 = get_df(df, "coordination status (mtp)".lower(), [ "rc complete", -], "V1 Coordination") +], "Coordination Stage 1 v1") filtered_dfs.append(v1) # V2 Coordination -# v2 = get_df(df, "v2 coordination status", ["rc v2 complete", "uploaded"], "V2 Coordination") +# v2 = get_df(df, "v2 coordination status", ["rc v2 complete", "uploaded"], "Coordination Stage 1 v2 remodel") # filtered_dfs.append(v2) # # V3 Coordination +# Coordination Stage 1 v3 remode # v3 = get_df(df, "") # # filtered_dfs.append(v3) # Coordination stage 2 Please complete -# cors2 = df[ -# df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) -# ] -# cors2["joby_type"] = "Coordination Stage 2" -# filtered_dfs.append(cors2) +cors2 = df[ + df["rc stage 2"].str.lower().isin(["to invoice"]) +].copy() +cors2["joby_type"] = "Coordination Stage 2" +filtered_dfs.append(cors2) # Design stage 1 # design1 = get_df(df, "rd invoiced", ["to invoice"], "Design") @@ -124,7 +142,7 @@ filtered_dfs.append(v1) # Full Lodgement Phase -full_lodgement = get_df(df, "trustmark lodgement".lower(), ["done"], "Full Lodgement") +full_lodgement = get_df(df, "trustmark lodgement".lower(), ["done"], "Full lodgement phase 2") filtered_dfs.append(full_lodgement) # POST EPC @@ -159,7 +177,7 @@ filtered_dfs.append(ra_ns) # ATT NO Show att_ns = df[ - (df(df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999))) & + (df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)) & (df["att no show evidence"] != 0) ].copy() att_ns["job_type"] = "ATT NO SHOW" @@ -170,9 +188,20 @@ filtered_dfs.append(att_ns) # epc_ns = df[ # df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999) # ].copy() -# epc_ns["job_type"] = "EPC NO SHOW" +# epc_ns["job_type"] = "post EPC no show" # filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) -final_df[['address', 'client', 'job_type']] \ No newline at end of file +final_df[['address', 'client', 'job_type']] + +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_excel(f'NCHA Almshouses_{timestamp}.xlsx', index=False)