From 444f2836160a4da569dbe8e7fed0559bd2152892 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 29 Jul 2025 14:00:00 +0000 Subject: [PATCH] save --- etl/month_end_automation_wave_2_no_11.py | 61 +++++++++++++------ etl/month_end_automation_wave_2_no_7.py | 75 +++++++++++++++++------- 2 files changed, 96 insertions(+), 40 deletions(-) diff --git a/etl/month_end_automation_wave_2_no_11.py b/etl/month_end_automation_wave_2_no_11.py index da17811..4ea345f 100644 --- a/etl/month_end_automation_wave_2_no_11.py +++ b/etl/month_end_automation_wave_2_no_11.py @@ -11,6 +11,23 @@ monday = MondayClient(monday_key) # Northumberland LAD2 & HUG2 board_ids = ["5121300882"] +rate_card_data = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + 259, 125, 280, 125, 125, + 650, 195, 175, 135, + 120, "Post EPR Please Marianne", 85, 125, 60, + 25, 25, 25 + ] +} + +rate_card_df = pd.DataFrame(rate_card_data) + for board in tqdm(board_ids): board_data = monday.boards.fetch_boards_by_id(board) columns = board_data["data"]["boards"][0]["columns"] @@ -66,7 +83,7 @@ df = pd.DataFrame(all_records) filtered_dfs = [] -def get_df(df, column_name, success_critera, job_name): +def get_df(df, column_name, success_critera, job_name=None): _ = df[ df[column_name].str.lower().isin(success_critera) ].copy() @@ -86,11 +103,11 @@ filtered_dfs.append(att) # V1 Coordination v1 = get_df(df, "lite ima status".lower(), [ "rc complete", -], "V1 Coordination") +], "Coordination Stage 1 v1") filtered_dfs.append(v1) # V2 Coordination -v2 = get_df(df, "ima-mtp status", ["ima-mtp completed"], "V2 Coordination") +v2 = get_df(df, "ima-mtp status", ["ima-mtp completed"], "Coordination Stage 1 v2 remodel") filtered_dfs.append(v2) # # V3 Coordination @@ -98,11 +115,11 @@ filtered_dfs.append(v2) # filtered_dfs.append(v3) # Coordination stage 2 Please complete -# cors2 = df[ -# df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) -# ] -# cors2["joby_type"] = "Coordination Stage 2" -# filtered_dfs.append(cors2) +cors2 = df[ + df["rc stg. 2"].str.lower().isin(["to invoice"]) +] +cors2["joby_type"] = "Coordination Stage 2" +filtered_dfs.append(cors2) # Design stage 1 # design1 = get_df(df, "", ["done"], "Design") @@ -123,7 +140,7 @@ filtered_dfs.append(lodg1) # Full Lodgement Phase -full_lodgement = get_df(df, "trustmark lodgement".lower(), ["done"], "Full Lodgement") +full_lodgement = get_df(df, "trustmark lodgement".lower(), ["done"], "Full lodgement phase 2") filtered_dfs.append(full_lodgement) # POST EPC @@ -131,20 +148,17 @@ post_epc = get_df(df, "post-epc status", ["uploaded & completed", "to invoice"], filtered_dfs.append(post_epc) -# # POST EPR -# post_epr = df[ -# df["post-epc status"].str.lower().isin(["post epr completed"]) -# ].copy() -# post_epr["job_type"] = "POST ATT" -# filtered_dfs.append(post_epr) +# POST EPR +post_epr = df[ + df["post-epc status"].str.lower().isin(["post epr completed"]) +].copy() +post_epr["job_type"] = "POST ATT" +filtered_dfs.append(post_epr) # Post ATT post_att = get_df(df, "post att status", ["uploaded & completed", "to invoice"], "POST ATT") filtered_dfs.append(post_att) -post_att = get_df(df, "post-test status", ["complete"], "POST ATT") -filtered_dfs.append(post_att) - # Retrofit Evaluation retro = get_df(df, "retrofit evaluation", ["done", "to invoice"], "Retrofit Evaluation") filtered_dfs.append(retro) @@ -175,4 +189,13 @@ filtered_dfs.append(att_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) -final_df[['address', 'client', 'job_type']] \ No newline at end of file +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_excel(f'Northumberland LAD2 & HUG2_{timestamp}.xlsx', index=False) diff --git a/etl/month_end_automation_wave_2_no_7.py b/etl/month_end_automation_wave_2_no_7.py index 46d0a33..cbb3e2b 100644 --- a/etl/month_end_automation_wave_2_no_7.py +++ b/etl/month_end_automation_wave_2_no_7.py @@ -12,6 +12,23 @@ monday = MondayClient(monday_key) #Home Group Wave 2SP+ board_ids = ["4254419092"] +rate_card_data = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + 259, 125, 280, 125, 125, + 650, 195, 175, 135, + 120, "Post EPR Please Marianne", 85, 125, 60, + 25, 25, 25 + ] +} + +rate_card_df = pd.DataFrame(rate_card_data) + for board in tqdm(board_ids): board_data = monday.boards.fetch_boards_by_id(board) @@ -68,11 +85,12 @@ df = pd.DataFrame(all_records) filtered_dfs = [] -def get_df(df, column_name, success_critera, job_name): +def get_df(df, column_name, success_critera, job_name=None): _ = df[ df[column_name].str.lower().isin(success_critera) ].copy() - _["job_type"] = job_name + if job_name: + _["job_type"] = job_name return _ @@ -86,13 +104,13 @@ att = get_df(df, "pre-att", ["completed"], "ATT") filtered_dfs.append(att) # V1 Coordination -v1 = get_df(df, "osmosis rc status".lower(), [ +v1 = get_df(df, "osmosis rc status".lower(), [ "rc completed", -], "V1 Coordination") +], "Coordination Stage 1 v1") filtered_dfs.append(v1) # V2 Coordination -v2 = get_df(df, "v2 ioe mtp", ["completed"], "V2 Coordination") +v2 = get_df(df, "v2 ioe mtp", ["completed"], "Coordination Stage 1 v2 remodel") filtered_dfs.append(v2) # # V3 Coordination @@ -100,14 +118,20 @@ filtered_dfs.append(v2) # # filtered_dfs.append(v3) # Coordination stage 2 Please complete -# cors2 = df[ -# df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) -# ] -# cors2["joby_type"] = "Coordination Stage 2" -# filtered_dfs.append(cors2) +cors2 = df[ + df["rc stage 2"].str.lower().isin(["to invoice"]) +] +cors2["joby_type"] = "Coordination Stage 2" +filtered_dfs.append(cors2) -# Design stage 1 -design1 = get_df(df, "design invoice status", ["to invoice"], "Design") +# Design stage Archetype +design1 = get_df(df, "design invoice status", ["to invoice"]) +design1 = get_df(design1, "design type for invoicing", ["archetype"], "Design Archetype") +filtered_dfs.append(design1) + +# Design stage Repetitive +design1 = get_df(df, "design invoice status", ["to invoice"]) +design1 = get_df(design1, "design type for invoicing", ["repetitive"], "Design Repetitive") filtered_dfs.append(design1) # Design revision @@ -120,11 +144,11 @@ filtered_dfs.append(design1) # filtered_dfs.append(design2) # Lodgement Phase 1 -lodg1 = get_df(df, "TM Phase 1 Invoicing Status".lower(), ["done", "to invoice"], "Lodgement Phase 1") +lodg1 = get_df(df, "TM Phase 1 Invoicing Status".lower(), ["done", "to invoice"], "Lodgement phase 1") filtered_dfs.append(lodg1) # Full Lodgement Phase -full_lodgement = get_df(df, "Jun-te TM Phase 2 Invoicing Status".lower(), ["to invoice"], "Full Lodgement") +full_lodgement = get_df(df, "Jun-te TM Phase 2 Invoicing Status".lower(), ["to invoice"], "Full lodgement phase 2") filtered_dfs.append(full_lodgement) # POST EPC @@ -132,12 +156,12 @@ post_epc = get_df(df, "post-epc status", ["complete & uploaded"], "POST EPC") filtered_dfs.append(post_epc) -# # POST EPR -# post_epr = df[ -# df["post-epc status"].str.lower().isin(["post epr completed"]) -# ].copy() -# post_epr["job_type"] = "POST ATT" -# filtered_dfs.append(post_epr) +# POST EPR +post_epr = df[ + df["post-epc status"].str.lower().isin(["post epr completed"]) +].copy() +post_epr["job_type"] = "POST ATT" +filtered_dfs.append(post_epr) # Post ATT post_att = get_df(df, "post-att status", ["complete & uploaded"], "POST ATT") @@ -176,4 +200,13 @@ filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) -final_df[['address', 'client', 'job_type']] \ No newline at end of file +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_excel(f'HomeGroup Wave 2SP+_{timestamp}.xlsx', index=False)