From 4255b3036b0c1fb5f2258cce9a55c669c0bddfa9 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 29 Jul 2025 15:51:25 +0000 Subject: [PATCH] post epr --- etl/month_end_automation_wave_2_no_10.py | 2 +- etl/month_end_automation_wave_2_no_11.py | 4 +- etl/month_end_automation_wave_2_no_12.py | 58 +++++--- etl/month_end_automation_wave_2_no_13.py | 58 +++++--- etl/month_end_automation_wave_2_no_14.py | 4 +- etl/month_end_automation_wave_2_no_15.py | 59 +++++--- etl/month_end_automation_wave_2_no_16.py | 2 +- etl/month_end_automation_wave_2_no_3.py | 84 ++++++++---- etl/month_end_automation_wave_2_no_4.py | 75 +++++++--- etl/month_end_automation_wave_2_no_5.py | 4 +- etl/month_end_automation_wave_2_no_6.py | 4 +- etl/month_end_automation_wave_2_no_7.py | 4 +- etl/month_end_automation_wave_2_no_8.py | 4 +- etl/month_end_automation_wave_2_no_9.py | 2 +- etl/month_end_automation_wave_3_layout.py | 158 +++++++++++++--------- 15 files changed, 347 insertions(+), 175 deletions(-) diff --git a/etl/month_end_automation_wave_2_no_10.py b/etl/month_end_automation_wave_2_no_10.py index acba26b..d854fdc 100644 --- a/etl/month_end_automation_wave_2_no_10.py +++ b/etl/month_end_automation_wave_2_no_10.py @@ -189,7 +189,7 @@ filtered_dfs.append(att_ns) # epc_ns = df[ # df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999) # ].copy() -# epc_ns["job_type"] = "EPC NO SHOW" +# epc_ns["job_type"] = "post EPC NO SHOW" # filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) diff --git a/etl/month_end_automation_wave_2_no_11.py b/etl/month_end_automation_wave_2_no_11.py index 4ea345f..f05ef13 100644 --- a/etl/month_end_automation_wave_2_no_11.py +++ b/etl/month_end_automation_wave_2_no_11.py @@ -152,7 +152,7 @@ filtered_dfs.append(post_epc) post_epr = df[ df["post-epc status"].str.lower().isin(["post epr completed"]) ].copy() -post_epr["job_type"] = "POST ATT" +post_epr["job_type"] = "POST epr" filtered_dfs.append(post_epr) # Post ATT @@ -184,7 +184,7 @@ filtered_dfs.append(att_ns) # epc_ns = df[ # df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999) # ].copy() -# epc_ns["job_type"] = "EPC NO SHOW" +# epc_ns["job_type"] = "post EPC NO SHOW" # filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) diff --git a/etl/month_end_automation_wave_2_no_12.py b/etl/month_end_automation_wave_2_no_12.py index 0de0630..962eb46 100644 --- a/etl/month_end_automation_wave_2_no_12.py +++ b/etl/month_end_automation_wave_2_no_12.py @@ -11,6 +11,23 @@ monday = MondayClient(monday_key) # Shropshire Council HUG2 board_ids = ["4718185486"] +rate_card_data = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + 259, 125, 280, 125, 125, + 650, 195, 175, 135, + 120, "Post EPR Please Marianne", 85, 125, 60, + 25, 25, 25 + ] +} + +rate_card_df = pd.DataFrame(rate_card_data) + for board in tqdm(board_ids): board_data = monday.boards.fetch_boards_by_id(board) columns = board_data["data"]["boards"][0]["columns"] @@ -86,11 +103,11 @@ filtered_dfs.append(att) # V1 Coordination v1 = get_df(df, "ima lite invoiced".lower(), [ "to invoice", -], "V1 Coordination") +], "Coordination Stage 1 v1") filtered_dfs.append(v1) # V2 Coordination -v2 = get_df(df, "coordination v2 invoiced", ["to invoice"], "V2 Coordination") +v2 = get_df(df, "coordination v2 invoiced", ["to invoice"], "Coordination Stage 1 v2 remodel") filtered_dfs.append(v2) # # V3 Coordination @@ -98,11 +115,11 @@ filtered_dfs.append(v2) # filtered_dfs.append(v3) # Coordination stage 2 Please complete -# cors2 = df[ -# df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) -# ] -# cors2["joby_type"] = "Coordination Stage 2" -# filtered_dfs.append(cors2) +cors2 = df[ + df["rc stage 2 invoice"].str.lower().isin(["to invoice"]) +] +cors2["joby_type"] = "Coordination Stage 2" +filtered_dfs.append(cors2) # Design stage 1 # design1 = get_df(df, "", ["done"], "Design") @@ -123,7 +140,7 @@ filtered_dfs.append(lodg1) # Full Lodgement Phase -full_lodgement = get_df(df, "tm ph2 invoice status".lower(), ["to invoice"], "Full Lodgement") +full_lodgement = get_df(df, "tm ph2 invoice status".lower(), ["to invoice"], "Full lodgement phase 2") filtered_dfs.append(full_lodgement) # POST EPC @@ -131,12 +148,12 @@ post_epc = get_df(df, "post-epc status", ["uploaded", "completed"], "POST EPC") filtered_dfs.append(post_epc) -# # POST EPR -# post_epr = df[ -# df["post-epc status"].str.lower().isin(["post epr completed"]) -# ].copy() -# post_epr["job_type"] = "POST ATT" -# filtered_dfs.append(post_epr) +# POST EPR +post_epr = df[ + df["post-epc status"].str.lower().isin(["post epr completed"]) +].copy() +post_epr["job_type"] = "POST epr" +filtered_dfs.append(post_epr) # Post ATT post_att = get_df(df, "post-att", ["uploaded", "completed", "to invoice"], "POST ATT") @@ -168,9 +185,18 @@ epc_ns = df[ (df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999)) & (df["epc no show evidence"] != 0) ].copy() -epc_ns["job_type"] = "EPC NO SHOW" +epc_ns["job_type"] = "post EPC NO SHOW" filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) -final_df[['address', 'client', 'job_type']] \ No newline at end of file +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_excel(f'Shropshire Council HUG2_{timestamp}.xlsx', index=False) diff --git a/etl/month_end_automation_wave_2_no_13.py b/etl/month_end_automation_wave_2_no_13.py index f0bfe90..d4a6a1d 100644 --- a/etl/month_end_automation_wave_2_no_13.py +++ b/etl/month_end_automation_wave_2_no_13.py @@ -11,6 +11,23 @@ monday = MondayClient(monday_key) # Stonewater SHDF 3.0 - Operations board_ids = ["6222522864"] +rate_card_data = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + 259, 125, 280, 125, 125, + 650, 195, 175, 135, + 120, "Post EPR Please Marianne", 85, 125, 60, + 25, 25, 25 + ] +} + +rate_card_df = pd.DataFrame(rate_card_data) + for board in tqdm(board_ids): board_data = monday.boards.fetch_boards_by_id(board) columns = board_data["data"]["boards"][0]["columns"] @@ -85,11 +102,11 @@ filtered_dfs.append(att) # V1 Coordination v1 = get_df(df, "coordination status (mtp)".lower(), [ "ima/mtp complete", -], "V1 Coordination") +], "Coordination Stage 1 v1") filtered_dfs.append(v1) # V2 Coordination -v2 = get_df(df, "v2 mtp status", ["ima/mtp complete"], "V2 Coordination") +v2 = get_df(df, "v2 mtp status", ["ima/mtp complete"], "Coordination Stage 1 v2 remodel") filtered_dfs.append(v2) # # V3 Coordination @@ -97,11 +114,11 @@ filtered_dfs.append(v2) # filtered_dfs.append(v3) # Coordination stage 2 Please complete -# cors2 = df[ -# df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) -# ] -# cors2["joby_type"] = "Coordination Stage 2" -# filtered_dfs.append(cors2) +cors2 = df[ + df["rc stage 2"].str.lower().isin(["to invoice"]) +] +cors2["joby_type"] = "Coordination Stage 2" +filtered_dfs.append(cors2) # Design stage 1 # design1 = get_df(df, "", ["done"], "Design") @@ -122,7 +139,7 @@ filtered_dfs.append(v2) # Full Lodgement Phase -# full_lodgement = get_df(df, "tm ph2 invoice status".lower(), ["to invoice"], "Full Lodgement") +# full_lodgement = get_df(df, "tm ph2 invoice status".lower(), ["to invoice"], "Full lodgement phase 2") # filtered_dfs.append(full_lodgement) # POST EPC @@ -130,12 +147,12 @@ post_epc = get_df(df, "post epc", ["done"], "POST EPC") filtered_dfs.append(post_epc) -# # POST EPR -# post_epr = df[ -# df["post-epc status"].str.lower().isin(["post epr completed"]) -# ].copy() -# post_epr["job_type"] = "POST ATT" -# filtered_dfs.append(post_epr) +# POST EPR +post_epr = df[ + df["post epc"].str.lower().isin(["post epr completed"]) +].copy() +post_epr["job_type"] = "POST EPR" +filtered_dfs.append(post_epr) # Post ATT # post_att = get_df(df, "post-att", ["uploaded", "completed", "to invoice"], "POST ATT") @@ -166,9 +183,18 @@ filtered_dfs.append(ra_ns) # epc_ns = df[ # df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999) # ].copy() -# epc_ns["job_type"] = "EPC NO SHOW" +# epc_ns["job_type"] = "post EPC NO SHOW" # filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) -final_df[['address', 'client', 'job_type']] \ No newline at end of file +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_excel(f'Stonewater SHDF 3.0 - Operations_{timestamp}.xlsx', index=False) diff --git a/etl/month_end_automation_wave_2_no_14.py b/etl/month_end_automation_wave_2_no_14.py index e3286ea..9203f2b 100644 --- a/etl/month_end_automation_wave_2_no_14.py +++ b/etl/month_end_automation_wave_2_no_14.py @@ -137,7 +137,7 @@ filtered_dfs.append(post_epc) # post_epr = df[ # df["post-epc status"].str.lower().isin(["post epr completed"]) # ].copy() -# post_epr["job_type"] = "POST ATT" +# post_epr["job_type"] = "POST epr" # filtered_dfs.append(post_epr) # Post ATT @@ -170,7 +170,7 @@ epc_ns = df[ (df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999)) & (df["post epc no show evidence"] != 0) ].copy() -epc_ns["job_type"] = "EPC NO SHOW" +epc_ns["job_type"] = "post EPC NO SHOW" filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) diff --git a/etl/month_end_automation_wave_2_no_15.py b/etl/month_end_automation_wave_2_no_15.py index bcdc46b..3686198 100644 --- a/etl/month_end_automation_wave_2_no_15.py +++ b/etl/month_end_automation_wave_2_no_15.py @@ -11,6 +11,24 @@ monday = MondayClient(monday_key) # WCHG SHDF 2.1 Mansard board_ids = ["5636990610"] +rate_card_data = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + 259, 125, 280, 125, 125, + 650, 195, 175, 135, + 120, "Post EPR Please Marianne", 85, 125, 60, + 25, 25, 25 + ] +} + +rate_card_df = pd.DataFrame(rate_card_data) + + for board in tqdm(board_ids): board_data = monday.boards.fetch_boards_by_id(board) columns = board_data["data"]["boards"][0]["columns"] @@ -85,7 +103,7 @@ filtered_dfs.append(att) # V1 Coordination v1 = get_df(df, "coordination status (ioe mtp)".lower(), [ "rc complete", -], "V1 Coordination") +], "Coordination Stage 1 v1") filtered_dfs.append(v1) # V2 Coordination @@ -101,15 +119,15 @@ filtered_dfs.append(v1) # filtered_dfs.append(v3) # Coordination stage 2 Please complete -# cors2 = df[ -# df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) -# ] -# cors2["joby_type"] = "Coordination Stage 2" -# filtered_dfs.append(cors2) +cors2 = df[ + df["rc stg 2"].str.lower().isin(["to invoice"]) +] +cors2["joby_type"] = "Coordination Stage 2" +filtered_dfs.append(cors2) # Design stage 1 -design1 = get_df(df, "design invoice status", ["to invoice"], "Design") -filtered_dfs.append(design1) +# design1 = get_df(df, "design invoice status", ["to invoice"], "Design") +# filtered_dfs.append(design1) # Design revision # design2 = get_df(df, "design revision invoice", [ @@ -126,7 +144,7 @@ filtered_dfs.append(lodg1) # Full Lodgement Phase -full_lodgement = get_df(df, "lodgement invoice status".lower(), ["to invoice"], "Full Lodgement") +full_lodgement = get_df(df, "lodgement invoice status".lower(), ["to invoice"], "Full lodgement phase 2") filtered_dfs.append(full_lodgement) # POST EPC @@ -135,11 +153,11 @@ filtered_dfs.append(post_epc) # # POST EPR -# post_epr = df[ -# df["post-epc status"].str.lower().isin(["post epr completed"]) -# ].copy() -# post_epr["job_type"] = "POST ATT" -# filtered_dfs.append(post_epr) +post_epr = df[ + df["post-epc status"].str.lower().isin(["post epr completed"]) +].copy() +post_epr["job_type"] = "POST epr" +filtered_dfs.append(post_epr) # Post ATT post_att = get_df(df, "post-att status", ["uploaded", "completed"], "POST ATT") @@ -172,9 +190,18 @@ epc_ns = df[ (df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999)) & (df["epc no show evidence"] != 0) ].copy() -epc_ns["job_type"] = "EPC NO SHOW" +epc_ns["job_type"] = "post EPC NO SHOW" filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) -final_df[['address', 'client', 'job_type']] \ No newline at end of file +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_excel(f'WCHG SHDF 2.1 Mansard {timestamp}.xlsx', index=False) diff --git a/etl/month_end_automation_wave_2_no_16.py b/etl/month_end_automation_wave_2_no_16.py index be52ffd..0af8bfe 100644 --- a/etl/month_end_automation_wave_2_no_16.py +++ b/etl/month_end_automation_wave_2_no_16.py @@ -190,7 +190,7 @@ filtered_dfs.append(att_ns) # epc_ns = df[ # df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999) # ].copy() -# epc_ns["job_type"] = "EPC NO SHOW" +# epc_ns["job_type"] = "post EPC NO SHOW" # filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) diff --git a/etl/month_end_automation_wave_2_no_3.py b/etl/month_end_automation_wave_2_no_3.py index 90253d5..6e497c5 100644 --- a/etl/month_end_automation_wave_2_no_3.py +++ b/etl/month_end_automation_wave_2_no_3.py @@ -12,6 +12,23 @@ monday = MondayClient(monday_key) # Platform Housing W2 (in use) board_ids = ["4796290860"] +rate_card_data = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + 259, 125, 280, 125, 125, + 650, 195, 175, 135, + 120, "Post EPR Please Marianne", 85, 125, 60, + 25, 25, 25 + ] +} + +rate_card_df = pd.DataFrame(rate_card_data) + for board in tqdm(board_ids): board_data = monday.boards.fetch_boards_by_id(board) @@ -87,41 +104,45 @@ filtered_dfs.append(att) v1 = df[ df["coordination status"].str.lower().isin(["ima/mtp completed"]) ].copy() -v1["job_type"] = "V1 Coordination" +v1["job_type"] = "Coordination Stage 1 v1" filtered_dfs.append(v1) # V2 Coordination _ = df[df["v2 mtp status"].fillna('').str.lower().isin(['v2 ima-mtp completed', 'v2 completed'])].copy() -_["job_type"] = "V2 Coordination" +_["job_type"] = "Coordination Stage 1 v2 remodel" filtered_dfs.append(_) # V3 Coordination # v3 = df[ # df["v3 invoiced"].str.lower().isin(["to be invoiced"]) # ].copy() -# v3["job_type"] = "V3 Coordination" +# v3["job_type"] = "Coordination Stage 1 v3 remodel" # filtered_dfs.append(v3) # Coordination stage 2 Please complete -# cors2 = df[ -# df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) -# ] -# cors2["joby_type"] = "Coordination Stage 2" -# filtered_dfs.append(cors2) +cors2 = df[ + df["rc stg. 2"].str.lower().isin(["to invoice"]) +] +cors2["joby_type"] = "Coordination Stage 2" +filtered_dfs.append(cors2) -# Design stage 1 +# Design Archetype design1 = df[ df["design invoice"].str.lower().isin(["complete pending rc"]) ].copy() -design1["job_type"] = "Design" +design1 = design1[design1["design type for invoicing"].str.lower().isin(['archetype'])].copy() +design1["job_type"] = "Design Archetype" filtered_dfs.append(design1) -# Design stage 2 -# design2 = df[ -# df["revision 2 design invoice"].str.lower().isin(["to invoice"]) -# ].copy() -# design2["job_type"] = "Design Revision" -# filtered_dfs.append(design2) +# Design Repetitive +design1 = df[ + df["design invoice"].str.lower().isin(["complete pending rc"]) +].copy() +design1 = design1[design1["design type for invoicing"].str.lower().isin(['repetitive'])].copy() +design1["job_type"] = "Design repetitive" +filtered_dfs.append(design1) + +# Design Revision # Lodgement Phase 1 lodg1 = df[ @@ -134,7 +155,7 @@ filtered_dfs.append(lodg1) lodg2 = df[ df["lodgement invoice status (lodgement)"].str.lower().isin(["to invoice"]) ].copy() -lodg2["job_type"] = "Full Lodgement" +lodg2["job_type"] = "Full lodgement phase 2" filtered_dfs.append(lodg2) # POST EPC @@ -145,13 +166,12 @@ post_epc["job_type"] = "POST EPC" filtered_dfs.append(post_epc) -# # POST EPR -# post_epr = df[ -# df["post-epc status"].str.lower().isin(["post epr completed"]) -# ].copy() -# post_epr["job_type"] = "POST ATT" -# filtered_dfs.append(post_epr) - +# POST EPR +post_epr = df[ + df["post epc"].str.lower().isin(["post epr completed"]) +].copy() +post_epr["job_type"] = "POST EPR" +filtered_dfs.append(post_epr) # Post ATT @@ -172,13 +192,12 @@ filtered_dfs.append(retro) # RA NO Show ra_ns = df[ (df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) & - (df[" ra no show evidence"] != 0) + (df["ra no show evidence"] != 0) ].copy() ra_ns["job_type"] = "RA NO SHOW" filtered_dfs.append(ra_ns) - # ATT NO Show att_ns = df[ (df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)) & @@ -193,9 +212,18 @@ epc_ns = df[ (df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999)) & (df["post epc no show evidence"] != 0) ].copy() -epc_ns["job_type"] = "EPC NO SHOW" +epc_ns["job_type"] = "Post EPC NO SHOW" filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) -final_df[['address', 'client', 'job_type']] \ No newline at end of file +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_excel(f'Platform Housing W2 (in use)_{timestamp}.xlsx', index=False) diff --git a/etl/month_end_automation_wave_2_no_4.py b/etl/month_end_automation_wave_2_no_4.py index b750850..1c17c7e 100644 --- a/etl/month_end_automation_wave_2_no_4.py +++ b/etl/month_end_automation_wave_2_no_4.py @@ -12,6 +12,22 @@ monday = MondayClient(monday_key) # Stonewater (in use) board_ids = ["3584401309"] +rate_card_data = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + 259, 125, 280, 125, 125, + 650, 195, 175, 135, + 120, "Post EPR Please Marianne", 85, 125, 60, + 25, 25, 25 + ] +} + +rate_card_df = pd.DataFrame(rate_card_data) for board in tqdm(board_ids): board_data = monday.boards.fetch_boards_by_id(board) @@ -87,34 +103,43 @@ filtered_dfs.append(att) v1 = df[ df["v1 coordination status (ioe,mtp)"].str.lower().isin(["rc complete"]) ].copy() -v1["job_type"] = "V1 Coordination" +v1["job_type"] = "Coordination Stage 1 v1" filtered_dfs.append(v1) # V2 Coordination _ = df[df["mtp v2 invoiced"].str.lower().isin(['done', 'needs to be invoiced'])].copy() -_["job_type"] = "V2 Coordination" +_["job_type"] = "Coordination Stage 1 v2 remodel" filtered_dfs.append(_) # V3 Coordination v3 = df[df["v3 rc status"].str.lower().isin(['uploaded'])].copy() -v3["job_type"] = "V3 Coordination" +v3["job_type"] = "Coordination Stage 1 v3 remodel" filtered_dfs.append(_) # Coordination stage 2 Please complete -# cors2 = df[ -# df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) -# ] -# cors2["joby_type"] = "Coordination Stage 2" -# filtered_dfs.append(cors2) +cors2 = df[ + df["rc stg. 2 status"].str.lower().isin(["to invoice", "completed"]) +] +cors2["joby_type"] = "Coordination Stage 2" +filtered_dfs.append(cors2) -# Design stage 1 +# Design Archetype design1 = df[ df["design invoice status"].str.lower().isin(["complete", "to invoice"]) ].copy() -design1["job_type"] = "Design" +design1 = design1[design1["design type"].str.lower().isin(["archetype"])].copy() +design1["job_type"] = "Design Archetype" filtered_dfs.append(design1) -# Design stage 2 +# Design Repetitive +design1 = df[ + df["design invoice status"].str.lower().isin(["complete", "to invoice"]) +].copy() +design1 = design1[design1["design type"].str.lower().isin(["repetitive"])].copy() +design1["job_type"] = "Design Repetitive" +filtered_dfs.append(design1) + +# Design revision # design2 = df[ # df[""].str.lower().isin(["to invoice"]) # ].copy() @@ -132,7 +157,7 @@ filtered_dfs.append(lodg1) _ = df[ df["lodgement invoice status"].str.lower().isin(["to invoice"]) ].copy() -_["job_type"] = "Full Lodgement" +_["job_type"] = "Full lodgement phase 2" filtered_dfs.append(_) # POST EPC @@ -143,12 +168,12 @@ post_epc["job_type"] = "POST EPC" filtered_dfs.append(post_epc) -# # POST EPR -# post_epr = df[ -# df["post-epc status"].str.lower().isin(["post epr completed"]) -# ].copy() -# post_epr["job_type"] = "POST ATT" -# filtered_dfs.append(post_epr) +# POST EPR +post_epr = df[ + df["post epc"].str.lower().isin(["post epr completed"]) +].copy() +post_epr["job_type"] = "POST EPR" +filtered_dfs.append(post_epr) # Post ATT post_att = df[ @@ -188,9 +213,19 @@ epc_ns = df[ (df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999)) & (df["post epc no show evidence"] != 0) ].copy() -epc_ns["job_type"] = "EPC NO SHOW" +epc_ns["job_type"] = "post EPC NO SHOW" filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) -final_df[['address', 'client', 'job_type']] \ No newline at end of file + +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_excel(f'Stonewater - (in use)_{timestamp}.xlsx', index=False) diff --git a/etl/month_end_automation_wave_2_no_5.py b/etl/month_end_automation_wave_2_no_5.py index 5d40b55..f26d3b0 100644 --- a/etl/month_end_automation_wave_2_no_5.py +++ b/etl/month_end_automation_wave_2_no_5.py @@ -175,7 +175,7 @@ filtered_dfs.append(post_epc) post_epr = df[ df["post epc status"].str.lower().isin(["post epr completed"]) ].copy() -post_epr["job_type"] = "POST ATT" +post_epr["job_type"] = "POST epr" filtered_dfs.append(post_epr) # Post ATT @@ -216,7 +216,7 @@ epc_ns = df[ (df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999)) & (df["epc no show evidence"] != 0 ) ].copy() -epc_ns["job_type"] = "EPC NO SHOW" +epc_ns["job_type"] = "post EPC NO SHOW" filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) diff --git a/etl/month_end_automation_wave_2_no_6.py b/etl/month_end_automation_wave_2_no_6.py index 69e9e53..0ad996f 100644 --- a/etl/month_end_automation_wave_2_no_6.py +++ b/etl/month_end_automation_wave_2_no_6.py @@ -158,7 +158,7 @@ filtered_dfs.append(post_epc) post_epr = df[ df["post epc & evaluation status"].str.lower().isin(["post epr completed"]) ].copy() -post_epr["job_type"] = "POST ATT" +post_epr["job_type"] = "POST epr" filtered_dfs.append(post_epr) # Post ATT @@ -190,7 +190,7 @@ filtered_dfs.append(retro) # epc_ns = df[ # df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999) # ].copy() -# epc_ns["job_type"] = "EPC NO SHOW" +# epc_ns["job_type"] = "post EPC NO SHOW" # filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) diff --git a/etl/month_end_automation_wave_2_no_7.py b/etl/month_end_automation_wave_2_no_7.py index cbb3e2b..0cc1749 100644 --- a/etl/month_end_automation_wave_2_no_7.py +++ b/etl/month_end_automation_wave_2_no_7.py @@ -160,7 +160,7 @@ filtered_dfs.append(post_epc) post_epr = df[ df["post-epc status"].str.lower().isin(["post epr completed"]) ].copy() -post_epr["job_type"] = "POST ATT" +post_epr["job_type"] = "POST epr" filtered_dfs.append(post_epr) # Post ATT @@ -195,7 +195,7 @@ epc_ns = df[ (df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999)) & (df["epc no show evidence"] != 0 ) ].copy() -epc_ns["job_type"] = "EPC NO SHOW" +epc_ns["job_type"] = "post EPC NO SHOW" filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) diff --git a/etl/month_end_automation_wave_2_no_8.py b/etl/month_end_automation_wave_2_no_8.py index 62950e3..d8f0a35 100644 --- a/etl/month_end_automation_wave_2_no_8.py +++ b/etl/month_end_automation_wave_2_no_8.py @@ -168,7 +168,7 @@ filtered_dfs.append(post_epc) post_epr = df[ df["post epc status"].str.lower().isin(["post epr completed"]) ].copy() -post_epr["job_type"] = "POST ATT" +post_epr["job_type"] = "POST epr" filtered_dfs.append(post_epr) # Post ATT @@ -202,7 +202,7 @@ filtered_dfs.append(att_ns) # epc_ns = df[ # df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999) # ].copy() -# epc_ns["job_type"] = "EPC NO SHOW" +# epc_ns["job_type"] = "post EPC NO SHOW" # filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) diff --git a/etl/month_end_automation_wave_2_no_9.py b/etl/month_end_automation_wave_2_no_9.py index 4946193..15bf2fb 100644 --- a/etl/month_end_automation_wave_2_no_9.py +++ b/etl/month_end_automation_wave_2_no_9.py @@ -154,7 +154,7 @@ filtered_dfs.append(full_lodgement) # post_epr = df[ # df["post-epc status"].str.lower().isin(["post epr completed"]) # ].copy() -# post_epr["job_type"] = "POST ATT" +# post_epr["job_type"] = "POST epr" # filtered_dfs.append(post_epr) # Post ATT diff --git a/etl/month_end_automation_wave_3_layout.py b/etl/month_end_automation_wave_3_layout.py index 3b5f53f..7e273f2 100644 --- a/etl/month_end_automation_wave_3_layout.py +++ b/etl/month_end_automation_wave_3_layout.py @@ -12,10 +12,27 @@ monday = MondayClient(monday_key) board_ids = [ # "9349630181", # WCHG Walkups-Operations # "8829428746", # 2502 Accent Housing - # "8830772914", # "L&Q London" - "9601691730", # Cardo Wales & West - Wave 3 + "8830772914", # "L&Q London" + # "9601691730", # Cardo Wales & West - Wave 3 ] +rate_card_data = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" + ], + "rate": [ + 259, 125, 280, 125, 125, + 650, 415, 195, 175, 135, + 120, "Post EPR Please Marianne", 85, 125, 60, + 25, 25, 25, "Mariann please input full cost mtp", "Marianne please input measure modelling" + ] +} + +rate_card_df = pd.DataFrame(rate_card_data) + for board in tqdm(board_ids): print(f"working on board {board}") @@ -42,26 +59,6 @@ for board in tqdm(board_ids): value = json.loads(col["value"]) no_of_files = len(value.get('files', [])) data.update({reversed_col_id_map[col.get("id")]: no_of_files}) - # elif "no show" in reversed_col_id_map[col.get("id")]: - # def extract_number_from_text(text): - # number_str = '' - - # for char in text: - # if char.isnumeric(): - # number_str += char - # elif number_str: - # break # stop once a number sequence ends - - # return int(number_str) if number_str else None - # text = col.get("text") - # if text is None: - # data.update({ - # reversed_col_id_map[col.get("id")]: col.get("text") - # }) - # else: - # data.update({ - # reversed_col_id_map[col.get("id")]: extract_number_from_text(text) - # }) else: data.update({ reversed_col_id_map[col.get("id")]: col.get("text") @@ -74,110 +71,143 @@ df = pd.DataFrame(all_records) filtered_dfs = [] -def get_df(df, column_name, success_critera, job_name): +def get_df(df, column_name, success_critera, job_name=None): + _ = pd.DataFrame() if column_name in col_id_map: _ = df[ df[column_name].str.lower().isin(success_critera) ].copy() - _["job_type"] = job_name - return _ - else: - print(f"failed to find {column_name}") + if job_name: + _["job_type"] = job_name + + + return _ # RA ra = get_df(df, "ra invoicing status", ["to invoice"], "RA") -if ra is not None: +if not ra.empty: filtered_dfs.append(ra) att = get_df(df, "post att invoicing status", ["to invoice"], "ATT") -if att is not None: +if not att.empty: filtered_dfs.append(att) -modeling = get_df(df, "mtp invoicing status", ["modelling to invoice"], "V3 Coordination") -if modeling is not None: +modeling = get_df(df, "mtp invoicing status", ["modelling to invoice"], "Measure Modelling") +if not modeling.empty: filtered_dfs.append(modeling) -v1 = get_df(df, "mtp invoicing status", ["(v1) ioe/mtp to invoice"], "V1 Coordination") -if v1 is not None: +try: + # Only needed for one board in wave 3 + full_cost = get_df(df, "mtp invoicing status", ["(V1) Full cost MTP to invoice (no previous modelling)".lower()], "full cost mtp") + if not full_cost.empty: + filtered_dfs(full_cost) +except Exception as e: + print(e) + +v1 = get_df(df, "mtp invoicing status", ["(v1) ioe/mtp to invoice"], "Coordination Stage 1 v1") +if not v1.empty: filtered_dfs.append(v1) -v2 = get_df(df, "mtp invoicing status", ["(v2) ioe/mtp to invoice"], "V2 Coordination") -if v2 is not None: +v2 = get_df(df, "mtp invoicing status", ["(v2) ioe/mtp to invoice"], "Coordination Stage 1 v2 remodel") +if not v2.empty: filtered_dfs.append(v2) -v3 = get_df(df, "mtp invoicing status", ["(v3) ioe/mtp to invoice"], "V3 Coordination") -if v3 is not None: +v3 = get_df(df, "mtp invoicing status", ["(v3) ioe/mtp to invoice"], "Coordination Stage 1 v3 remodel") +if not v3.empty: filtered_dfs.append(v3) # Coordination stage 2 Please complete -# cors2 = df[ -# df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) -# ] -# cors2["joby_type"] = "Coordination Stage 2" -# filtered_dfs.append(cors2) +cors2 = get_df(df, "rc stage 2", ["to invoice"], "Coordination Stage 2") +if not cors2.empty: + filtered_dfs.append(cors2) -design1 = get_df(df, "design invoicing status", ["to invoice"], "Design") -if design1 is not None: +# Design archetype complex +design = get_df(df, "design invoicing status", ["to invoice"]) +design1 = get_df(design, "design invoice type", ["archetype (complex)"], "Design Archetype Complex") +if not design1.empty : + filtered_dfs.append(design1) + +# Design archetype simple +design1 = get_df(design, "design invoice type", ["archetype (simple)"], "Design Archetype Simple") +if not design1.empty: + filtered_dfs.append(design1) + +# Design repetitive simple +design1 = get_df(design, "design invoice type", ["archetype (simple)"], "Design Archetype repetitive") +if not design1.empty: + filtered_dfs.append(design1) + +# Design repetitive complex +design1 = get_df(design, "design invoice type", ["archetype (complex)"], "Design Archetype complex") +if not design1.empty: filtered_dfs.append(design1) # Design Revision revision_letter = ['a', 'b', 'c', 'd'] for letter in revision_letter: design = get_df(df, "design revision invoice", [f"rev. {letter} to invoice"], "Design Revision") - if design is not None: + if not design.empty: filtered_dfs.append(design) # Lodgement Phase 1 lodg1 = get_df(df, "lodgement phase 1 invoicing status", ["to invoice"], "Lodgement Phase 1") -if lodg1 is not None: +if not lodg1.empty: filtered_dfs(lodg1) # Full Lodgement Phase -lodg2 = get_df(df, "full lodgement invoicing status", ["to invoice"], "Full Lodgement") -if lodg2 is not None: +lodg2 = get_df(df, "full lodgement invoicing status", ["to invoice"], "Full lodgement phase 2") +if not lodg2.empty: filtered_dfs.append(lodg2) # POST EPC -post_epc = get_df(df, "post epc & eval. invoicing status", ["to invoice"], "POST EPC") -if post_epc is not None: +post_epc = get_df(df, "post epc & eval. invoicing status", ["epc to invoice"], "POST EPC") +if not post_epc.empty: filtered_dfs.append(post_epc) -# # POST EPR, not right now -# post_epr = df[ -# df["post-epc status"].str.lower().isin(["post epr completed"]) -# ].copy() -# post_epr["job_type"] = "POST ATT" -# filtered_dfs.append(post_epr) +# POST EPR +post_epr = get_df(df, "post epc & eval. invoicing status", ["epr to invoice"], "POST EPR") +if not post_epr.empty: + filtered_dfs.append(post_epr) +# post att post_att = get_df(df, "post att invoicing status", ["to invoice"], "POST ATT") -if post_att is not None: +if not post_att.empty: filtered_dfs.append(post_epc) # Retrofit Evaluation -rc = get_df(df, "rc stage 2 invoicing status", ["to invoice"], "RC stage 2") -if rc is not None: +rc = get_df(df, "rc stage 2 invoicing status", ["to invoice"], "retrofit evaluation") +if not rc.empty: filtered_dfs.append(rc) # RA NO Show ra_ns = get_df(df,"ra no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "RA NO SHOW") -if ra_ns is not None: +if not ra_ns.empty: filtered_dfs.append(ra_ns) # ATT NO Show att_ns = get_df(df, "pre att no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "ATT NO SHOW") -if att_ns is not None: +if not att_ns.empty: filtered_dfs.append(att_ns) # Post visit no show -epc_ns = get_df(df, "post works no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "EPC NO SHOW") -if epc_ns is not None: +epc_ns = get_df(df, "post works no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "post EPC NO SHOW") +if not epc_ns.empty: filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) -final_df[['address', 'client', 'job_type']] \ No newline at end of file +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_excel(f'WCHG Walkups-Operations {timestamp}.xlsx', index=False)