From d1483fe07231d448e04223d90a41c282e25aeb05 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 28 Jul 2025 16:14:18 +0000 Subject: [PATCH] no show re write --- etl/month_end_automation_wave_2_layout.py | 80 ++++++++++++++++++----- etl/month_end_automation_wave_2_no_10.py | 6 +- etl/month_end_automation_wave_2_no_11.py | 3 +- etl/month_end_automation_wave_2_no_12.py | 6 +- etl/month_end_automation_wave_2_no_13.py | 3 +- etl/month_end_automation_wave_2_no_14.py | 6 +- etl/month_end_automation_wave_2_no_15.py | 9 ++- etl/month_end_automation_wave_2_no_16.py | 6 +- etl/month_end_automation_wave_2_no_3.py | 9 ++- etl/month_end_automation_wave_2_no_4.py | 9 ++- etl/month_end_automation_wave_2_no_5.py | 9 ++- etl/month_end_automation_wave_2_no_7.py | 9 ++- etl/month_end_automation_wave_2_no_8.py | 73 ++++++++++++++++++--- etl/month_end_automation_wave_2_no_9.py | 6 +- 14 files changed, 179 insertions(+), 55 deletions(-) diff --git a/etl/month_end_automation_wave_2_layout.py b/etl/month_end_automation_wave_2_layout.py index 843e2fa..174fe0b 100644 --- a/etl/month_end_automation_wave_2_layout.py +++ b/etl/month_end_automation_wave_2_layout.py @@ -9,9 +9,26 @@ import json monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) -# NCHK SHDF Westville Wave 1 & 2 +# NCHA SHDF Westville Wave 1 & 2 board_ids = ["3900434153"] +rate_card_data = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + 207.65, 101, 186.4, 98, 98, + 450, 150, 163, 135, 120, + 45, 90.5, 40, + 25, 25, 25 + ] +} + +rate_card_df = pd.DataFrame(rate_card_data) + for board in tqdm(board_ids): board_data = monday.boards.fetch_boards_by_id(board) @@ -87,55 +104,69 @@ filtered_dfs.append(att) v1 = df[ df["v1 coordination status"].str.lower().isin(["rc complete"]) ].copy() -v1["job_type"] = "V1 Coordination" +v1["job_type"] = "Coordination Stage 1 v1" filtered_dfs.append(v1) # V2 Coordination _ = df[df["v2 invoiced"].fillna('').str.lower().isin(['to be invoiced'])] v2 = _[_["v2 dc/ima/pas"] > 0].copy() -v2["job_type"] = "V2 Coordination" +v2["job_type"] = "Coordination Stage 1 v2 remodel" filtered_dfs.append(v2) # V3 Coordination v3 = df[ df["v3 invoiced"].str.lower().isin(["to be invoiced"]) ].copy() -v3["job_type"] = "V3 Coordination" +v3["job_type"] = "Coordination Stage 1 v3 remodel" filtered_dfs.append(v3) -# Design stage 1 +# Coordination stage 2 +cors2 = df[ + df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) +] +cors2["joby_type"] = "Coordination Stage 2" +filtered_dfs.append(cors2) + +# Design type archietype design1 = df[ - df["design invoice status"].str.lower().isin(["to invoice"]) + (df["design type for invoicing"].str.lower().isin(["archetype"])) & (df["design invoice status"].str.lower().isin(["to invoice"])) ].copy() -design1["job_type"] = "Design" +design1["job_type"] = "Design Archetype" filtered_dfs.append(design1) -# Design stage 2 +# design type reptitive +design1 = df[ + (df["design type for invoicing"].str.lower().isin(["repetitive"])) & df["design invoice status"].str.lower().isin(["to invoice"]) +].copy() +design1["job_type"] = "Design Repetitive" +filtered_dfs.append(design1) + +# Design stage revisions design2 = df[ df["revision 2 design invoice"].str.lower().isin(["to invoice"]) ].copy() -design2["job_type"] = "Design Revision" +design2["job_type"] = "Design Repetitive" filtered_dfs.append(design2) # Lodgement Phase 1 lodg1 = df[ df["lodg. phase 1 invoice status"].str.lower().isin(["to invoice"]) ].copy() -lodg1["job_type"] = "Lodgement Phase 1" +lodg1["job_type"] = "Lodgement phase 1" filtered_dfs.append(lodg1) # Full Lodgement Phase lodg2 = df[ df["full lodgement invoice status"].str.lower().isin(["to invoice"]) ].copy() -lodg2["job_type"] = "Full Lodgement" +lodg2["job_type"] = "Full Lodgement phase 2" filtered_dfs.append(lodg2) # POST EPC post_epc = df[ df["post-epc status"].str.lower().isin(["epc pics uploaded"]) ].copy() -post_epc["job_type"] = "POST EPC" +post_epc["job_type"] = "Post EPC" filtered_dfs.append(post_epc) @@ -143,7 +174,7 @@ filtered_dfs.append(post_epc) post_epr = df[ df["post-epc status"].str.lower().isin(["post epr completed"]) ].copy() -post_epr["job_type"] = "POST ATT" +post_epr["job_type"] = "Post ATT" filtered_dfs.append(post_epr) @@ -165,7 +196,8 @@ filtered_dfs.append(retro) # RA NO Show ra_ns = df[ - df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) + (df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) & + (df["ra no show evidence"] != 0) ].copy() ra_ns["job_type"] = "RA NO SHOW" filtered_dfs.append(ra_ns) @@ -174,7 +206,8 @@ filtered_dfs.append(ra_ns) # ATT NO Show att_ns = df[ - df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) + (df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)) & + (df["att no show evidence"] != 0) ].copy() att_ns["job_type"] = "ATT NO SHOW" filtered_dfs.append(att_ns) @@ -182,11 +215,22 @@ filtered_dfs.append(att_ns) # Post visit no show epc_ns = df[ - df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999) + (df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999)) & + (df["epc no show evidence"] != 0) ].copy() -epc_ns["job_type"] = "EPC NO SHOW" +epc_ns["job_type"] = "post EPC NO SHOW" filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) -final_df[['address', 'client', 'job_type']] \ No newline at end of file + +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_csv(f'NCHA SHDF Westville Wave 1 & 2_{timestamp}.csv', index=False) diff --git a/etl/month_end_automation_wave_2_no_10.py b/etl/month_end_automation_wave_2_no_10.py index 87580f2..d779e45 100644 --- a/etl/month_end_automation_wave_2_no_10.py +++ b/etl/month_end_automation_wave_2_no_10.py @@ -144,7 +144,8 @@ filtered_dfs.append(retro) # RA NO Show ra_ns = df[ - df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) + (df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) & + (df["ra no show evidence"] !=0 ) ].copy() ra_ns["job_type"] = "RA NO SHOW" filtered_dfs.append(ra_ns) @@ -152,7 +153,8 @@ filtered_dfs.append(ra_ns) # ATT NO Show att_ns = df[ - df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) + (df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)) & + (df["att no show evidence"] != 0) ].copy() att_ns["job_type"] = "ATT NO SHOW" filtered_dfs.append(att_ns) diff --git a/etl/month_end_automation_wave_2_no_11.py b/etl/month_end_automation_wave_2_no_11.py index babf1f8..2eeb76e 100644 --- a/etl/month_end_automation_wave_2_no_11.py +++ b/etl/month_end_automation_wave_2_no_11.py @@ -152,7 +152,8 @@ filtered_dfs.append(retro) # ATT NO Show att_ns = df[ - df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) + (df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)) & + (df["att no show evidence"] != 0) ].copy() att_ns["job_type"] = "ATT NO SHOW" filtered_dfs.append(att_ns) diff --git a/etl/month_end_automation_wave_2_no_12.py b/etl/month_end_automation_wave_2_no_12.py index 261ed2f..fbe4bf0 100644 --- a/etl/month_end_automation_wave_2_no_12.py +++ b/etl/month_end_automation_wave_2_no_12.py @@ -141,7 +141,8 @@ filtered_dfs.append(retro) # RA NO Show ra_ns = df[ - df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) + (df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) & + (df["ra no show evidence"] != 0) ].copy() ra_ns["job_type"] = "RA NO SHOW" filtered_dfs.append(ra_ns) @@ -157,7 +158,8 @@ filtered_dfs.append(ra_ns) # Post visit no show epc_ns = df[ - df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999) + (df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999)) & + (df["epc no show evidence"] != 0) ].copy() epc_ns["job_type"] = "EPC NO SHOW" filtered_dfs.append(epc_ns) diff --git a/etl/month_end_automation_wave_2_no_13.py b/etl/month_end_automation_wave_2_no_13.py index 412c6f7..30a09bf 100644 --- a/etl/month_end_automation_wave_2_no_13.py +++ b/etl/month_end_automation_wave_2_no_13.py @@ -140,7 +140,8 @@ filtered_dfs.append(retro) # RA NO Show ra_ns = df[ - df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) + (df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) & + (df["ra no show evidence"] != 0 ) ].copy() ra_ns["job_type"] = "RA NO SHOW" filtered_dfs.append(ra_ns) diff --git a/etl/month_end_automation_wave_2_no_14.py b/etl/month_end_automation_wave_2_no_14.py index 9f4e83b..733be63 100644 --- a/etl/month_end_automation_wave_2_no_14.py +++ b/etl/month_end_automation_wave_2_no_14.py @@ -143,7 +143,8 @@ filtered_dfs.append(retro) # RA NO Show ra_ns = df[ - df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) + (df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) & + (df["ra no show evidence"] != 0) ].copy() ra_ns["job_type"] = "RA NO SHOW" filtered_dfs.append(ra_ns) @@ -159,7 +160,8 @@ filtered_dfs.append(ra_ns) # Post visit no show epc_ns = df[ - df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999) + (df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999)) & + (df["post epc no show evidence"] != 0) ].copy() epc_ns["job_type"] = "EPC NO SHOW" filtered_dfs.append(epc_ns) diff --git a/etl/month_end_automation_wave_2_no_15.py b/etl/month_end_automation_wave_2_no_15.py index c4e3613..9348fe4 100644 --- a/etl/month_end_automation_wave_2_no_15.py +++ b/etl/month_end_automation_wave_2_no_15.py @@ -143,7 +143,8 @@ filtered_dfs.append(retro) # RA NO Show ra_ns = df[ - df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) + (df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) & + (df["ra no show evidence"] != 0) ].copy() ra_ns["job_type"] = "RA NO SHOW" filtered_dfs.append(ra_ns) @@ -151,7 +152,8 @@ filtered_dfs.append(ra_ns) # ATT NO Show att_ns = df[ - df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) + (df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)) & + (df["att no show evidence"] != 0) ].copy() att_ns["job_type"] = "ATT NO SHOW" filtered_dfs.append(att_ns) @@ -159,7 +161,8 @@ filtered_dfs.append(att_ns) # Post visit no show epc_ns = df[ - df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999) + (df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999)) & + (df["epc no show evidence"] != 0) ].copy() epc_ns["job_type"] = "EPC NO SHOW" filtered_dfs.append(epc_ns) diff --git a/etl/month_end_automation_wave_2_no_16.py b/etl/month_end_automation_wave_2_no_16.py index 0c6d3c1..ce43999 100644 --- a/etl/month_end_automation_wave_2_no_16.py +++ b/etl/month_end_automation_wave_2_no_16.py @@ -143,7 +143,8 @@ filtered_dfs.append(retro) # RA NO Show ra_ns = df[ - df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) + (df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) & + (df["ra no show evidence"] != 0) ].copy() ra_ns["job_type"] = "RA NO SHOW" filtered_dfs.append(ra_ns) @@ -151,7 +152,8 @@ filtered_dfs.append(ra_ns) # ATT NO Show att_ns = df[ - df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) + (df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)) & + (df["att no show evidence"] != 0) ].copy() att_ns["job_type"] = "ATT NO SHOW" filtered_dfs.append(att_ns) diff --git a/etl/month_end_automation_wave_2_no_3.py b/etl/month_end_automation_wave_2_no_3.py index 8a7ca4c..56b2efd 100644 --- a/etl/month_end_automation_wave_2_no_3.py +++ b/etl/month_end_automation_wave_2_no_3.py @@ -164,7 +164,8 @@ filtered_dfs.append(retro) # RA NO Show ra_ns = df[ - df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) + (df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) & + (df[" ra no show evidence"] != 0) ].copy() ra_ns["job_type"] = "RA NO SHOW" filtered_dfs.append(ra_ns) @@ -173,7 +174,8 @@ filtered_dfs.append(ra_ns) # ATT NO Show att_ns = df[ - df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) + (df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)) & + (df["att no show evidence"] != 0) ].copy() att_ns["job_type"] = "ATT NO SHOW" filtered_dfs.append(att_ns) @@ -181,7 +183,8 @@ filtered_dfs.append(att_ns) # Post visit no show epc_ns = df[ - df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999) + (df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999)) & + (df["post epc no show evidence"] != 0) ].copy() epc_ns["job_type"] = "EPC NO SHOW" filtered_dfs.append(epc_ns) diff --git a/etl/month_end_automation_wave_2_no_4.py b/etl/month_end_automation_wave_2_no_4.py index 6fe0d90..4678aba 100644 --- a/etl/month_end_automation_wave_2_no_4.py +++ b/etl/month_end_automation_wave_2_no_4.py @@ -160,7 +160,8 @@ filtered_dfs.append(retro) # RA NO Show ra_ns = df[ - df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) + (df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) & + (df["ra no show evidence"] != 0) ].copy() ra_ns["job_type"] = "RA NO SHOW" filtered_dfs.append(ra_ns) @@ -168,7 +169,8 @@ filtered_dfs.append(ra_ns) # ATT NO Show att_ns = df[ - df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) + (df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)) & + (df["att no show evidence"] != 0) ].copy() att_ns["job_type"] = "ATT NO SHOW" filtered_dfs.append(att_ns) @@ -176,7 +178,8 @@ filtered_dfs.append(att_ns) # Post visit no show epc_ns = df[ - df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999) + (df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999)) & + (df["post epc no show evidence"] != 0) ].copy() epc_ns["job_type"] = "EPC NO SHOW" filtered_dfs.append(epc_ns) diff --git a/etl/month_end_automation_wave_2_no_5.py b/etl/month_end_automation_wave_2_no_5.py index daaa8b0..f244dc5 100644 --- a/etl/month_end_automation_wave_2_no_5.py +++ b/etl/month_end_automation_wave_2_no_5.py @@ -160,7 +160,8 @@ filtered_dfs.append(retro) # RA NO Show ra_ns = df[ - df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) + (df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) & + (df["ra no show evidence"] != 0) ].copy() ra_ns["job_type"] = "RA NO SHOW" filtered_dfs.append(ra_ns) @@ -168,7 +169,8 @@ filtered_dfs.append(ra_ns) # ATT NO Show att_ns = df[ - df["pre att no show evidence"].fillna(-9999) != df["pre att no show invoice"].fillna(-9999) + (df["pre att no show evidence"].fillna(-9999) != df["pre att no show invoice"].fillna(-9999)) & + (df["pre att no show evidence"] ! = 0) ].copy() att_ns["job_type"] = "ATT NO SHOW" filtered_dfs.append(att_ns) @@ -176,7 +178,8 @@ filtered_dfs.append(att_ns) # Post visit no show epc_ns = df[ - df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999) + (df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999)) & + (df["epc no show evidence"] != 0 ) ].copy() epc_ns["job_type"] = "EPC NO SHOW" filtered_dfs.append(epc_ns) diff --git a/etl/month_end_automation_wave_2_no_7.py b/etl/month_end_automation_wave_2_no_7.py index fb15205..d834847 100644 --- a/etl/month_end_automation_wave_2_no_7.py +++ b/etl/month_end_automation_wave_2_no_7.py @@ -143,7 +143,8 @@ filtered_dfs.append(retro) # RA NO Show ra_ns = df[ - df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) + (df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) & + (df["ra no show evidence"] != 0) ].copy() ra_ns["job_type"] = "RA NO SHOW" filtered_dfs.append(ra_ns) @@ -151,7 +152,8 @@ filtered_dfs.append(ra_ns) # ATT NO Show att_ns = df[ - df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) + (df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)) & + (df["att no show evidence"] != 0 ) ].copy() att_ns["job_type"] = "ATT NO SHOW" filtered_dfs.append(att_ns) @@ -159,7 +161,8 @@ filtered_dfs.append(att_ns) # Post visit no show epc_ns = df[ - df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999) + (df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999)) & + (df["epc no show evidence"] != 0 ) ].copy() epc_ns["job_type"] = "EPC NO SHOW" filtered_dfs.append(epc_ns) diff --git a/etl/month_end_automation_wave_2_no_8.py b/etl/month_end_automation_wave_2_no_8.py index ddd2368..8951e71 100644 --- a/etl/month_end_automation_wave_2_no_8.py +++ b/etl/month_end_automation_wave_2_no_8.py @@ -11,6 +11,23 @@ monday = MondayClient(monday_key) # NCHA SHDF 2.1 SBS board_ids = ["8668578700"] +rate_card_data = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + 259, 125, 280, 125, 125, + 650, 415, 195, 175, 135, + 120, 85, 125, 60, + 25, 25, 25 + ] +} + +rate_card_df = pd.DataFrame(rate_card_data) + for board in tqdm(board_ids): board_data = monday.boards.fetch_boards_by_id(board) @@ -67,11 +84,12 @@ df = pd.DataFrame(all_records) filtered_dfs = [] -def get_df(df, column_name, success_critera, job_name): +def get_df(df, column_name, success_critera, job_name=None): _ = df[ df[column_name].str.lower().isin(success_critera) ].copy() - _["job_type"] = job_name + if job_name: + _["job_type"] = job_name return _ @@ -87,21 +105,43 @@ filtered_dfs.append(att) # V1 Coordination v1 = get_df(df, "v1 coordination status".lower(), [ "rc complete", -], "V1 Coordination") +], "Coordination Stage 1 v1") filtered_dfs.append(v1) # V2 Coordination -v2 = get_df(df, "v2 coordination status", ["rc v2 complete", "uploaded"], "V2 Coordination") +v2 = get_df(df, "v2 coordination status", ["rc v2 complete", "uploaded"], "Coordination Stage 1 v2 remodel") filtered_dfs.append(v2) # # V3 Coordination # v3 = get_df(df, "") # # filtered_dfs.append(v3) -# Design stage 1 -design1 = get_df(df, "rd invoiced", ["to invoice"], "Design") +# TODO check with Kevin [Coordination stage 2, design complex and revision] +# Corodination stage 2 +cors2 = get_df(df, "rc stg. 2", ["to invoice"], "Coordination Stage 2") +filtered_dfs.append(cors2) + +# Design Archtype Complex +design1 = get_df(df, "rd invoiced", ["to invoice"]) +design1 = get_df(design1, "design type", ["archetype (complex)"]) +design1 = get_df(design1, "design upload to sharepoint", ["done"], "Design Archetype Complex") filtered_dfs.append(design1) +# Design Archtype Simple +design1 = get_df(df, "rd invoiced", ["to invoice"]) +design1 = get_df(design1, "design type", ["archetype (simple)"]) +design1 = get_df(design1, "design upload to sharepoint", ["done"], "Design Archetype Simple") +filtered_dfs.append(design1) + +# Design Repitive Simple +design1 = get_df(df, "rd invoiced", ["to invoice"]) +design1 = get_df(design1, "design type", ["Design Repetitive Simple"]) +design1 = get_df(design1, "design upload to sharepoint", ["done"], "Design Repetitive Simple") +filtered_dfs.append(design1) + +# TOOD Kevin: +# Check with Design checks and check with cor dination stage 2 checks + # Design revision # design2 = get_df(df, "design revision invoice", [ # "Rev. A to invoice".lower(), @@ -120,7 +160,7 @@ lodg1 = get_df(df, "phase 1 to be invoiced".lower(), ["phase 1 to be invoiced"], filtered_dfs.append(lodg1) # Full Lodgement Phase -full_lodgement = get_df(df, "full lodgement invoice status".lower(), ["to be invoice"], "Full Lodgement") +full_lodgement = get_df(df, "full lodgement invoice status".lower(), ["to be invoice"], "Full Lodgement phase 2") filtered_dfs.append(full_lodgement) # POST EPC @@ -146,7 +186,8 @@ filtered_dfs.append(retro) # RA NO Show ra_ns = df[ - df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) + (df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) & + (df["ra no show evidence"] != 0 ) ].copy() ra_ns["job_type"] = "RA NO SHOW" filtered_dfs.append(ra_ns) @@ -154,7 +195,8 @@ filtered_dfs.append(ra_ns) # ATT NO Show att_ns = df[ - df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) + (df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)) & + (df["att no show evidence"] !=0 ) ].copy() att_ns["job_type"] = "ATT NO SHOW" filtered_dfs.append(att_ns) @@ -169,4 +211,15 @@ filtered_dfs.append(att_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) -final_df[['address', 'client', 'job_type']] \ No newline at end of file +final_df[['address', 'client', 'job_type']] + +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_csv(f'NCHA SHDF 2.1 SBS_{timestamp}.csv', index=False) diff --git a/etl/month_end_automation_wave_2_no_9.py b/etl/month_end_automation_wave_2_no_9.py index ca198f5..f295ca6 100644 --- a/etl/month_end_automation_wave_2_no_9.py +++ b/etl/month_end_automation_wave_2_no_9.py @@ -143,7 +143,8 @@ filtered_dfs.append(retro) # RA NO Show ra_ns = df[ - df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) + (df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) & + (df["ra no show evidence"] != 0) ].copy() ra_ns["job_type"] = "RA NO SHOW" filtered_dfs.append(ra_ns) @@ -151,7 +152,8 @@ filtered_dfs.append(ra_ns) # ATT NO Show att_ns = df[ - df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) + (df(df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999))) & + (df["att no show evidence"] != 0) ].copy() att_ns["job_type"] = "ATT NO SHOW" filtered_dfs.append(att_ns)