From 232509ef16c5dc507b046ad8b0ea13fbb308d574 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 17 Sep 2025 13:01:29 +0000 Subject: [PATCH] finish month end --- etl/month_end_automation_wave_2_layout.py | 16 ++++++---- etl/month_end_automation_wave_2_no_8.py | 39 +++++++++++++++-------- 2 files changed, 36 insertions(+), 19 deletions(-) diff --git a/etl/month_end_automation_wave_2_layout.py b/etl/month_end_automation_wave_2_layout.py index 5a1296b..1fb6dd1 100644 --- a/etl/month_end_automation_wave_2_layout.py +++ b/etl/month_end_automation_wave_2_layout.py @@ -20,13 +20,13 @@ rate_card_data = { "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", - "RA no show", "ATT no show", "post EPC no show", "Design Revision" + "RA no show", "ATT no show", "post EPC no show", "Design Revision", "Design Specify" ], "rate": [ 207.65, 101, 186.4, 98, 98, 450, 150, 163, 135, 120, "60 - Needs to be verified (Post EPR)", 45, 90.5, 40, - "25 - check number on right", "25 - check number on right", "25 - check number on right", "check with Kevin" + "25 - check number on right", "25 - check number on right", "25 - check number on right", "check with Kevin", "ask adreas" ] } @@ -147,8 +147,6 @@ design1 = df[ (df["design type for invoicing"].str.lower().isin(["archetype"])) & (df["design invoice status"].str.lower().isin(["to invoice"])) ].copy() design1["job_type"] = "Design Archetype" -design1["evidence_record"] = None -design1["invoice_record"] = None filtered_dfs.append(design1) # design type reptitive @@ -156,10 +154,16 @@ design1 = df[ (df["design type for invoicing"].str.lower().isin(["repetitive"])) & df["design invoice status"].str.lower().isin(["to invoice"]) ].copy() design1["job_type"] = "Design Repetitive" -design1["evidence_record"] = None -design1["invoice_record"] = None filtered_dfs.append(design1) +design_other = df[ + (~df["design type for invoicing"].str.lower().isin(["archetype", "repetitive"])) + & (df["design invoice status"].str.lower().isin(["to invoice"])) +].copy() + +design_other["job_type"] = "Design Specify" +filtered_dfs.append(design_other) + # Design stage revisions design2 = df[ df["design revision invoice status"].str.lower().isin(["to invoice"]) diff --git a/etl/month_end_automation_wave_2_no_8.py b/etl/month_end_automation_wave_2_no_8.py index 1d8d619..7c2dd56 100644 --- a/etl/month_end_automation_wave_2_no_8.py +++ b/etl/month_end_automation_wave_2_no_8.py @@ -18,13 +18,13 @@ rate_card_data = { "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", - "RA no show", "ATT no show", "post EPC no show", "Design Revision" + "RA no show", "ATT no show", "post EPC no show", "Design Revision", "Design Repetitive Complex", "design type not specified" ], "rate": [ 259, 125, 280, 125, 125, 650, 415, 195, 175, 135, 120, "60 - Double check with Kevin/Marianne EPR", 85, 125, 60, - "45- check with right", "45 - check with right", "45 - check with right", "Design Revision check with Kevin/Marianne" + "45- check with right", "45 - check with right", "45 - check with right", "Design Revision check with Kevin/Marianne", "ask junte to add", "ask andreas" ] } @@ -125,22 +125,35 @@ cors2 = get_df(df, "rc stg. 2", ["to invoice"], "Coordination Stage 2") filtered_dfs.append(cors2) # Design Archtype Complex -design1 = get_df(df, "rd invoiced", ["to invoice"]) -design1 = get_df(design1, "design type", ["archetype (complex)"]) +design = get_df(df, "rd invoiced", ["to invoice"]) +design1 = get_df(design, "design type", ["archetype (complex)"]) design1 = get_df(design1, "design upload to sharepoint", ["done"], "Design Archetype Complex") -filtered_dfs.append(design1) +if not design1.empty: + filtered_dfs.append(design1) # Design Archtype Simple -design1 = get_df(df, "rd invoiced", ["to invoice"]) -design1 = get_df(design1, "design type", ["archetype (simple)"]) -design1 = get_df(design1, "design upload to sharepoint", ["done"], "Design Archetype Simple") -filtered_dfs.append(design1) +design2 = get_df(design, "design type", ["archetype (simple)"]) +design2 = get_df(design2, "design upload to sharepoint", ["done"], "Design Archetype Simple") +if not design2.empty: + filtered_dfs.append(design2) # Design Repitive Simple -design1 = get_df(df, "rd invoiced", ["to invoice"]) -design1 = get_df(design1, "design type", ["Design Repetitive Simple"]) -design1 = get_df(design1, "design upload to sharepoint", ["done"], "Design Repetitive Simple") -filtered_dfs.append(design1) +design3 = get_df(design, "design type", ["repetitive (simple)"]) +design3 = get_df(design3, "design upload to sharepoint", ["done"], "Design Repetitive Simple") +if not design3.empty: + filtered_dfs.append(design3) + +# Design Reptitive Complex +design4 = get_df(design, "design type", ["repetitive (complex)"]) +design4 = get_df(design4, "design upload to sharepoint", ["done"], "Design Repetitive Complex") +if not design4.empty: + filtered_dfs.append(design4) + +all_filtered = pd.concat([df for df in (design1, design2, design3, design4) if not df.empty]) +design_remaining = design.loc[~design.index.isin(all_filtered.index)] +if not design_remaining.empty: + design_remaining["job_type"] = "design type not specified" + filtered_dfs.append(design_remaining) # Design revision design2 = get_df(df, "design revision invoice status", [