diff --git a/etl/month_end_automation_wave_2_no_10.py b/etl/month_end_automation_wave_2_no_10.py index 9b7b3e7..0ca72eb 100644 --- a/etl/month_end_automation_wave_2_no_10.py +++ b/etl/month_end_automation_wave_2_no_10.py @@ -121,7 +121,7 @@ filtered_dfs.append(v1) cors2 = df[ df["rc stg. 2"].str.lower().isin(["to invoice"]) ] -cors2["joby_type"] = "Coordination Stage 2" +cors2["job_type"] = "Coordination Stage 2" filtered_dfs.append(cors2) # # Design stage 1 diff --git a/etl/month_end_automation_wave_2_no_11.py b/etl/month_end_automation_wave_2_no_11.py index db24dbe..e9d2349 100644 --- a/etl/month_end_automation_wave_2_no_11.py +++ b/etl/month_end_automation_wave_2_no_11.py @@ -121,7 +121,7 @@ filtered_dfs.append(v2) cors2 = df[ df["rc stg. 2"].str.lower().isin(["to invoice"]) ] -cors2["joby_type"] = "Coordination Stage 2" +cors2["job_type"] = "Coordination Stage 2" filtered_dfs.append(cors2) # Design stage 1 diff --git a/etl/month_end_automation_wave_2_no_12.py b/etl/month_end_automation_wave_2_no_12.py index 464d211..54469d9 100644 --- a/etl/month_end_automation_wave_2_no_12.py +++ b/etl/month_end_automation_wave_2_no_12.py @@ -122,7 +122,7 @@ filtered_dfs.append(v2) cors2 = df[ df["rc stage 2 invoice"].str.lower().isin(["to invoice"]) ].copy() -cors2["joby_type"] = "Coordination Stage 2" +cors2["job_type"] = "Coordination Stage 2" filtered_dfs.append(cors2) # Design stage 1 diff --git a/etl/month_end_automation_wave_2_no_13.py b/etl/month_end_automation_wave_2_no_13.py index 958c126..53535d3 100644 --- a/etl/month_end_automation_wave_2_no_13.py +++ b/etl/month_end_automation_wave_2_no_13.py @@ -119,7 +119,7 @@ filtered_dfs.append(v2) cors2 = df[ df["rc stage 2"].str.lower().isin(["to invoice"]) ] -cors2["joby_type"] = "Coordination Stage 2" +cors2["job_type"] = "Coordination Stage 2" filtered_dfs.append(cors2) # Design stage 1 diff --git a/etl/month_end_automation_wave_2_no_14.py b/etl/month_end_automation_wave_2_no_14.py index 1099185..e436aec 100644 --- a/etl/month_end_automation_wave_2_no_14.py +++ b/etl/month_end_automation_wave_2_no_14.py @@ -121,9 +121,9 @@ filtered_dfs.append(v1) # Coordination stage 2 Please complete # cors2 = df[ -# df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) +# df["rc stage 2"].str.lower().isin(["to invoice", "done"]) # ] -# cors2["joby_type"] = "Coordination Stage 2" +# cors2["job_type"] = "Coordination Stage 2" # filtered_dfs.append(cors2) # Design stage 1 diff --git a/etl/month_end_automation_wave_2_no_15.py b/etl/month_end_automation_wave_2_no_15.py index 1b10f25..a175801 100644 --- a/etl/month_end_automation_wave_2_no_15.py +++ b/etl/month_end_automation_wave_2_no_15.py @@ -128,7 +128,7 @@ filtered_dfs.append(v2) cors2 = df[ df["rc stg 2"].str.lower().isin(["to invoice"]) ] -cors2["joby_type"] = "Coordination Stage 2" +cors2["job_type"] = "Coordination Stage 2" filtered_dfs.append(cors2) # Design stage diff --git a/etl/month_end_automation_wave_2_no_16.py b/etl/month_end_automation_wave_2_no_16.py index 5505f11..70b5175 100644 --- a/etl/month_end_automation_wave_2_no_16.py +++ b/etl/month_end_automation_wave_2_no_16.py @@ -122,7 +122,7 @@ filtered_dfs.append(v1) cors2 = df[ df["rc stg. 2"].str.lower().isin(["to invoice"]) ] -cors2["joby_type"] = "Coordination Stage 2" +cors2["job_type"] = "Coordination Stage 2" filtered_dfs.append(cors2) # Design Archetype diff --git a/etl/month_end_automation_wave_2_no_3.py b/etl/month_end_automation_wave_2_no_3.py index 2500d25..e8a46de 100644 --- a/etl/month_end_automation_wave_2_no_3.py +++ b/etl/month_end_automation_wave_2_no_3.py @@ -124,7 +124,7 @@ filtered_dfs.append(_) cors2 = df[ df["rc stg. 2"].str.lower().isin(["to invoice"]) ] -cors2["joby_type"] = "Coordination Stage 2" +cors2["job_type"] = "Coordination Stage 2" filtered_dfs.append(cors2) # Design Archetype diff --git a/etl/month_end_automation_wave_2_no_4.py b/etl/month_end_automation_wave_2_no_4.py index 1b7f6f4..787aede 100644 --- a/etl/month_end_automation_wave_2_no_4.py +++ b/etl/month_end_automation_wave_2_no_4.py @@ -131,7 +131,7 @@ filtered_dfs.append(_) cors2 = df[ df["rc stg. 2 status"].str.lower().isin(["to invoice", "completed"]) ] -cors2["joby_type"] = "Coordination Stage 2" +cors2["job_type"] = "Coordination Stage 2" filtered_dfs.append(cors2) # Design Archetype diff --git a/etl/month_end_automation_wave_2_no_5.py b/etl/month_end_automation_wave_2_no_5.py index c26834b..da06633 100644 --- a/etl/month_end_automation_wave_2_no_5.py +++ b/etl/month_end_automation_wave_2_no_5.py @@ -122,7 +122,7 @@ filtered_dfs.append(v1) cors2 = df[ df["rc stage 2"].str.lower().isin(["to invoice",]) ] -cors2["joby_type"] = "Coordination Stage 2" +cors2["job_type"] = "Coordination Stage 2" filtered_dfs.append(cors2) # Design stage 1 diff --git a/etl/month_end_automation_wave_2_no_6.py b/etl/month_end_automation_wave_2_no_6.py index 8091aa1..064e676 100644 --- a/etl/month_end_automation_wave_2_no_6.py +++ b/etl/month_end_automation_wave_2_no_6.py @@ -123,7 +123,7 @@ filtered_dfs.append(v1) cors2 = df[ df["rc stage 2 status"].str.lower().isin(["to invoice"]) ] -cors2["joby_type"] = "Coordination Stage 2" +cors2["job_type"] = "Coordination Stage 2" filtered_dfs.append(cors2) # Design stage 1 diff --git a/etl/month_end_automation_wave_2_no_7.py b/etl/month_end_automation_wave_2_no_7.py index e1fbf9c..3c4b6ef 100644 --- a/etl/month_end_automation_wave_2_no_7.py +++ b/etl/month_end_automation_wave_2_no_7.py @@ -156,7 +156,7 @@ filtered_dfs.append(v3) cors2 = df[ df["rc stage 2"].str.lower().isin(["to invoice"]) ] -cors2["joby_type"] = "Coordination Stage 2" +cors2["job_type"] = "Coordination Stage 2" filtered_dfs.append(cors2) # Design stage Archetype diff --git a/etl/month_end_automation_wave_2_no_9.py b/etl/month_end_automation_wave_2_no_9.py index 7d92c72..ad01a92 100644 --- a/etl/month_end_automation_wave_2_no_9.py +++ b/etl/month_end_automation_wave_2_no_9.py @@ -122,7 +122,7 @@ filtered_dfs.append(v1) cors2 = df[ df["rc stage 2"].str.lower().isin(["to invoice"]) ].copy() -cors2["joby_type"] = "Coordination Stage 2" +cors2["job_type"] = "Coordination Stage 2" filtered_dfs.append(cors2) # Design stage 1 diff --git a/etl/month_end_automation_wave_3_layout.py b/etl/month_end_automation_wave_3_layout.py index dd3765b..8b18355 100644 --- a/etl/month_end_automation_wave_3_layout.py +++ b/etl/month_end_automation_wave_3_layout.py @@ -36,19 +36,19 @@ empty = "Rate card info missing" rate_cards = {} - +check_with_andres = "Check with Andres!!!" rate_card_data_watford_warm_homes = { "job_type": [ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", - "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" + "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling", "design type not specified" ], "rate": [ 165, empty, empty, empty, empty, empty, empty, empty, empty, empty, empty, empty, empty, empty, empty, - empty, empty, empty, empty, empty, + empty, empty, empty, empty, empty, check_with_andres ] } rate_cards.update({"9641491000": pd.DataFrame(rate_card_data_watford_warm_homes)}) @@ -60,13 +60,13 @@ rate_card_data_l_and_q_london = { "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", - "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" + "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling", "design type not specified" ], "rate": [ empty, empty, empty, empty, empty, empty, empty, empty, empty, empty, empty, empty, empty, empty, empty, - empty, empty, empty, 280, 150 + empty, empty, empty, 280, 150, check_with_andres ] } @@ -78,13 +78,13 @@ rate_card_data_northhumberland_country_shdf_wave_3 = { "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Design Repetitive Complex", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", - "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" + "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling", "design type not specified" ], "rate": [ 259, 125, "280 or 310 depending on SH or LA", 125, 125, 650, 415, 195, 225, 175, 135, 120, "60 - check", "85 or 90 depending on SH or LA", "110 or 125 depending on SH or LA", - 60, 45, 45, 45, empty, empty + 60, 45, 45, 45, empty, empty, check_with_andres ] } @@ -96,13 +96,13 @@ rate_card_data_walk_ups = { "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", - "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" + "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling", "design type not specified" ], "rate": [ 259, 125, 280, 125, 125, 650, 415, 195, 175, 135, 120, "60 to check", 85, 125, 60, - 45, 45, 45, empty, empty, + 45, 45, 45, empty, empty, check_with_andres ] } @@ -113,13 +113,13 @@ rate_card_data_empty = { "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", - "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" + "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling", "design type not specified" ], "rate": [ empty, empty, empty, empty, empty, empty, empty, empty, empty, empty, empty,empty , empty, empty, empty, - empty, empty, empty, empty, empty, + empty, empty, empty, empty, empty, check_with_andres ] } @@ -128,13 +128,13 @@ rate_card_data_cardo = { "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple","Design Repetitive Complex", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", - "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" + "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling", "design type not specified" ], "rate": [ 259, 125, 260, 125, 125, 415, 415, 225, 225, 175, 135, 120 , "60 - check", 85, 125, - 60, 45, 45, 45, empty, empty, + 60, 45, 45, 45, empty, empty, check_with_andres ] } @@ -145,13 +145,13 @@ rate_card_data_seddon = { "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple","Design Repetitive Complex", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", - "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" + "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling", "design type not specified" ], "rate": [ 259, 125, 280, 125, 125, 650, 415, 195, 225, 175, 135, 120 , "60 - check", 85, 125, - 60, 45, 45, 45, empty, empty, + 60, 45, 45, 45, empty, empty,check_with_andres ] } @@ -251,19 +251,28 @@ for board, all_records in board_to_record.items(): filtered_dfs.append(design1) # Design archetype simple - design1 = get_df(design, "design invoice type", ["archetype (simple)"], "Design Archetype Simple") + design2 = get_df(design, "design invoice type", ["archetype (simple)"], "Design Archetype Simple") if not design1.empty: - filtered_dfs.append(design1) + filtered_dfs.append(design2) # Design repetitive simple - design1 = get_df(design, "design invoice type", ["archetype (simple)"], "Design Archetype repetitive") + design3 = get_df(design, "design invoice type", ["archetype (simple)"], "Design Archetype repetitive") if not design1.empty: - filtered_dfs.append(design1) + filtered_dfs.append(design3) # Design repetitive complex - design1 = get_df(design, "design invoice type", ["archetype (complex)"], "Design Archetype complex") + design4 = get_df(design, "design invoice type", ["archetype (complex)"], "Design Archetype complex") if not design1.empty: - filtered_dfs.append(design1) + filtered_dfs.append(design4) + + # Design not specified + all_filtered = pd.concat([design1, design2, design3, design4], ignore_index=True) + design_remaining = design.loc[~design.index.isin(all_filtered.index)] + if not design_remaining.empty: + design_remaining["job_type"] = "design type not specified" + filtered_dfs.append(design_remaining) + + # Design Revision revision_letter = ['a', 'b', 'c', 'd']