diff --git a/.github/workflows/months_end.yml b/.github/workflows/months_end.yml index 5c833de..850034e 100644 --- a/.github/workflows/months_end.yml +++ b/.github/workflows/months_end.yml @@ -6,6 +6,7 @@ on: - cron: '0 7 23 * *' # On the 23th of every month at 07:00 UTC - cron: '0 7 25 * *' # On the 25th of every month at 07:00 UTC - cron: '0 7 26 * *' # On the 26th of every month at 07:00 UTC + - cron: '0 7 27 * *' # On the 26th of every month at 07:00 UTC - cron: '0 7 29 * *' # On the 29th of every month at 07:00 UTC - cron: '0 7 19 * *' # On the 29th of every month at 07:00 UTC workflow_dispatch: diff --git a/etl/month_end_automation_wave_2_no_3.py b/etl/month_end_automation_wave_2_no_3.py index 424834c..c590751 100644 --- a/etl/month_end_automation_wave_2_no_3.py +++ b/etl/month_end_automation_wave_2_no_3.py @@ -186,7 +186,7 @@ filtered_dfs.append(lodg2) # POST EPC post_epc = df[ - df["post epc"].str.lower().isin(["success", "pics uploaded"]) + df["post epc/epr"].str.lower().isin(["success", "pics uploaded"]) ].copy() post_epc["job_type"] = "POST EPC" post_epc["evidence_record"] = None @@ -196,7 +196,7 @@ filtered_dfs.append(post_epc) # POST EPR post_epr = df[ - df["post epc"].str.lower().isin(["post epr completed"]) + df["post epc/epr"].str.lower().isin(["post epr completed"]) ].copy() post_epr["job_type"] = "POST EPR" post_epr["evidence_record"] = None diff --git a/etl/month_end_automation_wave_3_layout.py b/etl/month_end_automation_wave_3_layout.py index 1dd8947..27006d6 100644 --- a/etl/month_end_automation_wave_3_layout.py +++ b/etl/month_end_automation_wave_3_layout.py @@ -46,7 +46,7 @@ check_with_andres = "Check with Andres!!!" rate_card_data_powys = { "job_type": [ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", - "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Design Repetitive Complex", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Design Repetitive Complex", "Coordination Stage 2", "Lodgement", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling", "design type not specified" ], @@ -62,7 +62,7 @@ rate_cards.update({"18232420839": pd.DataFrame(rate_card_data_powys)}) rate_card_data_devon = { "job_type": [ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", - "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Design Repetitive Complex", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Design Repetitive Complex", "Coordination Stage 2", "Lodgement", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling", "design type not specified" ], @@ -79,7 +79,7 @@ rate_cards.update({"18380751311": pd.DataFrame(rate_card_data_devon)}) rate_card_data_watford_warm_homes = { "job_type": [ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", - "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling", "design type not specified" ], @@ -95,7 +95,7 @@ rate_cards.update({"9641491000": pd.DataFrame(rate_card_data_watford_warm_homes) rate_card_data_ncha_wave_3 = { "job_type": [ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", - "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling", "design type not specified" ], @@ -113,7 +113,7 @@ rate_cards.update({"9929454382": pd.DataFrame(rate_card_data_ncha_wave_3)}) rate_card_data_l_and_q_london = { "job_type": [ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", - "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling", "design type not specified" ], @@ -131,7 +131,7 @@ rate_cards.update({"8830772914": pd.DataFrame(rate_card_data_l_and_q_london)}) rate_card_data_northhumberland_country_shdf_wave_3 = { "job_type": [ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", - "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Design Repetitive Complex", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Design Repetitive Complex", "Coordination Stage 2", "Lodgement", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling", "design type not specified" ], @@ -149,7 +149,7 @@ rate_cards.update({"9660895490": pd.DataFrame(rate_card_data_northhumberland_cou rate_card_data_walk_ups = { "job_type": [ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", - "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling", "design type not specified" ], @@ -166,7 +166,7 @@ rate_cards.update({"9349630181": pd.DataFrame(rate_card_data_walk_ups)}) rate_card_data_empty = { "job_type": [ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", - "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling", "design type not specified" ], @@ -181,7 +181,7 @@ rate_card_data_empty = { rate_card_data_cardo = { "job_type": [ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", - "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple","Design Repetitive Complex", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple","Design Repetitive Complex", "Coordination Stage 2", "Lodgement", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling", "design type not specified" ], @@ -198,7 +198,7 @@ rate_cards.update({"9601691730": pd.DataFrame(rate_card_data_cardo)}) rate_card_data_seddon = { "job_type": [ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", - "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple","Design Repetitive Complex", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple","Design Repetitive Complex", "Coordination Stage 2", "Lodgement", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling", "design type not specified" ], @@ -344,23 +344,23 @@ for board, all_records in board_to_record.items(): if not design.empty: filtered_dfs.append(design) - # Lodgement Phase 1 - lodg1 = get_df(df, "lodgement phase 1 invoicing status", ["to invoice"], "Lodgement Phase 1") + # Lodgement + lodg1 = get_df(df, "lodgement invoicing status", ["to invoice"], "Lodgement") if not lodg1.empty: filtered_dfs.append(lodg1) - - # Full Lodgement Phase - lodg2 = get_df(df, "lodgement phase 2 invoicing status", ["to invoice"], "Full lodgement phase 2") - if not lodg2.empty: - filtered_dfs.append(lodg2) + + # Retrofit Evaluation + re = get_df(df, "evaluation invoicing status", ["to invoice"], "retrofit evaluation") + if not re.empty: + filtered_dfs.append(re) - # POST EPC + Retrofit Evaluation - post_epc = get_df(df, "post epc & eval. invoicing status", ["epc to invoice"], "POST EPC") + # POST EPC + post_epc = get_df(df, "Post EPC invoicing status", ["epc to invoice"], "POST EPC") if not post_epc.empty: filtered_dfs.append(post_epc) # POST EPR - post_epr = get_df(df, "post epc & eval. invoicing status", ["epr to invoice"], "POST EPR") + post_epr = get_df(df, "Post EPR invoicing status", ["epr to invoice"], "POST EPR") if not post_epr.empty: filtered_dfs.append(post_epr) diff --git a/etl/month_end_automation_wave_accent_housing.py b/etl/month_end_automation_wave_accent_housing.py index f79b29a..42f02ba 100644 --- a/etl/month_end_automation_wave_accent_housing.py +++ b/etl/month_end_automation_wave_accent_housing.py @@ -20,11 +20,11 @@ rate_card_data_2502_accent_housing = { "job_type": [ "First half of MTP", "Second half of MTP", "full cost mtp", "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Complex", "Design Repetitive Simple", - "Design Revision", "design type not specified", "Coordination Stage 2", "POST EPC", "POST EPR", "POST ATT", "post EPC NO SHOW" - + "Design Revision", "design type not specified", "Coordination Stage 2", "POST EPC", "POST EPR", "POST ATT", "post EPC NO SHOW", "RE", + "Lodgement" ], "rate": [ - 150, 130, 280, junte, junte, junte, junte, junte, "please ask andreas", junte, junte, junte, junte, junte + 150, 130, 280, junte, junte, junte, junte, junte, "please ask andreas", junte, junte, junte, junte, junte, junte, junte ] } # ToDO @@ -147,16 +147,25 @@ rc = get_df(df, "rc stage 2", ["to invoice"], "Coordination Stage 2") if not rc.empty: filtered_dfs.append(rc) -# POST EPC + Retrofit Evaluation -post_epc = get_df(df, "post epc & eval. invoicing status", ["epc to invoice"], "POST EPC") +# Retrofit Evaluation +re = get_df(df, "Retrofit Evaluation Status", ["to invoice"], "re") +if not re.empty: + filtered_dfs.append(re) + +# POST EPC +post_epc = get_df(df, "Post EPC invoicing status", ["epc to invoice"], "POST EPC") if not post_epc.empty: filtered_dfs.append(post_epc) # POST EPR -post_epr = get_df(df, "post epc & eval. invoicing status", ["epr to invoice"], "POST EPR") +post_epr = get_df(df, "Post EPR invoicing status", ["epr to invoice"], "POST EPR") if not post_epr.empty: filtered_dfs.append(post_epr) +lodg1 = get_df(df, "lodgement invoicing status", ["to invoice"], "Lodgement") +if not lodg1.empty: + filtered_dfs.append(lodg1) + # Post ATT post_att = get_df(df, "post att invoicing status", ["to invoice"], "POST ATT") if not post_att.empty: