From 0633591a2f8def7f3bdbddf83a1a6ea9b772b52b Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 26 Jan 2026 13:20:39 +0000 Subject: [PATCH 1/4] fixed post epr --- etl/month_end_automation_wave_2_no_3.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etl/month_end_automation_wave_2_no_3.py b/etl/month_end_automation_wave_2_no_3.py index 424834c..c590751 100644 --- a/etl/month_end_automation_wave_2_no_3.py +++ b/etl/month_end_automation_wave_2_no_3.py @@ -186,7 +186,7 @@ filtered_dfs.append(lodg2) # POST EPC post_epc = df[ - df["post epc"].str.lower().isin(["success", "pics uploaded"]) + df["post epc/epr"].str.lower().isin(["success", "pics uploaded"]) ].copy() post_epc["job_type"] = "POST EPC" post_epc["evidence_record"] = None @@ -196,7 +196,7 @@ filtered_dfs.append(post_epc) # POST EPR post_epr = df[ - df["post epc"].str.lower().isin(["post epr completed"]) + df["post epc/epr"].str.lower().isin(["post epr completed"]) ].copy() post_epr["job_type"] = "POST EPR" post_epr["evidence_record"] = None From 31c8ae4ff278dbbb95f655514e21844be7a570c3 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 26 Jan 2026 13:46:42 +0000 Subject: [PATCH 2/4] re run --- etl/month_end_automation_wave_3_layout.py | 56 +++++++++---------- ...onth_end_automation_wave_accent_housing.py | 21 +++++-- 2 files changed, 43 insertions(+), 34 deletions(-) diff --git a/etl/month_end_automation_wave_3_layout.py b/etl/month_end_automation_wave_3_layout.py index 1dd8947..0977ce4 100644 --- a/etl/month_end_automation_wave_3_layout.py +++ b/etl/month_end_automation_wave_3_layout.py @@ -27,15 +27,15 @@ class BoardID(Enum): board_ids = [ - # "9349630181", # WCHG Walkups-Operations - # "8830772914", # "L&Q London" - # "9601691730", # Cardo Wales & West - Wave 3 - # "9660895490", # Northumberland County SHDF Wave 3 - # "9641491000", # Watford Warm Homes - # "9671463094", # Seddon + "9349630181", # WCHG Walkups-Operations + "8830772914", # "L&Q London" + "9601691730", # Cardo Wales & West - Wave 3 + "9660895490", # Northumberland County SHDF Wave 3 + "9641491000", # Watford Warm Homes + "9671463094", # Seddon "9929454382", # NCHA SHDF Wave 3 Main Operation Board - # "18232420839", # Powys County Council: - # "18380751311", # Devon County Council + "18232420839", # Powys County Council: + "18380751311", # Devon County Council ] empty = "Rate card info missing" @@ -46,7 +46,7 @@ check_with_andres = "Check with Andres!!!" rate_card_data_powys = { "job_type": [ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", - "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Design Repetitive Complex", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Design Repetitive Complex", "Coordination Stage 2", "Lodgement", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling", "design type not specified" ], @@ -62,7 +62,7 @@ rate_cards.update({"18232420839": pd.DataFrame(rate_card_data_powys)}) rate_card_data_devon = { "job_type": [ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", - "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Design Repetitive Complex", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Design Repetitive Complex", "Coordination Stage 2", "Lodgement", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling", "design type not specified" ], @@ -79,7 +79,7 @@ rate_cards.update({"18380751311": pd.DataFrame(rate_card_data_devon)}) rate_card_data_watford_warm_homes = { "job_type": [ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", - "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling", "design type not specified" ], @@ -95,7 +95,7 @@ rate_cards.update({"9641491000": pd.DataFrame(rate_card_data_watford_warm_homes) rate_card_data_ncha_wave_3 = { "job_type": [ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", - "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling", "design type not specified" ], @@ -113,7 +113,7 @@ rate_cards.update({"9929454382": pd.DataFrame(rate_card_data_ncha_wave_3)}) rate_card_data_l_and_q_london = { "job_type": [ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", - "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling", "design type not specified" ], @@ -131,7 +131,7 @@ rate_cards.update({"8830772914": pd.DataFrame(rate_card_data_l_and_q_london)}) rate_card_data_northhumberland_country_shdf_wave_3 = { "job_type": [ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", - "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Design Repetitive Complex", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Design Repetitive Complex", "Coordination Stage 2", "Lodgement", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling", "design type not specified" ], @@ -149,7 +149,7 @@ rate_cards.update({"9660895490": pd.DataFrame(rate_card_data_northhumberland_cou rate_card_data_walk_ups = { "job_type": [ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", - "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling", "design type not specified" ], @@ -166,7 +166,7 @@ rate_cards.update({"9349630181": pd.DataFrame(rate_card_data_walk_ups)}) rate_card_data_empty = { "job_type": [ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", - "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling", "design type not specified" ], @@ -181,7 +181,7 @@ rate_card_data_empty = { rate_card_data_cardo = { "job_type": [ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", - "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple","Design Repetitive Complex", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple","Design Repetitive Complex", "Coordination Stage 2", "Lodgement", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling", "design type not specified" ], @@ -198,7 +198,7 @@ rate_cards.update({"9601691730": pd.DataFrame(rate_card_data_cardo)}) rate_card_data_seddon = { "job_type": [ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", - "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple","Design Repetitive Complex", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple","Design Repetitive Complex", "Coordination Stage 2", "Lodgement", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling", "design type not specified" ], @@ -344,23 +344,23 @@ for board, all_records in board_to_record.items(): if not design.empty: filtered_dfs.append(design) - # Lodgement Phase 1 - lodg1 = get_df(df, "lodgement phase 1 invoicing status", ["to invoice"], "Lodgement Phase 1") + # Lodgement + lodg1 = get_df(df, "TM Phase 1", ["measures lodged"], "Lodgement") if not lodg1.empty: filtered_dfs.append(lodg1) - - # Full Lodgement Phase - lodg2 = get_df(df, "lodgement phase 2 invoicing status", ["to invoice"], "Full lodgement phase 2") - if not lodg2.empty: - filtered_dfs.append(lodg2) + + # Retrofit Evaluation + re = get_df(df, "retrofit evaluation status", ["uploaded"], "retrofit evaluation") + if not re.empty: + filtered_dfs.append(re) - # POST EPC + Retrofit Evaluation - post_epc = get_df(df, "post epc & eval. invoicing status", ["epc to invoice"], "POST EPC") + # POST EPC + post_epc = get_df(df, "post epc/epr status", ["epc uploaded"], "POST EPC") if not post_epc.empty: filtered_dfs.append(post_epc) # POST EPR - post_epr = get_df(df, "post epc & eval. invoicing status", ["epr to invoice"], "POST EPR") + post_epr = get_df(df, "post epc/epr status", ["epr uploaded"], "POST EPR") if not post_epr.empty: filtered_dfs.append(post_epr) diff --git a/etl/month_end_automation_wave_accent_housing.py b/etl/month_end_automation_wave_accent_housing.py index f79b29a..17ffecd 100644 --- a/etl/month_end_automation_wave_accent_housing.py +++ b/etl/month_end_automation_wave_accent_housing.py @@ -20,11 +20,11 @@ rate_card_data_2502_accent_housing = { "job_type": [ "First half of MTP", "Second half of MTP", "full cost mtp", "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Complex", "Design Repetitive Simple", - "Design Revision", "design type not specified", "Coordination Stage 2", "POST EPC", "POST EPR", "POST ATT", "post EPC NO SHOW" - + "Design Revision", "design type not specified", "Coordination Stage 2", "POST EPC", "POST EPR", "POST ATT", "post EPC NO SHOW", "RE", + "Lodgement" ], "rate": [ - 150, 130, 280, junte, junte, junte, junte, junte, "please ask andreas", junte, junte, junte, junte, junte + 150, 130, 280, junte, junte, junte, junte, junte, "please ask andreas", junte, junte, junte, junte, junte, junte, junte ] } # ToDO @@ -147,16 +147,25 @@ rc = get_df(df, "rc stage 2", ["to invoice"], "Coordination Stage 2") if not rc.empty: filtered_dfs.append(rc) -# POST EPC + Retrofit Evaluation -post_epc = get_df(df, "post epc & eval. invoicing status", ["epc to invoice"], "POST EPC") +# Retrofit Evaluation +re = get_df(df, "retrofit evaluation status", ["uploaded"], "re") +if not re.empty: + filtered_dfs.append(re) + +# POST EPC +post_epc = get_df(df, "post epc/epr status", ["epc uploaded"], "POST EPC") if not post_epc.empty: filtered_dfs.append(post_epc) # POST EPR -post_epr = get_df(df, "post epc & eval. invoicing status", ["epr to invoice"], "POST EPR") +post_epr = get_df(df, "post epc/epr status", ["epr uploaded"], "POST EPR") if not post_epr.empty: filtered_dfs.append(post_epr) +lodg1 = get_df(df, "TM Phase 1", ["measures lodged"], "Lodgement") +if not lodg1.empty: + filtered_dfs.append(lodg1) + # Post ATT post_att = get_df(df, "post att invoicing status", ["to invoice"], "POST ATT") if not post_att.empty: From 3a94b28c04448973a937869010cbbd3bdd284573 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 26 Jan 2026 14:48:31 +0000 Subject: [PATCH 3/4] re run --- etl/month_end_automation_wave_3_layout.py | 6 +++--- etl/month_end_automation_wave_accent_housing.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/etl/month_end_automation_wave_3_layout.py b/etl/month_end_automation_wave_3_layout.py index 0977ce4..133a634 100644 --- a/etl/month_end_automation_wave_3_layout.py +++ b/etl/month_end_automation_wave_3_layout.py @@ -350,17 +350,17 @@ for board, all_records in board_to_record.items(): filtered_dfs.append(lodg1) # Retrofit Evaluation - re = get_df(df, "retrofit evaluation status", ["uploaded"], "retrofit evaluation") + re = get_df(df, "Retrofit Evaluation Status", ["uploaded"], "retrofit evaluation") if not re.empty: filtered_dfs.append(re) # POST EPC - post_epc = get_df(df, "post epc/epr status", ["epc uploaded"], "POST EPC") + post_epc = get_df(df, "Post EPC/EPR Status", ["epc uploaded"], "POST EPC") if not post_epc.empty: filtered_dfs.append(post_epc) # POST EPR - post_epr = get_df(df, "post epc/epr status", ["epr uploaded"], "POST EPR") + post_epr = get_df(df, "Post EPC/EPR Status", ["epr uploaded"], "POST EPR") if not post_epr.empty: filtered_dfs.append(post_epr) diff --git a/etl/month_end_automation_wave_accent_housing.py b/etl/month_end_automation_wave_accent_housing.py index 17ffecd..c3e97e4 100644 --- a/etl/month_end_automation_wave_accent_housing.py +++ b/etl/month_end_automation_wave_accent_housing.py @@ -148,17 +148,17 @@ if not rc.empty: filtered_dfs.append(rc) # Retrofit Evaluation -re = get_df(df, "retrofit evaluation status", ["uploaded"], "re") +re = get_df(df, "Retrofit Evaluation Status", ["uploaded"], "re") if not re.empty: filtered_dfs.append(re) # POST EPC -post_epc = get_df(df, "post epc/epr status", ["epc uploaded"], "POST EPC") +post_epc = get_df(df, "Post EPC/EPR Status", ["epc uploaded"], "POST EPC") if not post_epc.empty: filtered_dfs.append(post_epc) # POST EPR -post_epr = get_df(df, "post epc/epr status", ["epr uploaded"], "POST EPR") +post_epr = get_df(df, "POST EPC/EPR Status", ["epr uploaded"], "POST EPR") if not post_epr.empty: filtered_dfs.append(post_epr) From 50a2a5611f9ad3be625eaf2f23c1379c19b1583d Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 26 Jan 2026 16:56:55 +0000 Subject: [PATCH 4/4] 27th as well --- .github/workflows/months_end.yml | 1 + etl/month_end_automation_wave_3_layout.py | 24 +++++++++---------- ...onth_end_automation_wave_accent_housing.py | 8 +++---- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/.github/workflows/months_end.yml b/.github/workflows/months_end.yml index 5c833de..850034e 100644 --- a/.github/workflows/months_end.yml +++ b/.github/workflows/months_end.yml @@ -6,6 +6,7 @@ on: - cron: '0 7 23 * *' # On the 23th of every month at 07:00 UTC - cron: '0 7 25 * *' # On the 25th of every month at 07:00 UTC - cron: '0 7 26 * *' # On the 26th of every month at 07:00 UTC + - cron: '0 7 27 * *' # On the 26th of every month at 07:00 UTC - cron: '0 7 29 * *' # On the 29th of every month at 07:00 UTC - cron: '0 7 19 * *' # On the 29th of every month at 07:00 UTC workflow_dispatch: diff --git a/etl/month_end_automation_wave_3_layout.py b/etl/month_end_automation_wave_3_layout.py index 133a634..27006d6 100644 --- a/etl/month_end_automation_wave_3_layout.py +++ b/etl/month_end_automation_wave_3_layout.py @@ -27,15 +27,15 @@ class BoardID(Enum): board_ids = [ - "9349630181", # WCHG Walkups-Operations - "8830772914", # "L&Q London" - "9601691730", # Cardo Wales & West - Wave 3 - "9660895490", # Northumberland County SHDF Wave 3 - "9641491000", # Watford Warm Homes - "9671463094", # Seddon + # "9349630181", # WCHG Walkups-Operations + # "8830772914", # "L&Q London" + # "9601691730", # Cardo Wales & West - Wave 3 + # "9660895490", # Northumberland County SHDF Wave 3 + # "9641491000", # Watford Warm Homes + # "9671463094", # Seddon "9929454382", # NCHA SHDF Wave 3 Main Operation Board - "18232420839", # Powys County Council: - "18380751311", # Devon County Council + # "18232420839", # Powys County Council: + # "18380751311", # Devon County Council ] empty = "Rate card info missing" @@ -345,22 +345,22 @@ for board, all_records in board_to_record.items(): filtered_dfs.append(design) # Lodgement - lodg1 = get_df(df, "TM Phase 1", ["measures lodged"], "Lodgement") + lodg1 = get_df(df, "lodgement invoicing status", ["to invoice"], "Lodgement") if not lodg1.empty: filtered_dfs.append(lodg1) # Retrofit Evaluation - re = get_df(df, "Retrofit Evaluation Status", ["uploaded"], "retrofit evaluation") + re = get_df(df, "evaluation invoicing status", ["to invoice"], "retrofit evaluation") if not re.empty: filtered_dfs.append(re) # POST EPC - post_epc = get_df(df, "Post EPC/EPR Status", ["epc uploaded"], "POST EPC") + post_epc = get_df(df, "Post EPC invoicing status", ["epc to invoice"], "POST EPC") if not post_epc.empty: filtered_dfs.append(post_epc) # POST EPR - post_epr = get_df(df, "Post EPC/EPR Status", ["epr uploaded"], "POST EPR") + post_epr = get_df(df, "Post EPR invoicing status", ["epr to invoice"], "POST EPR") if not post_epr.empty: filtered_dfs.append(post_epr) diff --git a/etl/month_end_automation_wave_accent_housing.py b/etl/month_end_automation_wave_accent_housing.py index c3e97e4..42f02ba 100644 --- a/etl/month_end_automation_wave_accent_housing.py +++ b/etl/month_end_automation_wave_accent_housing.py @@ -148,21 +148,21 @@ if not rc.empty: filtered_dfs.append(rc) # Retrofit Evaluation -re = get_df(df, "Retrofit Evaluation Status", ["uploaded"], "re") +re = get_df(df, "Retrofit Evaluation Status", ["to invoice"], "re") if not re.empty: filtered_dfs.append(re) # POST EPC -post_epc = get_df(df, "Post EPC/EPR Status", ["epc uploaded"], "POST EPC") +post_epc = get_df(df, "Post EPC invoicing status", ["epc to invoice"], "POST EPC") if not post_epc.empty: filtered_dfs.append(post_epc) # POST EPR -post_epr = get_df(df, "POST EPC/EPR Status", ["epr uploaded"], "POST EPR") +post_epr = get_df(df, "Post EPR invoicing status", ["epr to invoice"], "POST EPR") if not post_epr.empty: filtered_dfs.append(post_epr) -lodg1 = get_df(df, "TM Phase 1", ["measures lodged"], "Lodgement") +lodg1 = get_df(df, "lodgement invoicing status", ["to invoice"], "Lodgement") if not lodg1.empty: filtered_dfs.append(lodg1)