From 3f59350ea34121f83288efa989062c11a4e10b11 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 16 Sep 2025 19:43:38 +0100 Subject: [PATCH 1/4] save --- deployment/lambda/walthamforest_etl/docker/app.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/deployment/lambda/walthamforest_etl/docker/app.py b/deployment/lambda/walthamforest_etl/docker/app.py index 535ddd0..1f9b570 100644 --- a/deployment/lambda/walthamforest_etl/docker/app.py +++ b/deployment/lambda/walthamforest_etl/docker/app.py @@ -108,6 +108,10 @@ def combine_records_for_flats(assets: dict, simple: list) -> dict: return assets +def get_energy_information(): + df = pd.read_excel("../../../../../home/Downloads/data.xlsx", sheet_name="") + # add uprn to everything + def handler(event, context): # read data for houses only assets = process_complex("Houses Asset Data") @@ -120,5 +124,7 @@ def handler(event, context): flats = combine_records_for_flats(assets, simple) - + + +# run a script that upload to s3 -> uprn -> jsonified -> walthamforest -> uri From d3cfd311de87d1e80c30ec9a153de775fd47b185 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 17 Sep 2025 10:03:47 +0000 Subject: [PATCH 2/4] added more jobs to accent housing --- ...onth_end_automation_wave_accent_housing.py | 35 ++++++++++++++++--- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/etl/month_end_automation_wave_accent_housing.py b/etl/month_end_automation_wave_accent_housing.py index 8e04c38..786c121 100644 --- a/etl/month_end_automation_wave_accent_housing.py +++ b/etl/month_end_automation_wave_accent_housing.py @@ -15,16 +15,16 @@ board_ids = [ ] empty = "Rate card info missing" -junte = "ask junte to update" +junte = "ask junte to update - He'll need to ask finance for ratecard and didn't change once they updated" rate_card_data_2502_accent_housing = { "job_type": [ - "First half of MTP", "Second half of MTP", "Full MTP", "Design Archetype Complex", + "First half of MTP", "Second half of MTP", "full cost mtp", "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Complex", "Design Repetitive Simple", - "Design Revision", "design type not specified", + "Design Revision", "design type not specified", "Coordination Stage 2", "POST EPC", "POST EPR", "POST ATT", "post EPC NO SHOW" ], "rate": [ - 150, 130, 280, junte, junte, junte, junte, junte, "please ask andreas" + 150, 130, 280, junte, junte, junte, junte, junte, "please ask andreas", junte, junte, junte, junte, junte ] } # ToDO @@ -133,6 +133,33 @@ for letter in revision_letter: if not design.empty: filtered_dfs.append(design) +# Coordination Stage 2 +rc = get_df(df, "rc stage 2", ["to invoice"], "Coordination Stage 2") +if not rc.empty: + filtered_dfs.append(rc) + +# POST EPC + Retrofit Evaluation +post_epc = get_df(df, "post epc & eval. invoicing status", ["epc to invoice"], "POST EPC") +if not post_epc.empty: + filtered_dfs.append(post_epc) + +# POST EPR +post_epr = get_df(df, "post epc & eval. invoicing status", ["epr to invoice"], "POST EPR") +if not post_epr.empty: + filtered_dfs.append(post_epr) + +# Post ATT +post_att = get_df(df, "post att invoicing status", ["to invoice"], "POST ATT") +if not post_att.empty: + filtered_dfs.append(post_epc) + +# Post visit no show +epc_ns = get_df(df, "post works no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "post EPC NO SHOW") +if not epc_ns.empty: + filtered_dfs.append(epc_ns) + + + final_df = pd.concat(filtered_dfs).reset_index(drop=True) From f86ffd67d4e2474737eeaede2d0f1c9a33f6544e Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 17 Sep 2025 12:42:18 +0000 Subject: [PATCH 3/4] save --- .github/workflows/months_end.yml | 27 +++++++------- etl/month_end_automation_wave_2_layout.py | 40 +++++++++++++++++++-- etl/month_end_automation_wave_2_no_11.py | 12 +++++-- etl/month_end_automation_wave_2_no_12.py | 10 ++++-- etl/month_end_automation_wave_2_no_14.py | 8 +++-- etl/month_end_automation_wave_2_no_15.py | 10 ++++-- etl/month_end_automation_wave_2_no_3.py | 38 ++++++++++++++++++-- etl/month_end_automation_wave_2_no_4.py | 44 +++++++++++++++++++++-- etl/month_end_automation_wave_2_no_7.py | 16 +++++++-- etl/month_end_automation_wave_2_no_8.py | 14 ++++++-- 10 files changed, 185 insertions(+), 34 deletions(-) diff --git a/.github/workflows/months_end.yml b/.github/workflows/months_end.yml index fb82aa1..e060615 100644 --- a/.github/workflows/months_end.yml +++ b/.github/workflows/months_end.yml @@ -28,18 +28,15 @@ jobs: run: | pwd ls -la - poetry run python etl/month_end_automation_wave_2_layout.py - poetry run python etl/month_end_automation_wave_2_no_3.py - poetry run python etl/month_end_automation_wave_2_no_4.py - poetry run python etl/month_end_automation_wave_2_no_6.py - poetry run python etl/month_end_automation_wave_2_no_7.py - poetry run python etl/month_end_automation_wave_2_no_8.py - # poetry run python etl/month_end_automation_wave_2_no_10.py - poetry run python etl/month_end_automation_wave_2_no_11.py - poetry run python etl/month_end_automation_wave_2_no_12.py - # poetry run python etl/month_end_automation_wave_2_no_13.py - poetry run python etl/month_end_automation_wave_2_no_14.py - poetry run python etl/month_end_automation_wave_2_no_15.py - # poetry run python etl/month_end_automation_wave_2_no_16.py - poetry run python etl/month_end_automation_wave_accent_housing.py - poetry run python etl/month_end_automation_wave_3_layout.py + poetry run python etl/month_end_automation_wave_2_layout.py # Done + poetry run python etl/month_end_automation_wave_2_no_3.py # Done + poetry run python etl/month_end_automation_wave_2_no_4.py # Done + poetry run python etl/month_end_automation_wave_2_no_6.py # Check with Matt if this can be deleted + poetry run python etl/month_end_automation_wave_2_no_7.py # Done + poetry run python etl/month_end_automation_wave_2_no_8.py # Done + poetry run python etl/month_end_automation_wave_2_no_11.py # Done + poetry run python etl/month_end_automation_wave_2_no_12.py # Done + poetry run python etl/month_end_automation_wave_2_no_14.py # Done + poetry run python etl/month_end_automation_wave_2_no_15.py # Done + poetry run python etl/month_end_automation_wave_accent_housing.py # Done + poetry run python etl/month_end_automation_wave_3_layout.py # Done diff --git a/etl/month_end_automation_wave_2_layout.py b/etl/month_end_automation_wave_2_layout.py index e6b4ff2..5a1296b 100644 --- a/etl/month_end_automation_wave_2_layout.py +++ b/etl/month_end_automation_wave_2_layout.py @@ -26,7 +26,7 @@ rate_card_data = { 207.65, 101, 186.4, 98, 98, 450, 150, 163, 135, 120, "60 - Needs to be verified (Post EPR)", 45, 90.5, 40, - 25, 25, 25, "check with Kevin" + "25 - check number on right", "25 - check number on right", "25 - check number on right", "check with Kevin" ] } @@ -93,6 +93,8 @@ ra = df[ df["ra"].str.lower().isin(["completed rdsap 10", "completed rdsap 9.9"]) ].copy() ra["job_type"] = "RA" +ra["evidence_record"] = None +ra["invoice_record"] = None filtered_dfs.append(ra) @@ -101,6 +103,8 @@ att = df[ df["att"].str.lower().isin(["completed"]) ].copy() att["job_type"] = "ATT" +att["evidence_record"] = None +att["invoice_record"] = None filtered_dfs.append(att) # V1 Coordination @@ -108,12 +112,16 @@ v1 = df[ df["v1 coordination status"].str.lower().isin(["rc complete"]) ].copy() v1["job_type"] = "Coordination Stage 1 v1" +v1["evidence_record"] = None +v1["invoice_record"] = None filtered_dfs.append(v1) # V2 Coordination _ = df[df["v2 invoiced"].fillna('').str.lower().isin(['to be invoiced'])] v2 = _[_["v2 dc/ima/pas"] > 0].copy() v2["job_type"] = "Coordination Stage 1 v2 remodel" +v2["evidence_record"] = None +v2["invoice_record"] = None filtered_dfs.append(v2) # V3 Coordination @@ -121,6 +129,8 @@ v3 = df[ df["v3 invoiced"].str.lower().isin(["to be invoiced"]) ].copy() v3["job_type"] = "Coordination Stage 1 v3 remodel" +v3["evidence_record"] = None +v3["invoice_record"] = None filtered_dfs.append(v3) # Coordination stage 2 Please complete @@ -128,6 +138,8 @@ cors2 = df[ df["rc stg. 2"].str.lower().isin(["to invoice"]) ].copy() cors2["job_type"] = "Coordination Stage 2" +cors2["evidence_record"] = None +cors2["invoice_record"] = None filtered_dfs.append(cors2) # Design type archietype @@ -135,6 +147,8 @@ design1 = df[ (df["design type for invoicing"].str.lower().isin(["archetype"])) & (df["design invoice status"].str.lower().isin(["to invoice"])) ].copy() design1["job_type"] = "Design Archetype" +design1["evidence_record"] = None +design1["invoice_record"] = None filtered_dfs.append(design1) # design type reptitive @@ -142,6 +156,8 @@ design1 = df[ (df["design type for invoicing"].str.lower().isin(["repetitive"])) & df["design invoice status"].str.lower().isin(["to invoice"]) ].copy() design1["job_type"] = "Design Repetitive" +design1["evidence_record"] = None +design1["invoice_record"] = None filtered_dfs.append(design1) # Design stage revisions @@ -149,6 +165,8 @@ design2 = df[ df["design revision invoice status"].str.lower().isin(["to invoice"]) ].copy() design2["job_type"] = "Design Revision" +design2["evidence_record"] = None +design2["invoice_record"] = None filtered_dfs.append(design2) # Lodgement Phase 1 @@ -156,6 +174,8 @@ lodg1 = df[ df["lodg. phase 1 invoice status"].str.lower().isin(["to invoice"]) ].copy() lodg1["job_type"] = "Lodgement phase 1" +lodg1["evidence_record"] = None +lodg1["invoice_record"] = None filtered_dfs.append(lodg1) # Full Lodgement Phase @@ -163,6 +183,8 @@ lodg2 = df[ df["full lodgement invoice status"].str.lower().isin(["to invoice"]) ].copy() lodg2["job_type"] = "Full Lodgement phase 2" +lodg2["evidence_record"] = None +lodg2["invoice_record"] = None filtered_dfs.append(lodg2) # POST EPC @@ -170,6 +192,8 @@ post_epc = df[ df["post-epc status"].str.lower().isin(["epc files uploaded"]) ].copy() post_epc["job_type"] = "Post EPC" +post_epc["evidence_record"] = None +post_epc["invoice_record"] = None filtered_dfs.append(post_epc) @@ -178,6 +202,8 @@ post_epr = df[ df["post-epc status"].str.lower().isin(["post epr completed"]) ].copy() post_epr["job_type"] = "Post EPR" +post_epr["evidence_record"] = None +post_epr["invoice_record"] = None filtered_dfs.append(post_epr) @@ -187,6 +213,8 @@ post_att = df[ df["post-att"].str.lower().isin(["post-att uploaded"]) ].copy() post_att["job_type"] = "POST ATT" +post_att["evidence_record"] = None +post_att["invoice_record"] = None filtered_dfs.append(post_att) @@ -195,6 +223,8 @@ retro = df[ df["retrofit evaluation"].str.lower().isin(["complete"]) ].copy() retro["job_type"] = "Retrofit Evaluation" +retro["evidence_record"] = None +retro["invoice_record"] = None filtered_dfs.append(retro) # RA NO Show @@ -203,6 +233,8 @@ ra_ns = df[ (df["ra no show evidence"] != 0) ].copy() ra_ns["job_type"] = "RA NO SHOW" +ra_ns["evidence_record"] = ra_ns["ra no show evidence"] +ra_ns["invoice_record"] = ra_ns["ra no show invoice"] filtered_dfs.append(ra_ns) @@ -213,6 +245,8 @@ att_ns = df[ (df["att no show evidence"] != 0) ].copy() att_ns["job_type"] = "ATT NO SHOW" +att_ns["evidence_record"] = att_ns["att no show evidence"] +att_ns["invoice_record"] = att_ns["att no show invoice"] filtered_dfs.append(att_ns) @@ -222,6 +256,8 @@ epc_ns = df[ (df["epc no show evidence"] != 0) ].copy() epc_ns["job_type"] = "post EPC NO SHOW" +epc_ns["evidence_record"] = epc_ns["epc no show evidence"] +epc_ns["invoice_record"] = epc_ns["epc no show invoice"] filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) @@ -236,7 +272,7 @@ import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') # Upload to sharepoint -attribute = ['address', 'client', 'job_type', 'rate'] +attribute = ['address', 'client', 'job_type', 'rate', "evidence_record", "invoice_record"] master_folder_name = "NHCF SHDF WESTVILLE WAVE 1 & 2" file_name = f"NCHA SHDF Westville Wave 1 & 2_{timestamp}.xlsx" combined_with_rates[attribute].to_excel(file_name, index=False) diff --git a/etl/month_end_automation_wave_2_no_11.py b/etl/month_end_automation_wave_2_no_11.py index e9d2349..a5a7f0a 100644 --- a/etl/month_end_automation_wave_2_no_11.py +++ b/etl/month_end_automation_wave_2_no_11.py @@ -25,7 +25,7 @@ rate_card_data = { 225, 92.65, check_wall_type, check_wall_type, check_wall_type, check_wall_type, check_wall_type, 165, 135, 120, "60 - check", 48.6, 85.55, 49.6, - 125, 125, 125 + "125 - check with right", "125 - check with right", "125 - check with right" ] } @@ -91,6 +91,8 @@ def get_df(df, column_name, success_critera, job_name=None): df[column_name].str.lower().isin(success_critera) ].copy() _["job_type"] = job_name + _["evidence_record"] = None + _["invoice_record"] = None return _ @@ -122,6 +124,8 @@ cors2 = df[ df["rc stg. 2"].str.lower().isin(["to invoice"]) ] cors2["job_type"] = "Coordination Stage 2" +cors2["evidence_record"] = None +cors2["invoice_record"] = None filtered_dfs.append(cors2) # Design stage 1 @@ -156,6 +160,8 @@ post_epr = df[ df["post-epc status"].str.lower().isin(["post epr completed", "Post epr desktop based completed"]) ].copy() post_epr["job_type"] = "POST epr" +post_epr["evidence_record"] = None +post_epr["invoice_record"] = None filtered_dfs.append(post_epr) # Post ATT @@ -180,6 +186,8 @@ att_ns = df[ (df["att no show evidence"] != 0) ].copy() att_ns["job_type"] = "ATT NO SHOW" +att_ns["evidence_record"] = att_ns["att no show evidence"] +att_ns["invoice_record"] = att_ns["att no show invoice"] filtered_dfs.append(att_ns) @@ -201,7 +209,7 @@ import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') # Upload to sharepoint -attribute = ['address', 'client', 'job_type', 'rate'] +attribute = ['address', 'client', 'job_type', 'rate', "evidence_record", "invoice_record"] master_folder_name = "Northumberland LAD2 & HUG2" file_name = f"{master_folder_name}_{timestamp}.xlsx" combined_with_rates[attribute].to_excel(file_name, index=False) diff --git a/etl/month_end_automation_wave_2_no_12.py b/etl/month_end_automation_wave_2_no_12.py index aa7bd7e..e8c5462 100644 --- a/etl/month_end_automation_wave_2_no_12.py +++ b/etl/month_end_automation_wave_2_no_12.py @@ -26,7 +26,7 @@ rate_card_data = { "(185) - Kevin Check as this depends on property size", 115, 200, 125, 125, empty, "(135)- Mariane said 'Remaining RC & design sign off", 185, 135, 120, "(60) Post EPR, please verify", 65, 115, 60, - 50, 50, 50, "(60) - Please add price for design sign off, check with kevin and marianne" + "50 - check with right", "50- check with right", "50- check with right", "(60) - Please add price for design sign off, check with kevin and marianne" ] } @@ -157,6 +157,8 @@ post_epr = df[ df["post-epc epr status"].str.lower().isin(["post epr completed"]) ].copy() post_epr["job_type"] = "POST epr" +post_epr["evidence_record"] = post_epr["ra no show evidence"] +post_epr["invoice_record"] = post_epr["ra no show invoice"] filtered_dfs.append(post_epr) # Post ATT @@ -173,6 +175,8 @@ ra_ns = df[ (df["ra no show evidence"] != 0) ].copy() ra_ns["job_type"] = "RA NO SHOW" +ra_ns["evidence_record"] = ra_ns["ra no show evidence"] +ra_ns["invoice_record"] = ra_ns["ra no show invoice"] filtered_dfs.append(ra_ns) @@ -190,6 +194,8 @@ epc_ns = df[ (df["epc no show evidence"] != 0) ].copy() epc_ns["job_type"] = "post EPC NO SHOW" +epc_ns["evidence_record"] = epc_ns["epc no show evidence"] +epc_ns["invoice_record"] = epc_ns["epc no show invoice"] filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) @@ -203,7 +209,7 @@ import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') # Upload to sharepoint -attribute = ['address', 'client', 'job_type', 'rate'] +attribute = ['address', 'client', 'job_type', 'rate', "evidence_record", "invoice_record"] master_folder_name = "Shropshire Council HUG 2" file_name = f"{master_folder_name}_{timestamp}.xlsx" combined_with_rates[attribute].to_excel(file_name, index=False) diff --git a/etl/month_end_automation_wave_2_no_14.py b/etl/month_end_automation_wave_2_no_14.py index e436aec..150dcdd 100644 --- a/etl/month_end_automation_wave_2_no_14.py +++ b/etl/month_end_automation_wave_2_no_14.py @@ -24,7 +24,7 @@ rate_card_data_example = { "rate": [ 259, 125, 280, 125, 135, "60 to check", 85, 125, - 60, 45, 45, 45, + 60, "45 - check with right", "45 - check with right", "45 - check with right", ] } @@ -174,6 +174,8 @@ ra_ns = df[ (df["ra no show evidence"] != 0) ].copy() ra_ns["job_type"] = "RA NO SHOW" +ra_ns["evidence_record"] = ra_ns["ra no show evidence"] +ra_ns["invoice_record"] = ra_ns["ra no show invoice"] filtered_dfs.append(ra_ns) @@ -191,6 +193,8 @@ epc_ns = df[ (df["post epc no show evidence"] != 0) ].copy() epc_ns["job_type"] = "post EPC NO SHOW" +epc_ns["evidence_record"] = epc_ns["post epc no show evidence"] +epc_ns["invoice_record"] = epc_ns["post epc no show invoice"] filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) @@ -205,7 +209,7 @@ import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') # Upload to sharepoint -attribute = ['address', 'client', 'job_type', 'rate'] +attribute = ['address', 'client', 'job_type', 'rate', "evidence_record", "invoice_record"] master_folder_name = "Decent Homes Stonewater - Operations" file_name = f"{master_folder_name}_{timestamp}.xlsx" combined_with_rates[attribute].to_excel(file_name, index=False) diff --git a/etl/month_end_automation_wave_2_no_15.py b/etl/month_end_automation_wave_2_no_15.py index a175801..e5e5d55 100644 --- a/etl/month_end_automation_wave_2_no_15.py +++ b/etl/month_end_automation_wave_2_no_15.py @@ -27,7 +27,7 @@ rate_card_data = { "259 (new RA rate for PAS2035:2023 - old rates for other works - to discuss with KN)", 40, 178.5, empty, empty, empty, 180, 275, 135, 120, "60 - please verify with Marianne", 45, 45, 40, - 25, 25, 25, "Please check price for design revision with Andreas" + "25 - check with right", "25 - check with right", "25 - check with right", "Please check price for design revision with Andreas" ] } @@ -182,6 +182,8 @@ ra_ns = df[ (df["ra no show evidence"] != 0) ].copy() ra_ns["job_type"] = "RA NO SHOW" +ra_ns["evidence_record"] = ra_ns["ra no show evidence"] +ra_ns["invoice_record"] = ra_ns["ra no show invoice"] filtered_dfs.append(ra_ns) @@ -191,6 +193,8 @@ att_ns = df[ (df["att no show evidence"] != 0) ].copy() att_ns["job_type"] = "ATT NO SHOW" +att_ns["evidence_record"] = att_ns["att no show evidence"] +att_ns["invoice_record"] = att_ns["att no show invoice"] filtered_dfs.append(att_ns) @@ -200,6 +204,8 @@ epc_ns = df[ (df["epc no show evidence"] != 0) ].copy() epc_ns["job_type"] = "post EPC NO SHOW" +epc_ns["evidence_record"] = epc_ns["epc no show evidence"] +epc_ns["invoice_record"] = epc_ns["epc no show invoice"] filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) @@ -213,7 +219,7 @@ import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') # Upload to sharepoint -attribute = ['address', 'client', 'job_type', 'rate'] +attribute = ['address', 'client', 'job_type', 'rate', "evidence_record", "invoice_record"] master_folder_name = "WCHG SHDF 2.1 Mansard" file_name = f"{master_folder_name}_{timestamp}.xlsx" combined_with_rates[attribute].to_excel(file_name, index=False) diff --git a/etl/month_end_automation_wave_2_no_3.py b/etl/month_end_automation_wave_2_no_3.py index e8a46de..b3b82a4 100644 --- a/etl/month_end_automation_wave_2_no_3.py +++ b/etl/month_end_automation_wave_2_no_3.py @@ -24,7 +24,7 @@ rate_card_data = { 259, 101, 210, 95, 95, 450, 150, 195, 135, 120, "(60)) - please confirm with Marianne, EPR", 45, 90.5, 42.4, - 25, 25, 25, "Please ask for Design Revision" + "25 - check on right", "25 - check on right", "25 - check on right", "Please ask for Design Revision" ] } @@ -91,6 +91,8 @@ ra = df[ df["ra"].str.lower().isin(["completed rdsap 10", "completed rdsap 9.9", "completed", "complete"]) ].copy() ra["job_type"] = "RA" +ra["evidence_record"] = None +ra["invoice_record"] = None filtered_dfs.append(ra) @@ -99,6 +101,8 @@ att = df[ df["att"].str.lower().isin(["completed"]) ].copy() att["job_type"] = "ATT" +att["evidence_record"] = None +att["invoice_record"] = None filtered_dfs.append(att) # V1 Coordination @@ -106,11 +110,15 @@ v1 = df[ df["coordination status"].str.lower().isin(["ima/mtp completed"]) ].copy() v1["job_type"] = "Coordination Stage 1 v1" +v1["evidence_record"] = None +v1["invoice_record"] = None filtered_dfs.append(v1) # V2 Coordination _ = df[df["v2 mtp status"].fillna('').str.lower().isin(['v2 ima-mtp completed', 'v2 completed'])].copy() _["job_type"] = "Coordination Stage 1 v2 remodel" +_["evidence_record"] = None +_["invoice_record"] = None filtered_dfs.append(_) # V3 Coordination @@ -125,6 +133,8 @@ cors2 = df[ df["rc stg. 2"].str.lower().isin(["to invoice"]) ] cors2["job_type"] = "Coordination Stage 2" +cors2["evidence_record"] = None +cors2["invoice_record"] = None filtered_dfs.append(cors2) # Design Archetype @@ -133,6 +143,8 @@ design1 = df[ ].copy() design1 = design1[design1["design type for invoicing"].str.lower().isin(['archetype'])].copy() design1["job_type"] = "Design Archetype" +design1["evidence_record"] = None +design1["invoice_record"] = None filtered_dfs.append(design1) # Design Repetitive @@ -141,6 +153,8 @@ design1 = df[ ].copy() design1 = design1[design1["design type for invoicing"].str.lower().isin(['repetitive'])].copy() design1["job_type"] = "Design repetitive" +design1["evidence_record"] = None +design1["invoice_record"] = None filtered_dfs.append(design1) # Design Revision @@ -148,6 +162,8 @@ design_revision = df[ df["design revision invoice status"].str.lower().isin(["to invoice"]) ].copy() design_revision["job_type"] = "Design repetitive" +design_revision["evidence_record"] = None +design_revision["invoice_record"] = None filtered_dfs.append(design_revision) # Lodgement Phase 1 @@ -155,6 +171,8 @@ lodg1 = df[ df["phase 1 invoice status (lodgement)"].str.lower().isin(["done"]) ].copy() lodg1["job_type"] = "Lodgement Phase 1" +lodg1["evidence_record"] = None +lodg1["invoice_record"] = None filtered_dfs.append(lodg1) # Full Lodgement Phase @@ -162,6 +180,8 @@ lodg2 = df[ df["lodgement invoice status (lodgement)"].str.lower().isin(["to invoice"]) ].copy() lodg2["job_type"] = "Full lodgement phase 2" +lodg2["evidence_record"] = None +lodg2["invoice_record"] = None filtered_dfs.append(lodg2) # POST EPC @@ -169,6 +189,8 @@ post_epc = df[ df["post epc"].str.lower().isin(["success", "pics uploaded"]) ].copy() post_epc["job_type"] = "POST EPC" +post_epc["evidence_record"] = None +post_epc["invoice_record"] = None filtered_dfs.append(post_epc) @@ -177,6 +199,8 @@ post_epr = df[ df["post epc"].str.lower().isin(["post epr completed"]) ].copy() post_epr["job_type"] = "POST EPR" +post_epr["evidence_record"] = None +post_epr["invoice_record"] = None filtered_dfs.append(post_epr) @@ -185,6 +209,8 @@ post_att = df[ df["post att"].str.lower().isin(["uploaded"]) ].copy() post_att["job_type"] = "POST ATT" +post_att["evidence_record"] = None +post_att["invoice_record"] = None filtered_dfs.append(post_att) @@ -193,6 +219,8 @@ retro = df[ df["retrofit evaluation"].str.lower().isin(["uploaded", "completed", "to invoice"]) ].copy() retro["job_type"] = "Retrofit Evaluation" +retro["evidence_record"] = None +retro["invoice_record"] = None filtered_dfs.append(retro) # RA NO Show @@ -201,6 +229,8 @@ ra_ns = df[ (df["ra no show evidence"] != 0) ].copy() ra_ns["job_type"] = "RA NO SHOW" +ra_ns["evidence_record"] = ra_ns["ra no show evidence"] +ra_ns["invoice_record"] = ra_ns["ra no show invoice"] filtered_dfs.append(ra_ns) @@ -210,6 +240,8 @@ att_ns = df[ (df["att no show evidence"] != 0) ].copy() att_ns["job_type"] = "ATT NO SHOW" +att_ns["evidence_record"] = att_ns["att no show evidence"] +att_ns["invoice_record"] = att_ns["att no show invoice"] filtered_dfs.append(att_ns) @@ -219,6 +251,8 @@ epc_ns = df[ (df["post epc no show evidence"] != 0) ].copy() epc_ns["job_type"] = "Post EPC NO SHOW" +epc_ns["evidence_record"] = epc_ns["post epc no show evidence"] +epc_ns["invoice_record"] = epc_ns["post epc no show invoice"] filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) @@ -232,7 +266,7 @@ import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') # Upload to sharepoint -attribute = ['address', 'client', 'job_type', 'rate'] +attribute = ['address', 'client', 'job_type', 'rate', "evidence_record", "invoice_record"] master_folder_name = "Platform Housing W2 (in use)" file_name = f"{master_folder_name}_{timestamp}.xlsx" combined_with_rates[attribute].to_excel(file_name, index=False) diff --git a/etl/month_end_automation_wave_2_no_4.py b/etl/month_end_automation_wave_2_no_4.py index 787aede..e0355b6 100644 --- a/etl/month_end_automation_wave_2_no_4.py +++ b/etl/month_end_automation_wave_2_no_4.py @@ -24,7 +24,7 @@ rate_card_data = { 165.75, 72.25, 174.25, 174.25, 174.25, 175, 175, 124.25, 135, 120, "(60) - please check with Marianne", 45, 63.75, 34, - 25, 25, 25, "Please ask marianne or kev for design revision" + "25 - check number on right", "25 - check number on right", "25- check with number on right", "Please ask marianne or kev for design revision" ] } @@ -90,6 +90,8 @@ ra = df[ df["ra"].str.lower().isin(["completed rdsap 10", "completed rdsap 9.9"]) ].copy() ra["job_type"] = "RA" +ra["evidence_record"] = None +ra["invoice_record"] = None filtered_dfs.append(ra) @@ -98,6 +100,8 @@ att = df[ df["att"].str.lower().isin(["completed"]) ].copy() att["job_type"] = "ATT" +att["evidence_record"] = None +att["invoice_record"] = None filtered_dfs.append(att) # V1 Coordination @@ -105,26 +109,36 @@ v1 = df[ df["v1 coordination status (ioe,mtp)"].str.lower().isin(["rc complete"]) ].copy() v1["job_type"] = "Coordination Stage 1 v1" +v1["evidence_record"] = None +v1["invoice_record"] = None filtered_dfs.append(v1) # V2 Coordination _ = df[df["mtp v2 status"].str.lower().isin(['rc v2 complete'])].copy() _["job_type"] = "Coordination Stage 1 v2 remodel" +_["evidence_record"] = None +_["invoice_record"] = None filtered_dfs.append(_) # V2 Coordination _ = df[df["mtp v2 invoiced"].str.lower().isin(['needs to be invoiced'])].copy() _["job_type"] = "Coordination Stage 1 v2 remodel" +_["evidence_record"] = None +_["invoice_record"] = None filtered_dfs.append(_) # V3 Coordination v3 = df[df["v3 rc status"].str.lower().isin(['uploaded'])].copy() v3["job_type"] = "Coordination Stage 1 v3 remodel" +v3["evidence_record"] = None +v3["invoice_record"] = None filtered_dfs.append(_) # V3 Coordination v3 = df[df["v3 invoice status"].str.lower().isin(['to be invoiced'])].copy() v3["job_type"] = "Coordination Stage 1 v3 remodel" +v3["evidence_record"] = None +v3["invoice_record"] = None filtered_dfs.append(_) # Coordination stage 2 Please complete @@ -132,6 +146,8 @@ cors2 = df[ df["rc stg. 2 status"].str.lower().isin(["to invoice", "completed"]) ] cors2["job_type"] = "Coordination Stage 2" +cors2["evidence_record"] = None +cors2["invoice_record"] = None filtered_dfs.append(cors2) # Design Archetype @@ -140,6 +156,8 @@ design1 = df[ ].copy() design1 = design1[design1["design type"].str.lower().isin(["archetype"])].copy() design1["job_type"] = "Design Archetype" +design1["evidence_record"] = None +design1["invoice_record"] = None filtered_dfs.append(design1) # Design Repetitive @@ -148,6 +166,8 @@ design1 = df[ ].copy() design1 = design1[design1["design type"].str.lower().isin(["repetitive"])].copy() design1["job_type"] = "Design Repetitive" +design1["evidence_record"] = None +design1["invoice_record"] = None filtered_dfs.append(design1) # Design Revision @@ -155,6 +175,8 @@ design_revision = df[ df["design revision invoice status"].str.lower().isin(["to invoice"]) ].copy() design_revision["job_type"] = "Design repetitive" +design_revision["evidence_record"] = None +design_revision["invoice_record"] = None filtered_dfs.append(design_revision) # Lodgement Phase 1 @@ -162,6 +184,8 @@ lodg1 = df[ df["phase 1 invoice status (lodgement)"].str.lower().isin(["done", "to be invoiced"]) ].copy() lodg1["job_type"] = "Lodgement Phase 1" +lodg1["evidence_record"] = None +lodg1["invoice_record"] = None filtered_dfs.append(lodg1) # Full Lodgement Phase @@ -169,6 +193,8 @@ _ = df[ df["lodgement invoice status"].str.lower().isin(["to invoice"]) ].copy() _["job_type"] = "Full lodgement phase 2" +_["evidence_record"] = None +_["invoice_record"] = None filtered_dfs.append(_) # POST EPC @@ -176,6 +202,8 @@ post_epc = df[ df["post epc"].str.lower().isin(["completed & uploaded"]) ].copy() post_epc["job_type"] = "POST EPC" +post_epc["evidence_record"] = None +post_epc["invoice_record"] = None filtered_dfs.append(post_epc) @@ -184,6 +212,8 @@ post_epr = df[ df["post epc"].str.lower().isin(["post epr completed"]) ].copy() post_epr["job_type"] = "POST EPR" +post_epr["evidence_record"] = None +post_epr["invoice_record"] = None filtered_dfs.append(post_epr) # Post ATT @@ -191,6 +221,8 @@ post_att = df[ df["post att"].str.lower().isin(["completed & uploaded"]) ].copy() post_att["job_type"] = "POST ATT" +post_att["evidence_record"] = None +post_att["invoice_record"] = None filtered_dfs.append(post_att) @@ -199,6 +231,8 @@ retro = df[ df["retrofit evaluation"].str.lower().isin(["completed & uploaded"]) ].copy() retro["job_type"] = "Retrofit Evaluation" +retro["evidence_record"] = None +retro["invoice_record"] = None filtered_dfs.append(retro) # RA NO Show @@ -207,6 +241,8 @@ ra_ns = df[ (df["ra no show evidence"] != 0) ].copy() ra_ns["job_type"] = "RA NO SHOW" +ra_ns["evidence_record"] = ra_ns["ra no show evidence"] +ra_ns["invoice_record"] = ra_ns["ra no show invoice"] filtered_dfs.append(ra_ns) @@ -216,6 +252,8 @@ att_ns = df[ (df["att no show evidence"] != 0) ].copy() att_ns["job_type"] = "ATT NO SHOW" +att_ns["evidence_record"] = att_ns["att no show evidence"] +att_ns["invoice_record"] = att_ns["att no show invoice"] filtered_dfs.append(att_ns) @@ -225,6 +263,8 @@ epc_ns = df[ (df["post epc no show evidence"] != 0) ].copy() epc_ns["job_type"] = "post EPC NO SHOW" +epc_ns["evidence_record"] = epc_ns["post epc no show evidence"] +epc_ns["invoice_record"] = epc_ns["post epc no show invoice"] filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) @@ -240,7 +280,7 @@ timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') # Upload to sharepoint -attribute = ['address', 'client', 'job_type', 'rate'] +attribute = ['address', 'client', 'job_type', 'rate', "evidence_record", "invoice_record"] master_folder_name = "Stonewater - (in use)" file_name = f"{master_folder_name}_{timestamp}.xlsx" combined_with_rates[attribute].to_excel(file_name, index=False) diff --git a/etl/month_end_automation_wave_2_no_7.py b/etl/month_end_automation_wave_2_no_7.py index 5cdf8f0..023ee94 100644 --- a/etl/month_end_automation_wave_2_no_7.py +++ b/etl/month_end_automation_wave_2_no_7.py @@ -24,7 +24,7 @@ rate_card_data_sp_plus = { 170, 70, 200, "check with Kevin", "check with Kevin", 470, 155, 165, 135, 120, "60 but check with Kevin as EPR", 45, 70, 40, - 30, 30, 30 + "30 - check number on right", "30 - check number on right", "30 - check number on right" ] } @@ -40,7 +40,7 @@ rate_card_data_net_zero = { 170, 70, 200, emp_msg, emp_msg, 325, 140, 165, 135, 120, "60 but check with Kevin as EPR", 45, 70, 40, - 30, 30, 30 + "30 - check number on right", "30- check number on right", "30 - check number on right" ] } @@ -126,6 +126,9 @@ def get_df(df, column_name, success_critera, job_name=None): ].copy() if job_name: _["job_type"] = job_name + + _["evidence_record"] = None + _["invoice_record"] = None return _ @@ -207,6 +210,8 @@ ra_ns = df[ (df["ra no show evidence"] != 0) ].copy() ra_ns["job_type"] = "RA NO SHOW" +ra_ns["evidence_record"] = ra_ns["epc no show evidence"] +ra_ns["invoice_record"] = ra_ns["epc no show invoice"] filtered_dfs.append(ra_ns) @@ -216,15 +221,20 @@ att_ns = df[ (df["att no show evidence"] != 0 ) ].copy() att_ns["job_type"] = "ATT NO SHOW" +att_ns["evidence_record"] = att_ns["att no show evidence"] +att_ns["invoice_record"] = att_ns["att no show invoice"] filtered_dfs.append(att_ns) + # Post visit no show epc_ns = df[ (df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999)) & (df["epc no show evidence"] != 0 ) ].copy() epc_ns["job_type"] = "post EPC NO SHOW" +epc_ns["evidence_record"] = epc_ns["epc no show evidence"] +epc_ns["invoice_record"] = epc_ns["epc no show invoice"] filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) @@ -247,7 +257,7 @@ import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') # Upload to sharepoint -attribute = ['address', 'client', 'job_type', 'rate'] +attribute = ['address', 'client', 'job_type', 'rate', "evidence_record", "invoice_record"] combined_with_rates_sp_plus[attribute].to_excel(f'HomeGroup Wave 2SP+_{timestamp}.xlsx', index=False) combined_with_rates_net_zero_df[attribute].to_excel(f'HomeGroup Wave NetZero_{timestamp}.xlsx', index=False) combined_with_rates_other_from_home_group[attribute].to_excel(f'HomeGroup Wave Unsure_who_to_bill_{timestamp}.xlsx', index=False) diff --git a/etl/month_end_automation_wave_2_no_8.py b/etl/month_end_automation_wave_2_no_8.py index 7f2ab25..1d8d619 100644 --- a/etl/month_end_automation_wave_2_no_8.py +++ b/etl/month_end_automation_wave_2_no_8.py @@ -24,7 +24,7 @@ rate_card_data = { 259, 125, 280, 125, 125, 650, 415, 195, 175, 135, 120, "60 - Double check with Kevin/Marianne EPR", 85, 125, 60, - 45, 45, 45, "Design Revision check with Kevin/Marianne" + "45- check with right", "45 - check with right", "45 - check with right", "Design Revision check with Kevin/Marianne" ] } @@ -92,6 +92,8 @@ def get_df(df, column_name, success_critera, job_name=None): ].copy() if job_name: _["job_type"] = job_name + _["evidence_record"] = None + _["invoice_record"] = None return _ @@ -168,6 +170,8 @@ post_epr = df[ df["post epc status"].str.lower().isin(["post epr completed"]) ].copy() post_epr["job_type"] = "POST epr" +post_epr["evidence_record"] = None +post_epr["invoice_record"] = None filtered_dfs.append(post_epr) # Post ATT @@ -185,6 +189,8 @@ ra_ns = df[ (df["ra no show evidence"] != 0 ) ].copy() ra_ns["job_type"] = "RA NO SHOW" +ra_ns["evidence_record"] = ra_ns["ra no show evidence"] +ra_ns["invoice_record"] = ra_ns["ra no show invoice"] filtered_dfs.append(ra_ns) @@ -194,6 +200,8 @@ att_ns = df[ (df["att no show evidence"] !=0 ) ].copy() att_ns["job_type"] = "ATT NO SHOW" +att_ns["evidence_record"] = att_ns["att no show evidence"] +att_ns["invoice_record"] = att_ns["att no show invoice"] filtered_dfs.append(att_ns) @@ -202,6 +210,8 @@ epc_ns = df[ df["post works no show evidence"].fillna(-9999) != df["post works no show invoice"].fillna(-9999) ].copy() epc_ns["job_type"] = "post EPC NO SHOW" +epc_ns["evidence_record"] = epc_ns["post works no show evidence"] +epc_ns["invoice_record"] = epc_ns["post works no show invoice"] filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) @@ -218,7 +228,7 @@ timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') # Upload to sharepoint -attribute = ['address', 'client', 'job_type', 'rate'] +attribute = ['address', 'client', 'job_type', 'rate', "evidence_record", "invoice_record"] master_folder_name = "NCHA SHDF 2.1 SBS" file_name = f"{master_folder_name}_{timestamp}.xlsx" combined_with_rates[attribute].to_excel(file_name, index=False) From 232509ef16c5dc507b046ad8b0ea13fbb308d574 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 17 Sep 2025 13:01:29 +0000 Subject: [PATCH 4/4] finish month end --- etl/month_end_automation_wave_2_layout.py | 16 ++++++---- etl/month_end_automation_wave_2_no_8.py | 39 +++++++++++++++-------- 2 files changed, 36 insertions(+), 19 deletions(-) diff --git a/etl/month_end_automation_wave_2_layout.py b/etl/month_end_automation_wave_2_layout.py index 5a1296b..1fb6dd1 100644 --- a/etl/month_end_automation_wave_2_layout.py +++ b/etl/month_end_automation_wave_2_layout.py @@ -20,13 +20,13 @@ rate_card_data = { "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", - "RA no show", "ATT no show", "post EPC no show", "Design Revision" + "RA no show", "ATT no show", "post EPC no show", "Design Revision", "Design Specify" ], "rate": [ 207.65, 101, 186.4, 98, 98, 450, 150, 163, 135, 120, "60 - Needs to be verified (Post EPR)", 45, 90.5, 40, - "25 - check number on right", "25 - check number on right", "25 - check number on right", "check with Kevin" + "25 - check number on right", "25 - check number on right", "25 - check number on right", "check with Kevin", "ask adreas" ] } @@ -147,8 +147,6 @@ design1 = df[ (df["design type for invoicing"].str.lower().isin(["archetype"])) & (df["design invoice status"].str.lower().isin(["to invoice"])) ].copy() design1["job_type"] = "Design Archetype" -design1["evidence_record"] = None -design1["invoice_record"] = None filtered_dfs.append(design1) # design type reptitive @@ -156,10 +154,16 @@ design1 = df[ (df["design type for invoicing"].str.lower().isin(["repetitive"])) & df["design invoice status"].str.lower().isin(["to invoice"]) ].copy() design1["job_type"] = "Design Repetitive" -design1["evidence_record"] = None -design1["invoice_record"] = None filtered_dfs.append(design1) +design_other = df[ + (~df["design type for invoicing"].str.lower().isin(["archetype", "repetitive"])) + & (df["design invoice status"].str.lower().isin(["to invoice"])) +].copy() + +design_other["job_type"] = "Design Specify" +filtered_dfs.append(design_other) + # Design stage revisions design2 = df[ df["design revision invoice status"].str.lower().isin(["to invoice"]) diff --git a/etl/month_end_automation_wave_2_no_8.py b/etl/month_end_automation_wave_2_no_8.py index 1d8d619..7c2dd56 100644 --- a/etl/month_end_automation_wave_2_no_8.py +++ b/etl/month_end_automation_wave_2_no_8.py @@ -18,13 +18,13 @@ rate_card_data = { "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", - "RA no show", "ATT no show", "post EPC no show", "Design Revision" + "RA no show", "ATT no show", "post EPC no show", "Design Revision", "Design Repetitive Complex", "design type not specified" ], "rate": [ 259, 125, 280, 125, 125, 650, 415, 195, 175, 135, 120, "60 - Double check with Kevin/Marianne EPR", 85, 125, 60, - "45- check with right", "45 - check with right", "45 - check with right", "Design Revision check with Kevin/Marianne" + "45- check with right", "45 - check with right", "45 - check with right", "Design Revision check with Kevin/Marianne", "ask junte to add", "ask andreas" ] } @@ -125,22 +125,35 @@ cors2 = get_df(df, "rc stg. 2", ["to invoice"], "Coordination Stage 2") filtered_dfs.append(cors2) # Design Archtype Complex -design1 = get_df(df, "rd invoiced", ["to invoice"]) -design1 = get_df(design1, "design type", ["archetype (complex)"]) +design = get_df(df, "rd invoiced", ["to invoice"]) +design1 = get_df(design, "design type", ["archetype (complex)"]) design1 = get_df(design1, "design upload to sharepoint", ["done"], "Design Archetype Complex") -filtered_dfs.append(design1) +if not design1.empty: + filtered_dfs.append(design1) # Design Archtype Simple -design1 = get_df(df, "rd invoiced", ["to invoice"]) -design1 = get_df(design1, "design type", ["archetype (simple)"]) -design1 = get_df(design1, "design upload to sharepoint", ["done"], "Design Archetype Simple") -filtered_dfs.append(design1) +design2 = get_df(design, "design type", ["archetype (simple)"]) +design2 = get_df(design2, "design upload to sharepoint", ["done"], "Design Archetype Simple") +if not design2.empty: + filtered_dfs.append(design2) # Design Repitive Simple -design1 = get_df(df, "rd invoiced", ["to invoice"]) -design1 = get_df(design1, "design type", ["Design Repetitive Simple"]) -design1 = get_df(design1, "design upload to sharepoint", ["done"], "Design Repetitive Simple") -filtered_dfs.append(design1) +design3 = get_df(design, "design type", ["repetitive (simple)"]) +design3 = get_df(design3, "design upload to sharepoint", ["done"], "Design Repetitive Simple") +if not design3.empty: + filtered_dfs.append(design3) + +# Design Reptitive Complex +design4 = get_df(design, "design type", ["repetitive (complex)"]) +design4 = get_df(design4, "design upload to sharepoint", ["done"], "Design Repetitive Complex") +if not design4.empty: + filtered_dfs.append(design4) + +all_filtered = pd.concat([df for df in (design1, design2, design3, design4) if not df.empty]) +design_remaining = design.loc[~design.index.isin(all_filtered.index)] +if not design_remaining.empty: + design_remaining["job_type"] = "design type not specified" + filtered_dfs.append(design_remaining) # Design revision design2 = get_df(df, "design revision invoice status", [