diff --git a/.github/workflows/months_end.yml b/.github/workflows/months_end.yml index fb82aa1..e060615 100644 --- a/.github/workflows/months_end.yml +++ b/.github/workflows/months_end.yml @@ -28,18 +28,15 @@ jobs: run: | pwd ls -la - poetry run python etl/month_end_automation_wave_2_layout.py - poetry run python etl/month_end_automation_wave_2_no_3.py - poetry run python etl/month_end_automation_wave_2_no_4.py - poetry run python etl/month_end_automation_wave_2_no_6.py - poetry run python etl/month_end_automation_wave_2_no_7.py - poetry run python etl/month_end_automation_wave_2_no_8.py - # poetry run python etl/month_end_automation_wave_2_no_10.py - poetry run python etl/month_end_automation_wave_2_no_11.py - poetry run python etl/month_end_automation_wave_2_no_12.py - # poetry run python etl/month_end_automation_wave_2_no_13.py - poetry run python etl/month_end_automation_wave_2_no_14.py - poetry run python etl/month_end_automation_wave_2_no_15.py - # poetry run python etl/month_end_automation_wave_2_no_16.py - poetry run python etl/month_end_automation_wave_accent_housing.py - poetry run python etl/month_end_automation_wave_3_layout.py + poetry run python etl/month_end_automation_wave_2_layout.py # Done + poetry run python etl/month_end_automation_wave_2_no_3.py # Done + poetry run python etl/month_end_automation_wave_2_no_4.py # Done + poetry run python etl/month_end_automation_wave_2_no_6.py # Check with Matt if this can be deleted + poetry run python etl/month_end_automation_wave_2_no_7.py # Done + poetry run python etl/month_end_automation_wave_2_no_8.py # Done + poetry run python etl/month_end_automation_wave_2_no_11.py # Done + poetry run python etl/month_end_automation_wave_2_no_12.py # Done + poetry run python etl/month_end_automation_wave_2_no_14.py # Done + poetry run python etl/month_end_automation_wave_2_no_15.py # Done + poetry run python etl/month_end_automation_wave_accent_housing.py # Done + poetry run python etl/month_end_automation_wave_3_layout.py # Done diff --git a/etl/month_end_automation_wave_2_layout.py b/etl/month_end_automation_wave_2_layout.py index e6b4ff2..5a1296b 100644 --- a/etl/month_end_automation_wave_2_layout.py +++ b/etl/month_end_automation_wave_2_layout.py @@ -26,7 +26,7 @@ rate_card_data = { 207.65, 101, 186.4, 98, 98, 450, 150, 163, 135, 120, "60 - Needs to be verified (Post EPR)", 45, 90.5, 40, - 25, 25, 25, "check with Kevin" + "25 - check number on right", "25 - check number on right", "25 - check number on right", "check with Kevin" ] } @@ -93,6 +93,8 @@ ra = df[ df["ra"].str.lower().isin(["completed rdsap 10", "completed rdsap 9.9"]) ].copy() ra["job_type"] = "RA" +ra["evidence_record"] = None +ra["invoice_record"] = None filtered_dfs.append(ra) @@ -101,6 +103,8 @@ att = df[ df["att"].str.lower().isin(["completed"]) ].copy() att["job_type"] = "ATT" +att["evidence_record"] = None +att["invoice_record"] = None filtered_dfs.append(att) # V1 Coordination @@ -108,12 +112,16 @@ v1 = df[ df["v1 coordination status"].str.lower().isin(["rc complete"]) ].copy() v1["job_type"] = "Coordination Stage 1 v1" +v1["evidence_record"] = None +v1["invoice_record"] = None filtered_dfs.append(v1) # V2 Coordination _ = df[df["v2 invoiced"].fillna('').str.lower().isin(['to be invoiced'])] v2 = _[_["v2 dc/ima/pas"] > 0].copy() v2["job_type"] = "Coordination Stage 1 v2 remodel" +v2["evidence_record"] = None +v2["invoice_record"] = None filtered_dfs.append(v2) # V3 Coordination @@ -121,6 +129,8 @@ v3 = df[ df["v3 invoiced"].str.lower().isin(["to be invoiced"]) ].copy() v3["job_type"] = "Coordination Stage 1 v3 remodel" +v3["evidence_record"] = None +v3["invoice_record"] = None filtered_dfs.append(v3) # Coordination stage 2 Please complete @@ -128,6 +138,8 @@ cors2 = df[ df["rc stg. 2"].str.lower().isin(["to invoice"]) ].copy() cors2["job_type"] = "Coordination Stage 2" +cors2["evidence_record"] = None +cors2["invoice_record"] = None filtered_dfs.append(cors2) # Design type archietype @@ -135,6 +147,8 @@ design1 = df[ (df["design type for invoicing"].str.lower().isin(["archetype"])) & (df["design invoice status"].str.lower().isin(["to invoice"])) ].copy() design1["job_type"] = "Design Archetype" +design1["evidence_record"] = None +design1["invoice_record"] = None filtered_dfs.append(design1) # design type reptitive @@ -142,6 +156,8 @@ design1 = df[ (df["design type for invoicing"].str.lower().isin(["repetitive"])) & df["design invoice status"].str.lower().isin(["to invoice"]) ].copy() design1["job_type"] = "Design Repetitive" +design1["evidence_record"] = None +design1["invoice_record"] = None filtered_dfs.append(design1) # Design stage revisions @@ -149,6 +165,8 @@ design2 = df[ df["design revision invoice status"].str.lower().isin(["to invoice"]) ].copy() design2["job_type"] = "Design Revision" +design2["evidence_record"] = None +design2["invoice_record"] = None filtered_dfs.append(design2) # Lodgement Phase 1 @@ -156,6 +174,8 @@ lodg1 = df[ df["lodg. phase 1 invoice status"].str.lower().isin(["to invoice"]) ].copy() lodg1["job_type"] = "Lodgement phase 1" +lodg1["evidence_record"] = None +lodg1["invoice_record"] = None filtered_dfs.append(lodg1) # Full Lodgement Phase @@ -163,6 +183,8 @@ lodg2 = df[ df["full lodgement invoice status"].str.lower().isin(["to invoice"]) ].copy() lodg2["job_type"] = "Full Lodgement phase 2" +lodg2["evidence_record"] = None +lodg2["invoice_record"] = None filtered_dfs.append(lodg2) # POST EPC @@ -170,6 +192,8 @@ post_epc = df[ df["post-epc status"].str.lower().isin(["epc files uploaded"]) ].copy() post_epc["job_type"] = "Post EPC" +post_epc["evidence_record"] = None +post_epc["invoice_record"] = None filtered_dfs.append(post_epc) @@ -178,6 +202,8 @@ post_epr = df[ df["post-epc status"].str.lower().isin(["post epr completed"]) ].copy() post_epr["job_type"] = "Post EPR" +post_epr["evidence_record"] = None +post_epr["invoice_record"] = None filtered_dfs.append(post_epr) @@ -187,6 +213,8 @@ post_att = df[ df["post-att"].str.lower().isin(["post-att uploaded"]) ].copy() post_att["job_type"] = "POST ATT" +post_att["evidence_record"] = None +post_att["invoice_record"] = None filtered_dfs.append(post_att) @@ -195,6 +223,8 @@ retro = df[ df["retrofit evaluation"].str.lower().isin(["complete"]) ].copy() retro["job_type"] = "Retrofit Evaluation" +retro["evidence_record"] = None +retro["invoice_record"] = None filtered_dfs.append(retro) # RA NO Show @@ -203,6 +233,8 @@ ra_ns = df[ (df["ra no show evidence"] != 0) ].copy() ra_ns["job_type"] = "RA NO SHOW" +ra_ns["evidence_record"] = ra_ns["ra no show evidence"] +ra_ns["invoice_record"] = ra_ns["ra no show invoice"] filtered_dfs.append(ra_ns) @@ -213,6 +245,8 @@ att_ns = df[ (df["att no show evidence"] != 0) ].copy() att_ns["job_type"] = "ATT NO SHOW" +att_ns["evidence_record"] = att_ns["att no show evidence"] +att_ns["invoice_record"] = att_ns["att no show invoice"] filtered_dfs.append(att_ns) @@ -222,6 +256,8 @@ epc_ns = df[ (df["epc no show evidence"] != 0) ].copy() epc_ns["job_type"] = "post EPC NO SHOW" +epc_ns["evidence_record"] = epc_ns["epc no show evidence"] +epc_ns["invoice_record"] = epc_ns["epc no show invoice"] filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) @@ -236,7 +272,7 @@ import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') # Upload to sharepoint -attribute = ['address', 'client', 'job_type', 'rate'] +attribute = ['address', 'client', 'job_type', 'rate', "evidence_record", "invoice_record"] master_folder_name = "NHCF SHDF WESTVILLE WAVE 1 & 2" file_name = f"NCHA SHDF Westville Wave 1 & 2_{timestamp}.xlsx" combined_with_rates[attribute].to_excel(file_name, index=False) diff --git a/etl/month_end_automation_wave_2_no_11.py b/etl/month_end_automation_wave_2_no_11.py index e9d2349..a5a7f0a 100644 --- a/etl/month_end_automation_wave_2_no_11.py +++ b/etl/month_end_automation_wave_2_no_11.py @@ -25,7 +25,7 @@ rate_card_data = { 225, 92.65, check_wall_type, check_wall_type, check_wall_type, check_wall_type, check_wall_type, 165, 135, 120, "60 - check", 48.6, 85.55, 49.6, - 125, 125, 125 + "125 - check with right", "125 - check with right", "125 - check with right" ] } @@ -91,6 +91,8 @@ def get_df(df, column_name, success_critera, job_name=None): df[column_name].str.lower().isin(success_critera) ].copy() _["job_type"] = job_name + _["evidence_record"] = None + _["invoice_record"] = None return _ @@ -122,6 +124,8 @@ cors2 = df[ df["rc stg. 2"].str.lower().isin(["to invoice"]) ] cors2["job_type"] = "Coordination Stage 2" +cors2["evidence_record"] = None +cors2["invoice_record"] = None filtered_dfs.append(cors2) # Design stage 1 @@ -156,6 +160,8 @@ post_epr = df[ df["post-epc status"].str.lower().isin(["post epr completed", "Post epr desktop based completed"]) ].copy() post_epr["job_type"] = "POST epr" +post_epr["evidence_record"] = None +post_epr["invoice_record"] = None filtered_dfs.append(post_epr) # Post ATT @@ -180,6 +186,8 @@ att_ns = df[ (df["att no show evidence"] != 0) ].copy() att_ns["job_type"] = "ATT NO SHOW" +att_ns["evidence_record"] = att_ns["att no show evidence"] +att_ns["invoice_record"] = att_ns["att no show invoice"] filtered_dfs.append(att_ns) @@ -201,7 +209,7 @@ import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') # Upload to sharepoint -attribute = ['address', 'client', 'job_type', 'rate'] +attribute = ['address', 'client', 'job_type', 'rate', "evidence_record", "invoice_record"] master_folder_name = "Northumberland LAD2 & HUG2" file_name = f"{master_folder_name}_{timestamp}.xlsx" combined_with_rates[attribute].to_excel(file_name, index=False) diff --git a/etl/month_end_automation_wave_2_no_12.py b/etl/month_end_automation_wave_2_no_12.py index aa7bd7e..e8c5462 100644 --- a/etl/month_end_automation_wave_2_no_12.py +++ b/etl/month_end_automation_wave_2_no_12.py @@ -26,7 +26,7 @@ rate_card_data = { "(185) - Kevin Check as this depends on property size", 115, 200, 125, 125, empty, "(135)- Mariane said 'Remaining RC & design sign off", 185, 135, 120, "(60) Post EPR, please verify", 65, 115, 60, - 50, 50, 50, "(60) - Please add price for design sign off, check with kevin and marianne" + "50 - check with right", "50- check with right", "50- check with right", "(60) - Please add price for design sign off, check with kevin and marianne" ] } @@ -157,6 +157,8 @@ post_epr = df[ df["post-epc epr status"].str.lower().isin(["post epr completed"]) ].copy() post_epr["job_type"] = "POST epr" +post_epr["evidence_record"] = post_epr["ra no show evidence"] +post_epr["invoice_record"] = post_epr["ra no show invoice"] filtered_dfs.append(post_epr) # Post ATT @@ -173,6 +175,8 @@ ra_ns = df[ (df["ra no show evidence"] != 0) ].copy() ra_ns["job_type"] = "RA NO SHOW" +ra_ns["evidence_record"] = ra_ns["ra no show evidence"] +ra_ns["invoice_record"] = ra_ns["ra no show invoice"] filtered_dfs.append(ra_ns) @@ -190,6 +194,8 @@ epc_ns = df[ (df["epc no show evidence"] != 0) ].copy() epc_ns["job_type"] = "post EPC NO SHOW" +epc_ns["evidence_record"] = epc_ns["epc no show evidence"] +epc_ns["invoice_record"] = epc_ns["epc no show invoice"] filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) @@ -203,7 +209,7 @@ import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') # Upload to sharepoint -attribute = ['address', 'client', 'job_type', 'rate'] +attribute = ['address', 'client', 'job_type', 'rate', "evidence_record", "invoice_record"] master_folder_name = "Shropshire Council HUG 2" file_name = f"{master_folder_name}_{timestamp}.xlsx" combined_with_rates[attribute].to_excel(file_name, index=False) diff --git a/etl/month_end_automation_wave_2_no_14.py b/etl/month_end_automation_wave_2_no_14.py index e436aec..150dcdd 100644 --- a/etl/month_end_automation_wave_2_no_14.py +++ b/etl/month_end_automation_wave_2_no_14.py @@ -24,7 +24,7 @@ rate_card_data_example = { "rate": [ 259, 125, 280, 125, 135, "60 to check", 85, 125, - 60, 45, 45, 45, + 60, "45 - check with right", "45 - check with right", "45 - check with right", ] } @@ -174,6 +174,8 @@ ra_ns = df[ (df["ra no show evidence"] != 0) ].copy() ra_ns["job_type"] = "RA NO SHOW" +ra_ns["evidence_record"] = ra_ns["ra no show evidence"] +ra_ns["invoice_record"] = ra_ns["ra no show invoice"] filtered_dfs.append(ra_ns) @@ -191,6 +193,8 @@ epc_ns = df[ (df["post epc no show evidence"] != 0) ].copy() epc_ns["job_type"] = "post EPC NO SHOW" +epc_ns["evidence_record"] = epc_ns["post epc no show evidence"] +epc_ns["invoice_record"] = epc_ns["post epc no show invoice"] filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) @@ -205,7 +209,7 @@ import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') # Upload to sharepoint -attribute = ['address', 'client', 'job_type', 'rate'] +attribute = ['address', 'client', 'job_type', 'rate', "evidence_record", "invoice_record"] master_folder_name = "Decent Homes Stonewater - Operations" file_name = f"{master_folder_name}_{timestamp}.xlsx" combined_with_rates[attribute].to_excel(file_name, index=False) diff --git a/etl/month_end_automation_wave_2_no_15.py b/etl/month_end_automation_wave_2_no_15.py index a175801..e5e5d55 100644 --- a/etl/month_end_automation_wave_2_no_15.py +++ b/etl/month_end_automation_wave_2_no_15.py @@ -27,7 +27,7 @@ rate_card_data = { "259 (new RA rate for PAS2035:2023 - old rates for other works - to discuss with KN)", 40, 178.5, empty, empty, empty, 180, 275, 135, 120, "60 - please verify with Marianne", 45, 45, 40, - 25, 25, 25, "Please check price for design revision with Andreas" + "25 - check with right", "25 - check with right", "25 - check with right", "Please check price for design revision with Andreas" ] } @@ -182,6 +182,8 @@ ra_ns = df[ (df["ra no show evidence"] != 0) ].copy() ra_ns["job_type"] = "RA NO SHOW" +ra_ns["evidence_record"] = ra_ns["ra no show evidence"] +ra_ns["invoice_record"] = ra_ns["ra no show invoice"] filtered_dfs.append(ra_ns) @@ -191,6 +193,8 @@ att_ns = df[ (df["att no show evidence"] != 0) ].copy() att_ns["job_type"] = "ATT NO SHOW" +att_ns["evidence_record"] = att_ns["att no show evidence"] +att_ns["invoice_record"] = att_ns["att no show invoice"] filtered_dfs.append(att_ns) @@ -200,6 +204,8 @@ epc_ns = df[ (df["epc no show evidence"] != 0) ].copy() epc_ns["job_type"] = "post EPC NO SHOW" +epc_ns["evidence_record"] = epc_ns["epc no show evidence"] +epc_ns["invoice_record"] = epc_ns["epc no show invoice"] filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) @@ -213,7 +219,7 @@ import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') # Upload to sharepoint -attribute = ['address', 'client', 'job_type', 'rate'] +attribute = ['address', 'client', 'job_type', 'rate', "evidence_record", "invoice_record"] master_folder_name = "WCHG SHDF 2.1 Mansard" file_name = f"{master_folder_name}_{timestamp}.xlsx" combined_with_rates[attribute].to_excel(file_name, index=False) diff --git a/etl/month_end_automation_wave_2_no_3.py b/etl/month_end_automation_wave_2_no_3.py index e8a46de..b3b82a4 100644 --- a/etl/month_end_automation_wave_2_no_3.py +++ b/etl/month_end_automation_wave_2_no_3.py @@ -24,7 +24,7 @@ rate_card_data = { 259, 101, 210, 95, 95, 450, 150, 195, 135, 120, "(60)) - please confirm with Marianne, EPR", 45, 90.5, 42.4, - 25, 25, 25, "Please ask for Design Revision" + "25 - check on right", "25 - check on right", "25 - check on right", "Please ask for Design Revision" ] } @@ -91,6 +91,8 @@ ra = df[ df["ra"].str.lower().isin(["completed rdsap 10", "completed rdsap 9.9", "completed", "complete"]) ].copy() ra["job_type"] = "RA" +ra["evidence_record"] = None +ra["invoice_record"] = None filtered_dfs.append(ra) @@ -99,6 +101,8 @@ att = df[ df["att"].str.lower().isin(["completed"]) ].copy() att["job_type"] = "ATT" +att["evidence_record"] = None +att["invoice_record"] = None filtered_dfs.append(att) # V1 Coordination @@ -106,11 +110,15 @@ v1 = df[ df["coordination status"].str.lower().isin(["ima/mtp completed"]) ].copy() v1["job_type"] = "Coordination Stage 1 v1" +v1["evidence_record"] = None +v1["invoice_record"] = None filtered_dfs.append(v1) # V2 Coordination _ = df[df["v2 mtp status"].fillna('').str.lower().isin(['v2 ima-mtp completed', 'v2 completed'])].copy() _["job_type"] = "Coordination Stage 1 v2 remodel" +_["evidence_record"] = None +_["invoice_record"] = None filtered_dfs.append(_) # V3 Coordination @@ -125,6 +133,8 @@ cors2 = df[ df["rc stg. 2"].str.lower().isin(["to invoice"]) ] cors2["job_type"] = "Coordination Stage 2" +cors2["evidence_record"] = None +cors2["invoice_record"] = None filtered_dfs.append(cors2) # Design Archetype @@ -133,6 +143,8 @@ design1 = df[ ].copy() design1 = design1[design1["design type for invoicing"].str.lower().isin(['archetype'])].copy() design1["job_type"] = "Design Archetype" +design1["evidence_record"] = None +design1["invoice_record"] = None filtered_dfs.append(design1) # Design Repetitive @@ -141,6 +153,8 @@ design1 = df[ ].copy() design1 = design1[design1["design type for invoicing"].str.lower().isin(['repetitive'])].copy() design1["job_type"] = "Design repetitive" +design1["evidence_record"] = None +design1["invoice_record"] = None filtered_dfs.append(design1) # Design Revision @@ -148,6 +162,8 @@ design_revision = df[ df["design revision invoice status"].str.lower().isin(["to invoice"]) ].copy() design_revision["job_type"] = "Design repetitive" +design_revision["evidence_record"] = None +design_revision["invoice_record"] = None filtered_dfs.append(design_revision) # Lodgement Phase 1 @@ -155,6 +171,8 @@ lodg1 = df[ df["phase 1 invoice status (lodgement)"].str.lower().isin(["done"]) ].copy() lodg1["job_type"] = "Lodgement Phase 1" +lodg1["evidence_record"] = None +lodg1["invoice_record"] = None filtered_dfs.append(lodg1) # Full Lodgement Phase @@ -162,6 +180,8 @@ lodg2 = df[ df["lodgement invoice status (lodgement)"].str.lower().isin(["to invoice"]) ].copy() lodg2["job_type"] = "Full lodgement phase 2" +lodg2["evidence_record"] = None +lodg2["invoice_record"] = None filtered_dfs.append(lodg2) # POST EPC @@ -169,6 +189,8 @@ post_epc = df[ df["post epc"].str.lower().isin(["success", "pics uploaded"]) ].copy() post_epc["job_type"] = "POST EPC" +post_epc["evidence_record"] = None +post_epc["invoice_record"] = None filtered_dfs.append(post_epc) @@ -177,6 +199,8 @@ post_epr = df[ df["post epc"].str.lower().isin(["post epr completed"]) ].copy() post_epr["job_type"] = "POST EPR" +post_epr["evidence_record"] = None +post_epr["invoice_record"] = None filtered_dfs.append(post_epr) @@ -185,6 +209,8 @@ post_att = df[ df["post att"].str.lower().isin(["uploaded"]) ].copy() post_att["job_type"] = "POST ATT" +post_att["evidence_record"] = None +post_att["invoice_record"] = None filtered_dfs.append(post_att) @@ -193,6 +219,8 @@ retro = df[ df["retrofit evaluation"].str.lower().isin(["uploaded", "completed", "to invoice"]) ].copy() retro["job_type"] = "Retrofit Evaluation" +retro["evidence_record"] = None +retro["invoice_record"] = None filtered_dfs.append(retro) # RA NO Show @@ -201,6 +229,8 @@ ra_ns = df[ (df["ra no show evidence"] != 0) ].copy() ra_ns["job_type"] = "RA NO SHOW" +ra_ns["evidence_record"] = ra_ns["ra no show evidence"] +ra_ns["invoice_record"] = ra_ns["ra no show invoice"] filtered_dfs.append(ra_ns) @@ -210,6 +240,8 @@ att_ns = df[ (df["att no show evidence"] != 0) ].copy() att_ns["job_type"] = "ATT NO SHOW" +att_ns["evidence_record"] = att_ns["att no show evidence"] +att_ns["invoice_record"] = att_ns["att no show invoice"] filtered_dfs.append(att_ns) @@ -219,6 +251,8 @@ epc_ns = df[ (df["post epc no show evidence"] != 0) ].copy() epc_ns["job_type"] = "Post EPC NO SHOW" +epc_ns["evidence_record"] = epc_ns["post epc no show evidence"] +epc_ns["invoice_record"] = epc_ns["post epc no show invoice"] filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) @@ -232,7 +266,7 @@ import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') # Upload to sharepoint -attribute = ['address', 'client', 'job_type', 'rate'] +attribute = ['address', 'client', 'job_type', 'rate', "evidence_record", "invoice_record"] master_folder_name = "Platform Housing W2 (in use)" file_name = f"{master_folder_name}_{timestamp}.xlsx" combined_with_rates[attribute].to_excel(file_name, index=False) diff --git a/etl/month_end_automation_wave_2_no_4.py b/etl/month_end_automation_wave_2_no_4.py index 787aede..e0355b6 100644 --- a/etl/month_end_automation_wave_2_no_4.py +++ b/etl/month_end_automation_wave_2_no_4.py @@ -24,7 +24,7 @@ rate_card_data = { 165.75, 72.25, 174.25, 174.25, 174.25, 175, 175, 124.25, 135, 120, "(60) - please check with Marianne", 45, 63.75, 34, - 25, 25, 25, "Please ask marianne or kev for design revision" + "25 - check number on right", "25 - check number on right", "25- check with number on right", "Please ask marianne or kev for design revision" ] } @@ -90,6 +90,8 @@ ra = df[ df["ra"].str.lower().isin(["completed rdsap 10", "completed rdsap 9.9"]) ].copy() ra["job_type"] = "RA" +ra["evidence_record"] = None +ra["invoice_record"] = None filtered_dfs.append(ra) @@ -98,6 +100,8 @@ att = df[ df["att"].str.lower().isin(["completed"]) ].copy() att["job_type"] = "ATT" +att["evidence_record"] = None +att["invoice_record"] = None filtered_dfs.append(att) # V1 Coordination @@ -105,26 +109,36 @@ v1 = df[ df["v1 coordination status (ioe,mtp)"].str.lower().isin(["rc complete"]) ].copy() v1["job_type"] = "Coordination Stage 1 v1" +v1["evidence_record"] = None +v1["invoice_record"] = None filtered_dfs.append(v1) # V2 Coordination _ = df[df["mtp v2 status"].str.lower().isin(['rc v2 complete'])].copy() _["job_type"] = "Coordination Stage 1 v2 remodel" +_["evidence_record"] = None +_["invoice_record"] = None filtered_dfs.append(_) # V2 Coordination _ = df[df["mtp v2 invoiced"].str.lower().isin(['needs to be invoiced'])].copy() _["job_type"] = "Coordination Stage 1 v2 remodel" +_["evidence_record"] = None +_["invoice_record"] = None filtered_dfs.append(_) # V3 Coordination v3 = df[df["v3 rc status"].str.lower().isin(['uploaded'])].copy() v3["job_type"] = "Coordination Stage 1 v3 remodel" +v3["evidence_record"] = None +v3["invoice_record"] = None filtered_dfs.append(_) # V3 Coordination v3 = df[df["v3 invoice status"].str.lower().isin(['to be invoiced'])].copy() v3["job_type"] = "Coordination Stage 1 v3 remodel" +v3["evidence_record"] = None +v3["invoice_record"] = None filtered_dfs.append(_) # Coordination stage 2 Please complete @@ -132,6 +146,8 @@ cors2 = df[ df["rc stg. 2 status"].str.lower().isin(["to invoice", "completed"]) ] cors2["job_type"] = "Coordination Stage 2" +cors2["evidence_record"] = None +cors2["invoice_record"] = None filtered_dfs.append(cors2) # Design Archetype @@ -140,6 +156,8 @@ design1 = df[ ].copy() design1 = design1[design1["design type"].str.lower().isin(["archetype"])].copy() design1["job_type"] = "Design Archetype" +design1["evidence_record"] = None +design1["invoice_record"] = None filtered_dfs.append(design1) # Design Repetitive @@ -148,6 +166,8 @@ design1 = df[ ].copy() design1 = design1[design1["design type"].str.lower().isin(["repetitive"])].copy() design1["job_type"] = "Design Repetitive" +design1["evidence_record"] = None +design1["invoice_record"] = None filtered_dfs.append(design1) # Design Revision @@ -155,6 +175,8 @@ design_revision = df[ df["design revision invoice status"].str.lower().isin(["to invoice"]) ].copy() design_revision["job_type"] = "Design repetitive" +design_revision["evidence_record"] = None +design_revision["invoice_record"] = None filtered_dfs.append(design_revision) # Lodgement Phase 1 @@ -162,6 +184,8 @@ lodg1 = df[ df["phase 1 invoice status (lodgement)"].str.lower().isin(["done", "to be invoiced"]) ].copy() lodg1["job_type"] = "Lodgement Phase 1" +lodg1["evidence_record"] = None +lodg1["invoice_record"] = None filtered_dfs.append(lodg1) # Full Lodgement Phase @@ -169,6 +193,8 @@ _ = df[ df["lodgement invoice status"].str.lower().isin(["to invoice"]) ].copy() _["job_type"] = "Full lodgement phase 2" +_["evidence_record"] = None +_["invoice_record"] = None filtered_dfs.append(_) # POST EPC @@ -176,6 +202,8 @@ post_epc = df[ df["post epc"].str.lower().isin(["completed & uploaded"]) ].copy() post_epc["job_type"] = "POST EPC" +post_epc["evidence_record"] = None +post_epc["invoice_record"] = None filtered_dfs.append(post_epc) @@ -184,6 +212,8 @@ post_epr = df[ df["post epc"].str.lower().isin(["post epr completed"]) ].copy() post_epr["job_type"] = "POST EPR" +post_epr["evidence_record"] = None +post_epr["invoice_record"] = None filtered_dfs.append(post_epr) # Post ATT @@ -191,6 +221,8 @@ post_att = df[ df["post att"].str.lower().isin(["completed & uploaded"]) ].copy() post_att["job_type"] = "POST ATT" +post_att["evidence_record"] = None +post_att["invoice_record"] = None filtered_dfs.append(post_att) @@ -199,6 +231,8 @@ retro = df[ df["retrofit evaluation"].str.lower().isin(["completed & uploaded"]) ].copy() retro["job_type"] = "Retrofit Evaluation" +retro["evidence_record"] = None +retro["invoice_record"] = None filtered_dfs.append(retro) # RA NO Show @@ -207,6 +241,8 @@ ra_ns = df[ (df["ra no show evidence"] != 0) ].copy() ra_ns["job_type"] = "RA NO SHOW" +ra_ns["evidence_record"] = ra_ns["ra no show evidence"] +ra_ns["invoice_record"] = ra_ns["ra no show invoice"] filtered_dfs.append(ra_ns) @@ -216,6 +252,8 @@ att_ns = df[ (df["att no show evidence"] != 0) ].copy() att_ns["job_type"] = "ATT NO SHOW" +att_ns["evidence_record"] = att_ns["att no show evidence"] +att_ns["invoice_record"] = att_ns["att no show invoice"] filtered_dfs.append(att_ns) @@ -225,6 +263,8 @@ epc_ns = df[ (df["post epc no show evidence"] != 0) ].copy() epc_ns["job_type"] = "post EPC NO SHOW" +epc_ns["evidence_record"] = epc_ns["post epc no show evidence"] +epc_ns["invoice_record"] = epc_ns["post epc no show invoice"] filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) @@ -240,7 +280,7 @@ timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') # Upload to sharepoint -attribute = ['address', 'client', 'job_type', 'rate'] +attribute = ['address', 'client', 'job_type', 'rate', "evidence_record", "invoice_record"] master_folder_name = "Stonewater - (in use)" file_name = f"{master_folder_name}_{timestamp}.xlsx" combined_with_rates[attribute].to_excel(file_name, index=False) diff --git a/etl/month_end_automation_wave_2_no_7.py b/etl/month_end_automation_wave_2_no_7.py index 5cdf8f0..023ee94 100644 --- a/etl/month_end_automation_wave_2_no_7.py +++ b/etl/month_end_automation_wave_2_no_7.py @@ -24,7 +24,7 @@ rate_card_data_sp_plus = { 170, 70, 200, "check with Kevin", "check with Kevin", 470, 155, 165, 135, 120, "60 but check with Kevin as EPR", 45, 70, 40, - 30, 30, 30 + "30 - check number on right", "30 - check number on right", "30 - check number on right" ] } @@ -40,7 +40,7 @@ rate_card_data_net_zero = { 170, 70, 200, emp_msg, emp_msg, 325, 140, 165, 135, 120, "60 but check with Kevin as EPR", 45, 70, 40, - 30, 30, 30 + "30 - check number on right", "30- check number on right", "30 - check number on right" ] } @@ -126,6 +126,9 @@ def get_df(df, column_name, success_critera, job_name=None): ].copy() if job_name: _["job_type"] = job_name + + _["evidence_record"] = None + _["invoice_record"] = None return _ @@ -207,6 +210,8 @@ ra_ns = df[ (df["ra no show evidence"] != 0) ].copy() ra_ns["job_type"] = "RA NO SHOW" +ra_ns["evidence_record"] = ra_ns["epc no show evidence"] +ra_ns["invoice_record"] = ra_ns["epc no show invoice"] filtered_dfs.append(ra_ns) @@ -216,15 +221,20 @@ att_ns = df[ (df["att no show evidence"] != 0 ) ].copy() att_ns["job_type"] = "ATT NO SHOW" +att_ns["evidence_record"] = att_ns["att no show evidence"] +att_ns["invoice_record"] = att_ns["att no show invoice"] filtered_dfs.append(att_ns) + # Post visit no show epc_ns = df[ (df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999)) & (df["epc no show evidence"] != 0 ) ].copy() epc_ns["job_type"] = "post EPC NO SHOW" +epc_ns["evidence_record"] = epc_ns["epc no show evidence"] +epc_ns["invoice_record"] = epc_ns["epc no show invoice"] filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) @@ -247,7 +257,7 @@ import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') # Upload to sharepoint -attribute = ['address', 'client', 'job_type', 'rate'] +attribute = ['address', 'client', 'job_type', 'rate', "evidence_record", "invoice_record"] combined_with_rates_sp_plus[attribute].to_excel(f'HomeGroup Wave 2SP+_{timestamp}.xlsx', index=False) combined_with_rates_net_zero_df[attribute].to_excel(f'HomeGroup Wave NetZero_{timestamp}.xlsx', index=False) combined_with_rates_other_from_home_group[attribute].to_excel(f'HomeGroup Wave Unsure_who_to_bill_{timestamp}.xlsx', index=False) diff --git a/etl/month_end_automation_wave_2_no_8.py b/etl/month_end_automation_wave_2_no_8.py index 7f2ab25..1d8d619 100644 --- a/etl/month_end_automation_wave_2_no_8.py +++ b/etl/month_end_automation_wave_2_no_8.py @@ -24,7 +24,7 @@ rate_card_data = { 259, 125, 280, 125, 125, 650, 415, 195, 175, 135, 120, "60 - Double check with Kevin/Marianne EPR", 85, 125, 60, - 45, 45, 45, "Design Revision check with Kevin/Marianne" + "45- check with right", "45 - check with right", "45 - check with right", "Design Revision check with Kevin/Marianne" ] } @@ -92,6 +92,8 @@ def get_df(df, column_name, success_critera, job_name=None): ].copy() if job_name: _["job_type"] = job_name + _["evidence_record"] = None + _["invoice_record"] = None return _ @@ -168,6 +170,8 @@ post_epr = df[ df["post epc status"].str.lower().isin(["post epr completed"]) ].copy() post_epr["job_type"] = "POST epr" +post_epr["evidence_record"] = None +post_epr["invoice_record"] = None filtered_dfs.append(post_epr) # Post ATT @@ -185,6 +189,8 @@ ra_ns = df[ (df["ra no show evidence"] != 0 ) ].copy() ra_ns["job_type"] = "RA NO SHOW" +ra_ns["evidence_record"] = ra_ns["ra no show evidence"] +ra_ns["invoice_record"] = ra_ns["ra no show invoice"] filtered_dfs.append(ra_ns) @@ -194,6 +200,8 @@ att_ns = df[ (df["att no show evidence"] !=0 ) ].copy() att_ns["job_type"] = "ATT NO SHOW" +att_ns["evidence_record"] = att_ns["att no show evidence"] +att_ns["invoice_record"] = att_ns["att no show invoice"] filtered_dfs.append(att_ns) @@ -202,6 +210,8 @@ epc_ns = df[ df["post works no show evidence"].fillna(-9999) != df["post works no show invoice"].fillna(-9999) ].copy() epc_ns["job_type"] = "post EPC NO SHOW" +epc_ns["evidence_record"] = epc_ns["post works no show evidence"] +epc_ns["invoice_record"] = epc_ns["post works no show invoice"] filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) @@ -218,7 +228,7 @@ timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') # Upload to sharepoint -attribute = ['address', 'client', 'job_type', 'rate'] +attribute = ['address', 'client', 'job_type', 'rate', "evidence_record", "invoice_record"] master_folder_name = "NCHA SHDF 2.1 SBS" file_name = f"{master_folder_name}_{timestamp}.xlsx" combined_with_rates[attribute].to_excel(file_name, index=False)