diff --git a/etl/month_end_automation_wave_2_no_16.py b/etl/month_end_automation_wave_2_no_16.py new file mode 100644 index 0000000..0c6d3c1 --- /dev/null +++ b/etl/month_end_automation_wave_2_no_16.py @@ -0,0 +1,169 @@ +# Wave 2's month end automation +from tqdm import tqdm +from monday import MondayClient +from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids +from pprint import pprint +import pandas as pd +import json + +monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" +monday = MondayClient(monday_key) +# NCHA SHDF Wave 3 On Hold +board_ids = ["6946967610"] + +for board in tqdm(board_ids): + board_data = monday.boards.fetch_boards_by_id(board) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} + reversed_col_id_map = {v: k for k, v in col_id_map.items()} + + + items = get_all_items(board, monday) + + all_records = [] + for row in tqdm(items): + data = {} + data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) + for col in row.get("column_values", []): + if col.get("id") in reversed_col_id_map: + if col.get("type") == "file": + value = col.get("value") + no_of_files = 0 + + if value: + value = json.loads(col["value"]) + no_of_files = len(value.get('files', [])) + data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + elif "no show" in reversed_col_id_map[col.get("id")]: + def extract_number_from_text(text): + number_str = '' + + for char in text: + if char.isnumeric(): + number_str += char + elif number_str: + break # stop once a number sequence ends + + return int(number_str) if number_str else None + text = col.get("text") + if text is None: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: extract_number_from_text(text) + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + all_records.append(data) + +# Convert to DataFrame +df = pd.DataFrame(all_records) + +filtered_dfs = [] + +def get_df(df, column_name, success_critera, job_name): + _ = df[ + df[column_name].str.lower().isin(success_critera) + ].copy() + _["job_type"] = job_name + return _ + +# RA +ra = get_df(df, "ra", ["completed"], "RA") +filtered_dfs.append(ra) + + +# PRE- ATT +att = get_df(df, "att", ["completed"], "ATT") +filtered_dfs.append(att) + +# V1 Coordination +v1 = get_df(df, "coordination status".lower(), [ + "rc complete", +], "V1 Coordination") +filtered_dfs.append(v1) + +# V2 Coordination +# v2 = get_df(df, "mtp v2 status", ["rc v2 complete"], "V2 Coordination") +# filtered_dfs.append(v2) + +# # V3 Coordination +# v3 = get_df(df, "v3 rc status", ["uploaded"], "V3 Coordination") +# filtered_dfs.append(v3) + +# v3 = get_df(df, "v3 invoice status", ["to be invoice"], "V3 Coordination") +# filtered_dfs.append(v3) + +# Design stage 1 +# design1 = get_df(df, "design invoice status", ["to invoice"], "Design") +# filtered_dfs.append(design1) + +# Design revision +# design2 = get_df(df, "design revision invoice", [ +# "Rev. A to invoice".lower(), +# "Rev. B to invoice".lower(), +# "Rev. C to invoice".lower(), +# "Rev. D to invoice".lower(), +# ], "Design Revision") +# filtered_dfs.append(design2) + +# Lodgement Phase 1 +lodg1 = get_df(df, "lodg. phase 1 invoice status".lower(), ["to invoice"], "Lodgement Phase 1") +filtered_dfs.append(lodg1) + + +# Full Lodgement Phase +full_lodgement = get_df(df, "full lodgement invoice status".lower(), ["to invoice"], "Full Lodgement") +filtered_dfs.append(full_lodgement) + +# POST EPC +post_epc = get_df(df, "lodged epc", ["complete", "complete & lodged",], "POST EPC") +filtered_dfs.append(post_epc) + + +# # POST EPR +# post_epr = df[ +# df["post-epc status"].str.lower().isin(["post epr completed"]) +# ].copy() +# post_epr["job_type"] = "POST ATT" +# filtered_dfs.append(post_epr) + +# Post ATT +post_att = get_df(df, "post att", ["done", "post att complete"], "POST ATT") +filtered_dfs.append(post_att) + +# Retrofit Evaluation +retro = get_df(df, "retrofit evaluation", ["done"], "Retrofit Evaluation") +filtered_dfs.append(retro) + +# RA NO Show +ra_ns = df[ + df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) +].copy() +ra_ns["job_type"] = "RA NO SHOW" +filtered_dfs.append(ra_ns) + + +# ATT NO Show +att_ns = df[ + df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) +].copy() +att_ns["job_type"] = "ATT NO SHOW" +filtered_dfs.append(att_ns) + + +# Post visit no show +# epc_ns = df[ +# df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999) +# ].copy() +# epc_ns["job_type"] = "EPC NO SHOW" +# filtered_dfs.append(epc_ns) + +final_df = pd.concat(filtered_dfs).reset_index(drop=True) + +final_df[['address', 'client', 'job_type']] \ No newline at end of file diff --git a/etl/month_end_automation_wave_3_layout.py b/etl/month_end_automation_wave_3_layout.py index 586b6da..e393890 100644 --- a/etl/month_end_automation_wave_3_layout.py +++ b/etl/month_end_automation_wave_3_layout.py @@ -9,11 +9,15 @@ import json monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) -# WCHG Walkups-Operations -board_ids = ["9349630181"] +board_ids = [ + # "9349630181", # WCHG Walkups-Operations + "8829428746", # 2502 Accent Housing + +] for board in tqdm(board_ids): + print(f"working on board {board}") board_data = monday.boards.fetch_boards_by_id(board) columns = board_data["data"]["boards"][0]["columns"] col_id_map = {col["title"].lower(): col["id"] for col in columns} @@ -68,81 +72,69 @@ df = pd.DataFrame(all_records) filtered_dfs = [] + +def get_df(df, column_name, success_critera, job_name): + if column_name in col_id_map: + _ = df[ + df[column_name].str.lower().isin(success_critera) + ].copy() + _["job_type"] = job_name + return _ + else: + print(f"failed to find {column_name}") + + # RA -ra = df[ - df["ra invoicing status"].str.lower().isin(["to invoice"]) -].copy() -ra["job_type"] = "RA" -filtered_dfs.append(ra) +ra = get_df(df, "ra invoicing status", ["to invoice"], "RA") +if ra is not None: + filtered_dfs.append(ra) -# ATT -att = df[ - df["pre att invoicing status"].str.lower().isin(["to invoice"]) -].copy() -att["job_type"] = "ATT" -filtered_dfs.append(att) +att = get_df(df, "post att invoicing status", ["to invoice"], "ATT") +if att is not None: + filtered_dfs.append(att) -# V1 Coordination -v1 = df[ - df["mtp invoicing status"].str.lower().isin(["(v1) ioe/mtp to invoice"]) -].copy() +modeling = get_df(df, "mtp invoicing status", ["modelling to invoice"], "V3 Coordination") +if modeling is not None: + filtered_dfs.append(modeling) -v1["job_type"] = "V1 Coordination" -filtered_dfs.append(v1) +v1 = get_df(df, "mtp invoicing status", ["(v1) ioe/mtp to invoice"], "V1 Coordination") +if v1 is not None: + filtered_dfs.append(v1) -# V2 Coordination -v2 = df[ - df["mtp invoicing status"].str.lower().isin(["(v2) ioe/mtp to invoice"]) -].copy() +v2 = get_df(df, "mtp invoicing status", ["(v2) ioe/mtp to invoice"], "V2 Coordination") +if v2 is not None: + filtered_dfs.append(v2) -v2["job_type"] = "V2 Coordination" -filtered_dfs.append(v2) +v3 = get_df(df, "mtp invoicing status", ["(v3) ioe/mtp to invoice"], "V3 Coordination") +if v3 is not None: + filtered_dfs.append(v3) -# V3 Coordination -v3 = df[ - df["mtp invoicing status"].str.lower().isin(["(v3) ioe/mtp to invoice"]) -].copy() - -v3["job_type"] = "V3 Coordination" -filtered_dfs.append(v3) - -# Design stage 1 -design1 = df[ - df["design invoicing status"].str.lower().isin(["to invoice"]) -].copy() -design1["job_type"] = "Design" -filtered_dfs.append(design1) +design1 = get_df(df, "design invoicing status", ["to invoice"], "Design") +if design1 is not None: + filtered_dfs.append(design1) # Design Revision revision_letter = ['a', 'b', 'c', 'd'] for letter in revision_letter: - design2 = df[ - df["design revision invoice"].str.lower().isin([f"rev. {letter} to invoice"]) - ].copy() - design2["job_type"] = "Design Revision" - filtered_dfs.append(design2) + design = get_df(df, "design revision invoice", [f"rev. {letter} to invoice"], "Design Revision") + if design is not None: + filtered_dfs.append(design) # Lodgement Phase 1 -lodg1 = df[ - df["lodgement phase 1 invoicing status"].str.lower().isin(["to invoice"]) -].copy() -lodg1["job_type"] = "Lodgement Phase 1" -filtered_dfs.append(lodg1) +lodg1 = get_df(df, "lodgement phase 1 invoicing status", ["to invoice"], "Lodgement Phase 1") +if lodg1 is not None: + filtered_dfs(lodg1) # Full Lodgement Phase -lodg2 = df[ - df["full lodgement invoicing status"].str.lower().isin(["to invoice"]) -].copy() -lodg2["job_type"] = "Full Lodgement" -filtered_dfs.append(lodg2) +lodg2 = get_df(df, "full lodgement invoicing status", ["to invoice"], "Full Lodgement") +if lodg2 is not None: + filtered_dfs.append(lodg2) # POST EPC -post_epc = df[ - df["post epc & eval. invoicing status"].str.lower().isin(["to invoice"]) -].copy() -post_epc["job_type"] = "POST EPC" -filtered_dfs.append(post_epc) +post_epc = get_df(df, "post epc & eval. invoicing status", ["to invoice"], "POST EPC") +if post_epc is not None: + filtered_dfs.append(post_epc) # # POST EPR, not right now @@ -152,45 +144,31 @@ filtered_dfs.append(post_epc) # post_epr["job_type"] = "POST ATT" # filtered_dfs.append(post_epr) - -# Post ATT -post_att = df[ - df["post att invoicing status"].str.lower().isin(["to invoice"]) -].copy() -post_att["job_type"] = "POST ATT" -filtered_dfs.append(post_att) - +post_att = get_df(df, "post att invoicing status", ["to invoice"], "POST ATT") +if post_att is not None: + filtered_dfs.append(post_epc) # Retrofit Evaluation -retro = df[ - df["rc stage 2 invoicing status"].str.lower().isin(["to invoice"]) -].copy() -retro["job_type"] = "RC stage 2" -filtered_dfs.append(retro) +rc = get_df(df, "rc stage 2 invoicing status", ["to invoice"], "RC stage 2") +if rc is not None: + filtered_dfs.append(rc) # RA NO Show -ra_ns = df[ - df["ra no show invoice"].str.lower().isin(["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"]) -].copy() -ra_ns["job_type"] = "RA NO SHOW" -filtered_dfs.append(ra_ns) +ra_ns = get_df(df,"ra no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "RA NO SHOW") +if ra_ns is not None: + filtered_dfs.append(ra_ns) # ATT NO Show -att_ns = df[ - df["pre att no show invoice"].str.lower().isin(["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"]) -].copy() -att_ns["job_type"] = "ATT NO SHOW" -filtered_dfs.append(att_ns) +att_ns = get_df(df, "pre att no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "ATT NO SHOW") +if att_ns is not None: + filtered_dfs.append(att_ns) # Post visit no show -#post work no show invoice -epc_ns = df[ - df["post works no show invoice"].str.lower().isin(["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"]) -].copy() -epc_ns["job_type"] = "EPC NO SHOW" -filtered_dfs.append(epc_ns) +epc_ns = get_df(df, "post works no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "EPC NO SHOW") +if epc_ns is not None: + filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True)