From c3b7345eabcf66944d66d46ddb980d7c70690d42 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 24 Jul 2025 14:22:32 +0000 Subject: [PATCH] completed --- etl/month_end_automation_wave_2_no_10.py | 170 +++++++++++++++++++++ etl/month_end_automation_wave_2_no_4.py | 18 +-- etl/month_end_automation_wave_2_no_5.py | 186 +++++++++++++++++++++++ etl/month_end_automation_wave_2_no_6.py | 175 +++++++++++++++++++++ etl/month_end_automation_wave_2_no_7.py | 169 ++++++++++++++++++++ etl/month_end_automation_wave_2_no_8.py | 172 +++++++++++++++++++++ etl/month_end_automation_wave_2_no_9.py | 169 ++++++++++++++++++++ 7 files changed, 1047 insertions(+), 12 deletions(-) create mode 100644 etl/month_end_automation_wave_2_no_10.py create mode 100644 etl/month_end_automation_wave_2_no_5.py create mode 100644 etl/month_end_automation_wave_2_no_6.py create mode 100644 etl/month_end_automation_wave_2_no_7.py create mode 100644 etl/month_end_automation_wave_2_no_8.py create mode 100644 etl/month_end_automation_wave_2_no_9.py diff --git a/etl/month_end_automation_wave_2_no_10.py b/etl/month_end_automation_wave_2_no_10.py new file mode 100644 index 0000000..87580f2 --- /dev/null +++ b/etl/month_end_automation_wave_2_no_10.py @@ -0,0 +1,170 @@ +# Wave 2's month end automation +from tqdm import tqdm +from monday import MondayClient +from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids +from pprint import pprint +import pandas as pd +import json + +monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" +monday = MondayClient(monday_key) +# NCHA Derbyshire Dales (DDDCC) SHDF +board_ids = ["6947307148"] + +for board in tqdm(board_ids): + board_data = monday.boards.fetch_boards_by_id(board) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} + reversed_col_id_map = {v: k for k, v in col_id_map.items()} + + + items = get_all_items(board, monday) + + all_records = [] + for row in tqdm(items): + data = {} + data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) + for col in row.get("column_values", []): + if col.get("id") in reversed_col_id_map: + if col.get("type") == "file": + value = col.get("value") + no_of_files = 0 + + if value: + value = json.loads(col["value"]) + no_of_files = len(value.get('files', [])) + data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + elif "no show" in reversed_col_id_map[col.get("id")]: + def extract_number_from_text(text): + number_str = '' + + for char in text: + if char.isnumeric(): + number_str += char + elif number_str: + break # stop once a number sequence ends + + return int(number_str) if number_str else None + text = col.get("text") + if text is None: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: extract_number_from_text(text) + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + all_records.append(data) + +# Convert to DataFrame +df = pd.DataFrame(all_records) + +filtered_dfs = [] + +def get_df(df, column_name, success_critera, job_name): + _ = df[ + df[column_name].str.lower().isin(success_critera) + ].copy() + _["job_type"] = job_name + return _ + + +# RA +ra = get_df(df, "ra", ["completed rdsap 9.9", "completed rd sap 10"], "RA") +filtered_dfs.append(ra) + + +# ATT +att = get_df(df, "att", ["completed"], "ATT") +filtered_dfs.append(att) + +# V1 Coordination +v1 = get_df(df, "coordination status".lower(), [ + "rc complete", +], "V1 Coordination") +filtered_dfs.append(v1) + +# V2 Coordination +# v2 = get_df(df, "v2 coordination status", ["rc v2 complete", "uploaded"], "V2 Coordination") +# filtered_dfs.append(v2) + +# # V3 Coordination +# v3 = get_df(df, "") +# # filtered_dfs.append(v3) + +# Design stage 1 +design1 = get_df(df, "design upload to sharepoint", ["done"], "Design") +filtered_dfs.append(design1) + +# Design revision +# design2 = get_df(df, "design revision invoice", [ +# "Rev. A to invoice".lower(), +# "Rev. B to invoice".lower(), +# "Rev. C to invoice".lower(), +# "Rev. D to invoice".lower(), +# ], "Design Revision") +# filtered_dfs.append(design2) + +# Lodgement Phase 1 +lodg1 = get_df(df, "Lodg. Phase 1 Invoice Status".lower(), ["to invoice"], "Lodgement Phase 1") +filtered_dfs.append(lodg1) + + +# Full Lodgement Phase +full_lodgement = get_df(df, "full lodgement invoice status".lower(), ["to invoice"], "Full Lodgement") +filtered_dfs.append(full_lodgement) + +# POST EPC +post_epc = get_df(df, "lodged epc", ["complete"], "POST EPC") +filtered_dfs.append(post_epc) + + +# # POST EPR +# post_epr = df[ +# df["post-epc status"].str.lower().isin(["post epr completed"]) +# ].copy() +# post_epr["job_type"] = "POST ATT" +# filtered_dfs.append(post_epr) + +# Post ATT +post_att = get_df(df, "post att", ["done"], "POST ATT") +filtered_dfs.append(post_att) + +post_att = get_df(df, "post-test status", ["complete"], "POST ATT") +filtered_dfs.append(post_att) + +# Retrofit Evaluation +retro = get_df(df, "retrofit evaluation", ["done"], "Retrofit Evaluation") +filtered_dfs.append(retro) + +# RA NO Show +ra_ns = df[ + df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) +].copy() +ra_ns["job_type"] = "RA NO SHOW" +filtered_dfs.append(ra_ns) + + +# ATT NO Show +att_ns = df[ + df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) +].copy() +att_ns["job_type"] = "ATT NO SHOW" +filtered_dfs.append(att_ns) + + +# Post visit no show +# epc_ns = df[ +# df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999) +# ].copy() +# epc_ns["job_type"] = "EPC NO SHOW" +# filtered_dfs.append(epc_ns) + +final_df = pd.concat(filtered_dfs).reset_index(drop=True) + +final_df[['address', 'client', 'job_type']] \ No newline at end of file diff --git a/etl/month_end_automation_wave_2_no_4.py b/etl/month_end_automation_wave_2_no_4.py index 7a4cc41..6fe0d90 100644 --- a/etl/month_end_automation_wave_2_no_4.py +++ b/etl/month_end_automation_wave_2_no_4.py @@ -123,17 +123,14 @@ filtered_dfs.append(lodg1) # Full Lodgement Phase _ = df[ - df["lodgement status"].str.lower().isin(["lodged"]) + df["lodgement invoice status"].str.lower().isin(["to invoice"]) ].copy() -lodg2 = _[ - _["lodgement invoice status"].str.lower().isin([None]) -].copy() -lodg2["job_type"] = "Full Lodgement" -filtered_dfs.append(lodg2) +_["job_type"] = "Full Lodgement" +filtered_dfs.append(_) # POST EPC post_epc = df[ - df["post epc"].str.lower().isin(["success", "pics uploaded"]) + df["post epc"].str.lower().isin(["completed & uploaded"]) ].copy() post_epc["job_type"] = "POST EPC" filtered_dfs.append(post_epc) @@ -146,11 +143,9 @@ filtered_dfs.append(post_epc) # post_epr["job_type"] = "POST ATT" # filtered_dfs.append(post_epr) - - # Post ATT post_att = df[ - df["post att"].str.lower().isin(["uploaded"]) + df["post att"].str.lower().isin(["completed & uploaded"]) ].copy() post_att["job_type"] = "POST ATT" filtered_dfs.append(post_att) @@ -158,7 +153,7 @@ filtered_dfs.append(post_att) # Retrofit Evaluation retro = df[ - df["retrofit evaluation"].str.lower().isin(["uploaded", "completed", "to invoice"]) + df["retrofit evaluation"].str.lower().isin(["completed & uploaded"]) ].copy() retro["job_type"] = "Retrofit Evaluation" filtered_dfs.append(retro) @@ -171,7 +166,6 @@ ra_ns["job_type"] = "RA NO SHOW" filtered_dfs.append(ra_ns) - # ATT NO Show att_ns = df[ df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) diff --git a/etl/month_end_automation_wave_2_no_5.py b/etl/month_end_automation_wave_2_no_5.py new file mode 100644 index 0000000..daaa8b0 --- /dev/null +++ b/etl/month_end_automation_wave_2_no_5.py @@ -0,0 +1,186 @@ +# Wave 2's month end automation + +from tqdm import tqdm +from monday import MondayClient +from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids +from pprint import pprint +import pandas as pd +import json + +monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" +monday = MondayClient(monday_key) +# ECO 4 NCHA Almshouses Operations +board_ids = ["9136254638"] + + +for board in tqdm(board_ids): + board_data = monday.boards.fetch_boards_by_id(board) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} + reversed_col_id_map = {v: k for k, v in col_id_map.items()} + + + items = get_all_items(board, monday) + + all_records = [] + for row in tqdm(items): + data = {} + data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) + for col in row.get("column_values", []): + if col.get("id") in reversed_col_id_map: + if col.get("type") == "file": + value = col.get("value") + no_of_files = 0 + + if value: + value = json.loads(col["value"]) + no_of_files = len(value.get('files', [])) + data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + elif "no show" in reversed_col_id_map[col.get("id")]: + def extract_number_from_text(text): + number_str = '' + + for char in text: + if char.isnumeric(): + number_str += char + elif number_str: + break # stop once a number sequence ends + + return int(number_str) if number_str else None + text = col.get("text") + if text is None: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: extract_number_from_text(text) + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + all_records.append(data) + +# Convert to DataFrame +df = pd.DataFrame(all_records) + +filtered_dfs = [] + +# RA +ra = df[ + df["retrofit assessment"].str.lower().isin(["completed rdsap 10", "completed rdsap 9.9"]) +].copy() +ra["job_type"] = "RA" +filtered_dfs.append(ra) + + +# ATT +att = df[ + df["pre att"].str.lower().isin(["completed"]) +].copy() +att["job_type"] = "ATT" +filtered_dfs.append(att) + +# V1 Coordination +v1 = df[ + df["coordination status"].str.lower().isin(["ioe/mtp complete"]) +].copy() +v1["job_type"] = "V1 Coordination" +filtered_dfs.append(v1) + +# V2 Coordination +# _ = df[df["mtp v2 invoiced"].str.lower().isin(['done', 'needs to be invoiced'])].copy() +# _["job_type"] = "V2 Coordination" +# filtered_dfs.append(_) + +# V3 Coordination +# v3 = df[df["v3 rc status"].str.lower().isin(['uploaded'])].copy() +# v3["job_type"] = "V3 Coordination" +# filtered_dfs.append(_) + +# Design stage 1 +design1 = df[ + df["retrofit design status"].str.lower().isin(["to invoice"]) +].copy() +design1["job_type"] = "Design" +filtered_dfs.append(design1) + +# Design stage 2 +# design2 = df[ +# df[""].str.lower().isin(["to invoice"]) +# ].copy() +# design2["job_type"] = "Design Revision" +# filtered_dfs.append(design2) + +# Lodgement Phase 1 +# lodg1 = df[ +# df["phase 1 invoice status (lodgement)"].str.lower().isin(["done", "to be invoiced"]) +# ].copy() +# lodg1["job_type"] = "Lodgement Phase 1" +# filtered_dfs.append(lodg1) + +# Full Lodgement Phase +_ = df[ + df["trustmark lodgement"].str.lower().isin(["done"]) +].copy() +_["job_type"] = "Full Lodgement" +filtered_dfs.append(_) + +# POST EPC +post_epc = df[ + df["post epc status"].str.lower().isin(["done"]) +].copy() +post_epc["job_type"] = "POST EPC" +filtered_dfs.append(post_epc) + + +# # POST EPR +# post_epr = df[ +# df["post-epc status"].str.lower().isin(["post epr completed"]) +# ].copy() +# post_epr["job_type"] = "POST ATT" +# filtered_dfs.append(post_epr) + +# Post ATT +post_att = df[ + df["post att status"].str.lower().isin(["done"]) +].copy() +post_att["job_type"] = "POST ATT" +filtered_dfs.append(post_att) + + +# Retrofit Evaluation +retro = df[ + df["retrofit evaluation"].str.lower().isin(["done"]) +].copy() +retro["job_type"] = "Retrofit Evaluation" +filtered_dfs.append(retro) + +# RA NO Show +ra_ns = df[ + df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) +].copy() +ra_ns["job_type"] = "RA NO SHOW" +filtered_dfs.append(ra_ns) + + +# ATT NO Show +att_ns = df[ + df["pre att no show evidence"].fillna(-9999) != df["pre att no show invoice"].fillna(-9999) +].copy() +att_ns["job_type"] = "ATT NO SHOW" +filtered_dfs.append(att_ns) + + +# Post visit no show +epc_ns = df[ + df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999) +].copy() +epc_ns["job_type"] = "EPC NO SHOW" +filtered_dfs.append(epc_ns) + +final_df = pd.concat(filtered_dfs).reset_index(drop=True) + +final_df[['address', 'client', 'job_type']] \ No newline at end of file diff --git a/etl/month_end_automation_wave_2_no_6.py b/etl/month_end_automation_wave_2_no_6.py new file mode 100644 index 0000000..d069018 --- /dev/null +++ b/etl/month_end_automation_wave_2_no_6.py @@ -0,0 +1,175 @@ +# Wave 2's month end automation + +from tqdm import tqdm +from monday import MondayClient +from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids +from pprint import pprint +import pandas as pd +import json + +monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" +monday = MondayClient(monday_key) +#ECO 4 Wates Operations +board_ids = ["9520779048"] + + +for board in tqdm(board_ids): + board_data = monday.boards.fetch_boards_by_id(board) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} + reversed_col_id_map = {v: k for k, v in col_id_map.items()} + + + items = get_all_items(board, monday) + + all_records = [] + for row in tqdm(items): + data = {} + data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) + for col in row.get("column_values", []): + if col.get("id") in reversed_col_id_map: + if col.get("type") == "file": + value = col.get("value") + no_of_files = 0 + + if value: + value = json.loads(col["value"]) + no_of_files = len(value.get('files', [])) + data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + elif "no show" in reversed_col_id_map[col.get("id")]: + def extract_number_from_text(text): + number_str = '' + + for char in text: + if char.isnumeric(): + number_str += char + elif number_str: + break # stop once a number sequence ends + + return int(number_str) if number_str else None + text = col.get("text") + if text is None: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: extract_number_from_text(text) + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + all_records.append(data) + +# Convert to DataFrame +df = pd.DataFrame(all_records) + +filtered_dfs = [] + +def get_df(df, column_name, success_critera, job_name): + _ = df[ + df[column_name].str.lower().isin(success_critera) + ].copy() + _["job_type"] = job_name + return _ + +# RA +ra = get_df(df, "ra status", ["completed & uploaded"], "RA") +filtered_dfs.append(ra) + + +# ATT +att = get_df(df, "pre att status", ["completed & uploaded"], "ATT") +filtered_dfs.append(att) + +# V1 Coordination +v1 = get_df(df, "Coordination Status IOE/MTP".lower(), [ + "(V1) IOE/MTP Complete".lower(), + "(V2) IOE/MTP Complete".lower(), + "(V3) IOE/MTP Complete".lower(), +], "V1 Coordination") +filtered_dfs.append(v1) + +# V2 Coordination +# _ = df[df["mtp v2 invoiced"].str.lower().isin(['done', 'needs to be invoiced'])].copy() +# _["job_type"] = "V2 Coordination" +# filtered_dfs.append(_) + +# V3 Coordination +# v3 = df[df["v3 rc status"].str.lower().isin(['uploaded'])].copy() +# v3["job_type"] = "V3 Coordination" +# filtered_dfs.append(_) + +# Design stage 1 +design1 = get_df(df, "retrofit design status", ["completed"], "Design") +filtered_dfs.append(design1) + +# Design stage 2 +design2 = get_df(df, "design revision invoice", [ + "Rev. A to invoice".lower(), + "Rev. B to invoice".lower(), + "Rev. C to invoice".lower(), + "Rev. D to invoice".lower(), +], "Design Revision") +filtered_dfs.append(design2) + +# Lodgement Phase 1 +# lodg1 = df[ +# df["phase 1 invoice status (lodgement)"].str.lower().isin(["done", "to be invoiced"]) +# ].copy() +# lodg1["job_type"] = "Lodgement Phase 1" +# filtered_dfs.append(lodg1) + +# Full Lodgement Phase +full_lodgement = get_df(df, "full lodgement", ["completed"], "Full Lodgement") +filtered_dfs.append(full_lodgement) + +# POST EPC +post_epc = get_df(df, "post epc & evaluation status", ["uploaded"], "POST EPC") +filtered_dfs.append(post_epc) + + +# # POST EPR +# post_epr = df[ +# df["post-epc status"].str.lower().isin(["post epr completed"]) +# ].copy() +# post_epr["job_type"] = "POST ATT" +# filtered_dfs.append(post_epr) + +# Post ATT +post_att = get_df(df, "post att status", ["uploaded"], "POST ATT") +filtered_dfs.append(post_att) + + +# Retrofit Evaluation +retro = get_df(df, "retrofit evaluation", ["done"], "Retrofit Evaluation") +filtered_dfs.append(retro) + +# RA NO Show +# ra_ns = df[ +# df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) +# ].copy() +# ra_ns["job_type"] = "RA NO SHOW" +# filtered_dfs.append(ra_ns) + + +# # ATT NO Show +# att_ns = df[ +# df["pre att no show evidence"].fillna(-9999) != df["pre att no show invoice"].fillna(-9999) +# ].copy() +# att_ns["job_type"] = "ATT NO SHOW" +# filtered_dfs.append(att_ns) + + +# # Post visit no show +# epc_ns = df[ +# df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999) +# ].copy() +# epc_ns["job_type"] = "EPC NO SHOW" +# filtered_dfs.append(epc_ns) + +final_df = pd.concat(filtered_dfs).reset_index(drop=True) + +final_df[['address', 'client', 'job_type']] \ No newline at end of file diff --git a/etl/month_end_automation_wave_2_no_7.py b/etl/month_end_automation_wave_2_no_7.py new file mode 100644 index 0000000..fb15205 --- /dev/null +++ b/etl/month_end_automation_wave_2_no_7.py @@ -0,0 +1,169 @@ +# Wave 2's month end automation + +from tqdm import tqdm +from monday import MondayClient +from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids +from pprint import pprint +import pandas as pd +import json + +monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" +monday = MondayClient(monday_key) +#Home Group Wave 2SP+ +board_ids = ["4254419092"] + + +for board in tqdm(board_ids): + board_data = monday.boards.fetch_boards_by_id(board) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} + reversed_col_id_map = {v: k for k, v in col_id_map.items()} + + + items = get_all_items(board, monday) + + all_records = [] + for row in tqdm(items): + data = {} + data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) + for col in row.get("column_values", []): + if col.get("id") in reversed_col_id_map: + if col.get("type") == "file": + value = col.get("value") + no_of_files = 0 + + if value: + value = json.loads(col["value"]) + no_of_files = len(value.get('files', [])) + data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + elif "no show" in reversed_col_id_map[col.get("id")]: + def extract_number_from_text(text): + number_str = '' + + for char in text: + if char.isnumeric(): + number_str += char + elif number_str: + break # stop once a number sequence ends + + return int(number_str) if number_str else None + text = col.get("text") + if text is None: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: extract_number_from_text(text) + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + all_records.append(data) + +# Convert to DataFrame +df = pd.DataFrame(all_records) + +filtered_dfs = [] + +def get_df(df, column_name, success_critera, job_name): + _ = df[ + df[column_name].str.lower().isin(success_critera) + ].copy() + _["job_type"] = job_name + return _ + + +# RA +ra = get_df(df, "ra", ["completed rdsap 9.9", "completed rdsap 10"], "RA") +filtered_dfs.append(ra) + + +# ATT +att = get_df(df, "pre-att", ["completed"], "ATT") +filtered_dfs.append(att) + +# V1 Coordination +v1 = get_df(df, "osmosis rc status".lower(), [ + "rc completed", +], "V1 Coordination") +filtered_dfs.append(v1) + +# V2 Coordination +v2 = get_df(df, "v2 ioe mtp", ["completed"], "V2 Coordination") +filtered_dfs.append(v2) + +# # V3 Coordination +# v3 = get_df(df, "") +# # filtered_dfs.append(v3) + +# Design stage 1 +design1 = get_df(df, "design invoice status", ["to invoice"], "Design") +filtered_dfs.append(design1) + +# Design revision +# design2 = get_df(df, "design revision invoice", [ +# "Rev. A to invoice".lower(), +# "Rev. B to invoice".lower(), +# "Rev. C to invoice".lower(), +# "Rev. D to invoice".lower(), +# ], "Design Revision") +# filtered_dfs.append(design2) + +# Lodgement Phase 1 +lodg1 = get_df(df, "TM Phase 1 Invoicing Status".lower(), ["done", "to invoice"], "Lodgement Phase 1") +filtered_dfs.append(lodg1) + +# Full Lodgement Phase +full_lodgement = get_df(df, "Jun-te TM Phase 2 Invoicing Status".lower(), ["to invoice"], "Full Lodgement") +filtered_dfs.append(full_lodgement) + +# POST EPC +post_epc = get_df(df, "post-epc status", ["complete & uploaded"], "POST EPC") +filtered_dfs.append(post_epc) + + +# # POST EPR +# post_epr = df[ +# df["post-epc status"].str.lower().isin(["post epr completed"]) +# ].copy() +# post_epr["job_type"] = "POST ATT" +# filtered_dfs.append(post_epr) + +# Post ATT +post_att = get_df(df, "post-att status", ["complete & uploaded"], "POST ATT") +filtered_dfs.append(post_att) + + +# Retrofit Evaluation +retro = get_df(df, "retrofit evaluation", ["completed & uploaded"], "Retrofit Evaluation") +filtered_dfs.append(retro) + +# RA NO Show +ra_ns = df[ + df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) +].copy() +ra_ns["job_type"] = "RA NO SHOW" +filtered_dfs.append(ra_ns) + + +# ATT NO Show +att_ns = df[ + df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) +].copy() +att_ns["job_type"] = "ATT NO SHOW" +filtered_dfs.append(att_ns) + + +# Post visit no show +epc_ns = df[ + df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999) +].copy() +epc_ns["job_type"] = "EPC NO SHOW" +filtered_dfs.append(epc_ns) + +final_df = pd.concat(filtered_dfs).reset_index(drop=True) + +final_df[['address', 'client', 'job_type']] \ No newline at end of file diff --git a/etl/month_end_automation_wave_2_no_8.py b/etl/month_end_automation_wave_2_no_8.py new file mode 100644 index 0000000..ddd2368 --- /dev/null +++ b/etl/month_end_automation_wave_2_no_8.py @@ -0,0 +1,172 @@ +# Wave 2's month end automation +from tqdm import tqdm +from monday import MondayClient +from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids +from pprint import pprint +import pandas as pd +import json + +monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" +monday = MondayClient(monday_key) +# NCHA SHDF 2.1 SBS +board_ids = ["8668578700"] + + +for board in tqdm(board_ids): + board_data = monday.boards.fetch_boards_by_id(board) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} + reversed_col_id_map = {v: k for k, v in col_id_map.items()} + + + items = get_all_items(board, monday) + + all_records = [] + for row in tqdm(items): + data = {} + data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) + for col in row.get("column_values", []): + if col.get("id") in reversed_col_id_map: + if col.get("type") == "file": + value = col.get("value") + no_of_files = 0 + + if value: + value = json.loads(col["value"]) + no_of_files = len(value.get('files', [])) + data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + elif "no show" in reversed_col_id_map[col.get("id")]: + def extract_number_from_text(text): + number_str = '' + + for char in text: + if char.isnumeric(): + number_str += char + elif number_str: + break # stop once a number sequence ends + + return int(number_str) if number_str else None + text = col.get("text") + if text is None: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: extract_number_from_text(text) + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + all_records.append(data) + +# Convert to DataFrame +df = pd.DataFrame(all_records) + +filtered_dfs = [] + +def get_df(df, column_name, success_critera, job_name): + _ = df[ + df[column_name].str.lower().isin(success_critera) + ].copy() + _["job_type"] = job_name + return _ + + +# RA +ra = get_df(df, "ra", ["completed rdsap 9.9", "completed rdsap 10"], "RA") +filtered_dfs.append(ra) + + +# ATT +att = get_df(df, "att", ["completed"], "ATT") +filtered_dfs.append(att) + +# V1 Coordination +v1 = get_df(df, "v1 coordination status".lower(), [ + "rc complete", +], "V1 Coordination") +filtered_dfs.append(v1) + +# V2 Coordination +v2 = get_df(df, "v2 coordination status", ["rc v2 complete", "uploaded"], "V2 Coordination") +filtered_dfs.append(v2) + +# # V3 Coordination +# v3 = get_df(df, "") +# # filtered_dfs.append(v3) + +# Design stage 1 +design1 = get_df(df, "rd invoiced", ["to invoice"], "Design") +filtered_dfs.append(design1) + +# Design revision +# design2 = get_df(df, "design revision invoice", [ +# "Rev. A to invoice".lower(), +# "Rev. B to invoice".lower(), +# "Rev. C to invoice".lower(), +# "Rev. D to invoice".lower(), +# ], "Design Revision") +# filtered_dfs.append(design2) + +# Lodgement Phase 1 +lodg1 = get_df(df, "Lodg. Phase 1 Invoice Status".lower(), ["to be invoiced"], "Lodgement Phase 1") +filtered_dfs.append(lodg1) + +# Lodgement Phase 1 +lodg1 = get_df(df, "phase 1 to be invoiced".lower(), ["phase 1 to be invoiced"], "Lodgement Phase 1") +filtered_dfs.append(lodg1) + +# Full Lodgement Phase +full_lodgement = get_df(df, "full lodgement invoice status".lower(), ["to be invoice"], "Full Lodgement") +filtered_dfs.append(full_lodgement) + +# POST EPC +post_epc = get_df(df, "post epc status", ["uploaded"], "POST EPC") +filtered_dfs.append(post_epc) + + +# # POST EPR +# post_epr = df[ +# df["post-epc status"].str.lower().isin(["post epr completed"]) +# ].copy() +# post_epr["job_type"] = "POST ATT" +# filtered_dfs.append(post_epr) + +# Post ATT +post_att = get_df(df, "post att", ["post att uploaded"], "POST ATT") +filtered_dfs.append(post_att) + + +# Retrofit Evaluation +retro = get_df(df, "retrofit evaluation", ["done"], "Retrofit Evaluation") +filtered_dfs.append(retro) + +# RA NO Show +ra_ns = df[ + df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) +].copy() +ra_ns["job_type"] = "RA NO SHOW" +filtered_dfs.append(ra_ns) + + +# ATT NO Show +att_ns = df[ + df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) +].copy() +att_ns["job_type"] = "ATT NO SHOW" +filtered_dfs.append(att_ns) + + +# Post visit no show +# epc_ns = df[ +# df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999) +# ].copy() +# epc_ns["job_type"] = "EPC NO SHOW" +# filtered_dfs.append(epc_ns) + +final_df = pd.concat(filtered_dfs).reset_index(drop=True) + +final_df[['address', 'client', 'job_type']] \ No newline at end of file diff --git a/etl/month_end_automation_wave_2_no_9.py b/etl/month_end_automation_wave_2_no_9.py new file mode 100644 index 0000000..ca198f5 --- /dev/null +++ b/etl/month_end_automation_wave_2_no_9.py @@ -0,0 +1,169 @@ +# Wave 2's month end automation +from tqdm import tqdm +from monday import MondayClient +from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids +from pprint import pprint +import pandas as pd +import json + +monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" +monday = MondayClient(monday_key) +# NCHA Almshouses +board_ids = ["5423364294"] + + +for board in tqdm(board_ids): + board_data = monday.boards.fetch_boards_by_id(board) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} + reversed_col_id_map = {v: k for k, v in col_id_map.items()} + + + items = get_all_items(board, monday) + + all_records = [] + for row in tqdm(items): + data = {} + data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) + for col in row.get("column_values", []): + if col.get("id") in reversed_col_id_map: + if col.get("type") == "file": + value = col.get("value") + no_of_files = 0 + + if value: + value = json.loads(col["value"]) + no_of_files = len(value.get('files', [])) + data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + elif "no show" in reversed_col_id_map[col.get("id")]: + def extract_number_from_text(text): + number_str = '' + + for char in text: + if char.isnumeric(): + number_str += char + elif number_str: + break # stop once a number sequence ends + + return int(number_str) if number_str else None + text = col.get("text") + if text is None: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: extract_number_from_text(text) + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + all_records.append(data) + +# Convert to DataFrame +df = pd.DataFrame(all_records) + +filtered_dfs = [] + +def get_df(df, column_name, success_critera, job_name): + _ = df[ + df[column_name].str.lower().isin(success_critera) + ].copy() + _["job_type"] = job_name + return _ + + +# RA +ra = get_df(df, "ra", ["completed rdsap 9.9", "completed rdsap 10"], "RA") +filtered_dfs.append(ra) + + +# ATT +att = get_df(df, "att", ["completed"], "ATT") +filtered_dfs.append(att) + +# V1 Coordination +v1 = get_df(df, "coordination status (mtp)".lower(), [ + "rc complete", +], "V1 Coordination") +filtered_dfs.append(v1) + +# V2 Coordination +# v2 = get_df(df, "v2 coordination status", ["rc v2 complete", "uploaded"], "V2 Coordination") +# filtered_dfs.append(v2) + +# # V3 Coordination +# v3 = get_df(df, "") +# # filtered_dfs.append(v3) + +# Design stage 1 +# design1 = get_df(df, "rd invoiced", ["to invoice"], "Design") +# filtered_dfs.append(design1) + +# Design revision +# design2 = get_df(df, "design revision invoice", [ +# "Rev. A to invoice".lower(), +# "Rev. B to invoice".lower(), +# "Rev. C to invoice".lower(), +# "Rev. D to invoice".lower(), +# ], "Design Revision") +# filtered_dfs.append(design2) + +# Lodgement Phase 1 +# lodg1 = get_df(df, "Lodg. Phase 1 Invoice Status".lower(), ["to be invoiced"], "Lodgement Phase 1") +# filtered_dfs.append(lodg1) + + +# Full Lodgement Phase +full_lodgement = get_df(df, "trustmark lodgement".lower(), ["done"], "Full Lodgement") +filtered_dfs.append(full_lodgement) + +# POST EPC +# post_epc = get_df(df, "post epc status", ["uploaded"], "POST EPC") +# filtered_dfs.append(post_epc) + + +# # POST EPR +# post_epr = df[ +# df["post-epc status"].str.lower().isin(["post epr completed"]) +# ].copy() +# post_epr["job_type"] = "POST ATT" +# filtered_dfs.append(post_epr) + +# Post ATT +# post_att = get_df(df, "post att", ["post att uploaded"], "POST ATT") +# filtered_dfs.append(post_att) + + +# Retrofit Evaluation +retro = get_df(df, "retrofit evaluation", ["done"], "Retrofit Evaluation") +filtered_dfs.append(retro) + +# RA NO Show +ra_ns = df[ + df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) +].copy() +ra_ns["job_type"] = "RA NO SHOW" +filtered_dfs.append(ra_ns) + + +# ATT NO Show +att_ns = df[ + df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) +].copy() +att_ns["job_type"] = "ATT NO SHOW" +filtered_dfs.append(att_ns) + + +# Post visit no show +# epc_ns = df[ +# df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999) +# ].copy() +# epc_ns["job_type"] = "EPC NO SHOW" +# filtered_dfs.append(epc_ns) + +final_df = pd.concat(filtered_dfs).reset_index(drop=True) + +final_df[['address', 'client', 'job_type']] \ No newline at end of file