From 542a788a85309bf90ce02b6858206bd9af96a270 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 23 Jul 2025 13:08:50 +0000 Subject: [PATCH] month end automation --- etl/month_end_automation.py | 3 +- etl/month_end_automation_wave_3_layout.py | 197 ++++++++++++++++++++++ 2 files changed, 199 insertions(+), 1 deletion(-) create mode 100644 etl/month_end_automation_wave_3_layout.py diff --git a/etl/month_end_automation.py b/etl/month_end_automation.py index a3be693..69c91ba 100644 --- a/etl/month_end_automation.py +++ b/etl/month_end_automation.py @@ -9,6 +9,7 @@ import json monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) +# NCHK SHDF Westville Wave 1 & 2 board_ids = ["3900434153"] @@ -17,7 +18,7 @@ for board in tqdm(board_ids): columns = board_data["data"]["boards"][0]["columns"] col_id_map = {col["title"].lower(): col["id"] for col in columns} reversed_col_id_map = {v: k for k, v in col_id_map.items()} - + items = get_all_items(board, monday) diff --git a/etl/month_end_automation_wave_3_layout.py b/etl/month_end_automation_wave_3_layout.py new file mode 100644 index 0000000..2fd7ebe --- /dev/null +++ b/etl/month_end_automation_wave_3_layout.py @@ -0,0 +1,197 @@ +# Wave 2's month end automation + +from tqdm import tqdm +from monday import MondayClient +from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids +from pprint import pprint +import pandas as pd +import json + +monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" +monday = MondayClient(monday_key) +# WCHG Walkups-Operations +board_ids = ["9349630181"] + + +for board in tqdm(board_ids): + board_data = monday.boards.fetch_boards_by_id(board) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} + reversed_col_id_map = {v: k for k, v in col_id_map.items()} + + + items = get_all_items(board, monday) + + all_records = [] + for row in tqdm(items): + data = {} + data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) + for col in row.get("column_values", []): + if col.get("id") in reversed_col_id_map: + if col.get("type") == "file": + value = col.get("value") + no_of_files = 0 + + if value: + value = json.loads(col["value"]) + no_of_files = len(value.get('files', [])) + data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + # elif "no show" in reversed_col_id_map[col.get("id")]: + # def extract_number_from_text(text): + # number_str = '' + + # for char in text: + # if char.isnumeric(): + # number_str += char + # elif number_str: + # break # stop once a number sequence ends + + # return int(number_str) if number_str else None + # text = col.get("text") + # if text is None: + # data.update({ + # reversed_col_id_map[col.get("id")]: col.get("text") + # }) + # else: + # data.update({ + # reversed_col_id_map[col.get("id")]: extract_number_from_text(text) + # }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + all_records.append(data) + +# Convert to DataFrame +df = pd.DataFrame(all_records) + +filtered_dfs = [] + +# RA +ra = df[ + df["ra invoicing status"].str.lower().isin(["to invoice"]) +].copy() +ra["job_type"] = "RA" +filtered_dfs.append(ra) + + +# ATT +att = df[ + df["pre att invoicing status"].str.lower().isin(["to invoice"]) +].copy() +att["job_type"] = "ATT" +filtered_dfs.append(att) + +# V1 Coordination +v1 = df[ + df["mtp invoicing status"].str.lower().isin(["(v1) ioe/mtp to invoice"]) +].copy() + +v1["job_type"] = "V1 Coordination" +filtered_dfs.append(v1) + +# V2 Coordination +v2 = df[ + df["mtp invoicing status"].str.lower().isin(["(v2) ioe/mtp to invoice"]) +].copy() + +v2["job_type"] = "V2 Coordination" +filtered_dfs.append(v2) + +# V3 Coordination +v3 = df[ + df["mtp invoicing status"].str.lower().isin(["(v3) ioe/mtp to invoice"]) +].copy() + +v3["job_type"] = "V3 Coordination" +filtered_dfs.append(v3) + +# Design stage 1 +design1 = df[ + df["design invoicing status"].str.lower().isin(["to invoice"]) +].copy() +design1["job_type"] = "Design" +filtered_dfs.append(design1) + +# Design Revision +revision_letter = ['a', 'b', 'c', 'd'] +for letter in revision_letter: + design2 = df[ + df["design revision invoice"].str.lower().isin([f"rev. {letter} to invoice"]) + ].copy() + design2["job_type"] = "Design Revision" + filtered_dfs.append(design2) + +# Lodgement Phase 1 +lodg1 = df[ + df["lodgement phase 1 invoicing status"].str.lower().isin(["to invoice"]) +].copy() +lodg1["job_type"] = "Lodgement Phase 1" +filtered_dfs.append(lodg1) + +# Full Lodgement Phase +lodg2 = df[ + df["full lodgement invoicing status"].str.lower().isin(["to invoice"]) +].copy() +lodg2["job_type"] = "Full Lodgement" +filtered_dfs.append(lodg2) + +# POST EPC +post_epc = df[ + df["post epc & eval. invoicing status"].str.lower().isin(["to invoice"]) +].copy() +post_epc["job_type"] = "POST EPC" +filtered_dfs.append(post_epc) + + +# # POST EPR, not right now +# post_epr = df[ +# df["post-epc status"].str.lower().isin(["post epr completed"]) +# ].copy() +# post_epr["job_type"] = "POST ATT" +# filtered_dfs.append(post_epr) + + +# Post ATT +post_att = df[ + df["post att invoicing status"].str.lower().isin(["to invoice"]) +].copy() +post_att["job_type"] = "POST ATT" +filtered_dfs.append(post_att) + + +# Retrofit Evaluation +retro = df[ + df["rc stage 2 invoicing status"].str.lower().isin(["to invoice"]) +].copy() +retro["job_type"] = "RC stage 2" +filtered_dfs.append(retro) + +# RA NO Show +ra_ns = df[ + df["ra no show invoice"].str.lower().isin(["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no show)"]) +].copy() +ra_ns["job_type"] = "RA NO SHOW" +filtered_dfs.append(ra_ns) + + +# ATT NO Show +att_ns = df[ + df["pre att no show invoice"].str.lower().isin(["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no show)"]) +].copy() +att_ns["job_type"] = "ATT NO SHOW" +filtered_dfs.append(att_ns) + + +# Post visit no show +#post work no show invoice +epc_ns = df[ + df["post works no show invoice"].str.lower().isin(["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no show)"]) +].copy() +epc_ns["job_type"] = "EPC NO SHOW" +filtered_dfs.append(epc_ns) + +final_df = pd.concat(filtered_dfs).drop_duplicates().reset_index(drop=True) + +final_df[['address', 'client', 'job_type']] \ No newline at end of file