diff --git a/etl/month_end_automation.py b/etl/month_end_automation.py index d396551..2eb9e68 100644 --- a/etl/month_end_automation.py +++ b/etl/month_end_automation.py @@ -1,3 +1,5 @@ +# Wave 2's month end automation + from tqdm import tqdm from monday import MondayClient from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids @@ -23,6 +25,7 @@ for board in tqdm(board_ids): for row in tqdm(items): data = {} data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) for col in row.get("column_values", []): if col.get("id") in reversed_col_id_map: if col.get("type") == "file": @@ -33,8 +36,27 @@ for board in tqdm(board_ids): value = json.loads(col["value"]) no_of_files = len(value.get('files', [])) data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + elif "no show" in reversed_col_id_map[col.get("id")]: + def extract_number_from_text(text): + number_str = '' + + for char in text: + if char.isnumeric(): + number_str += char + elif number_str: + break # stop once a number sequence ends + + return int(number_str) if number_str else None + text = col.get("text") + if text is None: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: extract_number_from_text(text) + }) else: - data.update({ reversed_col_id_map[col.get("id")]: col.get("text") }) @@ -43,22 +65,97 @@ for board in tqdm(board_ids): # Convert to DataFrame df = pd.DataFrame(all_records) +filtered_dfs = [] + # RA -filtered_df = df[ +ra = df[ df["ra"].str.lower().isin(["completed rdsap 10", "completed rdsap 9.9"]) ].copy() +filtered_dfs.append(ra) + # ATT -filtered_df = df[ +att = df[ df["att"].str.lower().isin(["completed"]) ].copy() +filtered_dfs.append(att) # V1 Coordination -filtered_df = df[ +v1 = df[ df["v1 coordination status"].str.lower().isin(["rc complete"]) ].copy() +filtered_dfs.append(v1) # V2 Coordination _ = df[df["v2 invoiced"].fillna('').str.lower().isin(['to be invoiced'])] -filtered_df = _[_["v2 dc/ima/pas"] > 0] +v2 = _[_["v2 dc/ima/pas"] > 0] +filtered_dfs.append(v2) + +# V3 Coordination +v3 = df[ + df["v3 invoiced"].str.lower().isin(["to be invoiced"]) +].copy() +filtered_dfs.append(v3) + +# Design stage 1 +design1 = df[ + df["design invoice status"].str.lower().isin(["to invoice"]) +] +filtered_dfs.append(design1) + +# Design stage 2 +design2 = df[ + df["revision 2 design invoice"].str.lower().isin(["to invoice"]) +] +filtered_dfs.append(design2) + +# Lodgement Phase 1 +lodg1 = df[ + df["lodg. phase 1 invoice status"].str.lower().isin(["to invoice"]) +] +filtered_dfs.append(lodg1) + +# Full Lodgement Phase +lodg2 = df[ + df["full lodgement invoice status"].str.lower().isin(["to invoice"]) +] +filtered_dfs.append(lodg2) + +# POST EPC +# !!! Need Kevins input ( how does he determine invoice in this one) + +# Post ATT +# !!! Check with kevin, if he uses epc pic uploaded ( to invoice ) or uses anything else +filtered_df = df[ + df["post-att"].str.lower().isin(["post-att uploaded"]) +] + +# Retrofit Evaluation +retro = df[ + df["retrofit evaluation"].str.lower().isin(["complete"]) +] +filtered_dfs.append(retro) + +# RA NO Show +ra_ns = df[ + df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) +] +filtered_dfs.append(ra_ns) + + + +# ATT NO Show +att_ns = df[ + df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) +] +filtered_dfs.append(att_ns) + + +# Post visit no show +epc_ns = df[ + df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999) +] +filtered_dfs.append(epc_ns) + +final_df = pd.concat(filtered_dfs).drop_duplicates().reset_index(drop=True)