diff --git a/etl/month_end_automation_wave_2_no_14.py b/etl/month_end_automation_wave_2_no_14.py index 3a670b9..27cadd0 100644 --- a/etl/month_end_automation_wave_2_no_14.py +++ b/etl/month_end_automation_wave_2_no_14.py @@ -96,9 +96,9 @@ ra = get_df(df, "ra", ["completed rdsap 9.9", "completed rdsap 10"], "RA") filtered_dfs.append(ra) -# ATT -att = get_df(df, "att", ["completed"], "ATT") -filtered_dfs.append(att) +# # ATT +# att = get_df(df, "att", ["completed"], "ATT") +# filtered_dfs.append(att) # V1 Coordination v1 = get_df(df, "v1 coordination status (ioe,mtp)".lower(), [ diff --git a/etl/month_end_automation_wave_accent_housing.py b/etl/month_end_automation_wave_accent_housing.py new file mode 100644 index 0000000..e02cdd9 --- /dev/null +++ b/etl/month_end_automation_wave_accent_housing.py @@ -0,0 +1,104 @@ +# Wave 3's month end automation + +from tqdm import tqdm +from monday import MondayClient +from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids +from pprint import pprint +import pandas as pd +import json + +monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" +monday = MondayClient(monday_key) +board_ids = [ + "8829428746", # 2502 Accent Housing +] + +empty = "Rate card info missing" + +rate_card_data_2502_accent_housing = { + "job_type": [ + "First half of MTP", "Second half of MTP", "Full MTP" + ], + "rate": [ + 150, 130, 280 + ] +} + +rate_card_df = pd.DataFrame(rate_card_data_2502_accent_housing) + + +for board in tqdm(board_ids): + print(f"working on board {board}") + board_data = monday.boards.fetch_boards_by_id(board) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} + reversed_col_id_map = {v: k for k, v in col_id_map.items()} + + + items = get_all_items(board, monday) + + all_records = [] + for row in tqdm(items): + data = {} + data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) + for col in row.get("column_values", []): + if col.get("id") in reversed_col_id_map: + if col.get("type") == "file": + value = col.get("value") + no_of_files = 0 + + if value: + value = json.loads(col["value"]) + no_of_files = len(value.get('files', [])) + data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + else: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + all_records.append(data) + +# Convert to DataFrame +df = pd.DataFrame(all_records) + +filtered_dfs = [] + + +def get_df(df, column_name, success_critera, job_name=None): + _ = pd.DataFrame() + if column_name in col_id_map: + _ = df[ + df[column_name].str.lower().isin(success_critera) + ].copy() + if job_name: + _["job_type"] = job_name + + + return _ + + + +modeling = get_df(df, "mtp invoicing status", ["modelling to invoice"], "First half of mtp") +if not modeling.empty: + filtered_dfs.append(modeling) + +second_payment = get_df(df, "mtp invoicing status", ["(V1) Remainder IOE/MTP to invoice".lower()], "second half of mtp") +if not second_payment.empty: + filtered_dfs.append(second_payment) + +full_cost = get_df(df, "mtp invoicing status", ["(v1) full cost mtp to invoice (no previous modelling)"], "full cost mtp") +if not full_cost.empty: + filtered_dfs(full_cost) + +final_df = pd.concat(filtered_dfs).reset_index(drop=True) + +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_excel(f'2502 Accent housing {timestamp}.xlsx', index=False)