completed

etl/month_end_automation_wave_2_no_10.py

@@ -0,0 +1,170 @@
+# Wave 2's month end automation
+from tqdm import tqdm
+from monday import MondayClient
+from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
+from pprint import pprint
+import pandas as pd
+import json
+
+monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
+monday = MondayClient(monday_key)
+# NCHA Derbyshire Dales (DDDCC) SHDF
+board_ids = ["6947307148"]
+
+for board in tqdm(board_ids):
+    board_data = monday.boards.fetch_boards_by_id(board)
+    columns = board_data["data"]["boards"][0]["columns"]
+    col_id_map = {col["title"].lower(): col["id"] for col in columns}
+    reversed_col_id_map = {v: k for k, v in col_id_map.items()}
+    
+    
+    items = get_all_items(board, monday)
+
+    all_records = []
+    for row in tqdm(items):
+        data = {}
+        data.update({"address": row['name']})
+        data.update({"client": row['group']['title']})
+        for col in row.get("column_values", []):
+            if col.get("id") in reversed_col_id_map:
+                if col.get("type") == "file":
+                    value = col.get("value")
+                    no_of_files = 0
+
+                    if value:
+                        value = json.loads(col["value"])
+                        no_of_files = len(value.get('files', []))
+                    data.update({reversed_col_id_map[col.get("id")]: no_of_files})
+                elif "no show" in reversed_col_id_map[col.get("id")]:
+                    def extract_number_from_text(text):
+                        number_str = ''
+                        
+                        for char in text:
+                            if char.isnumeric():
+                                number_str += char
+                            elif number_str:
+                                break  # stop once a number sequence ends
+
+                        return int(number_str) if number_str else None
+                    text = col.get("text")
+                    if text is None:
+                        data.update({
+                            reversed_col_id_map[col.get("id")]: col.get("text")
+                        })    
+                    else:
+                        data.update({
+                            reversed_col_id_map[col.get("id")]: extract_number_from_text(text)
+                        })    
+                else:
+                    data.update({
+                        reversed_col_id_map[col.get("id")]: col.get("text")
+                    })
+        all_records.append(data)
+
+# Convert to DataFrame
+df = pd.DataFrame(all_records)
+
+filtered_dfs = []
+
+def get_df(df, column_name, success_critera, job_name):
+    _ = df[
+        df[column_name].str.lower().isin(success_critera)
+    ].copy()
+    _["job_type"] = job_name
+    return _
+
+
+# RA
+ra = get_df(df, "ra", ["completed rdsap 9.9", "completed rd sap 10"], "RA")
+filtered_dfs.append(ra)
+
+
+# ATT
+att = get_df(df, "att", ["completed"], "ATT")
+filtered_dfs.append(att)
+
+# V1 Coordination
+v1 = get_df(df, "coordination status".lower(), [
+    "rc complete",
+], "V1 Coordination")
+filtered_dfs.append(v1)
+
+# V2 Coordination
+# v2 = get_df(df, "v2 coordination status", ["rc v2 complete", "uploaded"], "V2 Coordination")
+# filtered_dfs.append(v2)
+
+# # V3 Coordination
+# v3 = get_df(df, "")
+# # filtered_dfs.append(v3)
+
+# Design stage 1
+design1 = get_df(df, "design upload to sharepoint", ["done"], "Design")
+filtered_dfs.append(design1)
+
+# Design revision
+# design2 = get_df(df, "design revision invoice", [
+#     "Rev. A to invoice".lower(),
+#     "Rev. B to invoice".lower(),
+#     "Rev. C to invoice".lower(),
+#     "Rev. D to invoice".lower(),
+# ], "Design Revision")
+# filtered_dfs.append(design2)
+
+# Lodgement Phase 1
+lodg1 = get_df(df, "Lodg. Phase 1 Invoice Status".lower(), ["to invoice"], "Lodgement Phase 1")
+filtered_dfs.append(lodg1)
+
+
+# Full Lodgement Phase
+full_lodgement = get_df(df, "full lodgement invoice status".lower(), ["to invoice"], "Full Lodgement")
+filtered_dfs.append(full_lodgement)
+
+# POST EPC
+post_epc = get_df(df, "lodged epc", ["complete"], "POST EPC")
+filtered_dfs.append(post_epc)
+
+
+# # POST EPR
+# post_epr = df[
+#     df["post-epc status"].str.lower().isin(["post epr completed"])
+# ].copy()
+# post_epr["job_type"] = "POST ATT"
+# filtered_dfs.append(post_epr)
+
+# Post ATT
+post_att = get_df(df, "post att", ["done"], "POST ATT")
+filtered_dfs.append(post_att)
+
+post_att = get_df(df, "post-test status", ["complete"], "POST ATT")
+filtered_dfs.append(post_att)
+
+# Retrofit Evaluation
+retro = get_df(df, "retrofit evaluation", ["done"], "Retrofit Evaluation")
+filtered_dfs.append(retro)
+
+# RA NO Show
+ra_ns  = df[
+    df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)
+].copy()
+ra_ns["job_type"] = "RA NO SHOW"
+filtered_dfs.append(ra_ns)
+
+
+# ATT NO Show
+att_ns = df[
+    df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)
+].copy()
+att_ns["job_type"] = "ATT NO SHOW"
+filtered_dfs.append(att_ns)
+
+
+# Post visit no show
+# epc_ns = df[
+#     df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999)
+# ].copy()
+# epc_ns["job_type"] = "EPC NO SHOW"
+# filtered_dfs.append(epc_ns)
+
+final_df = pd.concat(filtered_dfs).reset_index(drop=True)
+
+final_df[['address', 'client', 'job_type']]

etl/month_end_automation_wave_2_no_4.py

@@ -123,17 +123,14 @@ filtered_dfs.append(lodg1)
 
 # Full Lodgement Phase
 _ = df[
-    df["lodgement status"].str.lower().isin(["lodged"])
+    df["lodgement invoice status"].str.lower().isin(["to invoice"])
 ].copy()
-lodg2 = _[
-    _["lodgement invoice status"].str.lower().isin([None])
-].copy()
-lodg2["job_type"] = "Full Lodgement"
-filtered_dfs.append(lodg2)
+_["job_type"] = "Full Lodgement"
+filtered_dfs.append(_)
 
 # POST EPC
 post_epc = df[
-    df["post epc"].str.lower().isin(["success", "pics uploaded"])
+    df["post epc"].str.lower().isin(["completed & uploaded"])
 ].copy()
 post_epc["job_type"] = "POST EPC"
 filtered_dfs.append(post_epc)
@@ -146,11 +143,9 @@ filtered_dfs.append(post_epc)
 # post_epr["job_type"] = "POST ATT"
 # filtered_dfs.append(post_epr)
 
-
-
 # Post ATT
 post_att = df[
-    df["post att"].str.lower().isin(["uploaded"])
+    df["post att"].str.lower().isin(["completed & uploaded"])
 ].copy()
 post_att["job_type"] = "POST ATT"
 filtered_dfs.append(post_att)
@@ -158,7 +153,7 @@ filtered_dfs.append(post_att)
 
 # Retrofit Evaluation
 retro = df[
-    df["retrofit evaluation"].str.lower().isin(["uploaded", "completed", "to invoice"])
+    df["retrofit evaluation"].str.lower().isin(["completed & uploaded"])
 ].copy()
 retro["job_type"] = "Retrofit Evaluation"
 filtered_dfs.append(retro)
@@ -171,7 +166,6 @@ ra_ns["job_type"] = "RA NO SHOW"
 filtered_dfs.append(ra_ns)
 
 
-
 # ATT NO Show
 att_ns = df[
     df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)

etl/month_end_automation_wave_2_no_5.py

@@ -0,0 +1,186 @@
+# Wave 2's month end automation
+
+from tqdm import tqdm
+from monday import MondayClient
+from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
+from pprint import pprint
+import pandas as pd
+import json
+
+monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
+monday = MondayClient(monday_key)
+# ECO 4 NCHA Almshouses Operations
+board_ids = ["9136254638"]
+
+
+for board in tqdm(board_ids):
+    board_data = monday.boards.fetch_boards_by_id(board)
+    columns = board_data["data"]["boards"][0]["columns"]
+    col_id_map = {col["title"].lower(): col["id"] for col in columns}
+    reversed_col_id_map = {v: k for k, v in col_id_map.items()}
+    
+    
+    items = get_all_items(board, monday)
+
+    all_records = []
+    for row in tqdm(items):
+        data = {}
+        data.update({"address": row['name']})
+        data.update({"client": row['group']['title']})
+        for col in row.get("column_values", []):
+            if col.get("id") in reversed_col_id_map:
+                if col.get("type") == "file":
+                    value = col.get("value")
+                    no_of_files = 0
+
+                    if value:
+                        value = json.loads(col["value"])
+                        no_of_files = len(value.get('files', []))
+                    data.update({reversed_col_id_map[col.get("id")]: no_of_files})
+                elif "no show" in reversed_col_id_map[col.get("id")]:
+                    def extract_number_from_text(text):
+                        number_str = ''
+                        
+                        for char in text:
+                            if char.isnumeric():
+                                number_str += char
+                            elif number_str:
+                                break  # stop once a number sequence ends
+
+                        return int(number_str) if number_str else None
+                    text = col.get("text")
+                    if text is None:
+                        data.update({
+                            reversed_col_id_map[col.get("id")]: col.get("text")
+                        })    
+                    else:
+                        data.update({
+                            reversed_col_id_map[col.get("id")]: extract_number_from_text(text)
+                        })    
+                else:
+                    data.update({
+                        reversed_col_id_map[col.get("id")]: col.get("text")
+                    })
+        all_records.append(data)
+
+# Convert to DataFrame
+df = pd.DataFrame(all_records)
+
+filtered_dfs = []
+
+# RA
+ra = df[
+    df["retrofit assessment"].str.lower().isin(["completed rdsap 10", "completed rdsap 9.9"])
+].copy()
+ra["job_type"] = "RA"
+filtered_dfs.append(ra)
+
+
+# ATT
+att = df[
+    df["pre att"].str.lower().isin(["completed"])
+].copy()
+att["job_type"] = "ATT"
+filtered_dfs.append(att)
+
+# V1 Coordination
+v1 = df[
+    df["coordination status"].str.lower().isin(["ioe/mtp complete"])
+].copy()
+v1["job_type"] = "V1 Coordination"
+filtered_dfs.append(v1)
+
+# V2 Coordination
+# _ = df[df["mtp v2 invoiced"].str.lower().isin(['done', 'needs to be invoiced'])].copy()
+# _["job_type"] = "V2 Coordination"
+# filtered_dfs.append(_)
+
+# V3 Coordination
+# v3 = df[df["v3 rc status"].str.lower().isin(['uploaded'])].copy()
+# v3["job_type"] = "V3 Coordination"
+# filtered_dfs.append(_)
+
+# Design stage 1
+design1 = df[
+    df["retrofit design status"].str.lower().isin(["to invoice"])
+].copy()
+design1["job_type"] = "Design"
+filtered_dfs.append(design1)
+
+# Design stage 2
+# design2 = df[
+#     df[""].str.lower().isin(["to invoice"])
+# ].copy()
+# design2["job_type"] = "Design Revision"
+# filtered_dfs.append(design2)
+
+# Lodgement Phase 1
+# lodg1 = df[
+#     df["phase 1 invoice status (lodgement)"].str.lower().isin(["done", "to be invoiced"])
+# ].copy()
+# lodg1["job_type"] = "Lodgement Phase 1"
+# filtered_dfs.append(lodg1)
+
+# Full Lodgement Phase
+_ = df[
+    df["trustmark lodgement"].str.lower().isin(["done"])
+].copy()
+_["job_type"] = "Full Lodgement"
+filtered_dfs.append(_)
+
+# POST EPC
+post_epc = df[
+    df["post epc status"].str.lower().isin(["done"])
+].copy()
+post_epc["job_type"] = "POST EPC"
+filtered_dfs.append(post_epc)
+
+
+# # POST EPR
+# post_epr = df[
+#     df["post-epc status"].str.lower().isin(["post epr completed"])
+# ].copy()
+# post_epr["job_type"] = "POST ATT"
+# filtered_dfs.append(post_epr)
+
+# Post ATT
+post_att = df[
+    df["post att status"].str.lower().isin(["done"])
+].copy()
+post_att["job_type"] = "POST ATT"
+filtered_dfs.append(post_att)
+
+
+# Retrofit Evaluation
+retro = df[
+    df["retrofit evaluation"].str.lower().isin(["done"])
+].copy()
+retro["job_type"] = "Retrofit Evaluation"
+filtered_dfs.append(retro)
+
+# RA NO Show
+ra_ns  = df[
+    df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)
+].copy()
+ra_ns["job_type"] = "RA NO SHOW"
+filtered_dfs.append(ra_ns)
+
+
+# ATT NO Show
+att_ns = df[
+    df["pre att no show evidence"].fillna(-9999) != df["pre att no show invoice"].fillna(-9999)
+].copy()
+att_ns["job_type"] = "ATT NO SHOW"
+filtered_dfs.append(att_ns)
+
+
+# Post visit no show
+epc_ns = df[
+    df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999)
+].copy()
+epc_ns["job_type"] = "EPC NO SHOW"
+filtered_dfs.append(epc_ns)
+
+final_df = pd.concat(filtered_dfs).reset_index(drop=True)
+
+final_df[['address', 'client', 'job_type']]

etl/month_end_automation_wave_2_no_6.py

@@ -0,0 +1,175 @@
+# Wave 2's month end automation
+
+from tqdm import tqdm
+from monday import MondayClient
+from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
+from pprint import pprint
+import pandas as pd
+import json
+
+monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
+monday = MondayClient(monday_key)
+#ECO 4 Wates Operations
+board_ids = ["9520779048"]
+
+
+for board in tqdm(board_ids):
+    board_data = monday.boards.fetch_boards_by_id(board)
+    columns = board_data["data"]["boards"][0]["columns"]
+    col_id_map = {col["title"].lower(): col["id"] for col in columns}
+    reversed_col_id_map = {v: k for k, v in col_id_map.items()}
+    
+    
+    items = get_all_items(board, monday)
+
+    all_records = []
+    for row in tqdm(items):
+        data = {}
+        data.update({"address": row['name']})
+        data.update({"client": row['group']['title']})
+        for col in row.get("column_values", []):
+            if col.get("id") in reversed_col_id_map:
+                if col.get("type") == "file":
+                    value = col.get("value")
+                    no_of_files = 0
+
+                    if value:
+                        value = json.loads(col["value"])
+                        no_of_files = len(value.get('files', []))
+                    data.update({reversed_col_id_map[col.get("id")]: no_of_files})
+                elif "no show" in reversed_col_id_map[col.get("id")]:
+                    def extract_number_from_text(text):
+                        number_str = ''
+                        
+                        for char in text:
+                            if char.isnumeric():
+                                number_str += char
+                            elif number_str:
+                                break  # stop once a number sequence ends
+
+                        return int(number_str) if number_str else None
+                    text = col.get("text")
+                    if text is None:
+                        data.update({
+                            reversed_col_id_map[col.get("id")]: col.get("text")
+                        })    
+                    else:
+                        data.update({
+                            reversed_col_id_map[col.get("id")]: extract_number_from_text(text)
+                        })    
+                else:
+                    data.update({
+                        reversed_col_id_map[col.get("id")]: col.get("text")
+                    })
+        all_records.append(data)
+
+# Convert to DataFrame
+df = pd.DataFrame(all_records)
+
+filtered_dfs = []
+
+def get_df(df, column_name, success_critera, job_name):
+    _ = df[
+        df[column_name].str.lower().isin(success_critera)
+    ].copy()
+    _["job_type"] = job_name
+    return _
+
+# RA
+ra = get_df(df, "ra status", ["completed & uploaded"], "RA")
+filtered_dfs.append(ra)
+
+
+# ATT
+att = get_df(df, "pre att status", ["completed & uploaded"], "ATT")
+filtered_dfs.append(att)
+
+# V1 Coordination
+v1 = get_df(df, "Coordination Status IOE/MTP".lower(), [
+    "(V1) IOE/MTP Complete".lower(),
+    "(V2) IOE/MTP Complete".lower(),
+    "(V3) IOE/MTP Complete".lower(),
+], "V1 Coordination")
+filtered_dfs.append(v1)
+
+# V2 Coordination
+# _ = df[df["mtp v2 invoiced"].str.lower().isin(['done', 'needs to be invoiced'])].copy()
+# _["job_type"] = "V2 Coordination"
+# filtered_dfs.append(_)
+
+# V3 Coordination
+# v3 = df[df["v3 rc status"].str.lower().isin(['uploaded'])].copy()
+# v3["job_type"] = "V3 Coordination"
+# filtered_dfs.append(_)
+
+# Design stage 1
+design1 = get_df(df, "retrofit design status", ["completed"], "Design")
+filtered_dfs.append(design1)
+
+# Design stage 2
+design2 = get_df(df, "design revision invoice", [
+    "Rev. A to invoice".lower(),
+    "Rev. B to invoice".lower(),
+    "Rev. C to invoice".lower(),
+    "Rev. D to invoice".lower(),
+], "Design Revision")
+filtered_dfs.append(design2)
+
+# Lodgement Phase 1
+# lodg1 = df[
+#     df["phase 1 invoice status (lodgement)"].str.lower().isin(["done", "to be invoiced"])
+# ].copy()
+# lodg1["job_type"] = "Lodgement Phase 1"
+# filtered_dfs.append(lodg1)
+
+# Full Lodgement Phase
+full_lodgement = get_df(df, "full lodgement", ["completed"], "Full Lodgement")
+filtered_dfs.append(full_lodgement)
+
+# POST EPC
+post_epc = get_df(df, "post epc & evaluation status", ["uploaded"], "POST EPC")
+filtered_dfs.append(post_epc)
+
+
+# # POST EPR
+# post_epr = df[
+#     df["post-epc status"].str.lower().isin(["post epr completed"])
+# ].copy()
+# post_epr["job_type"] = "POST ATT"
+# filtered_dfs.append(post_epr)
+
+# Post ATT
+post_att = get_df(df, "post att status", ["uploaded"], "POST ATT")
+filtered_dfs.append(post_att)
+
+
+# Retrofit Evaluation
+retro = get_df(df, "retrofit evaluation", ["done"], "Retrofit Evaluation")
+filtered_dfs.append(retro)
+
+# RA NO Show
+# ra_ns  = df[
+#     df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)
+# ].copy()
+# ra_ns["job_type"] = "RA NO SHOW"
+# filtered_dfs.append(ra_ns)
+
+
+# # ATT NO Show
+# att_ns = df[
+#     df["pre att no show evidence"].fillna(-9999) != df["pre att no show invoice"].fillna(-9999)
+# ].copy()
+# att_ns["job_type"] = "ATT NO SHOW"
+# filtered_dfs.append(att_ns)
+
+
+# # Post visit no show
+# epc_ns = df[
+#     df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999)
+# ].copy()
+# epc_ns["job_type"] = "EPC NO SHOW"
+# filtered_dfs.append(epc_ns)
+
+final_df = pd.concat(filtered_dfs).reset_index(drop=True)
+
+final_df[['address', 'client', 'job_type']]

etl/month_end_automation_wave_2_no_7.py

@@ -0,0 +1,169 @@
+# Wave 2's month end automation
+
+from tqdm import tqdm
+from monday import MondayClient
+from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
+from pprint import pprint
+import pandas as pd
+import json
+
+monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
+monday = MondayClient(monday_key)
+#Home Group Wave 2SP+
+board_ids = ["4254419092"]
+
+
+for board in tqdm(board_ids):
+    board_data = monday.boards.fetch_boards_by_id(board)
+    columns = board_data["data"]["boards"][0]["columns"]
+    col_id_map = {col["title"].lower(): col["id"] for col in columns}
+    reversed_col_id_map = {v: k for k, v in col_id_map.items()}
+    
+    
+    items = get_all_items(board, monday)
+
+    all_records = []
+    for row in tqdm(items):
+        data = {}
+        data.update({"address": row['name']})
+        data.update({"client": row['group']['title']})
+        for col in row.get("column_values", []):
+            if col.get("id") in reversed_col_id_map:
+                if col.get("type") == "file":
+                    value = col.get("value")
+                    no_of_files = 0
+
+                    if value:
+                        value = json.loads(col["value"])
+                        no_of_files = len(value.get('files', []))
+                    data.update({reversed_col_id_map[col.get("id")]: no_of_files})
+                elif "no show" in reversed_col_id_map[col.get("id")]:
+                    def extract_number_from_text(text):
+                        number_str = ''
+                        
+                        for char in text:
+                            if char.isnumeric():
+                                number_str += char
+                            elif number_str:
+                                break  # stop once a number sequence ends
+
+                        return int(number_str) if number_str else None
+                    text = col.get("text")
+                    if text is None:
+                        data.update({
+                            reversed_col_id_map[col.get("id")]: col.get("text")
+                        })    
+                    else:
+                        data.update({
+                            reversed_col_id_map[col.get("id")]: extract_number_from_text(text)
+                        })    
+                else:
+                    data.update({
+                        reversed_col_id_map[col.get("id")]: col.get("text")
+                    })
+        all_records.append(data)
+
+# Convert to DataFrame
+df = pd.DataFrame(all_records)
+
+filtered_dfs = []
+
+def get_df(df, column_name, success_critera, job_name):
+    _ = df[
+        df[column_name].str.lower().isin(success_critera)
+    ].copy()
+    _["job_type"] = job_name
+    return _
+
+
+# RA
+ra = get_df(df, "ra", ["completed rdsap 9.9", "completed rdsap 10"], "RA")
+filtered_dfs.append(ra)
+
+
+# ATT
+att = get_df(df, "pre-att", ["completed"], "ATT")
+filtered_dfs.append(att)
+
+# V1 Coordination
+v1 = get_df(df, "osmosis  rc status".lower(), [
+    "rc completed",
+], "V1 Coordination")
+filtered_dfs.append(v1)
+
+# V2 Coordination
+v2 = get_df(df, "v2 ioe mtp", ["completed"], "V2 Coordination")
+filtered_dfs.append(v2)
+
+# # V3 Coordination
+# v3 = get_df(df, "")
+# # filtered_dfs.append(v3)
+
+# Design stage 1
+design1 = get_df(df, "design invoice status", ["to invoice"], "Design")
+filtered_dfs.append(design1)
+
+# Design revision
+# design2 = get_df(df, "design revision invoice", [
+#     "Rev. A to invoice".lower(),
+#     "Rev. B to invoice".lower(),
+#     "Rev. C to invoice".lower(),
+#     "Rev. D to invoice".lower(),
+# ], "Design Revision")
+# filtered_dfs.append(design2)
+
+# Lodgement Phase 1
+lodg1 = get_df(df, "TM Phase 1 Invoicing Status".lower(), ["done", "to invoice"], "Lodgement Phase 1")
+filtered_dfs.append(lodg1)
+
+# Full Lodgement Phase
+full_lodgement = get_df(df, "Jun-te TM Phase 2 Invoicing Status".lower(), ["to invoice"], "Full Lodgement")
+filtered_dfs.append(full_lodgement)
+
+# POST EPC
+post_epc = get_df(df, "post-epc status", ["complete & uploaded"], "POST EPC")
+filtered_dfs.append(post_epc)
+
+
+# # POST EPR
+# post_epr = df[
+#     df["post-epc status"].str.lower().isin(["post epr completed"])
+# ].copy()
+# post_epr["job_type"] = "POST ATT"
+# filtered_dfs.append(post_epr)
+
+# Post ATT
+post_att = get_df(df, "post-att status", ["complete & uploaded"], "POST ATT")
+filtered_dfs.append(post_att)
+
+
+# Retrofit Evaluation
+retro = get_df(df, "retrofit evaluation", ["completed & uploaded"], "Retrofit Evaluation")
+filtered_dfs.append(retro)
+
+# RA NO Show
+ra_ns  = df[
+    df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)
+].copy()
+ra_ns["job_type"] = "RA NO SHOW"
+filtered_dfs.append(ra_ns)
+
+
+# ATT NO Show
+att_ns = df[
+    df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)
+].copy()
+att_ns["job_type"] = "ATT NO SHOW"
+filtered_dfs.append(att_ns)
+
+
+# Post visit no show
+epc_ns = df[
+    df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999)
+].copy()
+epc_ns["job_type"] = "EPC NO SHOW"
+filtered_dfs.append(epc_ns)
+
+final_df = pd.concat(filtered_dfs).reset_index(drop=True)
+
+final_df[['address', 'client', 'job_type']]

etl/month_end_automation_wave_2_no_8.py

@@ -0,0 +1,172 @@
+# Wave 2's month end automation
+from tqdm import tqdm
+from monday import MondayClient
+from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
+from pprint import pprint
+import pandas as pd
+import json
+
+monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
+monday = MondayClient(monday_key)
+# NCHA SHDF 2.1 SBS
+board_ids = ["8668578700"]
+
+
+for board in tqdm(board_ids):
+    board_data = monday.boards.fetch_boards_by_id(board)
+    columns = board_data["data"]["boards"][0]["columns"]
+    col_id_map = {col["title"].lower(): col["id"] for col in columns}
+    reversed_col_id_map = {v: k for k, v in col_id_map.items()}
+    
+    
+    items = get_all_items(board, monday)
+
+    all_records = []
+    for row in tqdm(items):
+        data = {}
+        data.update({"address": row['name']})
+        data.update({"client": row['group']['title']})
+        for col in row.get("column_values", []):
+            if col.get("id") in reversed_col_id_map:
+                if col.get("type") == "file":
+                    value = col.get("value")
+                    no_of_files = 0
+
+                    if value:
+                        value = json.loads(col["value"])
+                        no_of_files = len(value.get('files', []))
+                    data.update({reversed_col_id_map[col.get("id")]: no_of_files})
+                elif "no show" in reversed_col_id_map[col.get("id")]:
+                    def extract_number_from_text(text):
+                        number_str = ''
+                        
+                        for char in text:
+                            if char.isnumeric():
+                                number_str += char
+                            elif number_str:
+                                break  # stop once a number sequence ends
+
+                        return int(number_str) if number_str else None
+                    text = col.get("text")
+                    if text is None:
+                        data.update({
+                            reversed_col_id_map[col.get("id")]: col.get("text")
+                        })    
+                    else:
+                        data.update({
+                            reversed_col_id_map[col.get("id")]: extract_number_from_text(text)
+                        })    
+                else:
+                    data.update({
+                        reversed_col_id_map[col.get("id")]: col.get("text")
+                    })
+        all_records.append(data)
+
+# Convert to DataFrame
+df = pd.DataFrame(all_records)
+
+filtered_dfs = []
+
+def get_df(df, column_name, success_critera, job_name):
+    _ = df[
+        df[column_name].str.lower().isin(success_critera)
+    ].copy()
+    _["job_type"] = job_name
+    return _
+
+
+# RA
+ra = get_df(df, "ra", ["completed rdsap 9.9", "completed rdsap 10"], "RA")
+filtered_dfs.append(ra)
+
+
+# ATT
+att = get_df(df, "att", ["completed"], "ATT")
+filtered_dfs.append(att)
+
+# V1 Coordination
+v1 = get_df(df, "v1 coordination status".lower(), [
+    "rc complete",
+], "V1 Coordination")
+filtered_dfs.append(v1)
+
+# V2 Coordination
+v2 = get_df(df, "v2 coordination status", ["rc v2 complete", "uploaded"], "V2 Coordination")
+filtered_dfs.append(v2)
+
+# # V3 Coordination
+# v3 = get_df(df, "")
+# # filtered_dfs.append(v3)
+
+# Design stage 1
+design1 = get_df(df, "rd invoiced", ["to invoice"], "Design")
+filtered_dfs.append(design1)
+
+# Design revision
+# design2 = get_df(df, "design revision invoice", [
+#     "Rev. A to invoice".lower(),
+#     "Rev. B to invoice".lower(),
+#     "Rev. C to invoice".lower(),
+#     "Rev. D to invoice".lower(),
+# ], "Design Revision")
+# filtered_dfs.append(design2)
+
+# Lodgement Phase 1
+lodg1 = get_df(df, "Lodg. Phase 1 Invoice Status".lower(), ["to be invoiced"], "Lodgement Phase 1")
+filtered_dfs.append(lodg1)
+
+# Lodgement Phase 1
+lodg1 = get_df(df, "phase 1 to be invoiced".lower(), ["phase 1 to be invoiced"], "Lodgement Phase 1")
+filtered_dfs.append(lodg1)
+
+# Full Lodgement Phase
+full_lodgement = get_df(df, "full lodgement invoice status".lower(), ["to be invoice"], "Full Lodgement")
+filtered_dfs.append(full_lodgement)
+
+# POST EPC
+post_epc = get_df(df, "post epc status", ["uploaded"], "POST EPC")
+filtered_dfs.append(post_epc)
+
+
+# # POST EPR
+# post_epr = df[
+#     df["post-epc status"].str.lower().isin(["post epr completed"])
+# ].copy()
+# post_epr["job_type"] = "POST ATT"
+# filtered_dfs.append(post_epr)
+
+# Post ATT
+post_att = get_df(df, "post att", ["post att uploaded"], "POST ATT")
+filtered_dfs.append(post_att)
+
+
+# Retrofit Evaluation
+retro = get_df(df, "retrofit evaluation", ["done"], "Retrofit Evaluation")
+filtered_dfs.append(retro)
+
+# RA NO Show
+ra_ns  = df[
+    df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)
+].copy()
+ra_ns["job_type"] = "RA NO SHOW"
+filtered_dfs.append(ra_ns)
+
+
+# ATT NO Show
+att_ns = df[
+    df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)
+].copy()
+att_ns["job_type"] = "ATT NO SHOW"
+filtered_dfs.append(att_ns)
+
+
+# Post visit no show
+# epc_ns = df[
+#     df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999)
+# ].copy()
+# epc_ns["job_type"] = "EPC NO SHOW"
+# filtered_dfs.append(epc_ns)
+
+final_df = pd.concat(filtered_dfs).reset_index(drop=True)
+
+final_df[['address', 'client', 'job_type']]

etl/month_end_automation_wave_2_no_9.py

@@ -0,0 +1,169 @@
+# Wave 2's month end automation
+from tqdm import tqdm
+from monday import MondayClient
+from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
+from pprint import pprint
+import pandas as pd
+import json
+
+monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
+monday = MondayClient(monday_key)
+# NCHA Almshouses
+board_ids = ["5423364294"]
+
+
+for board in tqdm(board_ids):
+    board_data = monday.boards.fetch_boards_by_id(board)
+    columns = board_data["data"]["boards"][0]["columns"]
+    col_id_map = {col["title"].lower(): col["id"] for col in columns}
+    reversed_col_id_map = {v: k for k, v in col_id_map.items()}
+    
+    
+    items = get_all_items(board, monday)
+
+    all_records = []
+    for row in tqdm(items):
+        data = {}
+        data.update({"address": row['name']})
+        data.update({"client": row['group']['title']})
+        for col in row.get("column_values", []):
+            if col.get("id") in reversed_col_id_map:
+                if col.get("type") == "file":
+                    value = col.get("value")
+                    no_of_files = 0
+
+                    if value:
+                        value = json.loads(col["value"])
+                        no_of_files = len(value.get('files', []))
+                    data.update({reversed_col_id_map[col.get("id")]: no_of_files})
+                elif "no show" in reversed_col_id_map[col.get("id")]:
+                    def extract_number_from_text(text):
+                        number_str = ''
+                        
+                        for char in text:
+                            if char.isnumeric():
+                                number_str += char
+                            elif number_str:
+                                break  # stop once a number sequence ends
+
+                        return int(number_str) if number_str else None
+                    text = col.get("text")
+                    if text is None:
+                        data.update({
+                            reversed_col_id_map[col.get("id")]: col.get("text")
+                        })    
+                    else:
+                        data.update({
+                            reversed_col_id_map[col.get("id")]: extract_number_from_text(text)
+                        })    
+                else:
+                    data.update({
+                        reversed_col_id_map[col.get("id")]: col.get("text")
+                    })
+        all_records.append(data)
+
+# Convert to DataFrame
+df = pd.DataFrame(all_records)
+
+filtered_dfs = []
+
+def get_df(df, column_name, success_critera, job_name):
+    _ = df[
+        df[column_name].str.lower().isin(success_critera)
+    ].copy()
+    _["job_type"] = job_name
+    return _
+
+
+# RA
+ra = get_df(df, "ra", ["completed rdsap 9.9", "completed rdsap 10"], "RA")
+filtered_dfs.append(ra)
+
+
+# ATT
+att = get_df(df, "att", ["completed"], "ATT")
+filtered_dfs.append(att)
+
+# V1 Coordination
+v1 = get_df(df, "coordination status (mtp)".lower(), [
+    "rc complete",
+], "V1 Coordination")
+filtered_dfs.append(v1)
+
+# V2 Coordination
+# v2 = get_df(df, "v2 coordination status", ["rc v2 complete", "uploaded"], "V2 Coordination")
+# filtered_dfs.append(v2)
+
+# # V3 Coordination
+# v3 = get_df(df, "")
+# # filtered_dfs.append(v3)
+
+# Design stage 1
+# design1 = get_df(df, "rd invoiced", ["to invoice"], "Design")
+# filtered_dfs.append(design1)
+
+# Design revision
+# design2 = get_df(df, "design revision invoice", [
+#     "Rev. A to invoice".lower(),
+#     "Rev. B to invoice".lower(),
+#     "Rev. C to invoice".lower(),
+#     "Rev. D to invoice".lower(),
+# ], "Design Revision")
+# filtered_dfs.append(design2)
+
+# Lodgement Phase 1
+# lodg1 = get_df(df, "Lodg. Phase 1 Invoice Status".lower(), ["to be invoiced"], "Lodgement Phase 1")
+# filtered_dfs.append(lodg1)
+
+
+# Full Lodgement Phase
+full_lodgement = get_df(df, "trustmark lodgement".lower(), ["done"], "Full Lodgement")
+filtered_dfs.append(full_lodgement)
+
+# POST EPC
+# post_epc = get_df(df, "post epc status", ["uploaded"], "POST EPC")
+# filtered_dfs.append(post_epc)
+
+
+# # POST EPR
+# post_epr = df[
+#     df["post-epc status"].str.lower().isin(["post epr completed"])
+# ].copy()
+# post_epr["job_type"] = "POST ATT"
+# filtered_dfs.append(post_epr)
+
+# Post ATT
+# post_att = get_df(df, "post att", ["post att uploaded"], "POST ATT")
+# filtered_dfs.append(post_att)
+
+
+# Retrofit Evaluation
+retro = get_df(df, "retrofit evaluation", ["done"], "Retrofit Evaluation")
+filtered_dfs.append(retro)
+
+# RA NO Show
+ra_ns  = df[
+    df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)
+].copy()
+ra_ns["job_type"] = "RA NO SHOW"
+filtered_dfs.append(ra_ns)
+
+
+# ATT NO Show
+att_ns = df[
+    df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)
+].copy()
+att_ns["job_type"] = "ATT NO SHOW"
+filtered_dfs.append(att_ns)
+
+
+# Post visit no show
+# epc_ns = df[
+#     df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999)
+# ].copy()
+# epc_ns["job_type"] = "EPC NO SHOW"
+# filtered_dfs.append(epc_ns)
+
+final_df = pd.concat(filtered_dfs).reset_index(drop=True)
+
+final_df[['address', 'client', 'job_type']]