save month ened wave 2 one board example

etl/month_end_automation.py

@@ -1,3 +1,5 @@
+# Wave 2's month end automation
+
 from tqdm import tqdm
 from monday import MondayClient
 from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
@@ -23,6 +25,7 @@ for board in tqdm(board_ids):
     for row in tqdm(items):
         data = {}
         data.update({"address": row['name']})
+        data.update({"client": row['group']['title']})
         for col in row.get("column_values", []):
             if col.get("id") in reversed_col_id_map:
                 if col.get("type") == "file":
@@ -33,8 +36,27 @@ for board in tqdm(board_ids):
                         value = json.loads(col["value"])
                         no_of_files = len(value.get('files', []))
                     data.update({reversed_col_id_map[col.get("id")]: no_of_files})
+                elif "no show" in reversed_col_id_map[col.get("id")]:
+                    def extract_number_from_text(text):
+                        number_str = ''
+                        
+                        for char in text:
+                            if char.isnumeric():
+                                number_str += char
+                            elif number_str:
+                                break  # stop once a number sequence ends
+
+                        return int(number_str) if number_str else None
+                    text = col.get("text")
+                    if text is None:
+                        data.update({
+                            reversed_col_id_map[col.get("id")]: col.get("text")
+                        })    
+                    else:
+                        data.update({
+                            reversed_col_id_map[col.get("id")]: extract_number_from_text(text)
+                        })    
                 else:
-                    
                     data.update({
                         reversed_col_id_map[col.get("id")]: col.get("text")
                     })
@@ -43,22 +65,97 @@ for board in tqdm(board_ids):
 # Convert to DataFrame
 df = pd.DataFrame(all_records)
 
+filtered_dfs = []
+
 # RA
-filtered_df = df[
+ra = df[
     df["ra"].str.lower().isin(["completed rdsap 10", "completed rdsap 9.9"])
 ].copy()
+filtered_dfs.append(ra)
+
 
 # ATT
-filtered_df = df[
+att = df[
     df["att"].str.lower().isin(["completed"])
 ].copy()
+filtered_dfs.append(att)
 
 # V1 Coordination
-filtered_df = df[
+v1 = df[
     df["v1 coordination status"].str.lower().isin(["rc complete"])
 ].copy()
+filtered_dfs.append(v1)
 
 # V2 Coordination
 _ = df[df["v2 invoiced"].fillna('').str.lower().isin(['to be invoiced'])]
-filtered_df = _[_["v2 dc/ima/pas"] > 0]
+v2 = _[_["v2 dc/ima/pas"] > 0]
+filtered_dfs.append(v2)
+
+# V3 Coordination
+v3 = df[
+    df["v3  invoiced"].str.lower().isin(["to be invoiced"])
+].copy()
+filtered_dfs.append(v3)
+
+# Design stage 1
+design1 = df[
+    df["design invoice status"].str.lower().isin(["to invoice"])
+]
+filtered_dfs.append(design1)
+
+# Design stage 2
+design2 = df[
+    df["revision 2 design invoice"].str.lower().isin(["to invoice"])
+]
+filtered_dfs.append(design2)
+
+# Lodgement Phase 1
+lodg1 = df[
+    df["lodg. phase 1 invoice status"].str.lower().isin(["to invoice"])
+]
+filtered_dfs.append(lodg1)
+
+# Full Lodgement Phase
+lodg2 = df[
+    df["full lodgement invoice status"].str.lower().isin(["to invoice"])
+]
+filtered_dfs.append(lodg2)
+
+# POST EPC
+# !!! Need Kevins input ( how does he determine invoice in this one)
+
+# Post ATT
+# !!! Check with kevin, if he uses epc pic uploaded ( to invoice ) or uses anything else
+filtered_df = df[
+    df["post-att"].str.lower().isin(["post-att uploaded"])
+]
+
+# Retrofit Evaluation
+retro = df[
+    df["retrofit evaluation"].str.lower().isin(["complete"])
+]
+filtered_dfs.append(retro)
+
+# RA NO Show
+ra_ns  = df[
+    df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)
+]
+filtered_dfs.append(ra_ns)
+
+
+
+# ATT NO Show
+att_ns = df[
+    df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)
+]
+filtered_dfs.append(att_ns)
+
+
+# Post visit no show
+epc_ns = df[
+    df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999)
+]
+filtered_dfs.append(epc_ns)
+
+final_df = pd.concat(filtered_dfs).drop_duplicates().reset_index(drop=True)