From d96983c383444517ca0a3b9931500464a140445d Mon Sep 17 00:00:00 2001
From: Jun-te Kim <junte.kim@mealcraft.com>
Date: Thu, 27 Nov 2025 15:38:57 +0000
Subject: [PATCH] save

---
 etl/month_end_automation_wave_3_layout.py | 50 ++++++++++++-----------
 1 file changed, 26 insertions(+), 24 deletions(-)

diff --git a/etl/month_end_automation_wave_3_layout.py b/etl/month_end_automation_wave_3_layout.py
index 61476f6..1dd8947 100644
--- a/etl/month_end_automation_wave_3_layout.py
+++ b/etl/month_end_automation_wave_3_layout.py
@@ -27,15 +27,15 @@ class BoardID(Enum):
 
 
 board_ids = [
-    "9349630181", # WCHG Walkups-Operations
-    "8830772914", # "L&Q London"
-    "9601691730", # Cardo Wales & West - Wave 3
-    "9660895490", # Northumberland County SHDF Wave 3
-    "9641491000", # Watford Warm Homes
-    "9671463094", # Seddon
+    # "9349630181", # WCHG Walkups-Operations
+    # "8830772914", # "L&Q London"
+    # "9601691730", # Cardo Wales & West - Wave 3
+    # "9660895490", # Northumberland County SHDF Wave 3
+    # "9641491000", # Watford Warm Homes
+    # "9671463094", # Seddon
     "9929454382", # NCHA SHDF Wave 3 Main Operation Board
-    "18232420839", # Powys County Council:
-    "18380751311", # Devon County Council
+    # "18232420839", # Powys County Council:
+    # "18380751311", # Devon County Council
 ]
 
 empty = "Rate card info missing"
@@ -228,22 +228,24 @@ for board in tqdm(board_ids):
     for row in tqdm(items):
         data = {}
         data.update({"address": row['name']})
-        data.update({"client": row['group']['title']})
-        for col in row.get("column_values", []):
-            if col.get("id") in reversed_col_id_map:
-                if col.get("type") == "file":
-                    value = col.get("value")
-                    no_of_files = 0
+        client = row['group']['title']
+        if client != "Removed From Program" and client != "Removed From Project":
+            data.update({"client": row['group']['title']})
+            for col in row.get("column_values", []):
+                if col.get("id") in reversed_col_id_map:
+                    if col.get("type") == "file":
+                        value = col.get("value")
+                        no_of_files = 0
 
-                    if value:
-                        value = json.loads(col["value"])
-                        no_of_files = len(value.get('files', []))
-                    data.update({reversed_col_id_map[col.get("id")]: no_of_files})
-                else:
-                    data.update({
-                        reversed_col_id_map[col.get("id")]: col.get("text")
-                    })
-        all_records.append(data)
+                        if value:
+                            value = json.loads(col["value"])
+                            no_of_files = len(value.get('files', []))
+                        data.update({reversed_col_id_map[col.get("id")]: no_of_files})
+                    else:
+                        data.update({
+                            reversed_col_id_map[col.get("id")]: col.get("text")
+                        })
+            all_records.append(data)
     board_to_record.update({board: all_records})
 
 # Convert to DataFrame
@@ -346,7 +348,7 @@ for board, all_records in board_to_record.items():
     lodg1  = get_df(df, "lodgement phase 1 invoicing status", ["to invoice"], "Lodgement Phase 1")
     if not lodg1.empty:
         filtered_dfs.append(lodg1)
-
+    
     # Full Lodgement Phase
     lodg2 = get_df(df, "lodgement phase 2 invoicing status", ["to invoice"], "Full lodgement phase 2")
     if not lodg2.empty: