junte/survey-extraction
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/survey-extraction.git synced 2026-06-08 11:17:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #112 from Hestia-Homes/feature/tasks

Browse source 
save
This commit is contained in:
Jun-te Kim 2025-11-27 15:41:11 +00:00 • committed by GitHub
commit 1f835656d8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 26 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

50
etl/month_end_automation_wave_3_layout.py
View file

@ -27,15 +27,15 @@ class BoardID(Enum):
board_ids = [
"9349630181", # WCHG Walkups-Operations
"8830772914", # "L&Q London"
"9601691730", # Cardo Wales & West - Wave 3
"9660895490", # Northumberland County SHDF Wave 3
"9641491000", # Watford Warm Homes
"9671463094", # Seddon
# "9349630181", # WCHG Walkups-Operations
# "8830772914", # "L&Q London"
# "9601691730", # Cardo Wales & West - Wave 3
# "9660895490", # Northumberland County SHDF Wave 3
# "9641491000", # Watford Warm Homes
# "9671463094", # Seddon
"9929454382", # NCHA SHDF Wave 3 Main Operation Board
"18232420839", # Powys County Council:
"18380751311", # Devon County Council
# "18232420839", # Powys County Council:
# "18380751311", # Devon County Council
]
empty = "Rate card info missing"
@ -228,22 +228,24 @@ for board in tqdm(board_ids):
for row in tqdm(items):
data = {}
data.update({"address": row['name']})
data.update({"client": row['group']['title']})
for col in row.get("column_values", []):
if col.get("id") in reversed_col_id_map:
if col.get("type") == "file":
value = col.get("value")
no_of_files = 0
client = row['group']['title']
if client != "Removed From Program" and client != "Removed From Project":
data.update({"client": row['group']['title']})
for col in row.get("column_values", []):
if col.get("id") in reversed_col_id_map:
if col.get("type") == "file":
value = col.get("value")
no_of_files = 0
if value:
value = json.loads(col["value"])
no_of_files = len(value.get('files', []))
data.update({reversed_col_id_map[col.get("id")]: no_of_files})
else:
data.update({
reversed_col_id_map[col.get("id")]: col.get("text")
})
all_records.append(data)
if value:
value = json.loads(col["value"])
no_of_files = len(value.get('files', []))
data.update({reversed_col_id_map[col.get("id")]: no_of_files})
else:
data.update({
reversed_col_id_map[col.get("id")]: col.get("text")
})
all_records.append(data)
board_to_record.update({board: all_records})
# Convert to DataFrame
@ -346,7 +348,7 @@ for board, all_records in board_to_record.items():
lodg1 = get_df(df, "lodgement phase 1 invoicing status", ["to invoice"], "Lodgement Phase 1")
if not lodg1.empty:
filtered_dfs.append(lodg1)
# Full Lodgement Phase
lodg2 = get_df(df, "lodgement phase 2 invoicing status", ["to invoice"], "Full lodgement phase 2")
if not lodg2.empty: