junte/survey-extraction
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/survey-extraction.git synced 2026-06-30 13:10:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

re do month encd

Browse source
This commit is contained in:
Jun-te Kim 2025-09-22 10:30:13 +00:00
parent 54a4d5761a
commit f74d2671fd
2 changed files with 44 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

42
etl/month_end_automation_wave_3_layout.py
View file

@ -244,36 +244,42 @@ for board, all_records in board_to_record.items():
if not v3.empty:
filtered_dfs.append(v3)
# Design archetype complex
# Base filter
design = get_df(df, "design invoicing status", ["to invoice"])
# Build each slice
design1 = get_df(design, "design invoice type", ["archetype (complex)"], "Design Archetype Complex")
if not design1.empty :
if not design1.empty:
filtered_dfs.append(design1)
# Design archetype simple
design2 = get_df(design, "design invoice type", ["archetype (simple)"], "Design Archetype Simple")
if not design1.empty:
if not design2.empty:
filtered_dfs.append(design2)
# Design repetitive simple
design3 = get_df(design, "design invoice type", ["repetitive (simple)"], "Design repetitive simple")
if not design1.empty:
design3 = get_df(design, "design invoice type", ["repetitive (simple)"], "Design Repetitive Simple")
if not design3.empty:
filtered_dfs.append(design3)
# Design repetitive complex
design4 = get_df(design, "design invoice type", ["repetitive (complex)"], "Design Repetitive complex")
if not design1.empty:
design4 = get_df(design, "design invoice type", ["repetitive (complex)"], "Design Repetitive Complex")
if not design4.empty:
filtered_dfs.append(design4)
# Design not specified
all_filtered = pd.concat([df for df in (design1, design2, design3, design4) if not df.empty])
design_remaining = design.loc[~design.index.isin(all_filtered.index)]
# Safe concat of non-empty slices
parts = [d for d in (design1, design2, design3, design4) if not d.empty]
if parts:
all_filtered = pd.concat(parts, ignore_index=False) # keep original index so we can subtract cleanly
# Anything not matched by the 4 categories
remaining_idx = design.index.difference(all_filtered.index)
else:
# No matches in any category
all_filtered = design.iloc[0:0] # empty DF with same columns
remaining_idx = design.index
design_remaining = design.loc[remaining_idx].copy()
if not design_remaining.empty:
design_remaining["job_type"] = "design type not specified"
design_remaining["job_type"] = "Design Type Not Specified"
filtered_dfs.append(design_remaining)
# Design Revision
revision_letter = ['a', 'b', 'c', 'd']
for letter in revision_letter:

31
etl/month_end_automation_wave_accent_housing.py
View file

@ -97,33 +97,42 @@ full_cost = get_df(df, "mtp invoicing status", ["(v1) full cost mtp to invoice (
if not full_cost.empty:
filtered_dfs.append(full_cost)
# Design archetype complex
# Base filter
design = get_df(df, "design invoicing status", ["to invoice"])
# Design archetype complex
design1 = get_df(design, "design invoice type", ["archetype (complex)"], "Design Archetype Complex")
if not design1.empty :
if not design1.empty:
filtered_dfs.append(design1)
# Design archetype simple
design2 = get_df(design, "design invoice type", ["archetype (simple)"], "Design Archetype Simple")
if not design1.empty:
if not design2.empty:
filtered_dfs.append(design2)
# Design repetitive simple
design3 = get_df(design, "design invoice type", ["repetitive (simple)"], "Design repetitive simple")
if not design1.empty:
design3 = get_df(design, "design invoice type", ["repetitive (simple)"], "Design Repetitive Simple")
if not design3.empty:
filtered_dfs.append(design3)
# Design repetitive complex
design4 = get_df(design, "design invoice type", ["repetitive (complex)"], "Design repetitive complex")
if not design1.empty:
design4 = get_df(design, "design invoice type", ["repetitive (complex)"], "Design Repetitive Complex")
if not design4.empty:
filtered_dfs.append(design4)
# Design not specified
all_filtered = pd.concat([df for df in (design1, design2, design3, design4) if not df.empty])
design_remaining = design.loc[~design.index.isin(all_filtered.index)]
# Safe concat
parts = [d for d in (design1, design2, design3, design4) if not d.empty]
if parts:
all_filtered = pd.concat(parts, ignore_index=False) # keep original indices
remaining_idx = design.index.difference(all_filtered.index)
else:
all_filtered = design.iloc[0:0] # empty DF with same schema
remaining_idx = design.index
# Design not specified
design_remaining = design.loc[remaining_idx].copy()
if not design_remaining.empty:
design_remaining["job_type"] = "design type not specified"
design_remaining["job_type"] = "Design Type Not Specified"
filtered_dfs.append(design_remaining)
# Design Revision