From 5743495261e506c4278f71c67f0378aae20e4d5d Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 4 Aug 2025 16:08:25 +0000 Subject: [PATCH] save --- etl/month_end_automation_wave_3_layout.py | 292 +--------------------- 1 file changed, 5 insertions(+), 287 deletions(-) diff --git a/etl/month_end_automation_wave_3_layout.py b/etl/month_end_automation_wave_3_layout.py index d5649f6..704668c 100644 --- a/etl/month_end_automation_wave_3_layout.py +++ b/etl/month_end_automation_wave_3_layout.py @@ -10,11 +10,11 @@ import json monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) board_ids = [ - "9349630181", # WCHG Walkups-Operations + # "9349630181", # WCHG Walkups-Operations # "8830772914", # "L&Q London" # "9601691730", # Cardo Wales & West - Wave 3 # "9660895490", # Northumberland County SHDF Wave 3 - # "9641491000", #Watford Warm Homes + "9641491000", #Watford Warm Homes ] empty = "Rate card info missing" @@ -95,12 +95,12 @@ rate_card_data_walk_ups = { ] } - +rate_card_df = pd.DataFrame(rate_card_data_watford_warm_homes) # rate_card_df = pd.DataFrame(rate_card_data_example) # rate_card_df = pd.DataFrame(rate_card_data_2502_accent_housing) # rate_card_df = pd.DataFrame(rate_card_data_l_and_q_london) # rate_card_df = pd.DataFrame(rate_card_data_northhumberland_country_shdf_wave_3) -rate_card_df = pd.DataFrame(rate_card_data_walk_ups) +# rate_card_df = pd.DataFrame(rate_card_data_walk_ups) for board in tqdm(board_ids): @@ -279,286 +279,4 @@ import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'WCHG Walk up {timestamp}.xlsx', index=False) -# Wave 3's month end automation - -from tqdm import tqdm -from monday import MondayClient -from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids -from pprint import pprint -import pandas as pd -import json - -monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" -monday = MondayClient(monday_key) -board_ids = [ - "9349630181", # WCHG Walkups-Operations - # "8830772914", # "L&Q London" - # "9601691730", # Cardo Wales & West - Wave 3 - # "9660895490", # Northumberland County SHDF Wave 3 - # "9641491000", #Watford Warm Homes -] - -empty = "Rate card info missing" -rate_card_data_watford_warm_homes = { - "job_type": [ - "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", - "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", - "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", - "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" - ], - "rate": [ - 165, empty, empty, empty, empty, - empty, empty, empty, empty, empty, - empty, empty, empty, empty, empty, - empty, empty, empty, empty, empty, - ] -} - - -rate_card_data_2502_accent_housing = { - "job_type": [ - "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", - "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", - "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", - "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" - ], - "rate": [ - empty, empty, empty, empty, empty, - empty, empty, empty, empty, empty, - empty, empty, empty, empty, empty, - empty, empty, empty, 280, 150 - ] -} - -rate_card_data_l_and_q_london = { - "job_type": [ - "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", - "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", - "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", - "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" - ], - "rate": [ - empty, empty, empty, empty, empty, - empty, empty, empty, empty, empty, - empty, empty, empty, empty, empty, - empty, empty, empty, 280, 150 - ] -} - - -rate_card_data_northhumberland_country_shdf_wave_3 = { - "job_type": [ - "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", - "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", - "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", - "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" - ], - "rate": [ - empty, empty, empty, empty, empty, - empty, empty, empty, empty, empty, - empty, empty, empty, empty, empty, - empty, empty, empty, 280, 130, - ] -} - -rate_card_data_walk_ups = { - "job_type": [ - "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", - "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", - "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", - "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" - ], - "rate": [ - 259, 125, 280, 125, 125, - 650, 415, 195, 175, 135, - 120, "60 to check", 85, 125, 60, - 45, 45, 45, empty, empty, - ] -} - - -# rate_card_df = pd.DataFrame(rate_card_data_example) -# rate_card_df = pd.DataFrame(rate_card_data_2502_accent_housing) -# rate_card_df = pd.DataFrame(rate_card_data_l_and_q_london) -# rate_card_df = pd.DataFrame(rate_card_data_northhumberland_country_shdf_wave_3) -rate_card_df = pd.DataFrame(rate_card_data_walk_ups) - - -for board in tqdm(board_ids): - print(f"working on board {board}") - board_data = monday.boards.fetch_boards_by_id(board) - columns = board_data["data"]["boards"][0]["columns"] - col_id_map = {col["title"].lower(): col["id"] for col in columns} - reversed_col_id_map = {v: k for k, v in col_id_map.items()} - - - items = get_all_items(board, monday) - - all_records = [] - for row in tqdm(items): - data = {} - data.update({"address": row['name']}) - data.update({"client": row['group']['title']}) - for col in row.get("column_values", []): - if col.get("id") in reversed_col_id_map: - if col.get("type") == "file": - value = col.get("value") - no_of_files = 0 - - if value: - value = json.loads(col["value"]) - no_of_files = len(value.get('files', [])) - data.update({reversed_col_id_map[col.get("id")]: no_of_files}) - else: - data.update({ - reversed_col_id_map[col.get("id")]: col.get("text") - }) - all_records.append(data) - -# Convert to DataFrame -df = pd.DataFrame(all_records) - -filtered_dfs = [] - - -def get_df(df, column_name, success_critera, job_name=None): - _ = pd.DataFrame() - if column_name in col_id_map: - _ = df[ - df[column_name].str.lower().isin(success_critera) - ].copy() - if job_name: - _["job_type"] = job_name - - - return _ - - -# RA -ra = get_df(df, "ra invoicing status", ["to invoice"], "RA") -if not ra.empty: - filtered_dfs.append(ra) - - -att = get_df(df, "post att invoicing status", ["to invoice"], "ATT") -if not att.empty: - filtered_dfs.append(att) - -modeling = get_df(df, "mtp invoicing status", ["modelling to invoice"], "Measure Modelling") -if not modeling.empty: - filtered_dfs.append(modeling) - -try: - # Only needed for one board in wave 3 - full_cost = get_df(df, "mtp invoicing status", ["(V1) Full cost MTP to invoice (no previous modelling)".lower()], "full cost mtp") - if not full_cost.empty: - filtered_dfs(full_cost) -except Exception as e: - print(e) - -v1 = get_df(df, "mtp invoicing status", ["(v1) ioe/mtp to invoice"], "Coordination Stage 1 v1") -if not v1.empty: - filtered_dfs.append(v1) - -v2 = get_df(df, "mtp invoicing status", ["(v2) ioe/mtp to invoice"], "Coordination Stage 1 v2 remodel") -if not v2.empty: - filtered_dfs.append(v2) - -v3 = get_df(df, "mtp invoicing status", ["(v3) ioe/mtp to invoice"], "Coordination Stage 1 v3 remodel") -if not v3.empty: - filtered_dfs.append(v3) - -# Coordination stage 2 Please complete -cors2 = get_df(df, "rc stage 2", ["to invoice"], "Coordination Stage 2") -if not cors2.empty: - filtered_dfs.append(cors2) - -# Design archetype complex -design = get_df(df, "design invoicing status", ["to invoice"]) -design1 = get_df(design, "design invoice type", ["archetype (complex)"], "Design Archetype Complex") -if not design1.empty : - filtered_dfs.append(design1) - -# Design archetype simple -design1 = get_df(design, "design invoice type", ["archetype (simple)"], "Design Archetype Simple") -if not design1.empty: - filtered_dfs.append(design1) - -# Design repetitive simple -design1 = get_df(design, "design invoice type", ["archetype (simple)"], "Design Archetype repetitive") -if not design1.empty: - filtered_dfs.append(design1) - -# Design repetitive complex -design1 = get_df(design, "design invoice type", ["archetype (complex)"], "Design Archetype complex") -if not design1.empty: - filtered_dfs.append(design1) - -# Design Revision -revision_letter = ['a', 'b', 'c', 'd'] -for letter in revision_letter: - design = get_df(df, "design revision invoice", [f"rev. {letter} to invoice"], "Design Revision") - if not design.empty: - filtered_dfs.append(design) - -# Lodgement Phase 1 -lodg1 = get_df(df, "lodgement phase 1 invoicing status", ["to invoice"], "Lodgement Phase 1") -if not lodg1.empty: - filtered_dfs(lodg1) - -# Full Lodgement Phase -lodg2 = get_df(df, "full lodgement invoicing status", ["to invoice"], "Full lodgement phase 2") -if not lodg2.empty: - filtered_dfs.append(lodg2) - -# POST EPC -post_epc = get_df(df, "post epc & eval. invoicing status", ["epc to invoice"], "POST EPC") -if not post_epc.empty: - filtered_dfs.append(post_epc) - - -# POST EPR -post_epr = get_df(df, "post epc & eval. invoicing status", ["epr to invoice"], "POST EPR") -if not post_epr.empty: - filtered_dfs.append(post_epr) - -# post att -post_att = get_df(df, "post att invoicing status", ["to invoice"], "POST ATT") -if not post_att.empty: - filtered_dfs.append(post_epc) - -# Retrofit Evaluation -rc = get_df(df, "rc stage 2 invoicing status", ["to invoice"], "retrofit evaluation") -if not rc.empty: - filtered_dfs.append(rc) - -# RA NO Show -ra_ns = get_df(df,"ra no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "RA NO SHOW") -if not ra_ns.empty: - filtered_dfs.append(ra_ns) - - -# ATT NO Show -att_ns = get_df(df, "pre att no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "ATT NO SHOW") -if not att_ns.empty: - filtered_dfs.append(att_ns) - - -# Post visit no show -epc_ns = get_df(df, "post works no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "post EPC NO SHOW") -if not epc_ns.empty: - filtered_dfs.append(epc_ns) - -final_df = pd.concat(filtered_dfs).reset_index(drop=True) - -final_df["job_type"] = final_df["job_type"].str.lower() -rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() - -# Now perform the merge -combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") -import datetime -timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') - -attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'WCHG Walk up {timestamp}.xlsx', index=False) +combined_with_rates[attribute].to_excel(f'Watford Warm Homes {timestamp}.xlsx', index=False)