From 827be0c161f6d1f9c6b607df562cb048d9e101d9 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 1 Aug 2025 15:44:45 +0000 Subject: [PATCH 1/3] added --- etl/month_end_automation_wave_2_no_14.py | 6 +- ...onth_end_automation_wave_accent_housing.py | 104 ++++++++++++++++++ 2 files changed, 107 insertions(+), 3 deletions(-) create mode 100644 etl/month_end_automation_wave_accent_housing.py diff --git a/etl/month_end_automation_wave_2_no_14.py b/etl/month_end_automation_wave_2_no_14.py index 3a670b9..27cadd0 100644 --- a/etl/month_end_automation_wave_2_no_14.py +++ b/etl/month_end_automation_wave_2_no_14.py @@ -96,9 +96,9 @@ ra = get_df(df, "ra", ["completed rdsap 9.9", "completed rdsap 10"], "RA") filtered_dfs.append(ra) -# ATT -att = get_df(df, "att", ["completed"], "ATT") -filtered_dfs.append(att) +# # ATT +# att = get_df(df, "att", ["completed"], "ATT") +# filtered_dfs.append(att) # V1 Coordination v1 = get_df(df, "v1 coordination status (ioe,mtp)".lower(), [ diff --git a/etl/month_end_automation_wave_accent_housing.py b/etl/month_end_automation_wave_accent_housing.py new file mode 100644 index 0000000..e02cdd9 --- /dev/null +++ b/etl/month_end_automation_wave_accent_housing.py @@ -0,0 +1,104 @@ +# Wave 3's month end automation + +from tqdm import tqdm +from monday import MondayClient +from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids +from pprint import pprint +import pandas as pd +import json + +monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" +monday = MondayClient(monday_key) +board_ids = [ + "8829428746", # 2502 Accent Housing +] + +empty = "Rate card info missing" + +rate_card_data_2502_accent_housing = { + "job_type": [ + "First half of MTP", "Second half of MTP", "Full MTP" + ], + "rate": [ + 150, 130, 280 + ] +} + +rate_card_df = pd.DataFrame(rate_card_data_2502_accent_housing) + + +for board in tqdm(board_ids): + print(f"working on board {board}") + board_data = monday.boards.fetch_boards_by_id(board) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} + reversed_col_id_map = {v: k for k, v in col_id_map.items()} + + + items = get_all_items(board, monday) + + all_records = [] + for row in tqdm(items): + data = {} + data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) + for col in row.get("column_values", []): + if col.get("id") in reversed_col_id_map: + if col.get("type") == "file": + value = col.get("value") + no_of_files = 0 + + if value: + value = json.loads(col["value"]) + no_of_files = len(value.get('files', [])) + data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + else: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + all_records.append(data) + +# Convert to DataFrame +df = pd.DataFrame(all_records) + +filtered_dfs = [] + + +def get_df(df, column_name, success_critera, job_name=None): + _ = pd.DataFrame() + if column_name in col_id_map: + _ = df[ + df[column_name].str.lower().isin(success_critera) + ].copy() + if job_name: + _["job_type"] = job_name + + + return _ + + + +modeling = get_df(df, "mtp invoicing status", ["modelling to invoice"], "First half of mtp") +if not modeling.empty: + filtered_dfs.append(modeling) + +second_payment = get_df(df, "mtp invoicing status", ["(V1) Remainder IOE/MTP to invoice".lower()], "second half of mtp") +if not second_payment.empty: + filtered_dfs.append(second_payment) + +full_cost = get_df(df, "mtp invoicing status", ["(v1) full cost mtp to invoice (no previous modelling)"], "full cost mtp") +if not full_cost.empty: + filtered_dfs(full_cost) + +final_df = pd.concat(filtered_dfs).reset_index(drop=True) + +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_excel(f'2502 Accent housing {timestamp}.xlsx', index=False) From 5743495261e506c4278f71c67f0378aae20e4d5d Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 4 Aug 2025 16:08:25 +0000 Subject: [PATCH 2/3] save --- etl/month_end_automation_wave_3_layout.py | 292 +--------------------- 1 file changed, 5 insertions(+), 287 deletions(-) diff --git a/etl/month_end_automation_wave_3_layout.py b/etl/month_end_automation_wave_3_layout.py index d5649f6..704668c 100644 --- a/etl/month_end_automation_wave_3_layout.py +++ b/etl/month_end_automation_wave_3_layout.py @@ -10,11 +10,11 @@ import json monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) board_ids = [ - "9349630181", # WCHG Walkups-Operations + # "9349630181", # WCHG Walkups-Operations # "8830772914", # "L&Q London" # "9601691730", # Cardo Wales & West - Wave 3 # "9660895490", # Northumberland County SHDF Wave 3 - # "9641491000", #Watford Warm Homes + "9641491000", #Watford Warm Homes ] empty = "Rate card info missing" @@ -95,12 +95,12 @@ rate_card_data_walk_ups = { ] } - +rate_card_df = pd.DataFrame(rate_card_data_watford_warm_homes) # rate_card_df = pd.DataFrame(rate_card_data_example) # rate_card_df = pd.DataFrame(rate_card_data_2502_accent_housing) # rate_card_df = pd.DataFrame(rate_card_data_l_and_q_london) # rate_card_df = pd.DataFrame(rate_card_data_northhumberland_country_shdf_wave_3) -rate_card_df = pd.DataFrame(rate_card_data_walk_ups) +# rate_card_df = pd.DataFrame(rate_card_data_walk_ups) for board in tqdm(board_ids): @@ -279,286 +279,4 @@ import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'WCHG Walk up {timestamp}.xlsx', index=False) -# Wave 3's month end automation - -from tqdm import tqdm -from monday import MondayClient -from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids -from pprint import pprint -import pandas as pd -import json - -monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" -monday = MondayClient(monday_key) -board_ids = [ - "9349630181", # WCHG Walkups-Operations - # "8830772914", # "L&Q London" - # "9601691730", # Cardo Wales & West - Wave 3 - # "9660895490", # Northumberland County SHDF Wave 3 - # "9641491000", #Watford Warm Homes -] - -empty = "Rate card info missing" -rate_card_data_watford_warm_homes = { - "job_type": [ - "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", - "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", - "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", - "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" - ], - "rate": [ - 165, empty, empty, empty, empty, - empty, empty, empty, empty, empty, - empty, empty, empty, empty, empty, - empty, empty, empty, empty, empty, - ] -} - - -rate_card_data_2502_accent_housing = { - "job_type": [ - "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", - "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", - "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", - "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" - ], - "rate": [ - empty, empty, empty, empty, empty, - empty, empty, empty, empty, empty, - empty, empty, empty, empty, empty, - empty, empty, empty, 280, 150 - ] -} - -rate_card_data_l_and_q_london = { - "job_type": [ - "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", - "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", - "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", - "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" - ], - "rate": [ - empty, empty, empty, empty, empty, - empty, empty, empty, empty, empty, - empty, empty, empty, empty, empty, - empty, empty, empty, 280, 150 - ] -} - - -rate_card_data_northhumberland_country_shdf_wave_3 = { - "job_type": [ - "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", - "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", - "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", - "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" - ], - "rate": [ - empty, empty, empty, empty, empty, - empty, empty, empty, empty, empty, - empty, empty, empty, empty, empty, - empty, empty, empty, 280, 130, - ] -} - -rate_card_data_walk_ups = { - "job_type": [ - "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", - "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", - "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", - "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" - ], - "rate": [ - 259, 125, 280, 125, 125, - 650, 415, 195, 175, 135, - 120, "60 to check", 85, 125, 60, - 45, 45, 45, empty, empty, - ] -} - - -# rate_card_df = pd.DataFrame(rate_card_data_example) -# rate_card_df = pd.DataFrame(rate_card_data_2502_accent_housing) -# rate_card_df = pd.DataFrame(rate_card_data_l_and_q_london) -# rate_card_df = pd.DataFrame(rate_card_data_northhumberland_country_shdf_wave_3) -rate_card_df = pd.DataFrame(rate_card_data_walk_ups) - - -for board in tqdm(board_ids): - print(f"working on board {board}") - board_data = monday.boards.fetch_boards_by_id(board) - columns = board_data["data"]["boards"][0]["columns"] - col_id_map = {col["title"].lower(): col["id"] for col in columns} - reversed_col_id_map = {v: k for k, v in col_id_map.items()} - - - items = get_all_items(board, monday) - - all_records = [] - for row in tqdm(items): - data = {} - data.update({"address": row['name']}) - data.update({"client": row['group']['title']}) - for col in row.get("column_values", []): - if col.get("id") in reversed_col_id_map: - if col.get("type") == "file": - value = col.get("value") - no_of_files = 0 - - if value: - value = json.loads(col["value"]) - no_of_files = len(value.get('files', [])) - data.update({reversed_col_id_map[col.get("id")]: no_of_files}) - else: - data.update({ - reversed_col_id_map[col.get("id")]: col.get("text") - }) - all_records.append(data) - -# Convert to DataFrame -df = pd.DataFrame(all_records) - -filtered_dfs = [] - - -def get_df(df, column_name, success_critera, job_name=None): - _ = pd.DataFrame() - if column_name in col_id_map: - _ = df[ - df[column_name].str.lower().isin(success_critera) - ].copy() - if job_name: - _["job_type"] = job_name - - - return _ - - -# RA -ra = get_df(df, "ra invoicing status", ["to invoice"], "RA") -if not ra.empty: - filtered_dfs.append(ra) - - -att = get_df(df, "post att invoicing status", ["to invoice"], "ATT") -if not att.empty: - filtered_dfs.append(att) - -modeling = get_df(df, "mtp invoicing status", ["modelling to invoice"], "Measure Modelling") -if not modeling.empty: - filtered_dfs.append(modeling) - -try: - # Only needed for one board in wave 3 - full_cost = get_df(df, "mtp invoicing status", ["(V1) Full cost MTP to invoice (no previous modelling)".lower()], "full cost mtp") - if not full_cost.empty: - filtered_dfs(full_cost) -except Exception as e: - print(e) - -v1 = get_df(df, "mtp invoicing status", ["(v1) ioe/mtp to invoice"], "Coordination Stage 1 v1") -if not v1.empty: - filtered_dfs.append(v1) - -v2 = get_df(df, "mtp invoicing status", ["(v2) ioe/mtp to invoice"], "Coordination Stage 1 v2 remodel") -if not v2.empty: - filtered_dfs.append(v2) - -v3 = get_df(df, "mtp invoicing status", ["(v3) ioe/mtp to invoice"], "Coordination Stage 1 v3 remodel") -if not v3.empty: - filtered_dfs.append(v3) - -# Coordination stage 2 Please complete -cors2 = get_df(df, "rc stage 2", ["to invoice"], "Coordination Stage 2") -if not cors2.empty: - filtered_dfs.append(cors2) - -# Design archetype complex -design = get_df(df, "design invoicing status", ["to invoice"]) -design1 = get_df(design, "design invoice type", ["archetype (complex)"], "Design Archetype Complex") -if not design1.empty : - filtered_dfs.append(design1) - -# Design archetype simple -design1 = get_df(design, "design invoice type", ["archetype (simple)"], "Design Archetype Simple") -if not design1.empty: - filtered_dfs.append(design1) - -# Design repetitive simple -design1 = get_df(design, "design invoice type", ["archetype (simple)"], "Design Archetype repetitive") -if not design1.empty: - filtered_dfs.append(design1) - -# Design repetitive complex -design1 = get_df(design, "design invoice type", ["archetype (complex)"], "Design Archetype complex") -if not design1.empty: - filtered_dfs.append(design1) - -# Design Revision -revision_letter = ['a', 'b', 'c', 'd'] -for letter in revision_letter: - design = get_df(df, "design revision invoice", [f"rev. {letter} to invoice"], "Design Revision") - if not design.empty: - filtered_dfs.append(design) - -# Lodgement Phase 1 -lodg1 = get_df(df, "lodgement phase 1 invoicing status", ["to invoice"], "Lodgement Phase 1") -if not lodg1.empty: - filtered_dfs(lodg1) - -# Full Lodgement Phase -lodg2 = get_df(df, "full lodgement invoicing status", ["to invoice"], "Full lodgement phase 2") -if not lodg2.empty: - filtered_dfs.append(lodg2) - -# POST EPC -post_epc = get_df(df, "post epc & eval. invoicing status", ["epc to invoice"], "POST EPC") -if not post_epc.empty: - filtered_dfs.append(post_epc) - - -# POST EPR -post_epr = get_df(df, "post epc & eval. invoicing status", ["epr to invoice"], "POST EPR") -if not post_epr.empty: - filtered_dfs.append(post_epr) - -# post att -post_att = get_df(df, "post att invoicing status", ["to invoice"], "POST ATT") -if not post_att.empty: - filtered_dfs.append(post_epc) - -# Retrofit Evaluation -rc = get_df(df, "rc stage 2 invoicing status", ["to invoice"], "retrofit evaluation") -if not rc.empty: - filtered_dfs.append(rc) - -# RA NO Show -ra_ns = get_df(df,"ra no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "RA NO SHOW") -if not ra_ns.empty: - filtered_dfs.append(ra_ns) - - -# ATT NO Show -att_ns = get_df(df, "pre att no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "ATT NO SHOW") -if not att_ns.empty: - filtered_dfs.append(att_ns) - - -# Post visit no show -epc_ns = get_df(df, "post works no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "post EPC NO SHOW") -if not epc_ns.empty: - filtered_dfs.append(epc_ns) - -final_df = pd.concat(filtered_dfs).reset_index(drop=True) - -final_df["job_type"] = final_df["job_type"].str.lower() -rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() - -# Now perform the merge -combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") -import datetime -timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') - -attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'WCHG Walk up {timestamp}.xlsx', index=False) +combined_with_rates[attribute].to_excel(f'Watford Warm Homes {timestamp}.xlsx', index=False) From e441e3e97c755b0078df4f2bd0ebc046114f5b54 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 4 Aug 2025 16:25:53 +0000 Subject: [PATCH 3/3] fix error --- etl/hubspot_surveyed_needs_sign_off.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etl/hubspot_surveyed_needs_sign_off.py b/etl/hubspot_surveyed_needs_sign_off.py index 6b5f48c..9502e22 100644 --- a/etl/hubspot_surveyed_needs_sign_off.py +++ b/etl/hubspot_surveyed_needs_sign_off.py @@ -24,11 +24,11 @@ deals = hubspotClient.get_deals_from_deal_stage(DealStage.FILES_MISSING_FROM_ASS for deal in deals: - hubspotClient.move_deals_to_different_stage([deal.deal_id], DealStage.SURVEYED_COMPLETED_SIGNED_OFF.value) + hubspotClient.move_deals_to_different_stage([deal.deal_id], DealStage.SURVEYED_COMPLETED_SIGNED_OFF) # Survyed_complete deals = hubspotClient.get_deals_from_deal_stage(DealStage.SURVEYED_COMPLETE_NEEDS_SIGN_OFF) for deal in deals: - hubspotClient.move_deals_to_different_stage([deal.deal_id], DealStage.SURVEYED_COMPLETED_SIGNED_OFF.value) + hubspotClient.move_deals_to_different_stage([deal.deal_id], DealStage.SURVEYED_COMPLETED_SIGNED_OFF)