diff --git a/etl/hubSpotClient/types.py b/etl/hubSpotClient/types.py index 35afaa0..6131136 100644 --- a/etl/hubSpotClient/types.py +++ b/etl/hubSpotClient/types.py @@ -18,6 +18,8 @@ def string_to_installer(installer): return SharePointInstaller.SOUTH_COAST_INSULATION elif installer.upper() == "SGEC": return SharePointInstaller.JJC + elif installer.upper() == "WARM FRONT": + return SharePointInstaller.BAXTER_KELLY else: return None diff --git a/etl/month_end_automation_wave_2_no_11.py b/etl/month_end_automation_wave_2_no_11.py index f05ef13..41f4438 100644 --- a/etl/month_end_automation_wave_2_no_11.py +++ b/etl/month_end_automation_wave_2_no_11.py @@ -8,9 +8,10 @@ import json monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) -# Northumberland LAD2 & HUG2 -board_ids = ["5121300882"] +board_ids = ["5121300882"] +emp = "rate card is empty" +check_wall_type = "check wall type" rate_card_data = { "job_type": [ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", @@ -19,10 +20,10 @@ rate_card_data = { "RA no show", "ATT no show", "post EPC no show" ], "rate": [ - 259, 125, 280, 125, 125, - 650, 195, 175, 135, - 120, "Post EPR Please Marianne", 85, 125, 60, - 25, 25, 25 + 225, 92.65, check_wall_type, check_wall_type, check_wall_type, + check_wall_type, check_wall_type, 165, 135, + 120, "60 - check", 48.6, 85.55, 49.6, + 125, 125, 125 ] } @@ -92,7 +93,7 @@ def get_df(df, column_name, success_critera, job_name=None): # RA -ra = get_df(df, "ra", ["completed rdsap 9.9", "completed rd sap 10"], "RA") +ra = get_df(df, "ra", ["completed rdsap 10", "completed rdsap 9.9"], "RA") filtered_dfs.append(ra) diff --git a/etl/month_end_automation_wave_2_no_14.py b/etl/month_end_automation_wave_2_no_14.py index 4a644ee..3a670b9 100644 --- a/etl/month_end_automation_wave_2_no_14.py +++ b/etl/month_end_automation_wave_2_no_14.py @@ -21,8 +21,8 @@ rate_card_data_example = { ], "rate": [ 259, 125, 280, 125, - 260, 126, 281, 126, - 262, 127, 282, 127, + 135, "60 to check", 85, 125, + 60, 45, 45, 45, ] } @@ -193,4 +193,14 @@ filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) -final_df[['address', 'client', 'job_type']] \ No newline at end of file + +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_excel(f'Decent Homes Stonewater - Operations_{timestamp}.xlsx', index=False) diff --git a/etl/month_end_automation_wave_3_layout.py b/etl/month_end_automation_wave_3_layout.py index 673ea66..d5649f6 100644 --- a/etl/month_end_automation_wave_3_layout.py +++ b/etl/month_end_automation_wave_3_layout.py @@ -10,15 +10,15 @@ import json monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) board_ids = [ -# "9349630181", # WCHG Walkups-Operations - # "8829428746", # 2502 Accent Housing + "9349630181", # WCHG Walkups-Operations # "8830772914", # "L&Q London" # "9601691730", # Cardo Wales & West - Wave 3 - "9660895490", # Northumberland County SHDF Wave 3 + # "9660895490", # Northumberland County SHDF Wave 3 + # "9641491000", #Watford Warm Homes ] empty = "Rate card info missing" -rate_card_data_example = { +rate_card_data_watford_warm_homes = { "job_type": [ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", @@ -26,10 +26,10 @@ rate_card_data_example = { "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" ], "rate": [ - 259, 125, 280, 125, 125, - 650, 415, 195, 175, 135, - 120, "Post EPR Please Marianne", 85, 125, 60, - 25, 25, 25, "Mariann please input full cost mtp", "Marianne please input measure modelling" + 165, empty, empty, empty, empty, + empty, empty, empty, empty, empty, + empty, empty, empty, empty, empty, + empty, empty, empty, empty, empty, ] } @@ -76,16 +76,31 @@ rate_card_data_northhumberland_country_shdf_wave_3 = { empty, empty, empty, empty, empty, empty, empty, empty, empty, empty, empty, empty, empty, empty, empty, - empty, empty, empty, 280, 150 + empty, empty, empty, 280, 130, ] } +rate_card_data_walk_ups = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" + ], + "rate": [ + 259, 125, 280, 125, 125, + 650, 415, 195, 175, 135, + 120, "60 to check", 85, 125, 60, + 45, 45, 45, empty, empty, + ] +} # rate_card_df = pd.DataFrame(rate_card_data_example) # rate_card_df = pd.DataFrame(rate_card_data_2502_accent_housing) # rate_card_df = pd.DataFrame(rate_card_data_l_and_q_london) -rate_card = pd.DataFrame(rate_card_data_northhumberland_country_shdf_wave_3) +# rate_card_df = pd.DataFrame(rate_card_data_northhumberland_country_shdf_wave_3) +rate_card_df = pd.DataFrame(rate_card_data_walk_ups) for board in tqdm(board_ids): @@ -264,4 +279,286 @@ import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'L&Q London {timestamp}.xlsx', index=False) +combined_with_rates[attribute].to_excel(f'WCHG Walk up {timestamp}.xlsx', index=False) +# Wave 3's month end automation + +from tqdm import tqdm +from monday import MondayClient +from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids +from pprint import pprint +import pandas as pd +import json + +monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" +monday = MondayClient(monday_key) +board_ids = [ + "9349630181", # WCHG Walkups-Operations + # "8830772914", # "L&Q London" + # "9601691730", # Cardo Wales & West - Wave 3 + # "9660895490", # Northumberland County SHDF Wave 3 + # "9641491000", #Watford Warm Homes +] + +empty = "Rate card info missing" +rate_card_data_watford_warm_homes = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" + ], + "rate": [ + 165, empty, empty, empty, empty, + empty, empty, empty, empty, empty, + empty, empty, empty, empty, empty, + empty, empty, empty, empty, empty, + ] +} + + +rate_card_data_2502_accent_housing = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" + ], + "rate": [ + empty, empty, empty, empty, empty, + empty, empty, empty, empty, empty, + empty, empty, empty, empty, empty, + empty, empty, empty, 280, 150 + ] +} + +rate_card_data_l_and_q_london = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" + ], + "rate": [ + empty, empty, empty, empty, empty, + empty, empty, empty, empty, empty, + empty, empty, empty, empty, empty, + empty, empty, empty, 280, 150 + ] +} + + +rate_card_data_northhumberland_country_shdf_wave_3 = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" + ], + "rate": [ + empty, empty, empty, empty, empty, + empty, empty, empty, empty, empty, + empty, empty, empty, empty, empty, + empty, empty, empty, 280, 130, + ] +} + +rate_card_data_walk_ups = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" + ], + "rate": [ + 259, 125, 280, 125, 125, + 650, 415, 195, 175, 135, + 120, "60 to check", 85, 125, 60, + 45, 45, 45, empty, empty, + ] +} + + +# rate_card_df = pd.DataFrame(rate_card_data_example) +# rate_card_df = pd.DataFrame(rate_card_data_2502_accent_housing) +# rate_card_df = pd.DataFrame(rate_card_data_l_and_q_london) +# rate_card_df = pd.DataFrame(rate_card_data_northhumberland_country_shdf_wave_3) +rate_card_df = pd.DataFrame(rate_card_data_walk_ups) + + +for board in tqdm(board_ids): + print(f"working on board {board}") + board_data = monday.boards.fetch_boards_by_id(board) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} + reversed_col_id_map = {v: k for k, v in col_id_map.items()} + + + items = get_all_items(board, monday) + + all_records = [] + for row in tqdm(items): + data = {} + data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) + for col in row.get("column_values", []): + if col.get("id") in reversed_col_id_map: + if col.get("type") == "file": + value = col.get("value") + no_of_files = 0 + + if value: + value = json.loads(col["value"]) + no_of_files = len(value.get('files', [])) + data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + else: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + all_records.append(data) + +# Convert to DataFrame +df = pd.DataFrame(all_records) + +filtered_dfs = [] + + +def get_df(df, column_name, success_critera, job_name=None): + _ = pd.DataFrame() + if column_name in col_id_map: + _ = df[ + df[column_name].str.lower().isin(success_critera) + ].copy() + if job_name: + _["job_type"] = job_name + + + return _ + + +# RA +ra = get_df(df, "ra invoicing status", ["to invoice"], "RA") +if not ra.empty: + filtered_dfs.append(ra) + + +att = get_df(df, "post att invoicing status", ["to invoice"], "ATT") +if not att.empty: + filtered_dfs.append(att) + +modeling = get_df(df, "mtp invoicing status", ["modelling to invoice"], "Measure Modelling") +if not modeling.empty: + filtered_dfs.append(modeling) + +try: + # Only needed for one board in wave 3 + full_cost = get_df(df, "mtp invoicing status", ["(V1) Full cost MTP to invoice (no previous modelling)".lower()], "full cost mtp") + if not full_cost.empty: + filtered_dfs(full_cost) +except Exception as e: + print(e) + +v1 = get_df(df, "mtp invoicing status", ["(v1) ioe/mtp to invoice"], "Coordination Stage 1 v1") +if not v1.empty: + filtered_dfs.append(v1) + +v2 = get_df(df, "mtp invoicing status", ["(v2) ioe/mtp to invoice"], "Coordination Stage 1 v2 remodel") +if not v2.empty: + filtered_dfs.append(v2) + +v3 = get_df(df, "mtp invoicing status", ["(v3) ioe/mtp to invoice"], "Coordination Stage 1 v3 remodel") +if not v3.empty: + filtered_dfs.append(v3) + +# Coordination stage 2 Please complete +cors2 = get_df(df, "rc stage 2", ["to invoice"], "Coordination Stage 2") +if not cors2.empty: + filtered_dfs.append(cors2) + +# Design archetype complex +design = get_df(df, "design invoicing status", ["to invoice"]) +design1 = get_df(design, "design invoice type", ["archetype (complex)"], "Design Archetype Complex") +if not design1.empty : + filtered_dfs.append(design1) + +# Design archetype simple +design1 = get_df(design, "design invoice type", ["archetype (simple)"], "Design Archetype Simple") +if not design1.empty: + filtered_dfs.append(design1) + +# Design repetitive simple +design1 = get_df(design, "design invoice type", ["archetype (simple)"], "Design Archetype repetitive") +if not design1.empty: + filtered_dfs.append(design1) + +# Design repetitive complex +design1 = get_df(design, "design invoice type", ["archetype (complex)"], "Design Archetype complex") +if not design1.empty: + filtered_dfs.append(design1) + +# Design Revision +revision_letter = ['a', 'b', 'c', 'd'] +for letter in revision_letter: + design = get_df(df, "design revision invoice", [f"rev. {letter} to invoice"], "Design Revision") + if not design.empty: + filtered_dfs.append(design) + +# Lodgement Phase 1 +lodg1 = get_df(df, "lodgement phase 1 invoicing status", ["to invoice"], "Lodgement Phase 1") +if not lodg1.empty: + filtered_dfs(lodg1) + +# Full Lodgement Phase +lodg2 = get_df(df, "full lodgement invoicing status", ["to invoice"], "Full lodgement phase 2") +if not lodg2.empty: + filtered_dfs.append(lodg2) + +# POST EPC +post_epc = get_df(df, "post epc & eval. invoicing status", ["epc to invoice"], "POST EPC") +if not post_epc.empty: + filtered_dfs.append(post_epc) + + +# POST EPR +post_epr = get_df(df, "post epc & eval. invoicing status", ["epr to invoice"], "POST EPR") +if not post_epr.empty: + filtered_dfs.append(post_epr) + +# post att +post_att = get_df(df, "post att invoicing status", ["to invoice"], "POST ATT") +if not post_att.empty: + filtered_dfs.append(post_epc) + +# Retrofit Evaluation +rc = get_df(df, "rc stage 2 invoicing status", ["to invoice"], "retrofit evaluation") +if not rc.empty: + filtered_dfs.append(rc) + +# RA NO Show +ra_ns = get_df(df,"ra no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "RA NO SHOW") +if not ra_ns.empty: + filtered_dfs.append(ra_ns) + + +# ATT NO Show +att_ns = get_df(df, "pre att no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "ATT NO SHOW") +if not att_ns.empty: + filtered_dfs.append(att_ns) + + +# Post visit no show +epc_ns = get_df(df, "post works no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "post EPC NO SHOW") +if not epc_ns.empty: + filtered_dfs.append(epc_ns) + +final_df = pd.concat(filtered_dfs).reset_index(drop=True) + +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_excel(f'WCHG Walk up {timestamp}.xlsx', index=False) diff --git a/etl/scraper/scraper.py b/etl/scraper/scraper.py index fb289cd..b4f0691 100644 --- a/etl/scraper/scraper.py +++ b/etl/scraper/scraper.py @@ -20,8 +20,8 @@ WEEK_COMMENCING = os.getenv("WEEK_COMMENCING", previous_monday()) class SharePointInstaller(Enum): # https//{tenant}.sharepoint.com/sites/{site}/_api/site/id - SOUTH_COAST_INSULATION = os.getenv("SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID", None) - JJC = os.getenv("JJC_SERVICE_SHAREPOINT_ID", None) + SOUTH_COAST_INSULATION = os.getenv("SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID", "b5a51507-9427-4ee0-b03e-90ec7681e2d3") + JJC = os.getenv("JJC_SERVICE_SHAREPOINT_ID", "7fdd0485-bbf3-4b29-b30f-98c81c2a6284") SGEC = os.getenv("SGEC_SERVICE_SHAREPOINT_ID", "52018e5c-3215-4fe4-a4e3-bbf0d0aa7cd9") BAXTER_KELLY = os.getenv("BAXTER_KELLY_SERVICE_SHAREPOINT_ID", "6f930bf3-572d-4f91-b1ae-ec536fa319e2") DOMNA = os.getenv("DOMNA_SHAREPOINT_ID", "8ab64924-ccde-4b56-b0dc-4e11596446e4")