From 233a64f03c050cf76618c594c6ad233958c7c2e5 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 31 Jul 2025 14:29:03 +0000 Subject: [PATCH] submit this code --- etl/hubSpotClient/hubspot.py | 9 +-- etl/hubSpotClient/types.py | 22 +++--- etl/hubspot_surveyed_needs_sign_off.py | 4 +- etl/month_end_automation_wave_2_layout.py | 14 ++-- etl/month_end_automation_wave_2_no_12.py | 17 +++-- etl/month_end_automation_wave_2_no_14.py | 58 ++++++++++------ etl/month_end_automation_wave_2_no_15.py | 45 +++++++----- etl/month_end_automation_wave_2_no_3.py | 15 ++-- etl/month_end_automation_wave_2_no_4.py | 34 +++++---- etl/month_end_automation_wave_2_no_7.py | 85 +++++++++++++++++------ etl/month_end_automation_wave_2_no_8.py | 27 ++++--- etl/month_end_automation_wave_3_layout.py | 64 +++++++++++++++-- 12 files changed, 269 insertions(+), 125 deletions(-) diff --git a/etl/hubSpotClient/hubspot.py b/etl/hubSpotClient/hubspot.py index 269862c..27e6903 100644 --- a/etl/hubSpotClient/hubspot.py +++ b/etl/hubSpotClient/hubspot.py @@ -18,7 +18,7 @@ class DealStage(Enum): SURVEYED_NO_ACCESS_NEED_SIGN_OFF = "1617223915" CUSTOMER_CONTACTED = "888730834" SURVEYED_COMPLETED_SIGNED_OFF = "1617223916" - NEEDS_ADDITIONAL_INFORMATION_FROM_ASSESSOR = "1887736000" + FILES_MISSING_FROM_ASSESSOR = "1887736000" class HubSpotClient(): def __init__(self): @@ -226,9 +226,6 @@ class HubSpotClient(): domna_id = domna_id, uprn = uprn, )) - if i > 1: - # Break after just for speed - break except Exception as e: def format_error_note(e): note_text = "⚠️ Automated Verification Failed:

" @@ -266,10 +263,8 @@ class HubSpotClient(): self.add_note_to_deal(deal_id, format_error_note(e)) else: self.logger.error(f"Non-validation error occurred: {str(e)}", exc_info=True) - - self.logger.info(f"Deal name <{deal_name}> moving to 'needs additional information'") - self.move_deals_to_different_stage([deal_id], DealStage.NEEDS_ADDITIONAL_INFORMATION_FROM_ASSESSOR.value) + self.move_deals_to_different_stage([deal_id], DealStage.FILES_MISSING_FROM_ASSESSOR.value) return all_deals def print_all_pipeline_ids(self): diff --git a/etl/hubSpotClient/types.py b/etl/hubSpotClient/types.py index c042d6d..35afaa0 100644 --- a/etl/hubSpotClient/types.py +++ b/etl/hubSpotClient/types.py @@ -60,14 +60,20 @@ class SubmissionInfoFromDeal(BaseModel): raise ValueError(f"Error accessing SharePoint path: {self.submission_folder_path}. Error: {str(e)}") try: - # Check if sharepoint link is reachable and has any contents - files = sp.get_folders_in_path(path) - if "value" in files and len(files["value"]) > 0: - pass - else: - raise ValueError(f"SharePoint folder is empty: {self.submission_folder_path}") + try: + files = sp.get_folders_in_path(path) + if files.get("value"): + pass + except Exception as e: + print("Trying SGEC") + sp = SharePointScraper(SharePointInstaller.SGEC) + files = sp.get_folders_in_path(path) + if files.get("value"): + pass + else: + raise ValueError(f"[SharePoint Folder Empty] Folder has no contents after multiple attempts: {self.submission_folder_path}") except Exception as e: - raise ValueError(str(e)) + raise ValueError(f"[Folder Access Error] {str(e)}") # download files in url and check files are there: try: @@ -80,7 +86,7 @@ class SubmissionInfoFromDeal(BaseModel): if sdp.condition_report is None: missing_items.append("Condition Report") - if sdp.epr_summary_information is None: + if sdp.epr_with_data is None: missing_items.append("EPR Energy report with data") if sdp.rd_sap_xml is None: diff --git a/etl/hubspot_surveyed_needs_sign_off.py b/etl/hubspot_surveyed_needs_sign_off.py index 7794d07..6b5f48c 100644 --- a/etl/hubspot_surveyed_needs_sign_off.py +++ b/etl/hubspot_surveyed_needs_sign_off.py @@ -10,6 +10,8 @@ os.environ["SHAREPOINT_CLIENT_SECRET"] = "SOf8Q~-is4wdQiqvEEm9FlJQRAY9ELGaj5Qz-a os.environ["SHAREPOINT_TENANT_ID"] = "c3f7519c-2719-4547-af04-6da6cbfd8f8f" os.environ["SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID"] = "b5a51507-9427-4ee0-b03e-90ec7681e2d3" os.environ["JJC_SERVICE_SHAREPOINT_ID"] = "7fdd0485-bbf3-4b29-b30f-98c81c2a6284" +os.environ["SGEC_SERVICE_SHAREPOINT_ID"] = "52018e5c-3215-4fe4-a4e3-bbf0d0aa7cd9" + from etl.hubSpotClient.hubspot import DealStage, HubSpotClient # Local development @@ -18,7 +20,7 @@ os.environ["DATABASE_URL"] = "postgresql://postgres:makingwarmhomes@db:5432/post hubspotClient = HubSpotClient() # files missing from assessor column -deals = hubspotClient.get_deals_from_deal_stage(DealStage.NEEDS_ADDITIONAL_INFORMATION_FROM_ASSESSOR) +deals = hubspotClient.get_deals_from_deal_stage(DealStage.FILES_MISSING_FROM_ASSESSOR) for deal in deals: diff --git a/etl/month_end_automation_wave_2_layout.py b/etl/month_end_automation_wave_2_layout.py index 5d208e2..7dca011 100644 --- a/etl/month_end_automation_wave_2_layout.py +++ b/etl/month_end_automation_wave_2_layout.py @@ -17,13 +17,13 @@ rate_card_data = { "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", - "RA no show", "ATT no show", "post EPC no show" + "RA no show", "ATT no show", "post EPC no show", "Design Revision" ], "rate": [ 207.65, 101, 186.4, 98, 98, 450, 150, 163, 135, 120, - "Mariaane Please tell me", 45, 90.5, 40, - 25, 25, 25 + "60 - Needs to be verified (Post EPR)", 45, 90.5, 40, + 25, 25, 25, "check with Kevin" ] } @@ -124,7 +124,7 @@ filtered_dfs.append(v3) cors2 = df[ df["rc stg. 2"].str.lower().isin(["to invoice"]) ].copy() -cors2["joby_type"] = "Coordination Stage 2" +cors2["job_type"] = "Coordination Stage 2" filtered_dfs.append(cors2) # Design type archietype @@ -143,9 +143,9 @@ filtered_dfs.append(design1) # Design stage revisions design2 = df[ - df["revision 2 design invoice"].str.lower().isin(["to invoice"]) + df["design revision invoice status"].str.lower().isin(["to invoice"]) ].copy() -design2["job_type"] = "Design Repetitive" +design2["job_type"] = "Design Revision" filtered_dfs.append(design2) # Lodgement Phase 1 @@ -164,7 +164,7 @@ filtered_dfs.append(lodg2) # POST EPC post_epc = df[ - df["post-epc status"].str.lower().isin(["epc pics uploaded"]) + df["post-epc status"].str.lower().isin(["epc files uploaded"]) ].copy() post_epc["job_type"] = "Post EPC" filtered_dfs.append(post_epc) diff --git a/etl/month_end_automation_wave_2_no_12.py b/etl/month_end_automation_wave_2_no_12.py index 962eb46..7fa4c1a 100644 --- a/etl/month_end_automation_wave_2_no_12.py +++ b/etl/month_end_automation_wave_2_no_12.py @@ -11,18 +11,20 @@ monday = MondayClient(monday_key) # Shropshire Council HUG2 board_ids = ["4718185486"] +empty = "nothing on rate card" + rate_card_data = { "job_type": [ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", - "RA no show", "ATT no show", "post EPC no show" + "RA no show", "ATT no show", "post EPC no show", "Design Sign off" ], "rate": [ - 259, 125, 280, 125, 125, - 650, 195, 175, 135, - 120, "Post EPR Please Marianne", 85, 125, 60, - 25, 25, 25 + "(185) - Kevin Check as this depends on property size", 115, 200, 125, 125, + empty, "(135)- Mariane said 'Remaining RC & design sign off", 185, 135, + 120, "(60) Post EPR, please verify", 65, 115, 60, + 50, 50, 50, "(60) - Please add price for design sign off, check with kevin and marianne" ] } @@ -117,7 +119,7 @@ filtered_dfs.append(v2) # Coordination stage 2 Please complete cors2 = df[ df["rc stage 2 invoice"].str.lower().isin(["to invoice"]) -] +].copy() cors2["joby_type"] = "Coordination Stage 2" filtered_dfs.append(cors2) @@ -134,6 +136,9 @@ filtered_dfs.append(cors2) # ], "Design Revision") # filtered_dfs.append(design2) +# Design sign off +design_sign_pff = get_df(df, "design payment step 3", ["ready to invoice"], "design sign off") + # Lodgement Phase 1 lodg1 = get_df(df, "tm ph1 invoice status".lower(), ["to invoice"], "Lodgement Phase 1") filtered_dfs.append(lodg1) diff --git a/etl/month_end_automation_wave_2_no_14.py b/etl/month_end_automation_wave_2_no_14.py index 9203f2b..4a644ee 100644 --- a/etl/month_end_automation_wave_2_no_14.py +++ b/etl/month_end_automation_wave_2_no_14.py @@ -11,6 +11,24 @@ monday = MondayClient(monday_key) # Decent Homes Stonewater - Operations board_ids = ["9319118237"] +empty = "Rate card info missing" +rate_card_data_example = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", + "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + 259, 125, 280, 125, + 260, 126, 281, 126, + 262, 127, 282, 127, + ] +} + + +rate_card_df = pd.DataFrame(rate_card_data_example) + for board in tqdm(board_ids): board_data = monday.boards.fetch_boards_by_id(board) columns = board_data["data"]["boards"][0]["columns"] @@ -79,25 +97,25 @@ filtered_dfs.append(ra) # ATT -# att = get_df(df, "att", ["completed"], "ATT") -# filtered_dfs.append(att) +att = get_df(df, "att", ["completed"], "ATT") +filtered_dfs.append(att) # V1 Coordination v1 = get_df(df, "v1 coordination status (ioe,mtp)".lower(), [ "rc complete", -], "V1 Coordination") +], "Coordination Stage 1 v1") filtered_dfs.append(v1) -# V2 Coordination -v2 = get_df(df, "mtp v2 status", ["rc v2 complete"], "V2 Coordination") -filtered_dfs.append(v2) +# # V2 Coordination +# v2 = get_df(df, "mtp v2 status", ["rc v2 complete"], "V2 Coordination") +# filtered_dfs.append(v2) -# # V3 Coordination -v3 = get_df(df, "v3 rc status", ["uploaded"], "V3 Coordination") -filtered_dfs.append(v3) +# # # V3 Coordination +# v3 = get_df(df, "v3 rc status", ["uploaded"], "V3 Coordination") +# filtered_dfs.append(v3) -v3 = get_df(df, "v3 invoice status", ["to be invoice"], "V3 Coordination") -filtered_dfs.append(v3) +# v3 = get_df(df, "v3 invoice status", ["to be invoice"], "V3 Coordination") +# filtered_dfs.append(v3) # Coordination stage 2 Please complete # cors2 = df[ @@ -107,8 +125,8 @@ filtered_dfs.append(v3) # filtered_dfs.append(cors2) # Design stage 1 -design1 = get_df(df, "design invoice status", ["to invoice"], "Design") -filtered_dfs.append(design1) +# design1 = get_df(df, "design invoice status", ["to invoice"], "Design") +# filtered_dfs.append(design1) # Design revision # design2 = get_df(df, "design revision invoice", [ @@ -125,7 +143,7 @@ filtered_dfs.append(lodg1) # Full Lodgement Phase -full_lodgement = get_df(df, "lodgement invoice status".lower(), ["to invoice"], "Full Lodgement") +full_lodgement = get_df(df, "lodgement invoice status".lower(), ["to invoice"], "Full lodgement phase 2") filtered_dfs.append(full_lodgement) # POST EPC @@ -133,12 +151,12 @@ post_epc = get_df(df, "post epc", ["completed & uploaded"], "POST EPC") filtered_dfs.append(post_epc) -# # POST EPR -# post_epr = df[ -# df["post-epc status"].str.lower().isin(["post epr completed"]) -# ].copy() -# post_epr["job_type"] = "POST epr" -# filtered_dfs.append(post_epr) +# POST EPR +post_epr = df[ + df["post epc"].str.lower().isin(["post epr completed"]) +].copy() +post_epr["job_type"] = "POST epr" +filtered_dfs.append(post_epr) # Post ATT post_att = get_df(df, "post att", ["completed & uploaded"], "POST ATT") diff --git a/etl/month_end_automation_wave_2_no_15.py b/etl/month_end_automation_wave_2_no_15.py index 3686198..8644db9 100644 --- a/etl/month_end_automation_wave_2_no_15.py +++ b/etl/month_end_automation_wave_2_no_15.py @@ -11,18 +11,21 @@ monday = MondayClient(monday_key) # WCHG SHDF 2.1 Mansard board_ids = ["5636990610"] + +empty = "Rate card was empty" + rate_card_data = { "job_type": [ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", - "RA no show", "ATT no show", "post EPC no show" + "RA no show", "ATT no show", "post EPC no show", "Design Revision" ], "rate": [ - 259, 125, 280, 125, 125, - 650, 195, 175, 135, - 120, "Post EPR Please Marianne", 85, 125, 60, - 25, 25, 25 + "259 (new RA rate for PAS2035:2023 - old rates for other works - to discuss with KN)", 40, 178.5, empty, empty, + empty, 180, 275, 135, + 120, "60 - please verify with Marianne", 45, 45, 40, + 25, 25, 25, "Please check price for design revision with Andreas" ] } @@ -84,11 +87,12 @@ df = pd.DataFrame(all_records) filtered_dfs = [] -def get_df(df, column_name, success_critera, job_name): +def get_df(df, column_name, success_critera, job_name=None): _ = df[ df[column_name].str.lower().isin(success_critera) ].copy() - _["job_type"] = job_name + if job_name: + _["job_type"] = job_name return _ # RA @@ -107,8 +111,8 @@ v1 = get_df(df, "coordination status (ioe mtp)".lower(), [ filtered_dfs.append(v1) # V2 Coordination -# v2 = get_df(df, "mtp v2 status", ["rc v2 complete"], "V2 Coordination") -# filtered_dfs.append(v2) +v2 = get_df(df, "v2 coordination status (ioe mtp)", ["rc complete"], "Coordination Stage 1 v2 remodel") +filtered_dfs.append(v2) # # V3 Coordination # v3 = get_df(df, "v3 rc status", ["uploaded"], "V3 Coordination") @@ -125,18 +129,21 @@ cors2 = df[ cors2["joby_type"] = "Coordination Stage 2" filtered_dfs.append(cors2) -# Design stage 1 -# design1 = get_df(df, "design invoice status", ["to invoice"], "Design") -# filtered_dfs.append(design1) +# Design stage +design = get_df(df, "design invoice status", ["to invoice"]) + +# Design archeytpe +de = get_df(design, "prop type for invoicing", ["archetype"], "Design Archetype") +filtered_dfs.append(de) +# Design repetitive +de = get_df(design, "prop type for invoicing", ["repetitive"], "Design Repetitive") +filtered_dfs.append(de) # Design revision -# design2 = get_df(df, "design revision invoice", [ -# "Rev. A to invoice".lower(), -# "Rev. B to invoice".lower(), -# "Rev. C to invoice".lower(), -# "Rev. D to invoice".lower(), -# ], "Design Revision") -# filtered_dfs.append(design2) +design2 = get_df(df, "design revision invoice status", [ + "to invoice".lower(), +], "Design Revision") +filtered_dfs.append(design2) # Lodgement Phase 1 lodg1 = get_df(df, "tm phase 1 invoice satus (lodgment)".lower(), ["to invoice"], "Lodgement Phase 1") diff --git a/etl/month_end_automation_wave_2_no_3.py b/etl/month_end_automation_wave_2_no_3.py index 6e497c5..b349a7a 100644 --- a/etl/month_end_automation_wave_2_no_3.py +++ b/etl/month_end_automation_wave_2_no_3.py @@ -17,13 +17,13 @@ rate_card_data = { "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", - "RA no show", "ATT no show", "post EPC no show" + "RA no show", "ATT no show", "post EPC no show", "Design Revision" ], "rate": [ - 259, 125, 280, 125, 125, - 650, 195, 175, 135, - 120, "Post EPR Please Marianne", 85, 125, 60, - 25, 25, 25 + 259, 101, 210, 95, 95, + 450, 150, 195, 135, + 120, "(60)) - please confirm with Marianne, EPR", 45, 90.5, 42.4, + 25, 25, 25, "Please ask for Design Revision" ] } @@ -143,6 +143,11 @@ design1["job_type"] = "Design repetitive" filtered_dfs.append(design1) # Design Revision +design_revision = df[ + df["design revision invoice status"].str.lower().isin(["to invoice"]) +].copy() +design_revision["job_type"] = "Design repetitive" +filtered_dfs.append(design_revision) # Lodgement Phase 1 lodg1 = df[ diff --git a/etl/month_end_automation_wave_2_no_4.py b/etl/month_end_automation_wave_2_no_4.py index 1c17c7e..30b949f 100644 --- a/etl/month_end_automation_wave_2_no_4.py +++ b/etl/month_end_automation_wave_2_no_4.py @@ -17,13 +17,13 @@ rate_card_data = { "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", - "RA no show", "ATT no show", "post EPC no show" + "RA no show", "ATT no show", "post EPC no show", "Design Revision" ], "rate": [ - 259, 125, 280, 125, 125, - 650, 195, 175, 135, - 120, "Post EPR Please Marianne", 85, 125, 60, - 25, 25, 25 + 165.75, 72.25, 174.25, 174.25, 174.25, + 175, 175, 124.25, 135, + 120, "(60) - please check with Marianne", 45, 63.75, 34, + 25, 25, 25, "Please ask marianne or kev for design revision" ] } @@ -107,7 +107,12 @@ v1["job_type"] = "Coordination Stage 1 v1" filtered_dfs.append(v1) # V2 Coordination -_ = df[df["mtp v2 invoiced"].str.lower().isin(['done', 'needs to be invoiced'])].copy() +_ = df[df["mtp v2 status"].str.lower().isin(['rc v2 complete'])].copy() +_["job_type"] = "Coordination Stage 1 v2 remodel" +filtered_dfs.append(_) + +# V2 Coordination +_ = df[df["mtp v2 invoiced"].str.lower().isin(['needs to be invoiced'])].copy() _["job_type"] = "Coordination Stage 1 v2 remodel" filtered_dfs.append(_) @@ -116,6 +121,11 @@ v3 = df[df["v3 rc status"].str.lower().isin(['uploaded'])].copy() v3["job_type"] = "Coordination Stage 1 v3 remodel" filtered_dfs.append(_) +# V3 Coordination +v3 = df[df["v3 invoice status"].str.lower().isin(['to be invoiced'])].copy() +v3["job_type"] = "Coordination Stage 1 v3 remodel" +filtered_dfs.append(_) + # Coordination stage 2 Please complete cors2 = df[ df["rc stg. 2 status"].str.lower().isin(["to invoice", "completed"]) @@ -139,12 +149,12 @@ design1 = design1[design1["design type"].str.lower().isin(["repetitive"])].copy( design1["job_type"] = "Design Repetitive" filtered_dfs.append(design1) -# Design revision -# design2 = df[ -# df[""].str.lower().isin(["to invoice"]) -# ].copy() -# design2["job_type"] = "Design Revision" -# filtered_dfs.append(design2) +# Design Revision +design_revision = df[ + df["design revision invoice status"].str.lower().isin(["to invoice"]) +].copy() +design_revision["job_type"] = "Design repetitive" +filtered_dfs.append(design_revision) # Lodgement Phase 1 lodg1 = df[ diff --git a/etl/month_end_automation_wave_2_no_7.py b/etl/month_end_automation_wave_2_no_7.py index 0cc1749..214c233 100644 --- a/etl/month_end_automation_wave_2_no_7.py +++ b/etl/month_end_automation_wave_2_no_7.py @@ -12,7 +12,7 @@ monday = MondayClient(monday_key) #Home Group Wave 2SP+ board_ids = ["4254419092"] -rate_card_data = { +rate_card_data_sp_plus = { "job_type": [ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", @@ -20,14 +20,48 @@ rate_card_data = { "RA no show", "ATT no show", "post EPC no show" ], "rate": [ - 259, 125, 280, 125, 125, - 650, 195, 175, 135, - 120, "Post EPR Please Marianne", 85, 125, 60, - 25, 25, 25 + 170, 70, 200, "check with Kevin", "check with Kevin", + 470, 155, 165, 135, + 120, "60 but check with Kevin as EPR", 45, 70, 40, + 30, 30, 30 ] } -rate_card_df = pd.DataFrame(rate_card_data) +emp_msg = "was empty in rate card - ask Marianne/Kevin" +rate_card_data_net_zero = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + 170, 70, 200, emp_msg, emp_msg, + 325, 140, 165, 135, + 120, "60 but check with Kevin as EPR", 45, 70, 40, + 30, 30, 30 + ] +} + +error_message = "Unsure which client this one is - sorry!" +rate_card_data_error_msg= { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + error_message, error_message, error_message, error_message, error_message, + error_message, error_message, error_message, error_message, + error_message, error_message, error_message, error_message, error_message, + error_message, error_message, error_message + ] +} + +rate_card_df_sp_plus = pd.DataFrame(rate_card_data_sp_plus) +rate_card_df_net_zero = pd.DataFrame(rate_card_data_net_zero) +rate_card_df_error_message = pd.DataFrame(rate_card_data_error_msg) for board in tqdm(board_ids): @@ -113,9 +147,9 @@ filtered_dfs.append(v1) v2 = get_df(df, "v2 ioe mtp", ["completed"], "Coordination Stage 1 v2 remodel") filtered_dfs.append(v2) -# # V3 Coordination -# v3 = get_df(df, "") -# # filtered_dfs.append(v3) +# V3 Coordination +v3 = get_df(df, "v3 rc status", ["rc completed"], "Coordination Stage 1 v3 remodel") +filtered_dfs.append(v3) # Coordination stage 2 Please complete cors2 = df[ @@ -135,13 +169,10 @@ design1 = get_df(design1, "design type for invoicing", ["repetitive"], "Design R filtered_dfs.append(design1) # Design revision -# design2 = get_df(df, "design revision invoice", [ -# "Rev. A to invoice".lower(), -# "Rev. B to invoice".lower(), -# "Rev. C to invoice".lower(), -# "Rev. D to invoice".lower(), -# ], "Design Revision") -# filtered_dfs.append(design2) +design2 = get_df(df, "design revision invoice status", [ + "to invoice" +], "Design Revision") +filtered_dfs.append(design2) # Lodgement Phase 1 lodg1 = get_df(df, "TM Phase 1 Invoicing Status".lower(), ["done", "to invoice"], "Lodgement phase 1") @@ -164,7 +195,7 @@ post_epr["job_type"] = "POST epr" filtered_dfs.append(post_epr) # Post ATT -post_att = get_df(df, "post-att status", ["complete & uploaded"], "POST ATT") +post_att = get_df(df, "post att invoicing status ", ["to invoice"], "POST ATT") filtered_dfs.append(post_att) @@ -201,12 +232,26 @@ filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) final_df["job_type"] = final_df["job_type"].str.lower() -rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() +rate_card_df_net_zero["job_type"] = rate_card_df_net_zero["job_type"].str.lower() +rate_card_df_sp_plus["job_type"] = rate_card_df_sp_plus["job_type"].str.lower() +rate_card_df_error_message["job_type"] = rate_card_df_error_message["job_type"].str.lower() # Now perform the merge -combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +net_zero_df = final_df[final_df['client'].str.contains('shdf net zero'.lower(), case=False, na=False)] +sp_plus_df = final_df[final_df['client'].str.contains('SHDF 2.0 SP+'.lower(), case=False, na=False)] +other_df = final_df[~final_df.index.isin(net_zero_df.index) & ~final_df.index.isin(sp_plus_df.index)] + +combined_with_rates_net_zero_df = net_zero_df.merge(rate_card_df_net_zero, on="job_type", how="left") +combined_with_rates_sp_plus = sp_plus_df.merge(rate_card_df_sp_plus, on="job_type", how="left") +combined_with_rates_other_from_home_group = other_df.merge(rate_card_df_error_message, on="job_type", how="left") import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'HomeGroup Wave 2SP+_{timestamp}.xlsx', index=False) +combined_with_rates_sp_plus[attribute].to_excel(f'HomeGroup Wave 2SP+_{timestamp}.xlsx', index=False) +combined_with_rates_net_zero_df[attribute].to_excel(f'HomeGroup Wave NetZero_{timestamp}.xlsx', index=False) +combined_with_rates_other_from_home_group[attribute].to_excel(f'HomeGroup Wave Unsure_who_to_bill_{timestamp}.xlsx', index=False) + + +# TO DO check everything in excel +# make logic for seperation \ No newline at end of file diff --git a/etl/month_end_automation_wave_2_no_8.py b/etl/month_end_automation_wave_2_no_8.py index d8f0a35..0aff6fc 100644 --- a/etl/month_end_automation_wave_2_no_8.py +++ b/etl/month_end_automation_wave_2_no_8.py @@ -16,13 +16,13 @@ rate_card_data = { "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", - "RA no show", "ATT no show", "post EPC no show" + "RA no show", "ATT no show", "post EPC no show", "Design Revision" ], "rate": [ 259, 125, 280, 125, 125, 650, 415, 195, 175, 135, - 120, "Post EPR Please Marianne", 85, 125, 60, - 25, 25, 25 + 120, "60 - Double check with Kevin/Marianne EPR", 85, 125, 60, + 45, 45, 45, "Design Revision check with Kevin/Marianne" ] } @@ -139,13 +139,10 @@ design1 = get_df(design1, "design upload to sharepoint", ["done"], "Design Repet filtered_dfs.append(design1) # Design revision -# design2 = get_df(df, "design revision invoice", [ -# "Rev. A to invoice".lower(), -# "Rev. B to invoice".lower(), -# "Rev. C to invoice".lower(), -# "Rev. D to invoice".lower(), -# ], "Design Revision") -# filtered_dfs.append(design2) +design2 = get_df(df, "design revision invoice status", [ + "to invoice" +], "Design Revision") +filtered_dfs.append(design2) # Lodgement Phase 1 lodg1 = get_df(df, "Lodg. Phase 1 Invoice Status".lower(), ["to be invoiced"], "Lodgement Phase 1") @@ -199,11 +196,11 @@ filtered_dfs.append(att_ns) # Post visit no show -# epc_ns = df[ -# df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999) -# ].copy() -# epc_ns["job_type"] = "post EPC NO SHOW" -# filtered_dfs.append(epc_ns) +epc_ns = df[ + df["post works no show evidence"].fillna(-9999) != df["post works no show invoice"].fillna(-9999) +].copy() +epc_ns["job_type"] = "post EPC NO SHOW" +filtered_dfs.append(epc_ns) final_df = pd.concat(filtered_dfs).reset_index(drop=True) diff --git a/etl/month_end_automation_wave_3_layout.py b/etl/month_end_automation_wave_3_layout.py index 7e273f2..673ea66 100644 --- a/etl/month_end_automation_wave_3_layout.py +++ b/etl/month_end_automation_wave_3_layout.py @@ -10,13 +10,15 @@ import json monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) board_ids = [ - # "9349630181", # WCHG Walkups-Operations +# "9349630181", # WCHG Walkups-Operations # "8829428746", # 2502 Accent Housing - "8830772914", # "L&Q London" + # "8830772914", # "L&Q London" # "9601691730", # Cardo Wales & West - Wave 3 + "9660895490", # Northumberland County SHDF Wave 3 ] -rate_card_data = { +empty = "Rate card info missing" +rate_card_data_example = { "job_type": [ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", @@ -31,7 +33,59 @@ rate_card_data = { ] } -rate_card_df = pd.DataFrame(rate_card_data) + +rate_card_data_2502_accent_housing = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" + ], + "rate": [ + empty, empty, empty, empty, empty, + empty, empty, empty, empty, empty, + empty, empty, empty, empty, empty, + empty, empty, empty, 280, 150 + ] +} + +rate_card_data_l_and_q_london = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" + ], + "rate": [ + empty, empty, empty, empty, empty, + empty, empty, empty, empty, empty, + empty, empty, empty, empty, empty, + empty, empty, empty, 280, 150 + ] +} + + +rate_card_data_northhumberland_country_shdf_wave_3 = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" + ], + "rate": [ + empty, empty, empty, empty, empty, + empty, empty, empty, empty, empty, + empty, empty, empty, empty, empty, + empty, empty, empty, 280, 150 + ] +} + + + +# rate_card_df = pd.DataFrame(rate_card_data_example) +# rate_card_df = pd.DataFrame(rate_card_data_2502_accent_housing) +# rate_card_df = pd.DataFrame(rate_card_data_l_and_q_london) +rate_card = pd.DataFrame(rate_card_data_northhumberland_country_shdf_wave_3) for board in tqdm(board_ids): @@ -210,4 +264,4 @@ import datetime timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'WCHG Walkups-Operations {timestamp}.xlsx', index=False) +combined_with_rates[attribute].to_excel(f'L&Q London {timestamp}.xlsx', index=False)