diff --git a/.github/workflows/months_end.yml b/.github/workflows/months_end.yml index 80955ad..f014aa0 100644 --- a/.github/workflows/months_end.yml +++ b/.github/workflows/months_end.yml @@ -1,10 +1,9 @@ name: Months End on: - # schedule: - # # - cron: '0 17 * * 1-5' - #adslfhjk asdlfjldsakjf + schedule: + - cron: '0 7 29 * *' # Runs at 7:00 AM UTC on the 29th of every month push: - branches: [main, feature/month_end_automation_of_all] + branches: [main] workflow_dispatch: jobs: @@ -43,5 +42,6 @@ jobs: poetry run python etl/month_end_automation_wave_2_no_15.py poetry run python etl/month_end_automation_wave_2_no_16.py poetry run python etl/month_end_automation_wave_accent_housing.py + poetry run python etl/month_end_automation_wave_3_layout.py env: PYTHONPATH: ${{ github.workspace }} \ No newline at end of file diff --git a/etl/month_end_automation_wave_3_layout.py b/etl/month_end_automation_wave_3_layout.py index 704668c..32125a4 100644 --- a/etl/month_end_automation_wave_3_layout.py +++ b/etl/month_end_automation_wave_3_layout.py @@ -6,18 +6,37 @@ from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset from pprint import pprint import pandas as pd import json +import os +from MonthEndUploader import upload_to_month_end_folder +from enum import Enum + monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" monday = MondayClient(monday_key) + +class BoardID(Enum): + _9349630181 = "WCHG_WALKUPS_OPERATIONS" + _8830772914 = "LQ_LONDON" + _9601691730 = "CARDO_WALES_WEST_WAVE3" + _9660895490 = "NORTHUMBERLAND_COUNTY_SHDF_WAVE3" + _9641491000 = "WATFORD_WARM_HOMES" + _9671463094 = "SEDDON" + + board_ids = [ - # "9349630181", # WCHG Walkups-Operations - # "8830772914", # "L&Q London" - # "9601691730", # Cardo Wales & West - Wave 3 - # "9660895490", # Northumberland County SHDF Wave 3 - "9641491000", #Watford Warm Homes + "9349630181", # WCHG Walkups-Operations x + "8830772914", # "L&Q London" x + "9601691730", # Cardo Wales & West - Wave 3 x + "9660895490", # Northumberland County SHDF Wave 3 x + "9641491000", # Watford Warm Homes x + "9671463094", # Seddon ] empty = "Rate card info missing" + +rate_cards = {} + + rate_card_data_watford_warm_homes = { "job_type": [ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", @@ -32,22 +51,9 @@ rate_card_data_watford_warm_homes = { empty, empty, empty, empty, empty, ] } +rate_cards.update({"9641491000": pd.DataFrame(rate_card_data_watford_warm_homes)}) -rate_card_data_2502_accent_housing = { - "job_type": [ - "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", - "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", - "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", - "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" - ], - "rate": [ - empty, empty, empty, empty, empty, - empty, empty, empty, empty, empty, - empty, empty, empty, empty, empty, - empty, empty, empty, 280, 150 - ] -} rate_card_data_l_and_q_london = { "job_type": [ @@ -64,22 +70,27 @@ rate_card_data_l_and_q_london = { ] } +rate_cards.update({"8830772914": pd.DataFrame(rate_card_data_l_and_q_london)}) + rate_card_data_northhumberland_country_shdf_wave_3 = { "job_type": [ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", - "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Design Repetitive Complex", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" ], "rate": [ - empty, empty, empty, empty, empty, - empty, empty, empty, empty, empty, - empty, empty, empty, empty, empty, - empty, empty, empty, 280, 130, + 259, 125, "280 or 310 depending on SH or LA", 125, 125, + 650, 415, 195, 225, 175, + 135, 120, "60 - check", "85 or 90 depending on SH or LA", "110 or 125 depending on SH or LA", + 60, 45, 45, 45, empty, empty ] } +pd.DataFrame(rate_card_data_northhumberland_country_shdf_wave_3) +rate_cards.update({"9660895490": pd.DataFrame(rate_card_data_northhumberland_country_shdf_wave_3)}) + rate_card_data_walk_ups = { "job_type": [ "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", @@ -95,14 +106,59 @@ rate_card_data_walk_ups = { ] } -rate_card_df = pd.DataFrame(rate_card_data_watford_warm_homes) -# rate_card_df = pd.DataFrame(rate_card_data_example) -# rate_card_df = pd.DataFrame(rate_card_data_2502_accent_housing) -# rate_card_df = pd.DataFrame(rate_card_data_l_and_q_london) -# rate_card_df = pd.DataFrame(rate_card_data_northhumberland_country_shdf_wave_3) -# rate_card_df = pd.DataFrame(rate_card_data_walk_ups) +rate_cards.update({"9349630181": pd.DataFrame(rate_card_data_walk_ups)}) + +rate_card_data_empty = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" + ], + "rate": [ + empty, empty, empty, empty, empty, + empty, empty, empty, empty, empty, + empty,empty , empty, empty, empty, + empty, empty, empty, empty, empty, + ] +} + +rate_card_data_cardo = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple","Design Repetitive Complex", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" + ], + "rate": [ + 259, 125, 260, 125, 125, + 415, 415, 225, 225, 175, + 135, 120 , "60 - check", 85, 125, + 60, 45, 45, 45, empty, empty, + ] +} + +rate_cards.update({"9601691730": pd.DataFrame(rate_card_data_cardo)}) + +rate_card_data_seddon = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple","Design Repetitive Complex", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" + ], + "rate": [ + 259, 125, 280, 125, 125, + 650, 415, 195, 225, 175, + 135, 120 , "60 - check", 85, 125, + 60, 45, 45, 45, empty, empty, + ] +} + +rate_cards.update({"9671463094": pd.DataFrame(rate_card_data_seddon)}) +board_to_record = {} for board in tqdm(board_ids): print(f"working on board {board}") board_data = monday.boards.fetch_boards_by_id(board) @@ -133,150 +189,169 @@ for board in tqdm(board_ids): reversed_col_id_map[col.get("id")]: col.get("text") }) all_records.append(data) + board_to_record.update({board: all_records}) # Convert to DataFrame -df = pd.DataFrame(all_records) - -filtered_dfs = [] +for board, all_records in board_to_record.items(): + df = pd.DataFrame(all_records) + filtered_dfs = [] -def get_df(df, column_name, success_critera, job_name=None): - _ = pd.DataFrame() - if column_name in col_id_map: - _ = df[ - df[column_name].str.lower().isin(success_critera) - ].copy() - if job_name: - _["job_type"] = job_name - - - return _ + def get_df(df, column_name, success_critera, job_name=None): + _ = pd.DataFrame() + if column_name in col_id_map: + _ = df[ + df[column_name].str.lower().isin(success_critera) + ].copy() + if job_name: + _["job_type"] = job_name + + + return _ -# RA -ra = get_df(df, "ra invoicing status", ["to invoice"], "RA") -if not ra.empty: - filtered_dfs.append(ra) + # RA + ra = get_df(df, "ra invoicing status", ["to invoice"], "RA") + if not ra.empty: + filtered_dfs.append(ra) -att = get_df(df, "post att invoicing status", ["to invoice"], "ATT") -if not att.empty: - filtered_dfs.append(att) + att = get_df(df, "post att invoicing status", ["to invoice"], "ATT") + if not att.empty: + filtered_dfs.append(att) -modeling = get_df(df, "mtp invoicing status", ["modelling to invoice"], "Measure Modelling") -if not modeling.empty: - filtered_dfs.append(modeling) + modeling = get_df(df, "mtp invoicing status", ["modelling to invoice"], "Measure Modelling") + if not modeling.empty: + filtered_dfs.append(modeling) -try: - # Only needed for one board in wave 3 - full_cost = get_df(df, "mtp invoicing status", ["(V1) Full cost MTP to invoice (no previous modelling)".lower()], "full cost mtp") - if not full_cost.empty: - filtered_dfs(full_cost) -except Exception as e: - print(e) + try: + # Only needed for one board in wave 3 + full_cost = get_df(df, "mtp invoicing status", ["(V1) Full cost MTP to invoice (no previous modelling)".lower()], "full cost mtp") + if not full_cost.empty: + filtered_dfs(full_cost) + except Exception as e: + print(e) -v1 = get_df(df, "mtp invoicing status", ["(v1) ioe/mtp to invoice"], "Coordination Stage 1 v1") -if not v1.empty: - filtered_dfs.append(v1) + v1 = get_df(df, "mtp invoicing status", ["(v1) ioe/mtp to invoice"], "Coordination Stage 1 v1") + if not v1.empty: + filtered_dfs.append(v1) -v2 = get_df(df, "mtp invoicing status", ["(v2) ioe/mtp to invoice"], "Coordination Stage 1 v2 remodel") -if not v2.empty: - filtered_dfs.append(v2) + v2 = get_df(df, "mtp invoicing status", ["(v2) ioe/mtp to invoice"], "Coordination Stage 1 v2 remodel") + if not v2.empty: + filtered_dfs.append(v2) -v3 = get_df(df, "mtp invoicing status", ["(v3) ioe/mtp to invoice"], "Coordination Stage 1 v3 remodel") -if not v3.empty: - filtered_dfs.append(v3) + v3 = get_df(df, "mtp invoicing status", ["(v3) ioe/mtp to invoice"], "Coordination Stage 1 v3 remodel") + if not v3.empty: + filtered_dfs.append(v3) -# Coordination stage 2 Please complete -cors2 = get_df(df, "rc stage 2", ["to invoice"], "Coordination Stage 2") -if not cors2.empty: - filtered_dfs.append(cors2) + # Coordination stage 2 Please complete + cors2 = get_df(df, "rc stage 2", ["to invoice"], "Coordination Stage 2") + if not cors2.empty: + filtered_dfs.append(cors2) -# Design archetype complex -design = get_df(df, "design invoicing status", ["to invoice"]) -design1 = get_df(design, "design invoice type", ["archetype (complex)"], "Design Archetype Complex") -if not design1.empty : - filtered_dfs.append(design1) + # Design archetype complex + design = get_df(df, "design invoicing status", ["to invoice"]) + design1 = get_df(design, "design invoice type", ["archetype (complex)"], "Design Archetype Complex") + if not design1.empty : + filtered_dfs.append(design1) -# Design archetype simple -design1 = get_df(design, "design invoice type", ["archetype (simple)"], "Design Archetype Simple") -if not design1.empty: - filtered_dfs.append(design1) + # Design archetype simple + design1 = get_df(design, "design invoice type", ["archetype (simple)"], "Design Archetype Simple") + if not design1.empty: + filtered_dfs.append(design1) -# Design repetitive simple -design1 = get_df(design, "design invoice type", ["archetype (simple)"], "Design Archetype repetitive") -if not design1.empty: - filtered_dfs.append(design1) + # Design repetitive simple + design1 = get_df(design, "design invoice type", ["archetype (simple)"], "Design Archetype repetitive") + if not design1.empty: + filtered_dfs.append(design1) -# Design repetitive complex -design1 = get_df(design, "design invoice type", ["archetype (complex)"], "Design Archetype complex") -if not design1.empty: - filtered_dfs.append(design1) + # Design repetitive complex + design1 = get_df(design, "design invoice type", ["archetype (complex)"], "Design Archetype complex") + if not design1.empty: + filtered_dfs.append(design1) -# Design Revision -revision_letter = ['a', 'b', 'c', 'd'] -for letter in revision_letter: - design = get_df(df, "design revision invoice", [f"rev. {letter} to invoice"], "Design Revision") - if not design.empty: - filtered_dfs.append(design) + # Design Revision + revision_letter = ['a', 'b', 'c', 'd'] + for letter in revision_letter: + design = get_df(df, "design revision invoice", [f"rev. {letter} to invoice"], "Design Revision") + if not design.empty: + filtered_dfs.append(design) -# Lodgement Phase 1 -lodg1 = get_df(df, "lodgement phase 1 invoicing status", ["to invoice"], "Lodgement Phase 1") -if not lodg1.empty: - filtered_dfs(lodg1) + # Lodgement Phase 1 + lodg1 = get_df(df, "lodgement phase 1 invoicing status", ["to invoice"], "Lodgement Phase 1") + if not lodg1.empty: + filtered_dfs(lodg1) -# Full Lodgement Phase -lodg2 = get_df(df, "full lodgement invoicing status", ["to invoice"], "Full lodgement phase 2") -if not lodg2.empty: - filtered_dfs.append(lodg2) + # Full Lodgement Phase + lodg2 = get_df(df, "full lodgement invoicing status", ["to invoice"], "Full lodgement phase 2") + if not lodg2.empty: + filtered_dfs.append(lodg2) -# POST EPC -post_epc = get_df(df, "post epc & eval. invoicing status", ["epc to invoice"], "POST EPC") -if not post_epc.empty: - filtered_dfs.append(post_epc) + # POST EPC + post_epc = get_df(df, "post epc & eval. invoicing status", ["epc to invoice"], "POST EPC") + if not post_epc.empty: + filtered_dfs.append(post_epc) -# POST EPR -post_epr = get_df(df, "post epc & eval. invoicing status", ["epr to invoice"], "POST EPR") -if not post_epr.empty: - filtered_dfs.append(post_epr) + # POST EPR + post_epr = get_df(df, "post epc & eval. invoicing status", ["epr to invoice"], "POST EPR") + if not post_epr.empty: + filtered_dfs.append(post_epr) -# post att -post_att = get_df(df, "post att invoicing status", ["to invoice"], "POST ATT") -if not post_att.empty: - filtered_dfs.append(post_epc) + # post att + post_att = get_df(df, "post att invoicing status", ["to invoice"], "POST ATT") + if not post_att.empty: + filtered_dfs.append(post_epc) -# Retrofit Evaluation -rc = get_df(df, "rc stage 2 invoicing status", ["to invoice"], "retrofit evaluation") -if not rc.empty: - filtered_dfs.append(rc) + # Retrofit Evaluation + rc = get_df(df, "rc stage 2 invoicing status", ["to invoice"], "retrofit evaluation") + if not rc.empty: + filtered_dfs.append(rc) -# RA NO Show -ra_ns = get_df(df,"ra no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "RA NO SHOW") -if not ra_ns.empty: - filtered_dfs.append(ra_ns) + # RA NO Show + ra_ns = get_df(df,"ra no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "RA NO SHOW") + if not ra_ns.empty: + filtered_dfs.append(ra_ns) -# ATT NO Show -att_ns = get_df(df, "pre att no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "ATT NO SHOW") -if not att_ns.empty: - filtered_dfs.append(att_ns) + # ATT NO Show + att_ns = get_df(df, "pre att no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "ATT NO SHOW") + if not att_ns.empty: + filtered_dfs.append(att_ns) -# Post visit no show -epc_ns = get_df(df, "post works no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "post EPC NO SHOW") -if not epc_ns.empty: - filtered_dfs.append(epc_ns) + # Post visit no show + epc_ns = get_df(df, "post works no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "post EPC NO SHOW") + if not epc_ns.empty: + filtered_dfs.append(epc_ns) -final_df = pd.concat(filtered_dfs).reset_index(drop=True) + if len(filtered_dfs) != 0: + final_df = pd.concat(filtered_dfs).reset_index(drop=True) -final_df["job_type"] = final_df["job_type"].str.lower() -rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + final_df["job_type"] = final_df["job_type"].str.lower() + rate_card_df = rate_cards[board] + rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() -# Now perform the merge -combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") -import datetime -timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + # Now perform the merge + combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") + import datetime + timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + + attribute = ['address', 'client', 'job_type', 'rate'] + combined_with_rates[attribute].to_excel(f'Watford Warm Homes {timestamp}.xlsx', index=False) + + # Upload to sharepoint + attribute = ['address', 'client', 'job_type', 'rate'] + master_folder_name = BoardID[f"_{board}"].value + file_name = f"{master_folder_name}_{timestamp}.xlsx" + combined_with_rates[attribute].to_excel(file_name, index=False) + + file_path = os.path.abspath(file_name) + + upload_to_month_end_folder(file_name, file_path, master_folder_name) + + invoice_name = "rate_card.xlsx" + file_path = os.path.abspath(invoice_name) + rate_card_df.to_excel(invoice_name, index=False) + upload_to_month_end_folder(invoice_name, file_path, master_folder_name) -attribute = ['address', 'client', 'job_type', 'rate'] -combined_with_rates[attribute].to_excel(f'Watford Warm Homes {timestamp}.xlsx', index=False)