diff --git a/.github/workflows/months_end.yml b/.github/workflows/months_end.yml index f00009f..a575cd3 100644 --- a/.github/workflows/months_end.yml +++ b/.github/workflows/months_end.yml @@ -32,17 +32,17 @@ jobs: poetry run python etl/month_end_automation_wave_2_layout.py poetry run python etl/month_end_automation_wave_2_no_3.py poetry run python etl/month_end_automation_wave_2_no_4.py - poetry run python etl/month_end_automation_wave_2_no_5.py + # poetry run python etl/month_end_automation_wave_2_no_5.py poetry run python etl/month_end_automation_wave_2_no_6.py poetry run python etl/month_end_automation_wave_2_no_7.py poetry run python etl/month_end_automation_wave_2_no_8.py poetry run python etl/month_end_automation_wave_2_no_9.py - poetry run python etl/month_end_automation_wave_2_no_10.py + # poetry run python etl/month_end_automation_wave_2_no_10.py poetry run python etl/month_end_automation_wave_2_no_11.py poetry run python etl/month_end_automation_wave_2_no_12.py - poetry run python etl/month_end_automation_wave_2_no_13.py + # poetry run python etl/month_end_automation_wave_2_no_13.py poetry run python etl/month_end_automation_wave_2_no_14.py poetry run python etl/month_end_automation_wave_2_no_15.py - poetry run python etl/month_end_automation_wave_2_no_16.py + # poetry run python etl/month_end_automation_wave_2_no_16.py poetry run python etl/month_end_automation_wave_accent_housing.py poetry run python etl/month_end_automation_wave_3_layout.py diff --git a/etl/MonthEndUploader.py b/etl/MonthEndUploader.py index 0afa353..c3f4f65 100644 --- a/etl/MonthEndUploader.py +++ b/etl/MonthEndUploader.py @@ -12,10 +12,17 @@ def upload_to_month_end_folder(file_name_on_sp, local_file_path, add_to_path): parent_folder = "General/Junte Kim/month end" today = datetime.today() # Format as "Month YYYY" - formatted_date = today.strftime("%B %Y") + " Attempt 2" - + formatted_date = today.strftime("%B %Y") sharepoint.create_dir(formatted_date, parent_folder) sharepoint_path = parent_folder + "/" + formatted_date + + + # Make day month year folder + formatted_date = today.strftime("%d-%m-%y") + sharepoint.create_dir(formatted_date, sharepoint_path) + sharepoint_path += "/" + formatted_date + + # Make company folder sharepoint.create_dir(add_to_path, sharepoint_path) sharepoint_path += "/" + add_to_path diff --git a/etl/month_end_automation_wave_2_no_11.py b/etl/month_end_automation_wave_2_no_11.py index 874eaa3..db24dbe 100644 --- a/etl/month_end_automation_wave_2_no_11.py +++ b/etl/month_end_automation_wave_2_no_11.py @@ -153,7 +153,7 @@ filtered_dfs.append(post_epc) # POST EPR post_epr = df[ - df["post-epc status"].str.lower().isin(["post epr completed"]) + df["post-epc status"].str.lower().isin(["post epr completed", "Post epr desktop based completed"]) ].copy() post_epr["job_type"] = "POST epr" filtered_dfs.append(post_epr) diff --git a/etl/month_end_automation_wave_2_no_13.py b/etl/month_end_automation_wave_2_no_13.py index 7637222..958c126 100644 --- a/etl/month_end_automation_wave_2_no_13.py +++ b/etl/month_end_automation_wave_2_no_13.py @@ -200,7 +200,7 @@ timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') # Upload to sharepoint attribute = ['address', 'client', 'job_type', 'rate'] -master_folder_name = "Stonewater SHDF 2.0 - Operations" +master_folder_name = "Stonewater SHDF 3.0 - Operations" file_name = f"{master_folder_name}_{timestamp}.xlsx" combined_with_rates[attribute].to_excel(file_name, index=False) diff --git a/etl/month_end_automation_wave_2_no_16.py b/etl/month_end_automation_wave_2_no_16.py index 664ebff..5505f11 100644 --- a/etl/month_end_automation_wave_2_no_16.py +++ b/etl/month_end_automation_wave_2_no_16.py @@ -212,6 +212,10 @@ master_folder_name = "NCHA SHDF Wave 3 On Hold" file_name = f"{master_folder_name}_{timestamp}.xlsx" combined_with_rates[attribute].to_excel(file_name, index=False) + +print("hello world") +file_name = "WCHG SHDF 2.1 Mansard_2025-08-29_10-46.xlsx" +master_folder_name="junte was here" file_path = os.path.abspath(file_name) upload_to_month_end_folder(file_name, file_path, master_folder_name) diff --git a/etl/month_end_automation_wave_2_no_8.py b/etl/month_end_automation_wave_2_no_8.py index d465dd6..7f2ab25 100644 --- a/etl/month_end_automation_wave_2_no_8.py +++ b/etl/month_end_automation_wave_2_no_8.py @@ -96,7 +96,7 @@ def get_df(df, column_name, success_critera, job_name=None): # RA -ra = get_df(df, "ra", ["completed rdsap 9.9", "completed rdsap 10"], "RA") +ra = get_df(df, "ra", ["completed rdsap 9.9", "complete rdsap 10"], "RA") filtered_dfs.append(ra) diff --git a/etl/month_end_automation_wave_3_layout.py b/etl/month_end_automation_wave_3_layout.py index 32125a4..dd3765b 100644 --- a/etl/month_end_automation_wave_3_layout.py +++ b/etl/month_end_automation_wave_3_layout.py @@ -24,11 +24,11 @@ class BoardID(Enum): board_ids = [ - "9349630181", # WCHG Walkups-Operations x - "8830772914", # "L&Q London" x - "9601691730", # Cardo Wales & West - Wave 3 x - "9660895490", # Northumberland County SHDF Wave 3 x - "9641491000", # Watford Warm Homes x + "9349630181", # WCHG Walkups-Operations + "8830772914", # "L&Q London" + "9601691730", # Cardo Wales & West - Wave 3 + "9660895490", # Northumberland County SHDF Wave 3 + "9641491000", # Watford Warm Homes "9671463094", # Seddon ] @@ -244,11 +244,6 @@ for board, all_records in board_to_record.items(): if not v3.empty: filtered_dfs.append(v3) - # Coordination stage 2 Please complete - cors2 = get_df(df, "rc stage 2", ["to invoice"], "Coordination Stage 2") - if not cors2.empty: - filtered_dfs.append(cors2) - # Design archetype complex design = get_df(df, "design invoicing status", ["to invoice"]) design1 = get_df(design, "design invoice type", ["archetype (complex)"], "Design Archetype Complex") @@ -287,24 +282,25 @@ for board, all_records in board_to_record.items(): if not lodg2.empty: filtered_dfs.append(lodg2) - # POST EPC + # POST EPC + Retrofit Evaluation post_epc = get_df(df, "post epc & eval. invoicing status", ["epc to invoice"], "POST EPC") if not post_epc.empty: filtered_dfs.append(post_epc) - # POST EPR post_epr = get_df(df, "post epc & eval. invoicing status", ["epr to invoice"], "POST EPR") if not post_epr.empty: filtered_dfs.append(post_epr) + # Retrofit Evaluation might be need for just EPR + # post att post_att = get_df(df, "post att invoicing status", ["to invoice"], "POST ATT") if not post_att.empty: filtered_dfs.append(post_epc) - # Retrofit Evaluation - rc = get_df(df, "rc stage 2 invoicing status", ["to invoice"], "retrofit evaluation") + # Coordination Stage 2 + rc = get_df(df, "rc stage 2 invoicing status", ["to invoice"], "Coordination Stage 2") if not rc.empty: filtered_dfs.append(rc) diff --git a/etl/osmosis_data/asset_list.xlsx b/etl/osmosis_data/asset_list.xlsx index 822b436..d26670c 100644 Binary files a/etl/osmosis_data/asset_list.xlsx and b/etl/osmosis_data/asset_list.xlsx differ diff --git a/etl/osmosis_monday_to_sharepoint_automation.py b/etl/osmosis_monday_to_sharepoint_automation.py index 40118f4..18e544f 100644 --- a/etl/osmosis_monday_to_sharepoint_automation.py +++ b/etl/osmosis_monday_to_sharepoint_automation.py @@ -14,7 +14,7 @@ import time osmosis = SharePointScraper(SharePointInstaller.OSMOSIS_WAVE_3) -parent_folder = "/Osmosis-ACD Projects/Cardo/Cardo (Wales & West)/2506 Cardo Property Folders" +parent_folder = "/Osmosis-ACD Projects/NCHA WHSHF Wave 3/Property Folders" asset_list = pd.read_excel("osmosis_data/asset_list.xlsx", sheet_name="Sheet 1") @@ -66,23 +66,23 @@ for index, address in tqdm(asset_list.iterrows()): new_asset_list.append(asset_data) -# Run this is you just want to get url -def just_url(asset_list): - new_asset_list = [] - for index, address in tqdm(asset_list.iterrows()): - folder_name = address['Name'] + " " + address['Postcode'] - webUrl = osmosis.create_dir(folder_name, parent_folder) - asset_data = { - "Name": address['Name'], - "Postcode": address['Postcode'], - "Sharepoint": webUrl, - } - print(asset_data) +# # Run this is you just want to get url +# def just_url(asset_list): +# new_asset_list = [] +# for index, address in tqdm(asset_list.iterrows()): +# folder_name = address['Name'] + " " + address['Postcode'] +# webUrl = osmosis.create_dir(folder_name, parent_folder) +# asset_data = { +# "Name": address['Name'], +# "Postcode": address['Postcode'], +# "Sharepoint": webUrl, +# } +# print(asset_data) - new_asset_list.append(asset_data) - return new_asset_list +# new_asset_list.append(asset_data) +# return new_asset_list -new_asset_list = just_url(asset_list=asset_list) +# new_asset_list = just_url(asset_list=asset_list) df = pd.DataFrame(new_asset_list) df.to_csv("output.csv", index=False)