junte/survey-extraction
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/survey-extraction.git synced 2026-06-30 13:10:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

changes

Browse source
This commit is contained in:
Jun-te Kim 2025-09-05 09:40:29 +00:00
parent 27a6227311
commit ae116c43ad
9 changed files with 46 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
.github/workflows/months_end.yml vendored
View file

@ -32,17 +32,17 @@ jobs:
poetry run python etl/month_end_automation_wave_2_layout.py
poetry run python etl/month_end_automation_wave_2_no_3.py
poetry run python etl/month_end_automation_wave_2_no_4.py
poetry run python etl/month_end_automation_wave_2_no_5.py
# poetry run python etl/month_end_automation_wave_2_no_5.py
poetry run python etl/month_end_automation_wave_2_no_6.py
poetry run python etl/month_end_automation_wave_2_no_7.py
poetry run python etl/month_end_automation_wave_2_no_8.py
poetry run python etl/month_end_automation_wave_2_no_9.py
poetry run python etl/month_end_automation_wave_2_no_10.py
# poetry run python etl/month_end_automation_wave_2_no_10.py
poetry run python etl/month_end_automation_wave_2_no_11.py
poetry run python etl/month_end_automation_wave_2_no_12.py
poetry run python etl/month_end_automation_wave_2_no_13.py
# poetry run python etl/month_end_automation_wave_2_no_13.py
poetry run python etl/month_end_automation_wave_2_no_14.py
poetry run python etl/month_end_automation_wave_2_no_15.py
poetry run python etl/month_end_automation_wave_2_no_16.py
# poetry run python etl/month_end_automation_wave_2_no_16.py
poetry run python etl/month_end_automation_wave_accent_housing.py
poetry run python etl/month_end_automation_wave_3_layout.py

11
etl/MonthEndUploader.py
View file

@ -12,10 +12,17 @@ def upload_to_month_end_folder(file_name_on_sp, local_file_path, add_to_path):
parent_folder = "General/Junte Kim/month end"
today = datetime.today()
# Format as "Month YYYY"
formatted_date = today.strftime("%B %Y") + " Attempt 2"
formatted_date = today.strftime("%B %Y")
sharepoint.create_dir(formatted_date, parent_folder)
sharepoint_path = parent_folder + "/" + formatted_date
# Make day month year folder
formatted_date = today.strftime("%d-%m-%y")
sharepoint.create_dir(formatted_date, sharepoint_path)
sharepoint_path += "/" + formatted_date
# Make company folder
sharepoint.create_dir(add_to_path, sharepoint_path)
sharepoint_path += "/" + add_to_path

2
etl/month_end_automation_wave_2_no_11.py
View file

@ -153,7 +153,7 @@ filtered_dfs.append(post_epc)
# POST EPR
post_epr = df[
df["post-epc status"].str.lower().isin(["post epr completed"])
df["post-epc status"].str.lower().isin(["post epr completed", "Post epr desktop based completed"])
].copy()
post_epr["job_type"] = "POST epr"
filtered_dfs.append(post_epr)

2
etl/month_end_automation_wave_2_no_13.py
View file

@ -200,7 +200,7 @@ timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')
# Upload to sharepoint
attribute = ['address', 'client', 'job_type', 'rate']
master_folder_name = "Stonewater SHDF 2.0 - Operations"
master_folder_name = "Stonewater SHDF 3.0 - Operations"
file_name = f"{master_folder_name}_{timestamp}.xlsx"
combined_with_rates[attribute].to_excel(file_name, index=False)

4
etl/month_end_automation_wave_2_no_16.py
View file

@ -212,6 +212,10 @@ master_folder_name = "NCHA SHDF Wave 3 On Hold"
file_name = f"{master_folder_name}_{timestamp}.xlsx"
combined_with_rates[attribute].to_excel(file_name, index=False)
print("hello world")
file_name = "WCHG SHDF 2.1 Mansard_2025-08-29_10-46.xlsx"
master_folder_name="junte was here"
file_path = os.path.abspath(file_name)
upload_to_month_end_folder(file_name, file_path, master_folder_name)

2
etl/month_end_automation_wave_2_no_8.py
View file

@ -96,7 +96,7 @@ def get_df(df, column_name, success_critera, job_name=None):
# RA
ra = get_df(df, "ra", ["completed rdsap 9.9", "completed rdsap 10"], "RA")
ra = get_df(df, "ra", ["completed rdsap 9.9", "complete rdsap 10"], "RA")
filtered_dfs.append(ra)

24
etl/month_end_automation_wave_3_layout.py
View file

@ -24,11 +24,11 @@ class BoardID(Enum):
board_ids = [
"9349630181", # WCHG Walkups-Operations x
"8830772914", # "L&Q London" x
"9601691730", # Cardo Wales & West - Wave 3 x
"9660895490", # Northumberland County SHDF Wave 3 x
"9641491000", # Watford Warm Homes x
"9349630181", # WCHG Walkups-Operations
"8830772914", # "L&Q London"
"9601691730", # Cardo Wales & West - Wave 3
"9660895490", # Northumberland County SHDF Wave 3
"9641491000", # Watford Warm Homes
"9671463094", # Seddon
]
@ -244,11 +244,6 @@ for board, all_records in board_to_record.items():
if not v3.empty:
filtered_dfs.append(v3)
# Coordination stage 2 Please complete
cors2 = get_df(df, "rc stage 2", ["to invoice"], "Coordination Stage 2")
if not cors2.empty:
filtered_dfs.append(cors2)
# Design archetype complex
design = get_df(df, "design invoicing status", ["to invoice"])
design1 = get_df(design, "design invoice type", ["archetype (complex)"], "Design Archetype Complex")
@ -287,24 +282,25 @@ for board, all_records in board_to_record.items():
if not lodg2.empty:
filtered_dfs.append(lodg2)
# POST EPC
# POST EPC + Retrofit Evaluation
post_epc = get_df(df, "post epc & eval. invoicing status", ["epc to invoice"], "POST EPC")
if not post_epc.empty:
filtered_dfs.append(post_epc)
# POST EPR
post_epr = get_df(df, "post epc & eval. invoicing status", ["epr to invoice"], "POST EPR")
if not post_epr.empty:
filtered_dfs.append(post_epr)
# Retrofit Evaluation might be need for just EPR
# post att
post_att = get_df(df, "post att invoicing status", ["to invoice"], "POST ATT")
if not post_att.empty:
filtered_dfs.append(post_epc)
# Retrofit Evaluation
rc = get_df(df, "rc stage 2 invoicing status", ["to invoice"], "retrofit evaluation")
# Coordination Stage 2
rc = get_df(df, "rc stage 2 invoicing status", ["to invoice"], "Coordination Stage 2")
if not rc.empty:
filtered_dfs.append(rc)

BIN
etl/osmosis_data/asset_list.xlsx
View file

Binary file not shown.

32
etl/osmosis_monday_to_sharepoint_automation.py
View file

@ -14,7 +14,7 @@ import time
osmosis = SharePointScraper(SharePointInstaller.OSMOSIS_WAVE_3)
parent_folder = "/Osmosis-ACD Projects/Cardo/Cardo (Wales & West)/2506 Cardo Property Folders"
parent_folder = "/Osmosis-ACD Projects/NCHA WHSHF Wave 3/Property Folders"
asset_list = pd.read_excel("osmosis_data/asset_list.xlsx", sheet_name="Sheet 1")
@ -66,23 +66,23 @@ for index, address in tqdm(asset_list.iterrows()):
new_asset_list.append(asset_data)
# Run this is you just want to get url
def just_url(asset_list):
new_asset_list = []
for index, address in tqdm(asset_list.iterrows()):
folder_name = address['Name'] + " " + address['Postcode']
webUrl = osmosis.create_dir(folder_name, parent_folder)
asset_data = {
"Name": address['Name'],
"Postcode": address['Postcode'],
"Sharepoint": webUrl,
}
print(asset_data)
# # Run this is you just want to get url
# def just_url(asset_list):
# new_asset_list = []
# for index, address in tqdm(asset_list.iterrows()):
# folder_name = address['Name'] + " " + address['Postcode']
# webUrl = osmosis.create_dir(folder_name, parent_folder)
# asset_data = {
# "Name": address['Name'],
# "Postcode": address['Postcode'],
# "Sharepoint": webUrl,
# }
# print(asset_data)
new_asset_list.append(asset_data)
return new_asset_list
# new_asset_list.append(asset_data)
# return new_asset_list
new_asset_list = just_url(asset_list=asset_list)
# new_asset_list = just_url(asset_list=asset_list)
df = pd.DataFrame(new_asset_list)
df.to_csv("output.csv", index=False)