mirror of
https://github.com/Hestia-Homes/survey-extraction.git
synced 2026-06-30 13:10:56 +00:00
save
This commit is contained in:
Jun-te Kim
parent
827be0c161
commit
5743495261
1 changed files with 5 additions and 287 deletions
|
|
@ -10,11 +10,11 @@ import json
|
|||
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
|
||||
monday = MondayClient(monday_key)
|
||||
board_ids = [
|
||||
"9349630181", # WCHG Walkups-Operations
|
||||
# "9349630181", # WCHG Walkups-Operations
|
||||
# "8830772914", # "L&Q London"
|
||||
# "9601691730", # Cardo Wales & West - Wave 3
|
||||
# "9660895490", # Northumberland County SHDF Wave 3
|
||||
# "9641491000", #Watford Warm Homes
|
||||
"9641491000", #Watford Warm Homes
|
||||
]
|
||||
|
||||
empty = "Rate card info missing"
|
||||
|
|
@ -95,12 +95,12 @@ rate_card_data_walk_ups = {
|
|||
]
|
||||
}
|
||||
|
||||
|
||||
rate_card_df = pd.DataFrame(rate_card_data_watford_warm_homes)
|
||||
# rate_card_df = pd.DataFrame(rate_card_data_example)
|
||||
# rate_card_df = pd.DataFrame(rate_card_data_2502_accent_housing)
|
||||
# rate_card_df = pd.DataFrame(rate_card_data_l_and_q_london)
|
||||
# rate_card_df = pd.DataFrame(rate_card_data_northhumberland_country_shdf_wave_3)
|
||||
rate_card_df = pd.DataFrame(rate_card_data_walk_ups)
|
||||
# rate_card_df = pd.DataFrame(rate_card_data_walk_ups)
|
||||
|
||||
|
||||
for board in tqdm(board_ids):
|
||||
|
|
@ -279,286 +279,4 @@ import datetime
|
|||
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')
|
||||
|
||||
attribute = ['address', 'client', 'job_type', 'rate']
|
||||
combined_with_rates[attribute].to_excel(f'WCHG Walk up {timestamp}.xlsx', index=False)
|
||||
# Wave 3's month end automation
|
||||
|
||||
from tqdm import tqdm
|
||||
from monday import MondayClient
|
||||
from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
|
||||
from pprint import pprint
|
||||
import pandas as pd
|
||||
import json
|
||||
|
||||
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
|
||||
monday = MondayClient(monday_key)
|
||||
board_ids = [
|
||||
"9349630181", # WCHG Walkups-Operations
|
||||
# "8830772914", # "L&Q London"
|
||||
# "9601691730", # Cardo Wales & West - Wave 3
|
||||
# "9660895490", # Northumberland County SHDF Wave 3
|
||||
# "9641491000", #Watford Warm Homes
|
||||
]
|
||||
|
||||
empty = "Rate card info missing"
|
||||
rate_card_data_watford_warm_homes = {
|
||||
"job_type": [
|
||||
"RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
|
||||
"Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
|
||||
"Post EPR", "Post EPC", "Post ATT", "retrofit evaluation",
|
||||
"RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling"
|
||||
],
|
||||
"rate": [
|
||||
165, empty, empty, empty, empty,
|
||||
empty, empty, empty, empty, empty,
|
||||
empty, empty, empty, empty, empty,
|
||||
empty, empty, empty, empty, empty,
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
rate_card_data_2502_accent_housing = {
|
||||
"job_type": [
|
||||
"RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
|
||||
"Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
|
||||
"Post EPR", "Post EPC", "Post ATT", "retrofit evaluation",
|
||||
"RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling"
|
||||
],
|
||||
"rate": [
|
||||
empty, empty, empty, empty, empty,
|
||||
empty, empty, empty, empty, empty,
|
||||
empty, empty, empty, empty, empty,
|
||||
empty, empty, empty, 280, 150
|
||||
]
|
||||
}
|
||||
|
||||
rate_card_data_l_and_q_london = {
|
||||
"job_type": [
|
||||
"RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
|
||||
"Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
|
||||
"Post EPR", "Post EPC", "Post ATT", "retrofit evaluation",
|
||||
"RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling"
|
||||
],
|
||||
"rate": [
|
||||
empty, empty, empty, empty, empty,
|
||||
empty, empty, empty, empty, empty,
|
||||
empty, empty, empty, empty, empty,
|
||||
empty, empty, empty, 280, 150
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
rate_card_data_northhumberland_country_shdf_wave_3 = {
|
||||
"job_type": [
|
||||
"RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
|
||||
"Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
|
||||
"Post EPR", "Post EPC", "Post ATT", "retrofit evaluation",
|
||||
"RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling"
|
||||
],
|
||||
"rate": [
|
||||
empty, empty, empty, empty, empty,
|
||||
empty, empty, empty, empty, empty,
|
||||
empty, empty, empty, empty, empty,
|
||||
empty, empty, empty, 280, 130,
|
||||
]
|
||||
}
|
||||
|
||||
rate_card_data_walk_ups = {
|
||||
"job_type": [
|
||||
"RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
|
||||
"Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
|
||||
"Post EPR", "Post EPC", "Post ATT", "retrofit evaluation",
|
||||
"RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling"
|
||||
],
|
||||
"rate": [
|
||||
259, 125, 280, 125, 125,
|
||||
650, 415, 195, 175, 135,
|
||||
120, "60 to check", 85, 125, 60,
|
||||
45, 45, 45, empty, empty,
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
# rate_card_df = pd.DataFrame(rate_card_data_example)
|
||||
# rate_card_df = pd.DataFrame(rate_card_data_2502_accent_housing)
|
||||
# rate_card_df = pd.DataFrame(rate_card_data_l_and_q_london)
|
||||
# rate_card_df = pd.DataFrame(rate_card_data_northhumberland_country_shdf_wave_3)
|
||||
rate_card_df = pd.DataFrame(rate_card_data_walk_ups)
|
||||
|
||||
|
||||
for board in tqdm(board_ids):
|
||||
print(f"working on board {board}")
|
||||
board_data = monday.boards.fetch_boards_by_id(board)
|
||||
columns = board_data["data"]["boards"][0]["columns"]
|
||||
col_id_map = {col["title"].lower(): col["id"] for col in columns}
|
||||
reversed_col_id_map = {v: k for k, v in col_id_map.items()}
|
||||
|
||||
|
||||
items = get_all_items(board, monday)
|
||||
|
||||
all_records = []
|
||||
for row in tqdm(items):
|
||||
data = {}
|
||||
data.update({"address": row['name']})
|
||||
data.update({"client": row['group']['title']})
|
||||
for col in row.get("column_values", []):
|
||||
if col.get("id") in reversed_col_id_map:
|
||||
if col.get("type") == "file":
|
||||
value = col.get("value")
|
||||
no_of_files = 0
|
||||
|
||||
if value:
|
||||
value = json.loads(col["value"])
|
||||
no_of_files = len(value.get('files', []))
|
||||
data.update({reversed_col_id_map[col.get("id")]: no_of_files})
|
||||
else:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: col.get("text")
|
||||
})
|
||||
all_records.append(data)
|
||||
|
||||
# Convert to DataFrame
|
||||
df = pd.DataFrame(all_records)
|
||||
|
||||
filtered_dfs = []
|
||||
|
||||
|
||||
def get_df(df, column_name, success_critera, job_name=None):
|
||||
_ = pd.DataFrame()
|
||||
if column_name in col_id_map:
|
||||
_ = df[
|
||||
df[column_name].str.lower().isin(success_critera)
|
||||
].copy()
|
||||
if job_name:
|
||||
_["job_type"] = job_name
|
||||
|
||||
|
||||
return _
|
||||
|
||||
|
||||
# RA
|
||||
ra = get_df(df, "ra invoicing status", ["to invoice"], "RA")
|
||||
if not ra.empty:
|
||||
filtered_dfs.append(ra)
|
||||
|
||||
|
||||
att = get_df(df, "post att invoicing status", ["to invoice"], "ATT")
|
||||
if not att.empty:
|
||||
filtered_dfs.append(att)
|
||||
|
||||
modeling = get_df(df, "mtp invoicing status", ["modelling to invoice"], "Measure Modelling")
|
||||
if not modeling.empty:
|
||||
filtered_dfs.append(modeling)
|
||||
|
||||
try:
|
||||
# Only needed for one board in wave 3
|
||||
full_cost = get_df(df, "mtp invoicing status", ["(V1) Full cost MTP to invoice (no previous modelling)".lower()], "full cost mtp")
|
||||
if not full_cost.empty:
|
||||
filtered_dfs(full_cost)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
|
||||
v1 = get_df(df, "mtp invoicing status", ["(v1) ioe/mtp to invoice"], "Coordination Stage 1 v1")
|
||||
if not v1.empty:
|
||||
filtered_dfs.append(v1)
|
||||
|
||||
v2 = get_df(df, "mtp invoicing status", ["(v2) ioe/mtp to invoice"], "Coordination Stage 1 v2 remodel")
|
||||
if not v2.empty:
|
||||
filtered_dfs.append(v2)
|
||||
|
||||
v3 = get_df(df, "mtp invoicing status", ["(v3) ioe/mtp to invoice"], "Coordination Stage 1 v3 remodel")
|
||||
if not v3.empty:
|
||||
filtered_dfs.append(v3)
|
||||
|
||||
# Coordination stage 2 Please complete
|
||||
cors2 = get_df(df, "rc stage 2", ["to invoice"], "Coordination Stage 2")
|
||||
if not cors2.empty:
|
||||
filtered_dfs.append(cors2)
|
||||
|
||||
# Design archetype complex
|
||||
design = get_df(df, "design invoicing status", ["to invoice"])
|
||||
design1 = get_df(design, "design invoice type", ["archetype (complex)"], "Design Archetype Complex")
|
||||
if not design1.empty :
|
||||
filtered_dfs.append(design1)
|
||||
|
||||
# Design archetype simple
|
||||
design1 = get_df(design, "design invoice type", ["archetype (simple)"], "Design Archetype Simple")
|
||||
if not design1.empty:
|
||||
filtered_dfs.append(design1)
|
||||
|
||||
# Design repetitive simple
|
||||
design1 = get_df(design, "design invoice type", ["archetype (simple)"], "Design Archetype repetitive")
|
||||
if not design1.empty:
|
||||
filtered_dfs.append(design1)
|
||||
|
||||
# Design repetitive complex
|
||||
design1 = get_df(design, "design invoice type", ["archetype (complex)"], "Design Archetype complex")
|
||||
if not design1.empty:
|
||||
filtered_dfs.append(design1)
|
||||
|
||||
# Design Revision
|
||||
revision_letter = ['a', 'b', 'c', 'd']
|
||||
for letter in revision_letter:
|
||||
design = get_df(df, "design revision invoice", [f"rev. {letter} to invoice"], "Design Revision")
|
||||
if not design.empty:
|
||||
filtered_dfs.append(design)
|
||||
|
||||
# Lodgement Phase 1
|
||||
lodg1 = get_df(df, "lodgement phase 1 invoicing status", ["to invoice"], "Lodgement Phase 1")
|
||||
if not lodg1.empty:
|
||||
filtered_dfs(lodg1)
|
||||
|
||||
# Full Lodgement Phase
|
||||
lodg2 = get_df(df, "full lodgement invoicing status", ["to invoice"], "Full lodgement phase 2")
|
||||
if not lodg2.empty:
|
||||
filtered_dfs.append(lodg2)
|
||||
|
||||
# POST EPC
|
||||
post_epc = get_df(df, "post epc & eval. invoicing status", ["epc to invoice"], "POST EPC")
|
||||
if not post_epc.empty:
|
||||
filtered_dfs.append(post_epc)
|
||||
|
||||
|
||||
# POST EPR
|
||||
post_epr = get_df(df, "post epc & eval. invoicing status", ["epr to invoice"], "POST EPR")
|
||||
if not post_epr.empty:
|
||||
filtered_dfs.append(post_epr)
|
||||
|
||||
# post att
|
||||
post_att = get_df(df, "post att invoicing status", ["to invoice"], "POST ATT")
|
||||
if not post_att.empty:
|
||||
filtered_dfs.append(post_epc)
|
||||
|
||||
# Retrofit Evaluation
|
||||
rc = get_df(df, "rc stage 2 invoicing status", ["to invoice"], "retrofit evaluation")
|
||||
if not rc.empty:
|
||||
filtered_dfs.append(rc)
|
||||
|
||||
# RA NO Show
|
||||
ra_ns = get_df(df,"ra no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "RA NO SHOW")
|
||||
if not ra_ns.empty:
|
||||
filtered_dfs.append(ra_ns)
|
||||
|
||||
|
||||
# ATT NO Show
|
||||
att_ns = get_df(df, "pre att no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "ATT NO SHOW")
|
||||
if not att_ns.empty:
|
||||
filtered_dfs.append(att_ns)
|
||||
|
||||
|
||||
# Post visit no show
|
||||
epc_ns = get_df(df, "post works no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "post EPC NO SHOW")
|
||||
if not epc_ns.empty:
|
||||
filtered_dfs.append(epc_ns)
|
||||
|
||||
final_df = pd.concat(filtered_dfs).reset_index(drop=True)
|
||||
|
||||
final_df["job_type"] = final_df["job_type"].str.lower()
|
||||
rate_card_df["job_type"] = rate_card_df["job_type"].str.lower()
|
||||
|
||||
# Now perform the merge
|
||||
combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left")
|
||||
import datetime
|
||||
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')
|
||||
|
||||
attribute = ['address', 'client', 'job_type', 'rate']
|
||||
combined_with_rates[attribute].to_excel(f'WCHG Walk up {timestamp}.xlsx', index=False)
|
||||
combined_with_rates[attribute].to_excel(f'Watford Warm Homes {timestamp}.xlsx', index=False)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue