junte/Model
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/Model.git synced 2026-06-30 13:10:47 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fixing cleaning for stonewater

Browse source
This commit is contained in:
Khalim Conn-Kowlessar 2025-02-03 13:04:10 +00:00
parent 01a5077c17
commit 04eba60961
1 changed files with 35 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

58
etl/customers/stonewater/Wave 3 Preparation.py
View file

@ -3194,7 +3194,32 @@ def revised_model():
"12005", "12005",
wates_coordination["Asset ID"] wates_coordination["Asset ID"]
) )
# We fill the missing ids
missing_lookup = {
"4 Sydnall Fields": 31231,
"12 Sydnall Fields": 31239,
"12 Athena Gardens": 28061,
"49 Banner Lane": 41189,
"4 Jonathan Road": 41232,
"8 Jonathan Road": 41236,
"1 Jonathan Road": 41229,
"96 Taunton Way": 31417,
"94 Taunton Way": 31418,
"1 Lady Lane": 29430,
"10 Jonathan Road": 41283,
"21 Jonathan Road": 41246,
"12 Ashcroft Close": 26399
}
for name, asset_id in missing_lookup.items():
wates_coordination["Asset ID_x"] = np.where(
wates_coordination["Name"] == name,
asset_id,
wates_coordination["Asset ID_x"]
)
wates_coordination = wates_coordination[~pd.isnull(wates_coordination["Asset ID"])]
wates_coordination["folder_path"] = wates_coordination["Sharepoint Folder"].apply( wates_coordination["folder_path"] = wates_coordination["Sharepoint Folder"].apply(
lambda x: extract_sharepoint_url(x) lambda x: extract_sharepoint_url(x)
) )
@ -3412,6 +3437,7 @@ def revised_model():
# We get a match for all records # We get a match for all records
assert ccs_matching_lookup.shape[0] == ccs_coordination.shape[0] assert ccs_matching_lookup.shape[0] == ccs_coordination.shape[0]
assert not pd.isnull(ccs_matching_lookup["Asset ID.1"]).sum() assert not pd.isnull(ccs_matching_lookup["Asset ID.1"]).sum()
assert not ccs_matching_lookup["Asset ID.1"].duplicated().sum()
# We do the same for Wates # We do the same for Wates
wates_coordination = wates_coordination.rename( wates_coordination = wates_coordination.rename(
@ -3447,6 +3473,8 @@ def revised_model():
'3 Chanders Rd', '3 Chanders Rd',
'1 Orchard Close', '1 Orchard Close',
] ]
wates_coordination = wates_coordination[~wates_coordination["Name"].isin(wates_to_skip)]
for _, home in tqdm(wates_coordination.iterrows(), total=len(wates_coordination)): for _, home in tqdm(wates_coordination.iterrows(), total=len(wates_coordination)):
# Search the folder # Search the folder
@ -3547,34 +3575,18 @@ def revised_model():
raise Exception("No match") raise Exception("No match")
wates_matching_lookup = pd.DataFrame(wates_matching_lookup) wates_matching_lookup = pd.DataFrame(wates_matching_lookup)
# We get a match for all records
assert wates_matching_lookup.shape[0] == wates_coordination.shape[0]
assert not pd.isnull(wates_matching_lookup["Asset ID"]).sum()
assert not wates_matching_lookup["Asset ID"].duplicated().sum()
# Merge lookup tables onto the coordination sheets # Merge lookup tables onto the coordination sheets
wates_coordination = wates_coordination.merge( wates_coordination = wates_coordination.merge(
wates_matching_lookup, how="left", on="Name" wates_matching_lookup, how="left", on="Name"
) )
missed_asset_id = wates_coordination[pd.isnull(wates_coordination["Asset ID_x"])] missed_asset_id = wates_coordination[pd.isnull(wates_coordination["Asset ID_x"])]
if not missed_asset_id.empty: if not missed_asset_id.empty:
# We fill the missing ids raise Exception("Missing Asset ID")
missing_lookup = {
"4 Sydnall Fields": 31231,
"12 Sydnall Fields": 31239,
"12 Athena Gardens": 28061,
"49 Banner Lane": 41189,
"4 Jonathan Road": 41232,
"8 Jonathan Road": 41236,
"1 Jonathan Road": 41229,
"96 Taunton Way": 31417,
"94 Taunton Way": 31418,
"1 Lady Lane": 29430,
"10 Jonathan Road": 41283,
"21 Jonathan Road": 41246,
"12 Ashcroft Close": 26399
}
for name, asset_id in missing_lookup.items():
wates_coordination["Asset ID_x"] = np.where(
wates_coordination["Name"] == name,
asset_id,
wates_coordination["Asset ID_x"]
)
ccs_coordination = ccs_coordination.merge( ccs_coordination = ccs_coordination.merge(
ccs_matching_lookup, how="left", on="Name" ccs_matching_lookup, how="left", on="Name"