junte/Model
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/Model.git synced 2026-07-12 13:29:04 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

created a small dataset of uprn and directory name to keep track of all equal rows

Browse source
This commit is contained in:
Khalim Conn-Kowlessar 2023-10-18 10:14:52 +11:00
parent d5b2ff9c36
commit f0db6b69df
1 changed files with 5 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
etl/epc/property_change_app.py
View file

@ -411,8 +411,8 @@ def app():
dataset = [] dataset = []
cleaning_dataset = [] cleaning_dataset = []
# Keep track of the number of all equals # Keep track of the all equals
all_equal_count = 0 all_equal_rows = []
for directory in tqdm(directories): for directory in tqdm(directories):
@ -521,8 +521,8 @@ def app():
) )
if all_equal: if all_equal:
# Keep track of this for the moment # Keep track of this for the moment so we can analyse
all_equal_count += 1 all_equal_rows.append({"uprn": uprn, "directory_name": directory.name})
continue continue
features = pd.concat([starting_record, ending_record]) features = pd.concat([starting_record, ending_record])
@ -622,7 +622,7 @@ def app():
save_dataframe_to_s3_parquet( save_dataframe_to_s3_parquet(
df=output, df=output,
bucket_name="retrofit-data-dev", bucket_name="retrofit-data-dev",
file_key="sap_change_model/dataset_test.parquet", file_key="sap_change_model/dataset.parquet",
) )