From 0e7f56e35632401e7e54a96034f604160a3cbc67 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 18 Oct 2023 10:28:18 +1100 Subject: [PATCH] Added all_equal_rows storage --- etl/epc/property_change_app.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/etl/epc/property_change_app.py b/etl/epc/property_change_app.py index a6034e3d..435b668d 100644 --- a/etl/epc/property_change_app.py +++ b/etl/epc/property_change_app.py @@ -597,7 +597,7 @@ def app(): cleaning_averages["LOCAL_AUTHORITY"] = df["LOCAL_AUTHORITY"].values[0] cleaning_dataset.append(cleaning_averages) - print("Final all equal count: %s" % str(all_equal_count)) + print("Final all equal count: %s" % str(len(all_equal_rows))) # Store cleaning dataset in s3 as a parquet file cleaning_dataset = pd.concat(cleaning_dataset) @@ -625,6 +625,14 @@ def app(): file_key="sap_change_model/dataset.parquet", ) + # Store all_equal_rows + all_equal_rows = pd.DataFrame(all_equal_rows) + save_dataframe_to_s3_parquet( + df=all_equal_rows, + bucket_name="retrofit-data-dev", + file_key="sap_change_model/all_equal_rows.parquet", + ) + if __name__ == "__main__": app()