diff --git a/modules/ml-pipeline/src/pipeline/configs/feature_processor_logic.py b/modules/ml-pipeline/src/pipeline/configs/feature_processor_logic.py index 7b131dc..78c29a9 100644 --- a/modules/ml-pipeline/src/pipeline/configs/feature_processor_logic.py +++ b/modules/ml-pipeline/src/pipeline/configs/feature_processor_logic.py @@ -18,11 +18,42 @@ def remove_starting_columns(df): return df +def keep_negative_heat_change(df): + df = df[df["HEAT_DEMAND_CHANGE"] < 0] + return df + + def keep_negative_carbon_change(df): df = df[df["CARBON_CHANGE"] < 0] return df +# TODO: Move to ETL pipeline +def remove_unreasonable_habitable_rooms(df): + """ + Assumption is that proportion of floor area to habitable rooms should be at least 6.5m2 + """ + minimum_room_size_index = ( + df["TOTAL_FLOOR_AREA_ENDING"] / df["NUMBER_HABITABLE_ROOMS"] >= 6.5 + ) + df = df[minimum_room_size_index] + return df + + +def remove_top_1_percent_heat_demand(df): + # threshold_value = df.describe(percentiles=[0.99])['HEAT_DEMAND_STARTING']['99%'] + threshold_value = 860 + df = df[df["HEAT_DEMAND_STARTING"] < threshold_value] + return df + + +def remove_top_1_percent_carbon(df): + # threshold_value = df.describe(percentiles=[0.99])['CARBON_STARTING']['99%'] + threshold_value = 18 + df = df[df["CARBON_STARTING"] < threshold_value] + return df + + # def keep_ending_columns(df): # ending_column_index = [ col_name.endswith("_ENDING") for col_name in list(df.columns)] # keep_columns = df.columns[ending_column_index].to_list() @@ -32,7 +63,11 @@ def keep_negative_carbon_change(df): # return df business_logic = { - "keep_negative_carbon_change": keep_negative_carbon_change + "remove_unreasonable_habitable_rooms": remove_unreasonable_habitable_rooms, + "keep_negative_heat_change": keep_negative_heat_change, + "keep_negative_carbon_change": keep_negative_carbon_change, + "remove_top_1_percent_heat_demand": remove_top_1_percent_heat_demand, + "remove_top_1_percent_carbon": remove_top_1_percent_carbon, # "remove_starting_columns": remove_starting_columns # "keep_ENDING_COLUMNS": keep_ending_columns } diff --git a/modules/ml-pipeline/src/pipeline/dvc.lock b/modules/ml-pipeline/src/pipeline/dvc.lock index 627af99..e65dfe8 100644 --- a/modules/ml-pipeline/src/pipeline/dvc.lock +++ b/modules/ml-pipeline/src/pipeline/dvc.lock @@ -29,8 +29,8 @@ stages: outs: - path: data/prepared_data/ hash: md5 - md5: 73c1f7be21be8358a73c4ab5f9ec8e39.dir - size: 32943109 + md5: ca205aaf77cb9a9414a0c6a1affd8d82.dir + size: 30597800 nfiles: 2 build_model: cmd: python 2_build_model.py @@ -41,8 +41,8 @@ stages: size: 4149 - path: data/prepared_data hash: md5 - md5: 73c1f7be21be8358a73c4ab5f9ec8e39.dir - size: 32943109 + md5: ca205aaf77cb9a9414a0c6a1affd8d82.dir + size: 30597800 nfiles: 2 params: configs/build_model.yaml: @@ -68,12 +68,12 @@ stages: outs: - path: data/model/ hash: md5 - md5: dee1a60e6a9f4695272da8127196f714.dir - size: 326732699 + md5: f3be67a0a80e525d30665f2ffc367d9b.dir + size: 312133166 nfiles: 24 - path: metrics/fit_metrics.json hash: md5 - md5: 1fefa99c7bc50d09c31bf175d5b9ee9c + md5: 36912d423f975802ca3661992103e614 size: 226 generate_predictions: cmd: python 3_generate_predictions.py @@ -84,13 +84,13 @@ stages: size: 2464 - path: data/model hash: md5 - md5: dee1a60e6a9f4695272da8127196f714.dir - size: 326732699 + md5: f3be67a0a80e525d30665f2ffc367d9b.dir + size: 312133166 nfiles: 24 - path: data/prepared_data hash: md5 - md5: 73c1f7be21be8358a73c4ab5f9ec8e39.dir - size: 32943109 + md5: ca205aaf77cb9a9414a0c6a1affd8d82.dir + size: 30597800 nfiles: 2 params: configs/settings.yaml: @@ -102,8 +102,8 @@ stages: outs: - path: data/predictions/ hash: md5 - md5: d2da3b713811952b66e2c5f8c95f5407.dir - size: 410646 + md5: 2ae9ab85ca2551d6b0833337cacbcc3e.dir + size: 389118 nfiles: 1 generate_metrics: cmd: python 4_generate_metrics.py @@ -114,13 +114,13 @@ stages: size: 3485 - path: data/predictions hash: md5 - md5: d2da3b713811952b66e2c5f8c95f5407.dir - size: 410646 + md5: 2ae9ab85ca2551d6b0833337cacbcc3e.dir + size: 389118 nfiles: 1 - path: data/prepared_data hash: md5 - md5: 73c1f7be21be8358a73c4ab5f9ec8e39.dir - size: 32943109 + md5: ca205aaf77cb9a9414a0c6a1affd8d82.dir + size: 30597800 nfiles: 2 params: configs/settings.yaml: @@ -130,7 +130,7 @@ stages: outs: - path: metrics/metrics.json hash: md5 - md5: 4ed2edc06b4dad3c094a2d1be374a5de + md5: 6447c7b2b92a4057aecd3d227de1aadf size: 224 startup_cleanup: cmd: python 0_startup_cleanup.py