diff --git a/.github/workflows/Deploy.yml b/.github/workflows/Deploy.yml index c1b21f6..2432655 100644 --- a/.github/workflows/Deploy.yml +++ b/.github/workflows/Deploy.yml @@ -2,7 +2,7 @@ name: Sap Change Model Deploy on: push: - branches: [ sap-dev, sap-prod, heat-dev, heat-prod, carbon-dev, carbon-prod, lighting-dev, lighting-prod ] + branches: [ sap-dev, sap-prod, heat-dev, heat-prod, carbon-dev, carbon-prod, cost-dev, cost-prod] jobs: deploy: diff --git a/.github/workflows/MLPipelinePostMerge.yml b/.github/workflows/MLPipelinePostMerge.yml index 78ea05a..1eaba31 100644 --- a/.github/workflows/MLPipelinePostMerge.yml +++ b/.github/workflows/MLPipelinePostMerge.yml @@ -13,7 +13,7 @@ on: - "sap-dev" - "heat-dev" - "carbon-dev" - - "lighting-dev" + - "cost-dev" permissions: write-all diff --git a/.github/workflows/MLPipelinePullRequest.yml b/.github/workflows/MLPipelinePullRequest.yml index aa701db..c60ecb8 100644 --- a/.github/workflows/MLPipelinePullRequest.yml +++ b/.github/workflows/MLPipelinePullRequest.yml @@ -5,7 +5,7 @@ on: # branches: # - "model-**" pull_request: - branches: ["sap-dev", "heat-dev", "carbon-dev", "lighting-dev"] + branches: ["sap-dev", "heat-dev", "carbon-dev", "cost-dev"] label: types: ["created", "edited"] diff --git a/modules/ml-pipeline/src/pipeline/configs/feature_processor_logic.py b/modules/ml-pipeline/src/pipeline/configs/feature_processor_logic.py index 103168d..40ec912 100644 --- a/modules/ml-pipeline/src/pipeline/configs/feature_processor_logic.py +++ b/modules/ml-pipeline/src/pipeline/configs/feature_processor_logic.py @@ -44,6 +44,11 @@ def keep_non_zero_rdsap(df): df = df[df["rdsap_change"] != 0] return df +def create_overall_costs(df): + df["overall_costs_starting"] = df["lighting_cost_starting"] + df["heating_cost_starting"] + df["hot_water_cost_starting"] + df["overall_costs_ending"] = df["lighting_cost_ending"] + df["heating_cost_ending"] + df["hot_water_cost_ending"] + return df + # def keep_ending_columns(df): # ending_column_index = [ col_name.endswith("_ENDING") for col_name in list(df.columns)] @@ -54,6 +59,7 @@ def keep_non_zero_rdsap(df): # return df business_logic = { + "create_overall_costs": create_overall_costs # "keep_non_zero_rdsap": keep_non_zero_rdsap, # "keep_flats": keep_flats, # "remove_minimum_habitable_room_size": remove_minimum_habitable_room_size, diff --git a/modules/ml-pipeline/src/pipeline/configs/settings.yaml b/modules/ml-pipeline/src/pipeline/configs/settings.yaml index 3adbe55..e2c69ba 100644 --- a/modules/ml-pipeline/src/pipeline/configs/settings.yaml +++ b/modules/ml-pipeline/src/pipeline/configs/settings.yaml @@ -32,12 +32,12 @@ default: feature_processor_config: subsample_amount: null subsample_seed: 0 - target: lighting_cost_ending + target: overall_costs_ending identifier_columns: ["uprn"] # drop_columns: ["heat_demand_change", "carbon_change", "rdsap_change", "heat_demand_ending", "carbon_ending", "days_to_starting", "days_to_ending"] drop_columns: [ "sap_ending", "heat_demand_change", "carbon_change", "rdsap_change", "heat_demand_ending", "carbon_ending", - "heating_cost_ending", "hot_water_cost_ending", + "lighting_cost_ending", "heating_cost_ending", "hot_water_cost_ending", # "days_to_starting", "days_to_ending", 'number_habitable_rooms_starting', 'number_habitable_rooms_ending', 'number_heated_rooms_starting', 'number_heated_rooms_ending', 'number_habitable_rooms', 'number_heated_rooms'] diff --git a/modules/ml-pipeline/src/pipeline/dvc.lock b/modules/ml-pipeline/src/pipeline/dvc.lock index a34651c..f82557b 100644 --- a/modules/ml-pipeline/src/pipeline/dvc.lock +++ b/modules/ml-pipeline/src/pipeline/dvc.lock @@ -27,6 +27,7 @@ stages: - rdsap_change - heat_demand_ending - carbon_ending + - lighting_cost_ending - heating_cost_ending - hot_water_cost_ending - number_habitable_rooms_starting @@ -38,7 +39,7 @@ stages: default.feature_processor.feature_processor_config.retain_features: default.feature_processor.feature_processor_config.subsample_amount: default.feature_processor.feature_processor_config.subsample_seed: 0 - default.feature_processor.feature_processor_config.target: lighting_cost_ending + default.feature_processor.feature_processor_config.target: overall_costs_ending default.feature_processor.feature_processor_type: dataframe default.prepare_data.data_filepath: s3://retrofit-data-dev/sap_change_model/2024-07-03-23-11-39/dataset_rooms.parquet @@ -50,8 +51,8 @@ stages: outs: - path: data/prepared_data/ hash: md5 - md5: 0f11a02cf75c0421757c0b26184cec33.dir - size: 48971227 + md5: e484d5032d234731d7f0b218310161e6.dir + size: 50645982 nfiles: 2 build_model: cmd: python 2_build_model.py @@ -62,8 +63,8 @@ stages: size: 4820 - path: data/prepared_data hash: md5 - md5: 0f11a02cf75c0421757c0b26184cec33.dir - size: 48971227 + md5: e484d5032d234731d7f0b218310161e6.dir + size: 50645982 nfiles: 2 params: configs/build_model.yaml: @@ -95,18 +96,18 @@ stages: outs: - path: data/fit_predictions/ hash: md5 - md5: 36c41f88681ab90668c17ce63fd9c318.dir - size: 3444201 + md5: 815c326a9ef5c5bea201c06cb04fee7e.dir + size: 3445213 nfiles: 1 - path: data/model/ hash: md5 - md5: bb9c3f1538e02e20e918ec36a0b7546f.dir - size: 754271944 + md5: 3c5340b992ffb3162d65361ba9d09bbd.dir + size: 823603444 nfiles: 37 - path: metrics/fit_metrics.json hash: md5 - md5: 16ae1efa8ac48d8ed978bb3fa67be64a - size: 225 + md5: 7d0715b84211f5cdef7a41954e7bc8e5 + size: 222 generate_predictions: cmd: python 3_generate_predictions.py deps: @@ -116,13 +117,13 @@ stages: size: 2464 - path: data/model hash: md5 - md5: bb9c3f1538e02e20e918ec36a0b7546f.dir - size: 754271944 + md5: 3c5340b992ffb3162d65361ba9d09bbd.dir + size: 823603444 nfiles: 37 - path: data/prepared_data hash: md5 - md5: 0f11a02cf75c0421757c0b26184cec33.dir - size: 48971227 + md5: e484d5032d234731d7f0b218310161e6.dir + size: 50645982 nfiles: 2 params: configs/settings.yaml: @@ -134,8 +135,8 @@ stages: outs: - path: data/predictions/ hash: md5 - md5: 50909a5b19c2551410e921dc9a92bef7.dir - size: 480359 + md5: 7d223c9cf49cdbc520ba607e00868123.dir + size: 480423 nfiles: 1 generate_metrics: cmd: python 4_generate_metrics.py @@ -146,13 +147,13 @@ stages: size: 3484 - path: data/predictions hash: md5 - md5: 50909a5b19c2551410e921dc9a92bef7.dir - size: 480359 + md5: 7d223c9cf49cdbc520ba607e00868123.dir + size: 480423 nfiles: 1 - path: data/prepared_data hash: md5 - md5: 0f11a02cf75c0421757c0b26184cec33.dir - size: 48971227 + md5: e484d5032d234731d7f0b218310161e6.dir + size: 50645982 nfiles: 2 params: configs/settings.yaml: @@ -162,8 +163,8 @@ stages: outs: - path: metrics/metrics.json hash: md5 - md5: d74767b34a1042c9ab0e3d6535791be6 - size: 224 + md5: 9700491cc19bd64a616cf504b0106896 + size: 221 generate_scenerio_metrics: cmd: python 5_generate_scenarios.py deps: