junte/ML
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/ML.git synced 2026-06-08 11:17:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

train new 600 second model with new data

Browse source
This commit is contained in:
Michael Duong 2024-01-18 00:14:20 +00:00
parent 9271df34e0
commit 66f54a92e2
6 changed files with 48 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
modules/ml-pipeline/src/pipeline/configs/build_model.yaml
View file

@ -13,7 +13,7 @@ default:
output_filepath: ./data/model/allmodels/
problem_type: regression
eval_metric: mean_squared_error #mean_absolute_error
time_limit: 400
time_limit: 600
presets: medium_quality
excluded_model_types: ['KNN', 'RF']
infer_limit: 0.05

14
modules/ml-pipeline/src/pipeline/configs/feature_processor_logic.py
View file

@ -9,22 +9,22 @@ Business Logic dict + functions
def remove_starting_columns(df):
keep_column_index = [
False if col_name.endswith("_STARTING") else True
False if col_name.endswith("_starting") else True
for col_name in list(df.columns)
]
keep_columns = df.columns[keep_column_index].to_list()
keep_columns.append("SAP_STARTING")
keep_columns.append("sap_starting")
df = df[keep_columns]
return df
def keep_negative_heat_change(df):
df = df[df["HEAT_DEMAND_CHANGE"] < 0]
df = df[df["heat_demand_change"] < 0]
return df
def keep_negative_carbon_change(df):
df = df[df["CARBON_CHANGE"] < 0]
df = df[df["carbon_change"] < 0]
return df
@ -34,7 +34,7 @@ def remove_unreasonable_habitable_rooms(df):
Assumption is that proportion of floor area to habitable rooms should be at least 6.5m2
"""
minimum_room_size_index = (
df["TOTAL_FLOOR_AREA_ENDING"] / df["NUMBER_HABITABLE_ROOMS"] >= 6.5
df["total_floor_area_ending"] / df["number_habitable_rooms"] >= 6.5
)
df = df[minimum_room_size_index]
return df
@ -43,14 +43,14 @@ def remove_unreasonable_habitable_rooms(df):
def remove_top_1_percent_heat_demand(df):
# threshold_value = df.describe(percentiles=[0.99])['HEAT_DEMAND_STARTING']['99%']
threshold_value = 860
df = df[df["HEAT_DEMAND_STARTING"] < threshold_value]
df = df[df["heat_demand_starting"] < threshold_value]
return df
def remove_top_1_percent_carbon(df):
# threshold_value = df.describe(percentiles=[0.99])['CARBON_STARTING']['99%']
threshold_value = 18
df = df[df["CARBON_STARTING"] < threshold_value]
df = df[df["carbon_starting"] < threshold_value]
return df

4
modules/ml-pipeline/src/pipeline/configs/post_prediction_logic.py
View file

@ -13,10 +13,10 @@ def clip_predictions_to_minimum_value(
predictions_df = pd.concat([data, predictions], axis=1)
# We expect all prediction to be atleast one point improvement
replace_index = (
predictions_df["predictions"] > predictions_df["HEAT_DEMAND_STARTING"] - 1
predictions_df["predictions"] > predictions_df["heat_demand_starting"] - 1
)
predictions_df.loc[replace_index, "predictions"] = (
predictions_df.loc[replace_index, "HEAT_DEMAND_STARTING"] - minimum_value
predictions_df.loc[replace_index, "heat_demand_starting"] - minimum_value
)
predictions_new = predictions_df["predictions"]

8
modules/ml-pipeline/src/pipeline/configs/settings.yaml
View file

@ -21,7 +21,7 @@ default:
# data_filepath: s3://retrofit-data-dev/sap_change_model/dataset_with_differencing.parquet
# data_filepath: s3://retrofit-data-dev/sap_change_model/floor_area_clean_test.parquet
# data_filepath: s3://retrofit-data-dev/sap_change_model/dataset_without_differencing.parquet
data_filepath: s3://retrofit-data-dev/sap_change_model/dataset_test.parquet
data_filepath: s3://retrofit-data-dev/sap_change_model/dataset.parquet
train_proportion: 0.9
output_train_filepath: ./data/prepared_data/train.parquet
output_test_filepath: ./data/prepared_data/test.parquet
@ -31,9 +31,9 @@ default:
feature_processor_config:
subsample_amount: null
subsample_seed: 0
target: HEAT_DEMAND_ENDING
identifier_columns: ["UPRN"]
drop_columns: ["HEAT_DEMAND_CHANGE", "CARBON_CHANGE", "RDSAP_CHANGE", "SAP_ENDING", "CARBON_ENDING"]
target: heat_demand_ending
identifier_columns: ["uprn"]
drop_columns: ["heat_demand_change", "carbon_change", "rdsap_change", "sap_ending", "carbon_ending"]
# retain_features: ["SAP_STARTING", "TOTAL_FLOOR_AREA_DIFF"]
retain_features: null

60
modules/ml-pipeline/src/pipeline/dvc.lock
View file

@ -10,17 +10,17 @@ stages:
params:
configs/settings.yaml:
default.feature_processor.feature_processor_config.drop_columns:
- HEAT_DEMAND_CHANGE
- CARBON_CHANGE
- RDSAP_CHANGE
- SAP_ENDING
- CARBON_ENDING
- heat_demand_change
- carbon_change
- rdsap_change
- sap_ending
- carbon_ending
default.feature_processor.feature_processor_config.retain_features:
default.feature_processor.feature_processor_config.subsample_amount:
default.feature_processor.feature_processor_config.subsample_seed: 0
default.feature_processor.feature_processor_config.target: HEAT_DEMAND_ENDING
default.feature_processor.feature_processor_config.target: heat_demand_ending
default.feature_processor.feature_processor_type: dataframe
default.prepare_data.data_filepath: s3://retrofit-data-dev/sap_change_model/dataset_test.parquet
default.prepare_data.data_filepath: s3://retrofit-data-dev/sap_change_model/dataset.parquet
default.prepare_data.input_dataclient_type: aws-s3
default.prepare_data.output_dataclient_type: local
default.prepare_data.output_test_filepath: ./data/prepared_data/test.parquet
@ -29,8 +29,8 @@ stages:
outs:
- path: data/prepared_data/
hash: md5
md5: f235f38714fefcf6e4927ae95ba912c3.dir
size: 30774760
md5: 613ddd198a29002e6e05a2d60275d924.dir
size: 32746979
nfiles: 2
build_model:
cmd: python 2_build_model.py
@ -41,8 +41,8 @@ stages:
size: 4149
- path: data/prepared_data
hash: md5
md5: f235f38714fefcf6e4927ae95ba912c3.dir
size: 30774760
md5: 613ddd198a29002e6e05a2d60275d924.dir
size: 32746979
nfiles: 2
params:
configs/build_model.yaml:
@ -58,7 +58,7 @@ stages:
output_filepath: ./data/model/allmodels/
problem_type: regression
eval_metric: mean_squared_error
time_limit: 400
time_limit: 600
presets: medium_quality
excluded_model_types:
- KNN
@ -68,13 +68,13 @@ stages:
outs:
- path: data/model/
hash: md5
md5: a868845999b46e0272dc27f5cb5bc618.dir
size: 310555147
nfiles: 24
md5: 837a42a0655862229620495c645d5fed.dir
size: 342382387
nfiles: 26
- path: metrics/fit_metrics.json
hash: md5
md5: 809f27735c77cbcb62866b96018eedea
size: 216
md5: f8a394b86c33dc1b3ce97abed803c8f1
size: 220
generate_predictions:
cmd: python 3_generate_predictions.py
deps:
@ -84,13 +84,13 @@ stages:
size: 2464
- path: data/model
hash: md5
md5: a868845999b46e0272dc27f5cb5bc618.dir
size: 310555147
nfiles: 24
md5: 837a42a0655862229620495c645d5fed.dir
size: 342382387
nfiles: 26
- path: data/prepared_data
hash: md5
md5: f235f38714fefcf6e4927ae95ba912c3.dir
size: 30774760
md5: 613ddd198a29002e6e05a2d60275d924.dir
size: 32746979
nfiles: 2
params:
configs/settings.yaml:
@ -102,8 +102,8 @@ stages:
outs:
- path: data/predictions/
hash: md5
md5: 2098fe82304751025e427f2cc241a2ff.dir
size: 295849
md5: 75f8326e99eb9e1032728208229ec37b.dir
size: 314002
nfiles: 1
generate_metrics:
cmd: python 4_generate_metrics.py
@ -114,13 +114,13 @@ stages:
size: 3448
- path: data/predictions
hash: md5
md5: 2098fe82304751025e427f2cc241a2ff.dir
size: 295849
md5: 75f8326e99eb9e1032728208229ec37b.dir
size: 314002
nfiles: 1
- path: data/prepared_data
hash: md5
md5: f235f38714fefcf6e4927ae95ba912c3.dir
size: 30774760
md5: 613ddd198a29002e6e05a2d60275d924.dir
size: 32746979
nfiles: 2
params:
configs/settings.yaml:
@ -130,8 +130,8 @@ stages:
outs:
- path: metrics/metrics.json
hash: md5
md5: aa671878e1bd8c6a8d4b5f9788c817c4
size: 219
md5: 269e89593f5e7ceb507c31dac2c2dd35
size: 220
startup_cleanup:
cmd: python 0_startup_cleanup.py
deps:

8
modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt
View file

@ -1,4 +1,4 @@
dvc==3.18.0
dvc-s3==2.23.0
gto==1.0.4
pyOpenSSL==23.2.0
dvc==3.36.0
dvc-s3==3.0.1
gto==1.6.1
pyOpenSSL==23.3.0