Merge pull request #109 from Hestia-Homes/heat-dev-model

Heat dev model
2026-06-08 11:17:25 +00:00 · 2024-03-28 17:21:46 +00:00 · 2024-03-28 17:21:46 +00:00 · e5d5b97269
commit e5d5b97269
parent 5290a0c769 5415cc972d
18 changed files with 108 additions and 79 deletions
--- a/MODEL_REGISTRY.md
+++ b/MODEL_REGISTRY.md
@ -18,7 +18,7 @@
    "heat": {
        "version": "v0.4.0",
        "stage": {
-            "dev": "v0.4.0"
+            "dev": "v0.5.0"
        },
        "registered": true,
        "active": true
--- a/deployment/.dockerignore
+++ b/deployment/.dockerignore
@ -0,0 +1,4 @@
+modules/ml-pipeline/src/pipeline/data/predictions*
+modules/ml-pipeline/src/pipeline/data/prepared_data*
+modules/ml-pipeline/src/pipeline/data/model/allmodels*
+modules/ml-pipeline/src/pipeline/metrics*
--- a/deployment/Dockerfile.prediction.lambda
+++ b/deployment/Dockerfile.prediction.lambda
@ -9,7 +9,7 @@ ARG RUNTIME_ENVIRONMENT
 ENV RUNTIME_ENVIRONMENT=${RUNTIME_ENVIRONMENT}

 # Install necessary build tools - required to test locally
-RUN yum install -y gcc python3-devel
+RUN yum install -y gcc python3-devel gcc-c++

 # Install python packages
 COPY modules/ml-pipeline/src/pipeline/requirements/predictions/requirements.txt ./requirements.txt
--- a/modules/ml-pipeline/.dvc/.gitignore
+++ b/modules/ml-pipeline/.dvc/.gitignore
@ -1,3 +0,0 @@
-/config.local
-/tmp
-/cache
--- a/modules/ml-pipeline/.dvc/config
+++ b/modules/ml-pipeline/.dvc/config
@ -1,2 +0,0 @@
-['remote "myremote"']
-    url = /tmp/dvcstore
--- a/modules/ml-pipeline/.dvcignore
+++ b/modules/ml-pipeline/.dvcignore
@ -1,3 +0,0 @@
-# Add patterns of files dvc should ignore, which could improve
-# the performance. Learn more at
-# https://dvc.org/doc/user-guide/dvcignore
--- a/modules/ml-pipeline/.gto
+++ b/modules/ml-pipeline/.gto
@ -1,2 +0,0 @@
-# .gto config file
-stages: [dev, stage, prod] # list of allowed Stages
--- a/modules/ml-pipeline/src/.dockerignore
+++ b/modules/ml-pipeline/src/.dockerignore
@ -0,0 +1,4 @@
+pipeline/data/predictions*
+pipeline/data/prepared_data/train.parquet*
+pipeline/data/model/allmodels*
+pipeline/metrics*
--- a/modules/ml-pipeline/src/pipeline/4_generate_metrics.py
+++ b/modules/ml-pipeline/src/pipeline/4_generate_metrics.py
@ -31,7 +31,6 @@ predictions_output_filepath = generate_predictions_params["predictions_output_fi
 predictions_column_name = generate_predictions_params["predictions_column_name"]
 metrics_output_filepath = generate_metrics_params["metrics_output_filepath"]

-
 logger.info(f"--- Initiate MLModel ---")

 model = model_factory(build_model_params["model_type"])
--- a/modules/ml-pipeline/src/pipeline/configs/analysis.yaml
+++ b/modules/ml-pipeline/src/pipeline/configs/analysis.yaml
@ -13,4 +13,4 @@ default:
    dataclient_type: local
    nshap_samples: 100 # how many samples to use to approximate each Shapely value, larger values will be slower
    n_val: 30  # how many datapoints from validation data should we interpret predictions for, larger values will be slower
-    row_index: [0, 10, 20] # index of an example datapoint
+    row_index: [20695, 50243, 7653] # index of an example datapoint
--- a/modules/ml-pipeline/src/pipeline/configs/post_prediction_logic.py
+++ b/modules/ml-pipeline/src/pipeline/configs/post_prediction_logic.py
@ -1,6 +1,7 @@
 """
 After predictions, we may want to apply some post processing to the predictions
 """
+
 import pandas as pd


--- a/modules/ml-pipeline/src/pipeline/configs/settings.yaml
+++ b/modules/ml-pipeline/src/pipeline/configs/settings.yaml
@ -18,12 +18,9 @@ default:
  prepare_data:
    input_dataclient_type: aws-s3
    output_dataclient_type: local
-    # data_filepath: s3://retrofit-data-dev/sap_change_model/dataset_with_differencing.parquet
-    # data_filepath: s3://retrofit-data-dev/sap_change_model/floor_area_clean_test.parquet
-    # data_filepath: s3://retrofit-data-dev/sap_change_model/dataset_without_differencing.parquet
-    # data_filepath: s3://retrofit-data-dev/sap_change_model/dataset.parquet
-    data_filepath: s3://retrofit-datalake-dev/dataset_with0perm_all.parquet
-    train_proportion: 1
+    # data_filepath: s3://retrofit-datalake-dev/dataset_with0perm_all.parquet
+    data_filepath: s3://retrofit-data-dev/sap_change_model/2024-03-22-18-56-53/dataset_rooms.parquet
+    train_proportion: 0.9
    output_train_filepath: ./data/prepared_data/train.parquet
    output_test_filepath: ./data/prepared_data/test.parquet

@ -34,7 +31,10 @@ default:
      subsample_seed: 0
      target: heat_demand_ending
      identifier_columns: ["uprn"]
-      drop_columns: ["heat_demand_change", "carbon_change", "rdsap_change", "sap_ending", "carbon_ending"]
+      drop_columns: [
+        "heat_demand_change", "carbon_change", "rdsap_change", "sap_ending", "carbon_ending", "days_to_starting", "days_to_ending",
+        'number_habitable_rooms_starting', 'number_habitable_rooms_ending', 'number_heated_rooms_starting', 'number_heated_rooms_ending',
+        'number_habitable_rooms', 'number_heated_rooms']
      # retain_features: ["SAP_STARTING", "TOTAL_FLOOR_AREA_DIFF"]
      retain_features: null

--- a/modules/ml-pipeline/src/pipeline/dvc.lock
+++ b/modules/ml-pipeline/src/pipeline/dvc.lock
@ -1,5 +1,16 @@
 schema: '2.0'
 stages:
+  startup_cleanup:
+    cmd: python 0_startup_cleanup.py
+    deps:
+    - path: 0_startup_cleanup.py
+      hash: md5
+      md5: b1b12f6b6393fbf8b83d23684df0a3d4
+      size: 1220
+    params:
+      configs/settings.yaml:
+        default.startup_cleanup.artefacts: ./data
+        default.startup_cleanup.metrics: ./metrics
  prepare_data:
    cmd: python 1_prepare_data.py
    deps:
@ -15,22 +26,31 @@ stages:
        - rdsap_change
        - sap_ending
        - carbon_ending
+        - days_to_starting
+        - days_to_ending
+        - number_habitable_rooms_starting
+        - number_habitable_rooms_ending
+        - number_heated_rooms_starting
+        - number_heated_rooms_ending
+        - number_habitable_rooms
+        - number_heated_rooms
        default.feature_processor.feature_processor_config.retain_features:
        default.feature_processor.feature_processor_config.subsample_amount:
        default.feature_processor.feature_processor_config.subsample_seed: 0
        default.feature_processor.feature_processor_config.target: heat_demand_ending
        default.feature_processor.feature_processor_type: dataframe
-        default.prepare_data.data_filepath: s3://retrofit-datalake-dev/dataset_with0perm_all.parquet
+        default.prepare_data.data_filepath: 
+          s3://retrofit-data-dev/sap_change_model/2024-03-22-18-56-53/dataset_rooms.parquet
        default.prepare_data.input_dataclient_type: aws-s3
        default.prepare_data.output_dataclient_type: local
        default.prepare_data.output_test_filepath: ./data/prepared_data/test.parquet
        default.prepare_data.output_train_filepath: ./data/prepared_data/train.parquet
-        default.prepare_data.train_proportion: 1
+        default.prepare_data.train_proportion: 0.9
    outs:
    - path: data/prepared_data/
      hash: md5
-      md5: dcd41f841c67b474a81a14e683646237.dir
-      size: 36317761
+      md5: 4cec69f112537658f14eb3cb678f91e3.dir
+      size: 36889932
      nfiles: 2
  build_model:
    cmd: python 2_build_model.py
@ -41,8 +61,8 @@ stages:
      size: 4820
    - path: data/prepared_data
      hash: md5
-      md5: dcd41f841c67b474a81a14e683646237.dir
-      size: 36317761
+      md5: 4cec69f112537658f14eb3cb678f91e3.dir
+      size: 36889932
      nfiles: 2
    params:
      configs/build_model.yaml:
@ -73,18 +93,18 @@ stages:
    outs:
    - path: data/fit_predictions/
      hash: md5
-      md5: 89063bb3b725afe61b6ed5edb724bb06.dir
-      size: 3090627
+      md5: 7dda2f1dd257a6c5beaaa0b74eab6d5d.dir
+      size: 2901760
      nfiles: 1
    - path: data/model/
      hash: md5
-      md5: c90eef03b5a76175506c048e88a401dd.dir
-      size: 783489255
+      md5: 741f8aed57383e860c535feb8b0adb71.dir
+      size: 752079341
      nfiles: 32
    - path: metrics/fit_metrics.json
      hash: md5
-      md5: 33f18fa6b7dda535de09733d4792c0fc
-      size: 217
+      md5: 8eaa72b08074f735a9e54de871edc6e6
+      size: 221
  generate_predictions:
    cmd: python 3_generate_predictions.py
    deps:
@ -94,13 +114,13 @@ stages:
      size: 2464
    - path: data/model
      hash: md5
-      md5: c90eef03b5a76175506c048e88a401dd.dir
-      size: 783489255
+      md5: 741f8aed57383e860c535feb8b0adb71.dir
+      size: 752079341
      nfiles: 32
    - path: data/prepared_data
      hash: md5
-      md5: dcd41f841c67b474a81a14e683646237.dir
-      size: 36317761
+      md5: 4cec69f112537658f14eb3cb678f91e3.dir
+      size: 36889932
      nfiles: 2
    params:
      configs/settings.yaml:
@ -112,25 +132,25 @@ stages:
    outs:
    - path: data/predictions/
      hash: md5
-      md5: 406e2ebe33d6abed9042f137d8c0d2bf.dir
-      size: 520735
+      md5: d842fe5350a3330c4c17e7e21c6359b2.dir
+      size: 380489
      nfiles: 1
  generate_metrics:
    cmd: python 4_generate_metrics.py
    deps:
    - path: 4_generate_metrics.py
      hash: md5
-      md5: 567b1acb819e2ff432b989cdbdd4a2bf
-      size: 3448
+      md5: d61bb524f706917f6a3eb72b1ab8bc61
+      size: 3447
    - path: data/predictions
      hash: md5
-      md5: 406e2ebe33d6abed9042f137d8c0d2bf.dir
-      size: 520735
+      md5: d842fe5350a3330c4c17e7e21c6359b2.dir
+      size: 380489
      nfiles: 1
    - path: data/prepared_data
      hash: md5
-      md5: dcd41f841c67b474a81a14e683646237.dir
-      size: 36317761
+      md5: 4cec69f112537658f14eb3cb678f91e3.dir
+      size: 36889932
      nfiles: 2
    params:
      configs/settings.yaml:
@ -140,16 +160,5 @@ stages:
    outs:
    - path: metrics/metrics.json
      hash: md5
-      md5: cc1ad408f2d9d3128df71822a38ea85e
-      size: 218
-  startup_cleanup:
-    cmd: python 0_startup_cleanup.py
-    deps:
-    - path: 0_startup_cleanup.py
-      hash: md5
-      md5: b1b12f6b6393fbf8b83d23684df0a3d4
-      size: 1220
-    params:
-      configs/settings.yaml:
-        default.startup_cleanup.artefacts: ./data
-        default.startup_cleanup.metrics: ./metrics
+      md5: 2632fa5d0a38763c177bf0466a670c8b
+      size: 220
--- a/modules/ml-pipeline/src/pipeline/eda.py
+++ b/modules/ml-pipeline/src/pipeline/eda.py
@ -1,6 +1,7 @@
 """
 Doing some eda on dataset
 """
+
 # Look at response variable

 from matplotlib import pyplot as plt
@ -191,22 +192,32 @@ prediction_analysis_params = settings.prediction_analysis
 model = model_factory(build_model_params["model_type"])
 model.load_model(build_model_params["model_save_filepath"])
 dataclient_type = prediction_analysis_params["dataclient_type"]
-dataclient = dataclient_factory(
-    dataclient_type=dataclient_type,
-    dataclient_config=client_params[dataclient_type],
-)
+# dataclient_type = 'aws-s3'
+# dataclient = dataclient_factory(
+#     dataclient_type=dataclient_type,
+#     dataclient_config=client_params[dataclient_type],
+# )
+# data = dataclient.load_data("s3://retrofit-data-dev/sap_change_model/dataset.parquet")

 target = feature_process_params["feature_processor_config"]["target"]
 predictions_column_name = generate_predictions_params["predictions_column_name"]
 output_test_filepath = prepare_data_params["output_test_filepath"]
 predictions_output_filepath = generate_predictions_params["predictions_output_filepath"]

-test_df = dataclient.load_data(output_test_filepath)
-predictions = dataclient.load_data(predictions_output_filepath)
+# score_data = dataclient.load_data("s3://retrofit-data-dev/carbon_change_predictions/51/2023-11-28T21:01:21.869339.parquet")
+
+
+local_dataclient = dataclient_factory(
+    dataclient_type="local",
+    dataclient_config=client_params["local"],
+)
+test_df = local_dataclient.load_data(output_test_filepath)
+predictions = local_dataclient.load_data(predictions_output_filepath)
 mix_df = pd.concat([test_df.copy(), predictions], axis=1)
 mix_df["residual"] = abs(mix_df[predictions_column_name] - mix_df[target])
 mix_df = mix_df.sort_values("residual", ascending=False)

+cosine_similarity_df = mix_df[mix_df.columns.difference(["predictions", "residual"])]
 metrics = metrics_factory("Regression")
 metrics.generate_metrics(mix_df["predictions"], mix_df["HEAT_DEMAND_ENDING"])

@ -215,7 +226,7 @@ cosine_similarity_df = mix_df[
 ]
 from sklearn.metrics.pairwise import cosine_similarity

-row_index = 58199
+row_index = 0

 from sklearn.preprocessing import LabelEncoder

@ -229,7 +240,17 @@ feature_vector = cosine_similarity_df.loc[[row_index]]

 cosine_similarity_df["cosine"] = cosine_similarity(cosine_similarity_df, feature_vector)
 similar_index = (
-    cosine_similarity_df.sort_values("cosine", ascending=False).head(5).index
+    cosine_similarity_df.sort_values("cosine", ascending=False).head(15).index
 )

 check_df = mix_df.loc[similar_index]
+
+columns_to_check = [
+    "LOW_ENERGY_LIGHTING_ENDING",
+    "walls_thermal_transmittance_ENDING",
+    "floor_thermal_transmittance_ENDING",
+    "roof_thermal_transmittance_ENDING",
+    "roof_insulation_thickness_ENDING",
+]
+
+cosine_similarity_df = mix_df[columns_to_check]
--- a/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements-dev.txt
+++ b/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements-dev.txt
@ -1,7 +1,7 @@
 joblib==1.3.2
 boto3==1.28.17
-pandas==1.5.3
-autogluon==0.8.2
-dynaconf==3.2.0
+pandas==2.1.4
+autogluon==1.0.0
+dynaconf==3.2.1
 pyarrow==13.0.0
 pre-commit==3.3.3
--- a/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements.txt
+++ b/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements.txt
@ -1,7 +1,7 @@
 joblib==1.3.2
 boto3==1.28.17
-pandas==1.5.3
-autogluon==0.8.2
-dynaconf==3.2.0
+pandas==2.1.4
+autogluon==1.0.0
+dynaconf==3.2.1
 pyarrow==13.0.0
 PyYAML==6.0.1
--- a/modules/ml-pipeline/src/pipeline/requirements/training/requirements-dev.txt
+++ b/modules/ml-pipeline/src/pipeline/requirements/training/requirements-dev.txt
@ -1,9 +1,10 @@
 joblib==1.3.2
 boto3==1.28.17
-pandas==1.5.3
-autogluon==0.8.2
-dynaconf==3.2.0
-alibi==0.9.4
+pandas==2.1.4
+autogluon==1.0.0
+ray==2.6.3
+dynaconf==3.2.1
+alibi==0.9.5
 shap==0.42.1
 pyarrow==13.0.0
 pre-commit==3.3.3
--- a/modules/ml-pipeline/src/pipeline/requirements/training/requirements.txt
+++ b/modules/ml-pipeline/src/pipeline/requirements/training/requirements.txt
@ -1,4 +1,4 @@
 boto3==1.28.41
-pandas==1.5.3
-autogluon==0.8.2
-dynaconf==3.2.0
+pandas==2.1.4
+autogluon==1.0.0
+dynaconf==3.2.1