medium model with scenario and upgraded autogluon

.github/workflows/MLPipelinePullRequest.yml

@@ -98,6 +98,10 @@ jobs:
         git fetch --depth=1 origin ${TARGET_BRANCH}:${TARGET_BRANCH}
         dvc metrics diff --md --all ${TARGET_BRANCH} >> report.md
 
+        echo "## Scenario metrics" >> report.md
+
+        cat metrics/scenario_table.md >> report.md
+
         cml comment create report.md
 
         # echo "## Residuals plot from model" >> report.md

modules/ml-pipeline/src/pipeline/5_generate_scenarios.py

@@ -0,0 +1,125 @@
+"""
+Fourth part of the pipeline:
+After the model is built and metrics are generated,
+we want to test this model against known scenarios
+"""
+
+import os
+import pandas as pd
+from core.interface.InterfaceModels import MLModel
+from core.interface.InterfaceDataClient import DataClient
+from configs.post_prediction_logic import post_prediction_logic
+from core.DataClient import dataclient_factory
+from core.MLModels import model_factory
+from core.Logger import logger
+from config import settings
+
+logger.info(f"--- Initiate Parameters ---")
+
+RUNTIME_ENVIRONMENT = os.environ.get("RUNTIME_ENVIRONMENT", "local")
+
+client_params = settings.client
+prepare_data_params = settings.prepare_data
+build_model_params = settings.build_model
+generate_predictions_params = settings.generate_predictions
+generate_metrics_params = settings.generate_metrics
+feature_process_params = settings.feature_processor
+scenarios_params = settings.scenarios
+
+model_filepath = build_model_params["model_save_filepath"]
+target = feature_process_params["feature_processor_config"]["target"]
+scenario_data_filepaths = scenarios_params["scenario_data_filepaths"]
+predictions_column_name = generate_predictions_params["predictions_column_name"]
+output_filepath = scenarios_params["output_filepath"]
+
+logger.info(f"--- Initiate MLModel ---")
+
+model = model_factory(build_model_params["model_type"])
+
+logger.info(f"--- Initiate DataClient ---")
+
+# Use data client for input and output, as we use dvc to cache later to the cloud
+input_dataclient_type = scenarios_params["input_dataclient_type"]
+input_dataclient = dataclient_factory(
+    dataclient_type=input_dataclient_type,
+    dataclient_config=client_params[input_dataclient_type],
+)
+
+output_dataclient_type = scenarios_params["output_dataclient_type"]
+output_dataclient = dataclient_factory(
+    dataclient_type=output_dataclient_type,
+    dataclient_config=client_params[output_dataclient_type],
+)
+
+
+def generate_scenario_predictions(
+    input_dataclient: DataClient,
+    output_dataclient: DataClient,
+    model: MLModel,
+    model_filepath: str,
+    scenario_data_filepaths: list,
+    predictions_column_name: str,
+    output_filepath: str,
+):
+    """
+    Given the new model, we generate prediction for expected scenarios
+    """
+
+    logger.info("--- Loading Scenario Data ---")
+
+    scenario_data = pd.DataFrame()
+
+    # Can have multiple scenario data files
+    for scenario_data_filepath in scenario_data_filepaths:
+        scenario_data = pd.concat(
+            [
+                scenario_data,
+                input_dataclient.load_data(scenario_data_filepath, load_config=None),
+            ]
+        )
+
+    logger.info("--- Loading Model ---")
+
+    model.load_model(model_filepath)
+
+    logger.info("--- Generating Predictions ---")
+
+    predictions = model.predict(
+        data=scenario_data, post_prediction_logic=post_prediction_logic
+    )
+
+    logger.info("--- Generate Scenario Predicted Impact ---")
+
+    predictions_df = pd.DataFrame(predictions)
+    predictions_df.columns = [predictions_column_name]
+
+    scenario_data = pd.concat([scenario_data, predictions_df], axis=1)
+    scenario_data["predicted_impact"] = abs(
+        scenario_data[predictions_column_name] - scenario_data["sap_starting"]
+    )
+
+    logger.info("--- Save prediction into metrics ---")
+
+    output_df = scenario_data[["uprn", "id", "impact", "predicted_impact"]]
+
+    output_dataclient.save_data(
+        obj=output_df, location=output_filepath, save_config=None
+    )
+
+
+if __name__ == "__main__":
+    logger.info(f"--- {__file__} - Start! ---")
+
+    logger.info(f"--- Generate Scenario Predictions ---")
+
+    generate_scenario_predictions(
+        input_dataclient=input_dataclient,
+        output_dataclient=output_dataclient,
+        model=model,
+        model_filepath=model_filepath,
+        scenario_data_filepaths=scenario_data_filepaths,
+        predictions_column_name=predictions_column_name,
+        output_filepath=output_filepath,
+    )
+
+    logger.info(f"--- {__file__} - Complete! ---")

modules/ml-pipeline/src/pipeline/config.py

@@ -7,6 +7,7 @@ settings = Dynaconf(
         "./configs/settings.yaml",
         "./configs/build_model.yaml",
         "./configs/analysis.yaml",
+        "./configs/scenarios.yaml",
     ],
 )
 

modules/ml-pipeline/src/pipeline/configs/build_model.yaml

@@ -15,8 +15,8 @@ default:
       problem_type: regression
       eval_metric: mean_squared_error #mean_absolute_error
       time_limit: 1800
-      presets: good_quality
+      presets: medium_quality
-      excluded_model_types: ['RF', 'FASTAI', 'CAT', 'NN_TORCH', 'KNN', 'XT']
+      excluded_model_types: ['RF', 'CAT', 'NN_TORCH', 'KNN', 'XT']
       infer_limit: 0.05
       infer_limit_batch_size: 10000
       ag_args_ensemble: {'num_folds_parallel': 2}

modules/ml-pipeline/src/pipeline/configs/scenarios.yaml

@@ -0,0 +1,8 @@
+default:
+  scenarios:
+    input_dataclient_type: aws-s3
+    output_dataclient_type: local
+    scenario_data_filepaths:
+      # - s3://retrofit-data-dev/scenario_data/22-03-2024-19-20-09/recommendations_scoring_data.parquet
+      - s3://retrofit-data-dev/scenario_data/24-03-2024-20-23-25/recommendations_scoring_data.parquet
+    output_filepath: ./metrics/scenario_table.md

modules/ml-pipeline/src/pipeline/configs/settings.yaml

@@ -18,12 +18,7 @@ default:
   prepare_data:
     input_dataclient_type: aws-s3
     output_dataclient_type: local
-    # data_filepath: s3://retrofit-data-dev/sap_change_model/dataset_with_differencing.parquet
+    data_filepath: s3://retrofit-data-dev/sap_change_model/2024-03-22-18-56-53/dataset_rooms.parquet
-    # data_filepath: s3://retrofit-data-dev/sap_change_model/floor_area_clean_test.parquet
-    # data_filepath: s3://retrofit-data-dev/sap_change_model/dataset_without_differencing.parquet
-    # data_filepath: s3://retrofit-data-dev/sap_change_model/dataset_test.parquet
-    data_filepath: s3://retrofit-data-dev/sap_change_model/dataset.parquet
-    # data_filepath: s3://retrofit-datalake-dev/dataset_with0perm_all.parquet
     train_proportion: 0.9
     output_train_filepath: ./data/prepared_data/train.parquet
     output_test_filepath: ./data/prepared_data/test.parquet
@@ -35,9 +30,35 @@ default:
       subsample_seed: 0
       target: sap_ending
       identifier_columns: ["uprn"]
-      drop_columns: ["heat_demand_change", "carbon_change", "rdsap_change", "heat_demand_ending", "carbon_ending", "days_to_starting", "days_to_ending"]
+      # drop_columns: ["heat_demand_change", "carbon_change", "rdsap_change", "heat_demand_ending", "carbon_ending", "days_to_starting", "days_to_ending"]
-      # retain_features: ["SAP_STARTING", "TOTAL_FLOOR_AREA_DIFF"]
+      drop_columns: [
+        "heat_demand_change", "carbon_change", "rdsap_change", "heat_demand_ending", "carbon_ending", "days_to_starting", "days_to_ending",
+        'number_habitable_rooms_starting', 'number_habitable_rooms_ending', 'number_heated_rooms_starting', 'number_heated_rooms_ending',
+        'number_habitable_rooms', 'number_heated_rooms']
       retain_features: null
+      # retain_features: ['uprn', 'sap_starting', 'hot_water_energy_eff_ending',
+      #  'mainheat_energy_eff_ending', 'constituency', 'roof_energy_eff_ending',
+      #  'walls_energy_eff_ending', 'secondheat_description_ending',
+      #  'property_type', 'mainheatc_energy_eff_ending', 'built_form',
+      #  'walls_insulation_thickness_ending', 'potential_energy_efficiency',
+      #  'transaction_type_ending',
+      #  'floor_thermal_transmittance_ending',
+      #  'low_energy_lighting_ending', 'heat_demand_starting',
+      #  'photo_supply_ending', 'carbon_starting',
+      #  'walls_thermal_transmittance_ending',
+      #  'roof_insulation_thickness_ending',
+      #  'total_floor_area_ending', 'number_open_fireplaces_ending',
+      #  'windows_energy_eff_ending',
+      #  'floor_height_ending',
+      #  'extension_count_ending',
+      #  'has_air_source_heat_pump_ending',
+      #  'charging_system_ending', 'construction_age_band', 'glazed_type_ending',
+      #  'roof_thermal_transmittance_ending',
+      #  'floor_insulation_thickness_ending', 'has_mains_gas_ending',
+      #  'estimated_perimeter_starting', 'energy_consumption_potential',
+      #  'environment_impact_potential', 'heater_type_ending',
+      #  'multi_glaze_proportion_ending',
+      #  'lighting_energy_eff_ending', 'fixed_lighting_outlets_count']
 
   generate_predictions:
     input_dataclient_type: local

modules/ml-pipeline/src/pipeline/core/DataClient.py

@@ -245,7 +245,8 @@ class LocalClient:
 
         save_methods = {
             ".parquet": self._save_parquet,
-            ".json": self._save_json
+            ".json": self._save_json,
+            ".md": self._save_md,
             # "": _save_directory(**save_config),
             # ADD MORE save_methods HERE
         }
@@ -294,3 +295,10 @@ class LocalClient:
         # Write the contents of the buffer to the local file
         with open(location, "wb") as f:
             f.write(buffer.getvalue())
+
+    def _save_md(self, obj: pd.DataFrame, location: str, save_config: dict):
+        """
+        Save object as markdown
+        """
+
+        obj.to_markdown(location, **save_config)

modules/ml-pipeline/src/pipeline/dvc.lock

@@ -1,5 +1,16 @@
 schema: '2.0'
 stages:
+  startup_cleanup:
+    cmd: python 0_startup_cleanup.py
+    deps:
+    - path: 0_startup_cleanup.py
+      hash: md5
+      md5: b1b12f6b6393fbf8b83d23684df0a3d4
+      size: 1220
+    params:
+      configs/settings.yaml:
+        default.startup_cleanup.artefacts: ./data
+        default.startup_cleanup.metrics: ./metrics
   prepare_data:
     cmd: python 1_prepare_data.py
     deps:
@@ -17,12 +28,19 @@ stages:
         - carbon_ending
         - days_to_starting
         - days_to_ending
+        - number_habitable_rooms_starting
+        - number_habitable_rooms_ending
+        - number_heated_rooms_starting
+        - number_heated_rooms_ending
+        - number_habitable_rooms
+        - number_heated_rooms
         default.feature_processor.feature_processor_config.retain_features:
         default.feature_processor.feature_processor_config.subsample_amount:
         default.feature_processor.feature_processor_config.subsample_seed: 0
         default.feature_processor.feature_processor_config.target: sap_ending
         default.feature_processor.feature_processor_type: dataframe
-        default.prepare_data.data_filepath: s3://retrofit-data-dev/sap_change_model/dataset.parquet
+        default.prepare_data.data_filepath:
+          s3://retrofit-data-dev/sap_change_model/2024-03-22-18-56-53/dataset_rooms.parquet
         default.prepare_data.input_dataclient_type: aws-s3
         default.prepare_data.output_dataclient_type: local
         default.prepare_data.output_test_filepath: ./data/prepared_data/test.parquet
@@ -31,8 +49,8 @@ stages:
     outs:
     - path: data/prepared_data/
       hash: md5
-      md5: 3d1144848fce4ce50f6abfaec5235552.dir
+      md5: efa416abea618ae6220a0c3d597603cf.dir
-      size: 46392840
+      size: 44750997
       nfiles: 2
   build_model:
     cmd: python 2_build_model.py
@@ -43,8 +61,8 @@ stages:
       size: 4820
     - path: data/prepared_data
       hash: md5
-      md5: 3d1144848fce4ce50f6abfaec5235552.dir
+      md5: efa416abea618ae6220a0c3d597603cf.dir
-      size: 46392840
+      size: 44750997
       nfiles: 2
     params:
       configs/build_model.yaml:
@@ -62,10 +80,9 @@ stages:
               problem_type: regression
               eval_metric: mean_squared_error
               time_limit: 1800
-              presets: good_quality
+              presets: medium_quality
               excluded_model_types:
               - RF
-              - FASTAI
               - CAT
               - NN_TORCH
               - KNN
@@ -77,18 +94,18 @@ stages:
     outs:
     - path: data/fit_predictions/
       hash: md5
-      md5: 346b6611afbf2070e038bf945249a86e.dir
+      md5: de46250d454c4d713ab580b10ff3fd31.dir
-      size: 3384302
+      size: 3349318
       nfiles: 1
     - path: data/model/
       hash: md5
-      md5: 8e37f21728cd092660bafa8c32dc109f.dir
+      md5: 18bd7a93ece75a65d3a950b7dfdab4fb.dir
-      size: 423840922
+      size: 735951861
-      nfiles: 118
+      nfiles: 35
     - path: metrics/fit_metrics.json
       hash: md5
-      md5: d63e1a8d31503055835ac35149554e41
+      md5: 8a952a5e884c268e6059357a627b9251
-      size: 223
+      size: 224
   generate_predictions:
     cmd: python 3_generate_predictions.py
     deps:
@@ -98,13 +115,13 @@ stages:
       size: 2464
     - path: data/model
       hash: md5
-      md5: 8e37f21728cd092660bafa8c32dc109f.dir
+      md5: 18bd7a93ece75a65d3a950b7dfdab4fb.dir
-      size: 423840922
+      size: 735951861
-      nfiles: 118
+      nfiles: 35
     - path: data/prepared_data
       hash: md5
-      md5: 3d1144848fce4ce50f6abfaec5235552.dir
+      md5: efa416abea618ae6220a0c3d597603cf.dir
-      size: 46392840
+      size: 44750997
       nfiles: 2
     params:
       configs/settings.yaml:
@@ -116,8 +133,8 @@ stages:
     outs:
     - path: data/predictions/
       hash: md5
-      md5: d148baf508140353d62c16d6ab0fb6b7.dir
+      md5: 07ef721a0dc94a52e3ba7a70ac45b8ff.dir
-      size: 469224
+      size: 463563
       nfiles: 1
   generate_metrics:
     cmd: python 4_generate_metrics.py
@@ -128,13 +145,13 @@ stages:
       size: 3484
     - path: data/predictions
       hash: md5
-      md5: d148baf508140353d62c16d6ab0fb6b7.dir
+      md5: 07ef721a0dc94a52e3ba7a70ac45b8ff.dir
-      size: 469224
+      size: 463563
       nfiles: 1
     - path: data/prepared_data
       hash: md5
-      md5: 3d1144848fce4ce50f6abfaec5235552.dir
+      md5: efa416abea618ae6220a0c3d597603cf.dir
-      size: 46392840
+      size: 44750997
       nfiles: 2
     params:
       configs/settings.yaml:
@@ -144,16 +161,25 @@ stages:
     outs:
     - path: metrics/metrics.json
       hash: md5
-      md5: 196232f94b563ac525cf65ee5cc6d639
+      md5: 9f863f47799d42c101eba3b03a179455
-      size: 222
+      size: 224
-  startup_cleanup:
+  generate_scenerio_metrics:
-    cmd: python 0_startup_cleanup.py
+    cmd: python 5_generate_scenarios.py
     deps:
-    - path: 0_startup_cleanup.py
+    - path: 5_generate_scenarios.py
       hash: md5
-      md5: b1b12f6b6393fbf8b83d23684df0a3d4
+      md5: 30f80ffeb6ee50c5f7b82943a4dc7702
-      size: 1220
+      size: 4014
     params:
-      configs/settings.yaml:
+      configs/scenarios.yaml:
-        default.startup_cleanup.artefacts: ./data
+        default.scenarios:
-        default.startup_cleanup.metrics: ./metrics
+          input_dataclient_type: aws-s3
+          output_dataclient_type: local
+          scenario_data_filepaths:
+          - s3://retrofit-data-dev/scenario_data/24-03-2024-20-23-25/recommendations_scoring_data.parquet
+          output_filepath: ./metrics/scenario_table.md
+    outs:
+    - path: metrics/scenario_table.md
+      hash: md5
+      md5: 54856c66fca8b2ebd1fa4dea2d25734a
+      size: 2133

modules/ml-pipeline/src/pipeline/dvc.yaml

@@ -71,6 +71,16 @@ stages:
     outs:
     - metrics/metrics.json
     always_changed: true
+  generate_scenerio_metrics:
+    cmd: python 5_generate_scenarios.py
+    deps:
+    - 5_generate_scenarios.py
+    params:
+    - configs/scenarios.yaml:
+      - default.scenarios
+    outs:
+    - metrics/scenario_table.md
+    always_changed: true
 metrics:
   - metrics/metrics.json
   - metrics/fit_metrics.json

modules/ml-pipeline/src/pipeline/metrics/.gitignore

@@ -1,2 +1,3 @@
 /fit_metrics.json
 /metrics.json
+/scenario_table.md