Merge pull request #171 from Hestia-Homes/sap_baseline-dev-model

Sap baseline dev model

.github/workflows/Deploy.yml

@@ -2,7 +2,17 @@ name: Sap Change Model Deploy
 
 on:
   push:
-    branches: [ sap-dev, sap-prod, heat-dev, heat-prod, carbon-dev, carbon-prod]
+    branches:
+      [
+        sap-dev,
+        sap-prod,
+        heat-dev,
+        heat-prod,
+        carbon-dev,
+        carbon-prod,
+        sap_baseline-dev,
+        sap_baseline-prod,
+      ]
 
 jobs:
   deploy:
@@ -85,10 +95,19 @@ jobs:
           aws-secret-access-key: ${{ secrets.PROD_AWS_SECRET_ACCESS_KEY }}
           aws-region: eu-west-2
 
+      - name: Check Disk Usage
+        run: |
+          df -h /
+          du -sh /* 2>/dev/null | sort -hr | head -10
+          uname -m  # Bonus: Confirms x64 vs. aarch64
+
       - name: DVC Pull
         run: |
           cd modules/ml-pipeline/src/pipeline
           dvc pull -r ${{ steps.set_runtime_environment.outputs.runtime_environment }}
+          cd data/model
+          ls
+          rm -r allmodels
 
       - name: Setup Docker
         uses: docker/setup-buildx-action@v1

.github/workflows/MLPipelinePostMerge.yml

@@ -13,6 +13,7 @@ on:
       - "sap-dev"
       - "heat-dev"
       - "carbon-dev"
+      - "sap_baseline-dev"
 
 permissions: write-all
 
@@ -180,7 +181,12 @@ jobs:
           dvc push -r dev
 
   Register-New-Model-Dev:
-    needs: [Register-Major-Model-Dev, Register-Minor-Model-Dev, Register-Patch-Model-Dev]
+    needs:
+      [
+        Register-Major-Model-Dev,
+        Register-Minor-Model-Dev,
+        Register-Patch-Model-Dev,
+      ]
     if: |
       always() &&
       (needs.Register-Major-Model-Dev.result == 'success' || needs.Register-Major-Model-Dev.result == 'skipped') &&

.github/workflows/MLPipelinePullRequest.yml

@@ -5,14 +5,13 @@ on:
   #   branches:
   #     - "model-**"
   pull_request:
-    branches: ["sap-dev", "heat-dev", "carbon-dev"]
+    branches: ["sap-dev", "heat-dev", "carbon-dev", "sap_baseline-dev"]
   label:
     types: ["created", "edited"]
 
 permissions: write-all
 
 jobs:
-
   Check-Label:
     runs-on: ubuntu-latest
     steps:
@@ -32,8 +31,91 @@ jobs:
   #       echo "Please choose one of these tags: 'major', 'major', 'patch'"
   #       exit(1)
 
-  Verify-Model:
+  Verify-Lambda:
+    runs-on: ubuntu-latest
 
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install packages to retrieve artifacts
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }}
+        run: |
+          pip install --upgrade pip
+          pip install -r modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt
+
+      - name: Retrieve artifacts (dvc.lock)
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }}
+        run: |
+          cd modules/ml-pipeline/src/pipeline
+          dvc pull -r experiments
+
+      - name: Set timestamp
+        id: set_timestamp
+        run: |
+          echo "timestamp=$(date +%Y%m%d)" >> $GITHUB_ENV
+          echo "Generated timestamp: ${timestamp}"
+
+      - name: Upload sample row dataset to S3
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }}
+        run: |
+          cd modules/ml-pipeline/src/pipeline/data/prepared_data/
+          aws s3 cp sample_test.parquet s3://retrofit-data-dev/sap_change_model/sample_data_for_cicd/${timestamp}/sample_test.parquet
+
+      - name: Build Lambda docker Image
+        run: |
+          docker build . --file ./deployment/Dockerfile.prediction.lambda --tag lambda_test
+
+      - name: Run lambda docker container
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }}
+        run: |
+          docker run -d -p 9000:8080 \
+            -e AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} \
+            -e AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} \
+            -e RUNTIME_ENVIRONMENT=dev \
+            -e PREDICTIONS_BUCKET=retrofit-sap-predictions-dev lambda_test
+
+      - name: Test Lambda endpoint
+        run: |
+          sleep 2
+          curl -X POST "http://localhost:9000/2015-03-31/functions/function/invocations" \
+            -H "Content-Type: application/json" \
+            -d "{\"body\": \"{\\\"file_location\\\": \\\"s3://retrofit-data-dev/sap_change_model/sample_data_for_cicd/${timestamp}/sample_test.parquet\\\", \\\"property_id\\\": 1, \\\"portfolio_id\\\": 4, \\\"created_at\\\": \\\"now\\\", \\\"warm\\\": true}\"}"
+
+      - name: Get Lambda logs
+        run: |
+          docker logs $(docker ps -al -q)
+
+      - name: Test Lambda endpoint again
+        run: |
+          sleep 2
+          curl -X POST "http://localhost:9000/2015-03-31/functions/function/invocations" \
+            -H "Content-Type: application/json" \
+            -d "{\"body\": \"{\\\"file_location\\\": \\\"s3://retrofit-data-dev/sap_change_model/sample_data_for_cicd/${timestamp}/sample_test.parquet\\\", \\\"property_id\\\": 1, \\\"portfolio_id\\\": 4, \\\"created_at\\\": \\\"now\\\", \\\"testing\\\": true}\"}"
+
+      - name: Get Lambda logs
+        run: |
+          docker logs $(docker ps -al -q)
+
+      - name: Stop Lambda container
+        run: |
+          docker stop lambda_test || echo "Container already stopped"
+
+      - name: Remove uploaded sample row dataset from S3
+        if: always()
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.ROBOT_AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.ROBOT_AWS_SECRET_ACCESS_KEY }}
+        run: |
+          aws s3 rm --recursive s3://retrofit-data-dev/sap_change_model/sample_data_for_cicd/${timestamp}/
+
+  Verify-Model:
     runs-on: ubuntu-latest
 
     steps:
@@ -64,7 +146,6 @@ jobs:
           docker run prediction_test
 
   Trigger-CML:
-
     runs-on: ubuntu-latest
 
     steps:

MODEL_REGISTRY.md

@@ -8,25 +8,73 @@
         "active": true
     },
     "sap": {
-        "version": "v0.14.0",
+        "version": "v0.17.5",
         "stage": {
-            "dev": "v0.14.0"
+            "dev": "v0.17.5"
         },
         "registered": true,
         "active": true
     },
     "heat": {
-        "version": "v0.5.0",
+        "version": "v0.8.0",
         "stage": {
-            "dev": "v0.5.0"
+            "dev": "v0.8.0"
         },
         "registered": true,
         "active": true
     },
     "carbon": {
-        "version": "v0.5.0",
+        "version": "v0.8.0",
         "stage": {
-            "dev": "v0.5.0"
+            "dev": "v0.8.0"
+        },
+        "registered": true,
+        "active": true
+    },
+    "hotwater": {
+        "version": "v1.0.0",
+        "stage": {
+            "dev": "v1.0.0"
+        },
+        "registered": true,
+        "active": true
+    },
+    "heating": {
+        "version": "v1.0.0",
+        "stage": {
+            "dev": "v1.0.0"
+        },
+        "registered": true,
+        "active": true
+    },
+    "lighting": {
+        "version": "v1.0.0",
+        "stage": {
+            "dev": "v1.0.0"
+        },
+        "registered": true,
+        "active": true
+    },
+    "hotwaterkwh": {
+        "version": "v1.4.0",
+        "stage": {
+            "dev": "v1.4.0"
+        },
+        "registered": true,
+        "active": true
+    },
+    "heatingkwh": {
+        "version": "v1.6.0",
+        "stage": {
+            "dev": "v1.6.0"
+        },
+        "registered": true,
+        "active": true
+    },
+    "sap_baseline": {
+        "version": "v1.0.2",
+        "stage": {
+            "dev": "v1.0.2"
         },
         "registered": true,
         "active": true

README.md

@@ -83,3 +83,13 @@ curl -XPOST "http://localhost:9000/2015-03-31/functions/function/invocations" -d
 ```
 
 This will send a POST request to the running Lambda function and pass in the required data as JSON.
+
+For the testing of warm or testing of the lambda, use:
+
+```json
+curl -XPOST "http://localhost:9000/2015-03-31/functions/function/invocations" -d '{"body": "{\"file_location\": \"s3://retrofit-data-dev/sap_change_model/one_sample_test_dataset.parquet\", \"property_id\": 1, \"portfolio_id\": 4, \"created_at\": \"now\", \"testing\": \"true\"}"}'
+```
+or
+```json
+curl -XPOST "http://localhost:9000/2015-03-31/functions/function/invocations" -d '{"body": "{\"file_location\": \"s3://retrofit-data-dev/sap_change_model/one_sample_test_dataset.parquet\", \"property_id\": 1, \"portfolio_id\": 4, \"created_at\": \"now\", \"warm\": \"true\"}"}'
+```

deployment/Dockerfile.prediction.lambda

@@ -1,19 +1,24 @@
-FROM public.ecr.aws/lambda/python:3.10
+FROM public.ecr.aws/lambda/python:3.12
 
 # Set the working directory
 WORKDIR ${LAMBDA_TASK_ROOT}
-ENV PYTHONPATH "${PYTHONPATH}:${LAMBDA_TASK_ROOT}"
+ENV PYTHONPATH="${PYTHONPATH}:${LAMBDA_TASK_ROOT}"
+ENV MPLCONFIGDIR="/tmp/matplotlib"
 
 # Environment variables
 ARG RUNTIME_ENVIRONMENT
 ENV RUNTIME_ENVIRONMENT=${RUNTIME_ENVIRONMENT}
 
 # Install necessary build tools - required to test locally
-RUN yum install -y gcc python3-devel gcc-c++
+RUN dnf install -y gcc python3-devel gcc-c++
 
 # Install python packages
 COPY modules/ml-pipeline/src/pipeline/requirements/predictions/requirements.txt ./requirements.txt
-RUN pip install --no-cache-dir -r ./requirements.txt
+
+RUN pip install uv
+
+RUN uv pip install -r requirements.txt --system
+# RUN pip install --no-cache-dir -r ./requirements.txt
 
 # Copy the project code
 COPY modules/ml-pipeline/src/pipeline ./pipeline

deployment/handlers/prediction_app.py

@@ -47,6 +47,30 @@ def upload_dataframe_to_s3(df, bucket, s3_file_name):
         return False
 
 
+def warming_up_invocation(
+    model,
+    model_filepath: str,
+):
+    """
+    Function to handle warm up invocations
+    """
+    import pandas as pd
+    import numpy as np
+
+    model.load_model(model_filepath)
+
+    warmup_df = pd.DataFrame(
+        np.zeros((1, len(model.model.original_features))),
+        columns=model.model.original_features,
+    )
+
+    # model_names = model.model.model_names()
+    # if "NeuralNetFastAI" in model_names:
+    #     model.model.predict(warmup_df, model="NeuralNetFastAI")
+    # else:
+    model.predict(data=warmup_df)
+
+
 def handler(event, context):
     """
     Take in event and trigger the prediction pipeline
@@ -66,9 +90,6 @@ def handler(event, context):
         created_at = body["created_at"]
 
         # TODO: Implement the loading of the model and prediction
-
-        storage_filepath = f"s3://{PREDICTIONS_BUCKET}/{portfolio_id}/{property_id}/{created_at}.parquet"
-
         logger.info(f"--- Initiate MLModel ---")
 
         build_model_params = settings.build_model
@@ -78,6 +99,32 @@ def handler(event, context):
 
         model = model_factory(build_model_params["model_type"])
 
+        model_filepath = build_model_params["model_save_filepath"]
+
+        if "warm" in body:
+            logger.info("Warm up invocation - synthetic prediction")
+
+            warming_up_invocation(model=model, model_filepath=model_filepath)
+
+            return {
+                "statusCode": 200,
+                "body": json.dumps(
+                    {
+                        "message": "Successfully warmed up invocation",
+                    }
+                ),
+            }
+
+        if "testing" in body:
+            logger.info(
+                "Testing invocation for CI/CD - save file to same location in S3"
+            )
+            storage_filepath = body["file_location"].replace(
+                ".parquet", "_output.parquet"
+            )
+        else:
+            storage_filepath = f"s3://{PREDICTIONS_BUCKET}/{portfolio_id}/{property_id}/{created_at}.parquet"
+
         logger.info(f"--- Initiate Input DataClient ---")
         input_dataclient = dataclient_factory(
             dataclient_type="aws-s3",
@@ -95,7 +142,7 @@ def handler(event, context):
             output_dataclient=output_dataclient,
             model=model,
             target=feature_process_params["feature_processor_config"]["target"],
-            model_filepath=build_model_params["model_save_filepath"],
+            model_filepath=model_filepath,
             test_data_filepath=body["file_location"],
             predictions_output_filepath=storage_filepath,
             predictions_column_name=generate_predictions_params[

deployment/serverless.yml

@@ -51,3 +51,4 @@ functions:
           path: /predict
           method: POST
     timeout: 120 # Set max run time to 2 minutes - we shouldn't need this much time so this can be reviewed
+    memorySize: 3008

modules/ml-pipeline/Makefile

@@ -1,7 +1,8 @@
 export PYENV_ROOT=$(HOME)/.pyenv
 export PATH := $(PYENV_ROOT)/bin:$(PATH)
-PYTHON_VERSION ?= 3.10.12
+PYTHON_VERSION ?= 3.12.12
 CONDA_ENV=dev_env_pipeline
+CONDA_ACTIVATE=source $$(conda info --base)/etc/profile.d/conda.sh ; conda deactivate ; conda activate
 
 .PHONY: init
 init: dev-conda
@@ -12,11 +13,15 @@ dev-conda:
 	# conda remove --name ${CONDA_ENV} --all -y || echo "No environment created previously"
 	conda create --name ${CONDA_ENV} python=$(PYTHON_VERSION) -y
 	conda init bash
-	conda run -v -n ${CONDA_ENV} pip install --upgrade pip
+	${CONDA_ACTIVATE} ${CONDA_ENV} && \
-	conda run -v -n ${CONDA_ENV} pip install -r src/pipeline/requirements/training/requirements-dev.txt
+		which pip && \
-	conda run -v -n ${CONDA_ENV} pip install -r src/pipeline/requirements/version_control/requirements.txt
+		pip install --upgrade pip && \
-	conda run -v -n ${CONDA_ENV} pre-commit install
+		pip install uv && \
-	conda run -v -n ${CONDA_ENV} pip install ipykernel
+		uv pip install -r src/pipeline/requirements/training/requirements-dev.txt && \
+		uv pip install -r src/pipeline/requirements/version_control/requirements.txt && \
+		pre-commit install && \
+		uv pip install ipykernel && \
+		conda install llvm-openmp -y
 	echo "TO ACTIVATE ENVIRONMENT, USE THE FOLLOWING COMMAND"
 	echo "conda activate ${CONDA_ENV}"
 

modules/ml-pipeline/src/Prediction.Dockerfile

@@ -1,12 +1,17 @@
 # Dockerfile that can be used to test loading a model to generate a prediction (part of CI/CD flow)
-FROM python:3.10.12-slim
+FROM python:3.12.12-slim
 
 RUN apt-get update && apt-get install -y libgomp1 gcc python3-dev
 
 COPY pipeline/requirements/predictions/requirements.txt requirements.txt
 
 RUN pip install --upgrade pip
-RUN pip install -r requirements.txt
+
+RUN pip install uv
+
+RUN uv pip install -r requirements.txt --system
+
+# RUN pip install -r requirements.txt
 
 # Assuming in the CI/CD step, there will be a dvc pull step to get data and model, so will just need to run a single script
 COPY pipeline/ /home/pipeline/

modules/ml-pipeline/src/pipeline/1_prepare_data.py

@@ -29,6 +29,7 @@ data_filepath = prepare_data_params["data_filepath"]
 train_proportion = prepare_data_params["train_proportion"]
 output_train_filepath = prepare_data_params["output_train_filepath"]
 output_test_filepath = prepare_data_params["output_test_filepath"]
+sample_test_filepath = prepare_data_params["sample_test_filepath"]
 feature_processor_config = feature_process_params["feature_processor_config"]
 
 logger.info(f"--- Initiate DataClient ---")
@@ -99,6 +100,10 @@ def prepare_data(
 
     logger.info("--- Outputting data ---")
 
+    output_dataclient.save_data(
+        obj=data.sample(1), location=sample_test_filepath, save_config=None
+    )
+
     output_dataclient.save_data(
         obj=train, location=output_train_filepath, save_config=None
     )

modules/ml-pipeline/src/pipeline/5_generate_scenarios.py

@@ -99,6 +99,12 @@ def generate_scenario_predictions(
             ]
         )
 
+    # TEMPORARY FIX: ADD is_post_sap10_starting and is_post_sap10_ending if not present
+    if "is_post_sap10_starting" not in scenario_data.columns:
+        scenario_data["is_post_sap10_starting"] = False
+    if "is_post_sap10_ending" not in scenario_data.columns:
+        scenario_data["is_post_sap10_ending"] = False
+
     logger.info("--- Loading Model ---")
 
     model.load_model(model_filepath)

modules/ml-pipeline/src/pipeline/configs/build_model.yaml

@@ -14,9 +14,23 @@ default:
       output_filepath: ./data/model/allmodels/
       problem_type: regression
       eval_metric: mean_squared_error #mean_absolute_error
-      time_limit: 1800
+      time_limit: 3600
       presets: medium_quality
-      excluded_model_types: ['RF', 'CAT', 'NN_TORCH', 'KNN', 'XT']
+      excluded_model_types: ['RF', 'CAT', 'NN_TORCH', 'KNN', 'XT', 'FASTAI']
-      infer_limit: 0.05
+      infer_limit: 1
       infer_limit_batch_size: 10000
+      fit_strategy: "parallel"
       ag_args_ensemble: {'num_folds_parallel': 2}
+      num_gpus: 0
+      hyperparameters:
+        {
+        'NN_TORCH': [{}],
+        'GBM': [{'extra_trees': True, 'ag_args': {'name_suffix': 'XT'}}, {}, {'learning_rate': 0.03, 'num_leaves': 128, 'feature_fraction': 0.9, 'min_data_in_leaf': 3, 'ag_args': {'name_suffix': 'Large', 'priority': 0,}}],
+        # 'GBM': [{}],
+        'CAT': [{}],
+        'XGB': [{}],
+        'FASTAI': [{}],
+        'RF': [{'criterion': 'gini', 'ag_args': {'name_suffix': 'Gini', 'problem_types': ['binary', 'multiclass']}}, {'criterion': 'entropy', 'ag_args': {'name_suffix': 'Entr', 'problem_types': ['binary', 'multiclass']}}, {'criterion': 'squared_error', 'ag_args': {'name_suffix': 'MSE', 'problem_types': ['regression', 'quantile']}}],
+        'XT': [{'criterion': 'gini', 'ag_args': {'name_suffix': 'Gini', 'problem_types': ['binary', 'multiclass']}}, {'criterion': 'entropy', 'ag_args': {'name_suffix': 'Entr', 'problem_types': ['binary', 'multiclass']}}, {'criterion': 'squared_error', 'ag_args': {'name_suffix': 'MSE', 'problem_types': ['regression', 'quantile']}}],
+        'KNN': [{'weights': 'uniform', 'ag_args': {'name_suffix': 'Unif'}}, {'weights': 'distance', 'ag_args': {'name_suffix': 'Dist'}}],
+        }

modules/ml-pipeline/src/pipeline/configs/post_prediction_logic.py

@@ -1,6 +1,7 @@
 """
 After predictions, we may want to apply some post processing to the predictions
 """
+
 import pandas as pd
 
 
@@ -30,6 +31,6 @@ def clip_predictions_to_minimum_value(
 
 
 post_prediction_logic = {
-    "clip_predictions_to_minimum_value": clip_predictions_to_minimum_value,
+    # "clip_predictions_to_minimum_value": clip_predictions_to_minimum_value,
     # "round_predictions": round_predictions
 }

modules/ml-pipeline/src/pipeline/configs/scenarios.yaml

@@ -3,11 +3,10 @@ default:
     input_dataclient_type: aws-s3
     output_dataclient_type: local
     scenario_data_filepaths:
-      # - s3://retrofit-data-dev/scenario_data/22-03-2024-19-20-09/recommendations_scoring_data.parquet
+      # - s3://retrofit-data-dev/scenario_data/28-05-2024-19-22-41/recommendations_scoring_data.parquet
-      # - s3://retrofit-data-dev/scenario_data/24-03-2024-20-23-25/recommendations_scoring_data.parquet
+      # - s3://retrofit-data-dev/scenario_data/07-10-2024-16-26-06/recommendations_scoring_data.parquet
-      # - s3://retrofit-data-dev/scenario_data/27-03-2024-11-38-15/recommendations_scoring_data.parquet
+      # - s3://retrofit-data-dev/scenario_data/08-10-2024-15-07-33/recommendations_scoring_data.parquet
-      # - s3://retrofit-data-dev/scenario_data/26-05-2024-08-47-45/recommendations_scoring_data.parquet
+      # - s3://retrofit-data-dev/scenario_data/08-10-2024-22-18-44/recommendations_scoring_data.parquet
-      # - s3://retrofit-data-dev/scenario_data/26-05-2024-10-44-53/recommendations_scoring_data.parquet
+      # - s3://retrofit-data-dev/scenario_data/09-10-2024-18-21-08/recommendations_scoring_data.parquet
-      - s3://retrofit-data-dev/scenario_data/28-05-2024-19-22-41/recommendations_scoring_data.parquet
     comparison_output_filepath: ./metrics/scenario_table.md
     metrics_output_filepath: ./metrics/scenario_metrics.md

modules/ml-pipeline/src/pipeline/configs/settings.yaml

@@ -12,32 +12,165 @@ default:
       AWS_ACCESS_KEY_ID: minio
       AWS_SECRET_ACCESS_KEY: minio123
       ENDPOINT_URL: http://localhost:9000
-    local:
+    local: null
-      null
 
   prepare_data:
     input_dataclient_type: aws-s3
     output_dataclient_type: local
-    # data_filepath: s3://retrofit-data-dev/sap_change_model/2024-03-22-18-56-53/dataset_rooms.parquet
+    # data_filepath: s3://retrofit-data-dev/sap_change_model/2024-05-28-19-08-25/dataset_rooms.parquet
-    # data_filepath: s3://retrofit-data-dev/sap_change_model/2024-05-25-08-36-36/dataset_rooms.parquet
+    # data_filepath: s3://retrofit-data-dev/sap_change_model/2024-10-03-22-57-23/dataset_rooms.parquet
-    # data_filepath: s3://retrofit-data-dev/sap_change_model/2024-05-26-10-31-39/dataset_rooms.parquet
+    # data_filepath: s3://retrofit-data-dev/sap_change_model/2024-10-08-21-58-03/dataset_rooms.parquet
-    data_filepath: s3://retrofit-data-dev/sap_change_model/2024-05-28-19-08-25/dataset_rooms.parquet
+    # data_filepath: s3://retrofit-data-dev/sap_change_model/2025-09-05-14-05-32/dataset_rooms.parquet
+    data_filepath: s3://retrofit-data-dev/sap_change_model/2025-11-02-09-32-42/dataset_rooms.parquet
     train_proportion: 0.9
     output_train_filepath: ./data/prepared_data/train.parquet
     output_test_filepath: ./data/prepared_data/test.parquet
+    sample_test_filepath: ./data/prepared_data/sample_test.parquet
 
   feature_processor:
     feature_processor_type: dataframe
     feature_processor_config:
       subsample_amount: null
       subsample_seed: 0
-      target: sap_ending
+      target: sap_starting
       identifier_columns: ["uprn"]
       # drop_columns: ["heat_demand_change", "carbon_change", "rdsap_change", "heat_demand_ending", "carbon_ending", "days_to_starting", "days_to_ending"]
-      drop_columns: [
+      drop_columns:
-        "heat_demand_change", "carbon_change", "rdsap_change", "heat_demand_ending", "carbon_ending", "days_to_starting", "days_to_ending",
+        [
-        'number_habitable_rooms_starting', 'number_habitable_rooms_ending', 'number_heated_rooms_starting', 'number_heated_rooms_ending',
+          "sap_ending",
-        'number_habitable_rooms', 'number_heated_rooms']
+          "potential_energy_efficiency",
+          "environment_impact_potential",
+          "energy_consumption_potential",
+          "co2_emissions_potential",
+          "heat_demand_change",
+          "carbon_change",
+          "rdsap_change",
+          "heat_demand_starting",
+          "heat_demand_ending",
+          "carbon_starting",
+          "carbon_ending",
+          "days_to_starting",
+          "days_to_ending",
+          "number_habitable_rooms_starting",
+          "number_habitable_rooms_ending",
+          "number_heated_rooms_starting",
+          "number_heated_rooms_ending",
+          "number_habitable_rooms",
+          "number_heated_rooms",
+          "lighting_cost_starting",
+          "lighting_cost_ending",
+          "heating_cost_starting",
+          "heating_cost_ending",
+          "hot_water_cost_starting",
+          "hot_water_cost_ending",
+          "floor_thermal_transmittance",
+          "floor_thermal_transmittance_ending",
+          "lodgement_date_starting",
+          "lodgement_date_ending",
+          "walls_thermal_transmittance_ending",
+          "walls_thermal_transmittance_unit_ending",
+          "is_filled_cavity_ending",
+          "is_as_built_ending",
+          "walls_is_assumed_ending",
+          "is_park_home_ending",
+          "walls_insulation_thickness_ending",
+          "external_insulation_ending",
+          "internal_insulation_ending",
+          "floor_insulation_thickness_ending",
+          "roof_thermal_transmittance_ending",
+          "is_at_rafters_ending",
+          "roof_insulation_thickness_ending",
+          "heater_type_ending",
+          "system_type_ending",
+          "thermostat_characteristics_ending",
+          "heating_scope_ending",
+          "energy_recovery_ending",
+          "hotwater_tariff_type_ending",
+          "extra_features_ending",
+          "chp_systems_ending",
+          "distribution_system_ending",
+          "no_system_present_ending",
+          "appliance_ending",
+          "has_radiators_ending",
+          "has_fan_coil_units_ending",
+          "has_pipes_in_screed_above_insulation_ending",
+          "has_pipes_in_insulated_timber_floor_ending",
+          "has_pipes_in_concrete_slab_ending",
+          "has_boiler_ending",
+          "has_air_source_heat_pump_ending",
+          "has_room_heaters_ending",
+          "has_electric_storage_heaters_ending",
+          "has_warm_air_ending",
+          "has_electric_underfloor_heating_ending",
+          "has_electric_ceiling_heating_ending",
+          "has_community_scheme_ending",
+          "has_ground_source_heat_pump_ending",
+          "has_no_system_present_ending",
+          "has_portable_electric_heaters_ending",
+          "has_water_source_heat_pump_ending",
+          "has_electric_heat_pump_ending",
+          "has_micro-cogeneration_ending",
+          "has_solar_assisted_heat_pump_ending",
+          "has_exhaust_source_heat_pump_ending",
+          "has_community_heat_pump_ending",
+          "has_hot-water-only_ending",
+          "has_electric_ending",
+          "has_mains_gas_ending",
+          "has_wood_logs_ending",
+          "has_coal_ending",
+          "has_oil_ending",
+          "has_wood_pellets_ending",
+          "has_anthracite_ending",
+          "has_dual_fuel_mineral_and_wood_ending",
+          "has_smokeless_fuel_ending",
+          "has_lpg_ending",
+          "has_b30k_ending",
+          "has_mineral_and_wood_ending",
+          "has_dual_fuel_appliance_ending",
+          "has_electricaire_ending",
+          "has_assumed_for_most_rooms_ending",
+          "has_underfloor_heating_ending",
+          "thermostatic_control_ending",
+          "charging_system_ending",
+          "switch_system_ending",
+          "no_control_ending",
+          "dhw_control_ending",
+          "community_heating_ending",
+          "multiple_room_thermostats_ending",
+          "auxiliary_systems_ending",
+          "trvs_ending",
+          "rate_control_ending",
+          "glazing_type_ending",
+          "fuel_type_ending",
+          "main-fuel_tariff_type_ending",
+          "is_community_ending",
+          "no_individual_heating_or_community_network_ending",
+          "complex_fuel_type_ending",
+          "mechanical_ventilation_ending",
+          "secondheat_description_ending",
+          "glazed_type_ending",
+          "multi_glaze_proportion_ending",
+          "low_energy_lighting_ending",
+          "number_open_fireplaces_ending",
+          "solar_water_heating_flag_ending",
+          "photo_supply_ending",
+          "transaction_type_ending",
+          "energy_tariff_ending",
+          "extension_count_ending",
+          "total_floor_area_ending",
+          "floor_height_ending",
+          "hot_water_energy_eff_ending",
+          "floor_energy_eff_ending",
+          "windows_energy_eff_ending",
+          "walls_energy_eff_ending",
+          "sheating_energy_eff_ending",
+          "roof_energy_eff_ending",
+          "mainheat_energy_eff_ending",
+          "mainheatc_energy_eff_ending",
+          "lighting_energy_eff_ending",
+          "is_post_sap10_ending",
+          "estimated_perimeter_ending",
+        ]
       retain_features: null
       # retain_features: ['uprn', 'sap_starting', 'hot_water_energy_eff_ending',
       #  'mainheat_energy_eff_ending', 'constituency', 'roof_energy_eff_ending',

modules/ml-pipeline/src/pipeline/core/MLModels.py

@@ -11,9 +11,6 @@ import joblib
 import pandas as pd
 from pathlib import Path
 from typing import Union, List
-from sklearn import linear_model
-from sklearn.svm import SVR
-from autogluon.tabular import TabularDataset, TabularPredictor
 from core.interface.InterfaceModels import MLModel
 from core.Logger import logger
 
@@ -69,6 +66,8 @@ class SKLearnLinearRegression:
         """
         Method to train a model
         """
+        from sklearn import linear_model
+
         self.model = linear_model.LinearRegression()
 
         x_train = data.iloc[:, data.columns != target]
@@ -117,6 +116,7 @@ class SKLearnSVMRegression:
         """
         Method to train a model
         """
+        from sklearn.svm import SVR
 
         validate_dict_keys(
             list(model_hyperparameters.keys()),
@@ -152,12 +152,17 @@ class AutogluonAutoML:
         "infer_limit",
         "infer_limit_batch_size",
         "ag_args_ensemble",
+        "fit_strategy",
+        "num_gpus",
+        "hyperparameters",
     ]
 
     def load_model(self, path: Union[Path, str]) -> None:
         """
         Method to load a model
         """
+        from autogluon.tabular import TabularPredictor
+
         filepath = str(path)
         self.model = TabularPredictor.load(path=filepath)
 
@@ -183,6 +188,10 @@ class AutogluonAutoML:
         """
         Method to train a model
         """
+        from autogluon.tabular import TabularDataset, TabularPredictor
+
+        # Force Parallel Model fitting
+        os.environ["AG_FORCE_PARALLEL"] = "True"
 
         validate_dict_keys(
             keys_1=list(model_hyperparameters.keys()),
@@ -209,6 +218,9 @@ class AutogluonAutoML:
             infer_limit=model_hyperparameters["infer_limit"],
             infer_limit_batch_size=model_hyperparameters["infer_limit_batch_size"],
             ag_args_ensemble=model_hyperparameters["ag_args_ensemble"],
+            fit_strategy=model_hyperparameters["fit_strategy"],
+            num_gpus=model_hyperparameters["num_gpus"],
+            hyperparameters=model_hyperparameters["hyperparameters"].to_dict(),
         )
 
     def predict(

modules/ml-pipeline/src/pipeline/dvc.lock

@@ -16,15 +16,22 @@ stages:
     deps:
     - path: 1_prepare_data.py
       hash: md5
-      md5: 11a3b8bfdfe199ab7ecc39ccc5652649
+      md5: a5ce162e1c402c0f811a80ef78cf4dd5
-      size: 4298
+      size: 4481
     params:
       configs/settings.yaml:
         default.feature_processor.feature_processor_config.drop_columns:
+        - sap_ending
+        - potential_energy_efficiency
+        - environment_impact_potential
+        - energy_consumption_potential
+        - co2_emissions_potential
         - heat_demand_change
         - carbon_change
         - rdsap_change
+        - heat_demand_starting
         - heat_demand_ending
+        - carbon_starting
         - carbon_ending
         - days_to_starting
         - days_to_ending
@@ -34,24 +41,139 @@ stages:
         - number_heated_rooms_ending
         - number_habitable_rooms
         - number_heated_rooms
+        - lighting_cost_starting
+        - lighting_cost_ending
+        - heating_cost_starting
+        - heating_cost_ending
+        - hot_water_cost_starting
+        - hot_water_cost_ending
+        - floor_thermal_transmittance
+        - floor_thermal_transmittance_ending
+        - lodgement_date_starting
+        - lodgement_date_ending
+        - walls_thermal_transmittance_ending
+        - walls_thermal_transmittance_unit_ending
+        - is_filled_cavity_ending
+        - is_as_built_ending
+        - walls_is_assumed_ending
+        - is_park_home_ending
+        - walls_insulation_thickness_ending
+        - external_insulation_ending
+        - internal_insulation_ending
+        - floor_insulation_thickness_ending
+        - roof_thermal_transmittance_ending
+        - is_at_rafters_ending
+        - roof_insulation_thickness_ending
+        - heater_type_ending
+        - system_type_ending
+        - thermostat_characteristics_ending
+        - heating_scope_ending
+        - energy_recovery_ending
+        - hotwater_tariff_type_ending
+        - extra_features_ending
+        - chp_systems_ending
+        - distribution_system_ending
+        - no_system_present_ending
+        - appliance_ending
+        - has_radiators_ending
+        - has_fan_coil_units_ending
+        - has_pipes_in_screed_above_insulation_ending
+        - has_pipes_in_insulated_timber_floor_ending
+        - has_pipes_in_concrete_slab_ending
+        - has_boiler_ending
+        - has_air_source_heat_pump_ending
+        - has_room_heaters_ending
+        - has_electric_storage_heaters_ending
+        - has_warm_air_ending
+        - has_electric_underfloor_heating_ending
+        - has_electric_ceiling_heating_ending
+        - has_community_scheme_ending
+        - has_ground_source_heat_pump_ending
+        - has_no_system_present_ending
+        - has_portable_electric_heaters_ending
+        - has_water_source_heat_pump_ending
+        - has_electric_heat_pump_ending
+        - has_micro-cogeneration_ending
+        - has_solar_assisted_heat_pump_ending
+        - has_exhaust_source_heat_pump_ending
+        - has_community_heat_pump_ending
+        - has_hot-water-only_ending
+        - has_electric_ending
+        - has_mains_gas_ending
+        - has_wood_logs_ending
+        - has_coal_ending
+        - has_oil_ending
+        - has_wood_pellets_ending
+        - has_anthracite_ending
+        - has_dual_fuel_mineral_and_wood_ending
+        - has_smokeless_fuel_ending
+        - has_lpg_ending
+        - has_b30k_ending
+        - has_mineral_and_wood_ending
+        - has_dual_fuel_appliance_ending
+        - has_electricaire_ending
+        - has_assumed_for_most_rooms_ending
+        - has_underfloor_heating_ending
+        - thermostatic_control_ending
+        - charging_system_ending
+        - switch_system_ending
+        - no_control_ending
+        - dhw_control_ending
+        - community_heating_ending
+        - multiple_room_thermostats_ending
+        - auxiliary_systems_ending
+        - trvs_ending
+        - rate_control_ending
+        - glazing_type_ending
+        - fuel_type_ending
+        - main-fuel_tariff_type_ending
+        - is_community_ending
+        - no_individual_heating_or_community_network_ending
+        - complex_fuel_type_ending
+        - mechanical_ventilation_ending
+        - secondheat_description_ending
+        - glazed_type_ending
+        - multi_glaze_proportion_ending
+        - low_energy_lighting_ending
+        - number_open_fireplaces_ending
+        - solar_water_heating_flag_ending
+        - photo_supply_ending
+        - transaction_type_ending
+        - energy_tariff_ending
+        - extension_count_ending
+        - total_floor_area_ending
+        - floor_height_ending
+        - hot_water_energy_eff_ending
+        - floor_energy_eff_ending
+        - windows_energy_eff_ending
+        - walls_energy_eff_ending
+        - sheating_energy_eff_ending
+        - roof_energy_eff_ending
+        - mainheat_energy_eff_ending
+        - mainheatc_energy_eff_ending
+        - lighting_energy_eff_ending
+        - is_post_sap10_ending
+        - estimated_perimeter_ending
         default.feature_processor.feature_processor_config.retain_features:
         default.feature_processor.feature_processor_config.subsample_amount:
         default.feature_processor.feature_processor_config.subsample_seed: 0
-        default.feature_processor.feature_processor_config.target: sap_ending
+        default.feature_processor.feature_processor_config.target: sap_starting
         default.feature_processor.feature_processor_type: dataframe
         default.prepare_data.data_filepath: 
-          s3://retrofit-data-dev/sap_change_model/2024-05-28-19-08-25/dataset_rooms.parquet
+          s3://retrofit-data-dev/sap_change_model/2025-11-02-09-32-42/dataset_rooms.parquet
         default.prepare_data.input_dataclient_type: aws-s3
         default.prepare_data.output_dataclient_type: local
-        default.prepare_data.output_test_filepath: ./data/prepared_data/test.parquet
+        default.prepare_data.output_test_filepath: 
-        default.prepare_data.output_train_filepath: ./data/prepared_data/train.parquet
+          ./data/prepared_data/test.parquet
+        default.prepare_data.output_train_filepath: 
+          ./data/prepared_data/train.parquet
         default.prepare_data.train_proportion: 0.9
     outs:
     - path: data/prepared_data/
       hash: md5
-      md5: 80c9e138146a1d96b9d16091c207e2e8.dir
+      md5: 2e33489c17eff1a60079e26d370a9058.dir
-      size: 45056059
+      size: 26013963
-      nfiles: 2
+      nfiles: 3
   build_model:
     cmd: python 2_build_model.py
     deps:
@@ -61,9 +183,9 @@ stages:
       size: 4820
     - path: data/prepared_data
       hash: md5
-      md5: 80c9e138146a1d96b9d16091c207e2e8.dir
+      md5: 2e33489c17eff1a60079e26d370a9058.dir
-      size: 45056059
+      size: 26013963
-      nfiles: 2
+      nfiles: 3
     params:
       configs/build_model.yaml:
         default:
@@ -79,7 +201,7 @@ stages:
               output_filepath: ./data/model/allmodels/
               problem_type: regression
               eval_metric: mean_squared_error
-              time_limit: 1800
+              time_limit: 3600
               presets: medium_quality
               excluded_model_types:
               - RF
@@ -87,25 +209,94 @@ stages:
               - NN_TORCH
               - KNN
               - XT
-              infer_limit: 0.05
+              - FASTAI
+              infer_limit: 1
               infer_limit_batch_size: 10000
+              fit_strategy: parallel
               ag_args_ensemble:
                 num_folds_parallel: 2
+              num_gpus: 0
+              hyperparameters:
+                NN_TORCH:
+                - {}
+                GBM:
+                - extra_trees: true
+                  ag_args:
+                    name_suffix: XT
+                - {}
+                - learning_rate: 0.03
+                  num_leaves: 128
+                  feature_fraction: 0.9
+                  min_data_in_leaf: 3
+                  ag_args:
+                    name_suffix: Large
+                    priority: 0
+                CAT:
+                - {}
+                XGB:
+                - {}
+                FASTAI:
+                - {}
+                RF:
+                - criterion: gini
+                  ag_args:
+                    name_suffix: Gini
+                    problem_types:
+                    - binary
+                    - multiclass
+                - criterion: entropy
+                  ag_args:
+                    name_suffix: Entr
+                    problem_types:
+                    - binary
+                    - multiclass
+                - criterion: squared_error
+                  ag_args:
+                    name_suffix: MSE
+                    problem_types:
+                    - regression
+                    - quantile
+                XT:
+                - criterion: gini
+                  ag_args:
+                    name_suffix: Gini
+                    problem_types:
+                    - binary
+                    - multiclass
+                - criterion: entropy
+                  ag_args:
+                    name_suffix: Entr
+                    problem_types:
+                    - binary
+                    - multiclass
+                - criterion: squared_error
+                  ag_args:
+                    name_suffix: MSE
+                    problem_types:
+                    - regression
+                    - quantile
+                KNN:
+                - weights: uniform
+                  ag_args:
+                    name_suffix: Unif
+                - weights: distance
+                  ag_args:
+                    name_suffix: Dist
     outs:
     - path: data/fit_predictions/
       hash: md5
-      md5: d9c9afc05e8780db47c0548b19bf7d19.dir
+      md5: 701063eff6ee4444971742f05c4de861.dir
-      size: 3349989
+      size: 3453673
       nfiles: 1
     - path: data/model/
       hash: md5
-      md5: 13c3100e1486c27a83a8a47491077842.dir
+      md5: 9c3e945f96fc2c95f2962a98af5c04fe.dir
-      size: 773523079
+      size: 594127389
-      nfiles: 36
+      nfiles: 30
     - path: metrics/fit_metrics.json
       hash: md5
-      md5: 2ff70a2a45813e1bcdf2ea3aa8e07d4a
+      md5: f1df654cc38d2c73261bfaed0a1ef277
-      size: 224
+      size: 223
   generate_predictions:
     cmd: python 3_generate_predictions.py
     deps:
@@ -115,26 +306,28 @@ stages:
       size: 2464
     - path: data/model
       hash: md5
-      md5: 13c3100e1486c27a83a8a47491077842.dir
+      md5: 9c3e945f96fc2c95f2962a98af5c04fe.dir
-      size: 773523079
+      size: 594127389
-      nfiles: 36
+      nfiles: 30
     - path: data/prepared_data
       hash: md5
-      md5: 80c9e138146a1d96b9d16091c207e2e8.dir
+      md5: 2e33489c17eff1a60079e26d370a9058.dir
-      size: 45056059
+      size: 26013963
-      nfiles: 2
+      nfiles: 3
     params:
       configs/settings.yaml:
         default.generate_predictions.input_dataclient_type: local
         default.generate_predictions.output_dataclient_type: local
         default.generate_predictions.predictions_column_name: predictions
-        default.generate_predictions.predictions_output_filepath: ./data/predictions/predictions.parquet
+        default.generate_predictions.predictions_output_filepath: 
-        default.generate_predictions.test_data_filepath: ./data/prepared_data/test.parquet
+          ./data/predictions/predictions.parquet
+        default.generate_predictions.test_data_filepath: 
+          ./data/prepared_data/test.parquet
     outs:
     - path: data/predictions/
       hash: md5
-      md5: 5d07bcebf3160a72bb18dfd79106e85c.dir
+      md5: 3dfb21b2918e954e657f0c7cbeca308f.dir
-      size: 463197
+      size: 479298
       nfiles: 1
   generate_metrics:
     cmd: python 4_generate_metrics.py
@@ -145,14 +338,14 @@ stages:
       size: 3484
     - path: data/predictions
       hash: md5
-      md5: 5d07bcebf3160a72bb18dfd79106e85c.dir
+      md5: 3dfb21b2918e954e657f0c7cbeca308f.dir
-      size: 463197
+      size: 479298
       nfiles: 1
     - path: data/prepared_data
       hash: md5
-      md5: 80c9e138146a1d96b9d16091c207e2e8.dir
+      md5: 2e33489c17eff1a60079e26d370a9058.dir
-      size: 45056059
+      size: 26013963
-      nfiles: 2
+      nfiles: 3
     params:
       configs/settings.yaml:
         default.generate_metrics.dataclient_type: local
@@ -161,30 +354,29 @@ stages:
     outs:
     - path: metrics/metrics.json
       hash: md5
-      md5: 3e08df02fd5c5d094bcf936e1338d596
+      md5: 365cf4dc2ac89b4df55dc7ae6862d8b1
-      size: 223
+      size: 222
   generate_scenerio_metrics:
     cmd: python 5_generate_scenarios.py
     deps:
     - path: 5_generate_scenarios.py
       hash: md5
-      md5: 40506749fefd926d47c60ff5b16db307
+      md5: 872b0c762ce1c8933fcbc5f54d5d4b5d
-      size: 5337
+      size: 5658
     params:
       configs/scenarios.yaml:
         default.scenarios:
           input_dataclient_type: aws-s3
           output_dataclient_type: local
           scenario_data_filepaths:
-          - s3://retrofit-data-dev/scenario_data/28-05-2024-19-22-41/recommendations_scoring_data.parquet
           comparison_output_filepath: ./metrics/scenario_table.md
           metrics_output_filepath: ./metrics/scenario_metrics.md
     outs:
     - path: metrics/scenario_metrics.md
       hash: md5
-      md5: fa4d6d7bbd7818613800da5f8f37ea96
+      md5: d41d8cd98f00b204e9800998ecf8427e
-      size: 363
+      size: 0
     - path: metrics/scenario_table.md
       hash: md5
-      md5: d6baf100a1623cc2467c2f8221d314c9
+      md5: d41d8cd98f00b204e9800998ecf8427e
-      size: 2133
+      size: 0

modules/ml-pipeline/src/pipeline/requirements/predictions/requirements-dev.txt

@@ -1,7 +1,7 @@
-joblib==1.3.2
+joblib==1.5.2
-boto3==1.28.17
+boto3==1.40.61
-pandas==2.1.4
+pandas==2.3.3
-autogluon.tabular[all]==1.0.0
+autogluon.tabular[all]==1.4.0
-dynaconf==3.2.1
+dynaconf==3.2.12
-pyarrow==13.0.0
+pyarrow==20.0.0
-pre-commit==3.3.3
+pre-commit==4.3.0

modules/ml-pipeline/src/pipeline/requirements/predictions/requirements.txt

@@ -1,7 +1,7 @@
-joblib==1.3.2
+joblib==1.5.2
-boto3==1.28.17
+boto3==1.40.61
-pandas==2.1.4
+pandas==2.3.3
-autogluon.tabular[all]==1.0.0
+autogluon.tabular[all]==1.4.0
-dynaconf==3.2.1
+dynaconf==3.2.12
-pyarrow==13.0.0
+pyarrow==20.0.0
-PyYAML==6.0.1
+PyYAML==6.0.3

modules/ml-pipeline/src/pipeline/requirements/training/requirements-dev.txt

@@ -1,10 +1,10 @@
-joblib==1.3.2
+joblib==1.5.2
-boto3==1.28.17
+boto3==1.40.61
-pandas==2.1.4
+pandas==2.3.3
-autogluon.tabular[all]==1.0.0
+autogluon.tabular[all]==1.4.0
-ray==2.6.3
+ray==2.44.1
-dynaconf==3.2.1
+dynaconf==3.2.12
-alibi==0.9.5
+# alibi
-shap==0.42.1
+shap==0.49.1
-pyarrow==13.0.0
+pyarrow==20.0.0
-pre-commit==3.3.3
+pre-commit==4.3.0

modules/ml-pipeline/src/pipeline/requirements/training/requirements.txt

@@ -1,4 +1,4 @@
-boto3==1.28.41
+boto3==1.40.61
-pandas==2.1.4
+pandas==2.3.3
-autogluon.tabular[all]==1.0.0
+autogluon.tabular[all]==1.4.0
-dynaconf==3.2.1
+dynaconf==3.2.12

modules/ml-pipeline/src/pipeline/requirements/version_control/requirements.txt

@@ -1,4 +1,5 @@
-dvc==3.51.0
+dvc==3.66.0
-dvc-s3==3.2.0
+dvc-s3==3.2.2
-gto==1.7.1
+boto3==1.40.61
+gto==1.9.0
 pyOpenSSL==23.3.0