From 6956a80707212acc3fc0dc98a373d1f08b69adb7 Mon Sep 17 00:00:00 2001 From: Michael Duong Date: Tue, 29 Aug 2023 17:28:48 +0100 Subject: [PATCH] add pre-commit hook, no customisation on black --- model_data/simulation_system/.pre-commit-config.yaml | 11 +++++++++++ model_data/simulation_system/MLModel/BaseMLModel.py | 3 ++- model_data/simulation_system/MLModel/Models.py | 6 ++++++ model_data/simulation_system/predictions.py | 3 +-- model_data/simulation_system/training.py | 2 -- 5 files changed, 20 insertions(+), 5 deletions(-) create mode 100644 model_data/simulation_system/.pre-commit-config.yaml diff --git a/model_data/simulation_system/.pre-commit-config.yaml b/model_data/simulation_system/.pre-commit-config.yaml new file mode 100644 index 00000000..cbc34112 --- /dev/null +++ b/model_data/simulation_system/.pre-commit-config.yaml @@ -0,0 +1,11 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.3.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace +- repo: https://github.com/psf/black + rev: 22.10.0 + hooks: + - id: black \ No newline at end of file diff --git a/model_data/simulation_system/MLModel/BaseMLModel.py b/model_data/simulation_system/MLModel/BaseMLModel.py index 42106a33..40ed53df 100644 --- a/model_data/simulation_system/MLModel/BaseMLModel.py +++ b/model_data/simulation_system/MLModel/BaseMLModel.py @@ -54,6 +54,7 @@ class MLModel(Protocol): Perfomance post processing on Model to ensure ready for deployment """ - def generate_meta_data(self): + def model_metadata(self) -> dict: """ + Extract out model metadata as dictionary """ diff --git a/model_data/simulation_system/MLModel/Models.py b/model_data/simulation_system/MLModel/Models.py index 137f2f20..869ff02f 100644 --- a/model_data/simulation_system/MLModel/Models.py +++ b/model_data/simulation_system/MLModel/Models.py @@ -135,6 +135,12 @@ class AutogluonModel: # This will return a string path of the location return self.model.clone_for_deployment(deployment_path) + + def model_metadata(self) -> dict: + """ + For Autogluon model, use the inbuilt model info method + """ + return self.model.info() diff --git a/model_data/simulation_system/predictions.py b/model_data/simulation_system/predictions.py index bc1b113b..ba7db181 100644 --- a/model_data/simulation_system/predictions.py +++ b/model_data/simulation_system/predictions.py @@ -112,13 +112,12 @@ def prediction(target_column: str = "RDSAP_CHANGE", model_path: str = None, data output_base = PREDICTION_LOCATION / target_column / uprn / TIMESTAMP output_base.mkdir(parents=True, exist_ok=True) - # TODO: change model.model.info to a class method for MLModel json_prediction = prediction.to_json(output_base / PREDICTION_FILE) prediction_metadata = { "model_type": model_type, "model_name": model_name, "model_location": model_location, - "model_settings": model.model.info() + "model_settings": model.model_metadata() } pd.DataFrame([prediction_metadata]).to_json(output_base / METADATA_FILE) diff --git a/model_data/simulation_system/training.py b/model_data/simulation_system/training.py index b37e7154..9dc17b2c 100644 --- a/model_data/simulation_system/training.py +++ b/model_data/simulation_system/training.py @@ -1,10 +1,8 @@ import argparse # import boto3 -import os from pathlib import Path from datetime import datetime -from typing import List from core.Logger import logger from core.DataLoader import DataLoader from core.FeatureProcessor import FeatureProcessor