Compare commits

...

30 commits

Author SHA1 Message Date
KhalimCK
64e1b57b2d
Merge pull request #143 from Hestia-Homes/heatingkwh-dev-model
remove costing columns, photo supply and main-heating-control
2024-08-09 07:59:09 +01:00
Michael Duong
c2ad73743a remove costing columns, photo supply and main-heating-control 2024-08-08 23:15:53 +01:00
Github-Bot
2fa0353ea1 Update Registry 2024-08-07 09:00:46 +00:00
Github-Bot
cfb9272a7b Update Registry 2024-08-07 09:00:06 +00:00
KhalimCK
6f5857d644
Merge pull request #141 from Hestia-Homes/heatingkwh-dev-model
remove the area-to-heated rooms feature, and env features
2024-08-07 09:59:25 +01:00
Michael Duong
bcb505084f use retain features again with remove env features 2024-08-06 22:18:27 +01:00
Michael Duong
318a51589d remove the area-to-heated rooms feature, and env features 2024-08-06 21:15:23 +01:00
Github-Bot
9a49caa0cd Update Registry 2024-08-06 19:36:12 +00:00
Github-Bot
fb9f364da3 Update Registry 2024-08-06 19:35:38 +00:00
KhalimCK
8a053fc775
Merge pull request #140 from Hestia-Homes/heatingkwh-dev-model
remove the rounding the 100 kwh
2024-08-06 20:35:05 +01:00
Michael Duong
bdb55d3ffe add estimated kwh 2024-08-06 20:22:31 +01:00
Michael Duong
d9b08b98dc remove the rounding the 100 kwh 2024-08-06 16:54:38 +01:00
Github-Bot
a6f6bc6bb5 Update Registry 2024-08-06 11:39:11 +00:00
Github-Bot
b3564e3521 Update Registry 2024-08-06 11:38:37 +00:00
KhalimCK
5c41c45516
Merge pull request #137 from Hestia-Homes/heatingkwh-dev-model
removed features for new heatingkwh model
2024-08-06 12:37:37 +01:00
Michael Duong
7af43ecbef removed features for new model 2024-08-05 22:46:03 +01:00
Github-Bot
119ce13740 Update Registry 2024-08-02 13:15:59 +00:00
Github-Bot
2f26bdd2f5 Update Registry 2024-08-02 13:14:52 +00:00
KhalimCK
0051f9cf97
Merge pull request #135 from Hestia-Homes/heatingkwh-dev-model
try new model
2024-08-02 14:14:18 +01:00
Michael Duong
97b432bac9 try new model 2024-07-28 11:31:03 +01:00
Github-Bot
64e44d0637 Update Registry 2024-07-22 13:33:29 +00:00
Github-Bot
43e5cf5370 Update Registry 2024-07-22 13:32:54 +00:00
KhalimCK
23221c87da
Merge pull request #132 from Hestia-Homes/heatingkwh-dev-model
initial heatingkwh model commit
2024-07-22 14:32:23 +01:00
Michael Duong
5cb8a8a6aa clipped extremely small heating values 2024-07-12 23:03:31 +01:00
Michael Duong
9785181e80 remove hot_water_kwh feature, lower mean squared error 2024-07-12 22:46:32 +01:00
Michael Duong
99d28e8b61 initial model commit 2024-07-12 15:13:11 +01:00
Github-Bot
732ea48cd1 Update Registry 2024-07-05 12:12:30 +00:00
Github-Bot
e4ddad7abc Update Registry 2024-07-05 12:11:49 +00:00
KhalimCK
e78b9226b8
Merge pull request #124 from Hestia-Homes/lighting-dev-model
test lighting model
2024-07-05 13:11:14 +01:00
Michael Duong
d164bff8d2 test lighting model 2024-07-04 13:47:33 +01:00
10 changed files with 342 additions and 89 deletions

View file

@ -2,7 +2,7 @@ name: Sap Change Model Deploy
on: on:
push: push:
branches: [ sap-dev, sap-prod, heat-dev, heat-prod, carbon-dev, carbon-prod] branches: [ sap-dev, sap-prod, heat-dev, heat-prod, carbon-dev, carbon-prod, heatingkwh-dev, heatingkwh-prod]
jobs: jobs:
deploy: deploy:

View file

@ -13,6 +13,7 @@ on:
- "sap-dev" - "sap-dev"
- "heat-dev" - "heat-dev"
- "carbon-dev" - "carbon-dev"
- "heatingkwh-dev"
permissions: write-all permissions: write-all

View file

@ -5,7 +5,7 @@ on:
# branches: # branches:
# - "model-**" # - "model-**"
pull_request: pull_request:
branches: ["sap-dev", "heat-dev", "carbon-dev"] branches: ["sap-dev", "heat-dev", "carbon-dev", "heatingkwh-dev"]
label: label:
types: ["created", "edited"] types: ["created", "edited"]

View file

@ -16,17 +16,57 @@
"active": true "active": true
}, },
"heat": { "heat": {
"version": "v0.5.0", "version": "v0.6.0",
"stage": { "stage": {
"dev": "v0.5.0" "dev": "v0.6.0"
}, },
"registered": true, "registered": true,
"active": true "active": true
}, },
"carbon": { "carbon": {
"version": "v0.5.0", "version": "v0.6.0",
"stage": { "stage": {
"dev": "v0.5.0" "dev": "v0.6.0"
},
"registered": true,
"active": true
},
"hotwater": {
"version": "v1.0.0",
"stage": {
"dev": "v1.0.0"
},
"registered": true,
"active": true
},
"heating": {
"version": "v1.0.0",
"stage": {
"dev": "v1.0.0"
},
"registered": true,
"active": true
},
"lighting": {
"version": "v1.0.0",
"stage": {
"dev": "v1.0.0"
},
"registered": true,
"active": true
},
"hotwaterkwh": {
"version": "v1.1.0",
"stage": {
"dev": "v1.1.0"
},
"registered": true,
"active": true
},
"heatingkwh": {
"version": "v1.4.0",
"stage": {
"dev": "v1.4.0"
}, },
"registered": true, "registered": true,
"active": true "active": true

View file

@ -17,14 +17,15 @@ Within `src` folder, the structure is as follows:
# How to develop using this pipeline: # How to develop using this pipeline:
Run `make init`, which will: First, download miniconda to use conda to manage Python Environments
- Download pyenv (Python version management) Rund `conda init`, to initialise your terminal
- Download Python 3.X.X as defined in the `make` file - current 3.10.12
- Create a virtual environment with this version of python Change to this directory and run `make init`, which will:
- Create a conda virtual environment with this version of python - current 3.10.12
- Install packages in the training and version control directories in the pipeline folder (dev version if applicable) - Install packages in the training and version control directories in the pipeline folder (dev version if applicable)
- Install pre-commit to enable pre-commit hooks - Install pre-commit to enable pre-commit hooks
To use the environment, run `source .dev_env_pipeline/bin/activate`. To use the environment, run `conda activate dev_env_pipeline`
To enable the virtual envrionemnt created in vscode: To enable the virtual envrionemnt created in vscode:
- Open settings - Open settings

View file

@ -5,6 +5,18 @@ During the feature processor step, we can apply additional business logic and fe
""" """
Business Logic dict + functions Business Logic dict + functions
""" """
import pandas as pd
import numpy as np
import boto3
import msgpack
s3 = boto3.resource('s3')
# Get the MessagePack data from S3
obj = s3.Object("retrofit-data-dev", "cleaned_epc_data/cleaned.bson")
cleaned = obj.get()['Body'].read()
cleaned = msgpack.unpackb(cleaned, raw=False)
def remove_starting_columns(df): def remove_starting_columns(df):
@ -44,6 +56,111 @@ def keep_non_zero_rdsap(df):
df = df[df["rdsap_change"] != 0] df = df[df["rdsap_change"] != 0]
return df return df
def remove_heatingkwh_bottom_percentile(df, percentile=0.0001):
df = df[df["heating_kwh"] > df["heating_kwh"].quantile(percentile)]
return df
def add_features_from_code(df):
FEATURES = {
"heating_kwh": [
"lodgement-year", "lodgement-month", "current-energy-efficiency", "energy-consumption-current",
"heating-cost-current", "heating-cost-potential", "total-floor-area", "number-heated-rooms",
"mainheat-description", "mainheat-energy-eff", "main-fuel", "secondheat-description", "property-type",
"built-form", "mainheatcont-description", "hotwater-description", "hot-water-energy-eff",
"walls-energy-eff",
"roof-energy-eff", "windows-description", "windows-energy-eff", "floor-description", "flat-top-storey",
"flat-storey-count", "unheated-corridor-length", "solar-water-heating-flag", "mechanical-ventilation",
"low-energy-lighting", "environment-impact-current", "energy-tariff",
"county", "construction-age-band", "co2-emissions-current",
],
"hot_water_kwh": [
"lodgement-year", "lodgement-month",
"current-energy-efficiency",
"energy-consumption-current",
"hot-water-cost-current",
"total-floor-area", "number-heated-rooms",
"hotwater-description", "hot-water-energy-eff", "main-fuel", "property-type", "built-form",
"co2-emissions-current",
]
}
CATEGORICAL_COLUMNS = [
"lodgement-year", "lodgement-month", "main-fuel", "mainheat-description", "number-heated-rooms",
"number-habitable-rooms", "mainheat-energy-eff", "mainheatcont-description", "property-type", "built-form",
"construction-age-band", "secondheat-description", "hotwater-description", "hot-water-energy-eff",
"walls-description", "walls-energy-eff", "roof-description", "roof-energy-eff", "floor-description",
"county",
"windows-description", "windows-energy-eff", "flat-top-storey",
"flat-storey-count", "unheated-corridor-length", "solar-water-heating-flag", "mechanical-ventilation",
"low-energy-lighting", "environment-impact-current", "energy-tariff", "current-energy-rating"
]
NUMERICAL_COLUMNS = list({
x for x in FEATURES["heating_kwh"] + FEATURES["hot_water_kwh"]
if x not in CATEGORICAL_COLUMNS
})
"""Performs feature engineering on the dataset."""
df["lodgement-date"] = pd.to_datetime(df["lodgement-date"])
df["lodgement-year"] = df["lodgement-date"].dt.year
df["lodgement-month"] = df["lodgement-date"].dt.month
# For walls, roof, floor description where we have average thermal transmittance, to avoid too many categories
# we group them
ranges = {
"lessthan 0.1": (0, 0.1),
"0.1 - 0.3": (0.1, 0.3),
"0.3 - 0.5": (0.3, 0.5),
"morethan 0.5": (0.5, 2.5),
}
# Generate the lookup table
thermal_transmittance_lookup_table = []
for i in range(1, 251):
value = i / 100
for label, (low, high) in ranges.items():
if low < value <= high:
thermal_transmittance_lookup_table.append({"from": value, "to": label})
break
# Convert to DataFrame for display
thermal_transmittance_lookup_table = pd.DataFrame(thermal_transmittance_lookup_table)
thermal_transmittance_lookup_table["from"] = thermal_transmittance_lookup_table["from"].astype(str)
# Apply the lookup table to the data
for feature in ["walls-description", "roof-description", "floor-description"]:
cleaned_df = pd.DataFrame(cleaned[feature])[["original_description", "thermal_transmittance"]]
# Round to 2 decimal places and convert to string
cleaned_df["thermal_transmittance"] = cleaned_df["thermal_transmittance"].round(2).astype(str)
df = df.merge(
cleaned_df,
how="left",
left_on=feature,
right_on="original_description",
)
# We now have the thermal transmittance in the data, which we can use to group with the lookup table
df = df.merge(
thermal_transmittance_lookup_table,
how="left",
left_on="thermal_transmittance",
right_on="from",
)
# Where "to" is populated, replace feature with to
df[feature] = np.where(
~pd.isnull(df["to"]),
df["to"],
df[feature]
)
df = df.drop(columns=["original_description", "thermal_transmittance", "from", "to"])
# Convert data types
df[NUMERICAL_COLUMNS] = df[NUMERICAL_COLUMNS].apply(pd.to_numeric)
df[CATEGORICAL_COLUMNS] = df[CATEGORICAL_COLUMNS].astype(str)
return df
# def keep_ending_columns(df): # def keep_ending_columns(df):
# ending_column_index = [ col_name.endswith("_ENDING") for col_name in list(df.columns)] # ending_column_index = [ col_name.endswith("_ENDING") for col_name in list(df.columns)]
@ -53,7 +170,42 @@ def keep_non_zero_rdsap(df):
# df = df[keep_columns] # df = df[keep_columns]
# return df # return df
def enforce_minimum_habitable_room_size(df):
# Need minimum of 6.5m per habitable room
df = df[
df["total-floor-area"] / df["number-habitable-rooms"].astype(float) > 6.5
].reset_index(drop=True)
return df
def round_to_100s(df):
df['heating_kwh'] = (df['heating_kwh']/100).round()*100
return df
def remove_high_ratio_of_area_to_rooms(df):
df['area-to-heated-rooms'] = df['total-floor-area'] / df['number-heated-rooms'].astype(float)
# Remove na rows
df = df[(df['area-to-heated-rooms'].notna())].reset_index(drop=True)
# change any infinite values to 0
df['area-to-heated-rooms'] = df['area-to-heated-rooms'].replace([np.inf], 0)
# Remove top 0.05% of area-to-heated-rooms
df = df[df['area-to-heated-rooms'] < df['area-to-heated-rooms'].quantile(0.9995)].reset_index(drop=True)
df = df.drop(columns=['area-to-heated-rooms'])
return df
def add_estimate_annual_kwh(df):
df['estimate_annual_kwh'] = df['energy-consumption-current'] * df['total-floor-area']
return df
business_logic = { business_logic = {
"add_features_from_code": add_features_from_code,
"remove_heatingkwh_bottom_percentile": remove_heatingkwh_bottom_percentile,
# "round_to_100s": round_to_100s,
"enforce_minimum_habitable_room_size": enforce_minimum_habitable_room_size,
"remove_high_ratio_of_area_to_rooms": remove_high_ratio_of_area_to_rooms,
"add_estimate_annual_kwh": add_estimate_annual_kwh,
# "keep_non_zero_rdsap": keep_non_zero_rdsap, # "keep_non_zero_rdsap": keep_non_zero_rdsap,
# "keep_flats": keep_flats, # "keep_flats": keep_flats,
# "remove_minimum_habitable_room_size": remove_minimum_habitable_room_size, # "remove_minimum_habitable_room_size": remove_minimum_habitable_room_size,

View file

@ -30,6 +30,6 @@ def clip_predictions_to_minimum_value(
post_prediction_logic = { post_prediction_logic = {
"clip_predictions_to_minimum_value": clip_predictions_to_minimum_value, # "clip_predictions_to_minimum_value": clip_predictions_to_minimum_value,
# "round_predictions": round_predictions # "round_predictions": round_predictions
} }

View file

@ -8,6 +8,6 @@ default:
# - s3://retrofit-data-dev/scenario_data/27-03-2024-11-38-15/recommendations_scoring_data.parquet # - s3://retrofit-data-dev/scenario_data/27-03-2024-11-38-15/recommendations_scoring_data.parquet
# - s3://retrofit-data-dev/scenario_data/26-05-2024-08-47-45/recommendations_scoring_data.parquet # - s3://retrofit-data-dev/scenario_data/26-05-2024-08-47-45/recommendations_scoring_data.parquet
# - s3://retrofit-data-dev/scenario_data/26-05-2024-10-44-53/recommendations_scoring_data.parquet # - s3://retrofit-data-dev/scenario_data/26-05-2024-10-44-53/recommendations_scoring_data.parquet
- s3://retrofit-data-dev/scenario_data/28-05-2024-19-22-41/recommendations_scoring_data.parquet # - s3://retrofit-data-dev/scenario_data/28-05-2024-19-22-41/recommendations_scoring_data.parquet
comparison_output_filepath: ./metrics/scenario_table.md comparison_output_filepath: ./metrics/scenario_table.md
metrics_output_filepath: ./metrics/scenario_metrics.md metrics_output_filepath: ./metrics/scenario_metrics.md

View file

@ -21,7 +21,10 @@ default:
# data_filepath: s3://retrofit-data-dev/sap_change_model/2024-03-22-18-56-53/dataset_rooms.parquet # data_filepath: s3://retrofit-data-dev/sap_change_model/2024-03-22-18-56-53/dataset_rooms.parquet
# data_filepath: s3://retrofit-data-dev/sap_change_model/2024-05-25-08-36-36/dataset_rooms.parquet # data_filepath: s3://retrofit-data-dev/sap_change_model/2024-05-25-08-36-36/dataset_rooms.parquet
# data_filepath: s3://retrofit-data-dev/sap_change_model/2024-05-26-10-31-39/dataset_rooms.parquet # data_filepath: s3://retrofit-data-dev/sap_change_model/2024-05-26-10-31-39/dataset_rooms.parquet
data_filepath: s3://retrofit-data-dev/sap_change_model/2024-05-28-19-08-25/dataset_rooms.parquet # data_filepath: s3://retrofit-data-dev/sap_change_model/2024-05-28-19-08-25/dataset_rooms.parquet
# data_filepath: s3://retrofit-data-dev/sap_change_model/2024-07-03-23-11-39/dataset_rooms.parquet
# data_filepath: s3://retrofit-data-dev/energy_consumption/2024-07-08/energy_consumption_dataset.parquet
data_filepath: s3://retrofit-data-dev/energy_consumption/2024-07-25/energy_consumption_dataset.parquet
train_proportion: 0.9 train_proportion: 0.9
output_train_filepath: ./data/prepared_data/train.parquet output_train_filepath: ./data/prepared_data/train.parquet
output_test_filepath: ./data/prepared_data/test.parquet output_test_filepath: ./data/prepared_data/test.parquet
@ -31,37 +34,61 @@ default:
feature_processor_config: feature_processor_config:
subsample_amount: null subsample_amount: null
subsample_seed: 0 subsample_seed: 0
target: sap_ending target: heating_kwh
identifier_columns: ["uprn"] identifier_columns: ["uprn"]
# drop_columns: ["heat_demand_change", "carbon_change", "rdsap_change", "heat_demand_ending", "carbon_ending", "days_to_starting", "days_to_ending"] drop_columns: ["hot_water_kwh"]
drop_columns: [ retain_features: [
"heat_demand_change", "carbon_change", "rdsap_change", "heat_demand_ending", "carbon_ending", "days_to_starting", "days_to_ending", 'uprn',
'number_habitable_rooms_starting', 'number_habitable_rooms_ending', 'number_heated_rooms_starting', 'number_heated_rooms_ending', # 'heating-cost-current',
'number_habitable_rooms', 'number_heated_rooms'] 'co2-emissions-current',
retain_features: null # 'hot-water-cost-current',
# retain_features: ['uprn', 'sap_starting', 'hot_water_energy_eff_ending', 'total-floor-area',
# 'mainheat_energy_eff_ending', 'constituency', 'roof_energy_eff_ending', 'secondheat-description',
# 'walls_energy_eff_ending', 'secondheat_description_ending', 'floor-description',
# 'property_type', 'mainheatc_energy_eff_ending', 'built_form', 'mainheat-energy-eff',
# 'walls_insulation_thickness_ending', 'potential_energy_efficiency', 'current-energy-efficiency',
# 'transaction_type_ending', 'walls-energy-eff',
# 'floor_thermal_transmittance_ending', 'roof-energy-eff',
# 'low_energy_lighting_ending', 'heat_demand_starting', 'property-type',
# 'photo_supply_ending', 'carbon_starting', 'mainheat-description',
# 'walls_thermal_transmittance_ending', 'mechanical-ventilation',
# 'roof_insulation_thickness_ending', 'floor-level',
# 'total_floor_area_ending', 'number_open_fireplaces_ending', 'built-form',
# 'windows_energy_eff_ending', 'walls-description',
# 'floor_height_ending', 'mainheatcont-description',
# 'extension_count_ending', 'roof-description',
# 'has_air_source_heat_pump_ending', 'energy-consumption-current',
# 'charging_system_ending', 'construction_age_band', 'glazed_type_ending', 'construction-age-band',
# 'roof_thermal_transmittance_ending', 'hotwater-description',
# 'floor_insulation_thickness_ending', 'has_mains_gas_ending', 'main-fuel',
# 'estimated_perimeter_starting', 'energy_consumption_potential', 'hot-water-energy-eff',
# 'environment_impact_potential', 'heater_type_ending', 'co2-emiss-curr-per-floor-area',
# 'multi_glaze_proportion_ending', 'windows-energy-eff',
# 'lighting_energy_eff_ending', 'fixed_lighting_outlets_count'] 'current-energy-rating',
'lodgement-year',
'extension-count',
'number-open-fireplaces',
'number-heated-rooms',
'windows-description',
# 'photo-supply',
'heat-loss-corridor',
'flat-top-storey',
'unheated-corridor-length',
'fixed-lighting-outlets-count',
'tenure',
'multi-glaze-proportion',
'solar-water-heating-flag',
'energy-tariff',
'floor-height',
'constituency',
'transaction-type',
'floor-energy-eff',
'lodgement-month',
# 'lighting-cost-current',
'glazed-area',
# 'main-heating-controls',
'estimate_annual_kwh',
]
generate_predictions: generate_predictions:
input_dataclient_type: local input_dataclient_type: local

View file

@ -21,26 +21,59 @@ stages:
params: params:
configs/settings.yaml: configs/settings.yaml:
default.feature_processor.feature_processor_config.drop_columns: default.feature_processor.feature_processor_config.drop_columns:
- heat_demand_change - hot_water_kwh
- carbon_change
- rdsap_change
- heat_demand_ending
- carbon_ending
- days_to_starting
- days_to_ending
- number_habitable_rooms_starting
- number_habitable_rooms_ending
- number_heated_rooms_starting
- number_heated_rooms_ending
- number_habitable_rooms
- number_heated_rooms
default.feature_processor.feature_processor_config.retain_features: default.feature_processor.feature_processor_config.retain_features:
- uprn
- co2-emissions-current
- total-floor-area
- secondheat-description
- floor-description
- mainheat-energy-eff
- current-energy-efficiency
- walls-energy-eff
- roof-energy-eff
- property-type
- mainheat-description
- mechanical-ventilation
- floor-level
- built-form
- walls-description
- mainheatcont-description
- roof-description
- energy-consumption-current
- construction-age-band
- hotwater-description
- main-fuel
- hot-water-energy-eff
- co2-emiss-curr-per-floor-area
- windows-energy-eff
- current-energy-rating
- lodgement-year
- extension-count
- number-open-fireplaces
- number-heated-rooms
- windows-description
- heat-loss-corridor
- flat-top-storey
- unheated-corridor-length
- fixed-lighting-outlets-count
- tenure
- multi-glaze-proportion
- solar-water-heating-flag
- energy-tariff
- floor-height
- constituency
- transaction-type
- floor-energy-eff
- lodgement-month
- glazed-area
- estimate_annual_kwh
default.feature_processor.feature_processor_config.subsample_amount: default.feature_processor.feature_processor_config.subsample_amount:
default.feature_processor.feature_processor_config.subsample_seed: 0 default.feature_processor.feature_processor_config.subsample_seed: 0
default.feature_processor.feature_processor_config.target: sap_ending default.feature_processor.feature_processor_config.target: heating_kwh
default.feature_processor.feature_processor_type: dataframe default.feature_processor.feature_processor_type: dataframe
default.prepare_data.data_filepath: default.prepare_data.data_filepath:
s3://retrofit-data-dev/sap_change_model/2024-05-28-19-08-25/dataset_rooms.parquet s3://retrofit-data-dev/energy_consumption/2024-07-25/energy_consumption_dataset.parquet
default.prepare_data.input_dataclient_type: aws-s3 default.prepare_data.input_dataclient_type: aws-s3
default.prepare_data.output_dataclient_type: local default.prepare_data.output_dataclient_type: local
default.prepare_data.output_test_filepath: ./data/prepared_data/test.parquet default.prepare_data.output_test_filepath: ./data/prepared_data/test.parquet
@ -49,8 +82,8 @@ stages:
outs: outs:
- path: data/prepared_data/ - path: data/prepared_data/
hash: md5 hash: md5
md5: 80c9e138146a1d96b9d16091c207e2e8.dir md5: a6241dcbb3fe1d3b39d1a300ea64dfc9.dir
size: 45056059 size: 9606500
nfiles: 2 nfiles: 2
build_model: build_model:
cmd: python 2_build_model.py cmd: python 2_build_model.py
@ -61,8 +94,8 @@ stages:
size: 4820 size: 4820
- path: data/prepared_data - path: data/prepared_data
hash: md5 hash: md5
md5: 80c9e138146a1d96b9d16091c207e2e8.dir md5: a6241dcbb3fe1d3b39d1a300ea64dfc9.dir
size: 45056059 size: 9606500
nfiles: 2 nfiles: 2
params: params:
configs/build_model.yaml: configs/build_model.yaml:
@ -94,18 +127,18 @@ stages:
outs: outs:
- path: data/fit_predictions/ - path: data/fit_predictions/
hash: md5 hash: md5
md5: d9c9afc05e8780db47c0548b19bf7d19.dir md5: 5e07647b4dd0145a6d52d6ef729a3bde.dir
size: 3349989 size: 1545562
nfiles: 1 nfiles: 1
- path: data/model/ - path: data/model/
hash: md5 hash: md5
md5: 13c3100e1486c27a83a8a47491077842.dir md5: ce14e6f1e69c5513a04403eb00e0db0a.dir
size: 773523079 size: 99464470
nfiles: 36 nfiles: 35
- path: metrics/fit_metrics.json - path: metrics/fit_metrics.json
hash: md5 hash: md5
md5: 2ff70a2a45813e1bcdf2ea3aa8e07d4a md5: 425c0c6c13742d2d21051bf7ceb90127
size: 224 size: 218
generate_predictions: generate_predictions:
cmd: python 3_generate_predictions.py cmd: python 3_generate_predictions.py
deps: deps:
@ -115,13 +148,13 @@ stages:
size: 2464 size: 2464
- path: data/model - path: data/model
hash: md5 hash: md5
md5: 13c3100e1486c27a83a8a47491077842.dir md5: ce14e6f1e69c5513a04403eb00e0db0a.dir
size: 773523079 size: 99464470
nfiles: 36 nfiles: 35
- path: data/prepared_data - path: data/prepared_data
hash: md5 hash: md5
md5: 80c9e138146a1d96b9d16091c207e2e8.dir md5: a6241dcbb3fe1d3b39d1a300ea64dfc9.dir
size: 45056059 size: 9606500
nfiles: 2 nfiles: 2
params: params:
configs/settings.yaml: configs/settings.yaml:
@ -133,8 +166,8 @@ stages:
outs: outs:
- path: data/predictions/ - path: data/predictions/
hash: md5 hash: md5
md5: 5d07bcebf3160a72bb18dfd79106e85c.dir md5: ddaa04115c5dd4299974048080d762f5.dir
size: 463197 size: 163540
nfiles: 1 nfiles: 1
generate_metrics: generate_metrics:
cmd: python 4_generate_metrics.py cmd: python 4_generate_metrics.py
@ -145,13 +178,13 @@ stages:
size: 3484 size: 3484
- path: data/predictions - path: data/predictions
hash: md5 hash: md5
md5: 5d07bcebf3160a72bb18dfd79106e85c.dir md5: ddaa04115c5dd4299974048080d762f5.dir
size: 463197 size: 163540
nfiles: 1 nfiles: 1
- path: data/prepared_data - path: data/prepared_data
hash: md5 hash: md5
md5: 80c9e138146a1d96b9d16091c207e2e8.dir md5: a6241dcbb3fe1d3b39d1a300ea64dfc9.dir
size: 45056059 size: 9606500
nfiles: 2 nfiles: 2
params: params:
configs/settings.yaml: configs/settings.yaml:
@ -161,8 +194,8 @@ stages:
outs: outs:
- path: metrics/metrics.json - path: metrics/metrics.json
hash: md5 hash: md5
md5: 3e08df02fd5c5d094bcf936e1338d596 md5: 0beb72a28af4af37a619181b14c2e311
size: 223 size: 218
generate_scenerio_metrics: generate_scenerio_metrics:
cmd: python 5_generate_scenarios.py cmd: python 5_generate_scenarios.py
deps: deps:
@ -176,15 +209,14 @@ stages:
input_dataclient_type: aws-s3 input_dataclient_type: aws-s3
output_dataclient_type: local output_dataclient_type: local
scenario_data_filepaths: scenario_data_filepaths:
- s3://retrofit-data-dev/scenario_data/28-05-2024-19-22-41/recommendations_scoring_data.parquet
comparison_output_filepath: ./metrics/scenario_table.md comparison_output_filepath: ./metrics/scenario_table.md
metrics_output_filepath: ./metrics/scenario_metrics.md metrics_output_filepath: ./metrics/scenario_metrics.md
outs: outs:
- path: metrics/scenario_metrics.md - path: metrics/scenario_metrics.md
hash: md5 hash: md5
md5: fa4d6d7bbd7818613800da5f8f37ea96 md5: d41d8cd98f00b204e9800998ecf8427e
size: 363 size: 0
- path: metrics/scenario_table.md - path: metrics/scenario_table.md
hash: md5 hash: md5
md5: d6baf100a1623cc2467c2f8221d314c9 md5: d41d8cd98f00b204e9800998ecf8427e
size: 2133 size: 0