diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..84abbe6 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +modules/ml-pipeline/src/pipeline/data/predictions +modules/ml-pipeline/src/pipeline/data/fit_predictions +modules/ml-pipeline/src/pipeline/data/prepared_data +modules/ml-pipeline/src/pipeline/data/model/allmodels +modules/ml-pipeline/src/pipeline/metrics +modules/ml-pipeline/src/pipeline/__pycache__ +modules/ml-pipeline/src/pipeline/.dvc +modules/ml-pipeline/src/pipeline/analysis +modules/ml-pipeline/src/pipeline/metrics diff --git a/deployment/.dockerignore b/deployment/.dockerignore index e01cbd5..8b8a7fb 100644 --- a/deployment/.dockerignore +++ b/deployment/.dockerignore @@ -1,4 +1,8 @@ -modules/ml-pipeline/src/pipeline/data/predictions* -modules/ml-pipeline/src/pipeline/data/prepared_data* -modules/ml-pipeline/src/pipeline/data/model/allmodels* -modules/ml-pipeline/src/pipeline/metrics* +modules/ml-pipeline/src/pipeline/data/predictions +modules/ml-pipeline/src/pipeline/data/prepared_data +modules/ml-pipeline/src/pipeline/data/model/allmodels +modules/ml-pipeline/src/pipeline/metrics +modules/ml-pipeline/src/__pycache__ +modules/ml-pipeline/src/.dvc +modules/ml-pipeline/src/analysis +modules/ml-pipeline/src/metrics diff --git a/modules/ml-pipeline/src/.dockerignore b/modules/ml-pipeline/src/.dockerignore index 14f71d7..5feb57d 100644 --- a/modules/ml-pipeline/src/.dockerignore +++ b/modules/ml-pipeline/src/.dockerignore @@ -1,4 +1,6 @@ -pipeline/data/predictions* -pipeline/data/prepared_data/train.parquet* -pipeline/data/model/allmodels* -pipeline/metrics* +pipeline/data/predictions +pipeline/data/prepared_data/train.parquet +pipeline/data/model/allmodels +pipeline/metrics +pipeline/.dvc +pipeline/analysis diff --git a/modules/ml-pipeline/src/Prediction.Dockerfile b/modules/ml-pipeline/src/Prediction.Dockerfile index a6fc539..e0a292c 100644 --- a/modules/ml-pipeline/src/Prediction.Dockerfile +++ b/modules/ml-pipeline/src/Prediction.Dockerfile @@ -1,7 +1,7 @@ # Dockerfile that can be used to test loading a model to generate a prediction (part of CI/CD flow) FROM python:3.10.12-slim -RUN apt-get update && apt-get install -y libgomp1 +RUN apt-get update && apt-get install -y libgomp1 gcc python3-dev COPY pipeline/requirements/predictions/requirements.txt requirements.txt diff --git a/modules/ml-pipeline/src/pipeline/dvc.lock b/modules/ml-pipeline/src/pipeline/dvc.lock index 104dc83..ca4bfdd 100644 --- a/modules/ml-pipeline/src/pipeline/dvc.lock +++ b/modules/ml-pipeline/src/pipeline/dvc.lock @@ -35,46 +35,6 @@ stages: - number_habitable_rooms - number_heated_rooms default.feature_processor.feature_processor_config.retain_features: - - uprn - - sap_starting - - hot_water_energy_eff_ending - - mainheat_energy_eff_ending - - constituency - - roof_energy_eff_ending - - walls_energy_eff_ending - - secondheat_description_ending - - property_type - - mainheatc_energy_eff_ending - - built_form - - walls_insulation_thickness_ending - - potential_energy_efficiency - - transaction_type_ending - - floor_thermal_transmittance_ending - - low_energy_lighting_ending - - heat_demand_starting - - photo_supply_ending - - carbon_starting - - walls_thermal_transmittance_ending - - roof_insulation_thickness_ending - - total_floor_area_ending - - number_open_fireplaces_ending - - windows_energy_eff_ending - - floor_height_ending - - extension_count_ending - - has_air_source_heat_pump_ending - - charging_system_ending - - construction_age_band - - glazed_type_ending - - roof_thermal_transmittance_ending - - floor_insulation_thickness_ending - - has_mains_gas_ending - - estimated_perimeter_starting - - energy_consumption_potential - - environment_impact_potential - - heater_type_ending - - multi_glaze_proportion_ending - - lighting_energy_eff_ending - - fixed_lighting_outlets_count default.feature_processor.feature_processor_config.subsample_amount: default.feature_processor.feature_processor_config.subsample_seed: 0 default.feature_processor.feature_processor_config.target: sap_ending @@ -89,8 +49,8 @@ stages: outs: - path: data/prepared_data/ hash: md5 - md5: efa416abea618ae6220a0c3d597603cf.dir - size: 44750997 + md5: a1ff276b1cbd2db0b8e2982cfd524b40.dir + size: 44713582 nfiles: 2 build_model: cmd: python 2_build_model.py @@ -101,8 +61,8 @@ stages: size: 4820 - path: data/prepared_data hash: md5 - md5: efa416abea618ae6220a0c3d597603cf.dir - size: 44750997 + md5: a1ff276b1cbd2db0b8e2982cfd524b40.dir + size: 44713582 nfiles: 2 params: configs/build_model.yaml: @@ -134,18 +94,18 @@ stages: outs: - path: data/fit_predictions/ hash: md5 - md5: de46250d454c4d713ab580b10ff3fd31.dir - size: 3349318 + md5: 750cd7426e2909ed36bc05601b5e04c4.dir + size: 3349190 nfiles: 1 - path: data/model/ hash: md5 - md5: 18bd7a93ece75a65d3a950b7dfdab4fb.dir - size: 735951861 - nfiles: 35 + md5: aa5eb60be553f2a01e63783cf8f1fad1.dir + size: 765461992 + nfiles: 36 - path: metrics/fit_metrics.json hash: md5 - md5: 8a952a5e884c268e6059357a627b9251 - size: 224 + md5: ca5190b3292210c57a58668fdb48296c + size: 226 generate_predictions: cmd: python 3_generate_predictions.py deps: @@ -155,13 +115,13 @@ stages: size: 2464 - path: data/model hash: md5 - md5: 18bd7a93ece75a65d3a950b7dfdab4fb.dir - size: 735951861 - nfiles: 35 + md5: aa5eb60be553f2a01e63783cf8f1fad1.dir + size: 765461992 + nfiles: 36 - path: data/prepared_data hash: md5 - md5: efa416abea618ae6220a0c3d597603cf.dir - size: 44750997 + md5: a1ff276b1cbd2db0b8e2982cfd524b40.dir + size: 44713582 nfiles: 2 params: configs/settings.yaml: @@ -173,8 +133,8 @@ stages: outs: - path: data/predictions/ hash: md5 - md5: 07ef721a0dc94a52e3ba7a70ac45b8ff.dir - size: 463563 + md5: 934c94e5b1a2c70db3dc865ee056f460.dir + size: 463619 nfiles: 1 generate_metrics: cmd: python 4_generate_metrics.py @@ -185,13 +145,13 @@ stages: size: 3484 - path: data/predictions hash: md5 - md5: 07ef721a0dc94a52e3ba7a70ac45b8ff.dir - size: 463563 + md5: 934c94e5b1a2c70db3dc865ee056f460.dir + size: 463619 nfiles: 1 - path: data/prepared_data hash: md5 - md5: efa416abea618ae6220a0c3d597603cf.dir - size: 44750997 + md5: a1ff276b1cbd2db0b8e2982cfd524b40.dir + size: 44713582 nfiles: 2 params: configs/settings.yaml: @@ -201,15 +161,15 @@ stages: outs: - path: metrics/metrics.json hash: md5 - md5: 9f863f47799d42c101eba3b03a179455 + md5: aa2a511ac759225549636ba05d6b667c size: 224 generate_scenerio_metrics: cmd: python 5_generate_scenarios.py deps: - path: 5_generate_scenarios.py hash: md5 - md5: a18f6c6ae2082f038df47386cf3e418e - size: 4896 + md5: 40506749fefd926d47c60ff5b16db307 + size: 5337 params: configs/scenarios.yaml: default.scenarios: @@ -222,9 +182,9 @@ stages: outs: - path: metrics/scenario_metrics.md hash: md5 - md5: 64e7db945ff655ae03c20c9845f19106 + md5: 4085a6ea3d044ad2fe7ac63b0a685fed size: 363 - path: metrics/scenario_table.md hash: md5 - md5: d4f8afe07b774374aeaa48f1b7b8a5fc + md5: a2b3da77921b5dcc10f7068646e0eae3 size: 2133 diff --git a/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements-dev.txt b/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements-dev.txt index 734419a..35875d9 100644 --- a/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements-dev.txt +++ b/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements-dev.txt @@ -1,7 +1,7 @@ joblib==1.3.2 boto3==1.28.17 pandas==2.1.4 -autogluon==1.0.0 +autogluon.tabular[lightgbm,xgboost,fastai]==1.0.0 dynaconf==3.2.1 pyarrow==13.0.0 pre-commit==3.3.3 diff --git a/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements.txt b/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements.txt index 937b000..6a96822 100644 --- a/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements.txt +++ b/modules/ml-pipeline/src/pipeline/requirements/predictions/requirements.txt @@ -1,7 +1,7 @@ joblib==1.3.2 boto3==1.28.17 pandas==2.1.4 -autogluon==1.0.0 +autogluon.tabular[lightgbm,xgboost,fastai]==1.0.0 dynaconf==3.2.1 pyarrow==13.0.0 PyYAML==6.0.1 diff --git a/modules/ml-pipeline/src/pipeline/requirements/training/requirements-dev.txt b/modules/ml-pipeline/src/pipeline/requirements/training/requirements-dev.txt index fe06a4d..6e1ea2f 100644 --- a/modules/ml-pipeline/src/pipeline/requirements/training/requirements-dev.txt +++ b/modules/ml-pipeline/src/pipeline/requirements/training/requirements-dev.txt @@ -1,7 +1,7 @@ joblib==1.3.2 boto3==1.28.17 pandas==2.1.4 -autogluon==1.0.0 +autogluon.tabular[lightgbm,xgboost,fastai]==1.0.0 ray==2.6.3 dynaconf==3.2.1 alibi==0.9.5 diff --git a/modules/ml-pipeline/src/pipeline/requirements/training/requirements.txt b/modules/ml-pipeline/src/pipeline/requirements/training/requirements.txt index a5bccd3..e4e319c 100644 --- a/modules/ml-pipeline/src/pipeline/requirements/training/requirements.txt +++ b/modules/ml-pipeline/src/pipeline/requirements/training/requirements.txt @@ -1,4 +1,4 @@ boto3==1.28.41 pandas==2.1.4 -autogluon==1.0.0 +autogluon.tabular[lightgbm,xgboost,fastai]==1.0.0 dynaconf==3.2.1