From 642a224a7b570344d0f30fbccc1493de24200a43 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Mon, 25 Sep 2023 19:38:20 +0100 Subject: [PATCH] adding due considerations ecr --- infrastructure/terraform/main.tf | 5 ++++ .../simulation_system/core/DataProcessor.py | 5 +++- .../generate_rdsap_change.py | 23 ++++++++++++++++++- 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/infrastructure/terraform/main.tf b/infrastructure/terraform/main.tf index 948ea915..399a6ff5 100644 --- a/infrastructure/terraform/main.tf +++ b/infrastructure/terraform/main.tf @@ -143,3 +143,8 @@ module "lambda_sap_prediction_ecr" { ecr_name = "lambda-sap-prediction-${var.stage}" source = "./modules/ecr" } + +module "due_considerations_ecr" { + ecr_name = "due-considerations-${var.stage}" + source = "./modules/ecr" +} diff --git a/model_data/simulation_system/core/DataProcessor.py b/model_data/simulation_system/core/DataProcessor.py index 1252f6c6..439ce2e0 100644 --- a/model_data/simulation_system/core/DataProcessor.py +++ b/model_data/simulation_system/core/DataProcessor.py @@ -543,8 +543,11 @@ class DataProcessor: df[uvalue_col] = pd.to_numeric(df[uvalue_col]) key_columns = [ - "RDSAP_CHANGE", "HEAT_DEMAND_CHANGE", "CARBON_CHANGE", "SAP_STARTING", "HEAT_DEMAND_STARTING", + "RDSAP_CHANGE", "HEAT_DEMAND_CHANGE", "CARBON_CHANGE", + "SAP_STARTING", "HEAT_DEMAND_STARTING", "CARBON_STARTING", "UPRN", "CONSTITUENCY", + "SAP_ENDING", "CARBON_ENDING", "HEAT_DEMAND_ENDING", + "DAYS_TO_STARTING", "DAYS_TO_ENDING" ] ignore_cols = FIXED_FEATURES + FIXED_DESCRIPTON_MAPPED_FEATURES + key_columns diff --git a/model_data/simulation_system/generate_rdsap_change.py b/model_data/simulation_system/generate_rdsap_change.py index 003fa046..b317e52c 100644 --- a/model_data/simulation_system/generate_rdsap_change.py +++ b/model_data/simulation_system/generate_rdsap_change.py @@ -458,6 +458,10 @@ def app(): starting_heat_demand = earliest_record[HEAT_DEMAND_RESPONSE] starting_carbon = earliest_record[CARBON_RESPONSE] + ending_sap = latest_record[RDSAP_RESPONSE] + ending_heat_demand = latest_record[HEAT_DEMAND_RESPONSE] + ending_carbon = latest_record[CARBON_RESPONSE] + rdsap_change = latest_record[RDSAP_RESPONSE] - starting_sap heat_demand_change = latest_record[HEAT_DEMAND_RESPONSE] - starting_heat_demand carbon_change = latest_record[CARBON_RESPONSE] - starting_carbon @@ -469,6 +473,10 @@ def app(): starting_heat_demand = latest_record[HEAT_DEMAND_RESPONSE] starting_carbon = latest_record[CARBON_RESPONSE] + ending_sap = earliest_record[RDSAP_RESPONSE] + ending_heat_demand = earliest_record[HEAT_DEMAND_RESPONSE] + ending_carbon = earliest_record[CARBON_RESPONSE] + rdsap_change = earliest_record[RDSAP_RESPONSE] - starting_sap heat_demand_change = earliest_record[HEAT_DEMAND_RESPONSE] - starting_heat_demand carbon_change = earliest_record[CARBON_RESPONSE] - starting_carbon @@ -488,8 +496,11 @@ def app(): "HEAT_DEMAND_CHANGE": heat_demand_change, "CARBON_CHANGE": carbon_change, "SAP_STARTING": starting_sap, + "SAP_ENDING": ending_sap, "HEAT_DEMAND_STARTING": starting_heat_demand, + "HEAT_DEMAND_ENDING": ending_heat_demand, "CARBON_STARTING": starting_carbon, + "CARBON_ENDING": ending_carbon, **fixed_data, **features.to_dict(), } @@ -553,12 +564,22 @@ def app(): output = pd.concat(dataset) + uvalue_columns = [col for col in output.columns if "thermal_transmittance" in col] + for uvalue_col in uvalue_columns: + output[uvalue_col] = pd.to_numeric(output[uvalue_col]) + + save_dataframe_to_s3_parquet( + df=output, + bucket_name="retrofit-data-dev", + file_key="sap_change_model/dataset_without_differencing.parquet", + ) + output = DataProcessor.difference_data(output) save_dataframe_to_s3_parquet( df=output, bucket_name="retrofit-data-dev", - file_key="sap_change_model/dataset_new_not_diff.parquet", + file_key="sap_change_model/dataset_with_differencing.parquet", )