From 5544220e6fbf301d7de1457c353b0c569a50e1f5 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Wed, 4 Mar 2026 11:28:32 +0000 Subject: [PATCH 1/9] set environment variables on engine lambda from github --- .github/workflows/_deploy_lambda.yml | 59 +++++++++++++++++- .github/workflows/deploy_terraform.yml | 41 ++++++++++++ .../terraform/lambda/engine/main.tf | 55 ++++++++++++++-- .../terraform/lambda/engine/variables.tf | 62 ++++++++++++++++++- 4 files changed, 210 insertions(+), 7 deletions(-) diff --git a/.github/workflows/_deploy_lambda.yml b/.github/workflows/_deploy_lambda.yml index 528300f8..51024bd4 100644 --- a/.github/workflows/_deploy_lambda.yml +++ b/.github/workflows/_deploy_lambda.yml @@ -42,6 +42,34 @@ on: required: true AWS_REGION: required: true + TF_VAR_api_key: + required: false + TF_VAR_secret_key: + required: false + TF_VAR_domain_name: + required: false + TF_VAR_epc_auth_token: + required: false + TF_VAR_google_solar_api_key: + required: false + TF_VAR_plan_trigger_bucket: + required: false + TF_VAR_data_bucket: + required: false + TF_VAR_predictions_bucket: + required: false + TF_VAR_sap_predictions_bucket: + required: false + TF_VAR_carbon_predictions_bucket: + required: false + TF_VAR_heat_predictions_bucket: + required: false + TF_VAR_heating_kwh_predictions_bucket: + required: false + TF_VAR_hotwater_kwh_predictions_bucket: + required: false + TF_VAR_energy_assessments_bucket: + required: false jobs: deploy: @@ -90,6 +118,21 @@ jobs: - name: Terraform Plan working-directory: ${{ inputs.lambda_path }} + env: + TF_VAR_api_key: ${{ secrets.TF_VAR_api_key }} + TF_VAR_secret_key: ${{ secrets.TF_VAR_secret_key }} + TF_VAR_domain_name: ${{ secrets.TF_VAR_domain_name }} + TF_VAR_epc_auth_token: ${{ secrets.TF_VAR_epc_auth_token }} + TF_VAR_google_solar_api_key: ${{ secrets.TF_VAR_google_solar_api_key }} + TF_VAR_plan_trigger_bucket: ${{ secrets.TF_VAR_plan_trigger_bucket }} + TF_VAR_data_bucket: ${{ secrets.TF_VAR_data_bucket }} + TF_VAR_predictions_bucket: ${{ secrets.TF_VAR_predictions_bucket }} + TF_VAR_sap_predictions_bucket: ${{ secrets.TF_VAR_sap_predictions_bucket }} + TF_VAR_carbon_predictions_bucket: ${{ secrets.TF_VAR_carbon_predictions_bucket }} + TF_VAR_heat_predictions_bucket: ${{ secrets.TF_VAR_heat_predictions_bucket }} + TF_VAR_heating_kwh_predictions_bucket: ${{ secrets.TF_VAR_heating_kwh_predictions_bucket }} + TF_VAR_hotwater_kwh_predictions_bucket: ${{ secrets.TF_VAR_hotwater_kwh_predictions_bucket }} + TF_VAR_energy_assessments_bucket: ${{ secrets.TF_VAR_energy_assessments_bucket }} run: | terraform plan \ -var="stage=${{ inputs.stage }}" \ @@ -106,10 +149,24 @@ jobs: - name: Terraform Destroy if: inputs.terraform_destroy == 'true' && inputs.terraform_apply != 'true' working-directory: ${{ inputs.lambda_path }} + env: + TF_VAR_api_key: ${{ secrets.TF_VAR_api_key }} + TF_VAR_secret_key: ${{ secrets.TF_VAR_secret_key }} + TF_VAR_domain_name: ${{ secrets.TF_VAR_domain_name }} + TF_VAR_epc_auth_token: ${{ secrets.TF_VAR_epc_auth_token }} + TF_VAR_google_solar_api_key: ${{ secrets.TF_VAR_google_solar_api_key }} + TF_VAR_plan_trigger_bucket: ${{ secrets.TF_VAR_plan_trigger_bucket }} + TF_VAR_data_bucket: ${{ secrets.TF_VAR_data_bucket }} + TF_VAR_predictions_bucket: ${{ secrets.TF_VAR_predictions_bucket }} + TF_VAR_sap_predictions_bucket: ${{ secrets.TF_VAR_sap_predictions_bucket }} + TF_VAR_carbon_predictions_bucket: ${{ secrets.TF_VAR_carbon_predictions_bucket }} + TF_VAR_heat_predictions_bucket: ${{ secrets.TF_VAR_heat_predictions_bucket }} + TF_VAR_heating_kwh_predictions_bucket: ${{ secrets.TF_VAR_heating_kwh_predictions_bucket }} + TF_VAR_hotwater_kwh_predictions_bucket: ${{ secrets.TF_VAR_hotwater_kwh_predictions_bucket }} + TF_VAR_energy_assessments_bucket: ${{ secrets.TF_VAR_energy_assessments_bucket }} run: | terraform destroy -auto-approve \ -var="stage=${{ inputs.stage }}" \ -var="lambda_name=${{ inputs.lambda_name }}" \ -var="ecr_repo_url=${{ steps.repo.outputs.ecr_repo_url }}" \ -var="image_digest=${{ inputs.image_digest }}" - diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 4c9ce44a..4b0adbac 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -241,4 +241,45 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + # ============================================================ + # Ara Engine image and Push + # ============================================================ + ara_engine_image: + needs: [determine_stage, shared_terraform] + uses: ./.github/workflows/_build_image.yml + with: + ecr_repo: engine-${{ needs.determine_stage.outputs.stage }} + dockerfile_path: backend/docker/engine.Dockerfile + build_context: . + # ============================================================ + # Deploy Categorisation Lambda + # ============================================================ + ara_engine_lambda: + needs: [ara_engine_image, determine_stage] + uses: ./.github/workflows/_deploy_lambda.yml + with: + lambda_name: ara_engine + lambda_path: infrastructure/terraform/lambda/engine + stage: ${{ needs.determine_stage.outputs.stage }} + ecr_repo: engine-${{ needs.determine_stage.outputs.stage }} + image_digest: ${{ needs.ara_engine_image.outputs.image_digest }} + terraform_apply: ${{ needs.determine_stage.outputs.terraform_apply }} + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + TF_VAR_api_key: ${{ secrets.DEV_API_KEY }} + TF_VAR_secret_key: ${{ secrets.DEV_SECRET_KEY }} + TF_VAR_domain_name: ${{ secrets.DEV_DOMAIN_NAME }} + TF_VAR_epc_auth_token: ${{ secrets.DEV_EPC_AUTH_TOKEN }} + TF_VAR_google_solar_api_key: ${{ secrets.DEV_GOOGLE_SOLAR_API_KEY }} + TF_VAR_plan_trigger_bucket: ${{ secrets.DEV_PLAN_TRIGGER_BUCKET }} + TF_VAR_data_bucket: ${{ secrets.DEV_DATA_BUCKET }} + TF_VAR_predictions_bucket: ${{ secrets.DEV_PREDICTIONS_BUCKET }} + TF_VAR_sap_predictions_bucket: ${{ secrets.DEV_SAP_PREDICTIONS_BUCKET }} + TF_VAR_carbon_predictions_bucket: ${{ secrets.DEV_CARBON_PREDICTIONS_BUCKET }} + TF_VAR_heat_predictions_bucket: ${{ secrets.DEV_HEAT_PREDICTIONS_BUCKET }} + TF_VAR_heating_kwh_predictions_bucket: ${{ secrets.DEV_HEATING_KWH_PREDICTIONS_BUCKET }} + TF_VAR_hotwater_kwh_predictions_bucket: ${{ secrets.DEV_HOTWATER_KWH_PREDICTIONS_BUCKET }} + TF_VAR_energy_assessments_bucket: ${{ secrets.DEV_ENERGY_ASSESSMENTS_BUCKET }} \ No newline at end of file diff --git a/infrastructure/terraform/lambda/engine/main.tf b/infrastructure/terraform/lambda/engine/main.tf index c1cff8a3..f9cb4f46 100644 --- a/infrastructure/terraform/lambda/engine/main.tf +++ b/infrastructure/terraform/lambda/engine/main.tf @@ -7,6 +7,15 @@ data "terraform_remote_state" "shared" { } } +data "aws_secretsmanager_secret_version" "db_credentials" { + secret_id = "${var.stage}/assessment_model/db_credentials" +} + +locals { + db_credentials = jsondecode(data.aws_secretsmanager_secret_version.db_credentials.secret_string) +} + + module "lambda" { source = "../modules/lambda_with_sqs" @@ -18,8 +27,44 @@ module "lambda" { # Optional: Set maximum_concurrency to limit concurrent SQS-triggered invocations (2-1000) maximum_concurrency = var.maximum_concurrency - environment = { - STAGE = var.stage - LOG_LEVEL = "info" - } -} \ No newline at end of file + environment = merge( + { + STAGE = var.stage + LOG_LEVEL = "info" + + # DB from Secrets Manager + DB_USERNAME = local.db_credentials.db_assessment_model_username + DB_PASSWORD = local.db_credentials.db_assessment_model_password + + # Secrets from GitHub + DB_HOST = var.db_host + DB_NAME = var.db_name + DB_PORT = var.db_port + API_KEY = var.api_key + SECRET_KEY = var.secret_key + DOMAIN_NAME = var.domain_name + EPC_AUTH_TOKEN = var.epc_auth_token + GOOGLE_SOLAR_API_KEY = var.google_solar_api_key + + # Buckets + PLAN_TRIGGER_BUCKET = var.plan_trigger_bucket + DATA_BUCKET = var.data_bucket + PREDICTIONS_BUCKET = var.predictions_bucket + SAP_PREDICTIONS_BUCKET = var.sap_predictions_bucket + CARBON_PREDICTIONS_BUCKET = var.carbon_predictions_bucket + HEAT_PREDICTIONS_BUCKET = var.heat_predictions_bucket + HEATING_KWH_PREDICTIONS_BUCKET = var.heating_kwh_predictions_bucket + HOTWATER_KWH_PREDICTIONS_BUCKET = var.hotwater_kwh_predictions_bucket + ENERGY_ASSESSMENTS_BUCKET = var.energy_assessments_bucket + + # SQS + ENGINE_SQS_URL = module.lambda.sqs_queue_url + + # Deployment + ECR_URI = var.ecr_repo_url + GITHUB_SHA = var.image_digest + } + ) +} + +# Policies and IAM \ No newline at end of file diff --git a/infrastructure/terraform/lambda/engine/variables.tf b/infrastructure/terraform/lambda/engine/variables.tf index 503bf6c8..189cc848 100644 --- a/infrastructure/terraform/lambda/engine/variables.tf +++ b/infrastructure/terraform/lambda/engine/variables.tf @@ -23,10 +23,70 @@ variable "maximum_concurrency" { description = "Maximum number of concurrent Lambda invocations from SQS (2-1000). null = no limit." } +variable "api_key" { + type = string + sensitive = true +} + +variable "secret_key" { + type = string + sensitive = true +} + +variable "domain_name" { + type = string +} + +variable "epc_auth_token" { + type = string + sensitive = true +} + +variable "google_solar_api_key" { + type = string + sensitive = true +} + +variable "plan_trigger_bucket" { + type = string +} + +variable "data_bucket" { + type = string +} + +variable "predictions_bucket" { + type = string +} + +variable "sap_predictions_bucket" { + type = string +} + +variable "carbon_predictions_bucket" { + type = string +} + +variable "heat_predictions_bucket" { + type = string +} + +variable "heating_kwh_predictions_bucket" { + type = string +} + +variable "hotwater_kwh_predictions_bucket" { + type = string +} + +variable "energy_assessments_bucket" { + type = string +} + locals { image_uri = "${var.ecr_repo_url}@${var.image_digest}" } output "resolved_image_uri" { value = local.image_uri -} +} \ No newline at end of file From d10c9f98cc5b1b8877ea378b29a3a504ac8d2825 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Wed, 4 Mar 2026 15:49:50 +0000 Subject: [PATCH 2/9] get bucket names from terraform state --- .../terraform/lambda/engine/main.tf | 20 ++++++------- .../terraform/lambda/engine/variables.tf | 24 --------------- infrastructure/terraform/shared/main.tf | 30 +++++++++++++++++++ 3 files changed, 40 insertions(+), 34 deletions(-) diff --git a/infrastructure/terraform/lambda/engine/main.tf b/infrastructure/terraform/lambda/engine/main.tf index f9cb4f46..6d0603e6 100644 --- a/infrastructure/terraform/lambda/engine/main.tf +++ b/infrastructure/terraform/lambda/engine/main.tf @@ -37,25 +37,25 @@ module "lambda" { DB_PASSWORD = local.db_credentials.db_assessment_model_password # Secrets from GitHub - DB_HOST = var.db_host - DB_NAME = var.db_name - DB_PORT = var.db_port + DB_HOST = var.db_host + DB_NAME = var.db_name + DB_PORT = var.db_port API_KEY = var.api_key SECRET_KEY = var.secret_key DOMAIN_NAME = var.domain_name EPC_AUTH_TOKEN = var.epc_auth_token GOOGLE_SOLAR_API_KEY = var.google_solar_api_key - # Buckets + # Buckets - from terraform state PLAN_TRIGGER_BUCKET = var.plan_trigger_bucket DATA_BUCKET = var.data_bucket PREDICTIONS_BUCKET = var.predictions_bucket - SAP_PREDICTIONS_BUCKET = var.sap_predictions_bucket - CARBON_PREDICTIONS_BUCKET = var.carbon_predictions_bucket - HEAT_PREDICTIONS_BUCKET = var.heat_predictions_bucket - HEATING_KWH_PREDICTIONS_BUCKET = var.heating_kwh_predictions_bucket - HOTWATER_KWH_PREDICTIONS_BUCKET = var.hotwater_kwh_predictions_bucket - ENERGY_ASSESSMENTS_BUCKET = var.energy_assessments_bucket + SAP_PREDICTIONS_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_sap_predictions_bucket_name + CARBON_PREDICTIONS_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_carbon_predictions_bucket_name + HEAT_PREDICTIONS_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_heat_predictions_bucket_name + HEATING_KWH_PREDICTIONS_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_heating_kwh_predictions_bucket_name + HOTWATER_KWH_PREDICTIONS_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_hotwater_kwh_predictions_bucket_name + ENERGY_ASSESSMENTS_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_energy_assessments_bucket_name # SQS ENGINE_SQS_URL = module.lambda.sqs_queue_url diff --git a/infrastructure/terraform/lambda/engine/variables.tf b/infrastructure/terraform/lambda/engine/variables.tf index 189cc848..0a2277ff 100644 --- a/infrastructure/terraform/lambda/engine/variables.tf +++ b/infrastructure/terraform/lambda/engine/variables.tf @@ -59,30 +59,6 @@ variable "predictions_bucket" { type = string } -variable "sap_predictions_bucket" { - type = string -} - -variable "carbon_predictions_bucket" { - type = string -} - -variable "heat_predictions_bucket" { - type = string -} - -variable "heating_kwh_predictions_bucket" { - type = string -} - -variable "hotwater_kwh_predictions_bucket" { - type = string -} - -variable "energy_assessments_bucket" { - type = string -} - locals { image_uri = "${var.ecr_repo_url}@${var.image_digest}" } diff --git a/infrastructure/terraform/shared/main.tf b/infrastructure/terraform/shared/main.tf index c7ed5a1f..02431802 100644 --- a/infrastructure/terraform/shared/main.tf +++ b/infrastructure/terraform/shared/main.tf @@ -134,6 +134,11 @@ module "retrofit_sap_predictions" { allowed_origins = var.allowed_origins } +output "retrofit_sap_predictions_bucket_name" { + value = module.retrofit_sap_predictions.bucket_name + description = "Name of the retrofit SAP predictions bucket" +} + module "retrofit_sap_data" { source = "../modules/s3" bucketname = "retrofit-data-${var.stage}" @@ -151,12 +156,22 @@ module "retrofit_carbon_predictions" { allowed_origins = var.allowed_origins } +output "retrofit_carbon_predictions_bucket_name" { + value = module.retrofit_carbon_predictions.bucket_name + description = "Name of the retrofit carbon predictions bucket" +} + module "retrofit_heat_predictions" { source = "../modules/s3" bucketname = "retrofit-heat-predictions-${var.stage}" allowed_origins = var.allowed_origins } +output "retrofit_heat_predictions_bucket_name" { + value = module.retrofit_heat_predictions.bucket_name + description = "Name of the retrofit heat predictions bucket" +} + module "retrofit_lighting_cost_predictions" { source = "../modules/s3" bucketname = "retrofit-lighting-cost-predictions-${var.stage}" @@ -181,12 +196,22 @@ module "retrofit_heating_kwh_predictions" { allowed_origins = var.allowed_origins } +output "retrofit_heating_kwh_predictions_bucket_name" { + value = module.retrofit_heating_kwh_predictions.bucket_name + description = "Name of the retrofit heating kWh predictions bucket" +} + module "retrofit_hotwater_kwh_predictions" { source = "../modules/s3" bucketname = "retrofit-hotwater-kwh-predictions-${var.stage}" allowed_origins = var.allowed_origins } +output "retrofit_hotwater_kwh_predictions_bucket_name" { + value = module.retrofit_hotwater_kwh_predictions.bucket_name + description = "Name of the retrofit hotwater kWh predictions bucket" +} + module "retrofit_sap_baseline_predictions" { source = "../modules/s3" bucketname = "retrofit-sap-baseline-predictions-${var.stage}" @@ -201,6 +226,11 @@ module "retrofit_energy_assessments" { environment = var.stage } +output "retrofit_energy_assessments_bucket_name" { + value = module.retrofit_energy_assessments.bucket_name + description = "Name of the retrofit energy assessments bucket" +} + # Set up the route53 record for the API module "route53" { source = "../modules/route53" From 30c3a773688ce93206d811c5cd37e4da6a82e4aa Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Wed, 4 Mar 2026 15:57:01 +0000 Subject: [PATCH 3/9] remove bucket names as env vars in deploy_lambda --- .github/workflows/_deploy_lambda.yml | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/.github/workflows/_deploy_lambda.yml b/.github/workflows/_deploy_lambda.yml index 51024bd4..05ecc751 100644 --- a/.github/workflows/_deploy_lambda.yml +++ b/.github/workflows/_deploy_lambda.yml @@ -58,18 +58,6 @@ on: required: false TF_VAR_predictions_bucket: required: false - TF_VAR_sap_predictions_bucket: - required: false - TF_VAR_carbon_predictions_bucket: - required: false - TF_VAR_heat_predictions_bucket: - required: false - TF_VAR_heating_kwh_predictions_bucket: - required: false - TF_VAR_hotwater_kwh_predictions_bucket: - required: false - TF_VAR_energy_assessments_bucket: - required: false jobs: deploy: @@ -127,12 +115,6 @@ jobs: TF_VAR_plan_trigger_bucket: ${{ secrets.TF_VAR_plan_trigger_bucket }} TF_VAR_data_bucket: ${{ secrets.TF_VAR_data_bucket }} TF_VAR_predictions_bucket: ${{ secrets.TF_VAR_predictions_bucket }} - TF_VAR_sap_predictions_bucket: ${{ secrets.TF_VAR_sap_predictions_bucket }} - TF_VAR_carbon_predictions_bucket: ${{ secrets.TF_VAR_carbon_predictions_bucket }} - TF_VAR_heat_predictions_bucket: ${{ secrets.TF_VAR_heat_predictions_bucket }} - TF_VAR_heating_kwh_predictions_bucket: ${{ secrets.TF_VAR_heating_kwh_predictions_bucket }} - TF_VAR_hotwater_kwh_predictions_bucket: ${{ secrets.TF_VAR_hotwater_kwh_predictions_bucket }} - TF_VAR_energy_assessments_bucket: ${{ secrets.TF_VAR_energy_assessments_bucket }} run: | terraform plan \ -var="stage=${{ inputs.stage }}" \ @@ -158,12 +140,6 @@ jobs: TF_VAR_plan_trigger_bucket: ${{ secrets.TF_VAR_plan_trigger_bucket }} TF_VAR_data_bucket: ${{ secrets.TF_VAR_data_bucket }} TF_VAR_predictions_bucket: ${{ secrets.TF_VAR_predictions_bucket }} - TF_VAR_sap_predictions_bucket: ${{ secrets.TF_VAR_sap_predictions_bucket }} - TF_VAR_carbon_predictions_bucket: ${{ secrets.TF_VAR_carbon_predictions_bucket }} - TF_VAR_heat_predictions_bucket: ${{ secrets.TF_VAR_heat_predictions_bucket }} - TF_VAR_heating_kwh_predictions_bucket: ${{ secrets.TF_VAR_heating_kwh_predictions_bucket }} - TF_VAR_hotwater_kwh_predictions_bucket: ${{ secrets.TF_VAR_hotwater_kwh_predictions_bucket }} - TF_VAR_energy_assessments_bucket: ${{ secrets.TF_VAR_energy_assessments_bucket }} run: | terraform destroy -auto-approve \ -var="stage=${{ inputs.stage }}" \ From 19656f20a4f19d673c829de48b0421b4ff4c201f Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Wed, 4 Mar 2026 16:12:25 +0000 Subject: [PATCH 4/9] remove additional bucket names as env vars in deploy_lambda --- .github/workflows/_deploy_lambda.yml | 8 -------- infrastructure/terraform/lambda/engine/main.tf | 4 ++-- infrastructure/terraform/shared/main.tf | 5 +++++ 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/.github/workflows/_deploy_lambda.yml b/.github/workflows/_deploy_lambda.yml index 05ecc751..3b91f106 100644 --- a/.github/workflows/_deploy_lambda.yml +++ b/.github/workflows/_deploy_lambda.yml @@ -52,10 +52,6 @@ on: required: false TF_VAR_google_solar_api_key: required: false - TF_VAR_plan_trigger_bucket: - required: false - TF_VAR_data_bucket: - required: false TF_VAR_predictions_bucket: required: false @@ -112,8 +108,6 @@ jobs: TF_VAR_domain_name: ${{ secrets.TF_VAR_domain_name }} TF_VAR_epc_auth_token: ${{ secrets.TF_VAR_epc_auth_token }} TF_VAR_google_solar_api_key: ${{ secrets.TF_VAR_google_solar_api_key }} - TF_VAR_plan_trigger_bucket: ${{ secrets.TF_VAR_plan_trigger_bucket }} - TF_VAR_data_bucket: ${{ secrets.TF_VAR_data_bucket }} TF_VAR_predictions_bucket: ${{ secrets.TF_VAR_predictions_bucket }} run: | terraform plan \ @@ -137,8 +131,6 @@ jobs: TF_VAR_domain_name: ${{ secrets.TF_VAR_domain_name }} TF_VAR_epc_auth_token: ${{ secrets.TF_VAR_epc_auth_token }} TF_VAR_google_solar_api_key: ${{ secrets.TF_VAR_google_solar_api_key }} - TF_VAR_plan_trigger_bucket: ${{ secrets.TF_VAR_plan_trigger_bucket }} - TF_VAR_data_bucket: ${{ secrets.TF_VAR_data_bucket }} TF_VAR_predictions_bucket: ${{ secrets.TF_VAR_predictions_bucket }} run: | terraform destroy -auto-approve \ diff --git a/infrastructure/terraform/lambda/engine/main.tf b/infrastructure/terraform/lambda/engine/main.tf index 6d0603e6..6c3b89e3 100644 --- a/infrastructure/terraform/lambda/engine/main.tf +++ b/infrastructure/terraform/lambda/engine/main.tf @@ -47,8 +47,8 @@ module "lambda" { GOOGLE_SOLAR_API_KEY = var.google_solar_api_key # Buckets - from terraform state - PLAN_TRIGGER_BUCKET = var.plan_trigger_bucket - DATA_BUCKET = var.data_bucket + PLAN_TRIGGER_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_plan_trigger_bucket_name + DATA_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_sap_data_bucket_name PREDICTIONS_BUCKET = var.predictions_bucket SAP_PREDICTIONS_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_sap_predictions_bucket_name CARBON_PREDICTIONS_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_carbon_predictions_bucket_name diff --git a/infrastructure/terraform/shared/main.tf b/infrastructure/terraform/shared/main.tf index 02431802..96097690 100644 --- a/infrastructure/terraform/shared/main.tf +++ b/infrastructure/terraform/shared/main.tf @@ -102,6 +102,11 @@ module "s3_presignable_bucket" { allowed_origins = var.allowed_origins } +output "retrofit_plan_trigger_bucket_name" { + value = module.s3_presignable_bucket.bucket_name + description = "Name of the retrofit plan trigger bucket" +} + module "s3_due_considerations_bucket" { source = "../modules/s3_presignable_bucket" bucketname = "retrofit-due-considerations-${var.stage}" From b7c963eb2c3c260227460eb25d97f36d030fc6a9 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Wed, 4 Mar 2026 16:15:22 +0000 Subject: [PATCH 5/9] remove bucket names as secrets in deploy_terraform --- .github/workflows/deploy_terraform.yml | 10 +--------- infrastructure/terraform/lambda/engine/main.tf | 2 +- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 4b0adbac..5fbd2d83 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -274,12 +274,4 @@ jobs: TF_VAR_domain_name: ${{ secrets.DEV_DOMAIN_NAME }} TF_VAR_epc_auth_token: ${{ secrets.DEV_EPC_AUTH_TOKEN }} TF_VAR_google_solar_api_key: ${{ secrets.DEV_GOOGLE_SOLAR_API_KEY }} - TF_VAR_plan_trigger_bucket: ${{ secrets.DEV_PLAN_TRIGGER_BUCKET }} - TF_VAR_data_bucket: ${{ secrets.DEV_DATA_BUCKET }} - TF_VAR_predictions_bucket: ${{ secrets.DEV_PREDICTIONS_BUCKET }} - TF_VAR_sap_predictions_bucket: ${{ secrets.DEV_SAP_PREDICTIONS_BUCKET }} - TF_VAR_carbon_predictions_bucket: ${{ secrets.DEV_CARBON_PREDICTIONS_BUCKET }} - TF_VAR_heat_predictions_bucket: ${{ secrets.DEV_HEAT_PREDICTIONS_BUCKET }} - TF_VAR_heating_kwh_predictions_bucket: ${{ secrets.DEV_HEATING_KWH_PREDICTIONS_BUCKET }} - TF_VAR_hotwater_kwh_predictions_bucket: ${{ secrets.DEV_HOTWATER_KWH_PREDICTIONS_BUCKET }} - TF_VAR_energy_assessments_bucket: ${{ secrets.DEV_ENERGY_ASSESSMENTS_BUCKET }} \ No newline at end of file + TF_VAR_predictions_bucket: ${{ secrets.DEV_PREDICTIONS_BUCKET }} \ No newline at end of file diff --git a/infrastructure/terraform/lambda/engine/main.tf b/infrastructure/terraform/lambda/engine/main.tf index 6c3b89e3..ee1bf2e2 100644 --- a/infrastructure/terraform/lambda/engine/main.tf +++ b/infrastructure/terraform/lambda/engine/main.tf @@ -45,11 +45,11 @@ module "lambda" { DOMAIN_NAME = var.domain_name EPC_AUTH_TOKEN = var.epc_auth_token GOOGLE_SOLAR_API_KEY = var.google_solar_api_key + PREDICTIONS_BUCKET = var.predictions_bucket # Buckets - from terraform state PLAN_TRIGGER_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_plan_trigger_bucket_name DATA_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_sap_data_bucket_name - PREDICTIONS_BUCKET = var.predictions_bucket SAP_PREDICTIONS_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_sap_predictions_bucket_name CARBON_PREDICTIONS_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_carbon_predictions_bucket_name HEAT_PREDICTIONS_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_heat_predictions_bucket_name From 4b3621578883a4f9a3a8be2c92d83a33b3aa145d Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Wed, 4 Mar 2026 16:42:14 +0000 Subject: [PATCH 6/9] give engine permission to read and write necessary s3 buckets --- infrastructure/terraform/shared/main.tf | 27 ++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/infrastructure/terraform/shared/main.tf b/infrastructure/terraform/shared/main.tf index 96097690..3253e8e0 100644 --- a/infrastructure/terraform/shared/main.tf +++ b/infrastructure/terraform/shared/main.tf @@ -464,4 +464,29 @@ module "engine_registry" { source = "../modules/container_registry" name = "engine" stage = var.stage -} \ No newline at end of file +} + +# S3 policy for Engine to read and write from various S3 buckets +module "engine_s3_read_and_write" { + source = "../modules/s3_iam_policy" + + policy_name = "EngineReadandWriteS3" + policy_description = "Allow Engine Lambda to read from and write to various S3 buckets" + bucket_arns = [ + "arn:aws:s3:::retrofit-plan-inputs-${var.stage}", + "arn:aws:s3:::retrofit-data-${var.stage}", + "arn:aws:s3:::retrofit-sap-predictions-${var.stage}", + "arn:aws:s3:::retrofit-carbon-predictions-${var.stage}", + "arn:aws:s3:::retrofit-heat-predictions-${var.stage}", + "arn:aws:s3:::retrofit-carbon-predictions-${var.stage}", + "arn:aws:s3:::retrofit-heating-kwh-predictions-${var.stage}", + "arn:aws:s3:::retrofit-hotwater-kwh-predictions-${var.stage}", + "arn:aws:s3:::retrofit-energy-assessments-${var.stage}" + ] + actions = ["s3:*"] + resource_paths = ["/*"] +} + +output "engine_s3_read_and_write_arn" { + value = module.engine_s3_read_and_write.policy_arn +} From dad4d6d55cb4919526bd917c44d3ccbc5b17c616 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Wed, 4 Mar 2026 16:48:14 +0000 Subject: [PATCH 7/9] give engine permission to read and write necessary s3 buckets --- infrastructure/terraform/lambda/engine/main.tf | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/infrastructure/terraform/lambda/engine/main.tf b/infrastructure/terraform/lambda/engine/main.tf index ee1bf2e2..23c5261e 100644 --- a/infrastructure/terraform/lambda/engine/main.tf +++ b/infrastructure/terraform/lambda/engine/main.tf @@ -67,4 +67,8 @@ module "lambda" { ) } -# Policies and IAM \ No newline at end of file +# Policies and IAM +resource "aws_iam-role_policy_attachment" "engine_s3_read_and_write" { + role = module.lambda.role_name + policy_arn = data.terraform_remote_state.shared.outputs.engine_s3_read_and_write_arn +} \ No newline at end of file From 770e19c59948ebab23ba855e2f8143ac0367c604 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Thu, 5 Mar 2026 09:57:33 +0000 Subject: [PATCH 8/9] in policies - reference s3 buckets from the defined modules instead of hardcoding bucket names --- infrastructure/terraform/lambda/engine/main.tf | 10 ++++++++-- infrastructure/terraform/shared/main.tf | 17 ++++++++--------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/infrastructure/terraform/lambda/engine/main.tf b/infrastructure/terraform/lambda/engine/main.tf index 23c5261e..6f6b20ce 100644 --- a/infrastructure/terraform/lambda/engine/main.tf +++ b/infrastructure/terraform/lambda/engine/main.tf @@ -67,8 +67,14 @@ module "lambda" { ) } -# Policies and IAM +### Policies and IAM +# S3 resource "aws_iam-role_policy_attachment" "engine_s3_read_and_write" { role = module.lambda.role_name policy_arn = data.terraform_remote_state.shared.outputs.engine_s3_read_and_write_arn -} \ No newline at end of file +} + +# Logs + + +# SQS diff --git a/infrastructure/terraform/shared/main.tf b/infrastructure/terraform/shared/main.tf index 3253e8e0..c19e3a0c 100644 --- a/infrastructure/terraform/shared/main.tf +++ b/infrastructure/terraform/shared/main.tf @@ -473,15 +473,14 @@ module "engine_s3_read_and_write" { policy_name = "EngineReadandWriteS3" policy_description = "Allow Engine Lambda to read from and write to various S3 buckets" bucket_arns = [ - "arn:aws:s3:::retrofit-plan-inputs-${var.stage}", - "arn:aws:s3:::retrofit-data-${var.stage}", - "arn:aws:s3:::retrofit-sap-predictions-${var.stage}", - "arn:aws:s3:::retrofit-carbon-predictions-${var.stage}", - "arn:aws:s3:::retrofit-heat-predictions-${var.stage}", - "arn:aws:s3:::retrofit-carbon-predictions-${var.stage}", - "arn:aws:s3:::retrofit-heating-kwh-predictions-${var.stage}", - "arn:aws:s3:::retrofit-hotwater-kwh-predictions-${var.stage}", - "arn:aws:s3:::retrofit-energy-assessments-${var.stage}" + "arn:aws:s3:::${module.s3_presignable_bucket.bucket_name}", + "arn:aws:s3:::${module.retrofit_sap_data.bucket_name}", + "arn:aws:s3:::${module.retrofit_sap_predictions.bucket_name}", + "arn:aws:s3:::${module.retrofit_carbon_predictions.bucket_name}", + "arn:aws:s3:::${module.retrofit_heat_predictions.bucket_name}", + "arn:aws:s3:::${module.retrofit_heating_kwh_predictions.bucket_name}", + "arn:aws:s3:::${module.retrofit_hotwater_kwh_predictions.bucket_name}", + "arn:aws:s3:::${module.retrofit_energy_assessments.bucket_name}" ] actions = ["s3:*"] resource_paths = ["/*"] From 83df8e856e46c4fc8e8eb4da4081ecaf0c511355 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Thu, 5 Mar 2026 10:21:11 +0000 Subject: [PATCH 9/9] Fix typo --- infrastructure/terraform/lambda/engine/main.tf | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/infrastructure/terraform/lambda/engine/main.tf b/infrastructure/terraform/lambda/engine/main.tf index 6f6b20ce..2302aaf6 100644 --- a/infrastructure/terraform/lambda/engine/main.tf +++ b/infrastructure/terraform/lambda/engine/main.tf @@ -69,12 +69,7 @@ module "lambda" { ### Policies and IAM # S3 -resource "aws_iam-role_policy_attachment" "engine_s3_read_and_write" { +resource "aws_iam_role_policy_attachment" "engine_s3_read_and_write" { role = module.lambda.role_name policy_arn = data.terraform_remote_state.shared.outputs.engine_s3_read_and_write_arn -} - -# Logs - - -# SQS +} \ No newline at end of file