diff --git a/.github/workflows/_deploy_lambda.yml b/.github/workflows/_deploy_lambda.yml index 528300f8..3b91f106 100644 --- a/.github/workflows/_deploy_lambda.yml +++ b/.github/workflows/_deploy_lambda.yml @@ -42,6 +42,18 @@ on: required: true AWS_REGION: required: true + TF_VAR_api_key: + required: false + TF_VAR_secret_key: + required: false + TF_VAR_domain_name: + required: false + TF_VAR_epc_auth_token: + required: false + TF_VAR_google_solar_api_key: + required: false + TF_VAR_predictions_bucket: + required: false jobs: deploy: @@ -90,6 +102,13 @@ jobs: - name: Terraform Plan working-directory: ${{ inputs.lambda_path }} + env: + TF_VAR_api_key: ${{ secrets.TF_VAR_api_key }} + TF_VAR_secret_key: ${{ secrets.TF_VAR_secret_key }} + TF_VAR_domain_name: ${{ secrets.TF_VAR_domain_name }} + TF_VAR_epc_auth_token: ${{ secrets.TF_VAR_epc_auth_token }} + TF_VAR_google_solar_api_key: ${{ secrets.TF_VAR_google_solar_api_key }} + TF_VAR_predictions_bucket: ${{ secrets.TF_VAR_predictions_bucket }} run: | terraform plan \ -var="stage=${{ inputs.stage }}" \ @@ -106,10 +125,16 @@ jobs: - name: Terraform Destroy if: inputs.terraform_destroy == 'true' && inputs.terraform_apply != 'true' working-directory: ${{ inputs.lambda_path }} + env: + TF_VAR_api_key: ${{ secrets.TF_VAR_api_key }} + TF_VAR_secret_key: ${{ secrets.TF_VAR_secret_key }} + TF_VAR_domain_name: ${{ secrets.TF_VAR_domain_name }} + TF_VAR_epc_auth_token: ${{ secrets.TF_VAR_epc_auth_token }} + TF_VAR_google_solar_api_key: ${{ secrets.TF_VAR_google_solar_api_key }} + TF_VAR_predictions_bucket: ${{ secrets.TF_VAR_predictions_bucket }} run: | terraform destroy -auto-approve \ -var="stage=${{ inputs.stage }}" \ -var="lambda_name=${{ inputs.lambda_name }}" \ -var="ecr_repo_url=${{ steps.repo.outputs.ecr_repo_url }}" \ -var="image_digest=${{ inputs.image_digest }}" - diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 4c9ce44a..5fbd2d83 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -241,4 +241,37 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + # ============================================================ + # Ara Engine image and Push + # ============================================================ + ara_engine_image: + needs: [determine_stage, shared_terraform] + uses: ./.github/workflows/_build_image.yml + with: + ecr_repo: engine-${{ needs.determine_stage.outputs.stage }} + dockerfile_path: backend/docker/engine.Dockerfile + build_context: . + # ============================================================ + # Deploy Categorisation Lambda + # ============================================================ + ara_engine_lambda: + needs: [ara_engine_image, determine_stage] + uses: ./.github/workflows/_deploy_lambda.yml + with: + lambda_name: ara_engine + lambda_path: infrastructure/terraform/lambda/engine + stage: ${{ needs.determine_stage.outputs.stage }} + ecr_repo: engine-${{ needs.determine_stage.outputs.stage }} + image_digest: ${{ needs.ara_engine_image.outputs.image_digest }} + terraform_apply: ${{ needs.determine_stage.outputs.terraform_apply }} + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + TF_VAR_api_key: ${{ secrets.DEV_API_KEY }} + TF_VAR_secret_key: ${{ secrets.DEV_SECRET_KEY }} + TF_VAR_domain_name: ${{ secrets.DEV_DOMAIN_NAME }} + TF_VAR_epc_auth_token: ${{ secrets.DEV_EPC_AUTH_TOKEN }} + TF_VAR_google_solar_api_key: ${{ secrets.DEV_GOOGLE_SOLAR_API_KEY }} + TF_VAR_predictions_bucket: ${{ secrets.DEV_PREDICTIONS_BUCKET }} \ No newline at end of file diff --git a/infrastructure/terraform/lambda/engine/main.tf b/infrastructure/terraform/lambda/engine/main.tf index c1cff8a3..2302aaf6 100644 --- a/infrastructure/terraform/lambda/engine/main.tf +++ b/infrastructure/terraform/lambda/engine/main.tf @@ -7,6 +7,15 @@ data "terraform_remote_state" "shared" { } } +data "aws_secretsmanager_secret_version" "db_credentials" { + secret_id = "${var.stage}/assessment_model/db_credentials" +} + +locals { + db_credentials = jsondecode(data.aws_secretsmanager_secret_version.db_credentials.secret_string) +} + + module "lambda" { source = "../modules/lambda_with_sqs" @@ -18,8 +27,49 @@ module "lambda" { # Optional: Set maximum_concurrency to limit concurrent SQS-triggered invocations (2-1000) maximum_concurrency = var.maximum_concurrency - environment = { - STAGE = var.stage - LOG_LEVEL = "info" - } + environment = merge( + { + STAGE = var.stage + LOG_LEVEL = "info" + + # DB from Secrets Manager + DB_USERNAME = local.db_credentials.db_assessment_model_username + DB_PASSWORD = local.db_credentials.db_assessment_model_password + + # Secrets from GitHub + DB_HOST = var.db_host + DB_NAME = var.db_name + DB_PORT = var.db_port + API_KEY = var.api_key + SECRET_KEY = var.secret_key + DOMAIN_NAME = var.domain_name + EPC_AUTH_TOKEN = var.epc_auth_token + GOOGLE_SOLAR_API_KEY = var.google_solar_api_key + PREDICTIONS_BUCKET = var.predictions_bucket + + # Buckets - from terraform state + PLAN_TRIGGER_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_plan_trigger_bucket_name + DATA_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_sap_data_bucket_name + SAP_PREDICTIONS_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_sap_predictions_bucket_name + CARBON_PREDICTIONS_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_carbon_predictions_bucket_name + HEAT_PREDICTIONS_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_heat_predictions_bucket_name + HEATING_KWH_PREDICTIONS_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_heating_kwh_predictions_bucket_name + HOTWATER_KWH_PREDICTIONS_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_hotwater_kwh_predictions_bucket_name + ENERGY_ASSESSMENTS_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_energy_assessments_bucket_name + + # SQS + ENGINE_SQS_URL = module.lambda.sqs_queue_url + + # Deployment + ECR_URI = var.ecr_repo_url + GITHUB_SHA = var.image_digest + } + ) +} + +### Policies and IAM +# S3 +resource "aws_iam_role_policy_attachment" "engine_s3_read_and_write" { + role = module.lambda.role_name + policy_arn = data.terraform_remote_state.shared.outputs.engine_s3_read_and_write_arn } \ No newline at end of file diff --git a/infrastructure/terraform/lambda/engine/variables.tf b/infrastructure/terraform/lambda/engine/variables.tf index 503bf6c8..0a2277ff 100644 --- a/infrastructure/terraform/lambda/engine/variables.tf +++ b/infrastructure/terraform/lambda/engine/variables.tf @@ -23,10 +23,46 @@ variable "maximum_concurrency" { description = "Maximum number of concurrent Lambda invocations from SQS (2-1000). null = no limit." } +variable "api_key" { + type = string + sensitive = true +} + +variable "secret_key" { + type = string + sensitive = true +} + +variable "domain_name" { + type = string +} + +variable "epc_auth_token" { + type = string + sensitive = true +} + +variable "google_solar_api_key" { + type = string + sensitive = true +} + +variable "plan_trigger_bucket" { + type = string +} + +variable "data_bucket" { + type = string +} + +variable "predictions_bucket" { + type = string +} + locals { image_uri = "${var.ecr_repo_url}@${var.image_digest}" } output "resolved_image_uri" { value = local.image_uri -} +} \ No newline at end of file diff --git a/infrastructure/terraform/shared/main.tf b/infrastructure/terraform/shared/main.tf index c7ed5a1f..c19e3a0c 100644 --- a/infrastructure/terraform/shared/main.tf +++ b/infrastructure/terraform/shared/main.tf @@ -102,6 +102,11 @@ module "s3_presignable_bucket" { allowed_origins = var.allowed_origins } +output "retrofit_plan_trigger_bucket_name" { + value = module.s3_presignable_bucket.bucket_name + description = "Name of the retrofit plan trigger bucket" +} + module "s3_due_considerations_bucket" { source = "../modules/s3_presignable_bucket" bucketname = "retrofit-due-considerations-${var.stage}" @@ -134,6 +139,11 @@ module "retrofit_sap_predictions" { allowed_origins = var.allowed_origins } +output "retrofit_sap_predictions_bucket_name" { + value = module.retrofit_sap_predictions.bucket_name + description = "Name of the retrofit SAP predictions bucket" +} + module "retrofit_sap_data" { source = "../modules/s3" bucketname = "retrofit-data-${var.stage}" @@ -151,12 +161,22 @@ module "retrofit_carbon_predictions" { allowed_origins = var.allowed_origins } +output "retrofit_carbon_predictions_bucket_name" { + value = module.retrofit_carbon_predictions.bucket_name + description = "Name of the retrofit carbon predictions bucket" +} + module "retrofit_heat_predictions" { source = "../modules/s3" bucketname = "retrofit-heat-predictions-${var.stage}" allowed_origins = var.allowed_origins } +output "retrofit_heat_predictions_bucket_name" { + value = module.retrofit_heat_predictions.bucket_name + description = "Name of the retrofit heat predictions bucket" +} + module "retrofit_lighting_cost_predictions" { source = "../modules/s3" bucketname = "retrofit-lighting-cost-predictions-${var.stage}" @@ -181,12 +201,22 @@ module "retrofit_heating_kwh_predictions" { allowed_origins = var.allowed_origins } +output "retrofit_heating_kwh_predictions_bucket_name" { + value = module.retrofit_heating_kwh_predictions.bucket_name + description = "Name of the retrofit heating kWh predictions bucket" +} + module "retrofit_hotwater_kwh_predictions" { source = "../modules/s3" bucketname = "retrofit-hotwater-kwh-predictions-${var.stage}" allowed_origins = var.allowed_origins } +output "retrofit_hotwater_kwh_predictions_bucket_name" { + value = module.retrofit_hotwater_kwh_predictions.bucket_name + description = "Name of the retrofit hotwater kWh predictions bucket" +} + module "retrofit_sap_baseline_predictions" { source = "../modules/s3" bucketname = "retrofit-sap-baseline-predictions-${var.stage}" @@ -201,6 +231,11 @@ module "retrofit_energy_assessments" { environment = var.stage } +output "retrofit_energy_assessments_bucket_name" { + value = module.retrofit_energy_assessments.bucket_name + description = "Name of the retrofit energy assessments bucket" +} + # Set up the route53 record for the API module "route53" { source = "../modules/route53" @@ -429,4 +464,28 @@ module "engine_registry" { source = "../modules/container_registry" name = "engine" stage = var.stage -} \ No newline at end of file +} + +# S3 policy for Engine to read and write from various S3 buckets +module "engine_s3_read_and_write" { + source = "../modules/s3_iam_policy" + + policy_name = "EngineReadandWriteS3" + policy_description = "Allow Engine Lambda to read from and write to various S3 buckets" + bucket_arns = [ + "arn:aws:s3:::${module.s3_presignable_bucket.bucket_name}", + "arn:aws:s3:::${module.retrofit_sap_data.bucket_name}", + "arn:aws:s3:::${module.retrofit_sap_predictions.bucket_name}", + "arn:aws:s3:::${module.retrofit_carbon_predictions.bucket_name}", + "arn:aws:s3:::${module.retrofit_heat_predictions.bucket_name}", + "arn:aws:s3:::${module.retrofit_heating_kwh_predictions.bucket_name}", + "arn:aws:s3:::${module.retrofit_hotwater_kwh_predictions.bucket_name}", + "arn:aws:s3:::${module.retrofit_energy_assessments.bucket_name}" + ] + actions = ["s3:*"] + resource_paths = ["/*"] +} + +output "engine_s3_read_and_write_arn" { + value = module.engine_s3_read_and_write.policy_arn +}