Merge pull request #770 from Hestia-Homes/deploy-backend-with-terraform

Deploy backend with terraform #2: Deploy Engine

.github/workflows/_deploy_lambda.yml

@@ -42,6 +42,18 @@ on:
         required: true
       AWS_REGION:
         required: true
+      TF_VAR_api_key:
+        required: false
+      TF_VAR_secret_key:
+        required: false
+      TF_VAR_domain_name:
+        required: false
+      TF_VAR_epc_auth_token:
+        required: false
+      TF_VAR_google_solar_api_key:
+        required: false
+      TF_VAR_predictions_bucket:
+        required: false
 
 jobs:
   deploy:
@@ -90,6 +102,13 @@ jobs:
 
       - name: Terraform Plan
         working-directory: ${{ inputs.lambda_path }}
+        env:
+          TF_VAR_api_key: ${{ secrets.TF_VAR_api_key }}
+          TF_VAR_secret_key: ${{ secrets.TF_VAR_secret_key }}
+          TF_VAR_domain_name: ${{ secrets.TF_VAR_domain_name }}
+          TF_VAR_epc_auth_token: ${{ secrets.TF_VAR_epc_auth_token }}
+          TF_VAR_google_solar_api_key: ${{ secrets.TF_VAR_google_solar_api_key }}
+          TF_VAR_predictions_bucket: ${{ secrets.TF_VAR_predictions_bucket }}
         run: |
           terraform plan \
             -var="stage=${{ inputs.stage }}" \
@@ -106,10 +125,16 @@ jobs:
       - name: Terraform Destroy
         if: inputs.terraform_destroy == 'true' && inputs.terraform_apply != 'true'
         working-directory: ${{ inputs.lambda_path }}
+        env:
+          TF_VAR_api_key: ${{ secrets.TF_VAR_api_key }}
+          TF_VAR_secret_key: ${{ secrets.TF_VAR_secret_key }}
+          TF_VAR_domain_name: ${{ secrets.TF_VAR_domain_name }}
+          TF_VAR_epc_auth_token: ${{ secrets.TF_VAR_epc_auth_token }}
+          TF_VAR_google_solar_api_key: ${{ secrets.TF_VAR_google_solar_api_key }}
+          TF_VAR_predictions_bucket: ${{ secrets.TF_VAR_predictions_bucket }}
         run: |
           terraform destroy -auto-approve \
             -var="stage=${{ inputs.stage }}" \
             -var="lambda_name=${{ inputs.lambda_name }}" \
             -var="ecr_repo_url=${{ steps.repo.outputs.ecr_repo_url }}" \
             -var="image_digest=${{ inputs.image_digest }}"
-

.github/workflows/deploy_terraform.yml

@@ -241,4 +241,37 @@ jobs:
       AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }}
       AWS_REGION: ${{ secrets.DEV_AWS_REGION }}
 
+  # ============================================================
+  # Ara Engine image and Push
+  # ============================================================
+  ara_engine_image:
+    needs: [determine_stage, shared_terraform]
+    uses: ./.github/workflows/_build_image.yml
+    with:
+      ecr_repo: engine-${{ needs.determine_stage.outputs.stage }}
+      dockerfile_path: backend/docker/engine.Dockerfile
+      build_context: .
 
+  # ============================================================
+  # Deploy Categorisation Lambda
+  # ============================================================
+  ara_engine_lambda:
+    needs: [ara_engine_image, determine_stage]
+    uses: ./.github/workflows/_deploy_lambda.yml
+    with:
+      lambda_name: ara_engine
+      lambda_path: infrastructure/terraform/lambda/engine
+      stage: ${{ needs.determine_stage.outputs.stage }}
+      ecr_repo: engine-${{ needs.determine_stage.outputs.stage }}
+      image_digest: ${{ needs.ara_engine_image.outputs.image_digest }}
+      terraform_apply: ${{ needs.determine_stage.outputs.terraform_apply }}
+    secrets:
+      AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }}
+      AWS_REGION: ${{ secrets.DEV_AWS_REGION }}
+      TF_VAR_api_key: ${{ secrets.DEV_API_KEY }}
+      TF_VAR_secret_key: ${{ secrets.DEV_SECRET_KEY }}
+      TF_VAR_domain_name: ${{ secrets.DEV_DOMAIN_NAME }}
+      TF_VAR_epc_auth_token: ${{ secrets.DEV_EPC_AUTH_TOKEN }}
+      TF_VAR_google_solar_api_key: ${{ secrets.DEV_GOOGLE_SOLAR_API_KEY }}
+      TF_VAR_predictions_bucket: ${{ secrets.DEV_PREDICTIONS_BUCKET }}

infrastructure/terraform/lambda/engine/main.tf

@@ -7,6 +7,15 @@ data "terraform_remote_state" "shared" {
   }
 }
 
+data "aws_secretsmanager_secret_version" "db_credentials" {
+  secret_id = "${var.stage}/assessment_model/db_credentials"
+}
+
+locals {
+  db_credentials = jsondecode(data.aws_secretsmanager_secret_version.db_credentials.secret_string)
+}
+
+
 module "lambda" {
   source = "../modules/lambda_with_sqs"
 
@@ -18,8 +27,49 @@ module "lambda" {
   # Optional: Set maximum_concurrency to limit concurrent SQS-triggered invocations (2-1000)
   maximum_concurrency = var.maximum_concurrency
 
-  environment = {
+  environment = merge(
-    STAGE = var.stage
+    {
-    LOG_LEVEL = "info"
+      STAGE      = var.stage
-  }
+      LOG_LEVEL  = "info"
+
+      # DB from Secrets Manager
+      DB_USERNAME = local.db_credentials.db_assessment_model_username
+      DB_PASSWORD = local.db_credentials.db_assessment_model_password
+
+      # Secrets from GitHub
+      DB_HOST              = var.db_host
+      DB_NAME              = var.db_name
+      DB_PORT              = var.db_port
+      API_KEY              = var.api_key
+      SECRET_KEY           = var.secret_key
+      DOMAIN_NAME          = var.domain_name
+      EPC_AUTH_TOKEN       = var.epc_auth_token
+      GOOGLE_SOLAR_API_KEY = var.google_solar_api_key
+      PREDICTIONS_BUCKET   = var.predictions_bucket
+
+      # Buckets - from terraform state
+      PLAN_TRIGGER_BUCKET             = data.terraform_remote_state.shared.outputs.retrofit_plan_trigger_bucket_name
+      DATA_BUCKET                     = data.terraform_remote_state.shared.outputs.retrofit_sap_data_bucket_name
+      SAP_PREDICTIONS_BUCKET          = data.terraform_remote_state.shared.outputs.retrofit_sap_predictions_bucket_name
+      CARBON_PREDICTIONS_BUCKET       = data.terraform_remote_state.shared.outputs.retrofit_carbon_predictions_bucket_name
+      HEAT_PREDICTIONS_BUCKET         = data.terraform_remote_state.shared.outputs.retrofit_heat_predictions_bucket_name
+      HEATING_KWH_PREDICTIONS_BUCKET  = data.terraform_remote_state.shared.outputs.retrofit_heating_kwh_predictions_bucket_name
+      HOTWATER_KWH_PREDICTIONS_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_hotwater_kwh_predictions_bucket_name
+      ENERGY_ASSESSMENTS_BUCKET       = data.terraform_remote_state.shared.outputs.retrofit_energy_assessments_bucket_name
+
+      # SQS
+      ENGINE_SQS_URL         = module.lambda.sqs_queue_url
+
+      # Deployment
+      ECR_URI    = var.ecr_repo_url
+      GITHUB_SHA = var.image_digest
+    }
+  )
+}
+
+### Policies and IAM
+# S3
+resource "aws_iam_role_policy_attachment" "engine_s3_read_and_write" {
+  role = module.lambda.role_name
+  policy_arn = data.terraform_remote_state.shared.outputs.engine_s3_read_and_write_arn
 }

infrastructure/terraform/lambda/engine/variables.tf

@@ -23,6 +23,42 @@ variable "maximum_concurrency" {
   description = "Maximum number of concurrent Lambda invocations from SQS (2-1000). null = no limit."
 }
 
+variable "api_key" {
+  type      = string
+  sensitive = true
+}
+
+variable "secret_key" {
+  type      = string
+  sensitive = true
+}
+
+variable "domain_name" {
+  type = string
+}
+
+variable "epc_auth_token" {
+  type      = string
+  sensitive = true
+}
+
+variable "google_solar_api_key" {
+  type      = string
+  sensitive = true
+}
+
+variable "plan_trigger_bucket" {
+  type = string
+}
+
+variable "data_bucket" {
+  type = string
+}
+
+variable "predictions_bucket" {
+  type = string
+}
+
 locals {
   image_uri = "${var.ecr_repo_url}@${var.image_digest}"
 }

infrastructure/terraform/shared/main.tf

@@ -102,6 +102,11 @@ module "s3_presignable_bucket" {
   allowed_origins = var.allowed_origins
 }
 
+output "retrofit_plan_trigger_bucket_name" {
+  value = module.s3_presignable_bucket.bucket_name
+  description = "Name of the retrofit plan trigger bucket"
+}
+
 module "s3_due_considerations_bucket" {
   source          = "../modules/s3_presignable_bucket"
   bucketname      = "retrofit-due-considerations-${var.stage}"
@@ -134,6 +139,11 @@ module "retrofit_sap_predictions" {
   allowed_origins = var.allowed_origins
 }
 
+output "retrofit_sap_predictions_bucket_name" {
+  value = module.retrofit_sap_predictions.bucket_name
+  description = "Name of the retrofit SAP predictions bucket"
+}
+
 module "retrofit_sap_data" {
   source          = "../modules/s3"
   bucketname      = "retrofit-data-${var.stage}"
@@ -151,12 +161,22 @@ module "retrofit_carbon_predictions" {
   allowed_origins = var.allowed_origins
 }
 
+output "retrofit_carbon_predictions_bucket_name" {
+  value = module.retrofit_carbon_predictions.bucket_name
+  description = "Name of the retrofit carbon predictions bucket"
+}
+
 module "retrofit_heat_predictions" {
   source          = "../modules/s3"
   bucketname      = "retrofit-heat-predictions-${var.stage}"
   allowed_origins = var.allowed_origins
 }
 
+output "retrofit_heat_predictions_bucket_name" {
+  value = module.retrofit_heat_predictions.bucket_name
+  description = "Name of the retrofit heat predictions bucket"
+}
+
 module "retrofit_lighting_cost_predictions" {
   source          = "../modules/s3"
   bucketname      = "retrofit-lighting-cost-predictions-${var.stage}"
@@ -181,12 +201,22 @@ module "retrofit_heating_kwh_predictions" {
   allowed_origins = var.allowed_origins
 }
 
+output "retrofit_heating_kwh_predictions_bucket_name" {
+  value = module.retrofit_heating_kwh_predictions.bucket_name
+  description = "Name of the retrofit heating kWh predictions bucket"
+}
+
 module "retrofit_hotwater_kwh_predictions" {
   source          = "../modules/s3"
   bucketname      = "retrofit-hotwater-kwh-predictions-${var.stage}"
   allowed_origins = var.allowed_origins
 }
 
+output "retrofit_hotwater_kwh_predictions_bucket_name" {
+  value = module.retrofit_hotwater_kwh_predictions.bucket_name
+  description = "Name of the retrofit hotwater kWh predictions bucket"
+}
+
 module "retrofit_sap_baseline_predictions" {
   source          = "../modules/s3"
   bucketname      = "retrofit-sap-baseline-predictions-${var.stage}"
@@ -201,6 +231,11 @@ module "retrofit_energy_assessments" {
   environment     = var.stage
 }
 
+output "retrofit_energy_assessments_bucket_name" {
+  value = module.retrofit_energy_assessments.bucket_name
+  description = "Name of the retrofit energy assessments bucket"
+}
+
 # Set up the route53 record for the API
 module "route53" {
   source         = "../modules/route53"
@@ -430,3 +465,27 @@ module "engine_registry" {
   name   = "engine"
   stage = var.stage
 }
+
+# S3 policy for Engine to read and write from various S3 buckets
+module "engine_s3_read_and_write" {
+  source = "../modules/s3_iam_policy"
+
+  policy_name        = "EngineReadandWriteS3"
+  policy_description = "Allow Engine Lambda to read from and write to various S3 buckets"
+  bucket_arns        = [
+    "arn:aws:s3:::${module.s3_presignable_bucket.bucket_name}",
+    "arn:aws:s3:::${module.retrofit_sap_data.bucket_name}",
+    "arn:aws:s3:::${module.retrofit_sap_predictions.bucket_name}",
+    "arn:aws:s3:::${module.retrofit_carbon_predictions.bucket_name}",
+    "arn:aws:s3:::${module.retrofit_heat_predictions.bucket_name}",
+    "arn:aws:s3:::${module.retrofit_heating_kwh_predictions.bucket_name}",
+    "arn:aws:s3:::${module.retrofit_hotwater_kwh_predictions.bucket_name}",
+    "arn:aws:s3:::${module.retrofit_energy_assessments.bucket_name}"
+  ]
+  actions            = ["s3:*"]
+  resource_paths     = ["/*"]
+}
+
+output "engine_s3_read_and_write_arn" {
+  value = module.engine_s3_read_and_write.policy_arn
+}