Merge 33a237104c into 75ab7f3c20

.github/workflows/deploy_fastapi_backend.yml

@@ -77,9 +77,6 @@ jobs:
           echo "::set-output name=hotwater_kwh_predictions_bucket::${{ secrets[format('{0}_HOTWATER_KWH_PREDICTIONS_BUCKET', github.ref_name)] }}"
           echo "::set-output name=energy_asessments_bucket::${{ secrets[format('{0}_ENERGY_ASSESSMENTS_BUCKET', github.ref_name)] }}"
           echo "::set-output name=google_solar_api_key::${{ secrets[format('{0}_GOOGLE_SOLAR_API_KEY', github.ref_name)] }}"
-          echo "::set-output name=sap_baseline_predictions_bucket::${{ secrets[format('{0}_SAP_BASELINE_PREDICTIONS_BUCKET', github.ref_name)] }}"
-          echo "::set-output name=carbon_baseline_predictions_bucket::${{ secrets[format('{0}_CARBON_BASELINE_PREDICTIONS_BUCKET', github.ref_name)] }}"
-          echo "::set-output name=heat_baseline_predictions_bucket::${{ secrets[format('{0}_HEAT_BASELINE_PREDICTIONS_BUCKET', github.ref_name)] }}"
 
       - name: Setup Docker
         uses: docker/setup-buildx-action@v1
@@ -132,9 +129,6 @@ jobs:
           DB_NAME: ${{ steps.set_db_credentials.outputs.db_name }}
           ECR_URI: ${{ steps.set_ecr_credentials.outputs.ecr_uri }}
           GITHUB_SHA: ${{ github.sha }}
-          SAP_BASELINE_PREDICTIONS_BUCKET: ${{ steps.set_api_secrets.outputs.sap_baseline_predictions_bucket }}
-          CARBON_BASELINE_PREDICTIONS_BUCKET: ${{ steps.set_api_secrets.outputs.carbon_baseline_predictions_bucket }}
-          HEAT_BASELINE_PREDICTIONS_BUCKET: ${{ steps.set_api_secrets.outputs.heat_baseline_predictions_bucket }}
         run: |
           # Fetch database credentials from AWS Secrets Manager
           SECRET_VALUE=$(aws secretsmanager get-secret-value --secret-id ${{ github.ref_name }}/assessment_model/db_credentials --query SecretString)

.idea/Model.iml

@@ -6,7 +6,6 @@
       <sourceFolder url="file://$MODULE_DIR$/model_data" isTestSource="false" />
       <sourceFolder url="file://$MODULE_DIR$/open_uprn" isTestSource="false" />
       <sourceFolder url="file://$MODULE_DIR$/recommendations" isTestSource="false" />
-      <excludeFolder url="file://$MODULE_DIR$/infrastructure/terraform/.terraform" />
     </content>
     <orderEntry type="jdk" jdkName="Fastapi-backend" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />

infrastructure/terraform/lambda/fast-api/main.tf

@@ -5,7 +5,7 @@ data "terraform_remote_state" "shared" {
   backend = "s3"
   config = {
     bucket = "assessment-model-terraform-state"
-    key    = "env:/${var.stage}/terraform.tfstate"
+    key = "env:/${var.stage}/terraform.tfstate"
     region = "eu-west-2"
   }
 }
@@ -14,7 +14,7 @@ data "terraform_remote_state" "engine" {
   backend = "s3"
   config = {
     bucket = "ara-engine-terraform-state",
-    key    = "env:/${var.stage}/terraform.tfstate"
+    key = "env:/${var.stage}/terraform.tfstate"
     region = "eu-west-2"
   }
 }
@@ -23,7 +23,7 @@ data "terraform_remote_state" "categorisation" {
   backend = "s3"
   config = {
     bucket = "categorisation-terraform-state",
-    key    = "env:/${var.stage}/terraform.tfstate"
+    key = "env:/${var.stage}/terraform.tfstate"
     region = "eu-west-2"
   }
 }
@@ -43,26 +43,26 @@ locals {
 # FastAPI Lambda + API Gateway
 ############################################
 module "fastapi" {
-  source = "../../modules/lambda_with_api_gateway"
+  source     = "../../modules/lambda_with_api_gateway"
 
-  name              = "fastapi"
-  stage             = var.stage
-  source_dir        = "${path.root}/../../../../"
-  handler           = "backend.app.main.handler"
-  runtime           = "python3.11"
-  timeout           = 600
-  memory_size       = 512
-  artifact_bucket   = data.terraform_remote_state.shared.outputs.ara_fast_api_state_bucket
+  name        = "fastapi"
+  stage       = var.stage
+  source_dir  = "${path.root}/../../../../"
+  handler     = "backend.app.main.handler"
+  runtime     = "python3.11"
+  timeout     = 600
+  memory_size = 512
+  artifact_bucket = data.terraform_remote_state.shared.outputs.ara_fast_api_state_bucket
   requirements_file = "${path.root}/../../../../backend/app/requirements/requirements.txt"
 
   domain_name = "api.${var.domain_name}"
 
   environment = {
-    ENVIRONMENT = var.stage
-    API_KEY     = var.api_key
-    SECRET_KEY  = var.secret_key
+    ENVIRONMENT    = var.stage
+    API_KEY        = var.api_key
+    SECRET_KEY     = var.secret_key
     # DOMAIN_NAME    = var.domain_name
-    EPC_AUTH_TOKEN       = var.epc_auth_token
+    EPC_AUTH_TOKEN = var.epc_auth_token
     GOOGLE_SOLAR_API_KEY = var.google_solar_api_key
 
     DB_HOST     = var.db_host
@@ -71,17 +71,14 @@ module "fastapi" {
     DB_USERNAME = local.db_credentials.db_assessment_model_username
     DB_PASSWORD = local.db_credentials.db_assessment_model_password
 
-    PLAN_TRIGGER_BUCKET                = data.terraform_remote_state.shared.outputs.retrofit_plan_trigger_bucket_name
-    DATA_BUCKET                        = data.terraform_remote_state.shared.outputs.retrofit_sap_data_bucket_name
-    SAP_PREDICTIONS_BUCKET             = data.terraform_remote_state.shared.outputs.retrofit_sap_predictions_bucket_name
-    CARBON_PREDICTIONS_BUCKET          = data.terraform_remote_state.shared.outputs.retrofit_carbon_predictions_bucket_name
-    HEAT_PREDICTIONS_BUCKET            = data.terraform_remote_state.shared.outputs.retrofit_heat_predictions_bucket_name
-    HEATING_KWH_PREDICTIONS_BUCKET     = data.terraform_remote_state.shared.outputs.retrofit_heating_kwh_predictions_bucket_name
-    HOTWATER_KWH_PREDICTIONS_BUCKET    = data.terraform_remote_state.shared.outputs.retrofit_hotwater_kwh_predictions_bucket_name
-    ENERGY_ASSESSMENTS_BUCKET          = data.terraform_remote_state.shared.outputs.retrofit_energy_assessments_bucket_name
-    SAP_BASELINE_PREDICTIONS_BUCKET    = data.terraform_remote_state.shared.outputs.retrofit_sap_baseline_predictions_bucket_name
-    CARBON_BASELINE_PREDICTIONS_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_carbon_baseline_predictions_bucket_name
-    HEAT_BASELINE_PREDICTIONS_BUCKET   = data.terraform_remote_state.shared.outputs.retrofit_heat_baseline_predictions_bucket_name
+    PLAN_TRIGGER_BUCKET             = data.terraform_remote_state.shared.outputs.retrofit_plan_trigger_bucket_name
+    DATA_BUCKET                     = data.terraform_remote_state.shared.outputs.retrofit_sap_data_bucket_name
+    SAP_PREDICTIONS_BUCKET          = data.terraform_remote_state.shared.outputs.retrofit_sap_predictions_bucket_name
+    CARBON_PREDICTIONS_BUCKET       = data.terraform_remote_state.shared.outputs.retrofit_carbon_predictions_bucket_name
+    HEAT_PREDICTIONS_BUCKET         = data.terraform_remote_state.shared.outputs.retrofit_heat_predictions_bucket_name
+    HEATING_KWH_PREDICTIONS_BUCKET  = data.terraform_remote_state.shared.outputs.retrofit_heating_kwh_predictions_bucket_name
+    HOTWATER_KWH_PREDICTIONS_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_hotwater_kwh_predictions_bucket_name
+    ENERGY_ASSESSMENTS_BUCKET       = data.terraform_remote_state.shared.outputs.retrofit_energy_assessments_bucket_name
 
     ENGINE_SQS_URL         = data.terraform_remote_state.engine.outputs.ara_engine_queue_url
     CATEGORISATION_SQS_URL = data.terraform_remote_state.categorisation.outputs.categorisation_queue_url
@@ -124,4 +121,4 @@ resource "aws_iam_role_policy_attachment" "fastapi_sqs_send" {
 resource "aws_iam_role_policy_attachment" "fastapi_s3_read_and_write" {
   role       = module.fastapi.role_name
   policy_arn = data.terraform_remote_state.shared.outputs.fast_api_s3_read_and_write_arn
-}
+}

infrastructure/terraform/shared/main.tf

@@ -6,16 +6,16 @@ terraform {
     }
   }
   backend "s3" {
-    bucket = "assessment-model-terraform-state"
-    region = "eu-west-2"
-    key    = "terraform.tfstate"
+    bucket  = "assessment-model-terraform-state"
+    region  = "eu-west-2"
+    key     = "terraform.tfstate"
   }
 
   required_version = ">= 1.2.0"
 }
 
 provider "aws" {
-  region = var.region
+  region  = var.region
 }
 
 # Additional provider for resources that need to be in us-east-1, specifically the SSL certificate
@@ -47,30 +47,30 @@ resource "aws_security_group" "allow_db" {
 
   ingress {
     # TLS (change to whatever ports you need)
-    from_port   = 5432
-    to_port     = 5432
-    protocol    = "tcp"
+    from_port = 5432
+    to_port   = 5432
+    protocol  = "tcp"
     cidr_blocks = ["0.0.0.0/0"]
   }
 
   egress {
-    from_port   = 0
-    to_port     = 0
-    protocol    = "-1"
+    from_port = 0
+    to_port   = 0
+    protocol  = "-1"
     cidr_blocks = ["0.0.0.0/0"]
   }
 }
 
 resource "aws_db_instance" "default" {
-  allocated_storage      = var.allocated_storage
-  engine                 = "postgres"
-  engine_version         = "14.17"
-  instance_class         = var.instance_class
-  db_name                = var.database_name
-  username               = jsondecode(data.aws_secretsmanager_secret_version.db_credentials.secret_string)["db_assessment_model_username"]
-  password               = jsondecode(data.aws_secretsmanager_secret_version.db_credentials.secret_string)["db_assessment_model_password"]
-  parameter_group_name   = "default.postgres14"
-  skip_final_snapshot    = true
+  allocated_storage    = var.allocated_storage
+  engine               = "postgres"
+  engine_version       = "14.17"
+  instance_class       = var.instance_class
+  db_name              = var.database_name
+  username             = jsondecode(data.aws_secretsmanager_secret_version.db_credentials.secret_string)["db_assessment_model_username"]
+  password             = jsondecode(data.aws_secretsmanager_secret_version.db_credentials.secret_string)["db_assessment_model_password"]
+  parameter_group_name = "default.postgres14"
+  skip_final_snapshot  = true
   vpc_security_group_ids = [aws_security_group.allow_db.id]
   lifecycle {
     prevent_destroy = true
@@ -87,7 +87,7 @@ resource "aws_db_instance" "default" {
   storage_type = "gp3"
 
   # Automated backups configuration
-  backup_retention_period = 14
+  backup_retention_period = 14 
   backup_window           = "03:00-04:00"
   maintenance_window      = "Sun:02:00-Sun:02:30"
   copy_tags_to_snapshot   = true
@@ -103,7 +103,7 @@ module "s3_presignable_bucket" {
 }
 
 output "retrofit_plan_trigger_bucket_name" {
-  value       = module.s3_presignable_bucket.bucket_name
+  value = module.s3_presignable_bucket.bucket_name
   description = "Name of the retrofit plan trigger bucket"
 }
 
@@ -156,7 +156,7 @@ module "retrofit_sap_predictions" {
 }
 
 output "retrofit_sap_predictions_bucket_name" {
-  value       = module.retrofit_sap_predictions.bucket_name
+  value = module.retrofit_sap_predictions.bucket_name
   description = "Name of the retrofit SAP predictions bucket"
 }
 
@@ -167,7 +167,7 @@ module "retrofit_sap_data" {
 }
 
 output "retrofit_sap_data_bucket_name" {
-  value       = module.retrofit_sap_data.bucket_name
+  value = module.retrofit_sap_data.bucket_name
   description = "Name of the retrofit SAP data bucket"
 }
 
@@ -178,7 +178,7 @@ module "retrofit_carbon_predictions" {
 }
 
 output "retrofit_carbon_predictions_bucket_name" {
-  value       = module.retrofit_carbon_predictions.bucket_name
+  value = module.retrofit_carbon_predictions.bucket_name
   description = "Name of the retrofit carbon predictions bucket"
 }
 
@@ -189,7 +189,7 @@ module "retrofit_heat_predictions" {
 }
 
 output "retrofit_heat_predictions_bucket_name" {
-  value       = module.retrofit_heat_predictions.bucket_name
+  value = module.retrofit_heat_predictions.bucket_name
   description = "Name of the retrofit heat predictions bucket"
 }
 
@@ -218,7 +218,7 @@ module "retrofit_heating_kwh_predictions" {
 }
 
 output "retrofit_heating_kwh_predictions_bucket_name" {
-  value       = module.retrofit_heating_kwh_predictions.bucket_name
+  value = module.retrofit_heating_kwh_predictions.bucket_name
   description = "Name of the retrofit heating kWh predictions bucket"
 }
 
@@ -229,7 +229,7 @@ module "retrofit_hotwater_kwh_predictions" {
 }
 
 output "retrofit_hotwater_kwh_predictions_bucket_name" {
-  value       = module.retrofit_hotwater_kwh_predictions.bucket_name
+  value = module.retrofit_hotwater_kwh_predictions.bucket_name
   description = "Name of the retrofit hotwater kWh predictions bucket"
 }
 
@@ -239,33 +239,6 @@ module "retrofit_sap_baseline_predictions" {
   allowed_origins = var.allowed_origins
 }
 
-output "retrofit_sap_baseline_predictions_bucket_name" {
-  value       = module.retrofit_sap_baseline_predictions.bucket_name
-  description = "Name of the retrofit SAP baseline predictions bucket"
-}
-
-module "retrofit_carbon_baseline_predictions" {
-  source          = "../modules/s3"
-  bucketname      = "retrofit-carbon-baseline-predictions-${var.stage}"
-  allowed_origins = var.allowed_origins
-}
-
-output "retrofit_carbon_baseline_predictions_bucket_name" {
-  value       = module.retrofit_carbon_baseline_predictions.bucket_name
-  description = "Name of the retrofit carbon baseline predictions bucket"
-}
-
-module "retrofit_heat_baseline_predictions" {
-  source          = "../modules/s3"
-  bucketname      = "retrofit-heat-baseline-predictions-${var.stage}"
-  allowed_origins = var.allowed_origins
-}
-
-output "retrofit_heat_baseline_predictions_bucket_name" {
-  value       = module.retrofit_heat_baseline_predictions.bucket_name
-  description = "Name of the retrofit heat baseline predictions bucket"
-}
-
 // We make this bucket presignable, because we want to generate download links for the frontend
 module "retrofit_energy_assessments" {
   source          = "../modules/s3_presignable_bucket"
@@ -275,7 +248,7 @@ module "retrofit_energy_assessments" {
 }
 
 output "retrofit_energy_assessments_bucket_name" {
-  value       = module.retrofit_energy_assessments.bucket_name
+  value = module.retrofit_energy_assessments.bucket_name
   description = "Name of the retrofit energy assessments bucket"
 }
 
@@ -354,16 +327,6 @@ module "sap_baseline_ecr" {
   source   = "../modules/ecr"
 }
 
-module "heat_baseline_ecr" {
-  ecr_name = "heat-baseline-prediction-${var.stage}"
-  source   = "../modules/ecr"
-}
-
-module "carbon_baseline_ecr" {
-  ecr_name = "carbon-baseline-prediction-${var.stage}"
-  source   = "../modules/ecr"
-}
-
 ################################################
 # SES - Email sending
 ################################################
@@ -389,7 +352,7 @@ module "address2uprn_state_bucket" {
 module "address2uprn_registry" {
   source = "../modules/container_registry"
   name   = "address2uprn"
-  stage  = var.stage
+  stage = var.stage
 
 }
 
@@ -420,14 +383,14 @@ module "condition_etl_state_bucket" {
 module "condition_etl_registry" {
   source = "../modules/container_registry"
   name   = "condition-etl"
-  stage  = var.stage
+  stage = var.stage
 
 }
 
 # Condition Data S3 Bucket to store initial data
 module "condition_data_bucket" {
-  source          = "../modules/s3"
-  bucketname      = "condition-data-${var.stage}"
+  source      = "../modules/s3"
+  bucketname = "condition-data-${var.stage}"
   allowed_origins = var.allowed_origins
 }
 
@@ -458,7 +421,7 @@ module "postcode_splitter_state_bucket" {
 module "postcode_splitter_registry" {
   source = "../modules/container_registry"
   name   = "postcode_splitter"
-  stage  = var.stage
+  stage = var.stage
 
 }
 
@@ -489,7 +452,7 @@ module "categorisation_state_bucket" {
 module "categorisation_registry" {
   source = "../modules/container_registry"
   name   = "categorisation"
-  stage  = var.stage
+  stage = var.stage
 }
 
 
@@ -505,7 +468,7 @@ module "ordnance_state_bucket" {
 module "ordnance_registry" {
   source = "../modules/container_registry"
   name   = "ordnance"
-  stage  = var.stage
+  stage = var.stage
 
 }
 
@@ -536,7 +499,7 @@ module "engine_state_bucket" {
 module "engine_registry" {
   source = "../modules/container_registry"
   name   = "engine"
-  stage  = var.stage
+  stage = var.stage
 }
 
 # S3 policy for Engine to read and write from various S3 buckets
@@ -545,7 +508,7 @@ module "engine_s3_read_and_write" {
 
   policy_name        = "EngineReadandWriteS3"
   policy_description = "Allow Engine Lambda to read from and write to various S3 buckets"
-  bucket_arns = [
+  bucket_arns        = [
     "arn:aws:s3:::${module.s3_presignable_bucket.bucket_name}",
     "arn:aws:s3:::${module.retrofit_sap_data.bucket_name}",
     "arn:aws:s3:::${module.retrofit_sap_predictions.bucket_name}",
@@ -553,13 +516,10 @@ module "engine_s3_read_and_write" {
     "arn:aws:s3:::${module.retrofit_heat_predictions.bucket_name}",
     "arn:aws:s3:::${module.retrofit_heating_kwh_predictions.bucket_name}",
     "arn:aws:s3:::${module.retrofit_hotwater_kwh_predictions.bucket_name}",
-    "arn:aws:s3:::${module.retrofit_energy_assessments.bucket_name}",
-    "arn:aws:s3:::${module.retrofit_sap_baseline_predictions.bucket_name}",
-    "arn:aws:s3:::${module.retrofit_carbon_baseline_predictions.bucket_name}",
-    "arn:aws:s3:::${module.retrofit_heat_baseline_predictions.bucket_name}"
+    "arn:aws:s3:::${module.retrofit_energy_assessments.bucket_name}"
   ]
-  actions        = ["s3:*"]
-  resource_paths = ["/*"]
+  actions            = ["s3:*"]
+  resource_paths     = ["/*"]
 }
 
 output "engine_s3_read_and_write_arn" {
@@ -584,7 +544,7 @@ module "fast_api_s3_read_and_write" {
 
   policy_name        = "FastAPIReadandWriteS3"
   policy_description = "Allow FastAPI Lambda to read from and write to various S3 buckets"
-  bucket_arns = [
+  bucket_arns        = [
     "arn:aws:s3:::${module.s3_presignable_bucket.bucket_name}",
     "arn:aws:s3:::${module.retrofit_sap_data.bucket_name}",
     "arn:aws:s3:::${module.retrofit_sap_predictions.bucket_name}",
@@ -594,8 +554,8 @@ module "fast_api_s3_read_and_write" {
     "arn:aws:s3:::${module.retrofit_hotwater_kwh_predictions.bucket_name}",
     "arn:aws:s3:::${module.retrofit_energy_assessments.bucket_name}"
   ]
-  actions        = ["s3:GetObject", "s3:ListBucket"]
-  resource_paths = ["/*"]
+  actions            = ["s3:GetObject", "s3:ListBucket"]
+  resource_paths     = ["/*"]
 }
 
 output "fast_api_s3_read_and_write_arn" {

serverless.yml

@@ -28,9 +28,6 @@ provider:
     HOTWATER_KWH_PREDICTIONS_BUCKET: ${env:HOTWATER_KWH_PREDICTIONS_BUCKET}
     ENERGY_ASSESSMENTS_BUCKET: ${env:ENERGY_ASSESSMENTS_BUCKET}
     GOOGLE_SOLAR_API_KEY: ${env:GOOGLE_SOLAR_API_KEY}
-    SAP_BASELINE_PREDICTIONS_BUCKET: ${env:SAP_BASELINE_PREDICTIONS_BUCKET}
-    CARBON_BASELINE_PREDICTIONS_BUCKET: ${env:CARBON_BASELINE_PREDICTIONS_BUCKET}
-    HEAT_BASELINE_PREDICTIONS_BUCKET: ${env:HEAT_BASELINE_PREDICTIONS_BUCKET}
     ENGINE_SQS_URL:
       Ref: EngineQueue
     # hardcode the categorisation queue for now as it's created in terraform
@@ -180,9 +177,3 @@ resources:
                     - arn:aws:s3:::${env:HEATING_KWH_PREDICTIONS_BUCKET}/*
                     - arn:aws:s3:::${env:HOTWATER_KWH_PREDICTIONS_BUCKET}
                     - arn:aws:s3:::${env:HOTWATER_KWH_PREDICTIONS_BUCKET}/*
-                    - arn:aws:s3:::${env:SAP_BASELINE_PREDICTIONS_BUCKET}
-                    - arn:aws:s3:::${env:SAP_BASELINE_PREDICTIONS_BUCKET}/*
-                    - arn:aws:s3:::${env:CARBON_BASELINE_PREDICTIONS_BUCKET}
-                    - arn:aws:s3:::${env:CARBON_BASELINE_PREDICTIONS_BUCKET}/*
-                    - arn:aws:s3:::${env:HEAT_BASELINE_PREDICTIONS_BUCKET}
-                    - arn:aws:s3:::${env:HEAT_BASELINE_PREDICTIONS_BUCKET}/*