pull infrastructure changes from Jun-te's branch

infrastructure/terraform/lambda/_template/main.tf

@@ -1,3 +1,30 @@
+# ==============================================================================
+# TEMPLATE: Lambda Configuration with Optional S3 IAM Policy
+# ==============================================================================
+# Instructions:
+# 1. Replace "REPLACE ME" with your lambda name (e.g., "my-lambda-name")
+# 2. Add any additional environment variables as needed
+# 3. To attach S3 IAM policies from shared state:
+#    - Uncomment the S3 policy attachment section below
+#    - Update the policy_arn to match the output from shared/main.tf
+#    - Available shared outputs (examples):
+#      - data.terraform_remote_state.shared.outputs.condition_etl_s3_read_arn
+#      - data.terraform_remote_state.shared.outputs.postcode_splitter_s3_read_arn
+# 4. To create a NEW S3 policy:
+#    - Add a new module "lambda_s3_policy" in shared/main.tf using the
+#      s3_iam_policy module (see examples in shared/main.tf)
+#    - Then reference it here using data.terraform_remote_state.shared.outputs
+# ==============================================================================
+
+data "terraform_remote_state" "shared" {
+  backend = "s3"
+  config = {
+    bucket = "assessment-model-terraform-state"
+    key = "env:/${var.stage}/terraform.tfstate"
+    region = "eu-west-2"
+  }
+}
+
 module "lambda" {
   source = "../modules/lambda_with_sqs"
 
@@ -12,3 +39,25 @@ module "lambda" {
     LOG_LEVEL = "info"
   }
 }
+
+# ======================================================================
+# OPTIONAL: Attach S3 IAM policy to Lambda execution role
+# ======================================================================
+# Uncomment and configure the resource below to attach S3 permissions
+#
+# Example 1: Attach existing policy from shared state
+# resource "aws_iam_role_policy_attachment" "lambda_s3_policy" {
+#   role       = module.lambda.role_name
+#   policy_arn = data.terraform_remote_state.shared.outputs.YOUR_POLICY_OUTPUT_NAME_arn
+# }
+#
+# Example 2: Attach multiple policies
+# resource "aws_iam_role_policy_attachment" "lambda_read_policy" {
+#   role       = module.lambda.role_name
+#   policy_arn = data.terraform_remote_state.shared.outputs.postcode_splitter_s3_read_arn
+# }
+#
+# resource "aws_iam_role_policy_attachment" "lambda_write_policy" {
+#   role       = module.lambda.role_name
+#   policy_arn = data.terraform_remote_state.shared.outputs.another_policy_arn
+# }

infrastructure/terraform/lambda/address2UPRN/main.tf

@@ -1,3 +1,19 @@
+data "terraform_remote_state" "shared" {
+  backend = "s3"
+  config = {
+    bucket = "assessment-model-terraform-state"
+    key = "env:/${var.stage}/terraform.tfstate"
+    region = "eu-west-2"
+  }
+}
+data "aws_secretsmanager_secret_version" "db_credentials" {
+  secret_id = "${var.stage}/assessment_model/db_credentials"
+}
+
+locals {
+  db_credentials = jsondecode(data.aws_secretsmanager_secret_version.db_credentials.secret_string)
+}
+
 module "address2uprn" {
   source = "../modules/lambda_with_sqs"
 
@@ -6,9 +22,32 @@ module "address2uprn" {
 
   image_uri = local.image_uri
 
-
-  environment = {
-    STAGE     = var.stage
-    LOG_LEVEL = "info"
-  }
+  environment = merge(
+    {
+      STAGE     = var.stage
+      LOG_LEVEL = "info"
+      DB_USERNAME = local.db_credentials.db_assessment_model_username
+      DB_PASSWORD = local.db_credentials.db_assessment_model_password
+      GOOGLE_SOLAR_API_KEY = "test"
+      SAP_PREDICTIONS_BUCKET = "test"
+      CARBON_PREDICTIONS_BUCKET = "test"
+      HEAT_PREDICTIONS_BUCKET = "test"
+      HEATING_KWH_PREDICTIONS_BUCKET = "test"
+      HOTWATER_KWH_PREDICTIONS_BUCKET = "test"
+      API_KEY = "test"
+      ENVIRONMENT = "test"
+      SECRET_KEY = "test"
+      PLAN_TRIGGER_BUCKET = "test"
+      DATA_BUCKET = "test"
+      ENGINE_SQS_URL = "test"
+      ENERGY_ASSESSMENTS_BUCKET = "test"
+      S3_BUCKET_NAME = data.terraform_remote_state.shared.outputs.retrofit_sap_data_bucket_name
+    },
+  )
 }
+
+# Attach S3 read policy to the Lambda execution role
+resource "aws_iam_role_policy_attachment" "address2uprn_read_and_write" {
+  role       = module.address2uprn.role_name
+  policy_arn = data.terraform_remote_state.shared.outputs.address_2_uprn_s3_read_and_write_arn
+}

infrastructure/terraform/lambda/address2UPRN/outputs.tf

@@ -0,0 +1,14 @@
+output "address2uprn_queue_url" {
+  value       = module.address2uprn.queue_url
+  description = "URL of the address2UPRN SQS queue"
+}
+
+output "address2uprn_queue_arn" {
+  value       = module.address2uprn.queue_arn
+  description = "ARN of the address2UPRN SQS queue"
+}
+
+output "address2uprn_lambda_arn" {
+  value       = module.address2uprn.lambda_arn
+  description = "ARN of the address2UPRN Lambda function"
+}

infrastructure/terraform/lambda/condition-etl/main.tf

@@ -23,7 +23,6 @@ module "lambda" {
   stage = var.stage
 
   image_uri = local.image_uri
-  timeout = 180
 
 
   environment = merge(

infrastructure/terraform/lambda/modules/lambda_with_sqs/outputs.tf

@@ -9,3 +9,4 @@ output "queue_arn" {
 output "queue_url" {
   value = module.queue.queue_url
 }
+

infrastructure/terraform/lambda/postcodeSplitter/main.tf

@@ -1,3 +1,30 @@
+data "terraform_remote_state" "shared" {
+  backend = "s3"
+  config = {
+    bucket = "assessment-model-terraform-state"
+    key = "env:/${var.stage}/terraform.tfstate" 
+    region = "eu-west-2"
+  }
+}
+data "aws_secretsmanager_secret_version" "db_credentials" {
+  secret_id = "${var.stage}/assessment_model/db_credentials"
+}
+
+
+locals {
+  db_credentials = jsondecode(data.aws_secretsmanager_secret_version.db_credentials.secret_string)
+}
+
+# Reference the existing address2UPRN Lambda outputs from address2uprn state
+data "terraform_remote_state" "address2uprn" {
+  backend = "s3"
+  config = {
+    bucket = "address2uprn-terraform-state"
+    key = "env:/${var.stage}/terraform.tfstate"
+    region = "eu-west-2"
+  }
+}
+
 module "lambda" {
   source = "../modules/lambda_with_sqs"
 
@@ -7,8 +34,56 @@ module "lambda" {
   image_uri = local.image_uri
 
 
-  environment = {
-    STAGE = var.stage
-    LOG_LEVEL = "info"
-  }
+  environment = merge(
+    {
+      STAGE     = var.stage
+      LOG_LEVEL = "info"
+      DB_USERNAME = local.db_credentials.db_assessment_model_username
+      DB_PASSWORD = local.db_credentials.db_assessment_model_password
+      GOOGLE_SOLAR_API_KEY = "test"
+      SAP_PREDICTIONS_BUCKET = "test"
+      CARBON_PREDICTIONS_BUCKET = "test"
+      HEAT_PREDICTIONS_BUCKET = "test"
+      HEATING_KWH_PREDICTIONS_BUCKET = "test"
+      HOTWATER_KWH_PREDICTIONS_BUCKET = "test"
+      API_KEY = "test"
+      ENVIRONMENT = "test"
+      SECRET_KEY = "test"
+      PLAN_TRIGGER_BUCKET = "test"
+      DATA_BUCKET = "test"
+      EPC_AUTH_TOKEN = "test"
+      ENGINE_SQS_URL = "test"
+      ENERGY_ASSESSMENTS_BUCKET = "test"
+      ADDRESS2UPRN_QUEUE_URL = data.terraform_remote_state.address2uprn.outputs.address2uprn_queue_url
+      S3_BUCKET_NAME = data.terraform_remote_state.shared.outputs.retrofit_sap_data_bucket_name
+    },
+  )
 }
+
+# Attach S3 read policy to the Lambda execution role
+resource "aws_iam_role_policy_attachment" "postcode_splitter_s3_read" {
+  role       = module.lambda.role_name
+  policy_arn = data.terraform_remote_state.shared.outputs.postcode_splitter_s3_read_arn
+}
+
+# Create SQS send policy for address2UPRN queue
+module "postcode_splitter_sqs_policy" {
+  source = "../../modules/general_iam_policy"
+
+  policy_name        = "postcode-splitter-sqs-send-${var.stage}"
+  policy_description = "Allow postcode-splitter Lambda to send messages to address2UPRN queue"
+
+  actions = [
+    "sqs:SendMessage"
+  ]
+
+  resources = [
+    data.terraform_remote_state.address2uprn.outputs.address2uprn_queue_arn
+  ]
+}
+
+# Attach SQS policy to the Lambda execution role
+resource "aws_iam_role_policy_attachment" "postcode_splitter_sqs_send" {
+  role       = module.lambda.role_name
+  policy_arn = module.postcode_splitter_sqs_policy.policy_arn
+}

infrastructure/terraform/lambda/postcodeSplitter/variables.tf

@@ -24,3 +24,12 @@ locals {
 output "resolved_image_uri" {
   value = local.image_uri
 }
+
+
+
+
+
+
+
+
+

infrastructure/terraform/modules/general_iam_policy/main.tf

@@ -0,0 +1,21 @@
+# IAM Policy with dynamic actions and resources
+resource "aws_iam_policy" "policy" {
+  name        = var.policy_name
+  description = var.policy_description
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      merge(
+        {
+          Effect   = "Allow"
+          Action   = var.actions
+          Resource = var.resources
+        },
+        var.conditions != null ? { Condition = var.conditions } : {}
+      )
+    ]
+  })
+
+  tags = var.tags
+}

infrastructure/terraform/modules/general_iam_policy/outputs.tf

@@ -0,0 +1,9 @@
+output "policy_arn" {
+  value       = aws_iam_policy.policy.arn
+  description = "ARN of the created IAM policy"
+}
+
+output "policy_name" {
+  value       = aws_iam_policy.policy.name
+  description = "Name of the created IAM policy"
+}

infrastructure/terraform/modules/general_iam_policy/variables.tf

@@ -0,0 +1,32 @@
+variable "policy_name" {
+  description = "Name of the IAM policy"
+  type        = string
+}
+
+variable "policy_description" {
+  description = "Description of the IAM policy"
+  type        = string
+  default     = ""
+}
+
+variable "actions" {
+  description = "List of IAM actions allowed by this policy"
+  type        = list(string)
+}
+
+variable "resources" {
+  description = "List of AWS resources this policy applies to"
+  type        = list(string)
+}
+
+variable "conditions" {
+  description = "Optional IAM policy conditions"
+  type        = any
+  default     = null
+}
+
+variable "tags" {
+  description = "Tags to apply to the policy"
+  type        = map(string)
+  default     = {}
+}

infrastructure/terraform/modules/lambda_execution_role/main.tf

@@ -19,19 +19,3 @@ resource "aws_iam_role_policy_attachment" "basic_logs" {
   policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
 }
 
-resource "aws_iam_role_policy" "ecr_pull" {
-  role = aws_iam_role.this.name
-
-  policy = jsonencode({
-    Version = "2012-10-17"
-    Statement = [{
-      Effect = "Allow"
-      Action = [
-        "ecr:GetAuthorizationToken",
-        "ecr:BatchGetImage",
-        "ecr:GetDownloadUrlForLayer"
-      ]
-      Resource = "*"
-    }]
-  })
-}

infrastructure/terraform/modules/s3_iam_policy/main.tf

@@ -0,0 +1,31 @@
+# Dynamically build S3 resources list from bucket ARNs and resource paths
+locals {
+  # Generate full resource ARNs by combining bucket ARNs with resource paths
+  resources = flatten([
+    for bucket_arn in var.bucket_arns : [
+      for path in var.resource_paths : "${bucket_arn}${path}"
+    ]
+  ])
+}
+
+# IAM Policy with dynamic actions and resources
+resource "aws_iam_policy" "s3_policy" {
+  name        = var.policy_name
+  description = var.policy_description
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      merge(
+        {
+          Effect   = "Allow"
+          Action   = var.actions
+          Resource = local.resources
+        },
+        var.conditions != null ? { Condition = var.conditions } : {}
+      )
+    ]
+  })
+
+  tags = var.tags
+}

infrastructure/terraform/modules/s3_iam_policy/outputs.tf

@@ -0,0 +1,14 @@
+output "policy_arn" {
+  description = "ARN of the S3 IAM policy"
+  value       = aws_iam_policy.s3_policy.arn
+}
+
+output "policy_name" {
+  description = "Name of the S3 IAM policy"
+  value       = aws_iam_policy.s3_policy.name
+}
+
+output "policy_id" {
+  description = "ID of the S3 IAM policy"
+  value       = aws_iam_policy.s3_policy.id
+}

infrastructure/terraform/modules/s3_iam_policy/variables.tf

@@ -0,0 +1,42 @@
+variable "policy_name" {
+  description = "Name of the IAM policy"
+  type        = string
+}
+
+variable "policy_description" {
+  description = "Description of the IAM policy"
+  type        = string
+  default     = ""
+}
+
+variable "bucket_arns" {
+  description = "List of S3 bucket ARNs to grant access to"
+  type        = list(string)
+}
+
+variable "actions" {
+  description = "List of S3 actions to allow (e.g., ['s3:GetObject'], ['s3:PutObject'], ['s3:DeleteObject'])"
+  type        = list(string)
+  default     = ["s3:GetObject"]
+}
+
+variable "resource_paths" {
+  description = "List of resource paths within buckets (e.g., ['/*'] for all objects, ['/specific-prefix/*'] for specific prefix)"
+  type        = list(string)
+  default     = ["/*"]
+}
+
+variable "conditions" {
+  description = "Optional IAM policy conditions to apply to the statement"
+  type        = any
+  default     = null
+}
+
+variable "tags" {
+  description = "Tags to apply to the policy"
+  type        = map(string)
+  default     = {}
+}
+
+
+

infrastructure/terraform/shared/main.tf

@@ -133,6 +133,11 @@ module "retrofit_sap_data" {
   allowed_origins = var.allowed_origins
 }
 
+output "retrofit_sap_data_bucket_name" {
+  value = module.retrofit_sap_data.bucket_name
+  description = "Name of the retrofit SAP data bucket"
+}
+
 module "retrofit_carbon_predictions" {
   source          = "../modules/s3"
   bucketname      = "retrofit-carbon-predictions-${var.stage}"
@@ -305,6 +310,21 @@ module "address2uprn_registry" {
 
 }
 
+# S3 policy for postcode splitter to read from retrofit data bucket
+module "address2uprn_s3_read_and_write" {
+  source = "../modules/s3_iam_policy"
+
+  policy_name        = "Address2UPRNReadandWriteS3"
+  policy_description = "Allow address2uprn Lambda to read and write from retrofit-data bucket"
+  bucket_arns        = ["arn:aws:s3:::retrofit-data-${var.stage}"]
+  actions            = ["s3:GetObject", "s3:ListBucket", "s3:PutObject"]
+  resource_paths     = ["/*"]
+}
+
+output "address_2_uprn_s3_read_and_write_arn" {
+  value = module.address2uprn_s3_read_and_write.policy_arn
+}
+
 ################################################
 # Condition ETL – Lambda ECR
 ################################################
@@ -321,6 +341,28 @@ module "condition_etl_registry" {
 
 }
 
+# Condition Data S3 Bucket to store initial data
+module "condition_data_bucket" {
+  source      = "../modules/s3"
+  bucketname = "condition-data-${var.stage}"
+  allowed_origins = var.allowed_origins
+}
+
+module "condition_etl_s3_read" {
+  source = "../modules/s3_iam_policy"
+
+  policy_name        = "ConditionETLReadS3"
+  policy_description = "Allow Lambda to read objects from condition-data-${var.stage}"
+  bucket_arns        = ["arn:aws:s3:::condition-data-${var.stage}"]
+  actions            = ["s3:GetObject"]
+  resource_paths     = ["/*"]
+}
+
+output "condition_etl_s3_read_arn" {
+  value = module.condition_etl_s3_read.policy_arn
+}
+
+
 ################################################
 # Postcode Splitter – Lambda ECR
 ################################################
@@ -337,30 +379,17 @@ module "postcode_splitter_registry" {
 
 }
 
-################################################
-# Conidition data – S3 bucket
-################################################
-module "condition_data_bucket" {
-  source      = "../modules/s3"
-  bucketname = "condition-data-${var.stage}"
-  allowed_origins = var.allowed_origins
+# S3 policy for postcode splitter to read from retrofit data bucket
+module "postcode_splitter_s3_read" {
+  source = "../modules/s3_iam_policy"
+
+  policy_name        = "PostcodeSplitterReadS3"
+  policy_description = "Allow postcode splitter Lambda to read from retrofit-data bucket"
+  bucket_arns        = ["arn:aws:s3:::retrofit-data-${var.stage}"]
+  actions            = ["s3:GetObject", "s3:ListBucket", "s3:PutObject"]
+  resource_paths     = ["/*"]
 }
 
-resource "aws_iam_policy" "condition_etl_s3_read" {
-  name        = "ConditionETLReadS3"
-  description = "Allow Lambda to read objects from condition-data-${var.stage}"
-  policy      = jsonencode({
-    Version = "2012-10-17"
-    Statement = [
-      {
-        Effect = "Allow"
-        Action = ["s3:GetObject"]
-        Resource = "arn:aws:s3:::condition-data-${var.stage}/*"
-      }
-    ]
-  })
-}
-
-output "condition_etl_s3_read_arn" {
-  value = aws_iam_policy.condition_etl_s3_read.arn
+output "postcode_splitter_s3_read_arn" {
+  value = module.postcode_splitter_s3_read.policy_arn
 }