diff --git a/.github/workflows/_deploy_lambda.yml b/.github/workflows/_deploy_lambda.yml index 6d1430a2..dab98d8b 100644 --- a/.github/workflows/_deploy_lambda.yml +++ b/.github/workflows/_deploy_lambda.yml @@ -16,12 +16,14 @@ on: type: string ecr_repo: - required: true + required: false type: string + default: '' image_digest: - required: true + required: false type: string + default: '' terraform_apply: required: false @@ -117,13 +119,23 @@ jobs: TF_VAR_domain_name: ${{ secrets.TF_VAR_domain_name }} TF_VAR_epc_auth_token: ${{ secrets.TF_VAR_epc_auth_token }} TF_VAR_google_solar_api_key: ${{ secrets.TF_VAR_google_solar_api_key }} - TF_VAR_ordnance_survey_api_key: ${{ secrets.TF_VAR_ordnance_survey_api_key}} + TF_VAR_ordnance_survey_api_key: ${{ secrets.TF_VAR_ordnance_survey_api_key }} run: | + ECR_REPO_URL_VAR="" + if [[ -n "${{ inputs.ecr_repo }}" ]]; then + ECR_REPO_URL_VAR="-var=ecr_repo_url=${{ steps.repo.outputs.ecr_repo_url }}" + fi + + IMAGE_DIGEST_VAR="" + if [[ -n "${{ inputs.ecr_repo }}" ]]; then + IMAGE_DIGEST_VAR="-var=image_digest=${{ inputs.image_digest }}" + fi + terraform plan \ -var="stage=${{ inputs.stage }}" \ -var="lambda_name=${{ inputs.lambda_name }}" \ - -var="ecr_repo_url=${{ steps.repo.outputs.ecr_repo_url }}" \ - -var="image_digest=${{ inputs.image_digest }}" \ + $ECR_REPO_URL_VAR \ + $IMAGE_DIGEST_VAR \ -out=lambdaplan - name: Terraform Apply @@ -143,10 +155,14 @@ jobs: TF_VAR_domain_name: ${{ secrets.TF_VAR_domain_name }} TF_VAR_epc_auth_token: ${{ secrets.TF_VAR_epc_auth_token }} TF_VAR_google_solar_api_key: ${{ secrets.TF_VAR_google_solar_api_key }} - TF_VAR_ordnance_survey_api_key: ${{ secrets.TF_VAR_ordnance_survey_api_key}} + TF_VAR_ordnance_survey_api_key: ${{ secrets.TF_VAR_ordnance_survey_api_key }} run: | + EXTRA_VARS="" + if [[ -n "${{ inputs.ecr_repo }}" ]]; then + EXTRA_VARS="-var=ecr_repo_url=${{ steps.repo.outputs.ecr_repo_url }} -var=image_digest=${{ inputs.image_digest }}" + fi + terraform destroy -auto-approve \ -var="stage=${{ inputs.stage }}" \ -var="lambda_name=${{ inputs.lambda_name }}" \ - -var="ecr_repo_url=${{ steps.repo.outputs.ecr_repo_url }}" \ - -var="image_digest=${{ inputs.image_digest }}" + $EXTRA_VARS diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 596ab82d..e41534e6 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -41,7 +41,7 @@ jobs: fi # ============================================================ - # 1️⃣ Shared Terraform (infra) + # Shared Terraform (infra) # ============================================================ shared_terraform: needs: determine_stage @@ -77,9 +77,49 @@ jobs: if: env.TERRAFORM_APPLY == 'true' working-directory: infrastructure/terraform/shared run: terraform apply -auto-approve tfplan + + # ============================================================ + # Ara Engine image and Push + # ============================================================ + ara_engine_image: + needs: [determine_stage, shared_terraform] + uses: ./.github/workflows/_build_image.yml + with: + ecr_repo: engine-${{ needs.determine_stage.outputs.stage }} + dockerfile_path: backend/docker/engine.Dockerfile + build_context: . + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + + # ============================================================ + # Deploy Ara Engine Lambda + # ============================================================ + ara_engine_lambda: + needs: [ara_engine_image, determine_stage] + uses: ./.github/workflows/_deploy_lambda.yml + with: + lambda_name: ara_engine + lambda_path: infrastructure/terraform/lambda/engine + stage: ${{ needs.determine_stage.outputs.stage }} + ecr_repo: engine-${{ needs.determine_stage.outputs.stage }} + image_digest: ${{ needs.ara_engine_image.outputs.image_digest }} + terraform_apply: ${{ needs.determine_stage.outputs.terraform_apply }} + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + TF_VAR_db_host: ${{ secrets.DEV_DB_HOST }} + TF_VAR_db_name: ${{ secrets.DEV_DB_NAME }} + TF_VAR_db_port: ${{ secrets.DEV_DB_PORT }} + TF_VAR_api_key: ${{ secrets.DEV_API_KEY }} + TF_VAR_secret_key: ${{ secrets.DEV_SECRET_KEY }} + TF_VAR_epc_auth_token: ${{ secrets.DEV_EPC_AUTH_TOKEN }} + TF_VAR_google_solar_api_key: ${{ secrets.DEV_GOOGLE_SOLAR_API_KEY }} # ============================================================ - # 2️⃣ Build Address 2 UPRN image and Push + # Build Address 2 UPRN image and Push # ============================================================ address2uprn_image: needs: [determine_stage, shared_terraform] @@ -103,7 +143,7 @@ jobs: EPC_AUTH_TOKEN: ${{ secrets.DEV_EPC_AUTH_TOKEN }} # ============================================================ - # 3️⃣ Deploy Address 2 UPRN Lambda + # Deploy Address 2 UPRN Lambda # ============================================================ address2uprn_lambda: needs: [address2uprn_image, determine_stage] @@ -122,7 +162,7 @@ jobs: # ============================================================ - # 2️⃣ Build Postcode Splitter image and Push + # Build Postcode Splitter image and Push # ============================================================ postcodeSplitter_image: needs: [determine_stage, shared_terraform] @@ -144,7 +184,7 @@ jobs: DEV_DB_NAME: ${{ secrets.DEV_DB_NAME }} # ============================================================ - # 3️⃣ Deploy Postcode Splitter Lambda + # Deploy Postcode Splitter Lambda # ============================================================ postcodeSplitter_lambda: needs: [postcodeSplitter_image, determine_stage, address2uprn_lambda] @@ -242,48 +282,7 @@ jobs: AWS_REGION: ${{ secrets.DEV_AWS_REGION }} # ============================================================ - # Ara Engine image and Push - # ============================================================ - ara_engine_image: - needs: [determine_stage, shared_terraform] - uses: ./.github/workflows/_build_image.yml - with: - ecr_repo: engine-${{ needs.determine_stage.outputs.stage }} - dockerfile_path: backend/docker/engine.Dockerfile - build_context: . - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} - AWS_REGION: ${{ secrets.DEV_AWS_REGION }} - - # ============================================================ - # Deploy Ara Engine Lambda - # ============================================================ - ara_engine_lambda: - needs: [ara_engine_image, determine_stage] - uses: ./.github/workflows/_deploy_lambda.yml - with: - lambda_name: ara_engine - lambda_path: infrastructure/terraform/lambda/engine - stage: ${{ needs.determine_stage.outputs.stage }} - ecr_repo: engine-${{ needs.determine_stage.outputs.stage }} - image_digest: ${{ needs.ara_engine_image.outputs.image_digest }} - terraform_apply: ${{ needs.determine_stage.outputs.terraform_apply }} - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} - AWS_REGION: ${{ secrets.DEV_AWS_REGION }} - TF_VAR_db_host: ${{ secrets.DEV_DB_HOST }} - TF_VAR_db_name: ${{ secrets.DEV_DB_NAME }} - TF_VAR_db_port: ${{ secrets.DEV_DB_PORT }} - TF_VAR_api_key: ${{ secrets.DEV_API_KEY }} - TF_VAR_secret_key: ${{ secrets.DEV_SECRET_KEY }} - TF_VAR_domain_name: ${{ secrets.DEV_DOMAIN_NAME }} - TF_VAR_epc_auth_token: ${{ secrets.DEV_EPC_AUTH_TOKEN }} - TF_VAR_google_solar_api_key: ${{ secrets.DEV_GOOGLE_SOLAR_API_KEY }} - - # ============================================================ - # 2️⃣ Build OrdanceSurvey image and Push + # Build OrdanceSurvey image and Push # ============================================================ ordnanceSurvey_image: needs: [determine_stage, shared_terraform] @@ -305,7 +304,7 @@ jobs: DEV_DB_NAME: ${{ secrets.DEV_DB_NAME }} # ============================================================ - # 3️⃣ Deploy OrdanceSurvey Lambda + # Deploy OrdanceSurvey Lambda # ============================================================ ordnanceSurvey_lambda: needs: [ordnanceSurvey_image, determine_stage] @@ -322,3 +321,116 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.DEV_AWS_REGION }} TF_VAR_ORDNANCE_SURVEY_API_KEY: ${{ secrets.ORDNANCE_SURVEY_API_KEY }} + + # ============================================================ + # Deploy FastAPI Lambda + # ============================================================ + fast_api_lambda: + needs: [determine_stage, ara_engine_lambda, categorisation_lambda] + uses: ./.github/workflows/_deploy_lambda.yml + with: + lambda_name: ara_fast_api + lambda_path: infrastructure/terraform/lambda/fast-api + stage: ${{ needs.determine_stage.outputs.stage }} + terraform_apply: ${{ needs.determine_stage.outputs.terraform_apply }} + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + TF_VAR_db_host: ${{ secrets.DEV_DB_HOST }} + TF_VAR_db_name: ${{ secrets.DEV_DB_NAME }} + TF_VAR_db_port: ${{ secrets.DEV_DB_PORT }} + TF_VAR_api_key: ${{ secrets.FASTAPI_API_KEY }} + TF_VAR_secret_key: ${{ secrets.NEXTAUTH_SECRET }} + TF_VAR_domain_name: ${{ secrets.ARA_DEV_DOMAIN_NAME }} + TF_VAR_epc_auth_token: ${{ secrets.DEV_EPC_AUTH_TOKEN }} + TF_VAR_google_solar_api_key: ${{ secrets.DEV_GOOGLE_SOLAR_API_KEY }} + + # ============================================================ + # Deploy ACM Certificate for Cloudfront + # ============================================================ + cloudfront_acm: + needs: [determine_stage, shared_terraform, fast_api_lambda] + runs-on: ubuntu-latest + + env: + STAGE: ${{ needs.determine_stage.outputs.stage }} + TERRAFORM_APPLY: ${{ needs.determine_stage.outputs.terraform_apply }} + + steps: + - uses: actions/checkout@v4 + + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.DEV_AWS_REGION }} + + - uses: hashicorp/setup-terraform@v3 + + - name: Terraform Init + working-directory: infrastructure/terraform/cdn_certificate + run: terraform init -reconfigure + + - name: Terraform Workspace + working-directory: infrastructure/terraform/cdn_certificate + run: | + terraform workspace select $STAGE \ + || terraform workspace new $STAGE + + - name: Terraform Plan + working-directory: infrastructure/terraform/cdn_certificate + run: | + terraform plan \ + -var="stage=${STAGE}" \ + -out=tfplan + + - name: Terraform Apply + if: env.TERRAFORM_APPLY == 'true' + working-directory: infrastructure/terraform/cdn_certificate + run: terraform apply -auto-approve tfplan + + + # ============================================================ + # Deploy Cloudfront CDN + # ============================================================ + cloudfront_cdn: + needs: [determine_stage, cloudfront_acm] + runs-on: ubuntu-latest + + env: + STAGE: ${{ needs.determine_stage.outputs.stage }} + TERRAFORM_APPLY: ${{ needs.determine_stage.outputs.terraform_apply }} + + steps: + - uses: actions/checkout@v4 + + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.DEV_AWS_REGION }} + + - uses: hashicorp/setup-terraform@v3 + + - name: Terraform Init + working-directory: infrastructure/terraform/cdn + run: terraform init -reconfigure + + - name: Terraform Workspace + working-directory: infrastructure/terraform/cdn + run: | + terraform workspace select $STAGE \ + || terraform workspace new $STAGE + + - name: Terraform Plan + working-directory: infrastructure/terraform/cdn + run: | + terraform plan \ + -var="stage=${STAGE}" \ + -out=tfplan + + - name: Terraform Apply + if: env.TERRAFORM_APPLY == 'true' + working-directory: infrastructure/terraform/cdn + run: terraform apply -auto-approve tfplan \ No newline at end of file diff --git a/.gitignore b/.gitignore index 68e66052..15050bdd 100644 --- a/.gitignore +++ b/.gitignore @@ -246,7 +246,7 @@ etl/epc/local_data/* /backend/condition/sample_data/peabody/* *.DS_Store -infrastructure/terraform/.terraform* +**/.terraform* # Don't commit packages up serverless packages .serverless diff --git a/infrastructure/terraform/cdn/main.tf b/infrastructure/terraform/cdn/main.tf new file mode 100644 index 00000000..56b2f52b --- /dev/null +++ b/infrastructure/terraform/cdn/main.tf @@ -0,0 +1,64 @@ +############################################ +# Load Shared Terraform State +############################################ +data "terraform_remote_state" "shared" { + backend = "s3" + config = { + bucket = "assessment-model-terraform-state" + key = "env:/${var.stage}/terraform.tfstate" + region = "eu-west-2" + } +} + +############################################ +# Load FastAPI Terraform State +############################################ +data "terraform_remote_state" "fast_api" { + backend = "s3" + config = { + bucket = "ara-fast-api-terraform-state" + key = "env:/${var.stage}/terraform.tfstate" + region = "eu-west-2" + } +} + +############################################ +# Load CDN Certificate Terraform State +############################################ +data "terraform_remote_state" "cdn_certificate" { + backend = "s3" + config = { + bucket = "cdn-certificate-terraform-state" + key = "env:/${var.stage}/terraform.tfstate" + region = "eu-west-2" + } +} + +############################################ +# CloudFront for API +############################################ +module "cdn" { + source = "../modules/cloudfront" + + aliases = [data.terraform_remote_state.fast_api.outputs.domain_name] + + acm_certificate_arn = data.terraform_remote_state.cdn_certificate.outputs.certificate_arn + + origins = [ + # ---- S3 ---- + { + origin_type = "s3" + origin_domain_name = data.terraform_remote_state.shared.outputs.retrofit_datalake_bucket_domain_name + origin_id = "s3-origin" + bucket_id = data.terraform_remote_state.shared.outputs.retrofit_datalake_bucket_id + bucket_arn = data.terraform_remote_state.shared.outputs.retrofit_datalake_bucket_arn + }, + + # ---- API Gateway ---- + { + origin_type = "api" + origin_domain_name = replace(data.terraform_remote_state.fast_api.outputs.invoke_url, "/^https?://([^/]*).*/", "$1") + origin_id = "api-origin" + } + ] +} \ No newline at end of file diff --git a/infrastructure/terraform/cdn/variables.tf b/infrastructure/terraform/cdn/variables.tf new file mode 100644 index 00000000..423f0b0f --- /dev/null +++ b/infrastructure/terraform/cdn/variables.tf @@ -0,0 +1,3 @@ +variable "stage" { + type = string +} \ No newline at end of file diff --git a/infrastructure/terraform/cdn_certificate/main.tf b/infrastructure/terraform/cdn_certificate/main.tf new file mode 100644 index 00000000..5c0c178c --- /dev/null +++ b/infrastructure/terraform/cdn_certificate/main.tf @@ -0,0 +1,28 @@ +############################################ +# Load FastAPI Terraform State +############################################ +data "terraform_remote_state" "fast_api" { + backend = "s3" + config = { + bucket = "ara-fast-api-terraform-state" + key = "env:/${var.stage}/terraform.tfstate" + region = "eu-west-2" + } +} + +############################################ +# Define Certificate +############################################ +module "cdn_certificate" { + source = "../modules/acm_certificate" + + providers = { + aws = aws.us_east_1 + } + + domain_name = data.terraform_remote_state.fast_api.outputs.domain_name + + tags = { + Environment = var.stage + } +} \ No newline at end of file diff --git a/infrastructure/terraform/cdn_certificate/outputs.tf b/infrastructure/terraform/cdn_certificate/outputs.tf new file mode 100644 index 00000000..e72c6e13 --- /dev/null +++ b/infrastructure/terraform/cdn_certificate/outputs.tf @@ -0,0 +1,3 @@ +output "certificate_arn" { + value = module.cdn_certificate.certificate_arn +} \ No newline at end of file diff --git a/infrastructure/terraform/cdn_certificate/provider.tf b/infrastructure/terraform/cdn_certificate/provider.tf new file mode 100644 index 00000000..38b01d82 --- /dev/null +++ b/infrastructure/terraform/cdn_certificate/provider.tf @@ -0,0 +1,23 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.0" + } + } + + backend "s3" { + bucket = "cdn-certificate-terraform-state" + key = "terraform.tfstate" + region = "eu-west-2" + } +} + +provider "aws" { + region = var.region +} + +provider "aws" { + alias = "us_east_1" + region = "us-east-1" +} \ No newline at end of file diff --git a/infrastructure/terraform/cdn_certificate/variables.tf b/infrastructure/terraform/cdn_certificate/variables.tf new file mode 100644 index 00000000..423f0b0f --- /dev/null +++ b/infrastructure/terraform/cdn_certificate/variables.tf @@ -0,0 +1,3 @@ +variable "stage" { + type = string +} \ No newline at end of file diff --git a/infrastructure/terraform/lambda/_template/main.tf b/infrastructure/terraform/lambda/_template/main.tf index c6015ea1..81b1c7f1 100644 --- a/infrastructure/terraform/lambda/_template/main.tf +++ b/infrastructure/terraform/lambda/_template/main.tf @@ -26,7 +26,7 @@ data "terraform_remote_state" "shared" { } module "lambda" { - source = "../modules/lambda_with_sqs" + source = "../../modules/lambda_with_sqs" name = REPLACE ME #"address2uprn" for example stage = var.stage diff --git a/infrastructure/terraform/lambda/_template/provider.tf b/infrastructure/terraform/lambda/_template/provider.tf index 37c412ce..3d66f392 100644 --- a/infrastructure/terraform/lambda/_template/provider.tf +++ b/infrastructure/terraform/lambda/_template/provider.tf @@ -2,7 +2,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 4.16" + version = ">= 5.0" } } diff --git a/infrastructure/terraform/lambda/address2UPRN/main.tf b/infrastructure/terraform/lambda/address2UPRN/main.tf index 3afc8738..bc6f9e67 100644 --- a/infrastructure/terraform/lambda/address2UPRN/main.tf +++ b/infrastructure/terraform/lambda/address2UPRN/main.tf @@ -15,7 +15,7 @@ locals { } module "address2uprn" { - source = "../modules/lambda_with_sqs" + source = "../../modules/lambda_with_sqs" name = "address2uprn" stage = var.stage diff --git a/infrastructure/terraform/lambda/address2UPRN/provider.tf b/infrastructure/terraform/lambda/address2UPRN/provider.tf index ad873717..3cfa2400 100644 --- a/infrastructure/terraform/lambda/address2UPRN/provider.tf +++ b/infrastructure/terraform/lambda/address2UPRN/provider.tf @@ -2,7 +2,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 4.16" + version = ">= 5.0" } } diff --git a/infrastructure/terraform/lambda/categorisation/main.tf b/infrastructure/terraform/lambda/categorisation/main.tf index b7193da4..01e16213 100644 --- a/infrastructure/terraform/lambda/categorisation/main.tf +++ b/infrastructure/terraform/lambda/categorisation/main.tf @@ -16,7 +16,7 @@ locals { } module "lambda" { - source = "../modules/lambda_with_sqs" + source = "../../modules/lambda_with_sqs" name = "categorisation" stage = var.stage diff --git a/infrastructure/terraform/lambda/categorisation/outputs.tf b/infrastructure/terraform/lambda/categorisation/outputs.tf new file mode 100644 index 00000000..8e33b8e0 --- /dev/null +++ b/infrastructure/terraform/lambda/categorisation/outputs.tf @@ -0,0 +1,9 @@ +output "categorisation_queue_url" { + value = module.lambda.queue_url + description = "URL of the Categorisation SQS queue" +} + +output "categorisation_queue_arn" { + value = module.lambda.queue_arn + description = "ARN of the Categorisation SQS queue" +} \ No newline at end of file diff --git a/infrastructure/terraform/lambda/categorisation/provider.tf b/infrastructure/terraform/lambda/categorisation/provider.tf index fe497c81..30e73ed2 100644 --- a/infrastructure/terraform/lambda/categorisation/provider.tf +++ b/infrastructure/terraform/lambda/categorisation/provider.tf @@ -2,7 +2,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 4.16" + version = ">= 5.0" } } diff --git a/infrastructure/terraform/lambda/condition-etl/main.tf b/infrastructure/terraform/lambda/condition-etl/main.tf index 0128f975..d654223c 100644 --- a/infrastructure/terraform/lambda/condition-etl/main.tf +++ b/infrastructure/terraform/lambda/condition-etl/main.tf @@ -17,7 +17,7 @@ locals { module "lambda" { - source = "../modules/lambda_with_sqs" + source = "../../modules/lambda_with_sqs" name = "condition-etl" stage = var.stage diff --git a/infrastructure/terraform/lambda/condition-etl/provider.tf b/infrastructure/terraform/lambda/condition-etl/provider.tf index c633d238..f7adf65a 100644 --- a/infrastructure/terraform/lambda/condition-etl/provider.tf +++ b/infrastructure/terraform/lambda/condition-etl/provider.tf @@ -2,7 +2,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 4.16" + version = ">= 5.0" } } diff --git a/infrastructure/terraform/lambda/engine/main.tf b/infrastructure/terraform/lambda/engine/main.tf index 35c00fa3..1f3ce017 100644 --- a/infrastructure/terraform/lambda/engine/main.tf +++ b/infrastructure/terraform/lambda/engine/main.tf @@ -17,7 +17,7 @@ locals { module "lambda" { - source = "../modules/lambda_with_sqs" + source = "../../modules/lambda_with_sqs" name = "engine" stage = var.stage @@ -44,7 +44,6 @@ module "lambda" { DB_PORT = var.db_port API_KEY = var.api_key SECRET_KEY = var.secret_key - DOMAIN_NAME = var.domain_name EPC_AUTH_TOKEN = var.epc_auth_token GOOGLE_SOLAR_API_KEY = var.google_solar_api_key diff --git a/infrastructure/terraform/lambda/engine/outputs.tf b/infrastructure/terraform/lambda/engine/outputs.tf new file mode 100644 index 00000000..c59e0809 --- /dev/null +++ b/infrastructure/terraform/lambda/engine/outputs.tf @@ -0,0 +1,9 @@ +output "ara_engine_queue_url" { + value = module.lambda.queue_url + description = "URL of the Engine SQS queue" +} + +output "ara_engine_queue_arn" { + value = module.lambda.queue_arn + description = "ARN of the Engine SQS queue" +} \ No newline at end of file diff --git a/infrastructure/terraform/lambda/engine/provider.tf b/infrastructure/terraform/lambda/engine/provider.tf index 2895d039..74021fd0 100644 --- a/infrastructure/terraform/lambda/engine/provider.tf +++ b/infrastructure/terraform/lambda/engine/provider.tf @@ -2,7 +2,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 4.16" + version = ">= 5.0" } } diff --git a/infrastructure/terraform/lambda/engine/variables.tf b/infrastructure/terraform/lambda/engine/variables.tf index 0a74ad5b..bf0a42a2 100644 --- a/infrastructure/terraform/lambda/engine/variables.tf +++ b/infrastructure/terraform/lambda/engine/variables.tf @@ -65,10 +65,6 @@ variable "secret_key" { sensitive = true } -variable "domain_name" { - type = string -} - variable "epc_auth_token" { type = string sensitive = true diff --git a/infrastructure/terraform/lambda/fast-api/main.tf b/infrastructure/terraform/lambda/fast-api/main.tf new file mode 100644 index 00000000..f71b6f60 --- /dev/null +++ b/infrastructure/terraform/lambda/fast-api/main.tf @@ -0,0 +1,124 @@ +############################################ +# Load Terraform State +############################################ +data "terraform_remote_state" "shared" { + backend = "s3" + config = { + bucket = "assessment-model-terraform-state" + key = "env:/${var.stage}/terraform.tfstate" + region = "eu-west-2" + } +} + +data "terraform_remote_state" "engine" { + backend = "s3" + config = { + bucket = "ara-engine-terraform-state", + key = "env:/${var.stage}/terraform.tfstate" + region = "eu-west-2" + } +} + +data "terraform_remote_state" "categorisation" { + backend = "s3" + config = { + bucket = "categorisation-terraform-state", + key = "env:/${var.stage}/terraform.tfstate" + region = "eu-west-2" + } +} + +############################################ +# Load Credentials +############################################ +data "aws_secretsmanager_secret_version" "db_credentials" { + secret_id = "${var.stage}/assessment_model/db_credentials" +} + +locals { + db_credentials = jsondecode(data.aws_secretsmanager_secret_version.db_credentials.secret_string) +} + +############################################ +# FastAPI Lambda + API Gateway +############################################ +module "fastapi" { + source = "../../modules/lambda_with_api_gateway" + + name = "fastapi" + stage = var.stage + source_dir = "${path.root}/../../../../" + handler = "backend.app.main.handler" + runtime = "python3.11" + timeout = 600 + memory_size = 512 + artifact_bucket = data.terraform_remote_state.shared.outputs.ara_fast_api_state_bucket + requirements_file = "${path.root}/../../../../backend/app/requirements/requirements.txt" + + domain_name = "api.${var.domain_name}" + + environment = { + ENVIRONMENT = var.stage + API_KEY = var.api_key + SECRET_KEY = var.secret_key + # DOMAIN_NAME = var.domain_name + EPC_AUTH_TOKEN = var.epc_auth_token + GOOGLE_SOLAR_API_KEY = var.google_solar_api_key + + DB_HOST = var.db_host + DB_NAME = var.db_name + DB_PORT = var.db_port + DB_USERNAME = local.db_credentials.db_assessment_model_username + DB_PASSWORD = local.db_credentials.db_assessment_model_password + + PLAN_TRIGGER_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_plan_trigger_bucket_name + DATA_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_sap_data_bucket_name + SAP_PREDICTIONS_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_sap_predictions_bucket_name + CARBON_PREDICTIONS_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_carbon_predictions_bucket_name + HEAT_PREDICTIONS_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_heat_predictions_bucket_name + HEATING_KWH_PREDICTIONS_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_heating_kwh_predictions_bucket_name + HOTWATER_KWH_PREDICTIONS_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_hotwater_kwh_predictions_bucket_name + ENERGY_ASSESSMENTS_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_energy_assessments_bucket_name + + ENGINE_SQS_URL = data.terraform_remote_state.engine.outputs.ara_engine_queue_url + CATEGORISATION_SQS_URL = data.terraform_remote_state.categorisation.outputs.categorisation_queue_url + } +} + +############################################ +# IAM policy attachments +############################################ +# SQS +module "fastapi_sqs_policy" { + source = "../../modules/general_iam_policy" + + policy_name = "fastapi-sqs-send-${var.stage}" + policy_description = "Allow FastAPI to send messages to engine & categorisation queues" + + actions = [ + "sqs:SendMessage" + ] + + resources = [ + data.terraform_remote_state.engine.outputs.ara_engine_queue_arn, + data.terraform_remote_state.categorisation.outputs.categorisation_queue_arn + ] + + conditions = null + + tags = { + Service = "fastapi" + Stage = var.stage + } +} + +resource "aws_iam_role_policy_attachment" "fastapi_sqs_send" { + role = module.fastapi.role_name + policy_arn = module.fastapi_sqs_policy.policy_arn +} + +# S3 +resource "aws_iam_role_policy_attachment" "fastapi_s3_read_and_write" { + role = module.fastapi.role_name + policy_arn = data.terraform_remote_state.shared.outputs.fast_api_s3_read_and_write_arn +} \ No newline at end of file diff --git a/infrastructure/terraform/lambda/fast-api/outputs.tf b/infrastructure/terraform/lambda/fast-api/outputs.tf new file mode 100644 index 00000000..c9fc6f86 --- /dev/null +++ b/infrastructure/terraform/lambda/fast-api/outputs.tf @@ -0,0 +1,7 @@ +output "domain_name" { + value = module.fastapi.domain_name +} + +output "invoke_url" { + value = module.fastapi.invoke_url +} \ No newline at end of file diff --git a/infrastructure/terraform/lambda/fast-api/provider.tf b/infrastructure/terraform/lambda/fast-api/provider.tf new file mode 100644 index 00000000..afe6f3f6 --- /dev/null +++ b/infrastructure/terraform/lambda/fast-api/provider.tf @@ -0,0 +1,16 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.0" + } + } + + backend "s3" { + bucket = "ara-fast-api-terraform-state" + key = "terraform.tfstate" + region = "eu-west-2" + } + + required_version = ">= 1.2.0" +} \ No newline at end of file diff --git a/infrastructure/terraform/lambda/fast-api/variables.tf b/infrastructure/terraform/lambda/fast-api/variables.tf new file mode 100644 index 00000000..a3157590 --- /dev/null +++ b/infrastructure/terraform/lambda/fast-api/variables.tf @@ -0,0 +1,44 @@ +variable "lambda_name" { + type = string + description = "Logical name of the lambda (e.g. address2uprn)" +} + +variable "stage" { + type = string +} + +variable "db_host" { + type = string +} + +variable "db_name" { + type = string +} + +variable "db_port" { + type = string +} + +variable "api_key" { + type = string + sensitive = true +} + +variable "secret_key" { + type = string + sensitive = true +} + +variable "domain_name" { + type = string +} + +variable "epc_auth_token" { + type = string + sensitive = true +} + +variable "google_solar_api_key" { + type = string + sensitive = true +} \ No newline at end of file diff --git a/infrastructure/terraform/lambda/ordnanceSurvey/provider.tf b/infrastructure/terraform/lambda/ordnanceSurvey/provider.tf index b7f453f1..12bd0f85 100644 --- a/infrastructure/terraform/lambda/ordnanceSurvey/provider.tf +++ b/infrastructure/terraform/lambda/ordnanceSurvey/provider.tf @@ -2,7 +2,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 4.16" + version = ">= 5.0" } } diff --git a/infrastructure/terraform/lambda/postcodeSplitter/main.tf b/infrastructure/terraform/lambda/postcodeSplitter/main.tf index d37a01c9..94c5cd4e 100644 --- a/infrastructure/terraform/lambda/postcodeSplitter/main.tf +++ b/infrastructure/terraform/lambda/postcodeSplitter/main.tf @@ -26,7 +26,7 @@ data "terraform_remote_state" "address2uprn" { } module "lambda" { - source = "../modules/lambda_with_sqs" + source = "../../modules/lambda_with_sqs" name = "postcode-splitter" stage = var.stage diff --git a/infrastructure/terraform/lambda/postcodeSplitter/provider.tf b/infrastructure/terraform/lambda/postcodeSplitter/provider.tf index dbe323f2..5749143d 100644 --- a/infrastructure/terraform/lambda/postcodeSplitter/provider.tf +++ b/infrastructure/terraform/lambda/postcodeSplitter/provider.tf @@ -2,7 +2,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 4.16" + version = ">= 5.0" } } diff --git a/infrastructure/terraform/modules/acm_certificate/main.tf b/infrastructure/terraform/modules/acm_certificate/main.tf new file mode 100644 index 00000000..9f813a77 --- /dev/null +++ b/infrastructure/terraform/modules/acm_certificate/main.tf @@ -0,0 +1,11 @@ +resource "aws_acm_certificate" "this" { + domain_name = var.domain_name + subject_alternative_names = var.subject_alternative_names + validation_method = "DNS" + + lifecycle { + create_before_destroy = false + } + + tags = var.tags +} \ No newline at end of file diff --git a/infrastructure/terraform/modules/acm_certificate/outputs.tf b/infrastructure/terraform/modules/acm_certificate/outputs.tf new file mode 100644 index 00000000..990a6db9 --- /dev/null +++ b/infrastructure/terraform/modules/acm_certificate/outputs.tf @@ -0,0 +1,7 @@ +output "certificate_arn" { + value = aws_acm_certificate.this.arn +} + +output "domain_validation_options" { + value = aws_acm_certificate.this.domain_validation_options +} \ No newline at end of file diff --git a/infrastructure/terraform/modules/acm_certificate/variables.tf b/infrastructure/terraform/modules/acm_certificate/variables.tf new file mode 100644 index 00000000..bd30501f --- /dev/null +++ b/infrastructure/terraform/modules/acm_certificate/variables.tf @@ -0,0 +1,16 @@ +variable "domain_name" { + description = "Primary domain name for the certificate" + type = string +} + +variable "subject_alternative_names" { + description = "Additional domains for the certificate" + type = list(string) + default = [] +} + +variable "tags" { + description = "Tags to apply to the certificate" + type = map(string) + default = {} +} \ No newline at end of file diff --git a/infrastructure/terraform/modules/cloudfront/main.tf b/infrastructure/terraform/modules/cloudfront/main.tf index 281ff09f..03082604 100644 --- a/infrastructure/terraform/modules/cloudfront/main.tf +++ b/infrastructure/terraform/modules/cloudfront/main.tf @@ -1,65 +1,165 @@ -resource "aws_cloudfront_distribution" "s3_distribution" { - origin { - domain_name = var.bucket_domain_name - origin_id = "S3-${var.bucket_name}" +############################################# +# Use Managed Caching and Forwarding Policies +############################################# +data "aws_cloudfront_cache_policy" "caching_disabled" { + name = "Managed-CachingDisabled" +} - s3_origin_config { - origin_access_identity = aws_cloudfront_origin_access_identity.oai.cloudfront_access_identity_path +data "aws_cloudfront_origin_request_policy" "all_viewer_except_host_header" { + name = "Managed-AllViewerExceptHostHeader" +} + +############################################ +# CloudFront Distribution +############################################ + +resource "aws_cloudfront_distribution" "this" { + + ########################################## + # Origins + ########################################## + + dynamic "origin" { + for_each = { for o in var.origins : o.origin_id => o } + + content { + domain_name = origin.value.origin_domain_name + origin_id = origin.value.origin_id + + ###################################### + # S3 Origin + ###################################### + dynamic "s3_origin_config" { + for_each = origin.value.origin_type == "s3" ? [1] : [] + + content { + origin_access_identity = aws_cloudfront_origin_access_identity.oai[origin.key].cloudfront_access_identity_path + } + } + + ###################################### + # API Gateway Origin + ###################################### + dynamic "custom_origin_config" { + for_each = origin.value.origin_type == "api" ? [1] : [] + + content { + http_port = 80 + https_port = 443 + origin_protocol_policy = "https-only" + origin_ssl_protocols = ["TLSv1.2"] + } + } } } enabled = true + aliases = var.aliases + + ########################################## + # Default Cache Behavior (S3) + ########################################## default_cache_behavior { - allowed_methods = ["GET", "HEAD"] - cached_methods = ["GET", "HEAD"] - target_origin_id = "S3-${var.bucket_name}" + target_origin_id = "s3-origin" + viewer_protocol_policy = "redirect-to-https" - compress = true + + allowed_methods = ["GET", "HEAD"] + cached_methods = ["GET", "HEAD"] forwarded_values { query_string = false + cookies { forward = "none" } } - min_ttl = 0 - default_ttl = 86400 - max_ttl = 31536000 + compress = true + min_ttl = 0 + default_ttl = 3600 + max_ttl = 86400 + } + + ########################################## + # API Behavior + ########################################## + + ordered_cache_behavior { + path_pattern = "/v1/*" + target_origin_id = "api-origin" + + viewer_protocol_policy = "redirect-to-https" + + allowed_methods = ["GET","HEAD","OPTIONS","PUT","POST","PATCH","DELETE"] + cached_methods = ["GET","HEAD"] + + cache_policy_id = data.aws_cloudfront_cache_policy.caching_disabled.id + origin_request_policy_id = data.aws_cloudfront_origin_request_policy.all_viewer_except_host_header.id } price_class = "PriceClass_All" + ########################################## + # Geo Restrictions + ########################################## + restrictions { geo_restriction { restriction_type = "none" } } + ########################################## + # SSL Certificate + ########################################## + viewer_certificate { - cloudfront_default_certificate = true + acm_certificate_arn = var.acm_certificate_arn + ssl_support_method = var.acm_certificate_arn != null ? "sni-only" : null + minimum_protocol_version = var.acm_certificate_arn != null ? "TLSv1.2_2021" : null + + cloudfront_default_certificate = var.acm_certificate_arn == null } } +############################################ +# Origin Access Identities (S3 only) +############################################ + resource "aws_cloudfront_origin_access_identity" "oai" { - comment = "OAI for ${var.bucket_name}" + for_each = { + for o in var.origins : o.origin_id => o + if o.origin_type == "s3" + } + + comment = "OAI for ${each.key}" } +############################################ +# S3 Bucket Policy (S3 only) +############################################ + resource "aws_s3_bucket_policy" "bucket_policy" { - bucket = var.bucket_id + for_each = { + for o in var.origins : o.origin_id => o + if o.origin_type == "s3" + } + + bucket = each.value.bucket_id policy = jsonencode({ - Version = "2012-10-17" + Version = "2012-10-17" Statement = [ { - Effect = "Allow" + Effect = "Allow" Principal = { - AWS = "arn:aws:iam::cloudfront:user/CloudFront Origin Access Identity ${aws_cloudfront_origin_access_identity.oai.id}" + AWS = aws_cloudfront_origin_access_identity.oai[each.key].iam_arn } Action = "s3:GetObject" - Resource = "${var.bucket_arn}/*" - }, + Resource = "${each.value.bucket_arn}/*" + } ] }) -} +} \ No newline at end of file diff --git a/infrastructure/terraform/modules/cloudfront/variables.tf b/infrastructure/terraform/modules/cloudfront/variables.tf index 88f770a8..4721d3d1 100644 --- a/infrastructure/terraform/modules/cloudfront/variables.tf +++ b/infrastructure/terraform/modules/cloudfront/variables.tf @@ -1,24 +1,20 @@ -variable "bucket_name" { - description = "The name of the bucket" - type = string +variable "origins" { + type = list(object({ + origin_type = string # "s3" or "api" + origin_domain_name = string + origin_id = string + + bucket_id = optional(string) + bucket_arn = optional(string) + })) } -variable "stage" { - description = "The deployment stage" - type = string +variable "aliases" { + type = list(string) } -variable "bucket_id" { - description = "The ID of the S3 bucket" - type = string -} - -variable "bucket_arn" { - description = "The ARN of the S3 bucket" - type = string -} - -variable "bucket_domain_name" { - description = "The regional domain name of the S3 bucket" +variable "acm_certificate_arn" { + description = "ACM certificate ARN for custom aliases" type = string + default = null } \ No newline at end of file diff --git a/infrastructure/terraform/modules/lambda_service_zip/main.tf b/infrastructure/terraform/modules/lambda_service_zip/main.tf new file mode 100644 index 00000000..d52f5ba4 --- /dev/null +++ b/infrastructure/terraform/modules/lambda_service_zip/main.tf @@ -0,0 +1,27 @@ +resource "aws_lambda_function" "this" { + function_name = var.name + role = var.role_arn + package_type = "Zip" + # filename = var.filename + source_code_hash = var.source_code_hash + handler = var.handler + runtime = var.runtime + timeout = var.timeout + memory_size = var.memory_size + publish = true + + s3_bucket = var.s3_bucket + s3_key = var.s3_key + + environment { + variables = var.environment + } +} + +output "lambda_arn" { + value = aws_lambda_function.this.arn +} + +output "function_name" { + value = aws_lambda_function.this.function_name +} diff --git a/infrastructure/terraform/modules/lambda_service_zip/variables.tf b/infrastructure/terraform/modules/lambda_service_zip/variables.tf new file mode 100644 index 00000000..095d4a81 --- /dev/null +++ b/infrastructure/terraform/modules/lambda_service_zip/variables.tf @@ -0,0 +1,20 @@ +variable "name" { type = string } +variable "role_arn" { type = string } +# variable "filename" { type = string } +variable "source_code_hash" { type = string } +variable "handler" { type = string } +variable "runtime" { type = string } +variable "timeout" { + type = number + default = 30 +} +variable "memory_size" { + type = number + default = 128 +} +variable "environment" { + type = map(string) + default = {} +} +variable "s3_bucket" { type = string } +variable "s3_key" { type = string } \ No newline at end of file diff --git a/infrastructure/terraform/modules/lambda_with_api_gateway/main.tf b/infrastructure/terraform/modules/lambda_with_api_gateway/main.tf new file mode 100644 index 00000000..bef4a16c --- /dev/null +++ b/infrastructure/terraform/modules/lambda_with_api_gateway/main.tf @@ -0,0 +1,113 @@ +############################################ +# IAM role +############################################ +module "role" { + source = "../lambda_execution_role" + name = "${var.name}-lambda-${var.stage}" +} + +############################################ +# Cloudwatch log group +############################################ +resource "aws_cloudwatch_log_group" "api_logs" { + name = "/aws/apigateway/${var.name}-${var.stage}" + retention_in_days = 14 +} + +############################################ +# Install python packages +############################################ +resource "null_resource" "pip_install" { + count = var.requirements_file != null ? 1 : 0 + + provisioner "local-exec" { + command = "pip install -r ${var.requirements_file} -t ${var.source_dir} --platform manylinux2014_x86_64 --implementation cp --python-version 3.11 --only-binary=:all: --upgrade" + } +} + +############################################ +# Zip the source code +############################################ +data "archive_file" "this" { + depends_on = [null_resource.pip_install] + type = "zip" + source_dir = var.source_dir + output_path = "${path.module}/lambda_package.zip" + excludes = var.zip_excludes +} + +############################################ +# Upload zip to S3 +############################################ +resource "aws_s3_object" "lambda_zip" { + bucket = var.artifact_bucket + key = "env:/${var.stage}/${var.name}.zip" + source = data.archive_file.this.output_path + etag = data.archive_file.this.output_md5 +} + +############################################ +# Lambda +############################################ +module "lambda" { + source = "../lambda_service_zip" + + name = "${var.name}-${var.stage}" + role_arn = module.role.role_arn + s3_bucket = var.artifact_bucket + s3_key = aws_s3_object.lambda_zip.key + source_code_hash = data.archive_file.this.output_base64sha256 + handler = var.handler + runtime = var.runtime + timeout = var.timeout + memory_size = var.memory_size + environment = var.environment +} + +############################################ +# API Gateway +############################################ +resource "aws_apigatewayv2_api" "this" { + name = "${var.name}-api-${var.stage}" + protocol_type = "HTTP" +} + +resource "aws_apigatewayv2_stage" "this" { + api_id = aws_apigatewayv2_api.this.id + name = "$default" + auto_deploy = true + + access_log_settings { + destination_arn = aws_cloudwatch_log_group.api_logs.arn + + format = jsonencode({ + requestId = "$context.requestId" + domainName = "$context.domainName" + path = "$context.path" + status = "$context.status" + sourceIp = "$context.identity.sourceIp" + userAgent = "$context.identity.userAgent" + }) + } +} + +resource "aws_apigatewayv2_integration" "this" { + api_id = aws_apigatewayv2_api.this.id + integration_type = "AWS_PROXY" + integration_uri = module.lambda.lambda_arn + payload_format_version = "2.0" +} + +resource "aws_apigatewayv2_route" "catch_all" { + api_id = aws_apigatewayv2_api.this.id + route_key = "$default" + target = "integrations/${aws_apigatewayv2_integration.this.id}" +} + +resource "aws_lambda_permission" "apigw_invoke" { + statement_id = "AllowAPIGatewayInvoke" + action = "lambda:InvokeFunction" + function_name = module.lambda.lambda_arn + principal = "apigateway.amazonaws.com" + source_arn = "${aws_apigatewayv2_api.this.execution_arn}/*/*" +} diff --git a/infrastructure/terraform/modules/lambda_with_api_gateway/outputs.tf b/infrastructure/terraform/modules/lambda_with_api_gateway/outputs.tf new file mode 100644 index 00000000..eae0f7d7 --- /dev/null +++ b/infrastructure/terraform/modules/lambda_with_api_gateway/outputs.tf @@ -0,0 +1,11 @@ +output "role_name" { + value = module.role.role_name +} + +output "domain_name" { + value = var.domain_name +} + +output "invoke_url" { + value = aws_apigatewayv2_stage.this.invoke_url +} \ No newline at end of file diff --git a/infrastructure/terraform/modules/lambda_with_api_gateway/variables.tf b/infrastructure/terraform/modules/lambda_with_api_gateway/variables.tf new file mode 100644 index 00000000..95e5acd9 --- /dev/null +++ b/infrastructure/terraform/modules/lambda_with_api_gateway/variables.tf @@ -0,0 +1,48 @@ +variable "name" { type = string } +variable "stage" { type = string } +variable "source_dir" { type = string } +variable "handler" { type = string } +variable "runtime" { type = string } + +variable "zip_excludes" { + type = list(string) + default = [ + "**/__pycache__/**", + "**/*.pyc", + "**/.pytest_cache/**", + "**/tests/**", + "**/infrastructure/**" + ] +} + +variable "timeout" { + type = number + default = 600 +} +variable "memory_size" { + type = number + default = 512 +} +variable "environment" { + type = map(string) + default = {} +} + +variable "domain_name" { + type = string + default = null +} +variable "certificate_arn" { + type = string + default = null +} +variable "route53_zone_id" { + type = string + default = null +} +variable "artifact_bucket" { type = string } + +variable "requirements_file" { + type = string + default = null +} \ No newline at end of file diff --git a/infrastructure/terraform/lambda/modules/lambda_with_sqs/main.tf b/infrastructure/terraform/modules/lambda_with_sqs/main.tf similarity index 85% rename from infrastructure/terraform/lambda/modules/lambda_with_sqs/main.tf rename to infrastructure/terraform/modules/lambda_with_sqs/main.tf index 74345d24..35626487 100644 --- a/infrastructure/terraform/lambda/modules/lambda_with_sqs/main.tf +++ b/infrastructure/terraform/modules/lambda_with_sqs/main.tf @@ -2,7 +2,7 @@ # IAM role ############################################ module "role" { - source = "../../../modules/lambda_execution_role" + source = "../lambda_execution_role" name = "${var.name}-lambda-${var.stage}" } @@ -14,7 +14,7 @@ output "role_name" { # SQS queue + DLQ ############################################ module "queue" { - source = "../../../modules/sqs_queue" + source = "../sqs_queue" name = "${var.name}-queue-${var.stage}" } @@ -22,7 +22,7 @@ module "queue" { # Lambda ############################################ module "lambda" { - source = "../../../modules/lambda_service" + source = "../lambda_service" name = "${var.name}-${var.stage}" role_arn = module.role.role_arn @@ -38,7 +38,7 @@ module "lambda" { # SQS → Lambda trigger ############################################ module "sqs_trigger" { - source = "../../../modules/lambda_sqs_trigger" + source = "../lambda_sqs_trigger" lambda_arn = module.lambda.lambda_arn lambda_role_name = module.role.role_name diff --git a/infrastructure/terraform/lambda/modules/lambda_with_sqs/outputs.tf b/infrastructure/terraform/modules/lambda_with_sqs/outputs.tf similarity index 100% rename from infrastructure/terraform/lambda/modules/lambda_with_sqs/outputs.tf rename to infrastructure/terraform/modules/lambda_with_sqs/outputs.tf diff --git a/infrastructure/terraform/lambda/modules/lambda_with_sqs/variables.tf b/infrastructure/terraform/modules/lambda_with_sqs/variables.tf similarity index 100% rename from infrastructure/terraform/lambda/modules/lambda_with_sqs/variables.tf rename to infrastructure/terraform/modules/lambda_with_sqs/variables.tf diff --git a/infrastructure/terraform/modules/s3_iam_policy/main.tf b/infrastructure/terraform/modules/s3_iam_policy/main.tf index 397bd963..0ef5c4be 100644 --- a/infrastructure/terraform/modules/s3_iam_policy/main.tf +++ b/infrastructure/terraform/modules/s3_iam_policy/main.tf @@ -2,9 +2,10 @@ locals { # Generate full resource ARNs by combining bucket ARNs with resource paths resources = flatten([ - for bucket_arn in var.bucket_arns : [ - for path in var.resource_paths : "${bucket_arn}${path}" - ] + for bucket_arn in var.bucket_arns : concat( + [bucket_arn], # bare ARN for bucket-level actions like ListBucket + [for path in var.resource_paths : "${bucket_arn}${path}"] + ) ]) } diff --git a/infrastructure/terraform/shared/main.tf b/infrastructure/terraform/shared/main.tf index 4cf5ac46..486f79ca 100644 --- a/infrastructure/terraform/shared/main.tf +++ b/infrastructure/terraform/shared/main.tf @@ -2,7 +2,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 4.16" + version = ">= 5.0" } } backend "s3" { @@ -127,6 +127,22 @@ module "s3" { allowed_origins = var.allowed_origins } +output "retrofit_datalake_bucket_id" { + value = module.s3.bucket_id +} + +output "retrofit_datalake_bucket_arn" { + value = module.s3.bucket_arn +} + +output "retrofit_datalake_bucket_domain_name" { + value = module.s3.bucket_domain_name +} + +output "retrofit_datalake_bucket_name" { + value = module.s3.bucket_name +} + module "model_directory" { source = "../modules/s3" bucketname = "retrofit-model-directory-${var.stage}" @@ -311,18 +327,6 @@ module "sap_baseline_ecr" { source = "../modules/ecr" } -############################################## -# CDN - Cloudfront -############################################## -module "cloudfront_distribution" { - source = "../modules/cloudfront" - bucket_name = module.s3.bucket_name - bucket_id = module.s3.bucket_id - bucket_arn = module.s3.bucket_arn - bucket_domain_name = module.s3.bucket_domain_name - stage = var.stage -} - ################################################ # SES - Email sending ################################################ @@ -521,3 +525,52 @@ module "engine_s3_read_and_write" { output "engine_s3_read_and_write_arn" { value = module.engine_s3_read_and_write.policy_arn } + +################################################ +# FastAPI – Lambda +################################################ +module "ara_fast_api_state_bucket" { + source = "../modules/tf_state_bucket" + bucket_name = "ara-fast-api-terraform-state" +} + +output "ara_fast_api_state_bucket" { + value = module.ara_fast_api_state_bucket.bucket_name +} + +# S3 policy for FastAPI app to read and write from various S3 buckets +module "fast_api_s3_read_and_write" { + source = "../modules/s3_iam_policy" + + policy_name = "FastAPIReadandWriteS3" + policy_description = "Allow FastAPI Lambda to read from and write to various S3 buckets" + bucket_arns = [ + "arn:aws:s3:::${module.s3_presignable_bucket.bucket_name}", + "arn:aws:s3:::${module.retrofit_sap_data.bucket_name}", + "arn:aws:s3:::${module.retrofit_sap_predictions.bucket_name}", + "arn:aws:s3:::${module.retrofit_carbon_predictions.bucket_name}", + "arn:aws:s3:::${module.retrofit_heat_predictions.bucket_name}", + "arn:aws:s3:::${module.retrofit_heating_kwh_predictions.bucket_name}", + "arn:aws:s3:::${module.retrofit_hotwater_kwh_predictions.bucket_name}", + "arn:aws:s3:::${module.retrofit_energy_assessments.bucket_name}" + ] + actions = ["s3:GetObject", "s3:ListBucket"] + resource_paths = ["/*"] +} + +output "fast_api_s3_read_and_write_arn" { + value = module.fast_api_s3_read_and_write.policy_arn +} + +################################################ +# CDN Certificate +################################################ +module "cdn_certificate_state_bucket" { + source = "../modules/tf_state_bucket" + bucket_name = "cdn-certificate-terraform-state" +} + +output "cdn_certificate_state_bucket" { + value = module.cdn_certificate_state_bucket.bucket_name +} +