Merge branch 'main' into feature/hubspot_etl

This commit is contained in:
Jun-te Kim 2026-03-17 17:15:29 +00:00
commit 05cae87733
44 changed files with 994 additions and 135 deletions

View file

@ -16,12 +16,14 @@ on:
type: string
ecr_repo:
required: true
required: false
type: string
default: ''
image_digest:
required: true
required: false
type: string
default: ''
terraform_apply:
required: false
@ -117,13 +119,23 @@ jobs:
TF_VAR_domain_name: ${{ secrets.TF_VAR_domain_name }}
TF_VAR_epc_auth_token: ${{ secrets.TF_VAR_epc_auth_token }}
TF_VAR_google_solar_api_key: ${{ secrets.TF_VAR_google_solar_api_key }}
TF_VAR_ordnance_survey_api_key: ${{ secrets.TF_VAR_ordnance_survey_api_key}}
TF_VAR_ordnance_survey_api_key: ${{ secrets.TF_VAR_ordnance_survey_api_key }}
run: |
ECR_REPO_URL_VAR=""
if [[ -n "${{ inputs.ecr_repo }}" ]]; then
ECR_REPO_URL_VAR="-var=ecr_repo_url=${{ steps.repo.outputs.ecr_repo_url }}"
fi
IMAGE_DIGEST_VAR=""
if [[ -n "${{ inputs.ecr_repo }}" ]]; then
IMAGE_DIGEST_VAR="-var=image_digest=${{ inputs.image_digest }}"
fi
terraform plan \
-var="stage=${{ inputs.stage }}" \
-var="lambda_name=${{ inputs.lambda_name }}" \
-var="ecr_repo_url=${{ steps.repo.outputs.ecr_repo_url }}" \
-var="image_digest=${{ inputs.image_digest }}" \
$ECR_REPO_URL_VAR \
$IMAGE_DIGEST_VAR \
-out=lambdaplan
- name: Terraform Apply
@ -143,10 +155,14 @@ jobs:
TF_VAR_domain_name: ${{ secrets.TF_VAR_domain_name }}
TF_VAR_epc_auth_token: ${{ secrets.TF_VAR_epc_auth_token }}
TF_VAR_google_solar_api_key: ${{ secrets.TF_VAR_google_solar_api_key }}
TF_VAR_ordnance_survey_api_key: ${{ secrets.TF_VAR_ordnance_survey_api_key}}
TF_VAR_ordnance_survey_api_key: ${{ secrets.TF_VAR_ordnance_survey_api_key }}
run: |
EXTRA_VARS=""
if [[ -n "${{ inputs.ecr_repo }}" ]]; then
EXTRA_VARS="-var=ecr_repo_url=${{ steps.repo.outputs.ecr_repo_url }} -var=image_digest=${{ inputs.image_digest }}"
fi
terraform destroy -auto-approve \
-var="stage=${{ inputs.stage }}" \
-var="lambda_name=${{ inputs.lambda_name }}" \
-var="ecr_repo_url=${{ steps.repo.outputs.ecr_repo_url }}" \
-var="image_digest=${{ inputs.image_digest }}"
$EXTRA_VARS

View file

@ -41,7 +41,7 @@ jobs:
fi
# ============================================================
# 1 Shared Terraform (infra)
# Shared Terraform (infra)
# ============================================================
shared_terraform:
needs: determine_stage
@ -77,9 +77,49 @@ jobs:
if: env.TERRAFORM_APPLY == 'true'
working-directory: infrastructure/terraform/shared
run: terraform apply -auto-approve tfplan
# ============================================================
# Ara Engine image and Push
# ============================================================
ara_engine_image:
needs: [determine_stage, shared_terraform]
uses: ./.github/workflows/_build_image.yml
with:
ecr_repo: engine-${{ needs.determine_stage.outputs.stage }}
dockerfile_path: backend/docker/engine.Dockerfile
build_context: .
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ secrets.DEV_AWS_REGION }}
# ============================================================
# Deploy Ara Engine Lambda
# ============================================================
ara_engine_lambda:
needs: [ara_engine_image, determine_stage]
uses: ./.github/workflows/_deploy_lambda.yml
with:
lambda_name: ara_engine
lambda_path: infrastructure/terraform/lambda/engine
stage: ${{ needs.determine_stage.outputs.stage }}
ecr_repo: engine-${{ needs.determine_stage.outputs.stage }}
image_digest: ${{ needs.ara_engine_image.outputs.image_digest }}
terraform_apply: ${{ needs.determine_stage.outputs.terraform_apply }}
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ secrets.DEV_AWS_REGION }}
TF_VAR_db_host: ${{ secrets.DEV_DB_HOST }}
TF_VAR_db_name: ${{ secrets.DEV_DB_NAME }}
TF_VAR_db_port: ${{ secrets.DEV_DB_PORT }}
TF_VAR_api_key: ${{ secrets.DEV_API_KEY }}
TF_VAR_secret_key: ${{ secrets.DEV_SECRET_KEY }}
TF_VAR_epc_auth_token: ${{ secrets.DEV_EPC_AUTH_TOKEN }}
TF_VAR_google_solar_api_key: ${{ secrets.DEV_GOOGLE_SOLAR_API_KEY }}
# ============================================================
# 2⃣ Build Address 2 UPRN image and Push
# Build Address 2 UPRN image and Push
# ============================================================
address2uprn_image:
needs: [determine_stage, shared_terraform]
@ -103,7 +143,7 @@ jobs:
EPC_AUTH_TOKEN: ${{ secrets.DEV_EPC_AUTH_TOKEN }}
# ============================================================
# 3 Deploy Address 2 UPRN Lambda
# Deploy Address 2 UPRN Lambda
# ============================================================
address2uprn_lambda:
needs: [address2uprn_image, determine_stage]
@ -122,7 +162,7 @@ jobs:
# ============================================================
# 2 Build Postcode Splitter image and Push
# Build Postcode Splitter image and Push
# ============================================================
postcodeSplitter_image:
needs: [determine_stage, shared_terraform]
@ -144,7 +184,7 @@ jobs:
DEV_DB_NAME: ${{ secrets.DEV_DB_NAME }}
# ============================================================
# 3 Deploy Postcode Splitter Lambda
# Deploy Postcode Splitter Lambda
# ============================================================
postcodeSplitter_lambda:
needs: [postcodeSplitter_image, determine_stage, address2uprn_lambda]
@ -242,48 +282,7 @@ jobs:
AWS_REGION: ${{ secrets.DEV_AWS_REGION }}
# ============================================================
# Ara Engine image and Push
# ============================================================
ara_engine_image:
needs: [determine_stage, shared_terraform]
uses: ./.github/workflows/_build_image.yml
with:
ecr_repo: engine-${{ needs.determine_stage.outputs.stage }}
dockerfile_path: backend/docker/engine.Dockerfile
build_context: .
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ secrets.DEV_AWS_REGION }}
# ============================================================
# Deploy Ara Engine Lambda
# ============================================================
ara_engine_lambda:
needs: [ara_engine_image, determine_stage]
uses: ./.github/workflows/_deploy_lambda.yml
with:
lambda_name: ara_engine
lambda_path: infrastructure/terraform/lambda/engine
stage: ${{ needs.determine_stage.outputs.stage }}
ecr_repo: engine-${{ needs.determine_stage.outputs.stage }}
image_digest: ${{ needs.ara_engine_image.outputs.image_digest }}
terraform_apply: ${{ needs.determine_stage.outputs.terraform_apply }}
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ secrets.DEV_AWS_REGION }}
TF_VAR_db_host: ${{ secrets.DEV_DB_HOST }}
TF_VAR_db_name: ${{ secrets.DEV_DB_NAME }}
TF_VAR_db_port: ${{ secrets.DEV_DB_PORT }}
TF_VAR_api_key: ${{ secrets.DEV_API_KEY }}
TF_VAR_secret_key: ${{ secrets.DEV_SECRET_KEY }}
TF_VAR_domain_name: ${{ secrets.DEV_DOMAIN_NAME }}
TF_VAR_epc_auth_token: ${{ secrets.DEV_EPC_AUTH_TOKEN }}
TF_VAR_google_solar_api_key: ${{ secrets.DEV_GOOGLE_SOLAR_API_KEY }}
# ============================================================
# 2⃣ Build OrdanceSurvey image and Push
# Build OrdanceSurvey image and Push
# ============================================================
ordnanceSurvey_image:
needs: [determine_stage, shared_terraform]
@ -305,7 +304,7 @@ jobs:
DEV_DB_NAME: ${{ secrets.DEV_DB_NAME }}
# ============================================================
# 3 Deploy OrdanceSurvey Lambda
# Deploy OrdanceSurvey Lambda
# ============================================================
ordnanceSurvey_lambda:
needs: [ordnanceSurvey_image, determine_stage]
@ -322,3 +321,116 @@ jobs:
AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ secrets.DEV_AWS_REGION }}
TF_VAR_ORDNANCE_SURVEY_API_KEY: ${{ secrets.ORDNANCE_SURVEY_API_KEY }}
# ============================================================
# Deploy FastAPI Lambda
# ============================================================
fast_api_lambda:
needs: [determine_stage, ara_engine_lambda, categorisation_lambda]
uses: ./.github/workflows/_deploy_lambda.yml
with:
lambda_name: ara_fast_api
lambda_path: infrastructure/terraform/lambda/fast-api
stage: ${{ needs.determine_stage.outputs.stage }}
terraform_apply: ${{ needs.determine_stage.outputs.terraform_apply }}
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ secrets.DEV_AWS_REGION }}
TF_VAR_db_host: ${{ secrets.DEV_DB_HOST }}
TF_VAR_db_name: ${{ secrets.DEV_DB_NAME }}
TF_VAR_db_port: ${{ secrets.DEV_DB_PORT }}
TF_VAR_api_key: ${{ secrets.FASTAPI_API_KEY }}
TF_VAR_secret_key: ${{ secrets.NEXTAUTH_SECRET }}
TF_VAR_domain_name: ${{ secrets.ARA_DEV_DOMAIN_NAME }}
TF_VAR_epc_auth_token: ${{ secrets.DEV_EPC_AUTH_TOKEN }}
TF_VAR_google_solar_api_key: ${{ secrets.DEV_GOOGLE_SOLAR_API_KEY }}
# ============================================================
# Deploy ACM Certificate for Cloudfront
# ============================================================
cloudfront_acm:
needs: [determine_stage, shared_terraform, fast_api_lambda]
runs-on: ubuntu-latest
env:
STAGE: ${{ needs.determine_stage.outputs.stage }}
TERRAFORM_APPLY: ${{ needs.determine_stage.outputs.terraform_apply }}
steps:
- uses: actions/checkout@v4
- uses: aws-actions/configure-aws-credentials@v4
with:
aws-access-key-id: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ secrets.DEV_AWS_REGION }}
- uses: hashicorp/setup-terraform@v3
- name: Terraform Init
working-directory: infrastructure/terraform/cdn_certificate
run: terraform init -reconfigure
- name: Terraform Workspace
working-directory: infrastructure/terraform/cdn_certificate
run: |
terraform workspace select $STAGE \
|| terraform workspace new $STAGE
- name: Terraform Plan
working-directory: infrastructure/terraform/cdn_certificate
run: |
terraform plan \
-var="stage=${STAGE}" \
-out=tfplan
- name: Terraform Apply
if: env.TERRAFORM_APPLY == 'true'
working-directory: infrastructure/terraform/cdn_certificate
run: terraform apply -auto-approve tfplan
# ============================================================
# Deploy Cloudfront CDN
# ============================================================
cloudfront_cdn:
needs: [determine_stage, cloudfront_acm]
runs-on: ubuntu-latest
env:
STAGE: ${{ needs.determine_stage.outputs.stage }}
TERRAFORM_APPLY: ${{ needs.determine_stage.outputs.terraform_apply }}
steps:
- uses: actions/checkout@v4
- uses: aws-actions/configure-aws-credentials@v4
with:
aws-access-key-id: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ secrets.DEV_AWS_REGION }}
- uses: hashicorp/setup-terraform@v3
- name: Terraform Init
working-directory: infrastructure/terraform/cdn
run: terraform init -reconfigure
- name: Terraform Workspace
working-directory: infrastructure/terraform/cdn
run: |
terraform workspace select $STAGE \
|| terraform workspace new $STAGE
- name: Terraform Plan
working-directory: infrastructure/terraform/cdn
run: |
terraform plan \
-var="stage=${STAGE}" \
-out=tfplan
- name: Terraform Apply
if: env.TERRAFORM_APPLY == 'true'
working-directory: infrastructure/terraform/cdn
run: terraform apply -auto-approve tfplan

2
.gitignore vendored
View file

@ -246,7 +246,7 @@ etl/epc/local_data/*
/backend/condition/sample_data/peabody/*
*.DS_Store
infrastructure/terraform/.terraform*
**/.terraform*
# Don't commit packages up serverless packages
.serverless

View file

@ -0,0 +1,64 @@
############################################
# Load Shared Terraform State
############################################
data "terraform_remote_state" "shared" {
backend = "s3"
config = {
bucket = "assessment-model-terraform-state"
key = "env:/${var.stage}/terraform.tfstate"
region = "eu-west-2"
}
}
############################################
# Load FastAPI Terraform State
############################################
data "terraform_remote_state" "fast_api" {
backend = "s3"
config = {
bucket = "ara-fast-api-terraform-state"
key = "env:/${var.stage}/terraform.tfstate"
region = "eu-west-2"
}
}
############################################
# Load CDN Certificate Terraform State
############################################
data "terraform_remote_state" "cdn_certificate" {
backend = "s3"
config = {
bucket = "cdn-certificate-terraform-state"
key = "env:/${var.stage}/terraform.tfstate"
region = "eu-west-2"
}
}
############################################
# CloudFront for API
############################################
module "cdn" {
source = "../modules/cloudfront"
aliases = [data.terraform_remote_state.fast_api.outputs.domain_name]
acm_certificate_arn = data.terraform_remote_state.cdn_certificate.outputs.certificate_arn
origins = [
# ---- S3 ----
{
origin_type = "s3"
origin_domain_name = data.terraform_remote_state.shared.outputs.retrofit_datalake_bucket_domain_name
origin_id = "s3-origin"
bucket_id = data.terraform_remote_state.shared.outputs.retrofit_datalake_bucket_id
bucket_arn = data.terraform_remote_state.shared.outputs.retrofit_datalake_bucket_arn
},
# ---- API Gateway ----
{
origin_type = "api"
origin_domain_name = replace(data.terraform_remote_state.fast_api.outputs.invoke_url, "/^https?://([^/]*).*/", "$1")
origin_id = "api-origin"
}
]
}

View file

@ -0,0 +1,3 @@
variable "stage" {
type = string
}

View file

@ -0,0 +1,28 @@
############################################
# Load FastAPI Terraform State
############################################
data "terraform_remote_state" "fast_api" {
backend = "s3"
config = {
bucket = "ara-fast-api-terraform-state"
key = "env:/${var.stage}/terraform.tfstate"
region = "eu-west-2"
}
}
############################################
# Define Certificate
############################################
module "cdn_certificate" {
source = "../modules/acm_certificate"
providers = {
aws = aws.us_east_1
}
domain_name = data.terraform_remote_state.fast_api.outputs.domain_name
tags = {
Environment = var.stage
}
}

View file

@ -0,0 +1,3 @@
output "certificate_arn" {
value = module.cdn_certificate.certificate_arn
}

View file

@ -0,0 +1,23 @@
terraform {
required_providers {
aws = {
source = "hashicorp/aws"
version = ">= 5.0"
}
}
backend "s3" {
bucket = "cdn-certificate-terraform-state"
key = "terraform.tfstate"
region = "eu-west-2"
}
}
provider "aws" {
region = var.region
}
provider "aws" {
alias = "us_east_1"
region = "us-east-1"
}

View file

@ -0,0 +1,3 @@
variable "stage" {
type = string
}

View file

@ -26,7 +26,7 @@ data "terraform_remote_state" "shared" {
}
module "lambda" {
source = "../modules/lambda_with_sqs"
source = "../../modules/lambda_with_sqs"
name = REPLACE ME #"address2uprn" for example
stage = var.stage

View file

@ -2,7 +2,7 @@ terraform {
required_providers {
aws = {
source = "hashicorp/aws"
version = "~> 4.16"
version = ">= 5.0"
}
}

View file

@ -15,7 +15,7 @@ locals {
}
module "address2uprn" {
source = "../modules/lambda_with_sqs"
source = "../../modules/lambda_with_sqs"
name = "address2uprn"
stage = var.stage

View file

@ -2,7 +2,7 @@ terraform {
required_providers {
aws = {
source = "hashicorp/aws"
version = "~> 4.16"
version = ">= 5.0"
}
}

View file

@ -16,7 +16,7 @@ locals {
}
module "lambda" {
source = "../modules/lambda_with_sqs"
source = "../../modules/lambda_with_sqs"
name = "categorisation"
stage = var.stage

View file

@ -0,0 +1,9 @@
output "categorisation_queue_url" {
value = module.lambda.queue_url
description = "URL of the Categorisation SQS queue"
}
output "categorisation_queue_arn" {
value = module.lambda.queue_arn
description = "ARN of the Categorisation SQS queue"
}

View file

@ -2,7 +2,7 @@ terraform {
required_providers {
aws = {
source = "hashicorp/aws"
version = "~> 4.16"
version = ">= 5.0"
}
}

View file

@ -17,7 +17,7 @@ locals {
module "lambda" {
source = "../modules/lambda_with_sqs"
source = "../../modules/lambda_with_sqs"
name = "condition-etl"
stage = var.stage

View file

@ -2,7 +2,7 @@ terraform {
required_providers {
aws = {
source = "hashicorp/aws"
version = "~> 4.16"
version = ">= 5.0"
}
}

View file

@ -17,7 +17,7 @@ locals {
module "lambda" {
source = "../modules/lambda_with_sqs"
source = "../../modules/lambda_with_sqs"
name = "engine"
stage = var.stage
@ -44,7 +44,6 @@ module "lambda" {
DB_PORT = var.db_port
API_KEY = var.api_key
SECRET_KEY = var.secret_key
DOMAIN_NAME = var.domain_name
EPC_AUTH_TOKEN = var.epc_auth_token
GOOGLE_SOLAR_API_KEY = var.google_solar_api_key

View file

@ -0,0 +1,9 @@
output "ara_engine_queue_url" {
value = module.lambda.queue_url
description = "URL of the Engine SQS queue"
}
output "ara_engine_queue_arn" {
value = module.lambda.queue_arn
description = "ARN of the Engine SQS queue"
}

View file

@ -2,7 +2,7 @@ terraform {
required_providers {
aws = {
source = "hashicorp/aws"
version = "~> 4.16"
version = ">= 5.0"
}
}

View file

@ -65,10 +65,6 @@ variable "secret_key" {
sensitive = true
}
variable "domain_name" {
type = string
}
variable "epc_auth_token" {
type = string
sensitive = true

View file

@ -0,0 +1,124 @@
############################################
# Load Terraform State
############################################
data "terraform_remote_state" "shared" {
backend = "s3"
config = {
bucket = "assessment-model-terraform-state"
key = "env:/${var.stage}/terraform.tfstate"
region = "eu-west-2"
}
}
data "terraform_remote_state" "engine" {
backend = "s3"
config = {
bucket = "ara-engine-terraform-state",
key = "env:/${var.stage}/terraform.tfstate"
region = "eu-west-2"
}
}
data "terraform_remote_state" "categorisation" {
backend = "s3"
config = {
bucket = "categorisation-terraform-state",
key = "env:/${var.stage}/terraform.tfstate"
region = "eu-west-2"
}
}
############################################
# Load Credentials
############################################
data "aws_secretsmanager_secret_version" "db_credentials" {
secret_id = "${var.stage}/assessment_model/db_credentials"
}
locals {
db_credentials = jsondecode(data.aws_secretsmanager_secret_version.db_credentials.secret_string)
}
############################################
# FastAPI Lambda + API Gateway
############################################
module "fastapi" {
source = "../../modules/lambda_with_api_gateway"
name = "fastapi"
stage = var.stage
source_dir = "${path.root}/../../../../"
handler = "backend.app.main.handler"
runtime = "python3.11"
timeout = 600
memory_size = 512
artifact_bucket = data.terraform_remote_state.shared.outputs.ara_fast_api_state_bucket
requirements_file = "${path.root}/../../../../backend/app/requirements/requirements.txt"
domain_name = "api.${var.domain_name}"
environment = {
ENVIRONMENT = var.stage
API_KEY = var.api_key
SECRET_KEY = var.secret_key
# DOMAIN_NAME = var.domain_name
EPC_AUTH_TOKEN = var.epc_auth_token
GOOGLE_SOLAR_API_KEY = var.google_solar_api_key
DB_HOST = var.db_host
DB_NAME = var.db_name
DB_PORT = var.db_port
DB_USERNAME = local.db_credentials.db_assessment_model_username
DB_PASSWORD = local.db_credentials.db_assessment_model_password
PLAN_TRIGGER_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_plan_trigger_bucket_name
DATA_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_sap_data_bucket_name
SAP_PREDICTIONS_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_sap_predictions_bucket_name
CARBON_PREDICTIONS_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_carbon_predictions_bucket_name
HEAT_PREDICTIONS_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_heat_predictions_bucket_name
HEATING_KWH_PREDICTIONS_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_heating_kwh_predictions_bucket_name
HOTWATER_KWH_PREDICTIONS_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_hotwater_kwh_predictions_bucket_name
ENERGY_ASSESSMENTS_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_energy_assessments_bucket_name
ENGINE_SQS_URL = data.terraform_remote_state.engine.outputs.ara_engine_queue_url
CATEGORISATION_SQS_URL = data.terraform_remote_state.categorisation.outputs.categorisation_queue_url
}
}
############################################
# IAM policy attachments
############################################
# SQS
module "fastapi_sqs_policy" {
source = "../../modules/general_iam_policy"
policy_name = "fastapi-sqs-send-${var.stage}"
policy_description = "Allow FastAPI to send messages to engine & categorisation queues"
actions = [
"sqs:SendMessage"
]
resources = [
data.terraform_remote_state.engine.outputs.ara_engine_queue_arn,
data.terraform_remote_state.categorisation.outputs.categorisation_queue_arn
]
conditions = null
tags = {
Service = "fastapi"
Stage = var.stage
}
}
resource "aws_iam_role_policy_attachment" "fastapi_sqs_send" {
role = module.fastapi.role_name
policy_arn = module.fastapi_sqs_policy.policy_arn
}
# S3
resource "aws_iam_role_policy_attachment" "fastapi_s3_read_and_write" {
role = module.fastapi.role_name
policy_arn = data.terraform_remote_state.shared.outputs.fast_api_s3_read_and_write_arn
}

View file

@ -0,0 +1,7 @@
output "domain_name" {
value = module.fastapi.domain_name
}
output "invoke_url" {
value = module.fastapi.invoke_url
}

View file

@ -0,0 +1,16 @@
terraform {
required_providers {
aws = {
source = "hashicorp/aws"
version = ">= 5.0"
}
}
backend "s3" {
bucket = "ara-fast-api-terraform-state"
key = "terraform.tfstate"
region = "eu-west-2"
}
required_version = ">= 1.2.0"
}

View file

@ -0,0 +1,44 @@
variable "lambda_name" {
type = string
description = "Logical name of the lambda (e.g. address2uprn)"
}
variable "stage" {
type = string
}
variable "db_host" {
type = string
}
variable "db_name" {
type = string
}
variable "db_port" {
type = string
}
variable "api_key" {
type = string
sensitive = true
}
variable "secret_key" {
type = string
sensitive = true
}
variable "domain_name" {
type = string
}
variable "epc_auth_token" {
type = string
sensitive = true
}
variable "google_solar_api_key" {
type = string
sensitive = true
}

View file

@ -2,7 +2,7 @@ terraform {
required_providers {
aws = {
source = "hashicorp/aws"
version = "~> 4.16"
version = ">= 5.0"
}
}

View file

@ -26,7 +26,7 @@ data "terraform_remote_state" "address2uprn" {
}
module "lambda" {
source = "../modules/lambda_with_sqs"
source = "../../modules/lambda_with_sqs"
name = "postcode-splitter"
stage = var.stage

View file

@ -2,7 +2,7 @@ terraform {
required_providers {
aws = {
source = "hashicorp/aws"
version = "~> 4.16"
version = ">= 5.0"
}
}

View file

@ -0,0 +1,11 @@
resource "aws_acm_certificate" "this" {
domain_name = var.domain_name
subject_alternative_names = var.subject_alternative_names
validation_method = "DNS"
lifecycle {
create_before_destroy = false
}
tags = var.tags
}

View file

@ -0,0 +1,7 @@
output "certificate_arn" {
value = aws_acm_certificate.this.arn
}
output "domain_validation_options" {
value = aws_acm_certificate.this.domain_validation_options
}

View file

@ -0,0 +1,16 @@
variable "domain_name" {
description = "Primary domain name for the certificate"
type = string
}
variable "subject_alternative_names" {
description = "Additional domains for the certificate"
type = list(string)
default = []
}
variable "tags" {
description = "Tags to apply to the certificate"
type = map(string)
default = {}
}

View file

@ -1,65 +1,165 @@
resource "aws_cloudfront_distribution" "s3_distribution" {
origin {
domain_name = var.bucket_domain_name
origin_id = "S3-${var.bucket_name}"
#############################################
# Use Managed Caching and Forwarding Policies
#############################################
data "aws_cloudfront_cache_policy" "caching_disabled" {
name = "Managed-CachingDisabled"
}
s3_origin_config {
origin_access_identity = aws_cloudfront_origin_access_identity.oai.cloudfront_access_identity_path
data "aws_cloudfront_origin_request_policy" "all_viewer_except_host_header" {
name = "Managed-AllViewerExceptHostHeader"
}
############################################
# CloudFront Distribution
############################################
resource "aws_cloudfront_distribution" "this" {
##########################################
# Origins
##########################################
dynamic "origin" {
for_each = { for o in var.origins : o.origin_id => o }
content {
domain_name = origin.value.origin_domain_name
origin_id = origin.value.origin_id
######################################
# S3 Origin
######################################
dynamic "s3_origin_config" {
for_each = origin.value.origin_type == "s3" ? [1] : []
content {
origin_access_identity = aws_cloudfront_origin_access_identity.oai[origin.key].cloudfront_access_identity_path
}
}
######################################
# API Gateway Origin
######################################
dynamic "custom_origin_config" {
for_each = origin.value.origin_type == "api" ? [1] : []
content {
http_port = 80
https_port = 443
origin_protocol_policy = "https-only"
origin_ssl_protocols = ["TLSv1.2"]
}
}
}
}
enabled = true
aliases = var.aliases
##########################################
# Default Cache Behavior (S3)
##########################################
default_cache_behavior {
allowed_methods = ["GET", "HEAD"]
cached_methods = ["GET", "HEAD"]
target_origin_id = "S3-${var.bucket_name}"
target_origin_id = "s3-origin"
viewer_protocol_policy = "redirect-to-https"
compress = true
allowed_methods = ["GET", "HEAD"]
cached_methods = ["GET", "HEAD"]
forwarded_values {
query_string = false
cookies {
forward = "none"
}
}
min_ttl = 0
default_ttl = 86400
max_ttl = 31536000
compress = true
min_ttl = 0
default_ttl = 3600
max_ttl = 86400
}
##########################################
# API Behavior
##########################################
ordered_cache_behavior {
path_pattern = "/v1/*"
target_origin_id = "api-origin"
viewer_protocol_policy = "redirect-to-https"
allowed_methods = ["GET","HEAD","OPTIONS","PUT","POST","PATCH","DELETE"]
cached_methods = ["GET","HEAD"]
cache_policy_id = data.aws_cloudfront_cache_policy.caching_disabled.id
origin_request_policy_id = data.aws_cloudfront_origin_request_policy.all_viewer_except_host_header.id
}
price_class = "PriceClass_All"
##########################################
# Geo Restrictions
##########################################
restrictions {
geo_restriction {
restriction_type = "none"
}
}
##########################################
# SSL Certificate
##########################################
viewer_certificate {
cloudfront_default_certificate = true
acm_certificate_arn = var.acm_certificate_arn
ssl_support_method = var.acm_certificate_arn != null ? "sni-only" : null
minimum_protocol_version = var.acm_certificate_arn != null ? "TLSv1.2_2021" : null
cloudfront_default_certificate = var.acm_certificate_arn == null
}
}
############################################
# Origin Access Identities (S3 only)
############################################
resource "aws_cloudfront_origin_access_identity" "oai" {
comment = "OAI for ${var.bucket_name}"
for_each = {
for o in var.origins : o.origin_id => o
if o.origin_type == "s3"
}
comment = "OAI for ${each.key}"
}
############################################
# S3 Bucket Policy (S3 only)
############################################
resource "aws_s3_bucket_policy" "bucket_policy" {
bucket = var.bucket_id
for_each = {
for o in var.origins : o.origin_id => o
if o.origin_type == "s3"
}
bucket = each.value.bucket_id
policy = jsonencode({
Version = "2012-10-17"
Version = "2012-10-17"
Statement = [
{
Effect = "Allow"
Effect = "Allow"
Principal = {
AWS = "arn:aws:iam::cloudfront:user/CloudFront Origin Access Identity ${aws_cloudfront_origin_access_identity.oai.id}"
AWS = aws_cloudfront_origin_access_identity.oai[each.key].iam_arn
}
Action = "s3:GetObject"
Resource = "${var.bucket_arn}/*"
},
Resource = "${each.value.bucket_arn}/*"
}
]
})
}
}

View file

@ -1,24 +1,20 @@
variable "bucket_name" {
description = "The name of the bucket"
type = string
variable "origins" {
type = list(object({
origin_type = string # "s3" or "api"
origin_domain_name = string
origin_id = string
bucket_id = optional(string)
bucket_arn = optional(string)
}))
}
variable "stage" {
description = "The deployment stage"
type = string
variable "aliases" {
type = list(string)
}
variable "bucket_id" {
description = "The ID of the S3 bucket"
type = string
}
variable "bucket_arn" {
description = "The ARN of the S3 bucket"
type = string
}
variable "bucket_domain_name" {
description = "The regional domain name of the S3 bucket"
variable "acm_certificate_arn" {
description = "ACM certificate ARN for custom aliases"
type = string
default = null
}

View file

@ -0,0 +1,27 @@
resource "aws_lambda_function" "this" {
function_name = var.name
role = var.role_arn
package_type = "Zip"
# filename = var.filename
source_code_hash = var.source_code_hash
handler = var.handler
runtime = var.runtime
timeout = var.timeout
memory_size = var.memory_size
publish = true
s3_bucket = var.s3_bucket
s3_key = var.s3_key
environment {
variables = var.environment
}
}
output "lambda_arn" {
value = aws_lambda_function.this.arn
}
output "function_name" {
value = aws_lambda_function.this.function_name
}

View file

@ -0,0 +1,20 @@
variable "name" { type = string }
variable "role_arn" { type = string }
# variable "filename" { type = string }
variable "source_code_hash" { type = string }
variable "handler" { type = string }
variable "runtime" { type = string }
variable "timeout" {
type = number
default = 30
}
variable "memory_size" {
type = number
default = 128
}
variable "environment" {
type = map(string)
default = {}
}
variable "s3_bucket" { type = string }
variable "s3_key" { type = string }

View file

@ -0,0 +1,113 @@
############################################
# IAM role
############################################
module "role" {
source = "../lambda_execution_role"
name = "${var.name}-lambda-${var.stage}"
}
############################################
# Cloudwatch log group
############################################
resource "aws_cloudwatch_log_group" "api_logs" {
name = "/aws/apigateway/${var.name}-${var.stage}"
retention_in_days = 14
}
############################################
# Install python packages
############################################
resource "null_resource" "pip_install" {
count = var.requirements_file != null ? 1 : 0
provisioner "local-exec" {
command = "pip install -r ${var.requirements_file} -t ${var.source_dir} --platform manylinux2014_x86_64 --implementation cp --python-version 3.11 --only-binary=:all: --upgrade"
}
}
############################################
# Zip the source code
############################################
data "archive_file" "this" {
depends_on = [null_resource.pip_install]
type = "zip"
source_dir = var.source_dir
output_path = "${path.module}/lambda_package.zip"
excludes = var.zip_excludes
}
############################################
# Upload zip to S3
############################################
resource "aws_s3_object" "lambda_zip" {
bucket = var.artifact_bucket
key = "env:/${var.stage}/${var.name}.zip"
source = data.archive_file.this.output_path
etag = data.archive_file.this.output_md5
}
############################################
# Lambda
############################################
module "lambda" {
source = "../lambda_service_zip"
name = "${var.name}-${var.stage}"
role_arn = module.role.role_arn
s3_bucket = var.artifact_bucket
s3_key = aws_s3_object.lambda_zip.key
source_code_hash = data.archive_file.this.output_base64sha256
handler = var.handler
runtime = var.runtime
timeout = var.timeout
memory_size = var.memory_size
environment = var.environment
}
############################################
# API Gateway
############################################
resource "aws_apigatewayv2_api" "this" {
name = "${var.name}-api-${var.stage}"
protocol_type = "HTTP"
}
resource "aws_apigatewayv2_stage" "this" {
api_id = aws_apigatewayv2_api.this.id
name = "$default"
auto_deploy = true
access_log_settings {
destination_arn = aws_cloudwatch_log_group.api_logs.arn
format = jsonencode({
requestId = "$context.requestId"
domainName = "$context.domainName"
path = "$context.path"
status = "$context.status"
sourceIp = "$context.identity.sourceIp"
userAgent = "$context.identity.userAgent"
})
}
}
resource "aws_apigatewayv2_integration" "this" {
api_id = aws_apigatewayv2_api.this.id
integration_type = "AWS_PROXY"
integration_uri = module.lambda.lambda_arn
payload_format_version = "2.0"
}
resource "aws_apigatewayv2_route" "catch_all" {
api_id = aws_apigatewayv2_api.this.id
route_key = "$default"
target = "integrations/${aws_apigatewayv2_integration.this.id}"
}
resource "aws_lambda_permission" "apigw_invoke" {
statement_id = "AllowAPIGatewayInvoke"
action = "lambda:InvokeFunction"
function_name = module.lambda.lambda_arn
principal = "apigateway.amazonaws.com"
source_arn = "${aws_apigatewayv2_api.this.execution_arn}/*/*"
}

View file

@ -0,0 +1,11 @@
output "role_name" {
value = module.role.role_name
}
output "domain_name" {
value = var.domain_name
}
output "invoke_url" {
value = aws_apigatewayv2_stage.this.invoke_url
}

View file

@ -0,0 +1,48 @@
variable "name" { type = string }
variable "stage" { type = string }
variable "source_dir" { type = string }
variable "handler" { type = string }
variable "runtime" { type = string }
variable "zip_excludes" {
type = list(string)
default = [
"**/__pycache__/**",
"**/*.pyc",
"**/.pytest_cache/**",
"**/tests/**",
"**/infrastructure/**"
]
}
variable "timeout" {
type = number
default = 600
}
variable "memory_size" {
type = number
default = 512
}
variable "environment" {
type = map(string)
default = {}
}
variable "domain_name" {
type = string
default = null
}
variable "certificate_arn" {
type = string
default = null
}
variable "route53_zone_id" {
type = string
default = null
}
variable "artifact_bucket" { type = string }
variable "requirements_file" {
type = string
default = null
}

View file

@ -2,7 +2,7 @@
# IAM role
############################################
module "role" {
source = "../../../modules/lambda_execution_role"
source = "../lambda_execution_role"
name = "${var.name}-lambda-${var.stage}"
}
@ -14,7 +14,7 @@ output "role_name" {
# SQS queue + DLQ
############################################
module "queue" {
source = "../../../modules/sqs_queue"
source = "../sqs_queue"
name = "${var.name}-queue-${var.stage}"
}
@ -22,7 +22,7 @@ module "queue" {
# Lambda
############################################
module "lambda" {
source = "../../../modules/lambda_service"
source = "../lambda_service"
name = "${var.name}-${var.stage}"
role_arn = module.role.role_arn
@ -38,7 +38,7 @@ module "lambda" {
# SQS Lambda trigger
############################################
module "sqs_trigger" {
source = "../../../modules/lambda_sqs_trigger"
source = "../lambda_sqs_trigger"
lambda_arn = module.lambda.lambda_arn
lambda_role_name = module.role.role_name

View file

@ -2,9 +2,10 @@
locals {
# Generate full resource ARNs by combining bucket ARNs with resource paths
resources = flatten([
for bucket_arn in var.bucket_arns : [
for path in var.resource_paths : "${bucket_arn}${path}"
]
for bucket_arn in var.bucket_arns : concat(
[bucket_arn], # bare ARN for bucket-level actions like ListBucket
[for path in var.resource_paths : "${bucket_arn}${path}"]
)
])
}

View file

@ -2,7 +2,7 @@ terraform {
required_providers {
aws = {
source = "hashicorp/aws"
version = "~> 4.16"
version = ">= 5.0"
}
}
backend "s3" {
@ -127,6 +127,22 @@ module "s3" {
allowed_origins = var.allowed_origins
}
output "retrofit_datalake_bucket_id" {
value = module.s3.bucket_id
}
output "retrofit_datalake_bucket_arn" {
value = module.s3.bucket_arn
}
output "retrofit_datalake_bucket_domain_name" {
value = module.s3.bucket_domain_name
}
output "retrofit_datalake_bucket_name" {
value = module.s3.bucket_name
}
module "model_directory" {
source = "../modules/s3"
bucketname = "retrofit-model-directory-${var.stage}"
@ -311,18 +327,6 @@ module "sap_baseline_ecr" {
source = "../modules/ecr"
}
##############################################
# CDN - Cloudfront
##############################################
module "cloudfront_distribution" {
source = "../modules/cloudfront"
bucket_name = module.s3.bucket_name
bucket_id = module.s3.bucket_id
bucket_arn = module.s3.bucket_arn
bucket_domain_name = module.s3.bucket_domain_name
stage = var.stage
}
################################################
# SES - Email sending
################################################
@ -521,3 +525,52 @@ module "engine_s3_read_and_write" {
output "engine_s3_read_and_write_arn" {
value = module.engine_s3_read_and_write.policy_arn
}
################################################
# FastAPI Lambda
################################################
module "ara_fast_api_state_bucket" {
source = "../modules/tf_state_bucket"
bucket_name = "ara-fast-api-terraform-state"
}
output "ara_fast_api_state_bucket" {
value = module.ara_fast_api_state_bucket.bucket_name
}
# S3 policy for FastAPI app to read and write from various S3 buckets
module "fast_api_s3_read_and_write" {
source = "../modules/s3_iam_policy"
policy_name = "FastAPIReadandWriteS3"
policy_description = "Allow FastAPI Lambda to read from and write to various S3 buckets"
bucket_arns = [
"arn:aws:s3:::${module.s3_presignable_bucket.bucket_name}",
"arn:aws:s3:::${module.retrofit_sap_data.bucket_name}",
"arn:aws:s3:::${module.retrofit_sap_predictions.bucket_name}",
"arn:aws:s3:::${module.retrofit_carbon_predictions.bucket_name}",
"arn:aws:s3:::${module.retrofit_heat_predictions.bucket_name}",
"arn:aws:s3:::${module.retrofit_heating_kwh_predictions.bucket_name}",
"arn:aws:s3:::${module.retrofit_hotwater_kwh_predictions.bucket_name}",
"arn:aws:s3:::${module.retrofit_energy_assessments.bucket_name}"
]
actions = ["s3:GetObject", "s3:ListBucket"]
resource_paths = ["/*"]
}
output "fast_api_s3_read_and_write_arn" {
value = module.fast_api_s3_read_and_write.policy_arn
}
################################################
# CDN Certificate
################################################
module "cdn_certificate_state_bucket" {
source = "../modules/tf_state_bucket"
bucket_name = "cdn-certificate-terraform-state"
}
output "cdn_certificate_state_bucket" {
value = module.cdn_certificate_state_bucket.bucket_name
}