diff --git a/.github/workflows/_deploy_lambda.yml b/.github/workflows/_deploy_lambda.yml index 6d1430a2..3cef705e 100644 --- a/.github/workflows/_deploy_lambda.yml +++ b/.github/workflows/_deploy_lambda.yml @@ -16,12 +16,14 @@ on: type: string ecr_repo: - required: true + required: false type: string + default: '' image_digest: - required: true + required: false type: string + default: '' terraform_apply: required: false @@ -119,11 +121,21 @@ jobs: TF_VAR_google_solar_api_key: ${{ secrets.TF_VAR_google_solar_api_key }} TF_VAR_ordnance_survey_api_key: ${{ secrets.TF_VAR_ordnance_survey_api_key}} run: | + ECR_REPO_URL_VAR="" + if [[ -n "${{ inputs.ecr_repo }}" ]]; then + ECR_REPO_URL_VAR="-var=ecr_repo_url=${{ steps.repo.outputs.ecr_repo_url }}" + fi + + IMAGE_DIGEST_VAR="" + if [[ -n "${{ inputs.ecr_repo }}" ]]; then + IMAGE_DIGEST_VAR="-var=image_digest=${{ inputs.image_digest }}" + fi + terraform plan \ -var="stage=${{ inputs.stage }}" \ -var="lambda_name=${{ inputs.lambda_name }}" \ - -var="ecr_repo_url=${{ steps.repo.outputs.ecr_repo_url }}" \ - -var="image_digest=${{ inputs.image_digest }}" \ + $ECR_REPO_URL_VAR \ + $IMAGE_DIGEST_VAR \ -out=lambdaplan - name: Terraform Apply @@ -145,8 +157,12 @@ jobs: TF_VAR_google_solar_api_key: ${{ secrets.TF_VAR_google_solar_api_key }} TF_VAR_ordnance_survey_api_key: ${{ secrets.TF_VAR_ordnance_survey_api_key}} run: | + EXTRA_VARS="" + if [[ -n "${{ inputs.ecr_repo }}" ]]; then + EXTRA_VARS="-var=ecr_repo_url=${{ steps.repo.outputs.ecr_repo_url }} -var=image_digest=${{ inputs.image_digest }}" + fi + terraform destroy -auto-approve \ -var="stage=${{ inputs.stage }}" \ -var="lambda_name=${{ inputs.lambda_name }}" \ - -var="ecr_repo_url=${{ steps.repo.outputs.ecr_repo_url }}" \ - -var="image_digest=${{ inputs.image_digest }}" + $EXTRA_VARS diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 596ab82d..6291dd2a 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -41,7 +41,7 @@ jobs: fi # ============================================================ - # 1️⃣ Shared Terraform (infra) + # Shared Terraform (infra) # ============================================================ shared_terraform: needs: determine_stage @@ -77,9 +77,74 @@ jobs: if: env.TERRAFORM_APPLY == 'true' working-directory: infrastructure/terraform/shared run: terraform apply -auto-approve tfplan + + # ============================================================ + # Ara Engine image and Push + # ============================================================ + ara_engine_image: + needs: [determine_stage, shared_terraform] + uses: ./.github/workflows/_build_image.yml + with: + ecr_repo: engine-${{ needs.determine_stage.outputs.stage }} + dockerfile_path: backend/docker/engine.Dockerfile + build_context: . + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + + # ============================================================ + # Deploy Ara Engine Lambda + # ============================================================ + ara_engine_lambda: + needs: [ara_engine_image, determine_stage] + uses: ./.github/workflows/_deploy_lambda.yml + with: + lambda_name: ara_engine + lambda_path: infrastructure/terraform/lambda/engine + stage: ${{ needs.determine_stage.outputs.stage }} + ecr_repo: engine-${{ needs.determine_stage.outputs.stage }} + image_digest: ${{ needs.ara_engine_image.outputs.image_digest }} + terraform_apply: ${{ needs.determine_stage.outputs.terraform_apply }} + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + TF_VAR_db_host: ${{ secrets.DEV_DB_HOST }} + TF_VAR_db_name: ${{ secrets.DEV_DB_NAME }} + TF_VAR_db_port: ${{ secrets.DEV_DB_PORT }} + TF_VAR_api_key: ${{ secrets.DEV_API_KEY }} + TF_VAR_secret_key: ${{ secrets.DEV_SECRET_KEY }} + TF_VAR_domain_name: ${{ secrets.DEV_DOMAIN_NAME }} + TF_VAR_epc_auth_token: ${{ secrets.DEV_EPC_AUTH_TOKEN }} + TF_VAR_google_solar_api_key: ${{ secrets.DEV_GOOGLE_SOLAR_API_KEY }} + + # ============================================================ + # Deploy FastAPI Lambda + # ============================================================ + fast_api_lambda: + needs: [determine_stage] + uses: ./.github/workflows/_deploy_lambda.yml + with: + lambda_name: ara_fast_api + lambda_path: infrastructure/terraform/lambda/fast-api + stage: ${{ needs.determine_stage.outputs.stage }} + terraform_apply: ${{ needs.determine_stage.outputs.terraform_apply }} + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + TF_VAR_db_host: ${{ secrets.DEV_DB_HOST }} + TF_VAR_db_name: ${{ secrets.DEV_DB_NAME }} + TF_VAR_db_port: ${{ secrets.DEV_DB_PORT }} + TF_VAR_api_key: ${{ secrets.DEV_API_KEY }} + TF_VAR_secret_key: ${{ secrets.DEV_SECRET_KEY }} + TF_VAR_domain_name: ${{ secrets.DEV_DOMAIN_NAME }} + TF_VAR_epc_auth_token: ${{ secrets.DEV_EPC_AUTH_TOKEN }} + TF_VAR_google_solar_api_key: ${{ secrets.DEV_GOOGLE_SOLAR_API_KEY }} # ============================================================ - # 2️⃣ Build Address 2 UPRN image and Push + # Build Address 2 UPRN image and Push # ============================================================ address2uprn_image: needs: [determine_stage, shared_terraform] @@ -103,7 +168,7 @@ jobs: EPC_AUTH_TOKEN: ${{ secrets.DEV_EPC_AUTH_TOKEN }} # ============================================================ - # 3️⃣ Deploy Address 2 UPRN Lambda + # Deploy Address 2 UPRN Lambda # ============================================================ address2uprn_lambda: needs: [address2uprn_image, determine_stage] @@ -122,7 +187,7 @@ jobs: # ============================================================ - # 2️⃣ Build Postcode Splitter image and Push + # Build Postcode Splitter image and Push # ============================================================ postcodeSplitter_image: needs: [determine_stage, shared_terraform] @@ -144,7 +209,7 @@ jobs: DEV_DB_NAME: ${{ secrets.DEV_DB_NAME }} # ============================================================ - # 3️⃣ Deploy Postcode Splitter Lambda + # Deploy Postcode Splitter Lambda # ============================================================ postcodeSplitter_lambda: needs: [postcodeSplitter_image, determine_stage, address2uprn_lambda] @@ -242,48 +307,7 @@ jobs: AWS_REGION: ${{ secrets.DEV_AWS_REGION }} # ============================================================ - # Ara Engine image and Push - # ============================================================ - ara_engine_image: - needs: [determine_stage, shared_terraform] - uses: ./.github/workflows/_build_image.yml - with: - ecr_repo: engine-${{ needs.determine_stage.outputs.stage }} - dockerfile_path: backend/docker/engine.Dockerfile - build_context: . - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} - AWS_REGION: ${{ secrets.DEV_AWS_REGION }} - - # ============================================================ - # Deploy Ara Engine Lambda - # ============================================================ - ara_engine_lambda: - needs: [ara_engine_image, determine_stage] - uses: ./.github/workflows/_deploy_lambda.yml - with: - lambda_name: ara_engine - lambda_path: infrastructure/terraform/lambda/engine - stage: ${{ needs.determine_stage.outputs.stage }} - ecr_repo: engine-${{ needs.determine_stage.outputs.stage }} - image_digest: ${{ needs.ara_engine_image.outputs.image_digest }} - terraform_apply: ${{ needs.determine_stage.outputs.terraform_apply }} - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} - AWS_REGION: ${{ secrets.DEV_AWS_REGION }} - TF_VAR_db_host: ${{ secrets.DEV_DB_HOST }} - TF_VAR_db_name: ${{ secrets.DEV_DB_NAME }} - TF_VAR_db_port: ${{ secrets.DEV_DB_PORT }} - TF_VAR_api_key: ${{ secrets.DEV_API_KEY }} - TF_VAR_secret_key: ${{ secrets.DEV_SECRET_KEY }} - TF_VAR_domain_name: ${{ secrets.DEV_DOMAIN_NAME }} - TF_VAR_epc_auth_token: ${{ secrets.DEV_EPC_AUTH_TOKEN }} - TF_VAR_google_solar_api_key: ${{ secrets.DEV_GOOGLE_SOLAR_API_KEY }} - - # ============================================================ - # 2️⃣ Build OrdanceSurvey image and Push + # Build OrdanceSurvey image and Push # ============================================================ ordnanceSurvey_image: needs: [determine_stage, shared_terraform] @@ -305,7 +329,7 @@ jobs: DEV_DB_NAME: ${{ secrets.DEV_DB_NAME }} # ============================================================ - # 3️⃣ Deploy OrdanceSurvey Lambda + # Deploy OrdanceSurvey Lambda # ============================================================ ordnanceSurvey_lambda: needs: [ordnanceSurvey_image, determine_stage] @@ -322,3 +346,5 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.DEV_AWS_REGION }} TF_VAR_ORDNANCE_SURVEY_API_KEY: ${{ secrets.ORDNANCE_SURVEY_API_KEY }} + + diff --git a/infrastructure/terraform/lambda/categorisation/outputs.tf b/infrastructure/terraform/lambda/categorisation/outputs.tf new file mode 100644 index 00000000..be1ac118 --- /dev/null +++ b/infrastructure/terraform/lambda/categorisation/outputs.tf @@ -0,0 +1,9 @@ +output "categorisation_queue_url" { + value = module.lambda.queu_url + description = "URL of the Categorisation SQS queue" +} + +output "categorisation_queue_arn" { + value = module.lambda.queu_arn + description = "ARN of the Categorisation SQS queue" +} \ No newline at end of file diff --git a/infrastructure/terraform/lambda/engine/outputs.tf b/infrastructure/terraform/lambda/engine/outputs.tf new file mode 100644 index 00000000..bba2046b --- /dev/null +++ b/infrastructure/terraform/lambda/engine/outputs.tf @@ -0,0 +1,9 @@ +output "ara_engine_queue_url" { + value = module.lambda.queu_url + description = "URL of the Engine SQS queue" +} + +output "ara_engine_queue_arn" { + value = module.lambda.queu_arn + description = "ARN of the Engine SQS queue" +} \ No newline at end of file diff --git a/infrastructure/terraform/lambda/fast-api/main.tf b/infrastructure/terraform/lambda/fast-api/main.tf index 104d4a4d..0a40b14c 100644 --- a/infrastructure/terraform/lambda/fast-api/main.tf +++ b/infrastructure/terraform/lambda/fast-api/main.tf @@ -1,3 +1,6 @@ +############################################ +# Load Terraform State +############################################ data "terraform_remote_state" "shared" { backend = "s3" config = { @@ -7,43 +10,144 @@ data "terraform_remote_state" "shared" { } } -module "lambda" { - source = "../../modules/lambda_with_sqs" - - name = REPLACE ME #"address2uprn" for example - stage = var.stage - - image_uri = local.image_uri - - # Optional: Set maximum_concurrency to limit concurrent SQS-triggered invocations (2-1000) - maximum_concurrency = var.maximum_concurrency - - batch_size = var.batch_size - - environment = { - STAGE = var.stage - LOG_LEVEL = "info" +data "terraform_remote_state" "engine" { + backend = "s3" + config = { + bucket = "ara-engine-terraform-state", + key = "env:/${var.stage}/teraform.tfstate" + region = "eu-west-2" } } -# ====================================================================== -# OPTIONAL: Attach S3 IAM policy to Lambda execution role -# ====================================================================== -# Uncomment and configure the resource below to attach S3 permissions -# -# Example 1: Attach existing policy from shared state -# resource "aws_iam_role_policy_attachment" "lambda_s3_policy" { -# role = module.lambda.role_name -# policy_arn = data.terraform_remote_state.shared.outputs.YOUR_POLICY_OUTPUT_NAME_arn +data "terraform_remote_state" "categorisation" { + backend = "s3" + config = { + bucket = "categorisation-terraform-state", + key = "env:/${var.stage}/teraform.tfstate" + region = "eu-west-2" + } +} + +############################################ +# Load Credentials +############################################ +data "aws_secretsmanager_secret_version" "db_credentials" { + secret_id = "${var.stage}/assessment_model/db_credentials" +} + +locals { + db_credentials = jsondecode(data.aws_secretsmanager_secret_version.db_credentials.secret_string) +} + +# data "aws_ssm_parameter" "certificate_arn" { +# name = "/ssl_certificate_arn" # } -# -# Example 2: Attach multiple policies -# resource "aws_iam_role_policy_attachment" "lambda_read_policy" { -# role = module.lambda.role_name -# policy_arn = data.terraform_remote_state.shared.outputs.postcode_splitter_s3_read_arn -# } -# -# resource "aws_iam_role_policy_attachment" "lambda_write_policy" { -# role = module.lambda.role_name -# policy_arn = data.terraform_remote_state.shared.outputs.another_policy_arn + +# data "aws_route53_zone" "this" { +# name = var.domain_name # } + +############################################ +# Install Python requirements +############################################ +resource "null_resource" "pip_install" { + triggers = { + requirements_hash = filemd5("${path.root}/../../../../backend/app/requirements/requirements.txt") + } + + provisioner "local-exec" { + command = <