diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index e1e9b3c5..338ef11d 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -661,6 +661,42 @@ jobs: TF_VAR_magicplan_customer_id: ${{ secrets.MAGICPLAN_CUSTOMER_ID }} TF_VAR_magicplan_api_key: ${{ secrets.MAGICPLAN_API_KEY }} + # ============================================================ + # Build Audit Generator image + # ============================================================ + audit_generator_image: + needs: [determine_stage, shared_terraform] + uses: ./.github/workflows/_build_image.yml + with: + ecr_repo: audit-generator-${{ needs.determine_stage.outputs.stage }} + dockerfile_path: applications/audit_generator/handler/Dockerfile + build_context: . + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + + # ============================================================ + # Deploy Audit Generator Lambda + # ============================================================ + audit_generator_lambda: + needs: [audit_generator_image, determine_stage] + uses: ./.github/workflows/_deploy_lambda.yml + with: + lambda_name: audit_generator + lambda_path: deployment/terraform/lambda/audit_generator + stage: ${{ needs.determine_stage.outputs.stage }} + ecr_repo: audit-generator-${{ needs.determine_stage.outputs.stage }} + image_digest: ${{ needs.audit_generator_image.outputs.image_digest }} + terraform_apply: ${{ needs.determine_stage.outputs.terraform_apply }} + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + TF_VAR_db_host: ${{ secrets.DEV_DB_HOST }} + TF_VAR_db_name: ${{ secrets.DEV_DB_NAME }} + TF_VAR_db_port: ${{ secrets.DEV_DB_PORT }} + # ============================================================ # Deploy Hubspot ETL Lambda # ============================================================ diff --git a/.github/workflows/lambda_smoke_tests.yml b/.github/workflows/lambda_smoke_tests.yml index 329a1319..1830fcd9 100644 --- a/.github/workflows/lambda_smoke_tests.yml +++ b/.github/workflows/lambda_smoke_tests.yml @@ -123,6 +123,16 @@ jobs: build_context: . service_name: magic-plan + # ============================================================ + # Audit Generator + # ============================================================ + audit_generator_smoke_test: + uses: ./.github/workflows/_smoke_test_lambda.yml + with: + dockerfile_path: applications/audit_generator/handler/Dockerfile + build_context: . + service_name: audit-generator + # ============================================================ # HubSpot Scraper # ============================================================ diff --git a/applications/audit_generator/handler/Dockerfile b/applications/audit_generator/handler/Dockerfile new file mode 100644 index 00000000..e70b00a3 --- /dev/null +++ b/applications/audit_generator/handler/Dockerfile @@ -0,0 +1,17 @@ +FROM public.ecr.aws/lambda/python:3.11 + +WORKDIR /var/task + +COPY applications/audit_generator/handler/requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY utilities/ utilities/ +COPY backend/ backend/ +COPY applications/ applications/ +COPY domain/ domain/ +COPY datatypes/ datatypes/ +COPY orchestration/ orchestration/ +COPY repositories/ repositories/ +COPY infrastructure/ infrastructure/ + +CMD ["applications.audit_generator.handler.handler"] diff --git a/applications/audit_generator/handler/requirements.txt b/applications/audit_generator/handler/requirements.txt new file mode 100644 index 00000000..bf2c1bff --- /dev/null +++ b/applications/audit_generator/handler/requirements.txt @@ -0,0 +1,7 @@ +awslambdaric +sqlalchemy==2.0.36 +sqlmodel +psycopg2-binary==2.9.10 +pydantic-settings==2.6.0 +boto3==1.35.44 +openpyxl diff --git a/deployment/terraform/lambda/audit_generator/main.tf b/deployment/terraform/lambda/audit_generator/main.tf new file mode 100644 index 00000000..64e36aa4 --- /dev/null +++ b/deployment/terraform/lambda/audit_generator/main.tf @@ -0,0 +1,45 @@ +data "terraform_remote_state" "shared" { + backend = "s3" + config = { + bucket = "assessment-model-terraform-state" + key = "env:/${var.stage}/terraform.tfstate" + region = "eu-west-2" + } +} + +data "aws_secretsmanager_secret_version" "db_credentials" { + secret_id = "${var.stage}/assessment_model/db_credentials" +} + +locals { + db_credentials = jsondecode(data.aws_secretsmanager_secret_version.db_credentials.secret_string) +} + +resource "aws_iam_role_policy_attachment" "audit_generator_s3_write" { + role = module.lambda.role_name + policy_arn = data.terraform_remote_state.shared.outputs.energy_assessments_s3_write_arn +} + +module "lambda" { + source = "../../modules/lambda_with_sqs" + + name = "audit_generator" + stage = var.stage + + image_uri = local.image_uri + + maximum_concurrency = var.maximum_concurrency + reserved_concurrent_executions = var.reserved_concurrent_executions + batch_size = var.batch_size + + environment = { + STAGE = var.stage + LOG_LEVEL = "info" + S3_BUCKET_NAME = data.terraform_remote_state.shared.outputs.retrofit_energy_assessments_bucket_name + POSTGRES_USERNAME = local.db_credentials.db_assessment_model_username + POSTGRES_PASSWORD = local.db_credentials.db_assessment_model_password + POSTGRES_HOST = var.db_host + POSTGRES_DATABASE = var.db_name + POSTGRES_PORT = var.db_port + } +} diff --git a/deployment/terraform/lambda/audit_generator/outputs.tf b/deployment/terraform/lambda/audit_generator/outputs.tf new file mode 100644 index 00000000..04609ca6 --- /dev/null +++ b/deployment/terraform/lambda/audit_generator/outputs.tf @@ -0,0 +1,9 @@ +output "audit_generator_queue_url" { + value = module.lambda.queue_url + description = "URL of the Audit Generator SQS queue" +} + +output "audit_generator_queue_arn" { + value = module.lambda.queue_arn + description = "ARN of the Audit Generator SQS queue" +} diff --git a/deployment/terraform/lambda/audit_generator/provider.tf b/deployment/terraform/lambda/audit_generator/provider.tf new file mode 100644 index 00000000..3793fefd --- /dev/null +++ b/deployment/terraform/lambda/audit_generator/provider.tf @@ -0,0 +1,16 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.0" + } + } + + backend "s3" { + bucket = "audit-generator-terraform-state" + key = "terraform.tfstate" + region = "eu-west-2" + } + + required_version = ">= 1.2.0" +} diff --git a/deployment/terraform/lambda/audit_generator/variables.tf b/deployment/terraform/lambda/audit_generator/variables.tf new file mode 100644 index 00000000..4642d811 --- /dev/null +++ b/deployment/terraform/lambda/audit_generator/variables.tf @@ -0,0 +1,52 @@ +variable "stage" { + description = "Deployment stage (e.g. dev, prod)" + type = string +} + +variable "ecr_repo_url" { + type = string + description = "ECR repository URL (no tag, no digest)" +} + +variable "image_digest" { + type = string + description = "Image digest (sha256:...)" +} + +variable "maximum_concurrency" { + type = number + default = null +} + +variable "reserved_concurrent_executions" { + type = number + default = 1 +} + +variable "batch_size" { + type = number + default = 1 +} + +locals { + image_uri = "${var.ecr_repo_url}@${var.image_digest}" +} + +output "resolved_image_uri" { + value = local.image_uri +} + +variable "db_host" { + type = string + sensitive = true +} + +variable "db_name" { + type = string + sensitive = true +} + +variable "db_port" { + type = string + sensitive = true +} diff --git a/deployment/terraform/shared/main.tf b/deployment/terraform/shared/main.tf index 0bd7a966..7ca116e7 100644 --- a/deployment/terraform/shared/main.tf +++ b/deployment/terraform/shared/main.tf @@ -830,3 +830,17 @@ module "magic_plan_client_registry" { stage = var.stage } +################################################ +# Audit Generator – Lambda +################################################ +module "audit_generator_state_bucket" { + source = "../modules/tf_state_bucket" + bucket_name = "audit-generator-terraform-state" +} + +module "audit_generator_registry" { + source = "../modules/container_registry" + name = "audit-generator" + stage = var.stage +} +