diff --git a/etl/hubspot/scripts/scraper/handler/Dockerfile b/etl/hubspot/scripts/scraper/handler/Dockerfile index bbcc3e22..012da376 100644 --- a/etl/hubspot/scripts/scraper/handler/Dockerfile +++ b/etl/hubspot/scripts/scraper/handler/Dockerfile @@ -1,16 +1,6 @@ FROM public.ecr.aws/lambda/python:3.10 # FROM python:3.11.10-bullseye - -ARG DEV_DB_HOST -ARG DEV_DB_PORT -ARG DEV_DB_NAME - -ENV DB_HOST=${DEV_DB_HOST} -ENV DB_PORT=${DEV_DB_PORT} -ENV DB_NAME=${DEV_DB_NAME} - - # Set working directory (Lambda task root) WORKDIR /var/task diff --git a/infrastructure/terraform/lambda/hubspot_deal_etl/main.tf b/infrastructure/terraform/lambda/hubspot_deal_etl/main.tf new file mode 100644 index 00000000..ec2b18e3 --- /dev/null +++ b/infrastructure/terraform/lambda/hubspot_deal_etl/main.tf @@ -0,0 +1,44 @@ +data "terraform_remote_state" "shared" { + backend = "s3" + config = { + bucket = "assessment-model-terraform-state" + key = "env:/${var.stage}/terraform.tfstate" + region = "eu-west-2" + } +} + + +data "aws_secretsmanager_secret_version" "db_credentials" { + secret_id = "${var.stage}/assessment_model/db_credentials" +} + +locals { + db_credentials = jsondecode(data.aws_secretsmanager_secret_version.db_credentials.secret_string) +} + + +module "lambda" { + source = "../../modules/lambda_with_sqs" + + name = REPLACE ME #"address2uprn" for example + stage = var.stage + + image_uri = local.image_uri + + # Optional: Set maximum_concurrency to limit concurrent SQS-triggered invocations (2-1000) + maximum_concurrency = var.maximum_concurrency + + batch_size = var.batch_size + + environment = { + STAGE = var.stage + LOG_LEVEL = "info" + DB_USERNAME = local.db_credentials.db_assessment_model_username + DB_PASSWORD = local.db_credentials.db_assessment_model_password + } +} + +resource "aws_iam_role_policy_attachment" "lambda_s3_policy" { + role = module.lambda.role_name + policy_arn = data.terraform_remote_state.shared.outputs.hubspot_etl_s3_read_and_write_arn +} \ No newline at end of file diff --git a/infrastructure/terraform/lambda/hubspot_deal_etl/provider.tf b/infrastructure/terraform/lambda/hubspot_deal_etl/provider.tf new file mode 100644 index 00000000..3d66f392 --- /dev/null +++ b/infrastructure/terraform/lambda/hubspot_deal_etl/provider.tf @@ -0,0 +1,16 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.0" + } + } + + backend "s3" { + bucket = REPLACE_ME + key = "terraform.tfstate" + region = "eu-west-2" + } + + required_version = ">= 1.2.0" +} \ No newline at end of file diff --git a/infrastructure/terraform/lambda/hubspot_deal_etl/variables.tf b/infrastructure/terraform/lambda/hubspot_deal_etl/variables.tf new file mode 100644 index 00000000..e7646811 --- /dev/null +++ b/infrastructure/terraform/lambda/hubspot_deal_etl/variables.tf @@ -0,0 +1,37 @@ +variable "lambda_name" { + type = string + description = "Logical name of the lambda (e.g. address2uprn)" +} + +variable "stage" { + description = "Deployment stage (e.g. dev, prod)" + type = string +} +variable "ecr_repo_url" { + type = string + description = "ECR repository URL (no tag, no digest)" +} + +variable "image_digest" { + type = string + description = "Image digest (sha256:...)" +} + +variable "maximum_concurrency" { + type = number + default = null + description = "Maximum number of concurrent Lambda invocations from SQS (2-1000). null = no limit." +} + +variable "batch_size" { + type = number + default = 1 +} + +locals { + image_uri = "${var.ecr_repo_url}@${var.image_digest}" +} + +output "resolved_image_uri" { + value = local.image_uri +} diff --git a/infrastructure/terraform/shared/main.tf b/infrastructure/terraform/shared/main.tf index 8d645522..bc16dc70 100644 --- a/infrastructure/terraform/shared/main.tf +++ b/infrastructure/terraform/shared/main.tf @@ -670,6 +670,6 @@ module "hubspot_etl_s3_read_and_write" { resource_paths = ["/*"] } -output "ordnance_s3_read_and_write_arn" { +output "hubspot_etl_s3_read_and_write_arn" { value = module.hubspot_etl_s3_read_and_write.policy_arn } \ No newline at end of file