From d5830fb63b6c66d6e309f3f49a2e0db0fc189aad Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Wed, 25 Mar 2026 15:15:32 +0000 Subject: [PATCH] deploy pashub_to_ara lambda --- .github/workflows/_deploy_lambda.yml | 37 +++++++++++++- .github/workflows/deploy_terraform.yml | 51 +++++++++++++++++++ .../terraform/lambda/pashub_to_ara/main.tf | 27 ++++++++++ .../lambda/pashub_to_ara/provider.tf | 16 ++++++ .../lambda/pashub_to_ara/variables.tf | 37 ++++++++++++++ infrastructure/terraform/shared/main.tf | 5 ++ 6 files changed, 172 insertions(+), 1 deletion(-) create mode 100644 infrastructure/terraform/lambda/pashub_to_ara/main.tf create mode 100644 infrastructure/terraform/lambda/pashub_to_ara/provider.tf create mode 100644 infrastructure/terraform/lambda/pashub_to_ara/variables.tf diff --git a/.github/workflows/_deploy_lambda.yml b/.github/workflows/_deploy_lambda.yml index dab98d8b..707c9e00 100644 --- a/.github/workflows/_deploy_lambda.yml +++ b/.github/workflows/_deploy_lambda.yml @@ -62,7 +62,24 @@ on: required: false TF_VAR_ordnance_survey_api_key: required: false - + TF_VAR_sharepoint_client_id: + required: false + TF_VAR_sharepoint_client_secret: + required: false + TF_VAR_sharepoint_tenant_id: + required: false + TF_VAR_domna_sharepoint_id: + required: false + TF_VAR_osmosis_acd_sharepoint_id: + required: false + TF_VAR_private_pay_sharepoint_id: + required: false + TF_VAR_social_housing_wave_3_sharepoint_id: + required: false + TF_VAR_pashub_email: + required: false + TF_VAR_pashub_password: + required: false jobs: deploy: runs-on: ubuntu-latest @@ -120,6 +137,15 @@ jobs: TF_VAR_epc_auth_token: ${{ secrets.TF_VAR_epc_auth_token }} TF_VAR_google_solar_api_key: ${{ secrets.TF_VAR_google_solar_api_key }} TF_VAR_ordnance_survey_api_key: ${{ secrets.TF_VAR_ordnance_survey_api_key }} + TF_VAR_sharepoint_client_id: ${{ secrets.TF_VAR_sharepoint_client_id }} + TF_VAR_sharepoint_client_secret: ${{ secrets.TF_VAR_sharepoint_client_secret }} + TF_VAR_sharepoint_tenant_id: ${{ secrets.TF_VAR_sharepoint_tenant_id }} + TF_VAR_domna_sharepoint_id: ${{ secrets.TF_VAR_domna_sharepoint_id }} + TF_VAR_osmosis_acd_sharepoint_id: ${{ secrets.TF_VAR_osmosis_acd_sharepoint_id }} + TF_VAR_private_pay_sharepoint_id: ${{ secrets.TF_VAR_private_pay_sharepoint_id }} + TF_VAR_social_housing_wave_3_sharepoint_id: ${{ secrets.TF_VAR_social_housing_wave_3_sharepoint_id }} + TF_VAR_pashub_email: ${{ secrets.TF_VAR_pashub_email }} + TF_VAR_pashub_password: ${{ secrets.TF_VAR_pashub_password }} run: | ECR_REPO_URL_VAR="" if [[ -n "${{ inputs.ecr_repo }}" ]]; then @@ -156,6 +182,15 @@ jobs: TF_VAR_epc_auth_token: ${{ secrets.TF_VAR_epc_auth_token }} TF_VAR_google_solar_api_key: ${{ secrets.TF_VAR_google_solar_api_key }} TF_VAR_ordnance_survey_api_key: ${{ secrets.TF_VAR_ordnance_survey_api_key }} + TF_VAR_sharepoint_client_id: ${{ secrets.TF_VAR_sharepoint_client_id }} + TF_VAR_sharepoint_client_secret: ${{ secrets.TF_VAR_sharepoint_client_secret }} + TF_VAR_sharepoint_tenant_id: ${{ secrets.TF_VAR_sharepoint_tenant_id }} + TF_VAR_domna_sharepoint_id: ${{ secrets.TF_VAR_domna_sharepoint_id }} + TF_VAR_osmosis_acd_sharepoint_id: ${{ secrets.TF_VAR_osmosis_acd_sharepoint_id }} + TF_VAR_private_pay_sharepoint_id: ${{ secrets.TF_VAR_private_pay_sharepoint_id }} + TF_VAR_social_housing_wave_3_sharepoint_id: ${{ secrets.TF_VAR_social_housing_wave_3_sharepoint_id }} + TF_VAR_pashub_email: ${{ secrets.TF_VAR_pashub_email }} + TF_VAR_pashub_password: ${{ secrets.TF_VAR_pashub_password }} run: | EXTRA_VARS="" if [[ -n "${{ inputs.ecr_repo }}" ]]; then diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index e41534e6..ce75f4c9 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -322,6 +322,57 @@ jobs: AWS_REGION: ${{ secrets.DEV_AWS_REGION }} TF_VAR_ORDNANCE_SURVEY_API_KEY: ${{ secrets.ORDNANCE_SURVEY_API_KEY }} + # ============================================================ + # Build Pas Hub to Ara image and Push + # ============================================================ + pashub_to_ara_image: + needs: [determine_stage, shared_terraform] + uses: ./.github/workflows/_build_image.yml + with: + ecr_repo: pashub_to_ara-${{ needs.determine_stage.outputs.stage }} + dockerfile_path: backend/pashub_fetcher/handler/Dockerfile + build_context: . + build_args: | + DEV_DB_HOST=$DEV_DB_HOST + DEV_DB_PORT=$DEV_DB_PORT + DEV_DB_NAME=$DEV_DB_NAME + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + DEV_DB_HOST: ${{ secrets.DEV_DB_HOST }} + DEV_DB_PORT: ${{ secrets.DEV_DB_PORT }} + DEV_DB_NAME: ${{ secrets.DEV_DB_NAME }} + + + # ============================================================ + # Deploy Pas Hub to Ara Lambda + # ============================================================ + pashub_to_ara_lambda: + needs: [pashub_to_ara_image, determine_stage] + uses: ./.github/workflows/_deploy_lambda.yml + with: + lambda_name: pashub_to_ara + lambda_path: infrastructure/terraform/lambda/pashub_to_ara + stage: ${{ needs.determine_stage.outputs.stage }} + ecr_repo: ordnance-${{ needs.determine_stage.outputs.stage }} + image_digest: ${{ needs.pashub_to_ara_image.outputs.image_digest }} + terraform_apply: ${{ needs.determine_stage.outputs.terraform_apply }} + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + TF_VAR_sharepoint_client_id: ${{ secrets.SHAREPOINT_CLIENT_ID }} + TF_VAR_sharepoint_client_secret: ${{ secrets.SHAREPOINT_CLIENT_SECRET }} + TF_VAR_sharepoint_tenant_id: ${{ secrets.SHAREPOINT_TENANT_ID }} + TF_VAR_domna_sharepoint_id: ${{ secrets.DOMNA_SHAREPOINT_ID }} + TF_VAR_osmosis_acd_sharepoint_id: ${{ secrets.OSMOSIS_ACD_SHAREPOINT_ID }} + TF_VAR_private_pay_sharepoint_id: ${{ secrets.PRIVATE_PAY_SHAREPOINT_ID }} + TF_VAR_social_housing_wave_3_sharepoint_id: ${{ secrets.SOCIAL_HOUSING_WAVE_3_SHAREPOINT_ID }} + TF_VAR_pashub_email: ${{ secrets.PASHUB_EMAIL }} + TF_VAR_pashub_password: ${{ secrets.PASHUB_PASSWORD }} + + # ============================================================ # Deploy FastAPI Lambda # ============================================================ diff --git a/infrastructure/terraform/lambda/pashub_to_ara/main.tf b/infrastructure/terraform/lambda/pashub_to_ara/main.tf new file mode 100644 index 00000000..e898e949 --- /dev/null +++ b/infrastructure/terraform/lambda/pashub_to_ara/main.tf @@ -0,0 +1,27 @@ +data "terraform_remote_state" "shared" { + backend = "s3" + config = { + bucket = "assessment-model-terraform-state" + key = "env:/${var.stage}/terraform.tfstate" + region = "eu-west-2" + } +} + +module "lambda" { + source = "../../modules/lambda_with_sqs" + + name = "pashub_to_ara" + stage = var.stage + + image_uri = local.image_uri + + # Optional: Set maximum_concurrency to limit concurrent SQS-triggered invocations (2-1000) + maximum_concurrency = var.maximum_concurrency + + batch_size = var.batch_size + + environment = { + STAGE = var.stage + LOG_LEVEL = "info" + } +} diff --git a/infrastructure/terraform/lambda/pashub_to_ara/provider.tf b/infrastructure/terraform/lambda/pashub_to_ara/provider.tf new file mode 100644 index 00000000..ba7cfa89 --- /dev/null +++ b/infrastructure/terraform/lambda/pashub_to_ara/provider.tf @@ -0,0 +1,16 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.0" + } + } + + backend "s3" { + bucket = "pashub-to-ara-terraform-state" + key = "terraform.tfstate" + region = "eu-west-2" + } + + required_version = ">= 1.2.0" +} \ No newline at end of file diff --git a/infrastructure/terraform/lambda/pashub_to_ara/variables.tf b/infrastructure/terraform/lambda/pashub_to_ara/variables.tf new file mode 100644 index 00000000..e7646811 --- /dev/null +++ b/infrastructure/terraform/lambda/pashub_to_ara/variables.tf @@ -0,0 +1,37 @@ +variable "lambda_name" { + type = string + description = "Logical name of the lambda (e.g. address2uprn)" +} + +variable "stage" { + description = "Deployment stage (e.g. dev, prod)" + type = string +} +variable "ecr_repo_url" { + type = string + description = "ECR repository URL (no tag, no digest)" +} + +variable "image_digest" { + type = string + description = "Image digest (sha256:...)" +} + +variable "maximum_concurrency" { + type = number + default = null + description = "Maximum number of concurrent Lambda invocations from SQS (2-1000). null = no limit." +} + +variable "batch_size" { + type = number + default = 1 +} + +locals { + image_uri = "${var.ecr_repo_url}@${var.image_digest}" +} + +output "resolved_image_uri" { + value = local.image_uri +} diff --git a/infrastructure/terraform/shared/main.tf b/infrastructure/terraform/shared/main.tf index 84c6748b..3b12561c 100644 --- a/infrastructure/terraform/shared/main.tf +++ b/infrastructure/terraform/shared/main.tf @@ -527,6 +527,11 @@ output "ordnance_s3_read_and_write_arn" { ################################################ # Pas Hub to Ara – Lambda ################################################ +module "pashub_to_ara_state_bucket" { + source = "../modules/tf_state_bucket" + bucket_name = "pashub-to-ara-terraform-state" +} + module "pashub_to_ara_registry" { source = "../modules/container_registry" name = "pashub_to_ara"