diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 338ef11d..0780c580 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -495,6 +495,45 @@ jobs: TF_VAR_pashub_coordination_password: ${{ secrets.PASHUB_COORDINATION_PASSWORD }} + # ============================================================ + # Build SharePoint Renamer image and Push + # ============================================================ + sharepoint_renamer_image: + needs: [determine_stage, shared_terraform] + uses: ./.github/workflows/_build_image.yml + with: + ecr_repo: sharepoint-renamer-${{ needs.determine_stage.outputs.stage }} + dockerfile_path: applications/sharepoint_renamer/handler/Dockerfile + build_context: . + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + + + # ============================================================ + # Deploy SharePoint Renamer Lambda + # ============================================================ + sharepoint_renamer_lambda: + needs: [sharepoint_renamer_image, determine_stage] + uses: ./.github/workflows/_deploy_lambda.yml + with: + lambda_name: sharepoint_renamer + lambda_path: deployment/terraform/lambda/sharepoint_renamer + stage: ${{ needs.determine_stage.outputs.stage }} + ecr_repo: sharepoint-renamer-${{ needs.determine_stage.outputs.stage }} + image_digest: ${{ needs.sharepoint_renamer_image.outputs.image_digest }} + terraform_apply: ${{ needs.determine_stage.outputs.terraform_apply }} + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + TF_VAR_sharepoint_client_id: ${{ secrets.SHAREPOINT_CLIENT_ID }} + TF_VAR_sharepoint_client_secret: ${{ secrets.SHAREPOINT_CLIENT_SECRET }} + TF_VAR_sharepoint_tenant_id: ${{ secrets.SHAREPOINT_TENANT_ID }} + TF_VAR_social_housing_wave_3_sharepoint_id: ${{ secrets.SOCIAL_HOUSING_WAVE_3_SHAREPOINT_ID }} + + # ============================================================ # Deploy FastAPI Lambda # ============================================================ diff --git a/applications/sharepoint_renamer/handler/Dockerfile b/applications/sharepoint_renamer/handler/Dockerfile new file mode 100644 index 00000000..10c40e89 --- /dev/null +++ b/applications/sharepoint_renamer/handler/Dockerfile @@ -0,0 +1,16 @@ +FROM public.ecr.aws/lambda/python:3.11 + +WORKDIR /var/task + +COPY applications/sharepoint_renamer/handler/requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY utils/ utils/ +COPY backend/__init__.py backend/__init__.py +COPY backend/pashub_fetcher/ backend/pashub_fetcher/ +COPY applications/sharepoint_renamer/ applications/sharepoint_renamer/ +COPY scripts/__init__.py scripts/__init__.py +COPY scripts/rename_sharepoint_files.py scripts/rename_sharepoint_files.py +COPY scripts/sero_address_list.csv scripts/sero_address_list.csv + +CMD ["applications.sharepoint_renamer.handler.handler.handler"] diff --git a/applications/sharepoint_renamer/handler/handler.py b/applications/sharepoint_renamer/handler/handler.py new file mode 100644 index 00000000..850d1ae6 --- /dev/null +++ b/applications/sharepoint_renamer/handler/handler.py @@ -0,0 +1,7 @@ +from typing import Any + +from scripts.rename_sharepoint_files import main + + +def handler(event: dict[str, Any], context: Any) -> None: + main() diff --git a/applications/sharepoint_renamer/handler/requirements.txt b/applications/sharepoint_renamer/handler/requirements.txt new file mode 100644 index 00000000..94317b81 --- /dev/null +++ b/applications/sharepoint_renamer/handler/requirements.txt @@ -0,0 +1,2 @@ +msal +requests diff --git a/deployment/terraform/lambda/sharepoint_renamer/main.tf b/deployment/terraform/lambda/sharepoint_renamer/main.tf new file mode 100644 index 00000000..0c245061 --- /dev/null +++ b/deployment/terraform/lambda/sharepoint_renamer/main.tf @@ -0,0 +1,22 @@ +module "lambda" { + source = "../../modules/lambda_with_sqs" + + name = "sharepoint_renamer" + stage = var.stage + + image_uri = local.image_uri + timeout = var.timeout + + reserved_concurrent_executions = var.reserved_concurrent_executions + + batch_size = var.batch_size + + environment = { + STAGE = var.stage + + SHAREPOINT_CLIENT_ID = var.sharepoint_client_id + SHAREPOINT_CLIENT_SECRET = var.sharepoint_client_secret + SHAREPOINT_TENANT_ID = var.sharepoint_tenant_id + SOCIAL_HOUSING_WAVE_3_SHAREPOINT_ID = var.social_housing_wave_3_sharepoint_id + } +} diff --git a/deployment/terraform/lambda/sharepoint_renamer/outputs.tf b/deployment/terraform/lambda/sharepoint_renamer/outputs.tf new file mode 100644 index 00000000..e71fac8b --- /dev/null +++ b/deployment/terraform/lambda/sharepoint_renamer/outputs.tf @@ -0,0 +1,9 @@ +output "sharepoint_renamer_queue_url" { + value = module.lambda.queue_url + description = "URL of the SharePoint Renamer SQS queue" +} + +output "sharepoint_renamer_queue_arn" { + value = module.lambda.queue_arn + description = "ARN of the SharePoint Renamer SQS queue" +} diff --git a/deployment/terraform/lambda/sharepoint_renamer/provider.tf b/deployment/terraform/lambda/sharepoint_renamer/provider.tf new file mode 100644 index 00000000..e6f8e32c --- /dev/null +++ b/deployment/terraform/lambda/sharepoint_renamer/provider.tf @@ -0,0 +1,20 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.0" + } + } + + backend "s3" { + bucket = "sharepoint-renamer-terraform-state" + key = "terraform.tfstate" + region = "eu-west-2" + } + + required_version = ">= 1.2.0" +} + +provider "aws" { + region = "eu-west-2" +} diff --git a/deployment/terraform/lambda/sharepoint_renamer/variables.tf b/deployment/terraform/lambda/sharepoint_renamer/variables.tf new file mode 100644 index 00000000..97cca538 --- /dev/null +++ b/deployment/terraform/lambda/sharepoint_renamer/variables.tf @@ -0,0 +1,55 @@ +variable "stage" { + description = "Deployment stage (e.g. dev, prod)" + type = string +} + +variable "ecr_repo_url" { + type = string + description = "ECR repository URL (no tag, no digest)" +} + +variable "image_digest" { + type = string + description = "Image digest (sha256:...)" +} + +variable "timeout" { + type = number + default = 900 + description = "Lambda timeout in seconds." +} + +variable "reserved_concurrent_executions" { + type = number + default = 1 + description = "Prevent parallel renames causing race conditions on SharePoint." +} + +variable "batch_size" { + type = number + default = 1 +} + +variable "sharepoint_client_id" { + type = string + sensitive = true +} + +variable "sharepoint_client_secret" { + type = string + sensitive = true +} + +variable "sharepoint_tenant_id" { + type = string + sensitive = true +} + +variable "social_housing_wave_3_sharepoint_id" { + type = string + sensitive = true +} + +locals { + image_uri = "${var.ecr_repo_url}@${var.image_digest}" +} diff --git a/deployment/terraform/shared/main.tf b/deployment/terraform/shared/main.tf index 7ca116e7..3d6bbd39 100644 --- a/deployment/terraform/shared/main.tf +++ b/deployment/terraform/shared/main.tf @@ -844,3 +844,17 @@ module "audit_generator_registry" { stage = var.stage } +################################################ +# SharePoint Renamer – Lambda +################################################ +module "sharepoint_renamer_state_bucket" { + source = "../modules/tf_state_bucket" + bucket_name = "sharepoint-renamer-terraform-state" +} + +module "sharepoint_renamer_registry" { + source = "../modules/container_registry" + name = "sharepoint-renamer" + stage = var.stage +} + diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/scripts/rename_sharepoint_files.py b/scripts/rename_sharepoint_files.py index a7306d88..7ed126e3 100644 --- a/scripts/rename_sharepoint_files.py +++ b/scripts/rename_sharepoint_files.py @@ -17,7 +17,7 @@ from utils.sharepoint.domna_sharepoint_client import DomnaSharepointClient from utils.sharepoint.domna_sites import DomnaSites DRY_RUN: bool = False -CSV_PATH: str = "scripts/sero_address_list_test.csv" +CSV_PATH: str = "scripts/sero_address_list.csv" BASE_PATH = ( "Osmosis-ACD Projects/Sero-Clarion Housing/"