From bc8ca3ead36e13b71272845baf80d01adee63e41 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 19 May 2026 12:55:30 +0000 Subject: [PATCH 01/22] deployment from infrastructure --- .dockerignore | 2 +- .github/workflows/deploy_terraform.yml | 46 +++++++++---------- Dockerfile.test.dockerignore | 2 +- .../terraform/lambda/_template/README.md | 2 +- .../lambda_with_api_gateway/variables.tf | 2 +- 5 files changed, 27 insertions(+), 27 deletions(-) diff --git a/.dockerignore b/.dockerignore index 0c7d7749..90436ffc 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,7 +6,7 @@ backend/.idea/* backend/.env recommendations/tests/* model_data/tests/* -infrastructure/* +deployment/* data_collection/* node_modules/* conservation_areas/* diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index bd014e3d..923fc0a9 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -62,20 +62,20 @@ jobs: - uses: hashicorp/setup-terraform@v3 - name: Terraform Init - working-directory: infrastructure/terraform/shared + working-directory: deployment/terraform/shared run: terraform init -reconfigure - name: Terraform Workspace - working-directory: infrastructure/terraform/shared + working-directory: deployment/terraform/shared run: terraform workspace select ${STAGE} || terraform workspace new ${STAGE} - name: Terraform Plan - working-directory: infrastructure/terraform/shared + working-directory: deployment/terraform/shared run: terraform plan -var-file=${STAGE}.tfvars -out=tfplan - name: Terraform Apply if: env.TERRAFORM_APPLY == 'true' - working-directory: infrastructure/terraform/shared + working-directory: deployment/terraform/shared run: terraform apply -auto-approve tfplan # ============================================================ @@ -101,7 +101,7 @@ jobs: uses: ./.github/workflows/_deploy_lambda.yml with: lambda_name: ara_engine - lambda_path: infrastructure/terraform/lambda/engine + lambda_path: deployment/terraform/lambda/engine stage: ${{ needs.determine_stage.outputs.stage }} ecr_repo: engine-${{ needs.determine_stage.outputs.stage }} image_digest: ${{ needs.ara_engine_image.outputs.image_digest }} @@ -150,7 +150,7 @@ jobs: uses: ./.github/workflows/_deploy_lambda.yml with: lambda_name: address2uprn - lambda_path: infrastructure/terraform/lambda/address2UPRN + lambda_path: deployment/terraform/lambda/address2UPRN stage: ${{ needs.determine_stage.outputs.stage }} ecr_repo: address2uprn-${{ needs.determine_stage.outputs.stage }} image_digest: ${{ needs.address2uprn_image.outputs.image_digest }} @@ -191,7 +191,7 @@ jobs: uses: ./.github/workflows/_deploy_lambda.yml with: lambda_name: postcodeSplitter - lambda_path: infrastructure/terraform/lambda/postcodeSplitter + lambda_path: deployment/terraform/lambda/postcodeSplitter stage: ${{ needs.determine_stage.outputs.stage }} ecr_repo: postcode_splitter-${{ needs.determine_stage.outputs.stage }} image_digest: ${{ needs.postcodeSplitter_image.outputs.image_digest }} @@ -231,7 +231,7 @@ jobs: uses: ./.github/workflows/_deploy_lambda.yml with: lambda_name: bulk_address2uprn_combiner - lambda_path: infrastructure/terraform/lambda/bulk_address2uprn_combiner + lambda_path: deployment/terraform/lambda/bulk_address2uprn_combiner stage: ${{ needs.determine_stage.outputs.stage }} ecr_repo: bulk_address2uprn_combiner-${{ needs.determine_stage.outputs.stage }} image_digest: ${{ needs.bulk_address2uprn_combiner_image.outputs.image_digest }} @@ -271,7 +271,7 @@ jobs: uses: ./.github/workflows/_deploy_lambda.yml with: lambda_name: condition-etl - lambda_path: infrastructure/terraform/lambda/condition-etl + lambda_path: deployment/terraform/lambda/condition-etl stage: ${{ needs.determine_stage.outputs.stage }} ecr_repo: condition-etl-${{ needs.determine_stage.outputs.stage }} image_digest: ${{ needs.condition_etl_image.outputs.image_digest }} @@ -311,7 +311,7 @@ jobs: uses: ./.github/workflows/_deploy_lambda.yml with: lambda_name: categorisation - lambda_path: infrastructure/terraform/lambda/categorisation + lambda_path: deployment/terraform/lambda/categorisation stage: ${{ needs.determine_stage.outputs.stage }} ecr_repo: categorisation-${{ needs.determine_stage.outputs.stage }} image_digest: ${{ needs.categorisation_image.outputs.image_digest }} @@ -351,7 +351,7 @@ jobs: uses: ./.github/workflows/_deploy_lambda.yml with: lambda_name: ordnanceSurvey - lambda_path: infrastructure/terraform/lambda/ordnanceSurvey + lambda_path: deployment/terraform/lambda/ordnanceSurvey stage: ${{ needs.determine_stage.outputs.stage }} ecr_repo: ordnance-${{ needs.determine_stage.outputs.stage }} image_digest: ${{ needs.ordnanceSurvey_image.outputs.image_digest }} @@ -386,7 +386,7 @@ jobs: uses: ./.github/workflows/_deploy_lambda.yml with: lambda_name: pashub_to_ara - lambda_path: infrastructure/terraform/lambda/pashub_to_ara + lambda_path: deployment/terraform/lambda/pashub_to_ara stage: ${{ needs.determine_stage.outputs.stage }} ecr_repo: pashub_to_ara-${{ needs.determine_stage.outputs.stage }} image_digest: ${{ needs.pashub_to_ara_image.outputs.image_digest }} @@ -419,7 +419,7 @@ jobs: uses: ./.github/workflows/_deploy_lambda.yml with: lambda_name: ara_fast_api - lambda_path: infrastructure/terraform/lambda/fast-api + lambda_path: deployment/terraform/lambda/fast-api stage: ${{ needs.determine_stage.outputs.stage }} terraform_apply: ${{ needs.determine_stage.outputs.terraform_apply }} secrets: @@ -458,17 +458,17 @@ jobs: - uses: hashicorp/setup-terraform@v3 - name: Terraform Init - working-directory: infrastructure/terraform/cdn_certificate + working-directory: deployment/terraform/cdn_certificate run: terraform init -reconfigure - name: Terraform Workspace - working-directory: infrastructure/terraform/cdn_certificate + working-directory: deployment/terraform/cdn_certificate run: | terraform workspace select $STAGE \ || terraform workspace new $STAGE - name: Terraform Plan - working-directory: infrastructure/terraform/cdn_certificate + working-directory: deployment/terraform/cdn_certificate run: | terraform plan \ -var="stage=${STAGE}" \ @@ -476,7 +476,7 @@ jobs: - name: Terraform Apply if: env.TERRAFORM_APPLY == 'true' - working-directory: infrastructure/terraform/cdn_certificate + working-directory: deployment/terraform/cdn_certificate run: terraform apply -auto-approve tfplan @@ -503,17 +503,17 @@ jobs: - uses: hashicorp/setup-terraform@v3 - name: Terraform Init - working-directory: infrastructure/terraform/cdn + working-directory: deployment/terraform/cdn run: terraform init -reconfigure - name: Terraform Workspace - working-directory: infrastructure/terraform/cdn + working-directory: deployment/terraform/cdn run: | terraform workspace select $STAGE \ || terraform workspace new $STAGE - name: Terraform Plan - working-directory: infrastructure/terraform/cdn + working-directory: deployment/terraform/cdn run: | terraform plan \ -var="stage=${STAGE}" \ @@ -521,7 +521,7 @@ jobs: - name: Terraform Apply if: env.TERRAFORM_APPLY == 'true' - working-directory: infrastructure/terraform/cdn + working-directory: deployment/terraform/cdn run: terraform apply -auto-approve tfplan # ============================================================ @@ -562,7 +562,7 @@ jobs: uses: ./.github/workflows/_deploy_lambda.yml with: lambda_name: magic_plan - lambda_path: infrastructure/terraform/lambda/magic_plan + lambda_path: deployment/terraform/lambda/magic_plan stage: ${{ needs.determine_stage.outputs.stage }} ecr_repo: magic-plan-${{ needs.determine_stage.outputs.stage }} image_digest: ${{ needs.magic_plan_image.outputs.image_digest }} @@ -585,7 +585,7 @@ jobs: uses: ./.github/workflows/_deploy_lambda.yml with: lambda_name: hubspot-etl-to-ara - lambda_path: infrastructure/terraform/lambda/hubspot_deal_etl + lambda_path: deployment/terraform/lambda/hubspot_deal_etl stage: ${{ needs.determine_stage.outputs.stage }} ecr_repo: hubspot-etl-${{ needs.determine_stage.outputs.stage }} image_digest: ${{ needs.hubspot_etl_image.outputs.image_digest }} diff --git a/Dockerfile.test.dockerignore b/Dockerfile.test.dockerignore index 4f79c6ee..ed05c399 100644 --- a/Dockerfile.test.dockerignore +++ b/Dockerfile.test.dockerignore @@ -4,7 +4,7 @@ model_data/local_data/ backend/node_modules/ backend/.idea/ backend/.env -infrastructure/ +deployment/ data_collection/ node_modules/ conservation_areas/ diff --git a/infrastructure/terraform/lambda/_template/README.md b/infrastructure/terraform/lambda/_template/README.md index 5bb10627..f2a8638a 100644 --- a/infrastructure/terraform/lambda/_template/README.md +++ b/infrastructure/terraform/lambda/_template/README.md @@ -10,7 +10,7 @@ ### 2. Add infrastructure prerequisites (shared stack) - Add a new ECR repository in: - infrastructure/terraform/shared/main.tf + deployment/terraform/shared/main.tf - Create a PR to deploy this to main then dev in order to deploy the shared stack diff --git a/infrastructure/terraform/modules/lambda_with_api_gateway/variables.tf b/infrastructure/terraform/modules/lambda_with_api_gateway/variables.tf index 95e5acd9..b5d0515a 100644 --- a/infrastructure/terraform/modules/lambda_with_api_gateway/variables.tf +++ b/infrastructure/terraform/modules/lambda_with_api_gateway/variables.tf @@ -11,7 +11,7 @@ variable "zip_excludes" { "**/*.pyc", "**/.pytest_cache/**", "**/tests/**", - "**/infrastructure/**" + "**/deployment/**" ] } From 54a674b5c88bd7907e77ac83db755c47ff4d8028 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 19 May 2026 16:35:09 +0000 Subject: [PATCH 02/22] added postcode splitter rewrite to ddd --- AGENTS.md | 29 ---- CLAUDE.md | 29 ---- asset_list/app.py | 24 ++- .../terraform/README.md | 0 .../terraform/cdn/main.tf | 0 .../terraform/cdn/provider.tf | 0 .../terraform/cdn/variables.tf | 0 .../terraform/cdn_certificate/main.tf | 0 .../terraform/cdn_certificate/outputs.tf | 0 .../terraform/cdn_certificate/provider.tf | 0 .../terraform/cdn_certificate/variables.tf | 0 .../terraform/lambda/_template/README.md | 0 .../terraform/lambda/_template/main.tf | 0 .../terraform/lambda/_template/provider.tf | 0 .../terraform/lambda/_template/variables.tf | 0 .../terraform/lambda/address2UPRN/main.tf | 0 .../terraform/lambda/address2UPRN/outputs.tf | 0 .../terraform/lambda/address2UPRN/provider.tf | 0 .../lambda/address2UPRN/variables.tf | 0 .../lambda/bulk_address2uprn_combiner/main.tf | 0 .../bulk_address2uprn_combiner/outputs.tf | 0 .../bulk_address2uprn_combiner/provider.tf | 0 .../bulk_address2uprn_combiner/variables.tf | 0 .../terraform/lambda/categorisation/main.tf | 0 .../lambda/categorisation/outputs.tf | 0 .../lambda/categorisation/provider.tf | 0 .../lambda/categorisation/variables.tf | 0 .../terraform/lambda/condition-etl/main.tf | 0 .../lambda/condition-etl/provider.tf | 0 .../lambda/condition-etl/variables.tf | 0 .../terraform/lambda/ecmk_to_ara/main.tf | 0 .../terraform/lambda/ecmk_to_ara/provider.tf | 0 .../terraform/lambda/ecmk_to_ara/variables.tf | 0 .../terraform/lambda/engine/main.tf | 0 .../terraform/lambda/engine/outputs.tf | 0 .../terraform/lambda/engine/provider.tf | 0 .../terraform/lambda/engine/variables.tf | 0 .../terraform/lambda/fast-api/main.tf | 0 .../terraform/lambda/fast-api/outputs.tf | 0 .../terraform/lambda/fast-api/provider.tf | 0 .../terraform/lambda/fast-api/variables.tf | 0 .../terraform/lambda/hubspot_deal_etl/main.tf | 0 .../lambda/hubspot_deal_etl/provider.tf | 0 .../lambda/hubspot_deal_etl/variables.tf | 0 .../terraform/lambda/magic_plan/main.tf | 0 .../terraform/lambda/magic_plan/outputs.tf | 0 .../terraform/lambda/magic_plan/provider.tf | 0 .../terraform/lambda/magic_plan/variables.tf | 0 .../terraform/lambda/ordnanceSurvey/main.tf | 0 .../lambda/ordnanceSurvey/provider.tf | 0 .../lambda/ordnanceSurvey/variables.tf | 0 .../terraform/lambda/pashub_to_ara/main.tf | 0 .../terraform/lambda/pashub_to_ara/outputs.tf | 0 .../lambda/pashub_to_ara/provider.tf | 0 .../lambda/pashub_to_ara/variables.tf | 0 .../terraform/lambda/postcodeSplitter/main.tf | 0 .../lambda/postcodeSplitter/outputs.tf | 0 .../lambda/postcodeSplitter/provider.tf | 0 .../lambda/postcodeSplitter/variables.tf | 0 .../terraform/modules/acm_certificate/main.tf | 0 .../modules/acm_certificate/outputs.tf | 0 .../modules/acm_certificate/variables.tf | 0 .../terraform/modules/cloudfront/main.tf | 0 .../terraform/modules/cloudfront/variables.tf | 0 .../modules/container_registry/main.tf | 0 .../modules/container_registry/outputs.tf | 0 .../modules/container_registry/variables.tf | 0 .../terraform/modules/ecr/main.tf | 0 .../terraform/modules/ecr/outputs.tf | 0 .../terraform/modules/ecr/variables.tf | 0 .../modules/general_iam_policy/main.tf | 0 .../modules/general_iam_policy/outputs.tf | 0 .../modules/general_iam_policy/variables.tf | 0 .../modules/lambda_execution_role/main.tf | 0 .../modules/lambda_execution_role/outputs.tf | 0 .../lambda_execution_role/variables.tf | 0 .../terraform/modules/lambda_service/main.tf | 0 .../modules/lambda_service/outputs.tf | 0 .../modules/lambda_service/variables.tf | 0 .../modules/lambda_service_zip/main.tf | 0 .../modules/lambda_service_zip/variables.tf | 0 .../modules/lambda_sqs_trigger/main.tf | 0 .../modules/lambda_sqs_trigger/variables.tf | 0 .../modules/lambda_with_api_gateway/main.tf | 0 .../lambda_with_api_gateway/outputs.tf | 0 .../lambda_with_api_gateway/variables.tf | 0 .../terraform/modules/lambda_with_sqs/main.tf | 0 .../modules/lambda_with_sqs/outputs.tf | 0 .../modules/lambda_with_sqs/variables.tf | 0 .../terraform/modules/route53/main.tf | 0 .../terraform/modules/route53/variables.tf | 0 .../terraform/modules/s3/main.tf | 0 .../terraform/modules/s3/outputs.tf | 0 .../terraform/modules/s3/variables.tf | 0 .../terraform/modules/s3_iam_policy/main.tf | 0 .../modules/s3_iam_policy/outputs.tf | 0 .../modules/s3_iam_policy/variables.tf | 0 .../modules/s3_presignable_bucket/main.tf | 0 .../modules/s3_presignable_bucket/outputs.tf | 0 .../s3_presignable_bucket/variables.tf | 0 .../terraform/modules/ses/main.tf | 0 .../terraform/modules/ses/outputs.tf | 0 .../terraform/modules/ses/variables.tf | 0 .../terraform/modules/sqs_queue/main.tf | 0 .../terraform/modules/sqs_queue/outputs.tf | 0 .../terraform/modules/sqs_queue/variables.tf | 0 .../terraform/modules/tf_state_bucket/main.tf | 0 .../modules/tf_state_bucket/outputs.tf | 0 .../modules/tf_state_bucket/variables.tf | 0 .../terraform/shared/dev.tfvars | 0 .../terraform/shared/main.tf | 0 .../terraform/shared/secrets.tf | 0 .../terraform/shared/variables.tf | 0 domain/__init__.py | 0 domain/tasks/__init__.py | 0 domain/tasks/subtasks.py | 55 +++++++ domain/tasks/tasks.py | 94 +++++++++++ infrastructure/__init__.py | 0 infrastructure/postgres/__init__.py | 0 infrastructure/postgres/config.py | 33 ++++ infrastructure/postgres/engine.py | 18 +++ infrastructure/postgres/subtask_table.py | 21 +++ infrastructure/postgres/task_table.py | 36 +++++ orchestration/__init__.py | 0 orchestration/task_orchestrator.py | 96 +++++++++++ repositories/__init__.py | 0 repositories/tasks/__init__.py | 0 .../tasks/subtask_postgres_repository.py | 89 +++++++++++ repositories/tasks/subtask_repository.py | 18 +++ .../tasks/task_postgres_repository.py | 77 +++++++++ repositories/tasks/task_repository.py | 15 ++ run_backlog.sh | 2 - tests/__init__.py | 0 tests/domain/__init__.py | 0 tests/domain/tasks/__init__.py | 0 tests/domain/tasks/test_subtasks.py | 75 +++++++++ tests/domain/tasks/test_tasks.py | 104 ++++++++++++ tests/orchestration/__init__.py | 0 tests/orchestration/test_task_orchestrator.py | 151 ++++++++++++++++++ tests/repositories/__init__.py | 0 tests/repositories/tasks/__init__.py | 0 tests/repositories/tasks/postgres/__init__.py | 0 .../test_subtask_postgres_repository.py | 81 ++++++++++ .../postgres/test_task_postgres_repository.py | 68 ++++++++ utilities/__init__.py | 0 utilities/aws_lambda/__init__.py | 0 utilities/aws_lambda/default_orchestrator.py | 26 +++ utilities/aws_lambda/subtask_handler.py | 67 ++++++++ utilities/aws_lambda/subtask_trigger_body.py | 17 ++ utilities/aws_lambda/task_handler.py | 98 ++++++++++++ utilities/private.py | 33 ++++ 151 files changed, 1281 insertions(+), 75 deletions(-) delete mode 100644 AGENTS.md rename {infrastructure => deployment}/terraform/README.md (100%) rename {infrastructure => deployment}/terraform/cdn/main.tf (100%) rename {infrastructure => deployment}/terraform/cdn/provider.tf (100%) rename {infrastructure => deployment}/terraform/cdn/variables.tf (100%) rename {infrastructure => deployment}/terraform/cdn_certificate/main.tf (100%) rename {infrastructure => deployment}/terraform/cdn_certificate/outputs.tf (100%) rename {infrastructure => deployment}/terraform/cdn_certificate/provider.tf (100%) rename {infrastructure => deployment}/terraform/cdn_certificate/variables.tf (100%) rename {infrastructure => deployment}/terraform/lambda/_template/README.md (100%) rename {infrastructure => deployment}/terraform/lambda/_template/main.tf (100%) rename {infrastructure => deployment}/terraform/lambda/_template/provider.tf (100%) rename {infrastructure => deployment}/terraform/lambda/_template/variables.tf (100%) rename {infrastructure => deployment}/terraform/lambda/address2UPRN/main.tf (100%) rename {infrastructure => deployment}/terraform/lambda/address2UPRN/outputs.tf (100%) rename {infrastructure => deployment}/terraform/lambda/address2UPRN/provider.tf (100%) rename {infrastructure => deployment}/terraform/lambda/address2UPRN/variables.tf (100%) rename {infrastructure => deployment}/terraform/lambda/bulk_address2uprn_combiner/main.tf (100%) rename {infrastructure => deployment}/terraform/lambda/bulk_address2uprn_combiner/outputs.tf (100%) rename {infrastructure => deployment}/terraform/lambda/bulk_address2uprn_combiner/provider.tf (100%) rename {infrastructure => deployment}/terraform/lambda/bulk_address2uprn_combiner/variables.tf (100%) rename {infrastructure => deployment}/terraform/lambda/categorisation/main.tf (100%) rename {infrastructure => deployment}/terraform/lambda/categorisation/outputs.tf (100%) rename {infrastructure => deployment}/terraform/lambda/categorisation/provider.tf (100%) rename {infrastructure => deployment}/terraform/lambda/categorisation/variables.tf (100%) rename {infrastructure => deployment}/terraform/lambda/condition-etl/main.tf (100%) rename {infrastructure => deployment}/terraform/lambda/condition-etl/provider.tf (100%) rename {infrastructure => deployment}/terraform/lambda/condition-etl/variables.tf (100%) rename {infrastructure => deployment}/terraform/lambda/ecmk_to_ara/main.tf (100%) rename {infrastructure => deployment}/terraform/lambda/ecmk_to_ara/provider.tf (100%) rename {infrastructure => deployment}/terraform/lambda/ecmk_to_ara/variables.tf (100%) rename {infrastructure => deployment}/terraform/lambda/engine/main.tf (100%) rename {infrastructure => deployment}/terraform/lambda/engine/outputs.tf (100%) rename {infrastructure => deployment}/terraform/lambda/engine/provider.tf (100%) rename {infrastructure => deployment}/terraform/lambda/engine/variables.tf (100%) rename {infrastructure => deployment}/terraform/lambda/fast-api/main.tf (100%) rename {infrastructure => deployment}/terraform/lambda/fast-api/outputs.tf (100%) rename {infrastructure => deployment}/terraform/lambda/fast-api/provider.tf (100%) rename {infrastructure => deployment}/terraform/lambda/fast-api/variables.tf (100%) rename {infrastructure => deployment}/terraform/lambda/hubspot_deal_etl/main.tf (100%) rename {infrastructure => deployment}/terraform/lambda/hubspot_deal_etl/provider.tf (100%) rename {infrastructure => deployment}/terraform/lambda/hubspot_deal_etl/variables.tf (100%) rename {infrastructure => deployment}/terraform/lambda/magic_plan/main.tf (100%) rename {infrastructure => deployment}/terraform/lambda/magic_plan/outputs.tf (100%) rename {infrastructure => deployment}/terraform/lambda/magic_plan/provider.tf (100%) rename {infrastructure => deployment}/terraform/lambda/magic_plan/variables.tf (100%) rename {infrastructure => deployment}/terraform/lambda/ordnanceSurvey/main.tf (100%) rename {infrastructure => deployment}/terraform/lambda/ordnanceSurvey/provider.tf (100%) rename {infrastructure => deployment}/terraform/lambda/ordnanceSurvey/variables.tf (100%) rename {infrastructure => deployment}/terraform/lambda/pashub_to_ara/main.tf (100%) rename {infrastructure => deployment}/terraform/lambda/pashub_to_ara/outputs.tf (100%) rename {infrastructure => deployment}/terraform/lambda/pashub_to_ara/provider.tf (100%) rename {infrastructure => deployment}/terraform/lambda/pashub_to_ara/variables.tf (100%) rename {infrastructure => deployment}/terraform/lambda/postcodeSplitter/main.tf (100%) rename {infrastructure => deployment}/terraform/lambda/postcodeSplitter/outputs.tf (100%) rename {infrastructure => deployment}/terraform/lambda/postcodeSplitter/provider.tf (100%) rename {infrastructure => deployment}/terraform/lambda/postcodeSplitter/variables.tf (100%) rename {infrastructure => deployment}/terraform/modules/acm_certificate/main.tf (100%) rename {infrastructure => deployment}/terraform/modules/acm_certificate/outputs.tf (100%) rename {infrastructure => deployment}/terraform/modules/acm_certificate/variables.tf (100%) rename {infrastructure => deployment}/terraform/modules/cloudfront/main.tf (100%) rename {infrastructure => deployment}/terraform/modules/cloudfront/variables.tf (100%) rename {infrastructure => deployment}/terraform/modules/container_registry/main.tf (100%) rename {infrastructure => deployment}/terraform/modules/container_registry/outputs.tf (100%) rename {infrastructure => deployment}/terraform/modules/container_registry/variables.tf (100%) rename {infrastructure => deployment}/terraform/modules/ecr/main.tf (100%) rename {infrastructure => deployment}/terraform/modules/ecr/outputs.tf (100%) rename {infrastructure => deployment}/terraform/modules/ecr/variables.tf (100%) rename {infrastructure => deployment}/terraform/modules/general_iam_policy/main.tf (100%) rename {infrastructure => deployment}/terraform/modules/general_iam_policy/outputs.tf (100%) rename {infrastructure => deployment}/terraform/modules/general_iam_policy/variables.tf (100%) rename {infrastructure => deployment}/terraform/modules/lambda_execution_role/main.tf (100%) rename {infrastructure => deployment}/terraform/modules/lambda_execution_role/outputs.tf (100%) rename {infrastructure => deployment}/terraform/modules/lambda_execution_role/variables.tf (100%) rename {infrastructure => deployment}/terraform/modules/lambda_service/main.tf (100%) rename {infrastructure => deployment}/terraform/modules/lambda_service/outputs.tf (100%) rename {infrastructure => deployment}/terraform/modules/lambda_service/variables.tf (100%) rename {infrastructure => deployment}/terraform/modules/lambda_service_zip/main.tf (100%) rename {infrastructure => deployment}/terraform/modules/lambda_service_zip/variables.tf (100%) rename {infrastructure => deployment}/terraform/modules/lambda_sqs_trigger/main.tf (100%) rename {infrastructure => deployment}/terraform/modules/lambda_sqs_trigger/variables.tf (100%) rename {infrastructure => deployment}/terraform/modules/lambda_with_api_gateway/main.tf (100%) rename {infrastructure => deployment}/terraform/modules/lambda_with_api_gateway/outputs.tf (100%) rename {infrastructure => deployment}/terraform/modules/lambda_with_api_gateway/variables.tf (100%) rename {infrastructure => deployment}/terraform/modules/lambda_with_sqs/main.tf (100%) rename {infrastructure => deployment}/terraform/modules/lambda_with_sqs/outputs.tf (100%) rename {infrastructure => deployment}/terraform/modules/lambda_with_sqs/variables.tf (100%) rename {infrastructure => deployment}/terraform/modules/route53/main.tf (100%) rename {infrastructure => deployment}/terraform/modules/route53/variables.tf (100%) rename {infrastructure => deployment}/terraform/modules/s3/main.tf (100%) rename {infrastructure => deployment}/terraform/modules/s3/outputs.tf (100%) rename {infrastructure => deployment}/terraform/modules/s3/variables.tf (100%) rename {infrastructure => deployment}/terraform/modules/s3_iam_policy/main.tf (100%) rename {infrastructure => deployment}/terraform/modules/s3_iam_policy/outputs.tf (100%) rename {infrastructure => deployment}/terraform/modules/s3_iam_policy/variables.tf (100%) rename {infrastructure => deployment}/terraform/modules/s3_presignable_bucket/main.tf (100%) rename {infrastructure => deployment}/terraform/modules/s3_presignable_bucket/outputs.tf (100%) rename {infrastructure => deployment}/terraform/modules/s3_presignable_bucket/variables.tf (100%) rename {infrastructure => deployment}/terraform/modules/ses/main.tf (100%) rename {infrastructure => deployment}/terraform/modules/ses/outputs.tf (100%) rename {infrastructure => deployment}/terraform/modules/ses/variables.tf (100%) rename {infrastructure => deployment}/terraform/modules/sqs_queue/main.tf (100%) rename {infrastructure => deployment}/terraform/modules/sqs_queue/outputs.tf (100%) rename {infrastructure => deployment}/terraform/modules/sqs_queue/variables.tf (100%) rename {infrastructure => deployment}/terraform/modules/tf_state_bucket/main.tf (100%) rename {infrastructure => deployment}/terraform/modules/tf_state_bucket/outputs.tf (100%) rename {infrastructure => deployment}/terraform/modules/tf_state_bucket/variables.tf (100%) rename {infrastructure => deployment}/terraform/shared/dev.tfvars (100%) rename {infrastructure => deployment}/terraform/shared/main.tf (100%) rename {infrastructure => deployment}/terraform/shared/secrets.tf (100%) rename {infrastructure => deployment}/terraform/shared/variables.tf (100%) create mode 100644 domain/__init__.py create mode 100644 domain/tasks/__init__.py create mode 100644 domain/tasks/subtasks.py create mode 100644 domain/tasks/tasks.py create mode 100644 infrastructure/__init__.py create mode 100644 infrastructure/postgres/__init__.py create mode 100644 infrastructure/postgres/config.py create mode 100644 infrastructure/postgres/engine.py create mode 100644 infrastructure/postgres/subtask_table.py create mode 100644 infrastructure/postgres/task_table.py create mode 100644 orchestration/__init__.py create mode 100644 orchestration/task_orchestrator.py create mode 100644 repositories/__init__.py create mode 100644 repositories/tasks/__init__.py create mode 100644 repositories/tasks/subtask_postgres_repository.py create mode 100644 repositories/tasks/subtask_repository.py create mode 100644 repositories/tasks/task_postgres_repository.py create mode 100644 repositories/tasks/task_repository.py delete mode 100644 run_backlog.sh create mode 100644 tests/__init__.py create mode 100644 tests/domain/__init__.py create mode 100644 tests/domain/tasks/__init__.py create mode 100644 tests/domain/tasks/test_subtasks.py create mode 100644 tests/domain/tasks/test_tasks.py create mode 100644 tests/orchestration/__init__.py create mode 100644 tests/orchestration/test_task_orchestrator.py create mode 100644 tests/repositories/__init__.py create mode 100644 tests/repositories/tasks/__init__.py create mode 100644 tests/repositories/tasks/postgres/__init__.py create mode 100644 tests/repositories/tasks/postgres/test_subtask_postgres_repository.py create mode 100644 tests/repositories/tasks/postgres/test_task_postgres_repository.py create mode 100644 utilities/__init__.py create mode 100644 utilities/aws_lambda/__init__.py create mode 100644 utilities/aws_lambda/default_orchestrator.py create mode 100644 utilities/aws_lambda/subtask_handler.py create mode 100644 utilities/aws_lambda/subtask_trigger_body.py create mode 100644 utilities/aws_lambda/task_handler.py create mode 100644 utilities/private.py diff --git a/AGENTS.md b/AGENTS.md deleted file mode 100644 index aa0426a0..00000000 --- a/AGENTS.md +++ /dev/null @@ -1,29 +0,0 @@ - - - - - -## BACKLOG WORKFLOW INSTRUCTIONS - -This project uses Backlog.md MCP for all task and project management activities. - -**CRITICAL GUIDANCE** - -- If your client supports MCP resources, read `backlog://workflow/overview` to understand when and how to use Backlog for this project. -- If your client only supports tools or the above request fails, call `backlog.get_backlog_instructions()` to load the tool-oriented overview. Use the `instruction` selector when you need `task-creation`, `task-execution`, or `task-finalization`. - -- **First time working here?** Read the overview resource IMMEDIATELY to learn the workflow -- **Already familiar?** You should have the overview cached ("## Backlog.md Overview (MCP)") -- **When to read it**: BEFORE creating tasks, or when you're unsure whether to track work - -These guides cover: -- Decision framework for when to create tasks -- Search-first workflow to avoid duplicates -- Links to detailed guides for task creation, execution, and finalization -- MCP tools reference - -You MUST read the overview resource to understand the complete workflow. The information is NOT summarized here. - - - - diff --git a/CLAUDE.md b/CLAUDE.md index f88a59d5..2dabf532 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,33 +1,4 @@ - - - - -## BACKLOG WORKFLOW INSTRUCTIONS - -This project uses Backlog.md MCP for all task and project management activities. - -**CRITICAL GUIDANCE** - -- If your client supports MCP resources, read `backlog://workflow/overview` to understand when and how to use Backlog for this project. -- If your client only supports tools or the above request fails, call `backlog.get_backlog_instructions()` to load the tool-oriented overview. Use the `instruction` selector when you need `task-creation`, `task-execution`, or `task-finalization`. - -- **First time working here?** Read the overview resource IMMEDIATELY to learn the workflow -- **Already familiar?** You should have the overview cached ("## Backlog.md Overview (MCP)") -- **When to read it**: BEFORE creating tasks, or when you're unsure whether to track work - -These guides cover: -- Decision framework for when to create tasks -- Search-first workflow to avoid duplicates -- Links to detailed guides for task creation, execution, and finalization -- MCP tools reference - -You MUST read the overview resource to understand the complete workflow. The information is NOT summarized here. - - - - - ## Available Skills Five Claude Code skills are installed in this repo's dev container. Each maps to a phase of the feature lifecycle. diff --git a/asset_list/app.py b/asset_list/app.py index 7413c7cb..9b10d7f3 100644 --- a/asset_list/app.py +++ b/asset_list/app.py @@ -79,23 +79,23 @@ def app(): """ data_folder = "/workspaces/model/asset_list" - data_filename = "input.xlsx" - sheet_name = "Handovers" - postcode_column = "POSTCODE" - address1_column = "Full Addres" + data_filename = "lincs_address_list.xlsx" + sheet_name = "Sheet1" + postcode_column = "Postcode" + address1_column = "Deal Name" address1_method = None - fulladdress_column = "Full Addres" + fulladdress_column = "Deal Name" address_cols_to_concat = [] missing_postcodes_method = None landlord_year_built = None - landlord_os_uprn = "domna_found_uprn" - landlord_property_type = "PROPERTY TYPE" # Good to include if landlord gave - landlord_built_form = "Type Description" # Good to include if landlord gave + landlord_os_uprn = None + landlord_property_type = None # Good to include if landlord gave + landlord_built_form = None # Good to include if landlord gave landlord_wall_construction = None landlord_roof_construction = None landlord_heating_system = None landlord_existing_pv = None - landlord_property_id = "PROP REF" + landlord_property_id = "landlord_id" landlord_sap = None outcomes_filename = None outcomes_sheetname = None @@ -468,9 +468,3 @@ def app(): asset_list.duplicated_addresses.to_excel( writer, sheet_name="Duplicate Properties", index=False ) - - - - -for key,value in dict.items(): - lsakjfldsa \ No newline at end of file diff --git a/infrastructure/terraform/README.md b/deployment/terraform/README.md similarity index 100% rename from infrastructure/terraform/README.md rename to deployment/terraform/README.md diff --git a/infrastructure/terraform/cdn/main.tf b/deployment/terraform/cdn/main.tf similarity index 100% rename from infrastructure/terraform/cdn/main.tf rename to deployment/terraform/cdn/main.tf diff --git a/infrastructure/terraform/cdn/provider.tf b/deployment/terraform/cdn/provider.tf similarity index 100% rename from infrastructure/terraform/cdn/provider.tf rename to deployment/terraform/cdn/provider.tf diff --git a/infrastructure/terraform/cdn/variables.tf b/deployment/terraform/cdn/variables.tf similarity index 100% rename from infrastructure/terraform/cdn/variables.tf rename to deployment/terraform/cdn/variables.tf diff --git a/infrastructure/terraform/cdn_certificate/main.tf b/deployment/terraform/cdn_certificate/main.tf similarity index 100% rename from infrastructure/terraform/cdn_certificate/main.tf rename to deployment/terraform/cdn_certificate/main.tf diff --git a/infrastructure/terraform/cdn_certificate/outputs.tf b/deployment/terraform/cdn_certificate/outputs.tf similarity index 100% rename from infrastructure/terraform/cdn_certificate/outputs.tf rename to deployment/terraform/cdn_certificate/outputs.tf diff --git a/infrastructure/terraform/cdn_certificate/provider.tf b/deployment/terraform/cdn_certificate/provider.tf similarity index 100% rename from infrastructure/terraform/cdn_certificate/provider.tf rename to deployment/terraform/cdn_certificate/provider.tf diff --git a/infrastructure/terraform/cdn_certificate/variables.tf b/deployment/terraform/cdn_certificate/variables.tf similarity index 100% rename from infrastructure/terraform/cdn_certificate/variables.tf rename to deployment/terraform/cdn_certificate/variables.tf diff --git a/infrastructure/terraform/lambda/_template/README.md b/deployment/terraform/lambda/_template/README.md similarity index 100% rename from infrastructure/terraform/lambda/_template/README.md rename to deployment/terraform/lambda/_template/README.md diff --git a/infrastructure/terraform/lambda/_template/main.tf b/deployment/terraform/lambda/_template/main.tf similarity index 100% rename from infrastructure/terraform/lambda/_template/main.tf rename to deployment/terraform/lambda/_template/main.tf diff --git a/infrastructure/terraform/lambda/_template/provider.tf b/deployment/terraform/lambda/_template/provider.tf similarity index 100% rename from infrastructure/terraform/lambda/_template/provider.tf rename to deployment/terraform/lambda/_template/provider.tf diff --git a/infrastructure/terraform/lambda/_template/variables.tf b/deployment/terraform/lambda/_template/variables.tf similarity index 100% rename from infrastructure/terraform/lambda/_template/variables.tf rename to deployment/terraform/lambda/_template/variables.tf diff --git a/infrastructure/terraform/lambda/address2UPRN/main.tf b/deployment/terraform/lambda/address2UPRN/main.tf similarity index 100% rename from infrastructure/terraform/lambda/address2UPRN/main.tf rename to deployment/terraform/lambda/address2UPRN/main.tf diff --git a/infrastructure/terraform/lambda/address2UPRN/outputs.tf b/deployment/terraform/lambda/address2UPRN/outputs.tf similarity index 100% rename from infrastructure/terraform/lambda/address2UPRN/outputs.tf rename to deployment/terraform/lambda/address2UPRN/outputs.tf diff --git a/infrastructure/terraform/lambda/address2UPRN/provider.tf b/deployment/terraform/lambda/address2UPRN/provider.tf similarity index 100% rename from infrastructure/terraform/lambda/address2UPRN/provider.tf rename to deployment/terraform/lambda/address2UPRN/provider.tf diff --git a/infrastructure/terraform/lambda/address2UPRN/variables.tf b/deployment/terraform/lambda/address2UPRN/variables.tf similarity index 100% rename from infrastructure/terraform/lambda/address2UPRN/variables.tf rename to deployment/terraform/lambda/address2UPRN/variables.tf diff --git a/infrastructure/terraform/lambda/bulk_address2uprn_combiner/main.tf b/deployment/terraform/lambda/bulk_address2uprn_combiner/main.tf similarity index 100% rename from infrastructure/terraform/lambda/bulk_address2uprn_combiner/main.tf rename to deployment/terraform/lambda/bulk_address2uprn_combiner/main.tf diff --git a/infrastructure/terraform/lambda/bulk_address2uprn_combiner/outputs.tf b/deployment/terraform/lambda/bulk_address2uprn_combiner/outputs.tf similarity index 100% rename from infrastructure/terraform/lambda/bulk_address2uprn_combiner/outputs.tf rename to deployment/terraform/lambda/bulk_address2uprn_combiner/outputs.tf diff --git a/infrastructure/terraform/lambda/bulk_address2uprn_combiner/provider.tf b/deployment/terraform/lambda/bulk_address2uprn_combiner/provider.tf similarity index 100% rename from infrastructure/terraform/lambda/bulk_address2uprn_combiner/provider.tf rename to deployment/terraform/lambda/bulk_address2uprn_combiner/provider.tf diff --git a/infrastructure/terraform/lambda/bulk_address2uprn_combiner/variables.tf b/deployment/terraform/lambda/bulk_address2uprn_combiner/variables.tf similarity index 100% rename from infrastructure/terraform/lambda/bulk_address2uprn_combiner/variables.tf rename to deployment/terraform/lambda/bulk_address2uprn_combiner/variables.tf diff --git a/infrastructure/terraform/lambda/categorisation/main.tf b/deployment/terraform/lambda/categorisation/main.tf similarity index 100% rename from infrastructure/terraform/lambda/categorisation/main.tf rename to deployment/terraform/lambda/categorisation/main.tf diff --git a/infrastructure/terraform/lambda/categorisation/outputs.tf b/deployment/terraform/lambda/categorisation/outputs.tf similarity index 100% rename from infrastructure/terraform/lambda/categorisation/outputs.tf rename to deployment/terraform/lambda/categorisation/outputs.tf diff --git a/infrastructure/terraform/lambda/categorisation/provider.tf b/deployment/terraform/lambda/categorisation/provider.tf similarity index 100% rename from infrastructure/terraform/lambda/categorisation/provider.tf rename to deployment/terraform/lambda/categorisation/provider.tf diff --git a/infrastructure/terraform/lambda/categorisation/variables.tf b/deployment/terraform/lambda/categorisation/variables.tf similarity index 100% rename from infrastructure/terraform/lambda/categorisation/variables.tf rename to deployment/terraform/lambda/categorisation/variables.tf diff --git a/infrastructure/terraform/lambda/condition-etl/main.tf b/deployment/terraform/lambda/condition-etl/main.tf similarity index 100% rename from infrastructure/terraform/lambda/condition-etl/main.tf rename to deployment/terraform/lambda/condition-etl/main.tf diff --git a/infrastructure/terraform/lambda/condition-etl/provider.tf b/deployment/terraform/lambda/condition-etl/provider.tf similarity index 100% rename from infrastructure/terraform/lambda/condition-etl/provider.tf rename to deployment/terraform/lambda/condition-etl/provider.tf diff --git a/infrastructure/terraform/lambda/condition-etl/variables.tf b/deployment/terraform/lambda/condition-etl/variables.tf similarity index 100% rename from infrastructure/terraform/lambda/condition-etl/variables.tf rename to deployment/terraform/lambda/condition-etl/variables.tf diff --git a/infrastructure/terraform/lambda/ecmk_to_ara/main.tf b/deployment/terraform/lambda/ecmk_to_ara/main.tf similarity index 100% rename from infrastructure/terraform/lambda/ecmk_to_ara/main.tf rename to deployment/terraform/lambda/ecmk_to_ara/main.tf diff --git a/infrastructure/terraform/lambda/ecmk_to_ara/provider.tf b/deployment/terraform/lambda/ecmk_to_ara/provider.tf similarity index 100% rename from infrastructure/terraform/lambda/ecmk_to_ara/provider.tf rename to deployment/terraform/lambda/ecmk_to_ara/provider.tf diff --git a/infrastructure/terraform/lambda/ecmk_to_ara/variables.tf b/deployment/terraform/lambda/ecmk_to_ara/variables.tf similarity index 100% rename from infrastructure/terraform/lambda/ecmk_to_ara/variables.tf rename to deployment/terraform/lambda/ecmk_to_ara/variables.tf diff --git a/infrastructure/terraform/lambda/engine/main.tf b/deployment/terraform/lambda/engine/main.tf similarity index 100% rename from infrastructure/terraform/lambda/engine/main.tf rename to deployment/terraform/lambda/engine/main.tf diff --git a/infrastructure/terraform/lambda/engine/outputs.tf b/deployment/terraform/lambda/engine/outputs.tf similarity index 100% rename from infrastructure/terraform/lambda/engine/outputs.tf rename to deployment/terraform/lambda/engine/outputs.tf diff --git a/infrastructure/terraform/lambda/engine/provider.tf b/deployment/terraform/lambda/engine/provider.tf similarity index 100% rename from infrastructure/terraform/lambda/engine/provider.tf rename to deployment/terraform/lambda/engine/provider.tf diff --git a/infrastructure/terraform/lambda/engine/variables.tf b/deployment/terraform/lambda/engine/variables.tf similarity index 100% rename from infrastructure/terraform/lambda/engine/variables.tf rename to deployment/terraform/lambda/engine/variables.tf diff --git a/infrastructure/terraform/lambda/fast-api/main.tf b/deployment/terraform/lambda/fast-api/main.tf similarity index 100% rename from infrastructure/terraform/lambda/fast-api/main.tf rename to deployment/terraform/lambda/fast-api/main.tf diff --git a/infrastructure/terraform/lambda/fast-api/outputs.tf b/deployment/terraform/lambda/fast-api/outputs.tf similarity index 100% rename from infrastructure/terraform/lambda/fast-api/outputs.tf rename to deployment/terraform/lambda/fast-api/outputs.tf diff --git a/infrastructure/terraform/lambda/fast-api/provider.tf b/deployment/terraform/lambda/fast-api/provider.tf similarity index 100% rename from infrastructure/terraform/lambda/fast-api/provider.tf rename to deployment/terraform/lambda/fast-api/provider.tf diff --git a/infrastructure/terraform/lambda/fast-api/variables.tf b/deployment/terraform/lambda/fast-api/variables.tf similarity index 100% rename from infrastructure/terraform/lambda/fast-api/variables.tf rename to deployment/terraform/lambda/fast-api/variables.tf diff --git a/infrastructure/terraform/lambda/hubspot_deal_etl/main.tf b/deployment/terraform/lambda/hubspot_deal_etl/main.tf similarity index 100% rename from infrastructure/terraform/lambda/hubspot_deal_etl/main.tf rename to deployment/terraform/lambda/hubspot_deal_etl/main.tf diff --git a/infrastructure/terraform/lambda/hubspot_deal_etl/provider.tf b/deployment/terraform/lambda/hubspot_deal_etl/provider.tf similarity index 100% rename from infrastructure/terraform/lambda/hubspot_deal_etl/provider.tf rename to deployment/terraform/lambda/hubspot_deal_etl/provider.tf diff --git a/infrastructure/terraform/lambda/hubspot_deal_etl/variables.tf b/deployment/terraform/lambda/hubspot_deal_etl/variables.tf similarity index 100% rename from infrastructure/terraform/lambda/hubspot_deal_etl/variables.tf rename to deployment/terraform/lambda/hubspot_deal_etl/variables.tf diff --git a/infrastructure/terraform/lambda/magic_plan/main.tf b/deployment/terraform/lambda/magic_plan/main.tf similarity index 100% rename from infrastructure/terraform/lambda/magic_plan/main.tf rename to deployment/terraform/lambda/magic_plan/main.tf diff --git a/infrastructure/terraform/lambda/magic_plan/outputs.tf b/deployment/terraform/lambda/magic_plan/outputs.tf similarity index 100% rename from infrastructure/terraform/lambda/magic_plan/outputs.tf rename to deployment/terraform/lambda/magic_plan/outputs.tf diff --git a/infrastructure/terraform/lambda/magic_plan/provider.tf b/deployment/terraform/lambda/magic_plan/provider.tf similarity index 100% rename from infrastructure/terraform/lambda/magic_plan/provider.tf rename to deployment/terraform/lambda/magic_plan/provider.tf diff --git a/infrastructure/terraform/lambda/magic_plan/variables.tf b/deployment/terraform/lambda/magic_plan/variables.tf similarity index 100% rename from infrastructure/terraform/lambda/magic_plan/variables.tf rename to deployment/terraform/lambda/magic_plan/variables.tf diff --git a/infrastructure/terraform/lambda/ordnanceSurvey/main.tf b/deployment/terraform/lambda/ordnanceSurvey/main.tf similarity index 100% rename from infrastructure/terraform/lambda/ordnanceSurvey/main.tf rename to deployment/terraform/lambda/ordnanceSurvey/main.tf diff --git a/infrastructure/terraform/lambda/ordnanceSurvey/provider.tf b/deployment/terraform/lambda/ordnanceSurvey/provider.tf similarity index 100% rename from infrastructure/terraform/lambda/ordnanceSurvey/provider.tf rename to deployment/terraform/lambda/ordnanceSurvey/provider.tf diff --git a/infrastructure/terraform/lambda/ordnanceSurvey/variables.tf b/deployment/terraform/lambda/ordnanceSurvey/variables.tf similarity index 100% rename from infrastructure/terraform/lambda/ordnanceSurvey/variables.tf rename to deployment/terraform/lambda/ordnanceSurvey/variables.tf diff --git a/infrastructure/terraform/lambda/pashub_to_ara/main.tf b/deployment/terraform/lambda/pashub_to_ara/main.tf similarity index 100% rename from infrastructure/terraform/lambda/pashub_to_ara/main.tf rename to deployment/terraform/lambda/pashub_to_ara/main.tf diff --git a/infrastructure/terraform/lambda/pashub_to_ara/outputs.tf b/deployment/terraform/lambda/pashub_to_ara/outputs.tf similarity index 100% rename from infrastructure/terraform/lambda/pashub_to_ara/outputs.tf rename to deployment/terraform/lambda/pashub_to_ara/outputs.tf diff --git a/infrastructure/terraform/lambda/pashub_to_ara/provider.tf b/deployment/terraform/lambda/pashub_to_ara/provider.tf similarity index 100% rename from infrastructure/terraform/lambda/pashub_to_ara/provider.tf rename to deployment/terraform/lambda/pashub_to_ara/provider.tf diff --git a/infrastructure/terraform/lambda/pashub_to_ara/variables.tf b/deployment/terraform/lambda/pashub_to_ara/variables.tf similarity index 100% rename from infrastructure/terraform/lambda/pashub_to_ara/variables.tf rename to deployment/terraform/lambda/pashub_to_ara/variables.tf diff --git a/infrastructure/terraform/lambda/postcodeSplitter/main.tf b/deployment/terraform/lambda/postcodeSplitter/main.tf similarity index 100% rename from infrastructure/terraform/lambda/postcodeSplitter/main.tf rename to deployment/terraform/lambda/postcodeSplitter/main.tf diff --git a/infrastructure/terraform/lambda/postcodeSplitter/outputs.tf b/deployment/terraform/lambda/postcodeSplitter/outputs.tf similarity index 100% rename from infrastructure/terraform/lambda/postcodeSplitter/outputs.tf rename to deployment/terraform/lambda/postcodeSplitter/outputs.tf diff --git a/infrastructure/terraform/lambda/postcodeSplitter/provider.tf b/deployment/terraform/lambda/postcodeSplitter/provider.tf similarity index 100% rename from infrastructure/terraform/lambda/postcodeSplitter/provider.tf rename to deployment/terraform/lambda/postcodeSplitter/provider.tf diff --git a/infrastructure/terraform/lambda/postcodeSplitter/variables.tf b/deployment/terraform/lambda/postcodeSplitter/variables.tf similarity index 100% rename from infrastructure/terraform/lambda/postcodeSplitter/variables.tf rename to deployment/terraform/lambda/postcodeSplitter/variables.tf diff --git a/infrastructure/terraform/modules/acm_certificate/main.tf b/deployment/terraform/modules/acm_certificate/main.tf similarity index 100% rename from infrastructure/terraform/modules/acm_certificate/main.tf rename to deployment/terraform/modules/acm_certificate/main.tf diff --git a/infrastructure/terraform/modules/acm_certificate/outputs.tf b/deployment/terraform/modules/acm_certificate/outputs.tf similarity index 100% rename from infrastructure/terraform/modules/acm_certificate/outputs.tf rename to deployment/terraform/modules/acm_certificate/outputs.tf diff --git a/infrastructure/terraform/modules/acm_certificate/variables.tf b/deployment/terraform/modules/acm_certificate/variables.tf similarity index 100% rename from infrastructure/terraform/modules/acm_certificate/variables.tf rename to deployment/terraform/modules/acm_certificate/variables.tf diff --git a/infrastructure/terraform/modules/cloudfront/main.tf b/deployment/terraform/modules/cloudfront/main.tf similarity index 100% rename from infrastructure/terraform/modules/cloudfront/main.tf rename to deployment/terraform/modules/cloudfront/main.tf diff --git a/infrastructure/terraform/modules/cloudfront/variables.tf b/deployment/terraform/modules/cloudfront/variables.tf similarity index 100% rename from infrastructure/terraform/modules/cloudfront/variables.tf rename to deployment/terraform/modules/cloudfront/variables.tf diff --git a/infrastructure/terraform/modules/container_registry/main.tf b/deployment/terraform/modules/container_registry/main.tf similarity index 100% rename from infrastructure/terraform/modules/container_registry/main.tf rename to deployment/terraform/modules/container_registry/main.tf diff --git a/infrastructure/terraform/modules/container_registry/outputs.tf b/deployment/terraform/modules/container_registry/outputs.tf similarity index 100% rename from infrastructure/terraform/modules/container_registry/outputs.tf rename to deployment/terraform/modules/container_registry/outputs.tf diff --git a/infrastructure/terraform/modules/container_registry/variables.tf b/deployment/terraform/modules/container_registry/variables.tf similarity index 100% rename from infrastructure/terraform/modules/container_registry/variables.tf rename to deployment/terraform/modules/container_registry/variables.tf diff --git a/infrastructure/terraform/modules/ecr/main.tf b/deployment/terraform/modules/ecr/main.tf similarity index 100% rename from infrastructure/terraform/modules/ecr/main.tf rename to deployment/terraform/modules/ecr/main.tf diff --git a/infrastructure/terraform/modules/ecr/outputs.tf b/deployment/terraform/modules/ecr/outputs.tf similarity index 100% rename from infrastructure/terraform/modules/ecr/outputs.tf rename to deployment/terraform/modules/ecr/outputs.tf diff --git a/infrastructure/terraform/modules/ecr/variables.tf b/deployment/terraform/modules/ecr/variables.tf similarity index 100% rename from infrastructure/terraform/modules/ecr/variables.tf rename to deployment/terraform/modules/ecr/variables.tf diff --git a/infrastructure/terraform/modules/general_iam_policy/main.tf b/deployment/terraform/modules/general_iam_policy/main.tf similarity index 100% rename from infrastructure/terraform/modules/general_iam_policy/main.tf rename to deployment/terraform/modules/general_iam_policy/main.tf diff --git a/infrastructure/terraform/modules/general_iam_policy/outputs.tf b/deployment/terraform/modules/general_iam_policy/outputs.tf similarity index 100% rename from infrastructure/terraform/modules/general_iam_policy/outputs.tf rename to deployment/terraform/modules/general_iam_policy/outputs.tf diff --git a/infrastructure/terraform/modules/general_iam_policy/variables.tf b/deployment/terraform/modules/general_iam_policy/variables.tf similarity index 100% rename from infrastructure/terraform/modules/general_iam_policy/variables.tf rename to deployment/terraform/modules/general_iam_policy/variables.tf diff --git a/infrastructure/terraform/modules/lambda_execution_role/main.tf b/deployment/terraform/modules/lambda_execution_role/main.tf similarity index 100% rename from infrastructure/terraform/modules/lambda_execution_role/main.tf rename to deployment/terraform/modules/lambda_execution_role/main.tf diff --git a/infrastructure/terraform/modules/lambda_execution_role/outputs.tf b/deployment/terraform/modules/lambda_execution_role/outputs.tf similarity index 100% rename from infrastructure/terraform/modules/lambda_execution_role/outputs.tf rename to deployment/terraform/modules/lambda_execution_role/outputs.tf diff --git a/infrastructure/terraform/modules/lambda_execution_role/variables.tf b/deployment/terraform/modules/lambda_execution_role/variables.tf similarity index 100% rename from infrastructure/terraform/modules/lambda_execution_role/variables.tf rename to deployment/terraform/modules/lambda_execution_role/variables.tf diff --git a/infrastructure/terraform/modules/lambda_service/main.tf b/deployment/terraform/modules/lambda_service/main.tf similarity index 100% rename from infrastructure/terraform/modules/lambda_service/main.tf rename to deployment/terraform/modules/lambda_service/main.tf diff --git a/infrastructure/terraform/modules/lambda_service/outputs.tf b/deployment/terraform/modules/lambda_service/outputs.tf similarity index 100% rename from infrastructure/terraform/modules/lambda_service/outputs.tf rename to deployment/terraform/modules/lambda_service/outputs.tf diff --git a/infrastructure/terraform/modules/lambda_service/variables.tf b/deployment/terraform/modules/lambda_service/variables.tf similarity index 100% rename from infrastructure/terraform/modules/lambda_service/variables.tf rename to deployment/terraform/modules/lambda_service/variables.tf diff --git a/infrastructure/terraform/modules/lambda_service_zip/main.tf b/deployment/terraform/modules/lambda_service_zip/main.tf similarity index 100% rename from infrastructure/terraform/modules/lambda_service_zip/main.tf rename to deployment/terraform/modules/lambda_service_zip/main.tf diff --git a/infrastructure/terraform/modules/lambda_service_zip/variables.tf b/deployment/terraform/modules/lambda_service_zip/variables.tf similarity index 100% rename from infrastructure/terraform/modules/lambda_service_zip/variables.tf rename to deployment/terraform/modules/lambda_service_zip/variables.tf diff --git a/infrastructure/terraform/modules/lambda_sqs_trigger/main.tf b/deployment/terraform/modules/lambda_sqs_trigger/main.tf similarity index 100% rename from infrastructure/terraform/modules/lambda_sqs_trigger/main.tf rename to deployment/terraform/modules/lambda_sqs_trigger/main.tf diff --git a/infrastructure/terraform/modules/lambda_sqs_trigger/variables.tf b/deployment/terraform/modules/lambda_sqs_trigger/variables.tf similarity index 100% rename from infrastructure/terraform/modules/lambda_sqs_trigger/variables.tf rename to deployment/terraform/modules/lambda_sqs_trigger/variables.tf diff --git a/infrastructure/terraform/modules/lambda_with_api_gateway/main.tf b/deployment/terraform/modules/lambda_with_api_gateway/main.tf similarity index 100% rename from infrastructure/terraform/modules/lambda_with_api_gateway/main.tf rename to deployment/terraform/modules/lambda_with_api_gateway/main.tf diff --git a/infrastructure/terraform/modules/lambda_with_api_gateway/outputs.tf b/deployment/terraform/modules/lambda_with_api_gateway/outputs.tf similarity index 100% rename from infrastructure/terraform/modules/lambda_with_api_gateway/outputs.tf rename to deployment/terraform/modules/lambda_with_api_gateway/outputs.tf diff --git a/infrastructure/terraform/modules/lambda_with_api_gateway/variables.tf b/deployment/terraform/modules/lambda_with_api_gateway/variables.tf similarity index 100% rename from infrastructure/terraform/modules/lambda_with_api_gateway/variables.tf rename to deployment/terraform/modules/lambda_with_api_gateway/variables.tf diff --git a/infrastructure/terraform/modules/lambda_with_sqs/main.tf b/deployment/terraform/modules/lambda_with_sqs/main.tf similarity index 100% rename from infrastructure/terraform/modules/lambda_with_sqs/main.tf rename to deployment/terraform/modules/lambda_with_sqs/main.tf diff --git a/infrastructure/terraform/modules/lambda_with_sqs/outputs.tf b/deployment/terraform/modules/lambda_with_sqs/outputs.tf similarity index 100% rename from infrastructure/terraform/modules/lambda_with_sqs/outputs.tf rename to deployment/terraform/modules/lambda_with_sqs/outputs.tf diff --git a/infrastructure/terraform/modules/lambda_with_sqs/variables.tf b/deployment/terraform/modules/lambda_with_sqs/variables.tf similarity index 100% rename from infrastructure/terraform/modules/lambda_with_sqs/variables.tf rename to deployment/terraform/modules/lambda_with_sqs/variables.tf diff --git a/infrastructure/terraform/modules/route53/main.tf b/deployment/terraform/modules/route53/main.tf similarity index 100% rename from infrastructure/terraform/modules/route53/main.tf rename to deployment/terraform/modules/route53/main.tf diff --git a/infrastructure/terraform/modules/route53/variables.tf b/deployment/terraform/modules/route53/variables.tf similarity index 100% rename from infrastructure/terraform/modules/route53/variables.tf rename to deployment/terraform/modules/route53/variables.tf diff --git a/infrastructure/terraform/modules/s3/main.tf b/deployment/terraform/modules/s3/main.tf similarity index 100% rename from infrastructure/terraform/modules/s3/main.tf rename to deployment/terraform/modules/s3/main.tf diff --git a/infrastructure/terraform/modules/s3/outputs.tf b/deployment/terraform/modules/s3/outputs.tf similarity index 100% rename from infrastructure/terraform/modules/s3/outputs.tf rename to deployment/terraform/modules/s3/outputs.tf diff --git a/infrastructure/terraform/modules/s3/variables.tf b/deployment/terraform/modules/s3/variables.tf similarity index 100% rename from infrastructure/terraform/modules/s3/variables.tf rename to deployment/terraform/modules/s3/variables.tf diff --git a/infrastructure/terraform/modules/s3_iam_policy/main.tf b/deployment/terraform/modules/s3_iam_policy/main.tf similarity index 100% rename from infrastructure/terraform/modules/s3_iam_policy/main.tf rename to deployment/terraform/modules/s3_iam_policy/main.tf diff --git a/infrastructure/terraform/modules/s3_iam_policy/outputs.tf b/deployment/terraform/modules/s3_iam_policy/outputs.tf similarity index 100% rename from infrastructure/terraform/modules/s3_iam_policy/outputs.tf rename to deployment/terraform/modules/s3_iam_policy/outputs.tf diff --git a/infrastructure/terraform/modules/s3_iam_policy/variables.tf b/deployment/terraform/modules/s3_iam_policy/variables.tf similarity index 100% rename from infrastructure/terraform/modules/s3_iam_policy/variables.tf rename to deployment/terraform/modules/s3_iam_policy/variables.tf diff --git a/infrastructure/terraform/modules/s3_presignable_bucket/main.tf b/deployment/terraform/modules/s3_presignable_bucket/main.tf similarity index 100% rename from infrastructure/terraform/modules/s3_presignable_bucket/main.tf rename to deployment/terraform/modules/s3_presignable_bucket/main.tf diff --git a/infrastructure/terraform/modules/s3_presignable_bucket/outputs.tf b/deployment/terraform/modules/s3_presignable_bucket/outputs.tf similarity index 100% rename from infrastructure/terraform/modules/s3_presignable_bucket/outputs.tf rename to deployment/terraform/modules/s3_presignable_bucket/outputs.tf diff --git a/infrastructure/terraform/modules/s3_presignable_bucket/variables.tf b/deployment/terraform/modules/s3_presignable_bucket/variables.tf similarity index 100% rename from infrastructure/terraform/modules/s3_presignable_bucket/variables.tf rename to deployment/terraform/modules/s3_presignable_bucket/variables.tf diff --git a/infrastructure/terraform/modules/ses/main.tf b/deployment/terraform/modules/ses/main.tf similarity index 100% rename from infrastructure/terraform/modules/ses/main.tf rename to deployment/terraform/modules/ses/main.tf diff --git a/infrastructure/terraform/modules/ses/outputs.tf b/deployment/terraform/modules/ses/outputs.tf similarity index 100% rename from infrastructure/terraform/modules/ses/outputs.tf rename to deployment/terraform/modules/ses/outputs.tf diff --git a/infrastructure/terraform/modules/ses/variables.tf b/deployment/terraform/modules/ses/variables.tf similarity index 100% rename from infrastructure/terraform/modules/ses/variables.tf rename to deployment/terraform/modules/ses/variables.tf diff --git a/infrastructure/terraform/modules/sqs_queue/main.tf b/deployment/terraform/modules/sqs_queue/main.tf similarity index 100% rename from infrastructure/terraform/modules/sqs_queue/main.tf rename to deployment/terraform/modules/sqs_queue/main.tf diff --git a/infrastructure/terraform/modules/sqs_queue/outputs.tf b/deployment/terraform/modules/sqs_queue/outputs.tf similarity index 100% rename from infrastructure/terraform/modules/sqs_queue/outputs.tf rename to deployment/terraform/modules/sqs_queue/outputs.tf diff --git a/infrastructure/terraform/modules/sqs_queue/variables.tf b/deployment/terraform/modules/sqs_queue/variables.tf similarity index 100% rename from infrastructure/terraform/modules/sqs_queue/variables.tf rename to deployment/terraform/modules/sqs_queue/variables.tf diff --git a/infrastructure/terraform/modules/tf_state_bucket/main.tf b/deployment/terraform/modules/tf_state_bucket/main.tf similarity index 100% rename from infrastructure/terraform/modules/tf_state_bucket/main.tf rename to deployment/terraform/modules/tf_state_bucket/main.tf diff --git a/infrastructure/terraform/modules/tf_state_bucket/outputs.tf b/deployment/terraform/modules/tf_state_bucket/outputs.tf similarity index 100% rename from infrastructure/terraform/modules/tf_state_bucket/outputs.tf rename to deployment/terraform/modules/tf_state_bucket/outputs.tf diff --git a/infrastructure/terraform/modules/tf_state_bucket/variables.tf b/deployment/terraform/modules/tf_state_bucket/variables.tf similarity index 100% rename from infrastructure/terraform/modules/tf_state_bucket/variables.tf rename to deployment/terraform/modules/tf_state_bucket/variables.tf diff --git a/infrastructure/terraform/shared/dev.tfvars b/deployment/terraform/shared/dev.tfvars similarity index 100% rename from infrastructure/terraform/shared/dev.tfvars rename to deployment/terraform/shared/dev.tfvars diff --git a/infrastructure/terraform/shared/main.tf b/deployment/terraform/shared/main.tf similarity index 100% rename from infrastructure/terraform/shared/main.tf rename to deployment/terraform/shared/main.tf diff --git a/infrastructure/terraform/shared/secrets.tf b/deployment/terraform/shared/secrets.tf similarity index 100% rename from infrastructure/terraform/shared/secrets.tf rename to deployment/terraform/shared/secrets.tf diff --git a/infrastructure/terraform/shared/variables.tf b/deployment/terraform/shared/variables.tf similarity index 100% rename from infrastructure/terraform/shared/variables.tf rename to deployment/terraform/shared/variables.tf diff --git a/domain/__init__.py b/domain/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/domain/tasks/__init__.py b/domain/tasks/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/domain/tasks/subtasks.py b/domain/tasks/subtasks.py new file mode 100644 index 00000000..bd49a6ec --- /dev/null +++ b/domain/tasks/subtasks.py @@ -0,0 +1,55 @@ +from dataclasses import dataclass +from datetime import datetime, timezone +from enum import Enum +from typing import Any, Optional +from uuid import UUID, uuid4 + + +class SubTaskStatus(str, Enum): + WAITING = "waiting" + IN_PROGRESS = "in progress" + COMPLETE = "complete" + FAILED = "failed" + + +@dataclass +class SubTask: + id: UUID + task_id: UUID + status: SubTaskStatus = SubTaskStatus.WAITING + inputs: Optional[dict[str, Any]] = None + outputs: Optional[dict[str, Any]] = None + cloud_logs_url: Optional[str] = None + job_started: Optional[datetime] = None + job_completed: Optional[datetime] = None + + @classmethod + def create( + cls, *, task_id: UUID, inputs: Optional[dict[str, Any]] = None + ) -> "SubTask": + return cls( + id=uuid4(), + task_id=task_id, + status=SubTaskStatus.WAITING, + inputs=inputs, + ) + + def start(self, cloud_logs_url: Optional[str] = None) -> None: + if self.status not in (SubTaskStatus.WAITING, SubTaskStatus.IN_PROGRESS): + raise ValueError(f"cannot start subtask in status {self.status}") + if self.job_started is None: + self.job_started = datetime.now(timezone.utc) + self.status = SubTaskStatus.IN_PROGRESS + if cloud_logs_url is not None: + self.cloud_logs_url = cloud_logs_url + + def complete(self, result: Any = None) -> None: + self.status = SubTaskStatus.COMPLETE + self.job_completed = datetime.now(timezone.utc) + if result is not None: + self.outputs = {"result": result} + + def fail(self, error: BaseException) -> None: + self.status = SubTaskStatus.FAILED + self.job_completed = datetime.now(timezone.utc) + self.outputs = {"error": str(error)} diff --git a/domain/tasks/tasks.py b/domain/tasks/tasks.py new file mode 100644 index 00000000..177258d6 --- /dev/null +++ b/domain/tasks/tasks.py @@ -0,0 +1,94 @@ +from dataclasses import dataclass +from datetime import datetime, timezone +from enum import Enum +from typing import Optional +from uuid import UUID, uuid4 + +from domain.tasks.subtasks import SubTaskStatus + + +class TaskStatus(str, Enum): + WAITING = "waiting" + IN_PROGRESS = "in progress" + COMPLETE = "complete" + FAILED = "failed" + + +class Source(str, Enum): + PORTFOLIO = "portfolio_id" + HUBSPOT_DEAL = "hubspot_deal_id" + + +@dataclass +class Task: + id: UUID + task_source: str + status: TaskStatus = TaskStatus.WAITING + service: Optional[str] = None + source: Optional[Source] = None + source_id: Optional[str] = None + job_started: Optional[datetime] = None + job_completed: Optional[datetime] = None + + @classmethod + def create( + cls, + *, + task_source: str, + service: Optional[str] = None, + source: Optional[Source] = None, + source_id: Optional[str] = None, + ) -> "Task": + if not task_source.strip(): + raise ValueError("task_source must be non-empty") + return cls( + id=uuid4(), + task_source=task_source, + service=service, + source=source, + source_id=source_id, + status=TaskStatus.WAITING, + job_started=datetime.now(timezone.utc), + ) + + def start(self) -> None: + if self.status not in (TaskStatus.WAITING, TaskStatus.IN_PROGRESS): + raise ValueError(f"cannot start task in status {self.status}") + if self.job_started is None: + self.job_started = datetime.now(timezone.utc) + self.status = TaskStatus.IN_PROGRESS + + def complete(self) -> None: + self.status = TaskStatus.COMPLETE + self.job_completed = datetime.now(timezone.utc) + + def fail(self) -> None: + self.status = TaskStatus.FAILED + self.job_completed = datetime.now(timezone.utc) + + def recalculate_from_subtasks(self, statuses: list[SubTaskStatus]) -> None: + """Recompute Task.status from its SubTasks' statuses. + + Rule (preserved from legacy _update_task_progress): + - any FAILED → FAILED + - all COMPLETE → COMPLETE + - any IN_PROGRESS → IN_PROGRESS + - otherwise → WAITING + + Empty list is a no-op (newly-created task with no subtasks). + """ + if not statuses: + return + now = datetime.now(timezone.utc) + if SubTaskStatus.FAILED in statuses: + self.status = TaskStatus.FAILED + self.job_completed = now + elif all(s is SubTaskStatus.COMPLETE for s in statuses): + self.status = TaskStatus.COMPLETE + self.job_completed = now + elif SubTaskStatus.IN_PROGRESS in statuses: + self.status = TaskStatus.IN_PROGRESS + self.job_completed = None + else: + self.status = TaskStatus.WAITING + self.job_completed = None diff --git a/infrastructure/__init__.py b/infrastructure/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/infrastructure/postgres/__init__.py b/infrastructure/postgres/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/infrastructure/postgres/config.py b/infrastructure/postgres/config.py new file mode 100644 index 00000000..c39c6f30 --- /dev/null +++ b/infrastructure/postgres/config.py @@ -0,0 +1,33 @@ +from dataclasses import dataclass +from typing import Mapping + + +@dataclass(frozen=True) +class PostgresConfig: + host: str + port: int + username: str + password: str + database: str + driver: str = "psycopg2" + pool_size: int = 3 + max_overflow: int = 5 + pool_pre_ping: bool = True + pool_recycle: int = 300 + + def url(self) -> str: + return ( + f"postgresql+{self.driver}://" + f"{self.username}:{self.password}@{self.host}:{self.port}/{self.database}" + ) + + @classmethod + def from_env(cls, env: Mapping[str, str]) -> "PostgresConfig": + return cls( + host=env["POSTGRES_HOST"], + port=int(env["POSTGRES_PORT"]), + username=env["POSTGRES_USERNAME"], + password=env["POSTGRES_PASSWORD"], + database=env["POSTGRES_DATABASE"], + driver=env.get("POSTGRES_DRIVER", "psycopg2"), + ) diff --git a/infrastructure/postgres/engine.py b/infrastructure/postgres/engine.py new file mode 100644 index 00000000..0de9efcb --- /dev/null +++ b/infrastructure/postgres/engine.py @@ -0,0 +1,18 @@ +from sqlalchemy.engine import Engine +from sqlmodel import Session, create_engine + +from infrastructure.postgres.config import PostgresConfig + + +def make_engine(config: PostgresConfig) -> Engine: + return create_engine( + config.url(), + pool_size=config.pool_size, + max_overflow=config.max_overflow, + pool_pre_ping=config.pool_pre_ping, + pool_recycle=config.pool_recycle, + ) + + +def make_session(engine: Engine) -> Session: + return Session(engine) diff --git a/infrastructure/postgres/subtask_table.py b/infrastructure/postgres/subtask_table.py new file mode 100644 index 00000000..dec34fbf --- /dev/null +++ b/infrastructure/postgres/subtask_table.py @@ -0,0 +1,21 @@ +from datetime import datetime, timezone +from typing import ClassVar, Optional +from uuid import UUID, uuid4 + +from sqlmodel import Field, SQLModel + + +class SubTaskRow(SQLModel, table=True): + __tablename__: ClassVar[str] = "sub_task" # pyright: ignore[reportIncompatibleVariableOverride] + + id: UUID = Field(default_factory=uuid4, primary_key=True, index=True) + task_id: UUID = Field(foreign_key="tasks.id") + job_started: Optional[datetime] = None + job_completed: Optional[datetime] = None + status: str = Field(default="waiting") + inputs: Optional[str] = None + outputs: Optional[str] = None + cloud_logs_url: Optional[str] = None + updated_at: datetime = Field( + default_factory=lambda: datetime.now(timezone.utc) + ) diff --git a/infrastructure/postgres/task_table.py b/infrastructure/postgres/task_table.py new file mode 100644 index 00000000..32e5450b --- /dev/null +++ b/infrastructure/postgres/task_table.py @@ -0,0 +1,36 @@ +from datetime import datetime, timezone +from typing import ClassVar, Optional +from uuid import UUID, uuid4 + +from sqlalchemy import Column +from sqlalchemy import Enum as SAEnum +from sqlmodel import Field, SQLModel + +from domain.tasks.tasks import Source + + +class TaskRow(SQLModel, table=True): + __tablename__: ClassVar[str] = "tasks" # pyright: ignore[reportIncompatibleVariableOverride] + + id: UUID = Field(default_factory=uuid4, primary_key=True, index=True) + task_source: str + job_started: Optional[datetime] = None + job_completed: Optional[datetime] = None + status: str = Field(default="waiting") + service: Optional[str] = None + updated_at: datetime = Field( + default_factory=lambda: datetime.now(timezone.utc) + ) + + source: Optional[Source] = Field( + default=None, + sa_column=Column( + SAEnum( + Source, + name="source", + values_callable=lambda cls: [m.value for m in cls], # pyright: ignore[reportUnknownLambdaType, reportUnknownMemberType, reportUnknownVariableType] + ), + nullable=True, + ), + ) + source_id: Optional[str] = None diff --git a/orchestration/__init__.py b/orchestration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/orchestration/task_orchestrator.py b/orchestration/task_orchestrator.py new file mode 100644 index 00000000..6c67d1ce --- /dev/null +++ b/orchestration/task_orchestrator.py @@ -0,0 +1,96 @@ +from typing import Any, Callable, Optional +from uuid import UUID + +from domain.tasks.subtasks import SubTask +from domain.tasks.tasks import Source, Task +from repositories.tasks.subtask_repository import SubTaskRepository +from repositories.tasks.task_repository import TaskRepository +from utilities.private import private + + +class TaskOrchestrator: + """Coordinates Task + SubTask lifecycle. + + Exposes primitives (start/complete/fail_subtask) for handlers that want + fine-grained control, and a high-level run_subtask wrapper that owns the + try/except so it can replace the body of the legacy subtask_handler + decorator in backend/utils/subtasks.py. + + Each primitive saves the SubTask, then recomputes the parent Task's + status from all its children. + """ + + def __init__( + self, + task_repo: TaskRepository, + subtask_repo: SubTaskRepository, + ) -> None: + self._tasks = task_repo + self._subtasks = subtask_repo + + def create_task_with_subtask( + self, + *, + task_source: str, + inputs: Optional[dict[str, Any]] = None, + service: Optional[str] = None, + source: Optional[Source] = None, + source_id: Optional[str] = None, + ) -> tuple[Task, SubTask]: + task = Task.create( + task_source=task_source, + service=service, + source=source, + source_id=source_id, + ) + self._tasks.create(task) + subtask = SubTask.create(task_id=task.id, inputs=inputs) + self._subtasks.create(subtask) + return task, subtask + + def start_subtask( + self, subtask_id: UUID, cloud_logs_url: Optional[str] = None + ) -> SubTask: + subtask = self._subtasks.get(subtask_id) + subtask.start(cloud_logs_url) + self._subtasks.save(subtask) + self._cascade(subtask.task_id) + return subtask + + def complete_subtask( + self, subtask_id: UUID, result: Any = None + ) -> SubTask: + subtask = self._subtasks.get(subtask_id) + subtask.complete(result) + self._subtasks.save(subtask) + self._cascade(subtask.task_id) + return subtask + + def fail_subtask(self, subtask_id: UUID, error: BaseException) -> SubTask: + subtask = self._subtasks.get(subtask_id) + subtask.fail(error) + self._subtasks.save(subtask) + self._cascade(subtask.task_id) + return subtask + + def run_subtask( + self, + subtask_id: UUID, + work: Callable[[], Any], + cloud_logs_url: Optional[str] = None, + ) -> Any: + self.start_subtask(subtask_id, cloud_logs_url) + try: + result = work() + except Exception as e: + self.fail_subtask(subtask_id, e) + raise + self.complete_subtask(subtask_id, result) + return result + + @private + def _cascade(self, task_id: UUID) -> None: + statuses = [s.status for s in self._subtasks.list_by_task(task_id)] + task = self._tasks.get(task_id) + task.recalculate_from_subtasks(statuses) + self._tasks.save(task) diff --git a/repositories/__init__.py b/repositories/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/repositories/tasks/__init__.py b/repositories/tasks/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/repositories/tasks/subtask_postgres_repository.py b/repositories/tasks/subtask_postgres_repository.py new file mode 100644 index 00000000..affc280e --- /dev/null +++ b/repositories/tasks/subtask_postgres_repository.py @@ -0,0 +1,89 @@ +import json +from datetime import datetime, timezone +from typing import Any, Optional +from uuid import UUID + +from sqlmodel import Session, select + +from domain.tasks.subtasks import SubTask, SubTaskStatus +from infrastructure.postgres.subtask_table import SubTaskRow +from repositories.tasks.subtask_repository import SubTaskRepository +from utilities.private import private + + +class SubTaskPostgresRepository(SubTaskRepository): + def __init__(self, session: Session) -> None: + self._session = session + + def create(self, subtask: SubTask) -> SubTask: + row = self._to_row(subtask) + self._session.add(row) + self._session.commit() + self._session.refresh(row) + return self._to_domain(row) + + def get(self, subtask_id: UUID) -> SubTask: + row = self._session.get(SubTaskRow, subtask_id) + if row is None: + raise ValueError(f"SubTask {subtask_id} not found") + return self._to_domain(row) + + def save(self, subtask: SubTask) -> None: + row = self._session.get(SubTaskRow, subtask.id) + if row is None: + raise ValueError(f"SubTask {subtask.id} not found") + row.status = subtask.status.value + row.job_started = subtask.job_started + row.job_completed = subtask.job_completed + row.inputs = ( + json.dumps(subtask.inputs) if subtask.inputs is not None else None + ) + row.outputs = ( + json.dumps(subtask.outputs) if subtask.outputs is not None else None + ) + row.cloud_logs_url = subtask.cloud_logs_url + row.updated_at = datetime.now(timezone.utc) + self._session.add(row) + self._session.commit() + + def list_by_task(self, task_id: UUID) -> list[SubTask]: + rows = self._session.exec( + select(SubTaskRow).where(SubTaskRow.task_id == task_id) + ).all() + return [self._to_domain(r) for r in rows] + + @private + def _to_row(self, subtask: SubTask) -> SubTaskRow: + return SubTaskRow( + id=subtask.id, + task_id=subtask.task_id, + status=subtask.status.value, + inputs=( + json.dumps(subtask.inputs) if subtask.inputs is not None else None + ), + outputs=( + json.dumps(subtask.outputs) + if subtask.outputs is not None + else None + ), + cloud_logs_url=subtask.cloud_logs_url, + job_started=subtask.job_started, + job_completed=subtask.job_completed, + ) + + @private + def _to_domain(self, row: SubTaskRow) -> SubTask: + return SubTask( + id=row.id, + task_id=row.task_id, + status=SubTaskStatus(row.status.lower()), + inputs=_loads_or_none(row.inputs), + outputs=_loads_or_none(row.outputs), + cloud_logs_url=row.cloud_logs_url, + job_started=row.job_started, + job_completed=row.job_completed, + ) + + +def _loads_or_none(s: Optional[str]) -> Optional[dict[str, Any]]: + return json.loads(s) if s else None diff --git a/repositories/tasks/subtask_repository.py b/repositories/tasks/subtask_repository.py new file mode 100644 index 00000000..adb36f99 --- /dev/null +++ b/repositories/tasks/subtask_repository.py @@ -0,0 +1,18 @@ +from abc import ABC, abstractmethod +from uuid import UUID + +from domain.tasks.subtasks import SubTask + + +class SubTaskRepository(ABC): + @abstractmethod + def create(self, subtask: SubTask) -> SubTask: ... + + @abstractmethod + def get(self, subtask_id: UUID) -> SubTask: ... + + @abstractmethod + def save(self, subtask: SubTask) -> None: ... + + @abstractmethod + def list_by_task(self, task_id: UUID) -> list[SubTask]: ... diff --git a/repositories/tasks/task_postgres_repository.py b/repositories/tasks/task_postgres_repository.py new file mode 100644 index 00000000..d23fe91c --- /dev/null +++ b/repositories/tasks/task_postgres_repository.py @@ -0,0 +1,77 @@ +""" +Postgres implementation of TaskRepository. + +NOTE: this repository owns only the `tasks` table. Unlike the legacy +backend.app.db.functions.tasks.Tasks.TasksInterface.create_task, it does NOT +auto-create a child SubTask. Do not rewire existing Lambda callers to this +repo until the SubTask aggregate + TaskOrchestrator slice lands — they would +silently lose their initial SubTask row. +""" + +from datetime import datetime, timezone +from uuid import UUID + +from sqlmodel import Session + +from domain.tasks.tasks import Task, TaskStatus +from infrastructure.postgres.task_table import TaskRow +from repositories.tasks.task_repository import TaskRepository +from utilities.private import private + + +class TaskPostgresRepository(TaskRepository): + def __init__(self, session: Session) -> None: + self._session = session + + def create(self, task: Task) -> Task: + row = self._to_row(task) + self._session.add(row) + self._session.commit() + self._session.refresh(row) + return self._to_domain(row) + + def get(self, task_id: UUID) -> Task: + row = self._session.get(TaskRow, task_id) + if row is None: + raise ValueError(f"Task {task_id} not found") + return self._to_domain(row) + + def save(self, task: Task) -> None: + row = self._session.get(TaskRow, task.id) + if row is None: + raise ValueError(f"Task {task.id} not found") + row.status = task.status.value + row.job_started = task.job_started + row.job_completed = task.job_completed + row.service = task.service + row.source = task.source + row.source_id = task.source_id + row.updated_at = datetime.now(timezone.utc) + self._session.add(row) + self._session.commit() + + @private + def _to_row(self, task: Task) -> TaskRow: + return TaskRow( + id=task.id, + task_source=task.task_source, + status=task.status.value, + service=task.service, + source=task.source, + source_id=task.source_id, + job_started=task.job_started, + job_completed=task.job_completed, + ) + + @private + def _to_domain(self, row: TaskRow) -> Task: + return Task( + id=row.id, + task_source=row.task_source, + status=TaskStatus(row.status.lower()), + service=row.service, + source=row.source, + source_id=row.source_id, + job_started=row.job_started, + job_completed=row.job_completed, + ) diff --git a/repositories/tasks/task_repository.py b/repositories/tasks/task_repository.py new file mode 100644 index 00000000..8bdce0cc --- /dev/null +++ b/repositories/tasks/task_repository.py @@ -0,0 +1,15 @@ +from abc import ABC, abstractmethod +from uuid import UUID + +from domain.tasks.tasks import Task + + +class TaskRepository(ABC): + @abstractmethod + def create(self, task: Task) -> Task: ... + + @abstractmethod + def get(self, task_id: UUID) -> Task: ... + + @abstractmethod + def save(self, task: Task) -> None: ... diff --git a/run_backlog.sh b/run_backlog.sh deleted file mode 100644 index 398e921c..00000000 --- a/run_backlog.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -backlog browser --port 6421 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/domain/__init__.py b/tests/domain/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/domain/tasks/__init__.py b/tests/domain/tasks/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/domain/tasks/test_subtasks.py b/tests/domain/tasks/test_subtasks.py new file mode 100644 index 00000000..2721d38f --- /dev/null +++ b/tests/domain/tasks/test_subtasks.py @@ -0,0 +1,75 @@ +from uuid import uuid4 + +import pytest + +from domain.tasks.subtasks import SubTask, SubTaskStatus + + +def test_create_subtask_starts_waiting() -> None: + task_id = uuid4() + + st = SubTask.create(task_id=task_id, inputs={"foo": "bar"}) + + assert st.task_id == task_id + assert st.status is SubTaskStatus.WAITING + assert st.inputs == {"foo": "bar"} + assert st.outputs is None + assert st.job_started is None + assert st.job_completed is None + + +def test_start_transitions_to_in_progress_and_sets_cloud_logs_url() -> None: + st = SubTask.create(task_id=uuid4()) + + st.start(cloud_logs_url="https://example/log") + + assert st.status is SubTaskStatus.IN_PROGRESS + assert st.cloud_logs_url == "https://example/log" + assert st.job_started is not None + + +def test_start_is_idempotent_from_in_progress() -> None: + st = SubTask.create(task_id=uuid4()) + st.start() + first_start = st.job_started + + st.start(cloud_logs_url="https://other") + + assert st.status is SubTaskStatus.IN_PROGRESS + assert st.job_started == first_start # not overwritten + assert st.cloud_logs_url == "https://other" + + +def test_start_rejects_from_terminal_status() -> None: + st = SubTask.create(task_id=uuid4()) + st.complete() + with pytest.raises(ValueError): + st.start() + + +def test_complete_marks_outputs_and_job_completed() -> None: + st = SubTask.create(task_id=uuid4()) + st.start() + + st.complete({"uprn": "123"}) + + assert st.status is SubTaskStatus.COMPLETE + assert st.outputs == {"result": {"uprn": "123"}} + assert st.job_completed is not None + + +def test_complete_without_result_leaves_outputs_unset() -> None: + st = SubTask.create(task_id=uuid4()) + st.complete() + assert st.outputs is None + + +def test_fail_records_error_in_outputs() -> None: + st = SubTask.create(task_id=uuid4()) + err = RuntimeError("boom") + + st.fail(err) + + assert st.status is SubTaskStatus.FAILED + assert st.outputs == {"error": "boom"} + assert st.job_completed is not None diff --git a/tests/domain/tasks/test_tasks.py b/tests/domain/tasks/test_tasks.py new file mode 100644 index 00000000..f30c0aa1 --- /dev/null +++ b/tests/domain/tasks/test_tasks.py @@ -0,0 +1,104 @@ +import pytest + +from domain.tasks.subtasks import SubTaskStatus +from domain.tasks.tasks import Source, Task, TaskStatus + + +def test_create_task_starts_waiting() -> None: + # Arrange / Act + t = Task.create( + task_source="manual:test", source=Source.PORTFOLIO, source_id="abc-123" + ) + + # Assert + assert t.status is TaskStatus.WAITING + assert t.source is Source.PORTFOLIO + assert t.source_id == "abc-123" + assert t.job_started is not None + assert t.job_completed is None + + +def test_create_task_rejects_blank_task_source() -> None: + with pytest.raises(ValueError, match="task_source"): + Task.create(task_source=" ") + + +def test_start_transitions_to_in_progress() -> None: + t = Task.create(task_source="manual:test") + t.start() + assert t.status is TaskStatus.IN_PROGRESS + + +def test_complete_marks_job_completed() -> None: + t = Task.create(task_source="manual:test") + t.start() + t.complete() + assert t.status is TaskStatus.COMPLETE + assert t.job_completed is not None + + +def test_fail_marks_job_completed() -> None: + t = Task.create(task_source="manual:test") + t.fail() + assert t.status is TaskStatus.FAILED + assert t.job_completed is not None + + +def test_start_rejects_from_terminal_status() -> None: + t = Task.create(task_source="manual:test") + t.complete() + with pytest.raises(ValueError): + t.start() + + +def test_recalculate_with_empty_statuses_is_noop() -> None: + t = Task.create(task_source="manual:test") + original_status = t.status + original_completed = t.job_completed + + t.recalculate_from_subtasks([]) + + assert t.status is original_status + assert t.job_completed is original_completed + + +def test_recalculate_all_waiting_keeps_waiting() -> None: + t = Task.create(task_source="manual:test") + t.start() # task moved to IN_PROGRESS earlier + t.complete() # then COMPLETE, with job_completed set + + t.recalculate_from_subtasks([SubTaskStatus.WAITING, SubTaskStatus.WAITING]) + + assert t.status is TaskStatus.WAITING + assert t.job_completed is None + + +def test_recalculate_any_in_progress_marks_in_progress() -> None: + t = Task.create(task_source="manual:test") + + t.recalculate_from_subtasks( + [SubTaskStatus.WAITING, SubTaskStatus.IN_PROGRESS, SubTaskStatus.COMPLETE] + ) + + assert t.status is TaskStatus.IN_PROGRESS + assert t.job_completed is None + + +def test_recalculate_all_complete_marks_complete() -> None: + t = Task.create(task_source="manual:test") + + t.recalculate_from_subtasks([SubTaskStatus.COMPLETE, SubTaskStatus.COMPLETE]) + + assert t.status is TaskStatus.COMPLETE + assert t.job_completed is not None + + +def test_recalculate_any_failed_marks_failed_even_with_others() -> None: + t = Task.create(task_source="manual:test") + + t.recalculate_from_subtasks( + [SubTaskStatus.IN_PROGRESS, SubTaskStatus.COMPLETE, SubTaskStatus.FAILED] + ) + + assert t.status is TaskStatus.FAILED + assert t.job_completed is not None diff --git a/tests/orchestration/__init__.py b/tests/orchestration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/orchestration/test_task_orchestrator.py b/tests/orchestration/test_task_orchestrator.py new file mode 100644 index 00000000..1a48127f --- /dev/null +++ b/tests/orchestration/test_task_orchestrator.py @@ -0,0 +1,151 @@ +from collections.abc import Iterator +from dataclasses import dataclass + +import pytest +from sqlmodel import Session, SQLModel, create_engine + +from domain.tasks.subtasks import SubTask, SubTaskStatus +from domain.tasks.tasks import Source, TaskStatus +from orchestration.task_orchestrator import TaskOrchestrator +from repositories.tasks.subtask_postgres_repository import SubTaskPostgresRepository +from repositories.tasks.task_postgres_repository import TaskPostgresRepository + + +@dataclass +class Harness: + orchestrator: TaskOrchestrator + tasks: TaskPostgresRepository + subtasks: SubTaskPostgresRepository + + +@pytest.fixture +def harness() -> Iterator[Harness]: + engine = create_engine("sqlite://") + SQLModel.metadata.create_all(engine) + with Session(engine) as session: + tasks = TaskPostgresRepository(session=session) + subtasks = SubTaskPostgresRepository(session=session) + yield Harness( + orchestrator=TaskOrchestrator(task_repo=tasks, subtask_repo=subtasks), + tasks=tasks, + subtasks=subtasks, + ) + + +def test_create_task_with_subtask_creates_both_in_waiting( + harness: Harness, +) -> None: + task, subtask = harness.orchestrator.create_task_with_subtask( + task_source="manual:test", + inputs={"foo": "bar"}, + source=Source.PORTFOLIO, + source_id="abc", + ) + + assert task.status is TaskStatus.WAITING + assert subtask.status is SubTaskStatus.WAITING + assert subtask.task_id == task.id + assert subtask.inputs == {"foo": "bar"} + + +def test_start_subtask_cascades_to_in_progress(harness: Harness) -> None: + task, subtask = harness.orchestrator.create_task_with_subtask( + task_source="manual:test" + ) + + started = harness.orchestrator.start_subtask( + subtask.id, cloud_logs_url="https://example/log" + ) + + assert started.status is SubTaskStatus.IN_PROGRESS + assert started.cloud_logs_url == "https://example/log" + assert harness.tasks.get(task.id).status is TaskStatus.IN_PROGRESS + + +def test_complete_subtask_cascades_to_complete(harness: Harness) -> None: + task, subtask = harness.orchestrator.create_task_with_subtask( + task_source="manual:test" + ) + harness.orchestrator.start_subtask(subtask.id) + + harness.orchestrator.complete_subtask(subtask.id, {"value": 42}) + + done_subtask = harness.subtasks.get(subtask.id) + done_task = harness.tasks.get(task.id) + assert done_subtask.outputs == {"result": {"value": 42}} + assert done_task.status is TaskStatus.COMPLETE + assert done_task.job_completed is not None + + +def test_fail_subtask_cascades_to_failed(harness: Harness) -> None: + task, subtask = harness.orchestrator.create_task_with_subtask( + task_source="manual:test" + ) + + harness.orchestrator.fail_subtask(subtask.id, RuntimeError("boom")) + + failed_subtask = harness.subtasks.get(subtask.id) + failed_task = harness.tasks.get(task.id) + assert failed_subtask.outputs == {"error": "boom"} + assert failed_task.status is TaskStatus.FAILED + + +def test_failed_subtask_locks_task_failed_even_with_others_complete( + harness: Harness, +) -> None: + task, first = harness.orchestrator.create_task_with_subtask( + task_source="manual:test" + ) + second = SubTask.create(task_id=task.id) + harness.subtasks.create(second) + + harness.orchestrator.complete_subtask(first.id) + harness.orchestrator.fail_subtask(second.id, RuntimeError("nope")) + + assert harness.tasks.get(task.id).status is TaskStatus.FAILED + + +def test_mixed_complete_and_in_progress_keeps_task_in_progress( + harness: Harness, +) -> None: + task, first = harness.orchestrator.create_task_with_subtask( + task_source="manual:test" + ) + second = SubTask.create(task_id=task.id) + harness.subtasks.create(second) + + harness.orchestrator.complete_subtask(first.id) + harness.orchestrator.start_subtask(second.id) + + assert harness.tasks.get(task.id).status is TaskStatus.IN_PROGRESS + + +def test_run_subtask_happy_path_returns_result_and_cascades_complete( + harness: Harness, +) -> None: + task, subtask = harness.orchestrator.create_task_with_subtask( + task_source="manual:test" + ) + + result = harness.orchestrator.run_subtask(subtask.id, work=lambda: {"answer": 42}) + + assert result == {"answer": 42} + assert harness.subtasks.get(subtask.id).status is SubTaskStatus.COMPLETE + assert harness.tasks.get(task.id).status is TaskStatus.COMPLETE + + +def test_run_subtask_failing_work_marks_failed_and_reraises( + harness: Harness, +) -> None: + task, subtask = harness.orchestrator.create_task_with_subtask( + task_source="manual:test" + ) + + def boom() -> None: + raise RuntimeError("boom") + + with pytest.raises(RuntimeError, match="boom"): + harness.orchestrator.run_subtask(subtask.id, work=boom) + + assert harness.subtasks.get(subtask.id).status is SubTaskStatus.FAILED + assert harness.tasks.get(task.id).status is TaskStatus.FAILED diff --git a/tests/repositories/__init__.py b/tests/repositories/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/repositories/tasks/__init__.py b/tests/repositories/tasks/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/repositories/tasks/postgres/__init__.py b/tests/repositories/tasks/postgres/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/repositories/tasks/postgres/test_subtask_postgres_repository.py b/tests/repositories/tasks/postgres/test_subtask_postgres_repository.py new file mode 100644 index 00000000..ac39e089 --- /dev/null +++ b/tests/repositories/tasks/postgres/test_subtask_postgres_repository.py @@ -0,0 +1,81 @@ +from collections.abc import Iterator +from uuid import uuid4 + +import pytest +from sqlmodel import Session, SQLModel, create_engine + +# Importing the SQLModel row modules registers their tables in +# SQLModel.metadata so create_all builds both. Imports look unused; they aren't. +import infrastructure.postgres.subtask_table # noqa: F401 # pyright: ignore[reportUnusedImport] +import infrastructure.postgres.task_table # noqa: F401 # pyright: ignore[reportUnusedImport] +from domain.tasks.subtasks import SubTask, SubTaskStatus +from repositories.tasks.subtask_postgres_repository import SubTaskPostgresRepository + + +@pytest.fixture +def session() -> Iterator[Session]: + engine = create_engine("sqlite://") + SQLModel.metadata.create_all(engine) + with Session(engine) as s: + yield s + + +def test_create_and_get_round_trip_preserves_inputs(session: Session) -> None: + repo = SubTaskPostgresRepository(session=session) + task_id = uuid4() + st = SubTask.create(task_id=task_id, inputs={"address": "68 Glendon Way"}) + + repo.create(st) + fetched = repo.get(st.id) + + assert fetched.id == st.id + assert fetched.task_id == task_id + assert fetched.status is SubTaskStatus.WAITING + assert fetched.inputs == {"address": "68 Glendon Way"} + assert fetched.outputs is None + + +def test_save_persists_status_and_outputs(session: Session) -> None: + repo = SubTaskPostgresRepository(session=session) + st = SubTask.create(task_id=uuid4()) + repo.create(st) + + st.start(cloud_logs_url="https://example/log") + repo.save(st) + assert repo.get(st.id).status is SubTaskStatus.IN_PROGRESS + + st.complete({"uprn": "123"}) + repo.save(st) + done = repo.get(st.id) + assert done.status is SubTaskStatus.COMPLETE + assert done.outputs == {"result": {"uprn": "123"}} + assert done.cloud_logs_url == "https://example/log" + assert done.job_completed is not None + + +def test_list_by_task_filters_by_task_id(session: Session) -> None: + repo = SubTaskPostgresRepository(session=session) + task_a = uuid4() + task_b = uuid4() + repo.create(SubTask.create(task_id=task_a)) + repo.create(SubTask.create(task_id=task_a)) + repo.create(SubTask.create(task_id=task_b)) + + a_results = repo.list_by_task(task_a) + b_results = repo.list_by_task(task_b) + + assert len(a_results) == 2 + assert len(b_results) == 1 + assert all(s.task_id == task_a for s in a_results) + assert all(s.task_id == task_b for s in b_results) + + +def test_list_by_task_returns_empty_for_unknown_task(session: Session) -> None: + repo = SubTaskPostgresRepository(session=session) + assert repo.list_by_task(uuid4()) == [] + + +def test_get_missing_raises(session: Session) -> None: + repo = SubTaskPostgresRepository(session=session) + with pytest.raises(ValueError, match="not found"): + repo.get(uuid4()) diff --git a/tests/repositories/tasks/postgres/test_task_postgres_repository.py b/tests/repositories/tasks/postgres/test_task_postgres_repository.py new file mode 100644 index 00000000..3e1aa226 --- /dev/null +++ b/tests/repositories/tasks/postgres/test_task_postgres_repository.py @@ -0,0 +1,68 @@ +from collections.abc import Iterator +from uuid import uuid4 + +import pytest +from sqlmodel import Session, SQLModel, create_engine + +from domain.tasks.tasks import Source, Task, TaskStatus +from infrastructure.postgres.task_table import TaskRow +from repositories.tasks.task_postgres_repository import TaskPostgresRepository + + +@pytest.fixture +def session() -> Iterator[Session]: + engine = create_engine("sqlite://") + SQLModel.metadata.create_all(engine) + with Session(engine) as s: + yield s + + +def test_create_and_get_round_trip(session: Session) -> None: + # Arrange + repo = TaskPostgresRepository(session=session) + t = Task.create( + task_source="manual:test", source=Source.PORTFOLIO, source_id="abc-123" + ) + + # Act + repo.create(t) + fetched = repo.get(t.id) + + # Assert + assert fetched.id == t.id + assert fetched.status is TaskStatus.WAITING + assert fetched.source is Source.PORTFOLIO + assert fetched.source_id == "abc-123" + + +def test_save_persists_status_transition(session: Session) -> None: + repo = TaskPostgresRepository(session=session) + t = Task.create(task_source="manual:test") + repo.create(t) + + t.start() + repo.save(t) + assert repo.get(t.id).status is TaskStatus.IN_PROGRESS + + t.complete() + repo.save(t) + done = repo.get(t.id) + assert done.status is TaskStatus.COMPLETE + assert done.job_completed is not None + + +def test_get_missing_raises(session: Session) -> None: + repo = TaskPostgresRepository(session=session) + with pytest.raises(ValueError, match="not found"): + repo.get(uuid4()) + + +def test_get_normalises_legacy_capitalised_status(session: Session) -> None: + # Existing rows written by backend code use "In Progress" (capitalised). + repo = TaskPostgresRepository(session=session) + row = TaskRow(task_source="manual:test", status="In Progress") + session.add(row) + session.commit() + + fetched = repo.get(row.id) + assert fetched.status is TaskStatus.IN_PROGRESS diff --git a/utilities/__init__.py b/utilities/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/utilities/aws_lambda/__init__.py b/utilities/aws_lambda/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/utilities/aws_lambda/default_orchestrator.py b/utilities/aws_lambda/default_orchestrator.py new file mode 100644 index 00000000..f78886b9 --- /dev/null +++ b/utilities/aws_lambda/default_orchestrator.py @@ -0,0 +1,26 @@ +import os +from collections.abc import Generator +from contextlib import contextmanager + +from sqlmodel import Session + +from infrastructure.postgres.config import PostgresConfig +from infrastructure.postgres.engine import make_engine +from orchestration.task_orchestrator import TaskOrchestrator +from repositories.tasks.subtask_postgres_repository import SubTaskPostgresRepository +from repositories.tasks.task_postgres_repository import TaskPostgresRepository + + +@contextmanager +def default_orchestrator() -> Generator[TaskOrchestrator, None, None]: + """Yield a TaskOrchestrator wired to a fresh Postgres session. + + Connection params come from os.environ via PostgresConfig.from_env. Each + handler invocation gets its own session, cleaned up on context exit. + """ + engine = make_engine(PostgresConfig.from_env(dict(os.environ))) + with Session(engine) as session: + yield TaskOrchestrator( + task_repo=TaskPostgresRepository(session=session), + subtask_repo=SubTaskPostgresRepository(session=session), + ) diff --git a/utilities/aws_lambda/subtask_handler.py b/utilities/aws_lambda/subtask_handler.py new file mode 100644 index 00000000..64c1daa6 --- /dev/null +++ b/utilities/aws_lambda/subtask_handler.py @@ -0,0 +1,67 @@ +"""@subtask_handler decorator for Lambdas that operate on existing SubTasks. + +Translates an AWS Lambda invocation (SQS-shaped or direct) into +TaskOrchestrator.run_subtask(...) calls. +""" + +import json +from contextlib import AbstractContextManager +from functools import wraps +from typing import Any, Callable, Optional, cast + +from utilities.aws_lambda.default_orchestrator import default_orchestrator +from utilities.aws_lambda.subtask_trigger_body import SubtaskTriggerBody +from orchestration.task_orchestrator import TaskOrchestrator + +OrchestratorCM = Callable[[], AbstractContextManager[TaskOrchestrator]] + + +def subtask_handler( + *, + orchestrator_cm: Optional[OrchestratorCM] = None, +) -> Callable[[Callable[..., Any]], Callable[..., Any]]: + """Run the wrapped function as the body of an existing SubTask. + + For each record, validates the body via SubtaskTriggerBody (must contain + task_id and sub_task_id), then runs the function inside + orchestrator.run_subtask(...). The orchestrator owns the start/complete/ + fail lifecycle and cascades status into the parent Task. On failure the + underlying exception propagates after the SubTask is marked FAILED. + """ + factory = orchestrator_cm or default_orchestrator + + def decorator(func: Callable[..., Any]) -> Callable[..., Any]: + @wraps(func) + def wrapper(event: dict[str, Any], context: Any) -> None: + with factory() as orchestrator: + for record in _records(event): + body = _parse_body(record) + trigger = SubtaskTriggerBody.model_validate(body) + orchestrator.run_subtask( + trigger.sub_task_id, + work=lambda body=body: func(body, context), + ) + + return wrapper + + return decorator + + +def _parse_body(record: dict[str, Any]) -> dict[str, Any]: + raw = record.get("body", record) + if isinstance(raw, str): + try: + parsed = json.loads(raw) + except json.JSONDecodeError: + return {} + return cast(dict[str, Any], parsed) if isinstance(parsed, dict) else {} + if isinstance(raw, dict): + return cast(dict[str, Any], raw) + return {} + + +def _records(event: dict[str, Any]) -> list[dict[str, Any]]: + raw_records = event.get("Records") + if isinstance(raw_records, list): + return [r for r in cast(list[Any], raw_records) if isinstance(r, dict)] + return [event] diff --git a/utilities/aws_lambda/subtask_trigger_body.py b/utilities/aws_lambda/subtask_trigger_body.py new file mode 100644 index 00000000..a6b539e5 --- /dev/null +++ b/utilities/aws_lambda/subtask_trigger_body.py @@ -0,0 +1,17 @@ +from uuid import UUID + +from pydantic import BaseModel, ConfigDict + + +class SubtaskTriggerBody(BaseModel): + """The minimum the subtask_handler needs to dispatch lifecycle calls. + + `extra="allow"` so the rest of the work payload passes through to the + decorated function untouched — handlers do their own model_validate on + the full body for fields specific to their use case. + """ + + model_config = ConfigDict(extra="allow") + + task_id: UUID + sub_task_id: UUID diff --git a/utilities/aws_lambda/task_handler.py b/utilities/aws_lambda/task_handler.py new file mode 100644 index 00000000..82c7198e --- /dev/null +++ b/utilities/aws_lambda/task_handler.py @@ -0,0 +1,98 @@ +"""@task_handler decorator for Lambdas that own the entire pipeline. + +Translates an AWS Lambda invocation (SQS-shaped or direct) into +TaskOrchestrator.create_task_with_subtask(...) + run_subtask(...). +""" + +import json +from contextlib import AbstractContextManager +from functools import wraps +from typing import Any, Callable, Optional, cast + +from utilities.aws_lambda.default_orchestrator import default_orchestrator +from domain.tasks.tasks import Source +from orchestration.task_orchestrator import TaskOrchestrator + +OrchestratorCM = Callable[[], AbstractContextManager[TaskOrchestrator]] + + +def task_handler( + *, + task_source: str, + source: Source, + orchestrator_cm: Optional[OrchestratorCM] = None, +) -> Callable[[Callable[..., Any]], Callable[..., Any]]: + """Run the wrapped function as the body of a freshly-created Task + SubTask. + + For each record, creates a new Task + initial SubTask, then runs the + wrapped function inside orchestrator.run_subtask(...). `source_id` is + read from body[source.value] (silent None if absent — preserved from + legacy ADR-0001). + + Records-style events use SQS partial-batch-failure semantics: individual + failures are reported via {"batchItemFailures": [...]} rather than + propagating. Direct invocations re-raise. + """ + factory = orchestrator_cm or default_orchestrator + + def decorator(func: Callable[..., Any]) -> Callable[..., Any]: + @wraps(func) + def wrapper(event: dict[str, Any], context: Any) -> Any: + with factory() as orchestrator: + results: list[Any] = [] + failures: list[dict[str, Any]] = [] + + for record in _records(event): + body = _parse_body(record) + raw_source_id = body.get(source.value) + source_id = ( + str(raw_source_id) if raw_source_id is not None else None + ) + + _, subtask = orchestrator.create_task_with_subtask( + task_source=task_source, + inputs=body, + source=source, + source_id=source_id, + ) + + try: + result = orchestrator.run_subtask( + subtask.id, + work=lambda body=body: func(body, context), + ) + results.append(result) + except Exception: + if "Records" in event: + message_id = record.get("messageId", "") + failures.append({"itemIdentifier": message_id}) + else: + raise + + if "Records" in event: + return {"batchItemFailures": failures} + return results + + return wrapper + + return decorator + + +def _parse_body(record: dict[str, Any]) -> dict[str, Any]: + raw = record.get("body", record) + if isinstance(raw, str): + try: + parsed = json.loads(raw) + except json.JSONDecodeError: + return {} + return cast(dict[str, Any], parsed) if isinstance(parsed, dict) else {} + if isinstance(raw, dict): + return cast(dict[str, Any], raw) + return {} + + +def _records(event: dict[str, Any]) -> list[dict[str, Any]]: + raw_records = event.get("Records") + if isinstance(raw_records, list): + return [r for r in cast(list[Any], raw_records) if isinstance(r, dict)] + return [event] diff --git a/utilities/private.py b/utilities/private.py new file mode 100644 index 00000000..77a70578 --- /dev/null +++ b/utilities/private.py @@ -0,0 +1,33 @@ +import inspect +from typing import Any, Callable + + +class private: + """Decorator that raises if a _-prefixed method is called from outside its class.""" + + func: Callable[..., Any] + name: str + owner: type + + def __init__(self, func: Callable[..., Any]) -> None: + self.func = func + self.name = getattr(func, "__name__", "") + + def __set_name__(self, owner: type, name: str) -> None: + self.owner = owner + + def __get__(self, instance: Any, owner: type) -> Callable[..., Any]: + # Walk up one frame to see who's calling + frame = inspect.currentframe() + if frame is None or frame.f_back is None: + raise RuntimeError("cannot inspect caller frame") + caller_frame = frame.f_back + caller_self = caller_frame.f_locals.get("self") + + if not isinstance(caller_self, self.owner): + raise RuntimeError( + f"{self.owner.__name__}.{self.name} is private; " + f"called from {caller_frame.f_code.co_name}" + ) + + return getattr(self.func, "__get__")(instance, owner) From 6198d7a46db83ecf2b74e2b260fd0b0923010b39 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 19 May 2026 16:45:47 +0000 Subject: [PATCH 03/22] postcode_splitter: pure domain (UserAddress, sanitise_postcode, postcode_batching) Slice 1/6 of the postcode_splitter refactor (Hestia-Homes/Model#1100). Introduces the pure-domain foundation under domain/, with no AWS, Postgres, or pandas. UserAddress is a frozen dataclass that sanitises its postcode in __post_init__ via the canonical sanitise_postcode helper, and iter_postcode_grouped_batches preserves the legacy splitter's batching invariants (group-by-postcode in insertion order, never split a group, oversize single-postcode groups dispatched whole, final flush). Updates UBIQUITOUS_LANGUAGE.md so the User Address term covers both the dataclass sense (preferred in domain code) and the raw upstream-string sense. Co-Authored-By: Claude Opus 4.7 (1M context) --- UBIQUITOUS_LANGUAGE.md | 4 +- domain/addresses/__init__.py | 0 domain/addresses/postcode_batching.py | 87 +++++++++++++++++ domain/addresses/user_address.py | 36 +++++++ domain/postcodes/__init__.py | 0 domain/postcodes/sanitise.py | 23 +++++ tests/domain/addresses/__init__.py | 0 .../addresses/test_postcode_batching.py | 93 +++++++++++++++++++ tests/domain/addresses/test_user_address.py | 45 +++++++++ tests/domain/postcodes/__init__.py | 0 tests/domain/postcodes/test_sanitise.py | 28 ++++++ 11 files changed, 314 insertions(+), 2 deletions(-) create mode 100644 domain/addresses/__init__.py create mode 100644 domain/addresses/postcode_batching.py create mode 100644 domain/addresses/user_address.py create mode 100644 domain/postcodes/__init__.py create mode 100644 domain/postcodes/sanitise.py create mode 100644 tests/domain/addresses/__init__.py create mode 100644 tests/domain/addresses/test_postcode_batching.py create mode 100644 tests/domain/addresses/test_user_address.py create mode 100644 tests/domain/postcodes/__init__.py create mode 100644 tests/domain/postcodes/test_sanitise.py diff --git a/UBIQUITOUS_LANGUAGE.md b/UBIQUITOUS_LANGUAGE.md index 1765cbc8..c3074c02 100644 --- a/UBIQUITOUS_LANGUAGE.md +++ b/UBIQUITOUS_LANGUAGE.md @@ -23,7 +23,7 @@ Invoke `/ubiquitous-language` in any session to extract new terms from the conve |------|------------|------------------| | **UPRN** | Unique Property Reference Number — the government-issued permanent identifier for a physical address in the UK. | "property ID", "address ID", "code" | | **Postcode** | A UK postal code used to group nearby addresses; the primary search key for finding EPC records. | "zip code", "postal code" | -| **User Address** | A free-text address string provided by a user or imported from a customer dataset, before any normalisation or matching. | "user input", "raw address", "user_inputed_address" | +| **User Address** | A structured dataclass (`domain.addresses.user_address.UserAddress`) capturing a customer-supplied address: a free-text `user_address` line, a canonical `postcode` (sanitised on construction), and an optional `internal_reference`. The bare string sense -- the raw free-text address line as it arrives from upstream ingestion, before being wrapped -- remains valid when discussing CSV columns, API payloads, or other upstream contexts; in domain code, prefer the dataclass. | "user input", "raw address", "user_inputed_address" | | **Dwelling** | A single residential unit that can hold an EPC — a house, flat, or maisonette. | "property", "unit", "home" | ## Address Matching @@ -72,7 +72,7 @@ Invoke `/ubiquitous-language` in any session to extract new terms from the conve ## Flagged ambiguities -- **"address"** appears as both the raw **User Address** (free-text from customer data) and a structured field on an **EPC Search Result** (normalised address lines). Always qualify: "user address" vs "EPC address" or "address line 1". +- **"address"** appears as both the raw **User Address** (free-text from customer data, or the structured `UserAddress` dataclass that wraps it) and a structured field on an **EPC Search Result** (normalised address lines). Always qualify: "user address" vs "EPC address" or "address line 1". Within `domain/`, **User Address** specifically means the `UserAddress` dataclass; in upstream ingestion contexts (CSV columns, SQS payloads) it can still mean the raw string sense. - **"score"** is used for the `AddressMatch.score()` function output, the `lexiscore` DataFrame column, and informally in conversation. Prefer **Lexiscore** in domain discussions; reserve "score" for method-level code comments. - **"user_inputed_address"** in `backend/address2UPRN/main.py` is a misspelling and a synonym for **User Address** — the canonical term. New code should use `user_address`. - **"EPC"** is overloaded as both the document (an Energy Performance Certificate) and the rating band letter. Use **EPC** for the document and **EPC Band** for the letter. diff --git a/domain/addresses/__init__.py b/domain/addresses/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/domain/addresses/postcode_batching.py b/domain/addresses/postcode_batching.py new file mode 100644 index 00000000..209e0784 --- /dev/null +++ b/domain/addresses/postcode_batching.py @@ -0,0 +1,87 @@ +"""Pure-Python postcode-grouped batching. + +This module preserves the batching invariants from the legacy postcode +splitter (``backend/postcode_splitter/main.py``) without touching pandas, +S3, or SQS: + + * Addresses are grouped by **Postcode** in *insertion order* -- the first + Postcode seen produces the first group. + * A Postcode group is never split across two batches. + * If a single Postcode group is larger than ``max_batch_size``, it is + flushed as its own oversize batch (any buffered groups go out first, + untouched). + * Adding a group that would push the buffer past ``max_batch_size`` first + flushes the existing buffer, then starts a new buffer with the group. + * Whatever remains in the buffer after the input is exhausted is flushed + as the final batch. + * Empty input yields no batches. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Iterator + +from domain.addresses.user_address import UserAddress + + +def iter_postcode_grouped_batches( + addresses: Iterable[UserAddress], + *, + max_batch_size: int = 500, +) -> Iterator[list[UserAddress]]: + """Yield batches of ``UserAddress`` grouped by Postcode. + + Args: + addresses: An iterable of :class:`UserAddress`. Order is preserved + within each Postcode group, and groups are yielded in the order + their first member was seen. + max_batch_size: The soft upper bound on batch size, in number of + addresses. A single Postcode group larger than this cap is + dispatched whole (the cap is never used to split a group). + + Yields: + Lists of ``UserAddress``. Each list is non-empty. + """ + if max_batch_size < 1: + raise ValueError("max_batch_size must be >= 1") + + groups = _group_by_postcode_in_order(addresses) + + buffer: list[UserAddress] = [] + for group in groups.values(): + group_len = len(group) + + # Oversize single-Postcode group: flush buffer first, then dispatch + # the group as its own batch. Mirrors the legacy + # ``if group_len >= batch_size`` branch. + if group_len >= max_batch_size: + if buffer: + yield buffer + buffer = [] + yield group + continue + + # Adding this group would overflow: flush buffer before appending. + if len(buffer) + group_len > max_batch_size: + yield buffer + buffer = [] + + buffer.extend(group) + + # Final flush. + if buffer: + yield buffer + + +def _group_by_postcode_in_order( + addresses: Iterable[UserAddress], +) -> dict[str, list[UserAddress]]: + """Group addresses by ``postcode`` preserving first-seen order. + + Python dicts retain insertion order since 3.7, so a plain dict suffices + for the same effect as pandas ``groupby(..., sort=False)``. + """ + groups: dict[str, list[UserAddress]] = {} + for address in addresses: + groups.setdefault(address.postcode, []).append(address) + return groups diff --git a/domain/addresses/user_address.py b/domain/addresses/user_address.py new file mode 100644 index 00000000..e48dfdec --- /dev/null +++ b/domain/addresses/user_address.py @@ -0,0 +1,36 @@ +"""The :class:`UserAddress` value object. + +A frozen dataclass capturing the splitter's domain entity: the raw input +address line, a sanitised postcode, and an optional internal reference from +the customer dataset. Postcode sanitisation runs in ``__post_init__`` so no +caller can construct an instance with an un-normalised postcode. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Optional + +from domain.postcodes.sanitise import sanitise_postcode + + +@dataclass(frozen=True) +class UserAddress: + """A user-supplied address paired with its canonical postcode. + + Attributes: + user_address: The free-text address string as supplied upstream. + postcode: The postcode; always stored in canonical form + (uppercased, whitespace stripped). Sanitisation is enforced by + :meth:`__post_init__`. + internal_reference: Optional customer-side identifier preserved for + traceability through the matching pipeline. + """ + + user_address: str + postcode: str + internal_reference: Optional[str] = None + + def __post_init__(self) -> None: + # Frozen dataclass: bypass the descriptor with object.__setattr__. + object.__setattr__(self, "postcode", sanitise_postcode(self.postcode)) diff --git a/domain/postcodes/__init__.py b/domain/postcodes/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/domain/postcodes/sanitise.py b/domain/postcodes/sanitise.py new file mode 100644 index 00000000..94b0dcf7 --- /dev/null +++ b/domain/postcodes/sanitise.py @@ -0,0 +1,23 @@ +"""Canonical postcode sanitisation for the domain layer. + +The legacy postcode_splitter normalises postcodes inline with +``df["postcode"].str.upper().str.replace(" ", "")``. This module promotes +that operation to a pure, reusable function so the same canonical form is +applied wherever a postcode crosses a domain boundary -- including +:class:`domain.addresses.user_address.UserAddress` construction and future +migrations. +""" + +from __future__ import annotations + + +def sanitise_postcode(s: str) -> str: + """Return the canonical form of a postcode. + + The canonical form is uppercase with all whitespace removed. This matches + the legacy splitter's ``str.upper().str.replace(" ", "")`` for the + overwhelmingly common case of space-separated postcodes (e.g. ``"sw1a 1aa"`` + becomes ``"SW1A1AA"``) while also tolerating tabs/newlines that can creep + in from CSV ingestion. + """ + return "".join(s.split()).upper() diff --git a/tests/domain/addresses/__init__.py b/tests/domain/addresses/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/domain/addresses/test_postcode_batching.py b/tests/domain/addresses/test_postcode_batching.py new file mode 100644 index 00000000..2dac46cc --- /dev/null +++ b/tests/domain/addresses/test_postcode_batching.py @@ -0,0 +1,93 @@ +import pytest + +from domain.addresses.postcode_batching import iter_postcode_grouped_batches +from domain.addresses.user_address import UserAddress + + +def _addrs(postcode: str, n: int) -> list[UserAddress]: + """Build ``n`` addresses sharing a postcode, with distinct address lines.""" + return [ + UserAddress(user_address=f"{i} {postcode} Street", postcode=postcode) + for i in range(n) + ] + + +def test_empty_input_yields_no_batches() -> None: + assert list(iter_postcode_grouped_batches([])) == [] + + +def test_single_batch_under_cap() -> None: + addrs = _addrs("AA1 1AA", 3) + _addrs("BB2 2BB", 2) + batches = list(iter_postcode_grouped_batches(addrs, max_batch_size=500)) + assert len(batches) == 1 + assert batches[0] == addrs + + +def test_multiple_postcodes_packed_into_one_batch_up_to_cap() -> None: + # Two groups whose total exactly equals the cap pack into a single + # batch -- no premature flush. + addrs = _addrs("AA1 1AA", 3) + _addrs("BB2 2BB", 2) + batches = list(iter_postcode_grouped_batches(addrs, max_batch_size=5)) + assert len(batches) == 1 + assert len(batches[0]) == 5 + + +def test_flush_on_overflow_before_adding_next_postcode() -> None: + # Cap is 5. First group fills 3 slots; second group of 3 would overflow, + # so the buffer is flushed first and the next group starts a fresh batch. + addrs = _addrs("AA1 1AA", 3) + _addrs("BB2 2BB", 3) + batches = list(iter_postcode_grouped_batches(addrs, max_batch_size=5)) + assert len(batches) == 2 + assert [a.postcode for a in batches[0]] == ["AA11AA"] * 3 + assert [a.postcode for a in batches[1]] == ["BB22BB"] * 3 + + +def test_single_postcode_group_exceeding_cap_is_dispatched_whole() -> None: + # An oversize single-postcode group goes out as one batch larger than + # the cap -- the cap never splits a postcode. + addrs = _addrs("AA1 1AA", 7) + batches = list(iter_postcode_grouped_batches(addrs, max_batch_size=5)) + assert len(batches) == 1 + assert len(batches[0]) == 7 + + +def test_oversize_group_flushes_existing_buffer_first() -> None: + # Mirrors the legacy ``if buffer: flush`` branch when an oversize group + # is encountered: buffered work must not be lost or interleaved. + small = _addrs("AA1 1AA", 2) + big = _addrs("BB2 2BB", 7) + tail = _addrs("CC3 3CC", 1) + batches = list( + iter_postcode_grouped_batches(small + big + tail, max_batch_size=5) + ) + assert len(batches) == 3 + assert [a.postcode for a in batches[0]] == ["AA11AA", "AA11AA"] + assert [a.postcode for a in batches[1]] == ["BB22BB"] * 7 + assert [a.postcode for a in batches[2]] == ["CC33CC"] + + +def test_final_flush_yields_remaining_buffer() -> None: + # No overflow ever happens, but the trailing buffer must still come out. + addrs = _addrs("AA1 1AA", 2) + _addrs("BB2 2BB", 2) + batches = list(iter_postcode_grouped_batches(addrs, max_batch_size=500)) + assert batches == [addrs] + + +def test_postcode_grouping_preserves_first_seen_order() -> None: + # Interleaved input must still group by postcode and emit in first-seen + # order -- never alphabetical. + a1, a2 = _addrs("ZZ9 9ZZ", 2) + b1, b2 = _addrs("AA1 1AA", 2) + batches = list(iter_postcode_grouped_batches([a1, b1, a2, b2])) + assert len(batches) == 1 + assert [a.postcode for a in batches[0]] == [ + "ZZ99ZZ", + "ZZ99ZZ", + "AA11AA", + "AA11AA", + ] + + +def test_invalid_max_batch_size_raises() -> None: + with pytest.raises(ValueError, match="max_batch_size"): + list(iter_postcode_grouped_batches([], max_batch_size=0)) diff --git a/tests/domain/addresses/test_user_address.py b/tests/domain/addresses/test_user_address.py new file mode 100644 index 00000000..e722077d --- /dev/null +++ b/tests/domain/addresses/test_user_address.py @@ -0,0 +1,45 @@ +import dataclasses + +import pytest + +from domain.addresses.user_address import UserAddress + + +def test_user_address_sanitises_postcode_on_construction() -> None: + addr = UserAddress(user_address="1 The Street", postcode="sw1a 1aa") + assert addr.postcode == "SW1A1AA" + + +def test_user_address_preserves_user_address_verbatim() -> None: + # The free-text user_address string is intentionally NOT normalised -- + # only the postcode is canonicalised at the boundary. + addr = UserAddress(user_address=" 1 The Street ", postcode="sw1a 1aa") + assert addr.user_address == " 1 The Street " + + +def test_user_address_internal_reference_defaults_to_none() -> None: + addr = UserAddress(user_address="1 The Street", postcode="SW1A1AA") + assert addr.internal_reference is None + + +def test_user_address_internal_reference_accepted() -> None: + addr = UserAddress( + user_address="1 The Street", + postcode="SW1A1AA", + internal_reference="cust-42", + ) + assert addr.internal_reference == "cust-42" + + +def test_user_address_is_frozen() -> None: + addr = UserAddress(user_address="1 The Street", postcode="SW1A1AA") + with pytest.raises(dataclasses.FrozenInstanceError): + addr.postcode = "OTHER" # type: ignore[misc] + + +def test_user_address_equality_uses_sanitised_postcode() -> None: + # Two instances constructed with different surface forms of the same + # postcode must compare equal because sanitisation runs eagerly. + a = UserAddress(user_address="1 The Street", postcode="sw1a 1aa") + b = UserAddress(user_address="1 The Street", postcode="SW1A1AA") + assert a == b diff --git a/tests/domain/postcodes/__init__.py b/tests/domain/postcodes/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/domain/postcodes/test_sanitise.py b/tests/domain/postcodes/test_sanitise.py new file mode 100644 index 00000000..edd1679c --- /dev/null +++ b/tests/domain/postcodes/test_sanitise.py @@ -0,0 +1,28 @@ +from domain.postcodes.sanitise import sanitise_postcode + + +def test_sanitise_uppercases() -> None: + assert sanitise_postcode("sw1a1aa") == "SW1A1AA" + + +def test_sanitise_strips_internal_spaces() -> None: + assert sanitise_postcode("sw1a 1aa") == "SW1A1AA" + + +def test_sanitise_strips_leading_and_trailing_whitespace() -> None: + assert sanitise_postcode(" sw1a 1aa ") == "SW1A1AA" + + +def test_sanitise_strips_tabs_and_newlines() -> None: + # CSV ingestion occasionally introduces stray whitespace characters; the + # canonical form must absorb them just like literal spaces. + assert sanitise_postcode("sw1a\t1aa\n") == "SW1A1AA" + + +def test_sanitise_already_canonical_is_idempotent() -> None: + assert sanitise_postcode("SW1A1AA") == "SW1A1AA" + assert sanitise_postcode(sanitise_postcode("sw1a 1aa")) == "SW1A1AA" + + +def test_sanitise_empty_string() -> None: + assert sanitise_postcode("") == "" From 7b00a33cd242e9959ac47e4e207d67477d53b8a2 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 19 May 2026 17:12:21 +0000 Subject: [PATCH 04/22] infrastructure: typed S3/SQS clients (S3Client, CsvS3Client, SqsClient, Address2UprnQueueClient) Slice 3/6 of the postcode_splitter refactor (Hestia-Homes/Model#1101). Introduces a thin typed infrastructure layer wrapping boto3 for the AWS side of the splitter. S3Client/SqsClient are bucket-/queue-bound byte adapters; CsvS3Client subclasses S3Client to round-trip CSV row dicts via the existing parse_s3_uri helper in utils/s3.py; Address2UprnQueueClient subclasses SqsClient to publish the typed {task_id, sub_task_id, s3_uri} fan-out body the downstream consumer expects. moto[s3,sqs] is pulled into test.requirements.txt and the new tests/infrastructure/ suite exercises each client against the moto backend (S3 round-trip, CSV round-trip, SQS send + body inspection, typed publish + body inspection). pyright --strict is clean on the new modules. Co-Authored-By: Claude Opus 4.7 (1M context) --- infrastructure/address2uprn_queue_client.py | 27 ++++++++ infrastructure/csv_s3_client.py | 46 +++++++++++++ infrastructure/s3_client.py | 31 +++++++++ infrastructure/sqs_client.py | 28 ++++++++ test.requirements.txt | 3 +- tests/infrastructure/__init__.py | 17 +++++ tests/infrastructure/conftest.py | 32 +++++++++ .../test_address2uprn_queue_client.py | 65 +++++++++++++++++++ tests/infrastructure/test_csv_s3_client.py | 43 ++++++++++++ tests/infrastructure/test_s3_client.py | 31 +++++++++ tests/infrastructure/test_sqs_client.py | 38 +++++++++++ 11 files changed, 360 insertions(+), 1 deletion(-) create mode 100644 infrastructure/address2uprn_queue_client.py create mode 100644 infrastructure/csv_s3_client.py create mode 100644 infrastructure/s3_client.py create mode 100644 infrastructure/sqs_client.py create mode 100644 tests/infrastructure/__init__.py create mode 100644 tests/infrastructure/conftest.py create mode 100644 tests/infrastructure/test_address2uprn_queue_client.py create mode 100644 tests/infrastructure/test_csv_s3_client.py create mode 100644 tests/infrastructure/test_s3_client.py create mode 100644 tests/infrastructure/test_sqs_client.py diff --git a/infrastructure/address2uprn_queue_client.py b/infrastructure/address2uprn_queue_client.py new file mode 100644 index 00000000..d81e2dd1 --- /dev/null +++ b/infrastructure/address2uprn_queue_client.py @@ -0,0 +1,27 @@ +from uuid import UUID + +from infrastructure.sqs_client import SqsClient + + +class Address2UprnQueueClient(SqsClient): + """SQS client that publishes Address-to-UPRN fan-out messages. + + The body shape is fixed by the downstream consumer: + ``{"task_id": str, "sub_task_id": str, "s3_uri": str}`` + """ + + def publish( + self, + *, + parent_task_id: UUID, + child_subtask_id: UUID, + s3_uri: str, + ) -> str: + """Send a typed Address-to-UPRN message. Returns the SQS ``MessageId``.""" + return self.send( + { + "task_id": str(parent_task_id), + "sub_task_id": str(child_subtask_id), + "s3_uri": s3_uri, + } + ) diff --git a/infrastructure/csv_s3_client.py b/infrastructure/csv_s3_client.py new file mode 100644 index 00000000..5163705b --- /dev/null +++ b/infrastructure/csv_s3_client.py @@ -0,0 +1,46 @@ +import csv +from io import StringIO + +from infrastructure.s3_client import S3Client +from utils.s3 import parse_s3_uri + + +class CsvS3Client(S3Client): + """:class:`S3Client` subclass that round-trips CSV row dictionaries. + + Rows are represented as ``list[dict[str, str]]`` — the same shape used by + :func:`csv.DictReader`/``DictWriter`` — which keeps the API trivially + compatible with existing CSV helpers in ``utils/s3.py``. + """ + + def read_rows(self, s3_uri: str) -> list[dict[str, str]]: + """Fetch the object at ``s3_uri`` and decode it as a CSV. + + The bucket portion of the URI is validated against this client's + configured bucket so cross-bucket reads fail loudly rather than + silently fetching from the wrong place. + """ + bucket, key = parse_s3_uri(s3_uri) + if bucket != self.bucket: + raise ValueError( + f"s3_uri bucket {bucket!r} does not match client bucket {self.bucket!r}" + ) + raw = self.get_object(key) + text = raw.decode("utf-8-sig") + reader = csv.DictReader(StringIO(text)) + return [dict(row) for row in reader] + + def save_rows(self, rows: list[dict[str, str]], key: str) -> str: + """Serialise ``rows`` to CSV under ``key`` and return the ``s3://`` URI. + + An empty ``rows`` list is rejected because we cannot otherwise infer + a header row. + """ + if not rows: + raise ValueError("Cannot save an empty rows list: header is unknown") + buffer = StringIO() + fieldnames = list(rows[0].keys()) + writer = csv.DictWriter(buffer, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + return self.put_object(key, buffer.getvalue().encode("utf-8")) diff --git a/infrastructure/s3_client.py b/infrastructure/s3_client.py new file mode 100644 index 00000000..9e772881 --- /dev/null +++ b/infrastructure/s3_client.py @@ -0,0 +1,31 @@ +from typing import Any + + +class S3Client: + """Thin typed wrapper around a boto3 S3 client bound to a single bucket. + + The class is deliberately small: it exposes only the byte-level + operations needed by the wider infrastructure layer. Serialisation + (CSV, JSON, etc.) lives in subclasses such as :class:`CsvS3Client`. + """ + + def __init__(self, boto_s3_client: Any, bucket: str) -> None: + self._client = boto_s3_client + self._bucket = bucket + + @property + def bucket(self) -> str: + return self._bucket + + def get_object(self, key: str) -> bytes: + """Return the raw bytes stored at ``key`` in this client's bucket.""" + response: dict[str, Any] = self._client.get_object( + Bucket=self._bucket, Key=key + ) + body: bytes = response["Body"].read() + return body + + def put_object(self, key: str, body: bytes) -> str: + """Write ``body`` to ``key`` and return the canonical ``s3://`` URI.""" + self._client.put_object(Bucket=self._bucket, Key=key, Body=body) + return f"s3://{self._bucket}/{key}" diff --git a/infrastructure/sqs_client.py b/infrastructure/sqs_client.py new file mode 100644 index 00000000..fb053680 --- /dev/null +++ b/infrastructure/sqs_client.py @@ -0,0 +1,28 @@ +import json +from typing import Any + + +class SqsClient: + """Thin typed wrapper around a boto3 SQS client bound to one queue URL. + + The body is JSON-serialised here so callers can pass plain dictionaries + instead of constructing message strings themselves. Typed publish + helpers (e.g. :class:`Address2UprnQueueClient`) build on this contract. + """ + + def __init__(self, boto_sqs_client: Any, queue_url: str) -> None: + self._client = boto_sqs_client + self._queue_url = queue_url + + @property + def queue_url(self) -> str: + return self._queue_url + + def send(self, body: dict[str, Any]) -> str: + """JSON-serialise ``body`` and send it. Returns the SQS ``MessageId``.""" + response: dict[str, Any] = self._client.send_message( + QueueUrl=self._queue_url, + MessageBody=json.dumps(body), + ) + message_id: str = response["MessageId"] + return message_id diff --git a/test.requirements.txt b/test.requirements.txt index 7fdd7dc4..26125034 100644 --- a/test.requirements.txt +++ b/test.requirements.txt @@ -9,4 +9,5 @@ hubspot-api-client fuzzywuzzy pymupdf playwright==1.58.0 -msal \ No newline at end of file +msal +moto[s3,sqs] \ No newline at end of file diff --git a/tests/infrastructure/__init__.py b/tests/infrastructure/__init__.py new file mode 100644 index 00000000..3478bda9 --- /dev/null +++ b/tests/infrastructure/__init__.py @@ -0,0 +1,17 @@ +from typing import Any + +import boto3 + +REGION = "us-east-1" + + +def make_boto_client(service_name: str) -> Any: + """Construct a boto3 client typed as ``Any``. + + boto3's overloaded ``client`` signature uses ``Literal[...]`` per service + in the installed stubs, which forces every call site to satisfy + ``reportArgumentType`` and ``reportUnknownMemberType`` under strict + pyright. Centralising the cast keeps each test file clean. + """ + factory: Any = boto3.client # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] + return factory(service_name, region_name=REGION) diff --git a/tests/infrastructure/conftest.py b/tests/infrastructure/conftest.py new file mode 100644 index 00000000..7ed2fdd6 --- /dev/null +++ b/tests/infrastructure/conftest.py @@ -0,0 +1,32 @@ +import os +from collections.abc import Iterator +from typing import Optional + +import pytest + + +@pytest.fixture(autouse=True) +def _aws_creds() -> Iterator[None]: # pyright: ignore[reportUnusedFunction] + """Stub AWS creds so botocore doesn't probe the host environment. + + Applied automatically to every test in ``tests/infrastructure/``. + """ + keys = ( + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "AWS_DEFAULT_REGION", + ) + prev: dict[str, Optional[str]] = {k: os.environ.get(k) for k in keys} + os.environ["AWS_ACCESS_KEY_ID"] = "testing" + os.environ["AWS_SECRET_ACCESS_KEY"] = "testing" + os.environ["AWS_SESSION_TOKEN"] = "testing" + os.environ["AWS_DEFAULT_REGION"] = "us-east-1" + try: + yield + finally: + for k, v in prev.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v diff --git a/tests/infrastructure/test_address2uprn_queue_client.py b/tests/infrastructure/test_address2uprn_queue_client.py new file mode 100644 index 00000000..b4114742 --- /dev/null +++ b/tests/infrastructure/test_address2uprn_queue_client.py @@ -0,0 +1,65 @@ +import json +from collections.abc import Iterator +from typing import Any, cast +from uuid import uuid4 + +import pytest +from moto import mock_aws + +from infrastructure.address2uprn_queue_client import Address2UprnQueueClient +from tests.infrastructure import make_boto_client + + +@pytest.fixture +def queue_setup() -> Iterator[tuple[Address2UprnQueueClient, Any, str]]: + with mock_aws(): + boto_client = make_boto_client("sqs") + queue: dict[str, Any] = boto_client.create_queue( + QueueName="address2uprn-queue" + ) + queue_url = cast(str, queue["QueueUrl"]) + yield ( + Address2UprnQueueClient(boto_client, queue_url), + boto_client, + queue_url, + ) + + +def test_publish_returns_message_id( + queue_setup: tuple[Address2UprnQueueClient, Any, str], +) -> None: + client, _boto, _url = queue_setup + message_id = client.publish( + parent_task_id=uuid4(), + child_subtask_id=uuid4(), + s3_uri="s3://my-bucket/path/to/chunk.csv", + ) + assert isinstance(message_id, str) + assert message_id + + +def test_publish_body_uses_typed_shape( + queue_setup: tuple[Address2UprnQueueClient, Any, str], +) -> None: + client, boto_client, queue_url = queue_setup + parent_id = uuid4() + child_id = uuid4() + s3_uri = "s3://my-bucket/path/to/chunk.csv" + + client.publish( + parent_task_id=parent_id, + child_subtask_id=child_id, + s3_uri=s3_uri, + ) + + received: dict[str, Any] = boto_client.receive_message( + QueueUrl=queue_url, MaxNumberOfMessages=1 + ) + messages: list[dict[str, Any]] = received["Messages"] + assert len(messages) == 1 + body = json.loads(messages[0]["Body"]) + assert body == { + "task_id": str(parent_id), + "sub_task_id": str(child_id), + "s3_uri": s3_uri, + } diff --git a/tests/infrastructure/test_csv_s3_client.py b/tests/infrastructure/test_csv_s3_client.py new file mode 100644 index 00000000..4b9fc199 --- /dev/null +++ b/tests/infrastructure/test_csv_s3_client.py @@ -0,0 +1,43 @@ +from collections.abc import Iterator + +import pytest +from moto import mock_aws + +from infrastructure.csv_s3_client import CsvS3Client +from tests.infrastructure import make_boto_client + +BUCKET = "csv-bucket" + + +@pytest.fixture +def csv_client() -> Iterator[CsvS3Client]: + with mock_aws(): + boto_client = make_boto_client("s3") + boto_client.create_bucket(Bucket=BUCKET) + yield CsvS3Client(boto_client, BUCKET) + + +def test_save_rows_returns_s3_uri(csv_client: CsvS3Client) -> None: + rows = [{"address": "1 High St", "postcode": "AB1 2CD"}] + uri = csv_client.save_rows(rows, "uploads/addresses.csv") + assert uri == f"s3://{BUCKET}/uploads/addresses.csv" + + +def test_round_trip_preserves_rows(csv_client: CsvS3Client) -> None: + rows = [ + {"address": "1 High St", "postcode": "AB1 2CD"}, + {"address": "2 Low St", "postcode": "XY9 8ZW"}, + ] + uri = csv_client.save_rows(rows, "uploads/addresses.csv") + fetched = csv_client.read_rows(uri) + assert fetched == rows + + +def test_save_rows_rejects_empty_list(csv_client: CsvS3Client) -> None: + with pytest.raises(ValueError, match="empty"): + csv_client.save_rows([], "uploads/empty.csv") + + +def test_read_rows_rejects_wrong_bucket(csv_client: CsvS3Client) -> None: + with pytest.raises(ValueError, match="does not match client bucket"): + csv_client.read_rows("s3://other-bucket/uploads/addresses.csv") diff --git a/tests/infrastructure/test_s3_client.py b/tests/infrastructure/test_s3_client.py new file mode 100644 index 00000000..7ed4c30b --- /dev/null +++ b/tests/infrastructure/test_s3_client.py @@ -0,0 +1,31 @@ +from collections.abc import Iterator + +import pytest +from moto import mock_aws + +from infrastructure.s3_client import S3Client +from tests.infrastructure import make_boto_client + +BUCKET = "test-bucket" + + +@pytest.fixture +def s3_client() -> Iterator[S3Client]: + with mock_aws(): + boto_client = make_boto_client("s3") + boto_client.create_bucket(Bucket=BUCKET) + yield S3Client(boto_client, BUCKET) + + +def test_put_object_returns_s3_uri(s3_client: S3Client) -> None: + uri = s3_client.put_object("folder/data.bin", b"payload") + assert uri == f"s3://{BUCKET}/folder/data.bin" + + +def test_get_object_returns_bytes_written_by_put_object(s3_client: S3Client) -> None: + s3_client.put_object("round/trip.bin", b"hello world") + assert s3_client.get_object("round/trip.bin") == b"hello world" + + +def test_bucket_property_exposes_configured_bucket(s3_client: S3Client) -> None: + assert s3_client.bucket == BUCKET diff --git a/tests/infrastructure/test_sqs_client.py b/tests/infrastructure/test_sqs_client.py new file mode 100644 index 00000000..7f1e8f78 --- /dev/null +++ b/tests/infrastructure/test_sqs_client.py @@ -0,0 +1,38 @@ +import json +from collections.abc import Iterator +from typing import Any, cast + +import pytest +from moto import mock_aws + +from infrastructure.sqs_client import SqsClient +from tests.infrastructure import make_boto_client + + +@pytest.fixture +def sqs_setup() -> Iterator[tuple[SqsClient, Any, str]]: + with mock_aws(): + boto_client = make_boto_client("sqs") + queue: dict[str, Any] = boto_client.create_queue(QueueName="test-queue") + queue_url = cast(str, queue["QueueUrl"]) + yield SqsClient(boto_client, queue_url), boto_client, queue_url + + +def test_send_returns_message_id(sqs_setup: tuple[SqsClient, Any, str]) -> None: + client, _boto, _url = sqs_setup + message_id = client.send({"hello": "world"}) + assert isinstance(message_id, str) + assert message_id + + +def test_send_json_serialises_body(sqs_setup: tuple[SqsClient, Any, str]) -> None: + client, boto_client, queue_url = sqs_setup + body = {"hello": "world", "count": 3} + client.send(body) + + received: dict[str, Any] = boto_client.receive_message( + QueueUrl=queue_url, MaxNumberOfMessages=1 + ) + messages: list[dict[str, Any]] = received["Messages"] + assert len(messages) == 1 + assert json.loads(messages[0]["Body"]) == body From d7f14033ba76b355543ded5fb3ced93e0411b2ae Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 19 May 2026 17:19:41 +0000 Subject: [PATCH 05/22] orchestration: add TaskOrchestrator.create_child_subtask primitive Adds a primitive for creating a new WAITING SubTask under an existing parent Task, routing all SubTask creation through the orchestrator (replacing the legacy SubTaskInterface path used by the splitter). Skips _cascade because a new WAITING child against an IN_PROGRESS parent is a no-op under Task.recalculate_from_subtasks. --- orchestration/task_orchestrator.py | 16 ++++++++++++++ tests/orchestration/test_task_orchestrator.py | 22 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/orchestration/task_orchestrator.py b/orchestration/task_orchestrator.py index 6c67d1ce..82d95db1 100644 --- a/orchestration/task_orchestrator.py +++ b/orchestration/task_orchestrator.py @@ -48,6 +48,22 @@ class TaskOrchestrator: self._subtasks.create(subtask) return task, subtask + def create_child_subtask( + self, + parent_task_id: UUID, + *, + inputs: Optional[dict[str, Any]] = None, + ) -> SubTask: + """Add a new WAITING SubTask under an existing parent Task. + + Skips `_cascade`: a new WAITING child against an IN_PROGRESS parent + leaves the parent's status unchanged per `Task.recalculate_from_subtasks`, + so calling it here would be a no-op. + """ + subtask = SubTask.create(task_id=parent_task_id, inputs=inputs) + self._subtasks.create(subtask) + return subtask + def start_subtask( self, subtask_id: UUID, cloud_logs_url: Optional[str] = None ) -> SubTask: diff --git a/tests/orchestration/test_task_orchestrator.py b/tests/orchestration/test_task_orchestrator.py index 1a48127f..c0816d2d 100644 --- a/tests/orchestration/test_task_orchestrator.py +++ b/tests/orchestration/test_task_orchestrator.py @@ -134,6 +134,28 @@ def test_run_subtask_happy_path_returns_result_and_cascades_complete( assert harness.tasks.get(task.id).status is TaskStatus.COMPLETE +def test_create_child_subtask_adds_waiting_child_without_changing_parent_status( + harness: Harness, +) -> None: + task, first = harness.orchestrator.create_task_with_subtask( + task_source="manual:test" + ) + harness.orchestrator.start_subtask(first.id) + assert harness.tasks.get(task.id).status is TaskStatus.IN_PROGRESS + + child = harness.orchestrator.create_child_subtask( + task.id, inputs={"split": "a"} + ) + + persisted_child = harness.subtasks.get(child.id) + assert persisted_child.task_id == task.id + assert persisted_child.status is SubTaskStatus.WAITING + assert persisted_child.inputs == {"split": "a"} + assert persisted_child.id != first.id + # Cascade is a no-op: parent stays IN_PROGRESS. + assert harness.tasks.get(task.id).status is TaskStatus.IN_PROGRESS + + def test_run_subtask_failing_work_marks_failed_and_reraises( harness: Harness, ) -> None: From d70e8a9e53706cd09f2cc82b85ce499fed97bda2 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 19 May 2026 17:31:27 +0000 Subject: [PATCH 06/22] utilities/aws_lambda: @subtask_handler injects TaskOrchestrator as third positional arg MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The wrapped function now receives the decorator-owned TaskOrchestrator as a third positional argument so handlers can compose their own use-case orchestrator that shares the session, instead of opening a second Postgres connection per invocation. Both existing callers (backend/ordnanceSurvey/main.py and backend/bulk_address2uprn_combiner/main.py) have their signatures extended to accept the new positional argument (typed Optional[TaskOrchestrator] so the legacy backend.utils.subtasks.subtask_handler — which only passes two args — keeps working until the migration to the new decorator lands). @task_handler is intentionally unchanged in this slice; symmetry is deferred per issue #1103. --- backend/bulk_address2uprn_combiner/main.py | 14 +- backend/ordnanceSurvey/main.py | 12 +- tests/utilities/__init__.py | 0 tests/utilities/aws_lambda/__init__.py | 0 .../aws_lambda/test_subtask_handler.py | 144 ++++++++++++++++++ utilities/aws_lambda/subtask_handler.py | 2 +- 6 files changed, 168 insertions(+), 4 deletions(-) create mode 100644 tests/utilities/__init__.py create mode 100644 tests/utilities/aws_lambda/__init__.py create mode 100644 tests/utilities/aws_lambda/test_subtask_handler.py diff --git a/backend/bulk_address2uprn_combiner/main.py b/backend/bulk_address2uprn_combiner/main.py index 44f0b3f9..37136e52 100644 --- a/backend/bulk_address2uprn_combiner/main.py +++ b/backend/bulk_address2uprn_combiner/main.py @@ -2,7 +2,7 @@ import os import boto3 import pandas as pd from io import BytesIO -from typing import Any +from typing import Any, Optional from uuid import UUID from datetime import datetime, timezone @@ -12,6 +12,7 @@ from backend.app.db.functions.bulk_address_uploads_functions import ( set_combined_output_s3_uri, set_combining_status, ) +from orchestration.task_orchestrator import TaskOrchestrator logger = setup_logger() @@ -35,7 +36,16 @@ def download_csv(s3_client, bucket: str, key: str) -> pd.DataFrame: @subtask_handler() -def handler(body: dict[str, Any], context: Any) -> str: +def handler( + body: dict[str, Any], + context: Any, + orchestrator: Optional[TaskOrchestrator] = None, +) -> str: + # `orchestrator` is injected by the new utilities.aws_lambda.subtask_handler + # decorator; unused here but accepted so the contract is uniform across + # callers (see issue #1103). + del orchestrator + task_id_str: str = body.get("task_id", "") if not task_id_str: diff --git a/backend/ordnanceSurvey/main.py b/backend/ordnanceSurvey/main.py index 6e82b468..18c4e2f2 100644 --- a/backend/ordnanceSurvey/main.py +++ b/backend/ordnanceSurvey/main.py @@ -16,6 +16,7 @@ from backend.ordnanceSurvey.helpers import ( os_places_results_to_dataframe, ) from backend.app.config import get_settings +from orchestration.task_orchestrator import TaskOrchestrator from sqlalchemy import select from datetime import datetime import uuid @@ -105,7 +106,16 @@ def save_results_to_s3( @subtask_handler() # This assumes task_id and subtask_id is defined in event.Records.body -def handler(body: dict[str, Any], context: Any, local: bool = False) -> None: +def handler( + body: dict[str, Any], + context: Any, + orchestrator: Optional[TaskOrchestrator] = None, + local: bool = False, +) -> None: + # `orchestrator` is injected by the new utilities.aws_lambda.subtask_handler + # decorator; unused here but accepted so the contract is uniform across + # callers (see issue #1103). + del orchestrator # delete this line after test # local = True diff --git a/tests/utilities/__init__.py b/tests/utilities/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/utilities/aws_lambda/__init__.py b/tests/utilities/aws_lambda/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/utilities/aws_lambda/test_subtask_handler.py b/tests/utilities/aws_lambda/test_subtask_handler.py new file mode 100644 index 00000000..426b250f --- /dev/null +++ b/tests/utilities/aws_lambda/test_subtask_handler.py @@ -0,0 +1,144 @@ +"""Tests for the @subtask_handler decorator. + +Covers the contract that the decorator owns the parent SubTask lifecycle and +injects the decorator-owned TaskOrchestrator as a third positional argument +to the wrapped function — so the handler can compose its own use-case +orchestrator that shares the session. +""" + +from collections.abc import Generator, Iterator +from contextlib import contextmanager +from dataclasses import dataclass +from typing import Any +from uuid import UUID + +import pytest +from sqlmodel import Session, SQLModel, create_engine + +from domain.tasks.subtasks import SubTaskStatus +from domain.tasks.tasks import TaskStatus +from orchestration.task_orchestrator import TaskOrchestrator +from repositories.tasks.subtask_postgres_repository import SubTaskPostgresRepository +from repositories.tasks.task_postgres_repository import TaskPostgresRepository +from utilities.aws_lambda.subtask_handler import subtask_handler + + +@dataclass +class Harness: + orchestrator: TaskOrchestrator + tasks: TaskPostgresRepository + subtasks: SubTaskPostgresRepository + + @contextmanager + def factory(self) -> Generator[TaskOrchestrator, None, None]: + yield self.orchestrator + + +@pytest.fixture +def harness() -> Iterator[Harness]: + engine = create_engine("sqlite://") + SQLModel.metadata.create_all(engine) + with Session(engine) as session: + tasks = TaskPostgresRepository(session=session) + subtasks = SubTaskPostgresRepository(session=session) + yield Harness( + orchestrator=TaskOrchestrator(task_repo=tasks, subtask_repo=subtasks), + tasks=tasks, + subtasks=subtasks, + ) + + +def _direct_event(task_id: UUID, subtask_id: UUID) -> dict[str, Any]: + return {"task_id": str(task_id), "sub_task_id": str(subtask_id)} + + +def test_subtask_handler_injects_orchestrator_as_third_positional_argument( + harness: Harness, +) -> None: + """The wrapped function receives the decorator-owned TaskOrchestrator + so it can share the session with its own use-case orchestrator.""" + _, subtask = harness.orchestrator.create_task_with_subtask( + task_source="manual:test" + ) + + received: dict[str, Any] = {} + + @subtask_handler(orchestrator_cm=harness.factory) + def handler( + body: dict[str, Any], context: Any, orchestrator: TaskOrchestrator + ) -> None: + received["body"] = body + received["context"] = context + received["orchestrator"] = orchestrator + + handler(_direct_event(subtask.task_id, subtask.id), context="ctx-sentinel") + + assert received["orchestrator"] is harness.orchestrator + assert received["context"] == "ctx-sentinel" + assert received["body"]["sub_task_id"] == str(subtask.id) + + +def test_subtask_handler_completes_parent_subtask_on_success( + harness: Harness, +) -> None: + task, subtask = harness.orchestrator.create_task_with_subtask( + task_source="manual:test" + ) + + @subtask_handler(orchestrator_cm=harness.factory) + def handler( + body: dict[str, Any], context: Any, orchestrator: TaskOrchestrator + ) -> None: + return None + + handler(_direct_event(task.id, subtask.id), context=None) + + assert harness.subtasks.get(subtask.id).status is SubTaskStatus.COMPLETE + assert harness.tasks.get(task.id).status is TaskStatus.COMPLETE + + +def test_subtask_handler_marks_parent_failed_and_reraises_on_error( + harness: Harness, +) -> None: + task, subtask = harness.orchestrator.create_task_with_subtask( + task_source="manual:test" + ) + + @subtask_handler(orchestrator_cm=harness.factory) + def handler( + body: dict[str, Any], context: Any, orchestrator: TaskOrchestrator + ) -> None: + raise RuntimeError("boom") + + with pytest.raises(RuntimeError, match="boom"): + handler(_direct_event(task.id, subtask.id), context=None) + + assert harness.subtasks.get(subtask.id).status is SubTaskStatus.FAILED + assert harness.tasks.get(task.id).status is TaskStatus.FAILED + + +def test_subtask_handler_injected_orchestrator_can_create_child_subtask( + harness: Harness, +) -> None: + """Smoke check the share-the-session promise: the injected orchestrator + is the same one the decorator owns, so a handler can use it to create + child SubTasks under the same session.""" + task, subtask = harness.orchestrator.create_task_with_subtask( + task_source="manual:test" + ) + + child_ids: list[UUID] = [] + + @subtask_handler(orchestrator_cm=harness.factory) + def handler( + body: dict[str, Any], context: Any, orchestrator: TaskOrchestrator + ) -> None: + child = orchestrator.create_child_subtask(task.id, inputs={"split": 1}) + child_ids.append(child.id) + + handler(_direct_event(task.id, subtask.id), context=None) + + assert len(child_ids) == 1 + persisted_child = harness.subtasks.get(child_ids[0]) + assert persisted_child.task_id == task.id + assert persisted_child.status is SubTaskStatus.WAITING diff --git a/utilities/aws_lambda/subtask_handler.py b/utilities/aws_lambda/subtask_handler.py index 64c1daa6..5ad5f6e1 100644 --- a/utilities/aws_lambda/subtask_handler.py +++ b/utilities/aws_lambda/subtask_handler.py @@ -39,7 +39,7 @@ def subtask_handler( trigger = SubtaskTriggerBody.model_validate(body) orchestrator.run_subtask( trigger.sub_task_id, - work=lambda body=body: func(body, context), + work=lambda body=body, o=orchestrator: func(body, context, o), ) return wrapper From 708f1b5d189222793cd206cf883c3c427ca63917 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 19 May 2026 17:37:02 +0000 Subject: [PATCH 07/22] repositories: UserAddressRepository + UserAddressCsvS3Repository (CSV-on-S3 adapter) Adds the persistence layer for UserAddress batches: - Abstract UserAddressRepository with load_batch / save_batch. - Concrete UserAddressCsvS3Repository over CsvS3Client: - load_batch reads canonical upload columns (Address 1/2/3, Postcode, Internal Reference), comma-joins non-empty address parts, and passes Internal Reference through (None when missing/empty). - save_batch writes a 3-column CSV (user_address,postcode, internal_reference) to {path_prefix}/{ISO datetime}_{uuid8}.csv and returns the s3://bucket/key URI. - Postcode sanitisation flows through UserAddress.__post_init__; the repo never calls sanitise_postcode directly. Tests (moto-backed) cover: three-line address load, Address-1-only load, missing Internal Reference, save->reload round trip, and unique-filename-per-save. pyright --strict clean. Co-Authored-By: Claude Opus 4.7 (1M context) --- repositories/user_address/__init__.py | 0 .../user_address_csv_s3_repository.py | 87 +++++++++ .../user_address/user_address_repository.py | 30 +++ tests/repositories/user_address/__init__.py | 0 tests/repositories/user_address/conftest.py | 32 ++++ .../test_user_address_csv_s3_repository.py | 175 ++++++++++++++++++ 6 files changed, 324 insertions(+) create mode 100644 repositories/user_address/__init__.py create mode 100644 repositories/user_address/user_address_csv_s3_repository.py create mode 100644 repositories/user_address/user_address_repository.py create mode 100644 tests/repositories/user_address/__init__.py create mode 100644 tests/repositories/user_address/conftest.py create mode 100644 tests/repositories/user_address/test_user_address_csv_s3_repository.py diff --git a/repositories/user_address/__init__.py b/repositories/user_address/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/repositories/user_address/user_address_csv_s3_repository.py b/repositories/user_address/user_address_csv_s3_repository.py new file mode 100644 index 00000000..be2baa13 --- /dev/null +++ b/repositories/user_address/user_address_csv_s3_repository.py @@ -0,0 +1,87 @@ +"""CSV-on-S3 adapter for :class:`UserAddressRepository`. + +Reads canonical upload CSVs (``Address 1``, ``Address 2``, ``Address 3``, +``Postcode``, ``Internal Reference``) and writes the splitter's compact +3-column form (``user_address``, ``postcode``, ``internal_reference``). + +The frontend pre-applies the user's column mapping at upload time, so this +adapter does NOT consult any ``BulkAddressUpload.column_mapping``: it always +expects the canonical column names listed above. +""" + +from __future__ import annotations + +import uuid +from datetime import datetime, timezone +from typing import Optional + +from domain.addresses.user_address import UserAddress +from infrastructure.csv_s3_client import CsvS3Client +from repositories.user_address.user_address_repository import UserAddressRepository + +_ADDRESS_COLUMNS: tuple[str, str, str] = ("Address 1", "Address 2", "Address 3") +_POSTCODE_COLUMN: str = "Postcode" +_INTERNAL_REFERENCE_COLUMN: str = "Internal Reference" + + +class UserAddressCsvS3Repository(UserAddressRepository): + """Persist :class:`UserAddress` batches as CSV objects in S3. + + The repo owns the unique-filename-within-prefix convention + (``{ISO datetime}_{8-char uuid}.csv``); callers own the directory + hierarchy supplied as ``path_prefix``. + """ + + def __init__(self, csv_client: CsvS3Client, bucket: str) -> None: + self._csv_client = csv_client + self._bucket = bucket + + def load_batch(self, s3_uri: str) -> list[UserAddress]: + """Load canonical upload CSV rows into :class:`UserAddress` objects. + + Concatenates ``Address 1``/``Address 2``/``Address 3`` with ``", "``, + skipping missing or empty parts, into ``user_address``. Falls back to + just ``Address 1`` when 2 and 3 are absent. Passes ``Internal Reference`` + through to :attr:`UserAddress.internal_reference` (``None`` when the + column is missing or empty). + """ + rows = self._csv_client.read_rows(s3_uri) + addresses: list[UserAddress] = [] + for row in rows: + parts = [ + row[col].strip() + for col in _ADDRESS_COLUMNS + if col in row and row[col].strip() + ] + user_address = ", ".join(parts) + postcode = row.get(_POSTCODE_COLUMN, "") + raw_ref = row.get(_INTERNAL_REFERENCE_COLUMN, "").strip() + internal_reference: Optional[str] = raw_ref or None + addresses.append( + UserAddress( + user_address=user_address, + postcode=postcode, + internal_reference=internal_reference, + ) + ) + return addresses + + def save_batch(self, addresses: list[UserAddress], path_prefix: str) -> str: + """Write a 3-column CSV under a unique key beneath ``path_prefix``. + + The key is ``{path_prefix}/{ISO-8601 datetime}_{8-char uuid}.csv``. + Returns the full ``s3://bucket/key`` URI. + """ + rows: list[dict[str, str]] = [ + { + "user_address": addr.user_address, + "postcode": addr.postcode, + "internal_reference": addr.internal_reference or "", + } + for addr in addresses + ] + filename = ( + f"{datetime.now(timezone.utc).isoformat()}_{uuid.uuid4().hex[:8]}.csv" + ) + key = f"{path_prefix.rstrip('/')}/{filename}" + return self._csv_client.save_rows(rows, key) diff --git a/repositories/user_address/user_address_repository.py b/repositories/user_address/user_address_repository.py new file mode 100644 index 00000000..d8c12855 --- /dev/null +++ b/repositories/user_address/user_address_repository.py @@ -0,0 +1,30 @@ +"""Abstract repository for :class:`UserAddress` batches. + +Persistence-agnostic interface for loading and saving batches of +:class:`domain.addresses.user_address.UserAddress`. Concrete adapters -- +e.g. :class:`UserAddressCsvS3Repository` -- live alongside this module. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod + +from domain.addresses.user_address import UserAddress + + +class UserAddressRepository(ABC): + """Load and persist batches of :class:`UserAddress`. + + Implementations choose the underlying storage (S3 CSV, Postgres, + in-memory, ...) but must preserve the canonical column semantics: + the address text, postcode (sanitised by ``UserAddress.__post_init__``), + and an optional internal reference. + """ + + @abstractmethod + def load_batch(self, s3_uri: str) -> list[UserAddress]: + """Read a batch of addresses from ``s3_uri`` and return domain objects.""" + + @abstractmethod + def save_batch(self, addresses: list[UserAddress], path_prefix: str) -> str: + """Persist ``addresses`` under ``path_prefix`` and return the URI written.""" diff --git a/tests/repositories/user_address/__init__.py b/tests/repositories/user_address/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/repositories/user_address/conftest.py b/tests/repositories/user_address/conftest.py new file mode 100644 index 00000000..1859ff0a --- /dev/null +++ b/tests/repositories/user_address/conftest.py @@ -0,0 +1,32 @@ +import os +from collections.abc import Iterator +from typing import Optional + +import pytest + + +@pytest.fixture(autouse=True) +def _aws_creds() -> Iterator[None]: # pyright: ignore[reportUnusedFunction] + """Stub AWS creds so botocore doesn't probe the host environment. + + Applied automatically to every test in ``tests/repositories/user_address/``. + """ + keys = ( + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "AWS_DEFAULT_REGION", + ) + prev: dict[str, Optional[str]] = {k: os.environ.get(k) for k in keys} + os.environ["AWS_ACCESS_KEY_ID"] = "testing" + os.environ["AWS_SECRET_ACCESS_KEY"] = "testing" + os.environ["AWS_SESSION_TOKEN"] = "testing" + os.environ["AWS_DEFAULT_REGION"] = "us-east-1" + try: + yield + finally: + for k, v in prev.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v diff --git a/tests/repositories/user_address/test_user_address_csv_s3_repository.py b/tests/repositories/user_address/test_user_address_csv_s3_repository.py new file mode 100644 index 00000000..ca9e8a57 --- /dev/null +++ b/tests/repositories/user_address/test_user_address_csv_s3_repository.py @@ -0,0 +1,175 @@ +from collections.abc import Iterator + +import pytest +from moto import mock_aws + +from infrastructure.csv_s3_client import CsvS3Client +from repositories.user_address.user_address_csv_s3_repository import ( + UserAddressCsvS3Repository, +) +from tests.infrastructure import make_boto_client + +BUCKET = "user-address-bucket" + + +@pytest.fixture +def repo() -> Iterator[UserAddressCsvS3Repository]: + with mock_aws(): + boto_client = make_boto_client("s3") + boto_client.create_bucket(Bucket=BUCKET) + csv_client = CsvS3Client(boto_client, BUCKET) + yield UserAddressCsvS3Repository(csv_client, BUCKET) + + +def _upload_csv( + repo: UserAddressCsvS3Repository, rows: list[dict[str, str]], key: str +) -> str: + return repo._csv_client.save_rows(rows, key) # pyright: ignore[reportPrivateUsage] + + +def test_load_batch_concatenates_three_address_lines( + repo: UserAddressCsvS3Repository, +) -> None: + rows = [ + { + "Address 1": "1 High Street", + "Address 2": "Flat 2", + "Address 3": "Townville", + "Postcode": "sw1a 1aa", + "Internal Reference": "REF-001", + } + ] + uri = _upload_csv(repo, rows, "uploads/full.csv") + + addresses = repo.load_batch(uri) + + assert len(addresses) == 1 + address = addresses[0] + assert address.user_address == "1 High Street, Flat 2, Townville" + assert address.postcode == "SW1A1AA" + assert address.internal_reference == "REF-001" + + +def test_load_batch_uses_only_address_1_when_others_missing( + repo: UserAddressCsvS3Repository, +) -> None: + rows = [ + { + "Address 1": "10 Cardiff Road", + "Address 2": "", + "Address 3": "", + "Postcode": "CF10 1AA", + "Internal Reference": "REF-002", + } + ] + uri = _upload_csv(repo, rows, "uploads/address1-only.csv") + + addresses = repo.load_batch(uri) + + assert len(addresses) == 1 + assert addresses[0].user_address == "10 Cardiff Road" + assert addresses[0].postcode == "CF101AA" + assert addresses[0].internal_reference == "REF-002" + + +def test_load_batch_handles_missing_internal_reference( + repo: UserAddressCsvS3Repository, +) -> None: + rows = [ + { + "Address 1": "5 Park Lane", + "Address 2": "", + "Address 3": "", + "Postcode": "M1 1AA", + "Internal Reference": "", + } + ] + uri = _upload_csv(repo, rows, "uploads/no-ref.csv") + + addresses = repo.load_batch(uri) + + assert len(addresses) == 1 + assert addresses[0].user_address == "5 Park Lane" + assert addresses[0].postcode == "M11AA" + assert addresses[0].internal_reference is None + + +def test_save_batch_returns_uri_under_path_prefix( + repo: UserAddressCsvS3Repository, +) -> None: + from domain.addresses.user_address import UserAddress + + addresses = [ + UserAddress( + user_address="1 High Street, Flat 2, Townville", + postcode="SW1A 1AA", + internal_reference="REF-001", + ), + ] + + uri = repo.save_batch(addresses, "tasks/abc/batches") + + assert uri.startswith(f"s3://{BUCKET}/tasks/abc/batches/") + assert uri.endswith(".csv") + + +def test_save_then_reload_round_trip_preserves_values( + repo: UserAddressCsvS3Repository, +) -> None: + from domain.addresses.user_address import UserAddress + + # save_batch writes the splitter's compact schema + # (user_address/postcode/internal_reference); load_batch reads the + # canonical upload schema. To round-trip through the repo we re-upload + # the saved CSV under the upload schema's column names. + original = [ + UserAddress( + user_address="1 High Street", + postcode="SW1A 1AA", + internal_reference="REF-001", + ), + UserAddress( + user_address="2 Low Street", + postcode="XY9 8ZW", + internal_reference=None, + ), + ] + + saved_uri = repo.save_batch(original, "tasks/round-trip") + + # Re-shape the saved CSV into the canonical upload schema for reload. + saved_rows = repo._csv_client.read_rows(saved_uri) # pyright: ignore[reportPrivateUsage] + upload_rows: list[dict[str, str]] = [ + { + "Address 1": row["user_address"], + "Address 2": "", + "Address 3": "", + "Postcode": row["postcode"], + "Internal Reference": row["internal_reference"], + } + for row in saved_rows + ] + upload_uri = _upload_csv(repo, upload_rows, "uploads/round-trip.csv") + + reloaded = repo.load_batch(upload_uri) + + assert reloaded == original + + +def test_save_batch_uses_unique_filename_per_call( + repo: UserAddressCsvS3Repository, +) -> None: + from domain.addresses.user_address import UserAddress + + addresses = [ + UserAddress( + user_address="1 High Street", + postcode="SW1A 1AA", + internal_reference="REF-001", + ), + ] + + uri_1 = repo.save_batch(addresses, "tasks/uniqueness") + uri_2 = repo.save_batch(addresses, "tasks/uniqueness") + + assert uri_1 != uri_2 From 0a0444821756543f57625832681e28a09d264056 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 19 May 2026 17:46:12 +0000 Subject: [PATCH 08/22] applications/postcode_splitter: PostcodeSplitterOrchestrator + Lambda entrypoint slice Wires slice 1-5 primitives into a deployable splitter: - orchestration/postcode_splitter_orchestrator.py: PostcodeSplitterOrchestrator loads addresses via UserAddressRepository, groups by postcode via iter_postcode_grouped_batches, persists each batch under ara_postcode_splitter_batches/{task_id}/{subtask_id}/, creates a WAITING child SubTask, and publishes an address2UPRN SQS message per batch. - applications/postcode_splitter/: Lambda entrypoint. handler.py is decorated with @subtask_handler() so the parent SubTask lifecycle is decorator-owned; PostcodeSplitterTriggerBody validates the body. Dockerfile is the python:3.11 Lambda base with the DDD-shaped source layers and no pandas. - tests/orchestration/test_postcode_splitter_orchestrator.py: integration test using moto S3 + moto SQS + in-memory SQLite that exercises the full wiring against a fixture CSV spanning three postcode groups (one oversize) and asserts child count, persisted inputs, queue bodies, and dispatch order. backend/postcode_splitter/ and .github/workflows/deploy_terraform.yml are intentionally unchanged: the dockerfile_path flip is deferred until the companion backend/address2UPRN/ migration is also ready. --- applications/__init__.py | 0 applications/postcode_splitter/Dockerfile | 21 ++ applications/postcode_splitter/__init__.py | 0 applications/postcode_splitter/handler.py | 70 ++++ .../postcode_splitter_trigger_body.py | 32 ++ .../postcode_splitter/requirements.txt | 4 + .../postcode_splitter_orchestrator.py | 89 ++++++ .../test_postcode_splitter_orchestrator.py | 298 ++++++++++++++++++ 8 files changed, 514 insertions(+) create mode 100644 applications/__init__.py create mode 100644 applications/postcode_splitter/Dockerfile create mode 100644 applications/postcode_splitter/__init__.py create mode 100644 applications/postcode_splitter/handler.py create mode 100644 applications/postcode_splitter/postcode_splitter_trigger_body.py create mode 100644 applications/postcode_splitter/requirements.txt create mode 100644 orchestration/postcode_splitter_orchestrator.py create mode 100644 tests/orchestration/test_postcode_splitter_orchestrator.py diff --git a/applications/__init__.py b/applications/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/applications/postcode_splitter/Dockerfile b/applications/postcode_splitter/Dockerfile new file mode 100644 index 00000000..578ee7a7 --- /dev/null +++ b/applications/postcode_splitter/Dockerfile @@ -0,0 +1,21 @@ +FROM public.ecr.aws/lambda/python:3.11 + +WORKDIR /var/task + +COPY applications/postcode_splitter/requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Copy the layered source the handler imports from. The new splitter pulls +# only DDD-shaped packages — no pandas, no legacy backend/. +COPY domain/ domain/ +COPY infrastructure/ infrastructure/ +COPY orchestration/ orchestration/ +COPY repositories/ repositories/ +COPY utilities/ utilities/ +COPY applications/ applications/ + +# Place the handler at the Lambda task root so the runtime can resolve +# ``main.handler`` without an extra package prefix. +COPY applications/postcode_splitter/handler.py /var/task/main.py + +CMD ["main.handler"] diff --git a/applications/postcode_splitter/__init__.py b/applications/postcode_splitter/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/applications/postcode_splitter/handler.py b/applications/postcode_splitter/handler.py new file mode 100644 index 00000000..005227a9 --- /dev/null +++ b/applications/postcode_splitter/handler.py @@ -0,0 +1,70 @@ +"""Lambda entrypoint for the postcode splitter slice. + +The :func:`handler` function is decorated with ``@subtask_handler()`` so the +decorator owns the parent ``SubTask`` lifecycle (start/complete/fail) and +injects the decorator-owned :class:`TaskOrchestrator` as the third positional +argument. The handler itself does only two things: + +1. Build a :class:`PostcodeSplitterOrchestrator` from env-driven config. +2. Delegate to ``split_and_dispatch`` and return its result so it lands in + ``SubTask.outputs["result"]``. +""" + +from __future__ import annotations + +import os +from typing import Any + +import boto3 + +from applications.postcode_splitter.postcode_splitter_trigger_body import ( + PostcodeSplitterTriggerBody, +) +from infrastructure.address2uprn_queue_client import Address2UprnQueueClient +from infrastructure.csv_s3_client import CsvS3Client +from orchestration.postcode_splitter_orchestrator import PostcodeSplitterOrchestrator +from orchestration.task_orchestrator import TaskOrchestrator +from repositories.user_address.user_address_csv_s3_repository import ( + UserAddressCsvS3Repository, +) +from utilities.aws_lambda.subtask_handler import subtask_handler + + +@subtask_handler() +def handler( + body: dict[str, Any], context: Any, task_orchestrator: TaskOrchestrator +) -> dict[str, list[str]]: + """Validate the trigger body, build the splitter, dispatch children. + + Reads ``S3_BUCKET_NAME`` and ``ADDRESS2UPRN_QUEUE_URL`` from the + environment to construct the typed S3/SQS clients. The return value + lands in ``SubTask.outputs["result"]`` via the decorator. + """ + trigger = PostcodeSplitterTriggerBody.model_validate(body) + + bucket = os.environ["S3_BUCKET_NAME"] + queue_url = os.environ["ADDRESS2UPRN_QUEUE_URL"] + + # boto3.client is overloaded per-service in the installed stubs; cast + # to Any so the strict-mode checker treats it as opaque. + boto3_client: Any = boto3.client # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] + boto_s3: Any = boto3_client("s3") + boto_sqs: Any = boto3_client("sqs") + + csv_client = CsvS3Client(boto_s3, bucket) + user_address_repo = UserAddressCsvS3Repository(csv_client, bucket) + queue_client = Address2UprnQueueClient(boto_sqs, queue_url) + + splitter = PostcodeSplitterOrchestrator( + task_orchestrator=task_orchestrator, + user_address_repo=user_address_repo, + queue_client=queue_client, + ) + + child_ids = splitter.split_and_dispatch( + parent_task_id=trigger.task_id, + parent_subtask_id=trigger.sub_task_id, + input_s3_uri=trigger.s3_uri, + ) + + return {"child_subtask_ids": [str(cid) for cid in child_ids]} diff --git a/applications/postcode_splitter/postcode_splitter_trigger_body.py b/applications/postcode_splitter/postcode_splitter_trigger_body.py new file mode 100644 index 00000000..bc983abc --- /dev/null +++ b/applications/postcode_splitter/postcode_splitter_trigger_body.py @@ -0,0 +1,32 @@ +"""Trigger payload model for the postcode splitter Lambda. + +The decorator (``@subtask_handler``) already validates ``task_id`` and +``sub_task_id`` via :class:`SubtaskTriggerBody`; this model layers on the +splitter-specific ``s3_uri`` field while keeping ``extra="allow"`` so any +upstream-passthrough keys (e.g. ``portfolio_id``) survive untouched. +""" + +from uuid import UUID + +from pydantic import BaseModel, ConfigDict + + +class PostcodeSplitterTriggerBody(BaseModel): + """Validated body for the postcode splitter Lambda. + + Attributes: + task_id: Parent ``Task`` id; used as the ``task_id`` input on each + child ``SubTask`` and as the ``parent_task_id`` on the fan-out + SQS messages. + sub_task_id: The splitter's own ``SubTask`` id; used as the path + segment under ``ara_postcode_splitter_batches/{task_id}/{...}`` + so per-invocation outputs cannot collide. + s3_uri: ``s3://bucket/key`` URI of the uploaded address CSV the + splitter must read. + """ + + model_config = ConfigDict(extra="allow") + + task_id: UUID + sub_task_id: UUID + s3_uri: str diff --git a/applications/postcode_splitter/requirements.txt b/applications/postcode_splitter/requirements.txt new file mode 100644 index 00000000..6a85a255 --- /dev/null +++ b/applications/postcode_splitter/requirements.txt @@ -0,0 +1,4 @@ +boto3 +pydantic +sqlmodel +psycopg2-binary diff --git a/orchestration/postcode_splitter_orchestrator.py b/orchestration/postcode_splitter_orchestrator.py new file mode 100644 index 00000000..6afa2538 --- /dev/null +++ b/orchestration/postcode_splitter_orchestrator.py @@ -0,0 +1,89 @@ +"""Use-case orchestrator for the postcode splitter Lambda. + +Wires the slice-1 domain (``iter_postcode_grouped_batches``), the slice-3 +``UserAddressRepository``, the slice-2 ``Address2UprnQueueClient``, and the +slice-4 ``TaskOrchestrator.create_child_subtask`` primitive together. + +``split_and_dispatch`` loads the input batch, groups it into per-postcode +chunks, writes each chunk back to S3 under a deterministic prefix, creates a +WAITING child ``SubTask`` for it, and publishes the address-to-UPRN fan-out +message that downstream consumers pick up. +""" + +from __future__ import annotations + +from uuid import UUID + +from infrastructure.address2uprn_queue_client import Address2UprnQueueClient +from orchestration.task_orchestrator import TaskOrchestrator +from domain.addresses.postcode_batching import iter_postcode_grouped_batches +from repositories.user_address.user_address_repository import UserAddressRepository + + +class PostcodeSplitterOrchestrator: + """Split an uploaded address batch into postcode-grouped child SubTasks. + + The orchestrator owns the algorithm; the IO collaborators + (:class:`UserAddressRepository`, :class:`Address2UprnQueueClient`) and + the :class:`TaskOrchestrator` lifecycle primitive are injected so the + same wiring can be exercised against moto/SQLite in tests and against + real AWS in the Lambda entrypoint. + """ + + def __init__( + self, + task_orchestrator: TaskOrchestrator, + user_address_repo: UserAddressRepository, + queue_client: Address2UprnQueueClient, + max_batch_size: int = 500, + ) -> None: + self._task_orchestrator = task_orchestrator + self._user_address_repo = user_address_repo + self._queue_client = queue_client + self._max_batch_size = max_batch_size + + def split_and_dispatch( + self, + *, + parent_task_id: UUID, + parent_subtask_id: UUID, + input_s3_uri: str, + ) -> list[UUID]: + """Split ``input_s3_uri`` into postcode batches and dispatch each. + + For each yielded batch: + + 1. Persist it under + ``ara_postcode_splitter_batches/{parent_task_id}/{parent_subtask_id}``. + 2. Create a WAITING child ``SubTask`` with + ``inputs={"task_id": str(parent_task_id), "s3_uri": batch_uri}``. + 3. Publish an ``address2UPRN`` SQS message referencing the new child. + + Returns: + The list of child ``SubTask`` ids, in dispatch order. + """ + addresses = self._user_address_repo.load_batch(input_s3_uri) + path_prefix = ( + f"ara_postcode_splitter_batches/{parent_task_id}/{parent_subtask_id}" + ) + + child_ids: list[UUID] = [] + for batch in iter_postcode_grouped_batches( + addresses, max_batch_size=self._max_batch_size + ): + batch_uri = self._user_address_repo.save_batch(batch, path_prefix) + child = self._task_orchestrator.create_child_subtask( + parent_task_id, + inputs={ + "task_id": str(parent_task_id), + "s3_uri": batch_uri, + }, + ) + self._queue_client.publish( + parent_task_id=parent_task_id, + child_subtask_id=child.id, + s3_uri=batch_uri, + ) + child_ids.append(child.id) + + return child_ids diff --git a/tests/orchestration/test_postcode_splitter_orchestrator.py b/tests/orchestration/test_postcode_splitter_orchestrator.py new file mode 100644 index 00000000..57bd2133 --- /dev/null +++ b/tests/orchestration/test_postcode_splitter_orchestrator.py @@ -0,0 +1,298 @@ +"""Integration test: PostcodeSplitterOrchestrator wired end-to-end. + +Combines moto S3 + moto SQS + an in-memory SQLite session for the +``TaskOrchestrator`` so the full slice-6 wiring is exercised through real +infrastructure adapters (not mocks). The fixture CSV spans three postcodes +with one oversize group, which forces both the buffer-flush-then-oversize +branch and the final-flush branch of +``iter_postcode_grouped_batches`` — three batches in total. +""" + +from __future__ import annotations + +import json +import os +from collections.abc import Iterator +from dataclasses import dataclass +from typing import Any, cast + +import boto3 +import pytest +from moto import mock_aws +from sqlmodel import Session, SQLModel, create_engine + +from infrastructure.address2uprn_queue_client import Address2UprnQueueClient +from infrastructure.csv_s3_client import CsvS3Client +from orchestration.postcode_splitter_orchestrator import PostcodeSplitterOrchestrator +from orchestration.task_orchestrator import TaskOrchestrator +from repositories.tasks.subtask_postgres_repository import SubTaskPostgresRepository +from repositories.tasks.task_postgres_repository import TaskPostgresRepository +from repositories.user_address.user_address_csv_s3_repository import ( + UserAddressCsvS3Repository, +) + +BUCKET = "splitter-bucket" +REGION = "us-east-1" + + +def _make_boto_client(service_name: str) -> Any: + factory: Any = boto3.client # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] + return factory(service_name, region_name=REGION) + + +@pytest.fixture(autouse=True) +def _aws_creds() -> Iterator[None]: # pyright: ignore[reportUnusedFunction] + keys = ( + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "AWS_DEFAULT_REGION", + ) + prev: dict[str, Any] = {k: os.environ.get(k) for k in keys} + os.environ["AWS_ACCESS_KEY_ID"] = "testing" + os.environ["AWS_SECRET_ACCESS_KEY"] = "testing" + os.environ["AWS_SESSION_TOKEN"] = "testing" + os.environ["AWS_DEFAULT_REGION"] = REGION + try: + yield + finally: + for k, v in prev.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v + + +@dataclass +class Harness: + splitter: PostcodeSplitterOrchestrator + task_orchestrator: TaskOrchestrator + subtasks: SubTaskPostgresRepository + csv_client: CsvS3Client + boto_sqs: Any + queue_url: str + repo: UserAddressCsvS3Repository + + +@pytest.fixture +def harness() -> Iterator[Harness]: + with mock_aws(): + # Infra: S3 + SQS + boto_s3 = _make_boto_client("s3") + boto_s3.create_bucket(Bucket=BUCKET) + boto_sqs = _make_boto_client("sqs") + queue: dict[str, Any] = boto_sqs.create_queue(QueueName="address2uprn-queue") + queue_url = cast(str, queue["QueueUrl"]) + + csv_client = CsvS3Client(boto_s3, BUCKET) + repo = UserAddressCsvS3Repository(csv_client, BUCKET) + queue_client = Address2UprnQueueClient(boto_sqs, queue_url) + + # DB: in-memory SQLite TaskOrchestrator + engine = create_engine("sqlite://") + SQLModel.metadata.create_all(engine) + with Session(engine) as session: + task_repo = TaskPostgresRepository(session=session) + subtask_repo = SubTaskPostgresRepository(session=session) + task_orchestrator = TaskOrchestrator( + task_repo=task_repo, subtask_repo=subtask_repo + ) + + splitter = PostcodeSplitterOrchestrator( + task_orchestrator=task_orchestrator, + user_address_repo=repo, + queue_client=queue_client, + max_batch_size=3, + ) + + yield Harness( + splitter=splitter, + task_orchestrator=task_orchestrator, + subtasks=subtask_repo, + csv_client=csv_client, + boto_sqs=boto_sqs, + queue_url=queue_url, + repo=repo, + ) + + +def _upload_fixture_csv(csv_client: CsvS3Client) -> str: + # Three postcode groups: + # AA1 1AA × 2 (within cap) + # BB2 2BB × 4 (oversize: > max_batch_size=3) + # CC3 3CC × 1 (final flush) + # Expected batching with cap=3 and the algorithm in + # ``iter_postcode_grouped_batches``: + # batch 1: [AA1 1AA × 2] (flushed because oversize follows) + # batch 2: [BB2 2BB × 4] (oversize own batch) + # batch 3: [CC3 3CC × 1] (final flush) + rows: list[dict[str, str]] = [] + rows.extend( + { + "Address 1": f"{i} High St", + "Address 2": "", + "Address 3": "", + "Postcode": "AA1 1AA", + "Internal Reference": f"AA-{i}", + } + for i in range(1, 3) + ) + rows.extend( + { + "Address 1": f"{i} Long Road", + "Address 2": "", + "Address 3": "", + "Postcode": "BB2 2BB", + "Internal Reference": f"BB-{i}", + } + for i in range(1, 5) + ) + rows.append( + { + "Address 1": "1 Final Way", + "Address 2": "", + "Address 3": "", + "Postcode": "CC3 3CC", + "Internal Reference": "CC-1", + } + ) + return csv_client.save_rows(rows, "uploads/input.csv") + + +def _drain_queue(boto_sqs: Any, queue_url: str) -> list[dict[str, Any]]: + bodies: list[dict[str, Any]] = [] + while True: + received: dict[str, Any] = boto_sqs.receive_message( + QueueUrl=queue_url, MaxNumberOfMessages=10, WaitTimeSeconds=0 + ) + messages = cast(list[dict[str, Any]], received.get("Messages", [])) + if not messages: + break + for message in messages: + bodies.append(cast(dict[str, Any], json.loads(message["Body"]))) + boto_sqs.delete_message( + QueueUrl=queue_url, ReceiptHandle=message["ReceiptHandle"] + ) + return bodies + + +def test_split_and_dispatch_creates_three_children_for_fixture( + harness: Harness, +) -> None: + parent_task, parent_subtask = ( + harness.task_orchestrator.create_task_with_subtask( + task_source="manual:postcode-splitter-int" + ) + ) + input_uri = _upload_fixture_csv(harness.csv_client) + + child_ids = harness.splitter.split_and_dispatch( + parent_task_id=parent_task.id, + parent_subtask_id=parent_subtask.id, + input_s3_uri=input_uri, + ) + + assert len(child_ids) == 3 + # All child ids are unique and persisted as WAITING children of the + # parent task. + assert len(set(child_ids)) == 3 + for cid in child_ids: + child = harness.subtasks.get(cid) + assert child.task_id == parent_task.id + + +def test_split_and_dispatch_persists_child_inputs_with_task_id_and_s3_uri( + harness: Harness, +) -> None: + parent_task, parent_subtask = ( + harness.task_orchestrator.create_task_with_subtask( + task_source="manual:postcode-splitter-int" + ) + ) + input_uri = _upload_fixture_csv(harness.csv_client) + + child_ids = harness.splitter.split_and_dispatch( + parent_task_id=parent_task.id, + parent_subtask_id=parent_subtask.id, + input_s3_uri=input_uri, + ) + + for cid in child_ids: + child = harness.subtasks.get(cid) + assert child.inputs is not None + assert child.inputs["task_id"] == str(parent_task.id) + batch_uri = child.inputs["s3_uri"] + assert isinstance(batch_uri, str) + prefix = ( + f"s3://{BUCKET}/ara_postcode_splitter_batches/" + f"{parent_task.id}/{parent_subtask.id}/" + ) + assert batch_uri.startswith(prefix) + assert batch_uri.endswith(".csv") + + +def test_split_and_dispatch_publishes_one_message_per_child_with_matching_ids( + harness: Harness, +) -> None: + parent_task, parent_subtask = ( + harness.task_orchestrator.create_task_with_subtask( + task_source="manual:postcode-splitter-int" + ) + ) + input_uri = _upload_fixture_csv(harness.csv_client) + + child_ids = harness.splitter.split_and_dispatch( + parent_task_id=parent_task.id, + parent_subtask_id=parent_subtask.id, + input_s3_uri=input_uri, + ) + + bodies = _drain_queue(harness.boto_sqs, harness.queue_url) + assert len(bodies) == len(child_ids) + + # Match queue messages against persisted child inputs by child_subtask_id; + # the message body's task_id/s3_uri must agree with the SubTask inputs. + bodies_by_child = {body["sub_task_id"]: body for body in bodies} + assert set(bodies_by_child.keys()) == {str(cid) for cid in child_ids} + for cid in child_ids: + child = harness.subtasks.get(cid) + body = bodies_by_child[str(cid)] + assert child.inputs is not None + assert body == { + "task_id": str(parent_task.id), + "sub_task_id": str(cid), + "s3_uri": child.inputs["s3_uri"], + } + + +def test_split_and_dispatch_returns_child_ids_in_dispatch_order( + harness: Harness, +) -> None: + parent_task, parent_subtask = ( + harness.task_orchestrator.create_task_with_subtask( + task_source="manual:postcode-splitter-int" + ) + ) + input_uri = _upload_fixture_csv(harness.csv_client) + + child_ids = harness.splitter.split_and_dispatch( + parent_task_id=parent_task.id, + parent_subtask_id=parent_subtask.id, + input_s3_uri=input_uri, + ) + + # Re-load each child's saved batch and inspect the postcode column to + # confirm the dispatch order matches the postcode-batching algorithm: + # AA-batch first, BB oversize batch second, CC final-flush third. + postcodes_per_batch: list[set[str]] = [] + for cid in child_ids: + child = harness.subtasks.get(cid) + assert child.inputs is not None + rows = harness.csv_client.read_rows(child.inputs["s3_uri"]) + postcodes_per_batch.append({row["postcode"] for row in rows}) + + assert postcodes_per_batch == [ + {"AA11AA"}, + {"BB22BB"}, + {"CC33CC"}, + ] From 914a8ed51e13bdd1cb644de4972d0d4951269adc Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 20 May 2026 11:07:40 +0000 Subject: [PATCH 09/22] postcode splliter working e2e --- .gitignore | 1 + applications/postcode_splitter/Dockerfile | 13 ++ .../local_handler/.env.local.example | 34 +++++ .../local_handler/docker-compose.yml | 9 ++ .../local_handler/invoke_local_lambda.py | 37 +++++ .../local_handler/run_local.sh | 12 ++ backend/address2UPRN/handler/requirements.txt | 3 +- .../terraform/lambda/postcodeSplitter/main.tf | 14 -- domain/addresses/user_address.py | 16 ++- infrastructure/csv_s3_client.py | 2 +- infrastructure/s3_uri.py | 43 ++++++ .../user_address_csv_s3_repository.py | 57 +++++--- tests/domain/addresses/test_user_address.py | 26 ++++ tests/infrastructure/test_s3_uri.py | 32 +++++ .../test_postcode_splitter_orchestrator.py | 12 +- .../test_user_address_csv_s3_repository.py | 127 +++++++++++------- .../aws_lambda/test_subtask_handler.py | 111 +++++++++++++++ utilities/aws_lambda/subtask_handler.py | 67 ++++++++- 18 files changed, 523 insertions(+), 93 deletions(-) create mode 100644 applications/postcode_splitter/local_handler/.env.local.example create mode 100644 applications/postcode_splitter/local_handler/docker-compose.yml create mode 100755 applications/postcode_splitter/local_handler/invoke_local_lambda.py create mode 100755 applications/postcode_splitter/local_handler/run_local.sh create mode 100644 infrastructure/s3_uri.py create mode 100644 tests/infrastructure/test_s3_uri.py diff --git a/.gitignore b/.gitignore index 888d527a..9e5df0c7 100644 --- a/.gitignore +++ b/.gitignore @@ -121,6 +121,7 @@ celerybeat.pid # Environments .env +.env.local .venv env/ venv/ diff --git a/applications/postcode_splitter/Dockerfile b/applications/postcode_splitter/Dockerfile index 578ee7a7..aea1f914 100644 --- a/applications/postcode_splitter/Dockerfile +++ b/applications/postcode_splitter/Dockerfile @@ -1,5 +1,18 @@ FROM public.ecr.aws/lambda/python:3.11 +# Postgres host/port/database are baked into the image at build time from +# the deploy workflow's --build-arg values (GitHub Actions DEV_DB_* secrets), +# mirroring backend/postcode_splitter/handler/Dockerfile. They map onto the +# POSTGRES_* names PostgresConfig.from_env reads. Username/password are NOT +# baked in -- Terraform injects those as Lambda env vars from Secrets Manager. +ARG DEV_DB_HOST +ARG DEV_DB_PORT +ARG DEV_DB_NAME + +ENV POSTGRES_HOST=${DEV_DB_HOST} +ENV POSTGRES_PORT=${DEV_DB_PORT} +ENV POSTGRES_DATABASE=${DEV_DB_NAME} + WORKDIR /var/task COPY applications/postcode_splitter/requirements.txt . diff --git a/applications/postcode_splitter/local_handler/.env.local.example b/applications/postcode_splitter/local_handler/.env.local.example new file mode 100644 index 00000000..28fa8390 --- /dev/null +++ b/applications/postcode_splitter/local_handler/.env.local.example @@ -0,0 +1,34 @@ +# Local-test environment for the postcode_splitter Lambda. +# +# cp .env.local.example .env.local then fill in the values below. +# +# .env.local is gitignored. The container hits REAL AWS and a REAL Postgres, +# so every value here points at infrastructure that actually exists. +# +# NOTE: the new DDD code uses different env var names than the repo root +# .env. The mapping (root .env name -> var here) is given per section. +# Keep comments on their own lines — docker-compose's env_file parser folds a +# trailing "# ..." into the value. + +# --- Postgres (orchestration/default_orchestrator -> PostgresConfig.from_env) --- +# POSTGRES_HOST <- DB_HOST, PORT <- DB_PORT, USERNAME <- DB_USERNAME, +# PASSWORD <- DB_PASSWORD, DATABASE <- DB_NAME. +POSTGRES_HOST= +POSTGRES_PORT=5432 +POSTGRES_USERNAME= +POSTGRES_PASSWORD= +POSTGRES_DATABASE= +# POSTGRES_DRIVER=psycopg2 (optional; defaults to psycopg2) + +# --- Handler config (applications/postcode_splitter/handler.py) --- +# S3_BUCKET_NAME: bucket holding the input address CSV (root .env: DATA_BUCKET). +# ADDRESS2UPRN_QUEUE_URL: SQS queue the splitter fans batches out to; not in +# the root .env (Terraform sets it in prod). +S3_BUCKET_NAME= +ADDRESS2UPRN_QUEUE_URL= + +# --- AWS credentials for boto3 (S3 + SQS clients) --- +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= +AWS_DEFAULT_REGION=eu-west-2 +# AWS_SESSION_TOKEN= (only if using temporary/SSO credentials) diff --git a/applications/postcode_splitter/local_handler/docker-compose.yml b/applications/postcode_splitter/local_handler/docker-compose.yml new file mode 100644 index 00000000..68af1c40 --- /dev/null +++ b/applications/postcode_splitter/local_handler/docker-compose.yml @@ -0,0 +1,9 @@ +services: + postcode-splitter: + build: + context: ../../../ + dockerfile: applications/postcode_splitter/Dockerfile + ports: + - "9001:8080" + env_file: + - .env.local diff --git a/applications/postcode_splitter/local_handler/invoke_local_lambda.py b/applications/postcode_splitter/local_handler/invoke_local_lambda.py new file mode 100755 index 00000000..c0ca89ec --- /dev/null +++ b/applications/postcode_splitter/local_handler/invoke_local_lambda.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +"""POST a single SQS-shaped event at the locally-running splitter Lambda. + +The container built by docker-compose runs the AWS Lambda Runtime Interface +Emulator, which accepts invocations on the URL below. Replace the three +placeholder values with a real parent Task id, the splitter's own SubTask id +(both must already exist in the Postgres pointed at by .env.local), and the +s3://... URI of an uploaded address CSV. +""" + +import json +import requests + +HOST = "localhost" +PORT = "9001" + +LAMBDA_URL = f"http://{HOST}:{PORT}/2015-03-31/functions/function/invocations" + +payload = { + "Records": [ + { + "body": json.dumps( + { + "task_id": "f4b3332f-c0cc-481f-96a5-d39860a647cf", + "sub_task_id": "14c042de-40c4-473b-8cd8-72c983a94a8d", + "s3_uri": "s3://retrofit-data-dev/ara_raw_inputs/calico/Calico Homes Full list EPC Properties(Sheet2) (1) (1).csv", + } + ) + } + ] +} + +response = requests.post(LAMBDA_URL, json=payload) + +print("Status code:", response.status_code) +print("Response:") +print(response.text) diff --git a/applications/postcode_splitter/local_handler/run_local.sh b/applications/postcode_splitter/local_handler/run_local.sh new file mode 100755 index 00000000..345b60ee --- /dev/null +++ b/applications/postcode_splitter/local_handler/run_local.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")" + +if [ ! -f .env.local ]; then + cp .env.local.example .env.local + echo "Created .env.local from the template — fill it in, then re-run." >&2 + exit 1 +fi + +docker compose build --no-cache +docker compose up --force-recreate diff --git a/backend/address2UPRN/handler/requirements.txt b/backend/address2UPRN/handler/requirements.txt index 6ef41b2d..02aaefba 100644 --- a/backend/address2UPRN/handler/requirements.txt +++ b/backend/address2UPRN/handler/requirements.txt @@ -8,4 +8,5 @@ boto3==1.35.44 sqlmodel sqlalchemy==2.0.36 psycopg2-binary==2.9.10 -pydantic-settings==2.6.0 \ No newline at end of file +pydantic-settings==2.6.0 +httpx \ No newline at end of file diff --git a/deployment/terraform/lambda/postcodeSplitter/main.tf b/deployment/terraform/lambda/postcodeSplitter/main.tf index 94c5cd4e..325f7dc7 100644 --- a/deployment/terraform/lambda/postcodeSplitter/main.tf +++ b/deployment/terraform/lambda/postcodeSplitter/main.tf @@ -40,20 +40,6 @@ module "lambda" { LOG_LEVEL = "info" DB_USERNAME = local.db_credentials.db_assessment_model_username DB_PASSWORD = local.db_credentials.db_assessment_model_password - GOOGLE_SOLAR_API_KEY = "test" - SAP_PREDICTIONS_BUCKET = "test" - CARBON_PREDICTIONS_BUCKET = "test" - HEAT_PREDICTIONS_BUCKET = "test" - HEATING_KWH_PREDICTIONS_BUCKET = "test" - HOTWATER_KWH_PREDICTIONS_BUCKET = "test" - API_KEY = "test" - ENVIRONMENT = "test" - SECRET_KEY = "test" - PLAN_TRIGGER_BUCKET = "test" - DATA_BUCKET = "test" - EPC_AUTH_TOKEN = "test" - ENGINE_SQS_URL = "test" - ENERGY_ASSESSMENTS_BUCKET = "test" ADDRESS2UPRN_QUEUE_URL = data.terraform_remote_state.address2uprn.outputs.address2uprn_queue_url S3_BUCKET_NAME = data.terraform_remote_state.shared.outputs.retrofit_sap_data_bucket_name }, diff --git a/domain/addresses/user_address.py b/domain/addresses/user_address.py index e48dfdec..120a3659 100644 --- a/domain/addresses/user_address.py +++ b/domain/addresses/user_address.py @@ -8,12 +8,17 @@ caller can construct an instance with an un-normalised postcode. from __future__ import annotations -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Optional from domain.postcodes.sanitise import sanitise_postcode +def _empty_source_row() -> dict[str, str]: + """Typed default factory for :attr:`UserAddress.source_row`.""" + return {} + + @dataclass(frozen=True) class UserAddress: """A user-supplied address paired with its canonical postcode. @@ -25,11 +30,20 @@ class UserAddress: :meth:`__post_init__`. internal_reference: Optional customer-side identifier preserved for traceability through the matching pipeline. + source_row: The complete original CSV row this address was parsed + from, column name -> cell value. The splitter is a pass-through + router: it groups rows by postcode but must not drop the other + columns the downstream address2uprn stage relies on, so the raw + row travels alongside the parsed fields. Excluded from equality + and hashing -- identity stays defined by the parsed fields above. """ user_address: str postcode: str internal_reference: Optional[str] = None + source_row: dict[str, str] = field( + default_factory=_empty_source_row, compare=False + ) def __post_init__(self) -> None: # Frozen dataclass: bypass the descriptor with object.__setattr__. diff --git a/infrastructure/csv_s3_client.py b/infrastructure/csv_s3_client.py index 5163705b..0a576b81 100644 --- a/infrastructure/csv_s3_client.py +++ b/infrastructure/csv_s3_client.py @@ -2,7 +2,7 @@ import csv from io import StringIO from infrastructure.s3_client import S3Client -from utils.s3 import parse_s3_uri +from infrastructure.s3_uri import parse_s3_uri class CsvS3Client(S3Client): diff --git a/infrastructure/s3_uri.py b/infrastructure/s3_uri.py new file mode 100644 index 00000000..bf97100e --- /dev/null +++ b/infrastructure/s3_uri.py @@ -0,0 +1,43 @@ +"""Parse S3 URIs into ``(bucket, key)`` pairs. + +A pure-stdlib helper for the infrastructure layer. It deliberately pulls in +neither pandas, boto3, nor the legacy ``utils`` package, so slim Lambda images +that only need URI parsing do not drag the wider data stack along. + +Two input shapes are supported: + +* canonical S3 URIs --- ``s3://bucket/key`` +* AWS S3 console URLs --- ``https://.../s3/object/bucket?prefix=key`` +""" + +from urllib.parse import unquote + + +def parse_s3_uri(s3_uri: str) -> tuple[str, str]: + """Return the ``(bucket, key)`` pair addressed by ``s3_uri``. + + Raises: + ValueError: if ``s3_uri`` is neither a well-formed ``s3://`` URI nor + an AWS console URL carrying a ``prefix`` query parameter. + """ + if s3_uri.startswith("s3://"): + parts = s3_uri[len("s3://") :].split("/", 1) + if len(parts) < 2 or not parts[0] or not parts[1]: + raise ValueError("S3 URI must include both a bucket and a key") + return parts[0], parts[1] + + if "?" not in s3_uri: + raise ValueError(f"Not an s3:// URI and has no query string: {s3_uri!r}") + base, query = s3_uri.split("?", 1) + + if "/s3/object/" not in base: + raise ValueError(f"Console URL has no '/s3/object/' segment: {s3_uri!r}") + bucket = base.split("/s3/object/", 1)[1] + + params: dict[str, str] = {} + for item in query.split("&"): + if "=" in item: + name, value = item.split("=", 1) + params[name] = value + key = unquote(params.get("prefix", "")) + return bucket, key diff --git a/repositories/user_address/user_address_csv_s3_repository.py b/repositories/user_address/user_address_csv_s3_repository.py index be2baa13..7cd10bac 100644 --- a/repositories/user_address/user_address_csv_s3_repository.py +++ b/repositories/user_address/user_address_csv_s3_repository.py @@ -1,12 +1,16 @@ """CSV-on-S3 adapter for :class:`UserAddressRepository`. -Reads canonical upload CSVs (``Address 1``, ``Address 2``, ``Address 3``, -``Postcode``, ``Internal Reference``) and writes the splitter's compact -3-column form (``user_address``, ``postcode``, ``internal_reference``). +Reads upload CSVs that carry a ``postcode`` column (plus optional +``Address 1``/``Address 2``/``Address 3`` and ``Internal Reference``), and +writes batch CSVs that pass *every* original column through unchanged with +one column appended -- ``postcode_clean`` (uppercase, whitespace-stripped) -- +which the downstream address2uprn stage groups on. -The frontend pre-applies the user's column mapping at upload time, so this -adapter does NOT consult any ``BulkAddressUpload.column_mapping``: it always -expects the canonical column names listed above. +The splitter is a pass-through router: it must not reshape or drop columns, +because address2uprn has not been migrated and still consumes the legacy +splitter's full-row output. The frontend pre-applies the user's column +mapping at upload time, so this adapter does NOT consult any +``BulkAddressUpload.column_mapping``. """ from __future__ import annotations @@ -20,8 +24,9 @@ from infrastructure.csv_s3_client import CsvS3Client from repositories.user_address.user_address_repository import UserAddressRepository _ADDRESS_COLUMNS: tuple[str, str, str] = ("Address 1", "Address 2", "Address 3") -_POSTCODE_COLUMN: str = "Postcode" +_POSTCODE_COLUMN: str = "postcode" _INTERNAL_REFERENCE_COLUMN: str = "Internal Reference" +_POSTCODE_CLEAN_COLUMN: str = "postcode_clean" class UserAddressCsvS3Repository(UserAddressRepository): @@ -37,15 +42,27 @@ class UserAddressCsvS3Repository(UserAddressRepository): self._bucket = bucket def load_batch(self, s3_uri: str) -> list[UserAddress]: - """Load canonical upload CSV rows into :class:`UserAddress` objects. + """Load upload CSV rows into :class:`UserAddress` objects. - Concatenates ``Address 1``/``Address 2``/``Address 3`` with ``", "``, - skipping missing or empty parts, into ``user_address``. Falls back to - just ``Address 1`` when 2 and 3 are absent. Passes ``Internal Reference`` - through to :attr:`UserAddress.internal_reference` (``None`` when the - column is missing or empty). + Each row's complete column set is preserved on + :attr:`UserAddress.source_row` so :meth:`save_batch` can pass it + through untouched. The parsed convenience fields are also populated: + ``Address 1``/``Address 2``/``Address 3`` are concatenated with + ``", "`` (skipping missing/empty parts) into ``user_address``, and + ``Internal Reference`` is threaded to + :attr:`UserAddress.internal_reference` (``None`` when missing/empty). + + Raises: + ValueError: if the CSV has rows but no ``postcode`` column -- + without it the splitter cannot group, and silently emitting + empty postcodes would corrupt every downstream batch. """ rows = self._csv_client.read_rows(s3_uri) + if rows and _POSTCODE_COLUMN not in rows[0]: + raise ValueError( + f"Input CSV {s3_uri} has no {_POSTCODE_COLUMN!r} column; " + f"columns present: {sorted(rows[0])}" + ) addresses: list[UserAddress] = [] for row in rows: parts = [ @@ -62,22 +79,24 @@ class UserAddressCsvS3Repository(UserAddressRepository): user_address=user_address, postcode=postcode, internal_reference=internal_reference, + source_row=row, ) ) return addresses def save_batch(self, addresses: list[UserAddress], path_prefix: str) -> str: - """Write a 3-column CSV under a unique key beneath ``path_prefix``. + """Write a pass-through batch CSV under a unique key. + + Each output row is the address's original ``source_row`` with a + ``postcode_clean`` column appended (the canonical postcode the + downstream address2uprn stage groups on). No original column is + dropped or reshaped. The key is ``{path_prefix}/{ISO-8601 datetime}_{8-char uuid}.csv``. Returns the full ``s3://bucket/key`` URI. """ rows: list[dict[str, str]] = [ - { - "user_address": addr.user_address, - "postcode": addr.postcode, - "internal_reference": addr.internal_reference or "", - } + {**addr.source_row, _POSTCODE_CLEAN_COLUMN: addr.postcode} for addr in addresses ] filename = ( diff --git a/tests/domain/addresses/test_user_address.py b/tests/domain/addresses/test_user_address.py index e722077d..4d8322da 100644 --- a/tests/domain/addresses/test_user_address.py +++ b/tests/domain/addresses/test_user_address.py @@ -43,3 +43,29 @@ def test_user_address_equality_uses_sanitised_postcode() -> None: a = UserAddress(user_address="1 The Street", postcode="sw1a 1aa") b = UserAddress(user_address="1 The Street", postcode="SW1A1AA") assert a == b + + +def test_user_address_source_row_defaults_to_empty_dict() -> None: + addr = UserAddress(user_address="1 The Street", postcode="SW1A1AA") + assert addr.source_row == {} + + +def test_user_address_carries_source_row() -> None: + row = {"Address 1": "1 The Street", "postcode": "SW1A 1AA", "SAP Score": "72"} + addr = UserAddress( + user_address="1 The Street", postcode="SW1A 1AA", source_row=row + ) + assert addr.source_row == row + + +def test_user_address_equality_ignores_source_row() -> None: + # source_row is excluded from equality (and hashing): identity stays + # defined by the parsed fields, so two addresses parsed from rows with + # different incidental columns still compare equal. + a = UserAddress( + user_address="1 The Street", postcode="SW1A1AA", source_row={"x": "1"} + ) + b = UserAddress( + user_address="1 The Street", postcode="SW1A1AA", source_row={"y": "2"} + ) + assert a == b diff --git a/tests/infrastructure/test_s3_uri.py b/tests/infrastructure/test_s3_uri.py new file mode 100644 index 00000000..896c5959 --- /dev/null +++ b/tests/infrastructure/test_s3_uri.py @@ -0,0 +1,32 @@ +import pytest + +from infrastructure.s3_uri import parse_s3_uri + + +def test_parses_simple_s3_uri() -> None: + assert parse_s3_uri("s3://my-bucket/file.csv") == ("my-bucket", "file.csv") + + +def test_parses_s3_uri_with_nested_key() -> None: + bucket, key = parse_s3_uri("s3://my-bucket/nested/path/to/file.csv") + assert (bucket, key) == ("my-bucket", "nested/path/to/file.csv") + + +def test_rejects_s3_uri_without_key() -> None: + with pytest.raises(ValueError, match="bucket and a key"): + parse_s3_uri("s3://my-bucket") + + +def test_rejects_s3_uri_with_empty_key() -> None: + with pytest.raises(ValueError, match="bucket and a key"): + parse_s3_uri("s3://my-bucket/") + + +def test_parses_console_url_prefix() -> None: + url = "https://eu-west-2.console.aws.amazon.com/s3/object/my-bucket?prefix=nested%2Ffile.csv" + assert parse_s3_uri(url) == ("my-bucket", "nested/file.csv") + + +def test_rejects_unparseable_string() -> None: + with pytest.raises(ValueError): + parse_s3_uri("not-a-uri-at-all") diff --git a/tests/orchestration/test_postcode_splitter_orchestrator.py b/tests/orchestration/test_postcode_splitter_orchestrator.py index 57bd2133..79c60974 100644 --- a/tests/orchestration/test_postcode_splitter_orchestrator.py +++ b/tests/orchestration/test_postcode_splitter_orchestrator.py @@ -132,7 +132,7 @@ def _upload_fixture_csv(csv_client: CsvS3Client) -> str: "Address 1": f"{i} High St", "Address 2": "", "Address 3": "", - "Postcode": "AA1 1AA", + "postcode": "AA1 1AA", "Internal Reference": f"AA-{i}", } for i in range(1, 3) @@ -142,7 +142,7 @@ def _upload_fixture_csv(csv_client: CsvS3Client) -> str: "Address 1": f"{i} Long Road", "Address 2": "", "Address 3": "", - "Postcode": "BB2 2BB", + "postcode": "BB2 2BB", "Internal Reference": f"BB-{i}", } for i in range(1, 5) @@ -152,7 +152,7 @@ def _upload_fixture_csv(csv_client: CsvS3Client) -> str: "Address 1": "1 Final Way", "Address 2": "", "Address 3": "", - "Postcode": "CC3 3CC", + "postcode": "CC3 3CC", "Internal Reference": "CC-1", } ) @@ -281,15 +281,15 @@ def test_split_and_dispatch_returns_child_ids_in_dispatch_order( input_s3_uri=input_uri, ) - # Re-load each child's saved batch and inspect the postcode column to - # confirm the dispatch order matches the postcode-batching algorithm: + # Re-load each child's saved batch and inspect the postcode_clean column + # to confirm the dispatch order matches the postcode-batching algorithm: # AA-batch first, BB oversize batch second, CC final-flush third. postcodes_per_batch: list[set[str]] = [] for cid in child_ids: child = harness.subtasks.get(cid) assert child.inputs is not None rows = harness.csv_client.read_rows(child.inputs["s3_uri"]) - postcodes_per_batch.append({row["postcode"] for row in rows}) + postcodes_per_batch.append({row["postcode_clean"] for row in rows}) assert postcodes_per_batch == [ {"AA11AA"}, diff --git a/tests/repositories/user_address/test_user_address_csv_s3_repository.py b/tests/repositories/user_address/test_user_address_csv_s3_repository.py index ca9e8a57..48733b55 100644 --- a/tests/repositories/user_address/test_user_address_csv_s3_repository.py +++ b/tests/repositories/user_address/test_user_address_csv_s3_repository.py @@ -3,6 +3,7 @@ from collections.abc import Iterator import pytest from moto import mock_aws +from domain.addresses.user_address import UserAddress from infrastructure.csv_s3_client import CsvS3Client from repositories.user_address.user_address_csv_s3_repository import ( UserAddressCsvS3Repository, @@ -27,7 +28,7 @@ def _upload_csv( return repo._csv_client.save_rows(rows, key) # pyright: ignore[reportPrivateUsage] -def test_load_batch_concatenates_three_address_lines( +def test_load_batch_parses_address_postcode_and_reference( repo: UserAddressCsvS3Repository, ) -> None: rows = [ @@ -35,7 +36,7 @@ def test_load_batch_concatenates_three_address_lines( "Address 1": "1 High Street", "Address 2": "Flat 2", "Address 3": "Townville", - "Postcode": "sw1a 1aa", + "postcode": "sw1a 1aa", "Internal Reference": "REF-001", } ] @@ -58,7 +59,7 @@ def test_load_batch_uses_only_address_1_when_others_missing( "Address 1": "10 Cardiff Road", "Address 2": "", "Address 3": "", - "Postcode": "CF10 1AA", + "postcode": "CF10 1AA", "Internal Reference": "REF-002", } ] @@ -80,7 +81,7 @@ def test_load_batch_handles_missing_internal_reference( "Address 1": "5 Park Lane", "Address 2": "", "Address 3": "", - "Postcode": "M1 1AA", + "postcode": "M1 1AA", "Internal Reference": "", } ] @@ -94,16 +95,67 @@ def test_load_batch_handles_missing_internal_reference( assert addresses[0].internal_reference is None +def test_load_batch_captures_full_source_row( + repo: UserAddressCsvS3Repository, +) -> None: + # A raw EPC-export-shaped row: the splitter must preserve every column, + # not just the ones it parses into UserAddress fields. + row = { + "Asset Reference": "511", + "Address 1": "9 Abingdon Road Padiham Lancashire BB12 7BX", + "postcode": "BB12 7BX", + "Property Type": "House: End Terrace", + "SAP Score": "69", + } + uri = _upload_csv(repo, [row], "uploads/epc.csv") + + addresses = repo.load_batch(uri) + + assert addresses[0].source_row == row + + +def test_load_batch_raises_when_postcode_column_absent( + repo: UserAddressCsvS3Repository, +) -> None: + rows = [{"Address 1": "1 High Street", "Property Type": "Flat"}] + uri = _upload_csv(repo, rows, "uploads/no-postcode.csv") + + with pytest.raises(ValueError, match="no 'postcode' column"): + repo.load_batch(uri) + + +def test_save_batch_passes_through_all_columns_and_appends_postcode_clean( + repo: UserAddressCsvS3Repository, +) -> None: + row = { + "Asset Reference": "511", + "Address 1": "9 Abingdon Road Padiham Lancashire BB12 7BX", + "postcode": " BB12 7BX", + "Property Type": "House: End Terrace", + } + uri = _upload_csv(repo, [row], "uploads/epc.csv") + addresses = repo.load_batch(uri) + + saved_uri = repo.save_batch(addresses, "tasks/passthrough") + saved_rows = repo._csv_client.read_rows(saved_uri) # pyright: ignore[reportPrivateUsage] + + assert len(saved_rows) == 1 + saved = saved_rows[0] + # Every original column survives, byte-for-byte. + for column, value in row.items(): + assert saved[column] == value + # Plus the one appended column the downstream address2uprn stage groups on. + assert saved["postcode_clean"] == "BB127BX" + + def test_save_batch_returns_uri_under_path_prefix( repo: UserAddressCsvS3Repository, ) -> None: - from domain.addresses.user_address import UserAddress - addresses = [ UserAddress( - user_address="1 High Street, Flat 2, Townville", + user_address="1 High Street", postcode="SW1A 1AA", - internal_reference="REF-001", + source_row={"Address 1": "1 High Street", "postcode": "SW1A 1AA"}, ), ] @@ -113,59 +165,42 @@ def test_save_batch_returns_uri_under_path_prefix( assert uri.endswith(".csv") -def test_save_then_reload_round_trip_preserves_values( +def test_save_then_reload_round_trip_preserves_columns( repo: UserAddressCsvS3Repository, ) -> None: - from domain.addresses.user_address import UserAddress - - # save_batch writes the splitter's compact schema - # (user_address/postcode/internal_reference); load_batch reads the - # canonical upload schema. To round-trip through the repo we re-upload - # the saved CSV under the upload schema's column names. - original = [ - UserAddress( - user_address="1 High Street", - postcode="SW1A 1AA", - internal_reference="REF-001", - ), - UserAddress( - user_address="2 Low Street", - postcode="XY9 8ZW", - internal_reference=None, - ), - ] - - saved_uri = repo.save_batch(original, "tasks/round-trip") - - # Re-shape the saved CSV into the canonical upload schema for reload. - saved_rows = repo._csv_client.read_rows(saved_uri) # pyright: ignore[reportPrivateUsage] - upload_rows: list[dict[str, str]] = [ + rows = [ { - "Address 1": row["user_address"], - "Address 2": "", - "Address 3": "", - "Postcode": row["postcode"], - "Internal Reference": row["internal_reference"], - } - for row in saved_rows + "Address 1": "1 High Street", + "postcode": "SW1A 1AA", + "Internal Reference": "REF-001", + }, + { + "Address 1": "2 Low Street", + "postcode": "XY9 8ZW", + "Internal Reference": "", + }, ] - upload_uri = _upload_csv(repo, upload_rows, "uploads/round-trip.csv") + uri = _upload_csv(repo, rows, "uploads/round-trip.csv") + addresses = repo.load_batch(uri) - reloaded = repo.load_batch(upload_uri) + saved_uri = repo.save_batch(addresses, "tasks/round-trip") + saved_rows = repo._csv_client.read_rows(saved_uri) # pyright: ignore[reportPrivateUsage] - assert reloaded == original + # Original columns come back verbatim; postcode_clean is the only addition. + assert [ + {k: v for k, v in r.items() if k != "postcode_clean"} for r in saved_rows + ] == rows + assert [r["postcode_clean"] for r in saved_rows] == ["SW1A1AA", "XY98ZW"] def test_save_batch_uses_unique_filename_per_call( repo: UserAddressCsvS3Repository, ) -> None: - from domain.addresses.user_address import UserAddress - addresses = [ UserAddress( user_address="1 High Street", postcode="SW1A 1AA", - internal_reference="REF-001", + source_row={"Address 1": "1 High Street", "postcode": "SW1A 1AA"}, ), ] diff --git a/tests/utilities/aws_lambda/test_subtask_handler.py b/tests/utilities/aws_lambda/test_subtask_handler.py index 426b250f..771a49f8 100644 --- a/tests/utilities/aws_lambda/test_subtask_handler.py +++ b/tests/utilities/aws_lambda/test_subtask_handler.py @@ -6,6 +6,7 @@ to the wrapped function — so the handler can compose its own use-case orchestrator that shares the session. """ +import logging from collections.abc import Generator, Iterator from contextlib import contextmanager from dataclasses import dataclass @@ -13,6 +14,8 @@ from typing import Any from uuid import UUID import pytest + +_LOGGER_NAME = "utilities.aws_lambda.subtask_handler" from sqlmodel import Session, SQLModel, create_engine from domain.tasks.subtasks import SubTaskStatus @@ -142,3 +145,111 @@ def test_subtask_handler_injected_orchestrator_can_create_child_subtask( persisted_child = harness.subtasks.get(child_ids[0]) assert persisted_child.task_id == task.id assert persisted_child.status is SubTaskStatus.WAITING + + +def test_subtask_handler_logs_subtask_lifecycle_on_success( + harness: Harness, caplog: pytest.LogCaptureFixture +) -> None: + """Start and completion are logged at INFO so a successful invocation + leaves a CloudWatch breadcrumb (not just the Lambda runtime lines).""" + task, subtask = harness.orchestrator.create_task_with_subtask( + task_source="manual:test" + ) + + @subtask_handler(orchestrator_cm=harness.factory) + def handler( + body: dict[str, Any], context: Any, orchestrator: TaskOrchestrator + ) -> None: + return None + + with caplog.at_level(logging.INFO, logger=_LOGGER_NAME): + handler(_direct_event(task.id, subtask.id), context=None) + + assert f"Running subtask {subtask.id}" in caplog.text + assert f"Subtask {subtask.id} completed" in caplog.text + + +def test_subtask_handler_logs_exception_on_failure( + harness: Harness, caplog: pytest.LogCaptureFixture +) -> None: + """A failing subtask is logged at ERROR with the traceback attached, + before the exception propagates for the Lambda runtime to surface.""" + task, subtask = harness.orchestrator.create_task_with_subtask( + task_source="manual:test" + ) + + @subtask_handler(orchestrator_cm=harness.factory) + def handler( + body: dict[str, Any], context: Any, orchestrator: TaskOrchestrator + ) -> None: + raise RuntimeError("boom") + + with caplog.at_level(logging.INFO, logger=_LOGGER_NAME): + with pytest.raises(RuntimeError, match="boom"): + handler(_direct_event(task.id, subtask.id), context=None) + + failures = [r for r in caplog.records if r.levelno == logging.ERROR] + assert any( + f"Subtask {subtask.id} failed" in r.getMessage() for r in failures + ) + assert any(r.exc_info is not None for r in failures) + + +def test_subtask_handler_records_cloudwatch_url_on_subtask( + harness: Harness, monkeypatch: pytest.MonkeyPatch +) -> None: + """With the AWS Lambda runtime's log env vars present, a CloudWatch deep + link is built and persisted on the SubTask.""" + monkeypatch.setenv("AWS_REGION", "eu-west-2") + monkeypatch.setenv( + "AWS_LAMBDA_LOG_GROUP_NAME", "/aws/lambda/postcode-splitter" + ) + monkeypatch.setenv( + "AWS_LAMBDA_LOG_STREAM_NAME", "2026/05/20/[$LATEST]abc123" + ) + task, subtask = harness.orchestrator.create_task_with_subtask( + task_source="manual:test" + ) + + @subtask_handler(orchestrator_cm=harness.factory) + def handler( + body: dict[str, Any], context: Any, orchestrator: TaskOrchestrator + ) -> None: + return None + + handler(_direct_event(task.id, subtask.id), context=None) + + saved_url = harness.subtasks.get(subtask.id).cloud_logs_url + assert saved_url is not None + assert saved_url.startswith( + "https://eu-west-2.console.aws.amazon.com/cloudwatch/home" + ) + # Log group / stream are console-encoded ("/" -> "$252F"). + assert "$252Faws$252Flambda$252Fpostcode-splitter" in saved_url + assert "$255B$2524LATEST$255D" in saved_url + + +def test_subtask_handler_leaves_cloudwatch_url_unset_outside_lambda( + harness: Harness, monkeypatch: pytest.MonkeyPatch +) -> None: + """Outside a real Lambda (e.g. the local RIE) the runtime log env vars + are absent, so cloud_logs_url is left unset rather than storing junk.""" + for var in ( + "AWS_REGION", + "AWS_LAMBDA_LOG_GROUP_NAME", + "AWS_LAMBDA_LOG_STREAM_NAME", + ): + monkeypatch.delenv(var, raising=False) + task, subtask = harness.orchestrator.create_task_with_subtask( + task_source="manual:test" + ) + + @subtask_handler(orchestrator_cm=harness.factory) + def handler( + body: dict[str, Any], context: Any, orchestrator: TaskOrchestrator + ) -> None: + return None + + handler(_direct_event(task.id, subtask.id), context=None) + + assert harness.subtasks.get(subtask.id).cloud_logs_url is None diff --git a/utilities/aws_lambda/subtask_handler.py b/utilities/aws_lambda/subtask_handler.py index 5ad5f6e1..40f116ad 100644 --- a/utilities/aws_lambda/subtask_handler.py +++ b/utilities/aws_lambda/subtask_handler.py @@ -1,18 +1,32 @@ """@subtask_handler decorator for Lambdas that operate on existing SubTasks. Translates an AWS Lambda invocation (SQS-shaped or direct) into -TaskOrchestrator.run_subtask(...) calls. +TaskOrchestrator.run_subtask(...) calls, emitting an INFO log line for each +subtask's start and completion and a logged exception on failure. Those lines +land in CloudWatch via the Lambda runtime's stdout/stderr capture. + +Each subtask also records ``cloud_logs_url`` -- a deep link to this +invocation's CloudWatch log stream -- so an operator can jump from a SubTask +row straight to its logs. It is built from the environment variables the AWS +Lambda runtime sets, so it is populated only on real Lambda invocations and +left unset under the local RIE (which does not export them). """ import json +import logging +import os from contextlib import AbstractContextManager from functools import wraps from typing import Any, Callable, Optional, cast +from urllib.parse import quote from utilities.aws_lambda.default_orchestrator import default_orchestrator from utilities.aws_lambda.subtask_trigger_body import SubtaskTriggerBody from orchestration.task_orchestrator import TaskOrchestrator +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + OrchestratorCM = Callable[[], AbstractContextManager[TaskOrchestrator]] @@ -33,14 +47,26 @@ def subtask_handler( def decorator(func: Callable[..., Any]) -> Callable[..., Any]: @wraps(func) def wrapper(event: dict[str, Any], context: Any) -> None: + cloud_logs_url = _cloudwatch_url() with factory() as orchestrator: for record in _records(event): body = _parse_body(record) trigger = SubtaskTriggerBody.model_validate(body) - orchestrator.run_subtask( - trigger.sub_task_id, - work=lambda body=body, o=orchestrator: func(body, context, o), - ) + logger.info("Running subtask %s", trigger.sub_task_id) + try: + orchestrator.run_subtask( + trigger.sub_task_id, + work=lambda body=body, o=orchestrator: func( + body, context, o + ), + cloud_logs_url=cloud_logs_url, + ) + except Exception: + logger.exception( + "Subtask %s failed", trigger.sub_task_id + ) + raise + logger.info("Subtask %s completed", trigger.sub_task_id) return wrapper @@ -65,3 +91,34 @@ def _records(event: dict[str, Any]) -> list[dict[str, Any]]: if isinstance(raw_records, list): return [r for r in cast(list[Any], raw_records) if isinstance(r, dict)] return [event] + + +def _console_encode(value: str) -> str: + """Encode a value for a CloudWatch console deep link. + + The console expects URL-encoding with the percent signs themselves + re-encoded as ``$25`` -- e.g. ``/`` becomes ``%2F`` becomes ``$252F``. + """ + return quote(value, safe="").replace("%", "$25") + + +def _cloudwatch_url() -> Optional[str]: + """Build a CloudWatch console URL for this invocation's log stream. + + Sourced entirely from the environment variables the AWS Lambda runtime + sets -- ``AWS_REGION``, ``AWS_LAMBDA_LOG_GROUP_NAME`` and + ``AWS_LAMBDA_LOG_STREAM_NAME``. Returns None when any is absent, which is + the case outside a real Lambda (the local RIE does not export them) -- so + ``SubTask.cloud_logs_url`` is left unset rather than storing a link that + points nowhere. + """ + region = os.environ.get("AWS_REGION") + log_group = os.environ.get("AWS_LAMBDA_LOG_GROUP_NAME") + log_stream = os.environ.get("AWS_LAMBDA_LOG_STREAM_NAME") + if not (region and log_group and log_stream): + return None + return ( + f"https://{region}.console.aws.amazon.com/cloudwatch/home" + f"?region={region}#logsV2:log-groups/log-group/" + f"{_console_encode(log_group)}/log-events/{_console_encode(log_stream)}" + ) From 8bb90a5aa5beb495de799c481c2faa7899e6c5de Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 20 May 2026 12:57:03 +0000 Subject: [PATCH 10/22] sanitisation of postcode --- backend/bulk_address2uprn_combiner/main.py | 14 +----- backend/ordnanceSurvey/main.py | 12 +---- domain/addresses/postcode_batching.py | 8 ++-- domain/addresses/user_address.py | 20 ++++---- domain/postcode.py | 40 ++++++++++++++++ domain/postcodes/__init__.py | 0 domain/postcodes/sanitise.py | 23 --------- .../user_address_csv_s3_repository.py | 5 +- .../user_address/user_address_repository.py | 4 +- .../addresses/test_postcode_batching.py | 17 ++++--- tests/domain/addresses/test_user_address.py | 48 +++++++++++-------- tests/domain/postcodes/__init__.py | 0 tests/domain/postcodes/test_sanitise.py | 28 ----------- tests/domain/test_postcode.py | 48 +++++++++++++++++++ .../test_user_address_csv_s3_repository.py | 11 +++-- 15 files changed, 153 insertions(+), 125 deletions(-) create mode 100644 domain/postcode.py delete mode 100644 domain/postcodes/__init__.py delete mode 100644 domain/postcodes/sanitise.py delete mode 100644 tests/domain/postcodes/__init__.py delete mode 100644 tests/domain/postcodes/test_sanitise.py create mode 100644 tests/domain/test_postcode.py diff --git a/backend/bulk_address2uprn_combiner/main.py b/backend/bulk_address2uprn_combiner/main.py index 37136e52..44f0b3f9 100644 --- a/backend/bulk_address2uprn_combiner/main.py +++ b/backend/bulk_address2uprn_combiner/main.py @@ -2,7 +2,7 @@ import os import boto3 import pandas as pd from io import BytesIO -from typing import Any, Optional +from typing import Any from uuid import UUID from datetime import datetime, timezone @@ -12,7 +12,6 @@ from backend.app.db.functions.bulk_address_uploads_functions import ( set_combined_output_s3_uri, set_combining_status, ) -from orchestration.task_orchestrator import TaskOrchestrator logger = setup_logger() @@ -36,16 +35,7 @@ def download_csv(s3_client, bucket: str, key: str) -> pd.DataFrame: @subtask_handler() -def handler( - body: dict[str, Any], - context: Any, - orchestrator: Optional[TaskOrchestrator] = None, -) -> str: - # `orchestrator` is injected by the new utilities.aws_lambda.subtask_handler - # decorator; unused here but accepted so the contract is uniform across - # callers (see issue #1103). - del orchestrator - +def handler(body: dict[str, Any], context: Any) -> str: task_id_str: str = body.get("task_id", "") if not task_id_str: diff --git a/backend/ordnanceSurvey/main.py b/backend/ordnanceSurvey/main.py index 18c4e2f2..6e82b468 100644 --- a/backend/ordnanceSurvey/main.py +++ b/backend/ordnanceSurvey/main.py @@ -16,7 +16,6 @@ from backend.ordnanceSurvey.helpers import ( os_places_results_to_dataframe, ) from backend.app.config import get_settings -from orchestration.task_orchestrator import TaskOrchestrator from sqlalchemy import select from datetime import datetime import uuid @@ -106,16 +105,7 @@ def save_results_to_s3( @subtask_handler() # This assumes task_id and subtask_id is defined in event.Records.body -def handler( - body: dict[str, Any], - context: Any, - orchestrator: Optional[TaskOrchestrator] = None, - local: bool = False, -) -> None: - # `orchestrator` is injected by the new utilities.aws_lambda.subtask_handler - # decorator; unused here but accepted so the contract is uniform across - # callers (see issue #1103). - del orchestrator +def handler(body: dict[str, Any], context: Any, local: bool = False) -> None: # delete this line after test # local = True diff --git a/domain/addresses/postcode_batching.py b/domain/addresses/postcode_batching.py index 209e0784..b73dc1bb 100644 --- a/domain/addresses/postcode_batching.py +++ b/domain/addresses/postcode_batching.py @@ -22,6 +22,7 @@ from __future__ import annotations from collections.abc import Iterable, Iterator from domain.addresses.user_address import UserAddress +from domain.postcode import Postcode def iter_postcode_grouped_batches( @@ -75,13 +76,14 @@ def iter_postcode_grouped_batches( def _group_by_postcode_in_order( addresses: Iterable[UserAddress], -) -> dict[str, list[UserAddress]]: +) -> dict[Postcode, list[UserAddress]]: """Group addresses by ``postcode`` preserving first-seen order. Python dicts retain insertion order since 3.7, so a plain dict suffices - for the same effect as pandas ``groupby(..., sort=False)``. + for the same effect as pandas ``groupby(..., sort=False)``. ``Postcode`` + is a frozen value object, hence hashable and usable as the dict key. """ - groups: dict[str, list[UserAddress]] = {} + groups: dict[Postcode, list[UserAddress]] = {} for address in addresses: groups.setdefault(address.postcode, []).append(address) return groups diff --git a/domain/addresses/user_address.py b/domain/addresses/user_address.py index 120a3659..672b2c54 100644 --- a/domain/addresses/user_address.py +++ b/domain/addresses/user_address.py @@ -1,9 +1,9 @@ """The :class:`UserAddress` value object. A frozen dataclass capturing the splitter's domain entity: the raw input -address line, a sanitised postcode, and an optional internal reference from -the customer dataset. Postcode sanitisation runs in ``__post_init__`` so no -caller can construct an instance with an un-normalised postcode. +address line, a :class:`~domain.postcode.Postcode`, and an optional internal +reference from the customer dataset. The postcode is a value object that is +canonical by construction, so no caller can hold an un-normalised postcode. """ from __future__ import annotations @@ -11,7 +11,7 @@ from __future__ import annotations from dataclasses import dataclass, field from typing import Optional -from domain.postcodes.sanitise import sanitise_postcode +from domain.postcode import Postcode def _empty_source_row() -> dict[str, str]: @@ -25,9 +25,9 @@ class UserAddress: Attributes: user_address: The free-text address string as supplied upstream. - postcode: The postcode; always stored in canonical form - (uppercased, whitespace stripped). Sanitisation is enforced by - :meth:`__post_init__`. + postcode: The postcode as a :class:`~domain.postcode.Postcode` value + object -- canonical (uppercased, whitespace stripped) by + construction. internal_reference: Optional customer-side identifier preserved for traceability through the matching pipeline. source_row: The complete original CSV row this address was parsed @@ -39,12 +39,8 @@ class UserAddress: """ user_address: str - postcode: str + postcode: Postcode internal_reference: Optional[str] = None source_row: dict[str, str] = field( default_factory=_empty_source_row, compare=False ) - - def __post_init__(self) -> None: - # Frozen dataclass: bypass the descriptor with object.__setattr__. - object.__setattr__(self, "postcode", sanitise_postcode(self.postcode)) diff --git a/domain/postcode.py b/domain/postcode.py new file mode 100644 index 00000000..514e1a39 --- /dev/null +++ b/domain/postcode.py @@ -0,0 +1,40 @@ +"""The :class:`Postcode` value object. + +A frozen value object that owns postcode sanitisation. Constructing a +``Postcode`` always yields the canonical form -- uppercase with all +whitespace removed -- so no part of the domain can hold an un-normalised +postcode. This matches the legacy splitter's +``df["postcode"].str.upper().str.replace(" ", "")``. + +``Postcode`` is the single sanitisation point: anywhere a postcode crosses a +domain boundary it should be wrapped in one, and ``str(postcode)`` gives the +canonical string back for serialisation. +""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class Postcode: + """A postcode held in canonical form. + + The ``value`` passed to the constructor is sanitised eagerly in + :meth:`__post_init__` -- uppercased, with all whitespace (spaces, tabs, + newlines) removed -- so every ``Postcode`` instance is canonical by + construction. Two postcodes that differ only in surface whitespace or + case therefore compare equal. + + Attributes: + value: The canonical postcode string (e.g. ``"SW1A1AA"``). + """ + + value: str + + def __post_init__(self) -> None: + # Frozen dataclass: bypass the descriptor with object.__setattr__. + object.__setattr__(self, "value", "".join(self.value.split()).upper()) + + def __str__(self) -> str: + return self.value diff --git a/domain/postcodes/__init__.py b/domain/postcodes/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/domain/postcodes/sanitise.py b/domain/postcodes/sanitise.py deleted file mode 100644 index 94b0dcf7..00000000 --- a/domain/postcodes/sanitise.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Canonical postcode sanitisation for the domain layer. - -The legacy postcode_splitter normalises postcodes inline with -``df["postcode"].str.upper().str.replace(" ", "")``. This module promotes -that operation to a pure, reusable function so the same canonical form is -applied wherever a postcode crosses a domain boundary -- including -:class:`domain.addresses.user_address.UserAddress` construction and future -migrations. -""" - -from __future__ import annotations - - -def sanitise_postcode(s: str) -> str: - """Return the canonical form of a postcode. - - The canonical form is uppercase with all whitespace removed. This matches - the legacy splitter's ``str.upper().str.replace(" ", "")`` for the - overwhelmingly common case of space-separated postcodes (e.g. ``"sw1a 1aa"`` - becomes ``"SW1A1AA"``) while also tolerating tabs/newlines that can creep - in from CSV ingestion. - """ - return "".join(s.split()).upper() diff --git a/repositories/user_address/user_address_csv_s3_repository.py b/repositories/user_address/user_address_csv_s3_repository.py index 7cd10bac..2432d8e9 100644 --- a/repositories/user_address/user_address_csv_s3_repository.py +++ b/repositories/user_address/user_address_csv_s3_repository.py @@ -20,6 +20,7 @@ from datetime import datetime, timezone from typing import Optional from domain.addresses.user_address import UserAddress +from domain.postcode import Postcode from infrastructure.csv_s3_client import CsvS3Client from repositories.user_address.user_address_repository import UserAddressRepository @@ -77,7 +78,7 @@ class UserAddressCsvS3Repository(UserAddressRepository): addresses.append( UserAddress( user_address=user_address, - postcode=postcode, + postcode=Postcode(postcode), internal_reference=internal_reference, source_row=row, ) @@ -96,7 +97,7 @@ class UserAddressCsvS3Repository(UserAddressRepository): Returns the full ``s3://bucket/key`` URI. """ rows: list[dict[str, str]] = [ - {**addr.source_row, _POSTCODE_CLEAN_COLUMN: addr.postcode} + {**addr.source_row, _POSTCODE_CLEAN_COLUMN: str(addr.postcode)} for addr in addresses ] filename = ( diff --git a/repositories/user_address/user_address_repository.py b/repositories/user_address/user_address_repository.py index d8c12855..ab9b6671 100644 --- a/repositories/user_address/user_address_repository.py +++ b/repositories/user_address/user_address_repository.py @@ -17,8 +17,8 @@ class UserAddressRepository(ABC): Implementations choose the underlying storage (S3 CSV, Postgres, in-memory, ...) but must preserve the canonical column semantics: - the address text, postcode (sanitised by ``UserAddress.__post_init__``), - and an optional internal reference. + the address text, postcode (a :class:`~domain.postcode.Postcode` value + object), and an optional internal reference. """ @abstractmethod diff --git a/tests/domain/addresses/test_postcode_batching.py b/tests/domain/addresses/test_postcode_batching.py index 2dac46cc..6e52b581 100644 --- a/tests/domain/addresses/test_postcode_batching.py +++ b/tests/domain/addresses/test_postcode_batching.py @@ -2,12 +2,15 @@ import pytest from domain.addresses.postcode_batching import iter_postcode_grouped_batches from domain.addresses.user_address import UserAddress +from domain.postcode import Postcode def _addrs(postcode: str, n: int) -> list[UserAddress]: """Build ``n`` addresses sharing a postcode, with distinct address lines.""" return [ - UserAddress(user_address=f"{i} {postcode} Street", postcode=postcode) + UserAddress( + user_address=f"{i} {postcode} Street", postcode=Postcode(postcode) + ) for i in range(n) ] @@ -38,8 +41,8 @@ def test_flush_on_overflow_before_adding_next_postcode() -> None: addrs = _addrs("AA1 1AA", 3) + _addrs("BB2 2BB", 3) batches = list(iter_postcode_grouped_batches(addrs, max_batch_size=5)) assert len(batches) == 2 - assert [a.postcode for a in batches[0]] == ["AA11AA"] * 3 - assert [a.postcode for a in batches[1]] == ["BB22BB"] * 3 + assert [str(a.postcode) for a in batches[0]] == ["AA11AA"] * 3 + assert [str(a.postcode) for a in batches[1]] == ["BB22BB"] * 3 def test_single_postcode_group_exceeding_cap_is_dispatched_whole() -> None: @@ -61,9 +64,9 @@ def test_oversize_group_flushes_existing_buffer_first() -> None: iter_postcode_grouped_batches(small + big + tail, max_batch_size=5) ) assert len(batches) == 3 - assert [a.postcode for a in batches[0]] == ["AA11AA", "AA11AA"] - assert [a.postcode for a in batches[1]] == ["BB22BB"] * 7 - assert [a.postcode for a in batches[2]] == ["CC33CC"] + assert [str(a.postcode) for a in batches[0]] == ["AA11AA", "AA11AA"] + assert [str(a.postcode) for a in batches[1]] == ["BB22BB"] * 7 + assert [str(a.postcode) for a in batches[2]] == ["CC33CC"] def test_final_flush_yields_remaining_buffer() -> None: @@ -80,7 +83,7 @@ def test_postcode_grouping_preserves_first_seen_order() -> None: b1, b2 = _addrs("AA1 1AA", 2) batches = list(iter_postcode_grouped_batches([a1, b1, a2, b2])) assert len(batches) == 1 - assert [a.postcode for a in batches[0]] == [ + assert [str(a.postcode) for a in batches[0]] == [ "ZZ99ZZ", "ZZ99ZZ", "AA11AA", diff --git a/tests/domain/addresses/test_user_address.py b/tests/domain/addresses/test_user_address.py index 4d8322da..fa44ad61 100644 --- a/tests/domain/addresses/test_user_address.py +++ b/tests/domain/addresses/test_user_address.py @@ -3,69 +3,77 @@ import dataclasses import pytest from domain.addresses.user_address import UserAddress +from domain.postcode import Postcode -def test_user_address_sanitises_postcode_on_construction() -> None: - addr = UserAddress(user_address="1 The Street", postcode="sw1a 1aa") - assert addr.postcode == "SW1A1AA" +def test_user_address_holds_postcode_value_object() -> None: + addr = UserAddress(user_address="1 The Street", postcode=Postcode("sw1a 1aa")) + assert addr.postcode == Postcode("SW1A1AA") def test_user_address_preserves_user_address_verbatim() -> None: # The free-text user_address string is intentionally NOT normalised -- - # only the postcode is canonicalised at the boundary. - addr = UserAddress(user_address=" 1 The Street ", postcode="sw1a 1aa") + # only the postcode is canonicalised, and that happens inside Postcode. + addr = UserAddress( + user_address=" 1 The Street ", postcode=Postcode("SW1A1AA") + ) assert addr.user_address == " 1 The Street " def test_user_address_internal_reference_defaults_to_none() -> None: - addr = UserAddress(user_address="1 The Street", postcode="SW1A1AA") + addr = UserAddress(user_address="1 The Street", postcode=Postcode("SW1A1AA")) assert addr.internal_reference is None def test_user_address_internal_reference_accepted() -> None: addr = UserAddress( user_address="1 The Street", - postcode="SW1A1AA", + postcode=Postcode("SW1A1AA"), internal_reference="cust-42", ) assert addr.internal_reference == "cust-42" def test_user_address_is_frozen() -> None: - addr = UserAddress(user_address="1 The Street", postcode="SW1A1AA") + addr = UserAddress(user_address="1 The Street", postcode=Postcode("SW1A1AA")) with pytest.raises(dataclasses.FrozenInstanceError): - addr.postcode = "OTHER" # type: ignore[misc] + addr.postcode = Postcode("OTHER") # type: ignore[misc] -def test_user_address_equality_uses_sanitised_postcode() -> None: - # Two instances constructed with different surface forms of the same - # postcode must compare equal because sanitisation runs eagerly. - a = UserAddress(user_address="1 The Street", postcode="sw1a 1aa") - b = UserAddress(user_address="1 The Street", postcode="SW1A1AA") +def test_user_address_equality_uses_canonical_postcode() -> None: + # Postcode sanitises eagerly, so addresses built from different surface + # forms of the same postcode compare equal. + a = UserAddress(user_address="1 The Street", postcode=Postcode("sw1a 1aa")) + b = UserAddress(user_address="1 The Street", postcode=Postcode("SW1A1AA")) assert a == b def test_user_address_source_row_defaults_to_empty_dict() -> None: - addr = UserAddress(user_address="1 The Street", postcode="SW1A1AA") + addr = UserAddress(user_address="1 The Street", postcode=Postcode("SW1A1AA")) assert addr.source_row == {} def test_user_address_carries_source_row() -> None: row = {"Address 1": "1 The Street", "postcode": "SW1A 1AA", "SAP Score": "72"} addr = UserAddress( - user_address="1 The Street", postcode="SW1A 1AA", source_row=row + user_address="1 The Street", + postcode=Postcode("SW1A 1AA"), + source_row=row, ) assert addr.source_row == row def test_user_address_equality_ignores_source_row() -> None: # source_row is excluded from equality (and hashing): identity stays - # defined by the parsed fields, so two addresses parsed from rows with - # different incidental columns still compare equal. + # defined by the parsed fields. a = UserAddress( - user_address="1 The Street", postcode="SW1A1AA", source_row={"x": "1"} + user_address="1 The Street", + postcode=Postcode("SW1A1AA"), + source_row={"x": "1"}, ) b = UserAddress( - user_address="1 The Street", postcode="SW1A1AA", source_row={"y": "2"} + user_address="1 The Street", + postcode=Postcode("SW1A1AA"), + source_row={"y": "2"}, ) assert a == b diff --git a/tests/domain/postcodes/__init__.py b/tests/domain/postcodes/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/domain/postcodes/test_sanitise.py b/tests/domain/postcodes/test_sanitise.py deleted file mode 100644 index edd1679c..00000000 --- a/tests/domain/postcodes/test_sanitise.py +++ /dev/null @@ -1,28 +0,0 @@ -from domain.postcodes.sanitise import sanitise_postcode - - -def test_sanitise_uppercases() -> None: - assert sanitise_postcode("sw1a1aa") == "SW1A1AA" - - -def test_sanitise_strips_internal_spaces() -> None: - assert sanitise_postcode("sw1a 1aa") == "SW1A1AA" - - -def test_sanitise_strips_leading_and_trailing_whitespace() -> None: - assert sanitise_postcode(" sw1a 1aa ") == "SW1A1AA" - - -def test_sanitise_strips_tabs_and_newlines() -> None: - # CSV ingestion occasionally introduces stray whitespace characters; the - # canonical form must absorb them just like literal spaces. - assert sanitise_postcode("sw1a\t1aa\n") == "SW1A1AA" - - -def test_sanitise_already_canonical_is_idempotent() -> None: - assert sanitise_postcode("SW1A1AA") == "SW1A1AA" - assert sanitise_postcode(sanitise_postcode("sw1a 1aa")) == "SW1A1AA" - - -def test_sanitise_empty_string() -> None: - assert sanitise_postcode("") == "" diff --git a/tests/domain/test_postcode.py b/tests/domain/test_postcode.py new file mode 100644 index 00000000..89d5cdc8 --- /dev/null +++ b/tests/domain/test_postcode.py @@ -0,0 +1,48 @@ +import dataclasses + +import pytest + +from domain.postcode import Postcode + + +def test_postcode_uppercases() -> None: + assert Postcode("sw1a1aa").value == "SW1A1AA" + + +def test_postcode_strips_internal_spaces() -> None: + assert Postcode("sw1a 1aa").value == "SW1A1AA" + + +def test_postcode_strips_leading_and_trailing_whitespace() -> None: + assert Postcode(" sw1a 1aa ").value == "SW1A1AA" + + +def test_postcode_strips_tabs_and_newlines() -> None: + # CSV ingestion occasionally introduces stray whitespace characters; the + # canonical form must absorb them just like literal spaces. + assert Postcode("sw1a\t1aa\n").value == "SW1A1AA" + + +def test_postcode_construction_is_idempotent() -> None: + once = Postcode("sw1a 1aa") + assert Postcode(once.value).value == "SW1A1AA" + + +def test_postcode_empty_string() -> None: + assert Postcode("").value == "" + + +def test_postcode_str_returns_canonical_value() -> None: + assert str(Postcode("sw1a 1aa")) == "SW1A1AA" + + +def test_postcode_equality_ignores_surface_form() -> None: + # Differing case / whitespace sanitise to the same canonical value, so + # the value objects compare equal. + assert Postcode("sw1a 1aa") == Postcode("SW1A1AA") + + +def test_postcode_is_frozen() -> None: + postcode = Postcode("SW1A1AA") + with pytest.raises(dataclasses.FrozenInstanceError): + postcode.value = "OTHER" # type: ignore[misc] diff --git a/tests/repositories/user_address/test_user_address_csv_s3_repository.py b/tests/repositories/user_address/test_user_address_csv_s3_repository.py index 48733b55..c1acee32 100644 --- a/tests/repositories/user_address/test_user_address_csv_s3_repository.py +++ b/tests/repositories/user_address/test_user_address_csv_s3_repository.py @@ -4,6 +4,7 @@ import pytest from moto import mock_aws from domain.addresses.user_address import UserAddress +from domain.postcode import Postcode from infrastructure.csv_s3_client import CsvS3Client from repositories.user_address.user_address_csv_s3_repository import ( UserAddressCsvS3Repository, @@ -47,7 +48,7 @@ def test_load_batch_parses_address_postcode_and_reference( assert len(addresses) == 1 address = addresses[0] assert address.user_address == "1 High Street, Flat 2, Townville" - assert address.postcode == "SW1A1AA" + assert address.postcode == Postcode("SW1A1AA") assert address.internal_reference == "REF-001" @@ -69,7 +70,7 @@ def test_load_batch_uses_only_address_1_when_others_missing( assert len(addresses) == 1 assert addresses[0].user_address == "10 Cardiff Road" - assert addresses[0].postcode == "CF101AA" + assert addresses[0].postcode == Postcode("CF101AA") assert addresses[0].internal_reference == "REF-002" @@ -91,7 +92,7 @@ def test_load_batch_handles_missing_internal_reference( assert len(addresses) == 1 assert addresses[0].user_address == "5 Park Lane" - assert addresses[0].postcode == "M11AA" + assert addresses[0].postcode == Postcode("M11AA") assert addresses[0].internal_reference is None @@ -154,7 +155,7 @@ def test_save_batch_returns_uri_under_path_prefix( addresses = [ UserAddress( user_address="1 High Street", - postcode="SW1A 1AA", + postcode=Postcode("SW1A 1AA"), source_row={"Address 1": "1 High Street", "postcode": "SW1A 1AA"}, ), ] @@ -199,7 +200,7 @@ def test_save_batch_uses_unique_filename_per_call( addresses = [ UserAddress( user_address="1 High Street", - postcode="SW1A 1AA", + postcode=Postcode("SW1A 1AA"), source_row={"Address 1": "1 High Street", "postcode": "SW1A 1AA"}, ), ] From d0cf3d14ad5116d0b2926aceb23d642408ca71bc Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 20 May 2026 13:21:11 +0000 Subject: [PATCH 11/22] get rid of comments --- applications/postcode_splitter/handler.py | 18 ------- .../local_handler/invoke_local_lambda.py | 9 ---- .../postcode_splitter_trigger_body.py | 21 --------- domain/addresses/postcode_batching.py | 38 --------------- domain/addresses/user_address.py | 30 +----------- domain/postcode.py | 25 ---------- infrastructure/address2uprn_queue_client.py | 7 --- infrastructure/csv_s3_client.py | 18 ------- infrastructure/s3_client.py | 9 ---- infrastructure/s3_uri.py | 18 ------- infrastructure/sqs_client.py | 8 ---- .../postcode_splitter_orchestrator.py | 34 -------------- orchestration/task_orchestrator.py | 6 --- .../user_address_csv_s3_repository.py | 47 ------------------- .../user_address/user_address_repository.py | 19 +------- .../addresses/test_postcode_batching.py | 1 - tests/infrastructure/__init__.py | 7 --- tests/infrastructure/conftest.py | 4 -- .../test_postcode_splitter_orchestrator.py | 10 ---- tests/repositories/user_address/conftest.py | 4 -- .../aws_lambda/test_subtask_handler.py | 25 +--------- utilities/aws_lambda/subtask_handler.py | 24 +--------- 22 files changed, 6 insertions(+), 376 deletions(-) diff --git a/applications/postcode_splitter/handler.py b/applications/postcode_splitter/handler.py index 005227a9..9fb3ca6a 100644 --- a/applications/postcode_splitter/handler.py +++ b/applications/postcode_splitter/handler.py @@ -1,15 +1,3 @@ -"""Lambda entrypoint for the postcode splitter slice. - -The :func:`handler` function is decorated with ``@subtask_handler()`` so the -decorator owns the parent ``SubTask`` lifecycle (start/complete/fail) and -injects the decorator-owned :class:`TaskOrchestrator` as the third positional -argument. The handler itself does only two things: - -1. Build a :class:`PostcodeSplitterOrchestrator` from env-driven config. -2. Delegate to ``split_and_dispatch`` and return its result so it lands in - ``SubTask.outputs["result"]``. -""" - from __future__ import annotations import os @@ -34,12 +22,6 @@ from utilities.aws_lambda.subtask_handler import subtask_handler def handler( body: dict[str, Any], context: Any, task_orchestrator: TaskOrchestrator ) -> dict[str, list[str]]: - """Validate the trigger body, build the splitter, dispatch children. - - Reads ``S3_BUCKET_NAME`` and ``ADDRESS2UPRN_QUEUE_URL`` from the - environment to construct the typed S3/SQS clients. The return value - lands in ``SubTask.outputs["result"]`` via the decorator. - """ trigger = PostcodeSplitterTriggerBody.model_validate(body) bucket = os.environ["S3_BUCKET_NAME"] diff --git a/applications/postcode_splitter/local_handler/invoke_local_lambda.py b/applications/postcode_splitter/local_handler/invoke_local_lambda.py index c0ca89ec..21fa9b9e 100755 --- a/applications/postcode_splitter/local_handler/invoke_local_lambda.py +++ b/applications/postcode_splitter/local_handler/invoke_local_lambda.py @@ -1,13 +1,4 @@ #!/usr/bin/env python3 -"""POST a single SQS-shaped event at the locally-running splitter Lambda. - -The container built by docker-compose runs the AWS Lambda Runtime Interface -Emulator, which accepts invocations on the URL below. Replace the three -placeholder values with a real parent Task id, the splitter's own SubTask id -(both must already exist in the Postgres pointed at by .env.local), and the -s3://... URI of an uploaded address CSV. -""" - import json import requests diff --git a/applications/postcode_splitter/postcode_splitter_trigger_body.py b/applications/postcode_splitter/postcode_splitter_trigger_body.py index bc983abc..4c33f4a4 100644 --- a/applications/postcode_splitter/postcode_splitter_trigger_body.py +++ b/applications/postcode_splitter/postcode_splitter_trigger_body.py @@ -1,30 +1,9 @@ -"""Trigger payload model for the postcode splitter Lambda. - -The decorator (``@subtask_handler``) already validates ``task_id`` and -``sub_task_id`` via :class:`SubtaskTriggerBody`; this model layers on the -splitter-specific ``s3_uri`` field while keeping ``extra="allow"`` so any -upstream-passthrough keys (e.g. ``portfolio_id``) survive untouched. -""" - from uuid import UUID from pydantic import BaseModel, ConfigDict class PostcodeSplitterTriggerBody(BaseModel): - """Validated body for the postcode splitter Lambda. - - Attributes: - task_id: Parent ``Task`` id; used as the ``task_id`` input on each - child ``SubTask`` and as the ``parent_task_id`` on the fan-out - SQS messages. - sub_task_id: The splitter's own ``SubTask`` id; used as the path - segment under ``ara_postcode_splitter_batches/{task_id}/{...}`` - so per-invocation outputs cannot collide. - s3_uri: ``s3://bucket/key`` URI of the uploaded address CSV the - splitter must read. - """ - model_config = ConfigDict(extra="allow") task_id: UUID diff --git a/domain/addresses/postcode_batching.py b/domain/addresses/postcode_batching.py index b73dc1bb..44e4d967 100644 --- a/domain/addresses/postcode_batching.py +++ b/domain/addresses/postcode_batching.py @@ -1,22 +1,3 @@ -"""Pure-Python postcode-grouped batching. - -This module preserves the batching invariants from the legacy postcode -splitter (``backend/postcode_splitter/main.py``) without touching pandas, -S3, or SQS: - - * Addresses are grouped by **Postcode** in *insertion order* -- the first - Postcode seen produces the first group. - * A Postcode group is never split across two batches. - * If a single Postcode group is larger than ``max_batch_size``, it is - flushed as its own oversize batch (any buffered groups go out first, - untouched). - * Adding a group that would push the buffer past ``max_batch_size`` first - flushes the existing buffer, then starts a new buffer with the group. - * Whatever remains in the buffer after the input is exhausted is flushed - as the final batch. - * Empty input yields no batches. -""" - from __future__ import annotations from collections.abc import Iterable, Iterator @@ -30,19 +11,6 @@ def iter_postcode_grouped_batches( *, max_batch_size: int = 500, ) -> Iterator[list[UserAddress]]: - """Yield batches of ``UserAddress`` grouped by Postcode. - - Args: - addresses: An iterable of :class:`UserAddress`. Order is preserved - within each Postcode group, and groups are yielded in the order - their first member was seen. - max_batch_size: The soft upper bound on batch size, in number of - addresses. A single Postcode group larger than this cap is - dispatched whole (the cap is never used to split a group). - - Yields: - Lists of ``UserAddress``. Each list is non-empty. - """ if max_batch_size < 1: raise ValueError("max_batch_size must be >= 1") @@ -77,12 +45,6 @@ def iter_postcode_grouped_batches( def _group_by_postcode_in_order( addresses: Iterable[UserAddress], ) -> dict[Postcode, list[UserAddress]]: - """Group addresses by ``postcode`` preserving first-seen order. - - Python dicts retain insertion order since 3.7, so a plain dict suffices - for the same effect as pandas ``groupby(..., sort=False)``. ``Postcode`` - is a frozen value object, hence hashable and usable as the dict key. - """ groups: dict[Postcode, list[UserAddress]] = {} for address in addresses: groups.setdefault(address.postcode, []).append(address) diff --git a/domain/addresses/user_address.py b/domain/addresses/user_address.py index 672b2c54..9a28751b 100644 --- a/domain/addresses/user_address.py +++ b/domain/addresses/user_address.py @@ -1,11 +1,3 @@ -"""The :class:`UserAddress` value object. - -A frozen dataclass capturing the splitter's domain entity: the raw input -address line, a :class:`~domain.postcode.Postcode`, and an optional internal -reference from the customer dataset. The postcode is a value object that is -canonical by construction, so no caller can hold an un-normalised postcode. -""" - from __future__ import annotations from dataclasses import dataclass, field @@ -15,32 +7,12 @@ from domain.postcode import Postcode def _empty_source_row() -> dict[str, str]: - """Typed default factory for :attr:`UserAddress.source_row`.""" return {} @dataclass(frozen=True) class UserAddress: - """A user-supplied address paired with its canonical postcode. - - Attributes: - user_address: The free-text address string as supplied upstream. - postcode: The postcode as a :class:`~domain.postcode.Postcode` value - object -- canonical (uppercased, whitespace stripped) by - construction. - internal_reference: Optional customer-side identifier preserved for - traceability through the matching pipeline. - source_row: The complete original CSV row this address was parsed - from, column name -> cell value. The splitter is a pass-through - router: it groups rows by postcode but must not drop the other - columns the downstream address2uprn stage relies on, so the raw - row travels alongside the parsed fields. Excluded from equality - and hashing -- identity stays defined by the parsed fields above. - """ - user_address: str postcode: Postcode internal_reference: Optional[str] = None - source_row: dict[str, str] = field( - default_factory=_empty_source_row, compare=False - ) + source_row: dict[str, str] = field(default_factory=_empty_source_row, compare=False) diff --git a/domain/postcode.py b/domain/postcode.py index 514e1a39..8e4e7c79 100644 --- a/domain/postcode.py +++ b/domain/postcode.py @@ -1,16 +1,3 @@ -"""The :class:`Postcode` value object. - -A frozen value object that owns postcode sanitisation. Constructing a -``Postcode`` always yields the canonical form -- uppercase with all -whitespace removed -- so no part of the domain can hold an un-normalised -postcode. This matches the legacy splitter's -``df["postcode"].str.upper().str.replace(" ", "")``. - -``Postcode`` is the single sanitisation point: anywhere a postcode crosses a -domain boundary it should be wrapped in one, and ``str(postcode)`` gives the -canonical string back for serialisation. -""" - from __future__ import annotations from dataclasses import dataclass @@ -18,18 +5,6 @@ from dataclasses import dataclass @dataclass(frozen=True) class Postcode: - """A postcode held in canonical form. - - The ``value`` passed to the constructor is sanitised eagerly in - :meth:`__post_init__` -- uppercased, with all whitespace (spaces, tabs, - newlines) removed -- so every ``Postcode`` instance is canonical by - construction. Two postcodes that differ only in surface whitespace or - case therefore compare equal. - - Attributes: - value: The canonical postcode string (e.g. ``"SW1A1AA"``). - """ - value: str def __post_init__(self) -> None: diff --git a/infrastructure/address2uprn_queue_client.py b/infrastructure/address2uprn_queue_client.py index d81e2dd1..314e981f 100644 --- a/infrastructure/address2uprn_queue_client.py +++ b/infrastructure/address2uprn_queue_client.py @@ -4,12 +4,6 @@ from infrastructure.sqs_client import SqsClient class Address2UprnQueueClient(SqsClient): - """SQS client that publishes Address-to-UPRN fan-out messages. - - The body shape is fixed by the downstream consumer: - ``{"task_id": str, "sub_task_id": str, "s3_uri": str}`` - """ - def publish( self, *, @@ -17,7 +11,6 @@ class Address2UprnQueueClient(SqsClient): child_subtask_id: UUID, s3_uri: str, ) -> str: - """Send a typed Address-to-UPRN message. Returns the SQS ``MessageId``.""" return self.send( { "task_id": str(parent_task_id), diff --git a/infrastructure/csv_s3_client.py b/infrastructure/csv_s3_client.py index 0a576b81..055d1ce3 100644 --- a/infrastructure/csv_s3_client.py +++ b/infrastructure/csv_s3_client.py @@ -6,20 +6,7 @@ from infrastructure.s3_uri import parse_s3_uri class CsvS3Client(S3Client): - """:class:`S3Client` subclass that round-trips CSV row dictionaries. - - Rows are represented as ``list[dict[str, str]]`` — the same shape used by - :func:`csv.DictReader`/``DictWriter`` — which keeps the API trivially - compatible with existing CSV helpers in ``utils/s3.py``. - """ - def read_rows(self, s3_uri: str) -> list[dict[str, str]]: - """Fetch the object at ``s3_uri`` and decode it as a CSV. - - The bucket portion of the URI is validated against this client's - configured bucket so cross-bucket reads fail loudly rather than - silently fetching from the wrong place. - """ bucket, key = parse_s3_uri(s3_uri) if bucket != self.bucket: raise ValueError( @@ -31,11 +18,6 @@ class CsvS3Client(S3Client): return [dict(row) for row in reader] def save_rows(self, rows: list[dict[str, str]], key: str) -> str: - """Serialise ``rows`` to CSV under ``key`` and return the ``s3://`` URI. - - An empty ``rows`` list is rejected because we cannot otherwise infer - a header row. - """ if not rows: raise ValueError("Cannot save an empty rows list: header is unknown") buffer = StringIO() diff --git a/infrastructure/s3_client.py b/infrastructure/s3_client.py index 9e772881..a789fcc2 100644 --- a/infrastructure/s3_client.py +++ b/infrastructure/s3_client.py @@ -2,13 +2,6 @@ from typing import Any class S3Client: - """Thin typed wrapper around a boto3 S3 client bound to a single bucket. - - The class is deliberately small: it exposes only the byte-level - operations needed by the wider infrastructure layer. Serialisation - (CSV, JSON, etc.) lives in subclasses such as :class:`CsvS3Client`. - """ - def __init__(self, boto_s3_client: Any, bucket: str) -> None: self._client = boto_s3_client self._bucket = bucket @@ -18,7 +11,6 @@ class S3Client: return self._bucket def get_object(self, key: str) -> bytes: - """Return the raw bytes stored at ``key`` in this client's bucket.""" response: dict[str, Any] = self._client.get_object( Bucket=self._bucket, Key=key ) @@ -26,6 +18,5 @@ class S3Client: return body def put_object(self, key: str, body: bytes) -> str: - """Write ``body`` to ``key`` and return the canonical ``s3://`` URI.""" self._client.put_object(Bucket=self._bucket, Key=key, Body=body) return f"s3://{self._bucket}/{key}" diff --git a/infrastructure/s3_uri.py b/infrastructure/s3_uri.py index bf97100e..1dd5d967 100644 --- a/infrastructure/s3_uri.py +++ b/infrastructure/s3_uri.py @@ -1,25 +1,7 @@ -"""Parse S3 URIs into ``(bucket, key)`` pairs. - -A pure-stdlib helper for the infrastructure layer. It deliberately pulls in -neither pandas, boto3, nor the legacy ``utils`` package, so slim Lambda images -that only need URI parsing do not drag the wider data stack along. - -Two input shapes are supported: - -* canonical S3 URIs --- ``s3://bucket/key`` -* AWS S3 console URLs --- ``https://.../s3/object/bucket?prefix=key`` -""" - from urllib.parse import unquote def parse_s3_uri(s3_uri: str) -> tuple[str, str]: - """Return the ``(bucket, key)`` pair addressed by ``s3_uri``. - - Raises: - ValueError: if ``s3_uri`` is neither a well-formed ``s3://`` URI nor - an AWS console URL carrying a ``prefix`` query parameter. - """ if s3_uri.startswith("s3://"): parts = s3_uri[len("s3://") :].split("/", 1) if len(parts) < 2 or not parts[0] or not parts[1]: diff --git a/infrastructure/sqs_client.py b/infrastructure/sqs_client.py index fb053680..6fe8dd2e 100644 --- a/infrastructure/sqs_client.py +++ b/infrastructure/sqs_client.py @@ -3,13 +3,6 @@ from typing import Any class SqsClient: - """Thin typed wrapper around a boto3 SQS client bound to one queue URL. - - The body is JSON-serialised here so callers can pass plain dictionaries - instead of constructing message strings themselves. Typed publish - helpers (e.g. :class:`Address2UprnQueueClient`) build on this contract. - """ - def __init__(self, boto_sqs_client: Any, queue_url: str) -> None: self._client = boto_sqs_client self._queue_url = queue_url @@ -19,7 +12,6 @@ class SqsClient: return self._queue_url def send(self, body: dict[str, Any]) -> str: - """JSON-serialise ``body`` and send it. Returns the SQS ``MessageId``.""" response: dict[str, Any] = self._client.send_message( QueueUrl=self._queue_url, MessageBody=json.dumps(body), diff --git a/orchestration/postcode_splitter_orchestrator.py b/orchestration/postcode_splitter_orchestrator.py index 6afa2538..36f4b515 100644 --- a/orchestration/postcode_splitter_orchestrator.py +++ b/orchestration/postcode_splitter_orchestrator.py @@ -1,15 +1,3 @@ -"""Use-case orchestrator for the postcode splitter Lambda. - -Wires the slice-1 domain (``iter_postcode_grouped_batches``), the slice-3 -``UserAddressRepository``, the slice-2 ``Address2UprnQueueClient``, and the -slice-4 ``TaskOrchestrator.create_child_subtask`` primitive together. - -``split_and_dispatch`` loads the input batch, groups it into per-postcode -chunks, writes each chunk back to S3 under a deterministic prefix, creates a -WAITING child ``SubTask`` for it, and publishes the address-to-UPRN fan-out -message that downstream consumers pick up. -""" - from __future__ import annotations from uuid import UUID @@ -21,15 +9,6 @@ from repositories.user_address.user_address_repository import UserAddressReposit class PostcodeSplitterOrchestrator: - """Split an uploaded address batch into postcode-grouped child SubTasks. - - The orchestrator owns the algorithm; the IO collaborators - (:class:`UserAddressRepository`, :class:`Address2UprnQueueClient`) and - the :class:`TaskOrchestrator` lifecycle primitive are injected so the - same wiring can be exercised against moto/SQLite in tests and against - real AWS in the Lambda entrypoint. - """ - def __init__( self, task_orchestrator: TaskOrchestrator, @@ -49,19 +28,6 @@ class PostcodeSplitterOrchestrator: parent_subtask_id: UUID, input_s3_uri: str, ) -> list[UUID]: - """Split ``input_s3_uri`` into postcode batches and dispatch each. - - For each yielded batch: - - 1. Persist it under - ``ara_postcode_splitter_batches/{parent_task_id}/{parent_subtask_id}``. - 2. Create a WAITING child ``SubTask`` with - ``inputs={"task_id": str(parent_task_id), "s3_uri": batch_uri}``. - 3. Publish an ``address2UPRN`` SQS message referencing the new child. - - Returns: - The list of child ``SubTask`` ids, in dispatch order. - """ addresses = self._user_address_repo.load_batch(input_s3_uri) path_prefix = ( f"ara_postcode_splitter_batches/{parent_task_id}/{parent_subtask_id}" diff --git a/orchestration/task_orchestrator.py b/orchestration/task_orchestrator.py index 82d95db1..ebb71a32 100644 --- a/orchestration/task_orchestrator.py +++ b/orchestration/task_orchestrator.py @@ -54,12 +54,6 @@ class TaskOrchestrator: *, inputs: Optional[dict[str, Any]] = None, ) -> SubTask: - """Add a new WAITING SubTask under an existing parent Task. - - Skips `_cascade`: a new WAITING child against an IN_PROGRESS parent - leaves the parent's status unchanged per `Task.recalculate_from_subtasks`, - so calling it here would be a no-op. - """ subtask = SubTask.create(task_id=parent_task_id, inputs=inputs) self._subtasks.create(subtask) return subtask diff --git a/repositories/user_address/user_address_csv_s3_repository.py b/repositories/user_address/user_address_csv_s3_repository.py index 2432d8e9..9b93b638 100644 --- a/repositories/user_address/user_address_csv_s3_repository.py +++ b/repositories/user_address/user_address_csv_s3_repository.py @@ -1,18 +1,3 @@ -"""CSV-on-S3 adapter for :class:`UserAddressRepository`. - -Reads upload CSVs that carry a ``postcode`` column (plus optional -``Address 1``/``Address 2``/``Address 3`` and ``Internal Reference``), and -writes batch CSVs that pass *every* original column through unchanged with -one column appended -- ``postcode_clean`` (uppercase, whitespace-stripped) -- -which the downstream address2uprn stage groups on. - -The splitter is a pass-through router: it must not reshape or drop columns, -because address2uprn has not been migrated and still consumes the legacy -splitter's full-row output. The frontend pre-applies the user's column -mapping at upload time, so this adapter does NOT consult any -``BulkAddressUpload.column_mapping``. -""" - from __future__ import annotations import uuid @@ -31,33 +16,11 @@ _POSTCODE_CLEAN_COLUMN: str = "postcode_clean" class UserAddressCsvS3Repository(UserAddressRepository): - """Persist :class:`UserAddress` batches as CSV objects in S3. - - The repo owns the unique-filename-within-prefix convention - (``{ISO datetime}_{8-char uuid}.csv``); callers own the directory - hierarchy supplied as ``path_prefix``. - """ - def __init__(self, csv_client: CsvS3Client, bucket: str) -> None: self._csv_client = csv_client self._bucket = bucket def load_batch(self, s3_uri: str) -> list[UserAddress]: - """Load upload CSV rows into :class:`UserAddress` objects. - - Each row's complete column set is preserved on - :attr:`UserAddress.source_row` so :meth:`save_batch` can pass it - through untouched. The parsed convenience fields are also populated: - ``Address 1``/``Address 2``/``Address 3`` are concatenated with - ``", "`` (skipping missing/empty parts) into ``user_address``, and - ``Internal Reference`` is threaded to - :attr:`UserAddress.internal_reference` (``None`` when missing/empty). - - Raises: - ValueError: if the CSV has rows but no ``postcode`` column -- - without it the splitter cannot group, and silently emitting - empty postcodes would corrupt every downstream batch. - """ rows = self._csv_client.read_rows(s3_uri) if rows and _POSTCODE_COLUMN not in rows[0]: raise ValueError( @@ -86,16 +49,6 @@ class UserAddressCsvS3Repository(UserAddressRepository): return addresses def save_batch(self, addresses: list[UserAddress], path_prefix: str) -> str: - """Write a pass-through batch CSV under a unique key. - - Each output row is the address's original ``source_row`` with a - ``postcode_clean`` column appended (the canonical postcode the - downstream address2uprn stage groups on). No original column is - dropped or reshaped. - - The key is ``{path_prefix}/{ISO-8601 datetime}_{8-char uuid}.csv``. - Returns the full ``s3://bucket/key`` URI. - """ rows: list[dict[str, str]] = [ {**addr.source_row, _POSTCODE_CLEAN_COLUMN: str(addr.postcode)} for addr in addresses diff --git a/repositories/user_address/user_address_repository.py b/repositories/user_address/user_address_repository.py index ab9b6671..170f34dd 100644 --- a/repositories/user_address/user_address_repository.py +++ b/repositories/user_address/user_address_repository.py @@ -1,10 +1,3 @@ -"""Abstract repository for :class:`UserAddress` batches. - -Persistence-agnostic interface for loading and saving batches of -:class:`domain.addresses.user_address.UserAddress`. Concrete adapters -- -e.g. :class:`UserAddressCsvS3Repository` -- live alongside this module. -""" - from __future__ import annotations from abc import ABC, abstractmethod @@ -13,18 +6,10 @@ from domain.addresses.user_address import UserAddress class UserAddressRepository(ABC): - """Load and persist batches of :class:`UserAddress`. - - Implementations choose the underlying storage (S3 CSV, Postgres, - in-memory, ...) but must preserve the canonical column semantics: - the address text, postcode (a :class:`~domain.postcode.Postcode` value - object), and an optional internal reference. - """ - @abstractmethod def load_batch(self, s3_uri: str) -> list[UserAddress]: - """Read a batch of addresses from ``s3_uri`` and return domain objects.""" + ... @abstractmethod def save_batch(self, addresses: list[UserAddress], path_prefix: str) -> str: - """Persist ``addresses`` under ``path_prefix`` and return the URI written.""" + ... diff --git a/tests/domain/addresses/test_postcode_batching.py b/tests/domain/addresses/test_postcode_batching.py index 6e52b581..c69722ba 100644 --- a/tests/domain/addresses/test_postcode_batching.py +++ b/tests/domain/addresses/test_postcode_batching.py @@ -6,7 +6,6 @@ from domain.postcode import Postcode def _addrs(postcode: str, n: int) -> list[UserAddress]: - """Build ``n`` addresses sharing a postcode, with distinct address lines.""" return [ UserAddress( user_address=f"{i} {postcode} Street", postcode=Postcode(postcode) diff --git a/tests/infrastructure/__init__.py b/tests/infrastructure/__init__.py index 3478bda9..f5ad62d0 100644 --- a/tests/infrastructure/__init__.py +++ b/tests/infrastructure/__init__.py @@ -6,12 +6,5 @@ REGION = "us-east-1" def make_boto_client(service_name: str) -> Any: - """Construct a boto3 client typed as ``Any``. - - boto3's overloaded ``client`` signature uses ``Literal[...]`` per service - in the installed stubs, which forces every call site to satisfy - ``reportArgumentType`` and ``reportUnknownMemberType`` under strict - pyright. Centralising the cast keeps each test file clean. - """ factory: Any = boto3.client # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] return factory(service_name, region_name=REGION) diff --git a/tests/infrastructure/conftest.py b/tests/infrastructure/conftest.py index 7ed2fdd6..25c1ac3b 100644 --- a/tests/infrastructure/conftest.py +++ b/tests/infrastructure/conftest.py @@ -7,10 +7,6 @@ import pytest @pytest.fixture(autouse=True) def _aws_creds() -> Iterator[None]: # pyright: ignore[reportUnusedFunction] - """Stub AWS creds so botocore doesn't probe the host environment. - - Applied automatically to every test in ``tests/infrastructure/``. - """ keys = ( "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", diff --git a/tests/orchestration/test_postcode_splitter_orchestrator.py b/tests/orchestration/test_postcode_splitter_orchestrator.py index 79c60974..4ee2315e 100644 --- a/tests/orchestration/test_postcode_splitter_orchestrator.py +++ b/tests/orchestration/test_postcode_splitter_orchestrator.py @@ -1,13 +1,3 @@ -"""Integration test: PostcodeSplitterOrchestrator wired end-to-end. - -Combines moto S3 + moto SQS + an in-memory SQLite session for the -``TaskOrchestrator`` so the full slice-6 wiring is exercised through real -infrastructure adapters (not mocks). The fixture CSV spans three postcodes -with one oversize group, which forces both the buffer-flush-then-oversize -branch and the final-flush branch of -``iter_postcode_grouped_batches`` — three batches in total. -""" - from __future__ import annotations import json diff --git a/tests/repositories/user_address/conftest.py b/tests/repositories/user_address/conftest.py index 1859ff0a..25c1ac3b 100644 --- a/tests/repositories/user_address/conftest.py +++ b/tests/repositories/user_address/conftest.py @@ -7,10 +7,6 @@ import pytest @pytest.fixture(autouse=True) def _aws_creds() -> Iterator[None]: # pyright: ignore[reportUnusedFunction] - """Stub AWS creds so botocore doesn't probe the host environment. - - Applied automatically to every test in ``tests/repositories/user_address/``. - """ keys = ( "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", diff --git a/tests/utilities/aws_lambda/test_subtask_handler.py b/tests/utilities/aws_lambda/test_subtask_handler.py index 771a49f8..9cf68f28 100644 --- a/tests/utilities/aws_lambda/test_subtask_handler.py +++ b/tests/utilities/aws_lambda/test_subtask_handler.py @@ -1,11 +1,3 @@ -"""Tests for the @subtask_handler decorator. - -Covers the contract that the decorator owns the parent SubTask lifecycle and -injects the decorator-owned TaskOrchestrator as a third positional argument -to the wrapped function — so the handler can compose its own use-case -orchestrator that shares the session. -""" - import logging from collections.abc import Generator, Iterator from contextlib import contextmanager @@ -14,8 +6,6 @@ from typing import Any from uuid import UUID import pytest - -_LOGGER_NAME = "utilities.aws_lambda.subtask_handler" from sqlmodel import Session, SQLModel, create_engine from domain.tasks.subtasks import SubTaskStatus @@ -25,6 +15,8 @@ from repositories.tasks.subtask_postgres_repository import SubTaskPostgresReposi from repositories.tasks.task_postgres_repository import TaskPostgresRepository from utilities.aws_lambda.subtask_handler import subtask_handler +_LOGGER_NAME = "utilities.aws_lambda.subtask_handler" + @dataclass class Harness: @@ -58,8 +50,6 @@ def _direct_event(task_id: UUID, subtask_id: UUID) -> dict[str, Any]: def test_subtask_handler_injects_orchestrator_as_third_positional_argument( harness: Harness, ) -> None: - """The wrapped function receives the decorator-owned TaskOrchestrator - so it can share the session with its own use-case orchestrator.""" _, subtask = harness.orchestrator.create_task_with_subtask( task_source="manual:test" ) @@ -123,9 +113,6 @@ def test_subtask_handler_marks_parent_failed_and_reraises_on_error( def test_subtask_handler_injected_orchestrator_can_create_child_subtask( harness: Harness, ) -> None: - """Smoke check the share-the-session promise: the injected orchestrator - is the same one the decorator owns, so a handler can use it to create - child SubTasks under the same session.""" task, subtask = harness.orchestrator.create_task_with_subtask( task_source="manual:test" ) @@ -150,8 +137,6 @@ def test_subtask_handler_injected_orchestrator_can_create_child_subtask( def test_subtask_handler_logs_subtask_lifecycle_on_success( harness: Harness, caplog: pytest.LogCaptureFixture ) -> None: - """Start and completion are logged at INFO so a successful invocation - leaves a CloudWatch breadcrumb (not just the Lambda runtime lines).""" task, subtask = harness.orchestrator.create_task_with_subtask( task_source="manual:test" ) @@ -172,8 +157,6 @@ def test_subtask_handler_logs_subtask_lifecycle_on_success( def test_subtask_handler_logs_exception_on_failure( harness: Harness, caplog: pytest.LogCaptureFixture ) -> None: - """A failing subtask is logged at ERROR with the traceback attached, - before the exception propagates for the Lambda runtime to surface.""" task, subtask = harness.orchestrator.create_task_with_subtask( task_source="manual:test" ) @@ -198,8 +181,6 @@ def test_subtask_handler_logs_exception_on_failure( def test_subtask_handler_records_cloudwatch_url_on_subtask( harness: Harness, monkeypatch: pytest.MonkeyPatch ) -> None: - """With the AWS Lambda runtime's log env vars present, a CloudWatch deep - link is built and persisted on the SubTask.""" monkeypatch.setenv("AWS_REGION", "eu-west-2") monkeypatch.setenv( "AWS_LAMBDA_LOG_GROUP_NAME", "/aws/lambda/postcode-splitter" @@ -232,8 +213,6 @@ def test_subtask_handler_records_cloudwatch_url_on_subtask( def test_subtask_handler_leaves_cloudwatch_url_unset_outside_lambda( harness: Harness, monkeypatch: pytest.MonkeyPatch ) -> None: - """Outside a real Lambda (e.g. the local RIE) the runtime log env vars - are absent, so cloud_logs_url is left unset rather than storing junk.""" for var in ( "AWS_REGION", "AWS_LAMBDA_LOG_GROUP_NAME", diff --git a/utilities/aws_lambda/subtask_handler.py b/utilities/aws_lambda/subtask_handler.py index 40f116ad..592ffebf 100644 --- a/utilities/aws_lambda/subtask_handler.py +++ b/utilities/aws_lambda/subtask_handler.py @@ -1,15 +1,7 @@ """@subtask_handler decorator for Lambdas that operate on existing SubTasks. Translates an AWS Lambda invocation (SQS-shaped or direct) into -TaskOrchestrator.run_subtask(...) calls, emitting an INFO log line for each -subtask's start and completion and a logged exception on failure. Those lines -land in CloudWatch via the Lambda runtime's stdout/stderr capture. - -Each subtask also records ``cloud_logs_url`` -- a deep link to this -invocation's CloudWatch log stream -- so an operator can jump from a SubTask -row straight to its logs. It is built from the environment variables the AWS -Lambda runtime sets, so it is populated only on real Lambda invocations and -left unset under the local RIE (which does not export them). +TaskOrchestrator.run_subtask(...) calls. """ import json @@ -94,24 +86,10 @@ def _records(event: dict[str, Any]) -> list[dict[str, Any]]: def _console_encode(value: str) -> str: - """Encode a value for a CloudWatch console deep link. - - The console expects URL-encoding with the percent signs themselves - re-encoded as ``$25`` -- e.g. ``/`` becomes ``%2F`` becomes ``$252F``. - """ return quote(value, safe="").replace("%", "$25") def _cloudwatch_url() -> Optional[str]: - """Build a CloudWatch console URL for this invocation's log stream. - - Sourced entirely from the environment variables the AWS Lambda runtime - sets -- ``AWS_REGION``, ``AWS_LAMBDA_LOG_GROUP_NAME`` and - ``AWS_LAMBDA_LOG_STREAM_NAME``. Returns None when any is absent, which is - the case outside a real Lambda (the local RIE does not export them) -- so - ``SubTask.cloud_logs_url`` is left unset rather than storing a link that - points nowhere. - """ region = os.environ.get("AWS_REGION") log_group = os.environ.get("AWS_LAMBDA_LOG_GROUP_NAME") log_stream = os.environ.get("AWS_LAMBDA_LOG_STREAM_NAME") From dc159e0b457d8e72e0e64dc931d21a9ae9dfed39 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 20 May 2026 14:00:19 +0000 Subject: [PATCH 12/22] tests framework completed --- .../user_address_csv_s3_repository.py | 3 ++ .../user_address/user_address_repository.py | 6 +-- tests/conftest.py | 48 +++++++++++++++++++ .../addresses/test_postcode_batching.py | 23 +++++++++ tests/domain/addresses/test_user_address.py | 19 ++++++++ tests/domain/tasks/test_subtasks.py | 20 ++++++++ tests/domain/tasks/test_tasks.py | 31 +++++++++++- tests/domain/test_postcode.py | 11 +++++ .../test_address2uprn_queue_client.py | 6 +++ tests/infrastructure/test_csv_s3_client.py | 8 ++++ tests/infrastructure/test_s3_client.py | 5 ++ tests/infrastructure/test_s3_uri.py | 8 ++++ tests/infrastructure/test_sqs_client.py | 6 +++ .../test_postcode_splitter_orchestrator.py | 23 ++++++--- tests/orchestration/test_task_orchestrator.py | 34 +++++++++++-- .../test_subtask_postgres_repository.py | 47 ++++++++++++------ .../postgres/test_task_postgres_repository.py | 25 ++++++---- .../test_user_address_csv_s3_repository.py | 26 ++++++++++ .../aws_lambda/test_subtask_handler.py | 31 ++++++++++-- 19 files changed, 336 insertions(+), 44 deletions(-) create mode 100644 tests/conftest.py diff --git a/repositories/user_address/user_address_csv_s3_repository.py b/repositories/user_address/user_address_csv_s3_repository.py index 9b93b638..058fd5a5 100644 --- a/repositories/user_address/user_address_csv_s3_repository.py +++ b/repositories/user_address/user_address_csv_s3_repository.py @@ -53,6 +53,9 @@ class UserAddressCsvS3Repository(UserAddressRepository): {**addr.source_row, _POSTCODE_CLEAN_COLUMN: str(addr.postcode)} for addr in addresses ] + + # TODO: [New Starter Task] file_name generation can be standardised + # and also easier to read, test for future implementation. Buiild that! filename = ( f"{datetime.now(timezone.utc).isoformat()}_{uuid.uuid4().hex[:8]}.csv" ) diff --git a/repositories/user_address/user_address_repository.py b/repositories/user_address/user_address_repository.py index 170f34dd..b2c0f866 100644 --- a/repositories/user_address/user_address_repository.py +++ b/repositories/user_address/user_address_repository.py @@ -7,9 +7,7 @@ from domain.addresses.user_address import UserAddress class UserAddressRepository(ABC): @abstractmethod - def load_batch(self, s3_uri: str) -> list[UserAddress]: - ... + def load_batch(self, s3_uri: str) -> list[UserAddress]: ... @abstractmethod - def save_batch(self, addresses: list[UserAddress], path_prefix: str) -> str: - ... + def save_batch(self, addresses: list[UserAddress], path_prefix: str) -> str: ... diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..0a246372 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,48 @@ +"""Shared pytest fixtures for the ``tests/`` tree. + +Provides an ephemeral PostgreSQL engine for tests that exercise SQLModel +repositories. PostgreSQL has no true in-memory mode; ``pytest-postgresql`` +starts a real, throwaway server in a temp directory (the process is started +once per session and a fresh database is created/dropped per test). That is +the closest equivalent to "in-memory" and matches production behaviour far +better than SQLite (enums, JSONB, constraint semantics, etc.). +""" + +from __future__ import annotations + +import glob +from collections.abc import Iterator +from typing import Any + +import pytest +from psycopg import Connection +from pytest_postgresql import factories +from sqlalchemy import Engine +from sqlmodel import SQLModel, create_engine + +# Importing the SQLModel row modules registers their tables on +# SQLModel.metadata so ``create_all`` builds the full schema. Imports look +# unused; they aren't. + + +# pg_ctl ships under a versioned path and is not on PATH in the dev container. +_PG_CTL = next(iter(sorted(glob.glob("/usr/lib/postgresql/*/bin/pg_ctl"))), "pg_ctl") + +postgresql_proc = factories.postgresql_proc( + executable=_PG_CTL +) # pyright: ignore[reportUnknownMemberType] +postgresql = factories.postgresql("postgresql_proc") + + +@pytest.fixture +def db_engine(postgresql: Connection[Any]) -> Iterator[Engine]: + """A SQLModel engine bound to a fresh, ephemeral PostgreSQL database.""" + info = postgresql.info + url = f"postgresql+psycopg://{info.user}:@{info.host}:{info.port}/{info.dbname}" + engine = create_engine(url) + SQLModel.metadata.create_all(engine) + try: + yield engine + finally: + SQLModel.metadata.drop_all(engine) + engine.dispose() diff --git a/tests/domain/addresses/test_postcode_batching.py b/tests/domain/addresses/test_postcode_batching.py index c69722ba..8ffcf1b5 100644 --- a/tests/domain/addresses/test_postcode_batching.py +++ b/tests/domain/addresses/test_postcode_batching.py @@ -15,12 +15,16 @@ def _addrs(postcode: str, n: int) -> list[UserAddress]: def test_empty_input_yields_no_batches() -> None: + # act / assert assert list(iter_postcode_grouped_batches([])) == [] def test_single_batch_under_cap() -> None: + # arrange addrs = _addrs("AA1 1AA", 3) + _addrs("BB2 2BB", 2) + # act batches = list(iter_postcode_grouped_batches(addrs, max_batch_size=500)) + # assert assert len(batches) == 1 assert batches[0] == addrs @@ -28,8 +32,11 @@ def test_single_batch_under_cap() -> None: def test_multiple_postcodes_packed_into_one_batch_up_to_cap() -> None: # Two groups whose total exactly equals the cap pack into a single # batch -- no premature flush. + # arrange addrs = _addrs("AA1 1AA", 3) + _addrs("BB2 2BB", 2) + # act batches = list(iter_postcode_grouped_batches(addrs, max_batch_size=5)) + # assert assert len(batches) == 1 assert len(batches[0]) == 5 @@ -37,8 +44,11 @@ def test_multiple_postcodes_packed_into_one_batch_up_to_cap() -> None: def test_flush_on_overflow_before_adding_next_postcode() -> None: # Cap is 5. First group fills 3 slots; second group of 3 would overflow, # so the buffer is flushed first and the next group starts a fresh batch. + # arrange addrs = _addrs("AA1 1AA", 3) + _addrs("BB2 2BB", 3) + # act batches = list(iter_postcode_grouped_batches(addrs, max_batch_size=5)) + # assert assert len(batches) == 2 assert [str(a.postcode) for a in batches[0]] == ["AA11AA"] * 3 assert [str(a.postcode) for a in batches[1]] == ["BB22BB"] * 3 @@ -47,8 +57,11 @@ def test_flush_on_overflow_before_adding_next_postcode() -> None: def test_single_postcode_group_exceeding_cap_is_dispatched_whole() -> None: # An oversize single-postcode group goes out as one batch larger than # the cap -- the cap never splits a postcode. + # arrange addrs = _addrs("AA1 1AA", 7) + # act batches = list(iter_postcode_grouped_batches(addrs, max_batch_size=5)) + # assert assert len(batches) == 1 assert len(batches[0]) == 7 @@ -56,12 +69,15 @@ def test_single_postcode_group_exceeding_cap_is_dispatched_whole() -> None: def test_oversize_group_flushes_existing_buffer_first() -> None: # Mirrors the legacy ``if buffer: flush`` branch when an oversize group # is encountered: buffered work must not be lost or interleaved. + # arrange small = _addrs("AA1 1AA", 2) big = _addrs("BB2 2BB", 7) tail = _addrs("CC3 3CC", 1) + # act batches = list( iter_postcode_grouped_batches(small + big + tail, max_batch_size=5) ) + # assert assert len(batches) == 3 assert [str(a.postcode) for a in batches[0]] == ["AA11AA", "AA11AA"] assert [str(a.postcode) for a in batches[1]] == ["BB22BB"] * 7 @@ -70,17 +86,23 @@ def test_oversize_group_flushes_existing_buffer_first() -> None: def test_final_flush_yields_remaining_buffer() -> None: # No overflow ever happens, but the trailing buffer must still come out. + # arrange addrs = _addrs("AA1 1AA", 2) + _addrs("BB2 2BB", 2) + # act batches = list(iter_postcode_grouped_batches(addrs, max_batch_size=500)) + # assert assert batches == [addrs] def test_postcode_grouping_preserves_first_seen_order() -> None: # Interleaved input must still group by postcode and emit in first-seen # order -- never alphabetical. + # arrange a1, a2 = _addrs("ZZ9 9ZZ", 2) b1, b2 = _addrs("AA1 1AA", 2) + # act batches = list(iter_postcode_grouped_batches([a1, b1, a2, b2])) + # assert assert len(batches) == 1 assert [str(a.postcode) for a in batches[0]] == [ "ZZ99ZZ", @@ -91,5 +113,6 @@ def test_postcode_grouping_preserves_first_seen_order() -> None: def test_invalid_max_batch_size_raises() -> None: + # act / assert with pytest.raises(ValueError, match="max_batch_size"): list(iter_postcode_grouped_batches([], max_batch_size=0)) diff --git a/tests/domain/addresses/test_user_address.py b/tests/domain/addresses/test_user_address.py index fa44ad61..8d092df3 100644 --- a/tests/domain/addresses/test_user_address.py +++ b/tests/domain/addresses/test_user_address.py @@ -7,35 +7,45 @@ from domain.postcode import Postcode def test_user_address_holds_postcode_value_object() -> None: + # act addr = UserAddress(user_address="1 The Street", postcode=Postcode("sw1a 1aa")) + # assert assert addr.postcode == Postcode("SW1A1AA") def test_user_address_preserves_user_address_verbatim() -> None: # The free-text user_address string is intentionally NOT normalised -- # only the postcode is canonicalised, and that happens inside Postcode. + # act addr = UserAddress( user_address=" 1 The Street ", postcode=Postcode("SW1A1AA") ) + # assert assert addr.user_address == " 1 The Street " def test_user_address_internal_reference_defaults_to_none() -> None: + # act addr = UserAddress(user_address="1 The Street", postcode=Postcode("SW1A1AA")) + # assert assert addr.internal_reference is None def test_user_address_internal_reference_accepted() -> None: + # act addr = UserAddress( user_address="1 The Street", postcode=Postcode("SW1A1AA"), internal_reference="cust-42", ) + # assert assert addr.internal_reference == "cust-42" def test_user_address_is_frozen() -> None: + # arrange addr = UserAddress(user_address="1 The Street", postcode=Postcode("SW1A1AA")) + # act / assert with pytest.raises(dataclasses.FrozenInstanceError): addr.postcode = Postcode("OTHER") # type: ignore[misc] @@ -43,29 +53,37 @@ def test_user_address_is_frozen() -> None: def test_user_address_equality_uses_canonical_postcode() -> None: # Postcode sanitises eagerly, so addresses built from different surface # forms of the same postcode compare equal. + # arrange a = UserAddress(user_address="1 The Street", postcode=Postcode("sw1a 1aa")) b = UserAddress(user_address="1 The Street", postcode=Postcode("SW1A1AA")) + # act / assert assert a == b def test_user_address_source_row_defaults_to_empty_dict() -> None: + # act addr = UserAddress(user_address="1 The Street", postcode=Postcode("SW1A1AA")) + # assert assert addr.source_row == {} def test_user_address_carries_source_row() -> None: + # arrange row = {"Address 1": "1 The Street", "postcode": "SW1A 1AA", "SAP Score": "72"} + # act addr = UserAddress( user_address="1 The Street", postcode=Postcode("SW1A 1AA"), source_row=row, ) + # assert assert addr.source_row == row def test_user_address_equality_ignores_source_row() -> None: # source_row is excluded from equality (and hashing): identity stays # defined by the parsed fields. + # arrange a = UserAddress( user_address="1 The Street", postcode=Postcode("SW1A1AA"), @@ -76,4 +94,5 @@ def test_user_address_equality_ignores_source_row() -> None: postcode=Postcode("SW1A1AA"), source_row={"y": "2"}, ) + # act / assert assert a == b diff --git a/tests/domain/tasks/test_subtasks.py b/tests/domain/tasks/test_subtasks.py index 2721d38f..8cee4496 100644 --- a/tests/domain/tasks/test_subtasks.py +++ b/tests/domain/tasks/test_subtasks.py @@ -6,10 +6,13 @@ from domain.tasks.subtasks import SubTask, SubTaskStatus def test_create_subtask_starts_waiting() -> None: + # arrange task_id = uuid4() + # act st = SubTask.create(task_id=task_id, inputs={"foo": "bar"}) + # assert assert st.task_id == task_id assert st.status is SubTaskStatus.WAITING assert st.inputs == {"foo": "bar"} @@ -19,57 +22,74 @@ def test_create_subtask_starts_waiting() -> None: def test_start_transitions_to_in_progress_and_sets_cloud_logs_url() -> None: + # arrange st = SubTask.create(task_id=uuid4()) + # act st.start(cloud_logs_url="https://example/log") + # assert assert st.status is SubTaskStatus.IN_PROGRESS assert st.cloud_logs_url == "https://example/log" assert st.job_started is not None def test_start_is_idempotent_from_in_progress() -> None: + # arrange st = SubTask.create(task_id=uuid4()) st.start() first_start = st.job_started + # act st.start(cloud_logs_url="https://other") + # assert assert st.status is SubTaskStatus.IN_PROGRESS assert st.job_started == first_start # not overwritten assert st.cloud_logs_url == "https://other" def test_start_rejects_from_terminal_status() -> None: + # arrange st = SubTask.create(task_id=uuid4()) st.complete() + # act / assert with pytest.raises(ValueError): st.start() def test_complete_marks_outputs_and_job_completed() -> None: + # arrange st = SubTask.create(task_id=uuid4()) st.start() + # act st.complete({"uprn": "123"}) + # assert assert st.status is SubTaskStatus.COMPLETE assert st.outputs == {"result": {"uprn": "123"}} assert st.job_completed is not None def test_complete_without_result_leaves_outputs_unset() -> None: + # arrange st = SubTask.create(task_id=uuid4()) + # act st.complete() + # assert assert st.outputs is None def test_fail_records_error_in_outputs() -> None: + # arrange st = SubTask.create(task_id=uuid4()) err = RuntimeError("boom") + # act st.fail(err) + # assert assert st.status is SubTaskStatus.FAILED assert st.outputs == {"error": "boom"} assert st.job_completed is not None diff --git a/tests/domain/tasks/test_tasks.py b/tests/domain/tasks/test_tasks.py index f30c0aa1..ba82412b 100644 --- a/tests/domain/tasks/test_tasks.py +++ b/tests/domain/tasks/test_tasks.py @@ -5,12 +5,12 @@ from domain.tasks.tasks import Source, Task, TaskStatus def test_create_task_starts_waiting() -> None: - # Arrange / Act + # arrange / act t = Task.create( task_source="manual:test", source=Source.PORTFOLIO, source_id="abc-123" ) - # Assert + # assert assert t.status is TaskStatus.WAITING assert t.source is Source.PORTFOLIO assert t.source_id == "abc-123" @@ -19,86 +19,113 @@ def test_create_task_starts_waiting() -> None: def test_create_task_rejects_blank_task_source() -> None: + # act / assert with pytest.raises(ValueError, match="task_source"): Task.create(task_source=" ") def test_start_transitions_to_in_progress() -> None: + # arrange t = Task.create(task_source="manual:test") + # act t.start() + # assert assert t.status is TaskStatus.IN_PROGRESS def test_complete_marks_job_completed() -> None: + # arrange t = Task.create(task_source="manual:test") t.start() + # act t.complete() + # assert assert t.status is TaskStatus.COMPLETE assert t.job_completed is not None def test_fail_marks_job_completed() -> None: + # arrange t = Task.create(task_source="manual:test") + # act t.fail() + # assert assert t.status is TaskStatus.FAILED assert t.job_completed is not None def test_start_rejects_from_terminal_status() -> None: + # arrange t = Task.create(task_source="manual:test") t.complete() + # act / assert with pytest.raises(ValueError): t.start() def test_recalculate_with_empty_statuses_is_noop() -> None: + # arrange t = Task.create(task_source="manual:test") original_status = t.status original_completed = t.job_completed + # act t.recalculate_from_subtasks([]) + # assert assert t.status is original_status assert t.job_completed is original_completed def test_recalculate_all_waiting_keeps_waiting() -> None: + # arrange t = Task.create(task_source="manual:test") t.start() # task moved to IN_PROGRESS earlier t.complete() # then COMPLETE, with job_completed set + # act t.recalculate_from_subtasks([SubTaskStatus.WAITING, SubTaskStatus.WAITING]) + # assert assert t.status is TaskStatus.WAITING assert t.job_completed is None def test_recalculate_any_in_progress_marks_in_progress() -> None: + # arrange t = Task.create(task_source="manual:test") + # act t.recalculate_from_subtasks( [SubTaskStatus.WAITING, SubTaskStatus.IN_PROGRESS, SubTaskStatus.COMPLETE] ) + # assert assert t.status is TaskStatus.IN_PROGRESS assert t.job_completed is None def test_recalculate_all_complete_marks_complete() -> None: + # arrange t = Task.create(task_source="manual:test") + # act t.recalculate_from_subtasks([SubTaskStatus.COMPLETE, SubTaskStatus.COMPLETE]) + # assert assert t.status is TaskStatus.COMPLETE assert t.job_completed is not None def test_recalculate_any_failed_marks_failed_even_with_others() -> None: + # arrange t = Task.create(task_source="manual:test") + # act t.recalculate_from_subtasks( [SubTaskStatus.IN_PROGRESS, SubTaskStatus.COMPLETE, SubTaskStatus.FAILED] ) + # assert assert t.status is TaskStatus.FAILED assert t.job_completed is not None diff --git a/tests/domain/test_postcode.py b/tests/domain/test_postcode.py index 89d5cdc8..f7ce9015 100644 --- a/tests/domain/test_postcode.py +++ b/tests/domain/test_postcode.py @@ -6,43 +6,54 @@ from domain.postcode import Postcode def test_postcode_uppercases() -> None: + # act / assert assert Postcode("sw1a1aa").value == "SW1A1AA" def test_postcode_strips_internal_spaces() -> None: + # act / assert assert Postcode("sw1a 1aa").value == "SW1A1AA" def test_postcode_strips_leading_and_trailing_whitespace() -> None: + # act / assert assert Postcode(" sw1a 1aa ").value == "SW1A1AA" def test_postcode_strips_tabs_and_newlines() -> None: # CSV ingestion occasionally introduces stray whitespace characters; the # canonical form must absorb them just like literal spaces. + # act / assert assert Postcode("sw1a\t1aa\n").value == "SW1A1AA" def test_postcode_construction_is_idempotent() -> None: + # arrange once = Postcode("sw1a 1aa") + # act / assert assert Postcode(once.value).value == "SW1A1AA" def test_postcode_empty_string() -> None: + # act / assert assert Postcode("").value == "" def test_postcode_str_returns_canonical_value() -> None: + # act / assert assert str(Postcode("sw1a 1aa")) == "SW1A1AA" def test_postcode_equality_ignores_surface_form() -> None: # Differing case / whitespace sanitise to the same canonical value, so # the value objects compare equal. + # act / assert assert Postcode("sw1a 1aa") == Postcode("SW1A1AA") def test_postcode_is_frozen() -> None: + # arrange postcode = Postcode("SW1A1AA") + # act / assert with pytest.raises(dataclasses.FrozenInstanceError): postcode.value = "OTHER" # type: ignore[misc] diff --git a/tests/infrastructure/test_address2uprn_queue_client.py b/tests/infrastructure/test_address2uprn_queue_client.py index b4114742..c8e89ece 100644 --- a/tests/infrastructure/test_address2uprn_queue_client.py +++ b/tests/infrastructure/test_address2uprn_queue_client.py @@ -28,12 +28,15 @@ def queue_setup() -> Iterator[tuple[Address2UprnQueueClient, Any, str]]: def test_publish_returns_message_id( queue_setup: tuple[Address2UprnQueueClient, Any, str], ) -> None: + # arrange client, _boto, _url = queue_setup + # act message_id = client.publish( parent_task_id=uuid4(), child_subtask_id=uuid4(), s3_uri="s3://my-bucket/path/to/chunk.csv", ) + # assert assert isinstance(message_id, str) assert message_id @@ -41,17 +44,20 @@ def test_publish_returns_message_id( def test_publish_body_uses_typed_shape( queue_setup: tuple[Address2UprnQueueClient, Any, str], ) -> None: + # arrange client, boto_client, queue_url = queue_setup parent_id = uuid4() child_id = uuid4() s3_uri = "s3://my-bucket/path/to/chunk.csv" + # act client.publish( parent_task_id=parent_id, child_subtask_id=child_id, s3_uri=s3_uri, ) + # assert received: dict[str, Any] = boto_client.receive_message( QueueUrl=queue_url, MaxNumberOfMessages=1 ) diff --git a/tests/infrastructure/test_csv_s3_client.py b/tests/infrastructure/test_csv_s3_client.py index 4b9fc199..30e27164 100644 --- a/tests/infrastructure/test_csv_s3_client.py +++ b/tests/infrastructure/test_csv_s3_client.py @@ -18,26 +18,34 @@ def csv_client() -> Iterator[CsvS3Client]: def test_save_rows_returns_s3_uri(csv_client: CsvS3Client) -> None: + # arrange rows = [{"address": "1 High St", "postcode": "AB1 2CD"}] + # act uri = csv_client.save_rows(rows, "uploads/addresses.csv") + # assert assert uri == f"s3://{BUCKET}/uploads/addresses.csv" def test_round_trip_preserves_rows(csv_client: CsvS3Client) -> None: + # arrange rows = [ {"address": "1 High St", "postcode": "AB1 2CD"}, {"address": "2 Low St", "postcode": "XY9 8ZW"}, ] + # act uri = csv_client.save_rows(rows, "uploads/addresses.csv") fetched = csv_client.read_rows(uri) + # assert assert fetched == rows def test_save_rows_rejects_empty_list(csv_client: CsvS3Client) -> None: + # act / assert with pytest.raises(ValueError, match="empty"): csv_client.save_rows([], "uploads/empty.csv") def test_read_rows_rejects_wrong_bucket(csv_client: CsvS3Client) -> None: + # act / assert with pytest.raises(ValueError, match="does not match client bucket"): csv_client.read_rows("s3://other-bucket/uploads/addresses.csv") diff --git a/tests/infrastructure/test_s3_client.py b/tests/infrastructure/test_s3_client.py index 7ed4c30b..67db4f58 100644 --- a/tests/infrastructure/test_s3_client.py +++ b/tests/infrastructure/test_s3_client.py @@ -18,14 +18,19 @@ def s3_client() -> Iterator[S3Client]: def test_put_object_returns_s3_uri(s3_client: S3Client) -> None: + # act uri = s3_client.put_object("folder/data.bin", b"payload") + # assert assert uri == f"s3://{BUCKET}/folder/data.bin" def test_get_object_returns_bytes_written_by_put_object(s3_client: S3Client) -> None: + # arrange s3_client.put_object("round/trip.bin", b"hello world") + # act / assert assert s3_client.get_object("round/trip.bin") == b"hello world" def test_bucket_property_exposes_configured_bucket(s3_client: S3Client) -> None: + # act / assert assert s3_client.bucket == BUCKET diff --git a/tests/infrastructure/test_s3_uri.py b/tests/infrastructure/test_s3_uri.py index 896c5959..32fd710f 100644 --- a/tests/infrastructure/test_s3_uri.py +++ b/tests/infrastructure/test_s3_uri.py @@ -4,29 +4,37 @@ from infrastructure.s3_uri import parse_s3_uri def test_parses_simple_s3_uri() -> None: + # act / assert assert parse_s3_uri("s3://my-bucket/file.csv") == ("my-bucket", "file.csv") def test_parses_s3_uri_with_nested_key() -> None: + # act bucket, key = parse_s3_uri("s3://my-bucket/nested/path/to/file.csv") + # assert assert (bucket, key) == ("my-bucket", "nested/path/to/file.csv") def test_rejects_s3_uri_without_key() -> None: + # act / assert with pytest.raises(ValueError, match="bucket and a key"): parse_s3_uri("s3://my-bucket") def test_rejects_s3_uri_with_empty_key() -> None: + # act / assert with pytest.raises(ValueError, match="bucket and a key"): parse_s3_uri("s3://my-bucket/") def test_parses_console_url_prefix() -> None: + # arrange url = "https://eu-west-2.console.aws.amazon.com/s3/object/my-bucket?prefix=nested%2Ffile.csv" + # act / assert assert parse_s3_uri(url) == ("my-bucket", "nested/file.csv") def test_rejects_unparseable_string() -> None: + # act / assert with pytest.raises(ValueError): parse_s3_uri("not-a-uri-at-all") diff --git a/tests/infrastructure/test_sqs_client.py b/tests/infrastructure/test_sqs_client.py index 7f1e8f78..44186bbb 100644 --- a/tests/infrastructure/test_sqs_client.py +++ b/tests/infrastructure/test_sqs_client.py @@ -19,17 +19,23 @@ def sqs_setup() -> Iterator[tuple[SqsClient, Any, str]]: def test_send_returns_message_id(sqs_setup: tuple[SqsClient, Any, str]) -> None: + # arrange client, _boto, _url = sqs_setup + # act message_id = client.send({"hello": "world"}) + # assert assert isinstance(message_id, str) assert message_id def test_send_json_serialises_body(sqs_setup: tuple[SqsClient, Any, str]) -> None: + # arrange client, boto_client, queue_url = sqs_setup body = {"hello": "world", "count": 3} + # act client.send(body) + # assert received: dict[str, Any] = boto_client.receive_message( QueueUrl=queue_url, MaxNumberOfMessages=1 ) diff --git a/tests/orchestration/test_postcode_splitter_orchestrator.py b/tests/orchestration/test_postcode_splitter_orchestrator.py index 4ee2315e..a718ffbc 100644 --- a/tests/orchestration/test_postcode_splitter_orchestrator.py +++ b/tests/orchestration/test_postcode_splitter_orchestrator.py @@ -9,7 +9,8 @@ from typing import Any, cast import boto3 import pytest from moto import mock_aws -from sqlmodel import Session, SQLModel, create_engine +from sqlalchemy import Engine +from sqlmodel import Session from infrastructure.address2uprn_queue_client import Address2UprnQueueClient from infrastructure.csv_s3_client import CsvS3Client @@ -65,7 +66,7 @@ class Harness: @pytest.fixture -def harness() -> Iterator[Harness]: +def harness(db_engine: Engine) -> Iterator[Harness]: with mock_aws(): # Infra: S3 + SQS boto_s3 = _make_boto_client("s3") @@ -78,10 +79,8 @@ def harness() -> Iterator[Harness]: repo = UserAddressCsvS3Repository(csv_client, BUCKET) queue_client = Address2UprnQueueClient(boto_sqs, queue_url) - # DB: in-memory SQLite TaskOrchestrator - engine = create_engine("sqlite://") - SQLModel.metadata.create_all(engine) - with Session(engine) as session: + # DB: ephemeral PostgreSQL TaskOrchestrator + with Session(db_engine) as session: task_repo = TaskPostgresRepository(session=session) subtask_repo = SubTaskPostgresRepository(session=session) task_orchestrator = TaskOrchestrator( @@ -169,6 +168,7 @@ def _drain_queue(boto_sqs: Any, queue_url: str) -> list[dict[str, Any]]: def test_split_and_dispatch_creates_three_children_for_fixture( harness: Harness, ) -> None: + # arrange parent_task, parent_subtask = ( harness.task_orchestrator.create_task_with_subtask( task_source="manual:postcode-splitter-int" @@ -176,12 +176,14 @@ def test_split_and_dispatch_creates_three_children_for_fixture( ) input_uri = _upload_fixture_csv(harness.csv_client) + # act child_ids = harness.splitter.split_and_dispatch( parent_task_id=parent_task.id, parent_subtask_id=parent_subtask.id, input_s3_uri=input_uri, ) + # assert assert len(child_ids) == 3 # All child ids are unique and persisted as WAITING children of the # parent task. @@ -194,6 +196,7 @@ def test_split_and_dispatch_creates_three_children_for_fixture( def test_split_and_dispatch_persists_child_inputs_with_task_id_and_s3_uri( harness: Harness, ) -> None: + # arrange parent_task, parent_subtask = ( harness.task_orchestrator.create_task_with_subtask( task_source="manual:postcode-splitter-int" @@ -201,12 +204,14 @@ def test_split_and_dispatch_persists_child_inputs_with_task_id_and_s3_uri( ) input_uri = _upload_fixture_csv(harness.csv_client) + # act child_ids = harness.splitter.split_and_dispatch( parent_task_id=parent_task.id, parent_subtask_id=parent_subtask.id, input_s3_uri=input_uri, ) + # assert for cid in child_ids: child = harness.subtasks.get(cid) assert child.inputs is not None @@ -224,6 +229,7 @@ def test_split_and_dispatch_persists_child_inputs_with_task_id_and_s3_uri( def test_split_and_dispatch_publishes_one_message_per_child_with_matching_ids( harness: Harness, ) -> None: + # arrange parent_task, parent_subtask = ( harness.task_orchestrator.create_task_with_subtask( task_source="manual:postcode-splitter-int" @@ -231,12 +237,14 @@ def test_split_and_dispatch_publishes_one_message_per_child_with_matching_ids( ) input_uri = _upload_fixture_csv(harness.csv_client) + # act child_ids = harness.splitter.split_and_dispatch( parent_task_id=parent_task.id, parent_subtask_id=parent_subtask.id, input_s3_uri=input_uri, ) + # assert bodies = _drain_queue(harness.boto_sqs, harness.queue_url) assert len(bodies) == len(child_ids) @@ -258,6 +266,7 @@ def test_split_and_dispatch_publishes_one_message_per_child_with_matching_ids( def test_split_and_dispatch_returns_child_ids_in_dispatch_order( harness: Harness, ) -> None: + # arrange parent_task, parent_subtask = ( harness.task_orchestrator.create_task_with_subtask( task_source="manual:postcode-splitter-int" @@ -265,12 +274,14 @@ def test_split_and_dispatch_returns_child_ids_in_dispatch_order( ) input_uri = _upload_fixture_csv(harness.csv_client) + # act child_ids = harness.splitter.split_and_dispatch( parent_task_id=parent_task.id, parent_subtask_id=parent_subtask.id, input_s3_uri=input_uri, ) + # assert # Re-load each child's saved batch and inspect the postcode_clean column # to confirm the dispatch order matches the postcode-batching algorithm: # AA-batch first, BB oversize batch second, CC final-flush third. diff --git a/tests/orchestration/test_task_orchestrator.py b/tests/orchestration/test_task_orchestrator.py index c0816d2d..ae89991d 100644 --- a/tests/orchestration/test_task_orchestrator.py +++ b/tests/orchestration/test_task_orchestrator.py @@ -2,7 +2,8 @@ from collections.abc import Iterator from dataclasses import dataclass import pytest -from sqlmodel import Session, SQLModel, create_engine +from sqlalchemy import Engine +from sqlmodel import Session from domain.tasks.subtasks import SubTask, SubTaskStatus from domain.tasks.tasks import Source, TaskStatus @@ -19,10 +20,8 @@ class Harness: @pytest.fixture -def harness() -> Iterator[Harness]: - engine = create_engine("sqlite://") - SQLModel.metadata.create_all(engine) - with Session(engine) as session: +def harness(db_engine: Engine) -> Iterator[Harness]: + with Session(db_engine) as session: tasks = TaskPostgresRepository(session=session) subtasks = SubTaskPostgresRepository(session=session) yield Harness( @@ -35,6 +34,7 @@ def harness() -> Iterator[Harness]: def test_create_task_with_subtask_creates_both_in_waiting( harness: Harness, ) -> None: + # act task, subtask = harness.orchestrator.create_task_with_subtask( task_source="manual:test", inputs={"foo": "bar"}, @@ -42,6 +42,7 @@ def test_create_task_with_subtask_creates_both_in_waiting( source_id="abc", ) + # assert assert task.status is TaskStatus.WAITING assert subtask.status is SubTaskStatus.WAITING assert subtask.task_id == task.id @@ -49,27 +50,33 @@ def test_create_task_with_subtask_creates_both_in_waiting( def test_start_subtask_cascades_to_in_progress(harness: Harness) -> None: + # arrange task, subtask = harness.orchestrator.create_task_with_subtask( task_source="manual:test" ) + # act started = harness.orchestrator.start_subtask( subtask.id, cloud_logs_url="https://example/log" ) + # assert assert started.status is SubTaskStatus.IN_PROGRESS assert started.cloud_logs_url == "https://example/log" assert harness.tasks.get(task.id).status is TaskStatus.IN_PROGRESS def test_complete_subtask_cascades_to_complete(harness: Harness) -> None: + # arrange task, subtask = harness.orchestrator.create_task_with_subtask( task_source="manual:test" ) harness.orchestrator.start_subtask(subtask.id) + # act harness.orchestrator.complete_subtask(subtask.id, {"value": 42}) + # assert done_subtask = harness.subtasks.get(subtask.id) done_task = harness.tasks.get(task.id) assert done_subtask.outputs == {"result": {"value": 42}} @@ -78,12 +85,15 @@ def test_complete_subtask_cascades_to_complete(harness: Harness) -> None: def test_fail_subtask_cascades_to_failed(harness: Harness) -> None: + # arrange task, subtask = harness.orchestrator.create_task_with_subtask( task_source="manual:test" ) + # act harness.orchestrator.fail_subtask(subtask.id, RuntimeError("boom")) + # assert failed_subtask = harness.subtasks.get(subtask.id) failed_task = harness.tasks.get(task.id) assert failed_subtask.outputs == {"error": "boom"} @@ -93,42 +103,51 @@ def test_fail_subtask_cascades_to_failed(harness: Harness) -> None: def test_failed_subtask_locks_task_failed_even_with_others_complete( harness: Harness, ) -> None: + # arrange task, first = harness.orchestrator.create_task_with_subtask( task_source="manual:test" ) second = SubTask.create(task_id=task.id) harness.subtasks.create(second) + # act harness.orchestrator.complete_subtask(first.id) harness.orchestrator.fail_subtask(second.id, RuntimeError("nope")) + # assert assert harness.tasks.get(task.id).status is TaskStatus.FAILED def test_mixed_complete_and_in_progress_keeps_task_in_progress( harness: Harness, ) -> None: + # arrange task, first = harness.orchestrator.create_task_with_subtask( task_source="manual:test" ) second = SubTask.create(task_id=task.id) harness.subtasks.create(second) + # act harness.orchestrator.complete_subtask(first.id) harness.orchestrator.start_subtask(second.id) + # assert assert harness.tasks.get(task.id).status is TaskStatus.IN_PROGRESS def test_run_subtask_happy_path_returns_result_and_cascades_complete( harness: Harness, ) -> None: + # arrange task, subtask = harness.orchestrator.create_task_with_subtask( task_source="manual:test" ) + # act result = harness.orchestrator.run_subtask(subtask.id, work=lambda: {"answer": 42}) + # assert assert result == {"answer": 42} assert harness.subtasks.get(subtask.id).status is SubTaskStatus.COMPLETE assert harness.tasks.get(task.id).status is TaskStatus.COMPLETE @@ -137,16 +156,19 @@ def test_run_subtask_happy_path_returns_result_and_cascades_complete( def test_create_child_subtask_adds_waiting_child_without_changing_parent_status( harness: Harness, ) -> None: + # arrange task, first = harness.orchestrator.create_task_with_subtask( task_source="manual:test" ) harness.orchestrator.start_subtask(first.id) assert harness.tasks.get(task.id).status is TaskStatus.IN_PROGRESS + # act child = harness.orchestrator.create_child_subtask( task.id, inputs={"split": "a"} ) + # assert persisted_child = harness.subtasks.get(child.id) assert persisted_child.task_id == task.id assert persisted_child.status is SubTaskStatus.WAITING @@ -159,6 +181,7 @@ def test_create_child_subtask_adds_waiting_child_without_changing_parent_status( def test_run_subtask_failing_work_marks_failed_and_reraises( harness: Harness, ) -> None: + # arrange task, subtask = harness.orchestrator.create_task_with_subtask( task_source="manual:test" ) @@ -166,6 +189,7 @@ def test_run_subtask_failing_work_marks_failed_and_reraises( def boom() -> None: raise RuntimeError("boom") + # act / assert with pytest.raises(RuntimeError, match="boom"): harness.orchestrator.run_subtask(subtask.id, work=boom) diff --git a/tests/repositories/tasks/postgres/test_subtask_postgres_repository.py b/tests/repositories/tasks/postgres/test_subtask_postgres_repository.py index ac39e089..9cec52ea 100644 --- a/tests/repositories/tasks/postgres/test_subtask_postgres_repository.py +++ b/tests/repositories/tasks/postgres/test_subtask_postgres_repository.py @@ -1,33 +1,40 @@ from collections.abc import Iterator -from uuid import uuid4 +from uuid import UUID, uuid4 import pytest -from sqlmodel import Session, SQLModel, create_engine +from sqlalchemy import Engine +from sqlmodel import Session -# Importing the SQLModel row modules registers their tables in -# SQLModel.metadata so create_all builds both. Imports look unused; they aren't. -import infrastructure.postgres.subtask_table # noqa: F401 # pyright: ignore[reportUnusedImport] -import infrastructure.postgres.task_table # noqa: F401 # pyright: ignore[reportUnusedImport] from domain.tasks.subtasks import SubTask, SubTaskStatus +from domain.tasks.tasks import Task from repositories.tasks.subtask_postgres_repository import SubTaskPostgresRepository +from repositories.tasks.task_postgres_repository import TaskPostgresRepository @pytest.fixture -def session() -> Iterator[Session]: - engine = create_engine("sqlite://") - SQLModel.metadata.create_all(engine) - with Session(engine) as s: +def session(db_engine: Engine) -> Iterator[Session]: + with Session(db_engine) as s: yield s +def _persisted_task_id(session: Session) -> UUID: + """Create a parent Task row so SubTask FK constraints are satisfied.""" + task = Task.create(task_source="manual:test") + TaskPostgresRepository(session=session).create(task) + return task.id + + def test_create_and_get_round_trip_preserves_inputs(session: Session) -> None: + # arrange repo = SubTaskPostgresRepository(session=session) - task_id = uuid4() + task_id = _persisted_task_id(session) st = SubTask.create(task_id=task_id, inputs={"address": "68 Glendon Way"}) + # act repo.create(st) fetched = repo.get(st.id) + # assert assert fetched.id == st.id assert fetched.task_id == task_id assert fetched.status is SubTaskStatus.WAITING @@ -36,16 +43,21 @@ def test_create_and_get_round_trip_preserves_inputs(session: Session) -> None: def test_save_persists_status_and_outputs(session: Session) -> None: + # arrange repo = SubTaskPostgresRepository(session=session) - st = SubTask.create(task_id=uuid4()) + st = SubTask.create(task_id=_persisted_task_id(session)) repo.create(st) + # act st.start(cloud_logs_url="https://example/log") repo.save(st) + # assert assert repo.get(st.id).status is SubTaskStatus.IN_PROGRESS + # act st.complete({"uprn": "123"}) repo.save(st) + # assert done = repo.get(st.id) assert done.status is SubTaskStatus.COMPLETE assert done.outputs == {"result": {"uprn": "123"}} @@ -54,16 +66,19 @@ def test_save_persists_status_and_outputs(session: Session) -> None: def test_list_by_task_filters_by_task_id(session: Session) -> None: + # arrange repo = SubTaskPostgresRepository(session=session) - task_a = uuid4() - task_b = uuid4() + task_a = _persisted_task_id(session) + task_b = _persisted_task_id(session) repo.create(SubTask.create(task_id=task_a)) repo.create(SubTask.create(task_id=task_a)) repo.create(SubTask.create(task_id=task_b)) + # act a_results = repo.list_by_task(task_a) b_results = repo.list_by_task(task_b) + # assert assert len(a_results) == 2 assert len(b_results) == 1 assert all(s.task_id == task_a for s in a_results) @@ -71,11 +86,15 @@ def test_list_by_task_filters_by_task_id(session: Session) -> None: def test_list_by_task_returns_empty_for_unknown_task(session: Session) -> None: + # arrange repo = SubTaskPostgresRepository(session=session) + # act / assert assert repo.list_by_task(uuid4()) == [] def test_get_missing_raises(session: Session) -> None: + # arrange repo = SubTaskPostgresRepository(session=session) + # act / assert with pytest.raises(ValueError, match="not found"): repo.get(uuid4()) diff --git a/tests/repositories/tasks/postgres/test_task_postgres_repository.py b/tests/repositories/tasks/postgres/test_task_postgres_repository.py index 3e1aa226..8a49a861 100644 --- a/tests/repositories/tasks/postgres/test_task_postgres_repository.py +++ b/tests/repositories/tasks/postgres/test_task_postgres_repository.py @@ -2,7 +2,8 @@ from collections.abc import Iterator from uuid import uuid4 import pytest -from sqlmodel import Session, SQLModel, create_engine +from sqlalchemy import Engine +from sqlmodel import Session from domain.tasks.tasks import Source, Task, TaskStatus from infrastructure.postgres.task_table import TaskRow @@ -10,25 +11,23 @@ from repositories.tasks.task_postgres_repository import TaskPostgresRepository @pytest.fixture -def session() -> Iterator[Session]: - engine = create_engine("sqlite://") - SQLModel.metadata.create_all(engine) - with Session(engine) as s: +def session(db_engine: Engine) -> Iterator[Session]: + with Session(db_engine) as s: yield s def test_create_and_get_round_trip(session: Session) -> None: - # Arrange + # arrange repo = TaskPostgresRepository(session=session) t = Task.create( task_source="manual:test", source=Source.PORTFOLIO, source_id="abc-123" ) - # Act + # act repo.create(t) fetched = repo.get(t.id) - # Assert + # assert assert fetched.id == t.id assert fetched.status is TaskStatus.WAITING assert fetched.source is Source.PORTFOLIO @@ -36,33 +35,43 @@ def test_create_and_get_round_trip(session: Session) -> None: def test_save_persists_status_transition(session: Session) -> None: + # arrange repo = TaskPostgresRepository(session=session) t = Task.create(task_source="manual:test") repo.create(t) + # act t.start() repo.save(t) + # assert assert repo.get(t.id).status is TaskStatus.IN_PROGRESS + # act t.complete() repo.save(t) + # assert done = repo.get(t.id) assert done.status is TaskStatus.COMPLETE assert done.job_completed is not None def test_get_missing_raises(session: Session) -> None: + # arrange repo = TaskPostgresRepository(session=session) + # act / assert with pytest.raises(ValueError, match="not found"): repo.get(uuid4()) def test_get_normalises_legacy_capitalised_status(session: Session) -> None: # Existing rows written by backend code use "In Progress" (capitalised). + # arrange repo = TaskPostgresRepository(session=session) row = TaskRow(task_source="manual:test", status="In Progress") session.add(row) session.commit() + # act fetched = repo.get(row.id) + # assert assert fetched.status is TaskStatus.IN_PROGRESS diff --git a/tests/repositories/user_address/test_user_address_csv_s3_repository.py b/tests/repositories/user_address/test_user_address_csv_s3_repository.py index c1acee32..9ffb250a 100644 --- a/tests/repositories/user_address/test_user_address_csv_s3_repository.py +++ b/tests/repositories/user_address/test_user_address_csv_s3_repository.py @@ -32,6 +32,7 @@ def _upload_csv( def test_load_batch_parses_address_postcode_and_reference( repo: UserAddressCsvS3Repository, ) -> None: + # arrange rows = [ { "Address 1": "1 High Street", @@ -43,8 +44,10 @@ def test_load_batch_parses_address_postcode_and_reference( ] uri = _upload_csv(repo, rows, "uploads/full.csv") + # act addresses = repo.load_batch(uri) + # assert assert len(addresses) == 1 address = addresses[0] assert address.user_address == "1 High Street, Flat 2, Townville" @@ -55,6 +58,7 @@ def test_load_batch_parses_address_postcode_and_reference( def test_load_batch_uses_only_address_1_when_others_missing( repo: UserAddressCsvS3Repository, ) -> None: + # arrange rows = [ { "Address 1": "10 Cardiff Road", @@ -66,8 +70,10 @@ def test_load_batch_uses_only_address_1_when_others_missing( ] uri = _upload_csv(repo, rows, "uploads/address1-only.csv") + # act addresses = repo.load_batch(uri) + # assert assert len(addresses) == 1 assert addresses[0].user_address == "10 Cardiff Road" assert addresses[0].postcode == Postcode("CF101AA") @@ -77,6 +83,7 @@ def test_load_batch_uses_only_address_1_when_others_missing( def test_load_batch_handles_missing_internal_reference( repo: UserAddressCsvS3Repository, ) -> None: + # arrange rows = [ { "Address 1": "5 Park Lane", @@ -88,8 +95,10 @@ def test_load_batch_handles_missing_internal_reference( ] uri = _upload_csv(repo, rows, "uploads/no-ref.csv") + # act addresses = repo.load_batch(uri) + # assert assert len(addresses) == 1 assert addresses[0].user_address == "5 Park Lane" assert addresses[0].postcode == Postcode("M11AA") @@ -101,6 +110,7 @@ def test_load_batch_captures_full_source_row( ) -> None: # A raw EPC-export-shaped row: the splitter must preserve every column, # not just the ones it parses into UserAddress fields. + # arrange row = { "Asset Reference": "511", "Address 1": "9 Abingdon Road Padiham Lancashire BB12 7BX", @@ -110,17 +120,21 @@ def test_load_batch_captures_full_source_row( } uri = _upload_csv(repo, [row], "uploads/epc.csv") + # act addresses = repo.load_batch(uri) + # assert assert addresses[0].source_row == row def test_load_batch_raises_when_postcode_column_absent( repo: UserAddressCsvS3Repository, ) -> None: + # arrange rows = [{"Address 1": "1 High Street", "Property Type": "Flat"}] uri = _upload_csv(repo, rows, "uploads/no-postcode.csv") + # act / assert with pytest.raises(ValueError, match="no 'postcode' column"): repo.load_batch(uri) @@ -128,6 +142,7 @@ def test_load_batch_raises_when_postcode_column_absent( def test_save_batch_passes_through_all_columns_and_appends_postcode_clean( repo: UserAddressCsvS3Repository, ) -> None: + # arrange row = { "Asset Reference": "511", "Address 1": "9 Abingdon Road Padiham Lancashire BB12 7BX", @@ -137,9 +152,11 @@ def test_save_batch_passes_through_all_columns_and_appends_postcode_clean( uri = _upload_csv(repo, [row], "uploads/epc.csv") addresses = repo.load_batch(uri) + # act saved_uri = repo.save_batch(addresses, "tasks/passthrough") saved_rows = repo._csv_client.read_rows(saved_uri) # pyright: ignore[reportPrivateUsage] + # assert assert len(saved_rows) == 1 saved = saved_rows[0] # Every original column survives, byte-for-byte. @@ -152,6 +169,7 @@ def test_save_batch_passes_through_all_columns_and_appends_postcode_clean( def test_save_batch_returns_uri_under_path_prefix( repo: UserAddressCsvS3Repository, ) -> None: + # arrange addresses = [ UserAddress( user_address="1 High Street", @@ -160,8 +178,10 @@ def test_save_batch_returns_uri_under_path_prefix( ), ] + # act uri = repo.save_batch(addresses, "tasks/abc/batches") + # assert assert uri.startswith(f"s3://{BUCKET}/tasks/abc/batches/") assert uri.endswith(".csv") @@ -169,6 +189,7 @@ def test_save_batch_returns_uri_under_path_prefix( def test_save_then_reload_round_trip_preserves_columns( repo: UserAddressCsvS3Repository, ) -> None: + # arrange rows = [ { "Address 1": "1 High Street", @@ -184,9 +205,11 @@ def test_save_then_reload_round_trip_preserves_columns( uri = _upload_csv(repo, rows, "uploads/round-trip.csv") addresses = repo.load_batch(uri) + # act saved_uri = repo.save_batch(addresses, "tasks/round-trip") saved_rows = repo._csv_client.read_rows(saved_uri) # pyright: ignore[reportPrivateUsage] + # assert # Original columns come back verbatim; postcode_clean is the only addition. assert [ {k: v for k, v in r.items() if k != "postcode_clean"} for r in saved_rows @@ -197,6 +220,7 @@ def test_save_then_reload_round_trip_preserves_columns( def test_save_batch_uses_unique_filename_per_call( repo: UserAddressCsvS3Repository, ) -> None: + # arrange addresses = [ UserAddress( user_address="1 High Street", @@ -205,7 +229,9 @@ def test_save_batch_uses_unique_filename_per_call( ), ] + # act uri_1 = repo.save_batch(addresses, "tasks/uniqueness") uri_2 = repo.save_batch(addresses, "tasks/uniqueness") + # assert assert uri_1 != uri_2 diff --git a/tests/utilities/aws_lambda/test_subtask_handler.py b/tests/utilities/aws_lambda/test_subtask_handler.py index 9cf68f28..d671adc4 100644 --- a/tests/utilities/aws_lambda/test_subtask_handler.py +++ b/tests/utilities/aws_lambda/test_subtask_handler.py @@ -6,7 +6,8 @@ from typing import Any from uuid import UUID import pytest -from sqlmodel import Session, SQLModel, create_engine +from sqlalchemy import Engine +from sqlmodel import Session from domain.tasks.subtasks import SubTaskStatus from domain.tasks.tasks import TaskStatus @@ -30,10 +31,8 @@ class Harness: @pytest.fixture -def harness() -> Iterator[Harness]: - engine = create_engine("sqlite://") - SQLModel.metadata.create_all(engine) - with Session(engine) as session: +def harness(db_engine: Engine) -> Iterator[Harness]: + with Session(db_engine) as session: tasks = TaskPostgresRepository(session=session) subtasks = SubTaskPostgresRepository(session=session) yield Harness( @@ -50,6 +49,7 @@ def _direct_event(task_id: UUID, subtask_id: UUID) -> dict[str, Any]: def test_subtask_handler_injects_orchestrator_as_third_positional_argument( harness: Harness, ) -> None: + # arrange _, subtask = harness.orchestrator.create_task_with_subtask( task_source="manual:test" ) @@ -64,8 +64,10 @@ def test_subtask_handler_injects_orchestrator_as_third_positional_argument( received["context"] = context received["orchestrator"] = orchestrator + # act handler(_direct_event(subtask.task_id, subtask.id), context="ctx-sentinel") + # assert assert received["orchestrator"] is harness.orchestrator assert received["context"] == "ctx-sentinel" assert received["body"]["sub_task_id"] == str(subtask.id) @@ -74,6 +76,7 @@ def test_subtask_handler_injects_orchestrator_as_third_positional_argument( def test_subtask_handler_completes_parent_subtask_on_success( harness: Harness, ) -> None: + # arrange task, subtask = harness.orchestrator.create_task_with_subtask( task_source="manual:test" ) @@ -84,8 +87,10 @@ def test_subtask_handler_completes_parent_subtask_on_success( ) -> None: return None + # act handler(_direct_event(task.id, subtask.id), context=None) + # assert assert harness.subtasks.get(subtask.id).status is SubTaskStatus.COMPLETE assert harness.tasks.get(task.id).status is TaskStatus.COMPLETE @@ -93,6 +98,7 @@ def test_subtask_handler_completes_parent_subtask_on_success( def test_subtask_handler_marks_parent_failed_and_reraises_on_error( harness: Harness, ) -> None: + # arrange task, subtask = harness.orchestrator.create_task_with_subtask( task_source="manual:test" ) @@ -103,6 +109,7 @@ def test_subtask_handler_marks_parent_failed_and_reraises_on_error( ) -> None: raise RuntimeError("boom") + # act / assert with pytest.raises(RuntimeError, match="boom"): handler(_direct_event(task.id, subtask.id), context=None) @@ -113,6 +120,7 @@ def test_subtask_handler_marks_parent_failed_and_reraises_on_error( def test_subtask_handler_injected_orchestrator_can_create_child_subtask( harness: Harness, ) -> None: + # arrange task, subtask = harness.orchestrator.create_task_with_subtask( task_source="manual:test" ) @@ -126,8 +134,10 @@ def test_subtask_handler_injected_orchestrator_can_create_child_subtask( child = orchestrator.create_child_subtask(task.id, inputs={"split": 1}) child_ids.append(child.id) + # act handler(_direct_event(task.id, subtask.id), context=None) + # assert assert len(child_ids) == 1 persisted_child = harness.subtasks.get(child_ids[0]) assert persisted_child.task_id == task.id @@ -137,6 +147,7 @@ def test_subtask_handler_injected_orchestrator_can_create_child_subtask( def test_subtask_handler_logs_subtask_lifecycle_on_success( harness: Harness, caplog: pytest.LogCaptureFixture ) -> None: + # arrange task, subtask = harness.orchestrator.create_task_with_subtask( task_source="manual:test" ) @@ -147,9 +158,11 @@ def test_subtask_handler_logs_subtask_lifecycle_on_success( ) -> None: return None + # act with caplog.at_level(logging.INFO, logger=_LOGGER_NAME): handler(_direct_event(task.id, subtask.id), context=None) + # assert assert f"Running subtask {subtask.id}" in caplog.text assert f"Subtask {subtask.id} completed" in caplog.text @@ -157,6 +170,7 @@ def test_subtask_handler_logs_subtask_lifecycle_on_success( def test_subtask_handler_logs_exception_on_failure( harness: Harness, caplog: pytest.LogCaptureFixture ) -> None: + # arrange task, subtask = harness.orchestrator.create_task_with_subtask( task_source="manual:test" ) @@ -167,6 +181,7 @@ def test_subtask_handler_logs_exception_on_failure( ) -> None: raise RuntimeError("boom") + # act / assert with caplog.at_level(logging.INFO, logger=_LOGGER_NAME): with pytest.raises(RuntimeError, match="boom"): handler(_direct_event(task.id, subtask.id), context=None) @@ -181,6 +196,7 @@ def test_subtask_handler_logs_exception_on_failure( def test_subtask_handler_records_cloudwatch_url_on_subtask( harness: Harness, monkeypatch: pytest.MonkeyPatch ) -> None: + # arrange monkeypatch.setenv("AWS_REGION", "eu-west-2") monkeypatch.setenv( "AWS_LAMBDA_LOG_GROUP_NAME", "/aws/lambda/postcode-splitter" @@ -198,8 +214,10 @@ def test_subtask_handler_records_cloudwatch_url_on_subtask( ) -> None: return None + # act handler(_direct_event(task.id, subtask.id), context=None) + # assert saved_url = harness.subtasks.get(subtask.id).cloud_logs_url assert saved_url is not None assert saved_url.startswith( @@ -213,6 +231,7 @@ def test_subtask_handler_records_cloudwatch_url_on_subtask( def test_subtask_handler_leaves_cloudwatch_url_unset_outside_lambda( harness: Harness, monkeypatch: pytest.MonkeyPatch ) -> None: + # arrange for var in ( "AWS_REGION", "AWS_LAMBDA_LOG_GROUP_NAME", @@ -229,6 +248,8 @@ def test_subtask_handler_leaves_cloudwatch_url_unset_outside_lambda( ) -> None: return None + # act handler(_direct_event(task.id, subtask.id), context=None) + # assert assert harness.subtasks.get(subtask.id).cloud_logs_url is None From f10947699eca992b3cbc5ef9b69b744acaf73226 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 20 May 2026 14:13:04 +0000 Subject: [PATCH 13/22] pytest.ini --- pytest.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/pytest.ini b/pytest.ini index 99cc8e1b..5044465b 100644 --- a/pytest.ini +++ b/pytest.ini @@ -4,6 +4,7 @@ log_cli = true log_cli_level = INFO addopts = --cov-report term-missing --cov=etl/epc --cov=recommendations --cov=backend --cov=etl/epc_clean --cov=etl/spatial testpaths = + tests recommendations/tests backend/tests backend/address2UPRN/tests From 154b820b29f7b6ba2c24c34a3a60a98435a79df8 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 20 May 2026 14:26:46 +0000 Subject: [PATCH 14/22] pytest.ini --- .github/workflows/unit_tests.yml | 12 ++++++++++++ pytest.ini | 1 - 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index fa4fdf2a..15d4cfe9 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -60,3 +60,15 @@ jobs: -e DB_PASSWORD=test \ -e DB_PORT=5432 \ model-test pytest -vv -m 'not integration' + + # The DDD rewrite (tests/) defines SQLModel table classes that map to the + # same physical tables as the legacy backend models. Both sets share the + # one global SQLModel.metadata, so they cannot be imported into the same + # pytest process. It runs as a separate invocation until the legacy + # models are retired. Its DB is spawned in-process by pytest-postgresql, + # so no DB service or env is required. + - name: Run DDD tests + run: | + docker run --rm \ + --network host \ + model-test pytest -vv tests/ diff --git a/pytest.ini b/pytest.ini index 5044465b..99cc8e1b 100644 --- a/pytest.ini +++ b/pytest.ini @@ -4,7 +4,6 @@ log_cli = true log_cli_level = INFO addopts = --cov-report term-missing --cov=etl/epc --cov=recommendations --cov=backend --cov=etl/epc_clean --cov=etl/spatial testpaths = - tests recommendations/tests backend/tests backend/address2UPRN/tests From 8610a0c87518c3dd7c2625b839218aa4593b9e4c Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 20 May 2026 15:17:55 +0000 Subject: [PATCH 15/22] actually deploy postcode splitter --- .github/workflows/deploy_terraform.yml | 2 +- deployment/terraform/lambda/postcodeSplitter/main.tf | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 923fc0a9..8ba473ca 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -169,7 +169,7 @@ jobs: uses: ./.github/workflows/_build_image.yml with: ecr_repo: postcode_splitter-${{ needs.determine_stage.outputs.stage }} - dockerfile_path: backend/postcode_splitter/handler/Dockerfile + dockerfile_path: applications/postcode_splitter/Dockerfile build_context: . build_args: | DEV_DB_HOST=$DEV_DB_HOST diff --git a/deployment/terraform/lambda/postcodeSplitter/main.tf b/deployment/terraform/lambda/postcodeSplitter/main.tf index 325f7dc7..721cb2ea 100644 --- a/deployment/terraform/lambda/postcodeSplitter/main.tf +++ b/deployment/terraform/lambda/postcodeSplitter/main.tf @@ -38,8 +38,8 @@ module "lambda" { { STAGE = var.stage LOG_LEVEL = "info" - DB_USERNAME = local.db_credentials.db_assessment_model_username - DB_PASSWORD = local.db_credentials.db_assessment_model_password + POSTGRES_USERNAME = local.db_credentials.db_assessment_model_username + POSTGRES_PASSWORD = local.db_credentials.db_assessment_model_password ADDRESS2UPRN_QUEUE_URL = data.terraform_remote_state.address2uprn.outputs.address2uprn_queue_url S3_BUCKET_NAME = data.terraform_remote_state.shared.outputs.retrofit_sap_data_bucket_name }, From 78c1d150fa2552ad4386cf113a0ee61523d8aa9a Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 20 May 2026 15:25:42 +0000 Subject: [PATCH 16/22] added smoke test --- .github/workflows/lambda_smoke_tests.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/lambda_smoke_tests.yml b/.github/workflows/lambda_smoke_tests.yml index 5ff5420a..b562f91e 100644 --- a/.github/workflows/lambda_smoke_tests.yml +++ b/.github/workflows/lambda_smoke_tests.yml @@ -36,6 +36,13 @@ jobs: build_context: . service_name: postcode-splitter + postcode_splitter_ddd_smoke_test: + uses: ./.github/workflows/_smoke_test_lambda.yml + with: + dockerfile_path: applications/postcode_splitter/Dockerfile + build_context: . + service_name: postcode-splitter-ddd + # ============================================================ # Bulk Address2UPRN Combiner # ============================================================ From 53b211e951c1b2eb71ac0fce20aefeab6cd9ddc5 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 20 May 2026 15:43:41 +0000 Subject: [PATCH 17/22] epc token added --- .github/workflows/_build_image.yml | 3 +++ .github/workflows/deploy_terraform.yml | 2 ++ backend/address2UPRN/handler/Dockerfile | 2 ++ 3 files changed, 7 insertions(+) diff --git a/.github/workflows/_build_image.yml b/.github/workflows/_build_image.yml index 3435c92d..e7ad9424 100644 --- a/.github/workflows/_build_image.yml +++ b/.github/workflows/_build_image.yml @@ -40,6 +40,8 @@ on: required: false EPC_AUTH_TOKEN: required: false + OPEN_EPC_API_TOKEN: + required: false jobs: build: @@ -50,6 +52,7 @@ jobs: DEV_DB_PORT: ${{ secrets.DEV_DB_PORT }} DEV_DB_NAME: ${{ secrets.DEV_DB_NAME }} EPC_AUTH_TOKEN: ${{ secrets.EPC_AUTH_TOKEN }} + OPEN_EPC_API_TOKEN: ${{ secrets.OPEN_EPC_API_TOKEN }} outputs: image_digest: ${{ steps.digest.outputs.image_digest }} diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 8ba473ca..7f2eb890 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -133,6 +133,7 @@ jobs: DEV_DB_PORT=$DEV_DB_PORT DEV_DB_NAME=$DEV_DB_NAME EPC_AUTH_TOKEN=$EPC_AUTH_TOKEN + OPEN_EPC_API_TOKEN=$OPEN_EPC_API_TOKEN secrets: AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} @@ -141,6 +142,7 @@ jobs: DEV_DB_PORT: ${{ secrets.DEV_DB_PORT }} DEV_DB_NAME: ${{ secrets.DEV_DB_NAME }} EPC_AUTH_TOKEN: ${{ secrets.DEV_EPC_AUTH_TOKEN }} + OPEN_EPC_API_TOKEN: ${{ secrets.DEV_OPEN_EPC_API_TOKEN }} # ============================================================ # Deploy Address 2 UPRN Lambda diff --git a/backend/address2UPRN/handler/Dockerfile b/backend/address2UPRN/handler/Dockerfile index 07159357..7d174152 100644 --- a/backend/address2UPRN/handler/Dockerfile +++ b/backend/address2UPRN/handler/Dockerfile @@ -6,11 +6,13 @@ ARG DEV_DB_HOST ARG DEV_DB_PORT ARG DEV_DB_NAME ARG EPC_AUTH_TOKEN +ARG OPEN_EPC_API_TOKEN ENV DB_HOST=${DEV_DB_HOST} ENV DB_PORT=${DEV_DB_PORT} ENV DB_NAME=${DEV_DB_NAME} ENV EPC_AUTH_TOKEN=${EPC_AUTH_TOKEN} +ENV OPEN_EPC_API_TOKEN=${OPEN_EPC_API_TOKEN} # Set working directory (Lambda task root) From e5583aac1f693fa58ed1d1f5501751d97b38bd01 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 20 May 2026 17:36:20 +0000 Subject: [PATCH 18/22] some excel files are formatted differently --- .../postcode_splitter/local_handler/invoke_local_lambda.py | 2 +- infrastructure/csv_s3_client.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/applications/postcode_splitter/local_handler/invoke_local_lambda.py b/applications/postcode_splitter/local_handler/invoke_local_lambda.py index 21fa9b9e..17d7e345 100755 --- a/applications/postcode_splitter/local_handler/invoke_local_lambda.py +++ b/applications/postcode_splitter/local_handler/invoke_local_lambda.py @@ -14,7 +14,7 @@ payload = { { "task_id": "f4b3332f-c0cc-481f-96a5-d39860a647cf", "sub_task_id": "14c042de-40c4-473b-8cd8-72c983a94a8d", - "s3_uri": "s3://retrofit-data-dev/ara_raw_inputs/calico/Calico Homes Full list EPC Properties(Sheet2) (1) (1).csv", + "s3_uri": "s3://retrofit-data-dev/bulk_onboarding_inputs/hyde2.csv", } ) } diff --git a/infrastructure/csv_s3_client.py b/infrastructure/csv_s3_client.py index 055d1ce3..8af8de73 100644 --- a/infrastructure/csv_s3_client.py +++ b/infrastructure/csv_s3_client.py @@ -13,7 +13,12 @@ class CsvS3Client(S3Client): f"s3_uri bucket {bucket!r} does not match client bucket {self.bucket!r}" ) raw = self.get_object(key) - text = raw.decode("utf-8-sig") + try: + text = raw.decode("utf-8-sig") + except UnicodeDecodeError: + # Some uploads are Windows-1252 (e.g. £ as byte 0xA3), not UTF-8. + text = raw.decode("cp1252") + reader = csv.DictReader(StringIO(text)) return [dict(row) for row in reader] From 714478a99a7a221e26367bb2a762d1a31f938ac0 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Wed, 20 May 2026 17:51:45 +0000 Subject: [PATCH 19/22] clean up sanitise postcode --- backend/epc_client/epc_client_service.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/backend/epc_client/epc_client_service.py b/backend/epc_client/epc_client_service.py index 86caeea3..72dbf142 100644 --- a/backend/epc_client/epc_client_service.py +++ b/backend/epc_client/epc_client_service.py @@ -47,8 +47,14 @@ class EpcClientService: latest = max(results, key=lambda r: r.registration_date) return self.get_by_certificate_number(latest.certificate_number) + @staticmethod + def _normalise_postcode(postcode: str) -> str: + """Return the postcode with all spaces removed and uppercased.""" + return postcode.replace(" ", "").upper() + def search_by_postcode(self, postcode: str) -> list[EpcSearchResult]: - return call_with_retry(lambda: self._search(postcode=postcode)) + normalised = self._normalise_postcode(postcode) + return call_with_retry(lambda: self._search(postcode=normalised)) # ------------------------------------------------------------------ # Private helperEpcRateLimpolarss From c5ab795f851402145bc7ed65e3b17a10cd8cd494 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 21 May 2026 09:46:47 +0000 Subject: [PATCH 20/22] redeploy old postcode splitter --- .github/workflows/deploy_terraform.yml | 4 +++- asset_list/app.py | 13 ++++++----- .../terraform/lambda/postcodeSplitter/main.tf | 22 +++++++++++++++++-- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 8ba473ca..1af90291 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -169,7 +169,9 @@ jobs: uses: ./.github/workflows/_build_image.yml with: ecr_repo: postcode_splitter-${{ needs.determine_stage.outputs.stage }} - dockerfile_path: applications/postcode_splitter/Dockerfile + # dockerfile_path: applications/postcode_splitter/Dockerfile + # Switch back to the old postcode_splitter due to hyde priority - interface for the new one isn't working atm + dockerfile_path: backend/postcode_splitter/handler/Dockerfile build_context: . build_args: | DEV_DB_HOST=$DEV_DB_HOST diff --git a/asset_list/app.py b/asset_list/app.py index 9b10d7f3..424f4df6 100644 --- a/asset_list/app.py +++ b/asset_list/app.py @@ -79,23 +79,23 @@ def app(): """ data_folder = "/workspaces/model/asset_list" - data_filename = "lincs_address_list.xlsx" - sheet_name = "Sheet1" + data_filename = "hyde.xlsx" + sheet_name = "AddressProfilingResults" postcode_column = "Postcode" - address1_column = "Deal Name" + address1_column = "Address" address1_method = None - fulladdress_column = "Deal Name" + fulladdress_column = "Postcode" address_cols_to_concat = [] missing_postcodes_method = None landlord_year_built = None landlord_os_uprn = None - landlord_property_type = None # Good to include if landlord gave + landlord_property_type = "Property Type" # Good to include if landlord gave landlord_built_form = None # Good to include if landlord gave landlord_wall_construction = None landlord_roof_construction = None landlord_heating_system = None landlord_existing_pv = None - landlord_property_id = "landlord_id" + landlord_property_id = "Organisation Reference" landlord_sap = None outcomes_filename = None outcomes_sheetname = None @@ -468,3 +468,4 @@ def app(): asset_list.duplicated_addresses.to_excel( writer, sheet_name="Duplicate Properties", index=False ) + diff --git a/deployment/terraform/lambda/postcodeSplitter/main.tf b/deployment/terraform/lambda/postcodeSplitter/main.tf index 721cb2ea..e04ae00f 100644 --- a/deployment/terraform/lambda/postcodeSplitter/main.tf +++ b/deployment/terraform/lambda/postcodeSplitter/main.tf @@ -38,8 +38,26 @@ module "lambda" { { STAGE = var.stage LOG_LEVEL = "info" - POSTGRES_USERNAME = local.db_credentials.db_assessment_model_username - POSTGRES_PASSWORD = local.db_credentials.db_assessment_model_password + # POSTGRES_USERNAME = local.db_credentials.db_assessment_model_username + # POSTGRES_PASSWORD = local.db_credentials.db_assessment_model_password + # Switch back to the old postcode_splitter due to hyde priority - interface for the new one isn't working atm + DB_USERNAME = local.db_credentials.db_assessment_model_username + DB_PASSWORD = local.db_credentials.db_assessment_model_password + # Placeholder values so backend/app/config.py Settings doesn't fall back to "changeme" + GOOGLE_SOLAR_API_KEY = "test" + SAP_PREDICTIONS_BUCKET = "test" + CARBON_PREDICTIONS_BUCKET = "test" + HEAT_PREDICTIONS_BUCKET = "test" + HEATING_KWH_PREDICTIONS_BUCKET = "test" + HOTWATER_KWH_PREDICTIONS_BUCKET = "test" + API_KEY = "test" + ENVIRONMENT = "test" + SECRET_KEY = "test" + PLAN_TRIGGER_BUCKET = "test" + DATA_BUCKET = "test" + EPC_AUTH_TOKEN = "test" + ENGINE_SQS_URL = "test" + ENERGY_ASSESSMENTS_BUCKET = "test" ADDRESS2UPRN_QUEUE_URL = data.terraform_remote_state.address2uprn.outputs.address2uprn_queue_url S3_BUCKET_NAME = data.terraform_remote_state.shared.outputs.retrofit_sap_data_bucket_name }, From 856ea6eb9358f10e89e6b574a3a4367b0e92a874 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 21 May 2026 10:12:08 +0000 Subject: [PATCH 21/22] undo postcodesplitter changes --- .github/workflows/deploy_terraform.yml | 4 +--- .../terraform/lambda/postcodeSplitter/main.tf | 22 ++----------------- 2 files changed, 3 insertions(+), 23 deletions(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 1af90291..8ba473ca 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -169,9 +169,7 @@ jobs: uses: ./.github/workflows/_build_image.yml with: ecr_repo: postcode_splitter-${{ needs.determine_stage.outputs.stage }} - # dockerfile_path: applications/postcode_splitter/Dockerfile - # Switch back to the old postcode_splitter due to hyde priority - interface for the new one isn't working atm - dockerfile_path: backend/postcode_splitter/handler/Dockerfile + dockerfile_path: applications/postcode_splitter/Dockerfile build_context: . build_args: | DEV_DB_HOST=$DEV_DB_HOST diff --git a/deployment/terraform/lambda/postcodeSplitter/main.tf b/deployment/terraform/lambda/postcodeSplitter/main.tf index e04ae00f..721cb2ea 100644 --- a/deployment/terraform/lambda/postcodeSplitter/main.tf +++ b/deployment/terraform/lambda/postcodeSplitter/main.tf @@ -38,26 +38,8 @@ module "lambda" { { STAGE = var.stage LOG_LEVEL = "info" - # POSTGRES_USERNAME = local.db_credentials.db_assessment_model_username - # POSTGRES_PASSWORD = local.db_credentials.db_assessment_model_password - # Switch back to the old postcode_splitter due to hyde priority - interface for the new one isn't working atm - DB_USERNAME = local.db_credentials.db_assessment_model_username - DB_PASSWORD = local.db_credentials.db_assessment_model_password - # Placeholder values so backend/app/config.py Settings doesn't fall back to "changeme" - GOOGLE_SOLAR_API_KEY = "test" - SAP_PREDICTIONS_BUCKET = "test" - CARBON_PREDICTIONS_BUCKET = "test" - HEAT_PREDICTIONS_BUCKET = "test" - HEATING_KWH_PREDICTIONS_BUCKET = "test" - HOTWATER_KWH_PREDICTIONS_BUCKET = "test" - API_KEY = "test" - ENVIRONMENT = "test" - SECRET_KEY = "test" - PLAN_TRIGGER_BUCKET = "test" - DATA_BUCKET = "test" - EPC_AUTH_TOKEN = "test" - ENGINE_SQS_URL = "test" - ENERGY_ASSESSMENTS_BUCKET = "test" + POSTGRES_USERNAME = local.db_credentials.db_assessment_model_username + POSTGRES_PASSWORD = local.db_credentials.db_assessment_model_password ADDRESS2UPRN_QUEUE_URL = data.terraform_remote_state.address2uprn.outputs.address2uprn_queue_url S3_BUCKET_NAME = data.terraform_remote_state.shared.outputs.retrofit_sap_data_bucket_name }, From dbd03de842933fa189de077d48e5c13ecf9729f4 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 21 May 2026 10:37:13 +0000 Subject: [PATCH 22/22] local run changes --- .../postcode_splitter/local_handler/invoke_local_lambda.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/applications/postcode_splitter/local_handler/invoke_local_lambda.py b/applications/postcode_splitter/local_handler/invoke_local_lambda.py index 17d7e345..5f4b1d36 100755 --- a/applications/postcode_splitter/local_handler/invoke_local_lambda.py +++ b/applications/postcode_splitter/local_handler/invoke_local_lambda.py @@ -12,9 +12,9 @@ payload = { { "body": json.dumps( { - "task_id": "f4b3332f-c0cc-481f-96a5-d39860a647cf", - "sub_task_id": "14c042de-40c4-473b-8cd8-72c983a94a8d", - "s3_uri": "s3://retrofit-data-dev/bulk_onboarding_inputs/hyde2.csv", + "task_id": "e295d89b-a7c5-4a9a-8b4e-b405fab1f298", + "sub_task_id": "f4a9944f-41f0-4a33-8669-5016ec574068", + "s3_uri": "s3://retrofit-data-dev/bulk_onboarding_inputs/hyde2 (1).csv", } ) }