From 572fcc1406d93ed7b0a8c32e1ab53b99183fd6e2 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 14 May 2026 16:38:22 +0000 Subject: [PATCH 1/5] smoke tests --- .github/workflows/_smoke_test_lambda.yml | 63 +++++++++++++ .github/workflows/lambda_smoke_tests.yml | 107 +++++++++++++++++++++++ 2 files changed, 170 insertions(+) create mode 100644 .github/workflows/_smoke_test_lambda.yml create mode 100644 .github/workflows/lambda_smoke_tests.yml diff --git a/.github/workflows/_smoke_test_lambda.yml b/.github/workflows/_smoke_test_lambda.yml new file mode 100644 index 00000000..63ec0af4 --- /dev/null +++ b/.github/workflows/_smoke_test_lambda.yml @@ -0,0 +1,63 @@ +name: Lambda smoke test + +on: + workflow_call: + inputs: + dockerfile_path: + required: true + type: string + build_context: + required: false + default: "." + type: string + service_name: + required: true + type: string + +jobs: + smoke-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Build Lambda image + run: | + docker build \ + --platform linux/amd64 \ + -f ${{ inputs.dockerfile_path }} \ + -t ${{ inputs.service_name }}-smoke-test:latest \ + ${{ inputs.build_context }} + + - name: Start Lambda container + run: | + docker run -d --name ${{ inputs.service_name }}-smoke-test \ + -p 9000:8080 \ + ${{ inputs.service_name }}-smoke-test:latest + + - name: Invoke Lambda and check for import errors + run: | + sleep 2 + response=$(curl -s -X POST \ + http://localhost:9000/2015-03-31/functions/function/invocations \ + -H "Content-Type: application/json" \ + -d '{"Records":[{"body":"{}"}]}') + + echo "Response: $response" + + if [ -z "$response" ]; then + echo "No response from Lambda RIE" + exit 1 + fi + + if echo "$response" | grep -qE 'ImportModuleError|ModuleNotFoundError|ImportError'; then + echo "Import error detected in handler" + exit 1 + fi + + - name: Dump container logs + if: always() + run: docker logs ${{ inputs.service_name }}-smoke-test + + - name: Tear down container + if: always() + run: docker rm -f ${{ inputs.service_name }}-smoke-test diff --git a/.github/workflows/lambda_smoke_tests.yml b/.github/workflows/lambda_smoke_tests.yml new file mode 100644 index 00000000..5ff5420a --- /dev/null +++ b/.github/workflows/lambda_smoke_tests.yml @@ -0,0 +1,107 @@ +name: Lambda Smoke Tests + +on: + pull_request: + branches: + - main + +jobs: + # ============================================================ + # Ara Engine + # ============================================================ + ara_engine_smoke_test: + uses: ./.github/workflows/_smoke_test_lambda.yml + with: + dockerfile_path: backend/docker/engine.Dockerfile + build_context: . + service_name: ara-engine + + # ============================================================ + # Address 2 UPRN + # ============================================================ + address2uprn_smoke_test: + uses: ./.github/workflows/_smoke_test_lambda.yml + with: + dockerfile_path: backend/address2UPRN/handler/Dockerfile + build_context: . + service_name: address2uprn + + # ============================================================ + # Postcode Splitter + # ============================================================ + postcode_splitter_smoke_test: + uses: ./.github/workflows/_smoke_test_lambda.yml + with: + dockerfile_path: backend/postcode_splitter/handler/Dockerfile + build_context: . + service_name: postcode-splitter + + # ============================================================ + # Bulk Address2UPRN Combiner + # ============================================================ + bulk_address2uprn_combiner_smoke_test: + uses: ./.github/workflows/_smoke_test_lambda.yml + with: + dockerfile_path: backend/bulk_address2uprn_combiner/handler/Dockerfile + build_context: . + service_name: bulk-address2uprn-combiner + + # ============================================================ + # Condition ETL + # ============================================================ + condition_etl_smoke_test: + uses: ./.github/workflows/_smoke_test_lambda.yml + with: + dockerfile_path: backend/condition/handler/Dockerfile + build_context: . + service_name: condition-etl + + # ============================================================ + # Categorisation + # ============================================================ + categorisation_smoke_test: + uses: ./.github/workflows/_smoke_test_lambda.yml + with: + dockerfile_path: backend/categorisation/handler/Dockerfile + build_context: . + service_name: categorisation + + # ============================================================ + # Ordnance Survey + # ============================================================ + ordnance_survey_smoke_test: + uses: ./.github/workflows/_smoke_test_lambda.yml + with: + dockerfile_path: backend/ordnanceSurvey/handler/Dockerfile + build_context: . + service_name: ordnance-survey + + # ============================================================ + # Pas Hub Fetcher + # ============================================================ + pashub_smoke_test: + uses: ./.github/workflows/_smoke_test_lambda.yml + with: + dockerfile_path: backend/pashub_fetcher/handler/Dockerfile + build_context: . + service_name: pashub + + # ============================================================ + # MagicPlan + # ============================================================ + magic_plan_smoke_test: + uses: ./.github/workflows/_smoke_test_lambda.yml + with: + dockerfile_path: backend/magic_plan/handler/Dockerfile + build_context: . + service_name: magic-plan + + # ============================================================ + # HubSpot Scraper + # ============================================================ + hubspot_scraper_smoke_test: + uses: ./.github/workflows/_smoke_test_lambda.yml + with: + dockerfile_path: etl/hubspot/scripts/scraper/handler/Dockerfile + build_context: . + service_name: hubspot-scraper From 16e60001800fca1db834d90adf843fdd15b419ce Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 14 May 2026 16:44:18 +0000 Subject: [PATCH 2/5] smoke tests --- .github/workflows/_smoke_test_lambda.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_smoke_test_lambda.yml b/.github/workflows/_smoke_test_lambda.yml index 63ec0af4..9b564f73 100644 --- a/.github/workflows/_smoke_test_lambda.yml +++ b/.github/workflows/_smoke_test_lambda.yml @@ -36,8 +36,8 @@ jobs: - name: Invoke Lambda and check for import errors run: | - sleep 2 - response=$(curl -s -X POST \ + response=$(curl -s --retry-connrefused --retry 15 --retry-delay 1 \ + -X POST \ http://localhost:9000/2015-03-31/functions/function/invocations \ -H "Content-Type: application/json" \ -d '{"Records":[{"body":"{}"}]}') From 0c3a31ed81a094d0907b321ab2d7ff3ad061e523 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 14 May 2026 16:49:45 +0000 Subject: [PATCH 3/5] smoke tests --- .github/workflows/_smoke_test_lambda.yml | 28 +++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/.github/workflows/_smoke_test_lambda.yml b/.github/workflows/_smoke_test_lambda.yml index 9b564f73..3fcf0de4 100644 --- a/.github/workflows/_smoke_test_lambda.yml +++ b/.github/workflows/_smoke_test_lambda.yml @@ -20,6 +20,13 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Download AWS Lambda RIE + run: | + mkdir -p ~/.aws-lambda-rie + curl -fsSL -o ~/.aws-lambda-rie/aws-lambda-rie \ + https://github.com/aws/aws-lambda-runtime-interface-emulator/releases/latest/download/aws-lambda-rie + chmod +x ~/.aws-lambda-rie/aws-lambda-rie + - name: Build Lambda image run: | docker build \ @@ -30,9 +37,24 @@ jobs: - name: Start Lambda container run: | - docker run -d --name ${{ inputs.service_name }}-smoke-test \ - -p 9000:8080 \ - ${{ inputs.service_name }}-smoke-test:latest + IMG=${{ inputs.service_name }}-smoke-test:latest + ENTRY=$(docker inspect --format='{{range .Config.Entrypoint}}{{.}} {{end}}' "$IMG") + CMD_ARGS=$(docker inspect --format='{{range .Config.Cmd}}{{.}} {{end}}' "$IMG") + + if echo "$ENTRY" | grep -q "lambda-entrypoint.sh"; then + # AWS base image — RIE is bundled + docker run -d --name ${{ inputs.service_name }}-smoke-test \ + -p 9000:8080 \ + "$IMG" + else + # Custom base — mount RIE from runner and re-wire entrypoint + docker run -d --name ${{ inputs.service_name }}-smoke-test \ + -v "$HOME/.aws-lambda-rie:/aws-lambda-rie" \ + -p 9000:8080 \ + --entrypoint /aws-lambda-rie/aws-lambda-rie \ + "$IMG" \ + $ENTRY $CMD_ARGS + fi - name: Invoke Lambda and check for import errors run: | From 6c8080ef6203694db127edb9aa9b7824bbc76898 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 14 May 2026 16:57:31 +0000 Subject: [PATCH 4/5] smoke tests --- backend/condition/handler/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/condition/handler/Dockerfile b/backend/condition/handler/Dockerfile index 71556895..fa130573 100644 --- a/backend/condition/handler/Dockerfile +++ b/backend/condition/handler/Dockerfile @@ -32,6 +32,7 @@ COPY utils/ utils/ COPY backend/condition/ backend/condition/ COPY backend/app/db/models/condition.py backend/app/db/models/condition.py +COPY backend/app/db/base.py backend/app/db/base.py COPY backend/app/db/connection.py backend/app/db/connection.py COPY backend/app/config.py backend/app/config.py From 6afd07600598a0a92883319345e4918aecd46cc1 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 15 May 2026 11:28:04 +0000 Subject: [PATCH 5/5] added 5 second rest every 100 tests --- backend/address2UPRN/tests/test_csv.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/backend/address2UPRN/tests/test_csv.py b/backend/address2UPRN/tests/test_csv.py index 73d94388..5c97e691 100644 --- a/backend/address2UPRN/tests/test_csv.py +++ b/backend/address2UPRN/tests/test_csv.py @@ -12,12 +12,21 @@ FIXTURE_PATH = Path(__file__).parent / "test_data.csv" # Each parametrized case fires at least one EPC request; without throttling, # GitHub-hosted runners burst fast enough to hit 429s. EPC_THROTTLE_SECONDS = 1.0 +EPC_LONG_PAUSE_EVERY = 100 +EPC_LONG_PAUSE_SECONDS = 5.0 + +_epc_request_count = 0 @pytest.fixture(autouse=True) def _throttle_epc_requests(): + global _epc_request_count yield - time.sleep(EPC_THROTTLE_SECONDS) + _epc_request_count += 1 + if _epc_request_count % EPC_LONG_PAUSE_EVERY == 0: + time.sleep(EPC_LONG_PAUSE_SECONDS) + else: + time.sleep(EPC_THROTTLE_SECONDS) def load_test_cases():