diff --git a/.github/workflows/_smoke_test_lambda.yml b/.github/workflows/_smoke_test_lambda.yml new file mode 100644 index 00000000..3fcf0de4 --- /dev/null +++ b/.github/workflows/_smoke_test_lambda.yml @@ -0,0 +1,85 @@ +name: Lambda smoke test + +on: + workflow_call: + inputs: + dockerfile_path: + required: true + type: string + build_context: + required: false + default: "." + type: string + service_name: + required: true + type: string + +jobs: + smoke-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Download AWS Lambda RIE + run: | + mkdir -p ~/.aws-lambda-rie + curl -fsSL -o ~/.aws-lambda-rie/aws-lambda-rie \ + https://github.com/aws/aws-lambda-runtime-interface-emulator/releases/latest/download/aws-lambda-rie + chmod +x ~/.aws-lambda-rie/aws-lambda-rie + + - name: Build Lambda image + run: | + docker build \ + --platform linux/amd64 \ + -f ${{ inputs.dockerfile_path }} \ + -t ${{ inputs.service_name }}-smoke-test:latest \ + ${{ inputs.build_context }} + + - name: Start Lambda container + run: | + IMG=${{ inputs.service_name }}-smoke-test:latest + ENTRY=$(docker inspect --format='{{range .Config.Entrypoint}}{{.}} {{end}}' "$IMG") + CMD_ARGS=$(docker inspect --format='{{range .Config.Cmd}}{{.}} {{end}}' "$IMG") + + if echo "$ENTRY" | grep -q "lambda-entrypoint.sh"; then + # AWS base image — RIE is bundled + docker run -d --name ${{ inputs.service_name }}-smoke-test \ + -p 9000:8080 \ + "$IMG" + else + # Custom base — mount RIE from runner and re-wire entrypoint + docker run -d --name ${{ inputs.service_name }}-smoke-test \ + -v "$HOME/.aws-lambda-rie:/aws-lambda-rie" \ + -p 9000:8080 \ + --entrypoint /aws-lambda-rie/aws-lambda-rie \ + "$IMG" \ + $ENTRY $CMD_ARGS + fi + + - name: Invoke Lambda and check for import errors + run: | + response=$(curl -s --retry-connrefused --retry 15 --retry-delay 1 \ + -X POST \ + http://localhost:9000/2015-03-31/functions/function/invocations \ + -H "Content-Type: application/json" \ + -d '{"Records":[{"body":"{}"}]}') + + echo "Response: $response" + + if [ -z "$response" ]; then + echo "No response from Lambda RIE" + exit 1 + fi + + if echo "$response" | grep -qE 'ImportModuleError|ModuleNotFoundError|ImportError'; then + echo "Import error detected in handler" + exit 1 + fi + + - name: Dump container logs + if: always() + run: docker logs ${{ inputs.service_name }}-smoke-test + + - name: Tear down container + if: always() + run: docker rm -f ${{ inputs.service_name }}-smoke-test diff --git a/.github/workflows/lambda_smoke_tests.yml b/.github/workflows/lambda_smoke_tests.yml new file mode 100644 index 00000000..5ff5420a --- /dev/null +++ b/.github/workflows/lambda_smoke_tests.yml @@ -0,0 +1,107 @@ +name: Lambda Smoke Tests + +on: + pull_request: + branches: + - main + +jobs: + # ============================================================ + # Ara Engine + # ============================================================ + ara_engine_smoke_test: + uses: ./.github/workflows/_smoke_test_lambda.yml + with: + dockerfile_path: backend/docker/engine.Dockerfile + build_context: . + service_name: ara-engine + + # ============================================================ + # Address 2 UPRN + # ============================================================ + address2uprn_smoke_test: + uses: ./.github/workflows/_smoke_test_lambda.yml + with: + dockerfile_path: backend/address2UPRN/handler/Dockerfile + build_context: . + service_name: address2uprn + + # ============================================================ + # Postcode Splitter + # ============================================================ + postcode_splitter_smoke_test: + uses: ./.github/workflows/_smoke_test_lambda.yml + with: + dockerfile_path: backend/postcode_splitter/handler/Dockerfile + build_context: . + service_name: postcode-splitter + + # ============================================================ + # Bulk Address2UPRN Combiner + # ============================================================ + bulk_address2uprn_combiner_smoke_test: + uses: ./.github/workflows/_smoke_test_lambda.yml + with: + dockerfile_path: backend/bulk_address2uprn_combiner/handler/Dockerfile + build_context: . + service_name: bulk-address2uprn-combiner + + # ============================================================ + # Condition ETL + # ============================================================ + condition_etl_smoke_test: + uses: ./.github/workflows/_smoke_test_lambda.yml + with: + dockerfile_path: backend/condition/handler/Dockerfile + build_context: . + service_name: condition-etl + + # ============================================================ + # Categorisation + # ============================================================ + categorisation_smoke_test: + uses: ./.github/workflows/_smoke_test_lambda.yml + with: + dockerfile_path: backend/categorisation/handler/Dockerfile + build_context: . + service_name: categorisation + + # ============================================================ + # Ordnance Survey + # ============================================================ + ordnance_survey_smoke_test: + uses: ./.github/workflows/_smoke_test_lambda.yml + with: + dockerfile_path: backend/ordnanceSurvey/handler/Dockerfile + build_context: . + service_name: ordnance-survey + + # ============================================================ + # Pas Hub Fetcher + # ============================================================ + pashub_smoke_test: + uses: ./.github/workflows/_smoke_test_lambda.yml + with: + dockerfile_path: backend/pashub_fetcher/handler/Dockerfile + build_context: . + service_name: pashub + + # ============================================================ + # MagicPlan + # ============================================================ + magic_plan_smoke_test: + uses: ./.github/workflows/_smoke_test_lambda.yml + with: + dockerfile_path: backend/magic_plan/handler/Dockerfile + build_context: . + service_name: magic-plan + + # ============================================================ + # HubSpot Scraper + # ============================================================ + hubspot_scraper_smoke_test: + uses: ./.github/workflows/_smoke_test_lambda.yml + with: + dockerfile_path: etl/hubspot/scripts/scraper/handler/Dockerfile + build_context: . + service_name: hubspot-scraper diff --git a/backend/address2UPRN/tests/test_csv.py b/backend/address2UPRN/tests/test_csv.py index 73d94388..5c97e691 100644 --- a/backend/address2UPRN/tests/test_csv.py +++ b/backend/address2UPRN/tests/test_csv.py @@ -12,12 +12,21 @@ FIXTURE_PATH = Path(__file__).parent / "test_data.csv" # Each parametrized case fires at least one EPC request; without throttling, # GitHub-hosted runners burst fast enough to hit 429s. EPC_THROTTLE_SECONDS = 1.0 +EPC_LONG_PAUSE_EVERY = 100 +EPC_LONG_PAUSE_SECONDS = 5.0 + +_epc_request_count = 0 @pytest.fixture(autouse=True) def _throttle_epc_requests(): + global _epc_request_count yield - time.sleep(EPC_THROTTLE_SECONDS) + _epc_request_count += 1 + if _epc_request_count % EPC_LONG_PAUSE_EVERY == 0: + time.sleep(EPC_LONG_PAUSE_SECONDS) + else: + time.sleep(EPC_THROTTLE_SECONDS) def load_test_cases(): diff --git a/backend/condition/handler/Dockerfile b/backend/condition/handler/Dockerfile index 71556895..fa130573 100644 --- a/backend/condition/handler/Dockerfile +++ b/backend/condition/handler/Dockerfile @@ -32,6 +32,7 @@ COPY utils/ utils/ COPY backend/condition/ backend/condition/ COPY backend/app/db/models/condition.py backend/app/db/models/condition.py +COPY backend/app/db/base.py backend/app/db/base.py COPY backend/app/db/connection.py backend/app/db/connection.py COPY backend/app/config.py backend/app/config.py