Merge pull request #1469 from Hestia-Homes/fix/fastapi-lambda-zip-size

Fix fastapi lambda exceeding AWS's 250MB unzipped size limit

.github/workflows/_deploy_lambda.yml

@@ -148,6 +148,7 @@ jobs:
       - name: Terraform Plan
         working-directory: ${{ inputs.lambda_path }}
         env:
+          TF_VAR_github_sha: ${{ github.sha }}
           TF_VAR_db_host: ${{ secrets.TF_VAR_db_host }}
           TF_VAR_db_name: ${{ secrets.TF_VAR_db_name }}
           TF_VAR_db_port: ${{ secrets.TF_VAR_db_port }}

.github/workflows/check_lambda_zip_size.yml

@@ -0,0 +1,23 @@
+name: Check FastAPI Lambda package size
+
+# Runs on PRs targeting main so a Lambda-size regression fails the PR before
+# it can merge to main and roll into the dev deploy — deploy_terraform.yml
+# only triggers on push to dev/prod, which is too late (see PR #1469: this
+# broke dev for weeks before anyone noticed).
+on:
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  fast_api_lambda_zip_size_check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Check projected unzipped Lambda size
+        run: python3 backend/app/requirements/check_lambda_zip_size.py

backend/app/main.py

@@ -53,7 +53,9 @@ async def log_requests(request: Request, call_next):
 
 @app.get("/health")
 async def health():
-    return {"status": "ok", "sha": os.getenv("GITHUB_SHA", "unknown")}
+    sha = os.getenv("GITHUB_SHA", "unknown")
+    logger.info(f"Health check OK - deployed sha={sha}")
+    return {"status": "ok", "sha": sha}
 
 
 app.include_router(tasks_router.router, prefix="/v1")

backend/app/requirements/check_lambda_zip_size.py

@@ -0,0 +1,189 @@
+#!/usr/bin/env python3
+"""Estimate the unzipped size of the fastapi Lambda before deploying.
+
+Mirrors what `deployment/terraform/modules/lambda_with_api_gateway/main.tf` does
+for the `ara_fast_api` lambda (`deployment/terraform/lambda/fast-api`):
+`pip install -r requirements.txt -t <source_dir> ...` followed by zipping
+`source_dir` minus `zip_excludes`. AWS Lambda's hard limit is 262144000 bytes
+(250 MiB) unzipped. We install the dependencies into a scratch directory
+instead of the real source tree so this can run locally or in CI without
+mutating the repo.
+
+Usage (from the repo root):
+    python3 backend/app/requirements/check_lambda_zip_size.py --source-dir .
+
+--requirements defaults to the requirements.txt next to this script, and
+--exclude defaults to whatever `zip_excludes` is set to in
+`deployment/terraform/modules/lambda_with_api_gateway/variables.tf` — read
+straight from that file, so this can never silently drift from what
+Terraform actually excludes. Pass --exclude explicitly to override.
+
+Exits non-zero if the projected unzipped size exceeds --limit (or exceeds
+--warn-pct of the limit, when --strict is not passed the warning is just
+printed).
+"""
+from __future__ import annotations
+
+import argparse
+import fnmatch
+import re
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+LAMBDA_UNZIPPED_LIMIT_BYTES = 262_144_000
+
+REPO_ROOT = Path(__file__).resolve().parents[3]
+DEFAULT_REQUIREMENTS = Path(__file__).resolve().parent / "requirements.txt"
+DEFAULT_TF_VARIABLES_FILE = (
+    REPO_ROOT / "deployment/terraform/modules/lambda_with_api_gateway/variables.tf"
+)
+
+
+def parse_zip_excludes_from_tf(tf_variables_file: Path) -> list[str]:
+    text = tf_variables_file.read_text()
+    match = re.search(
+        r'variable\s+"zip_excludes"\s*{.*?default\s*=\s*\[(.*?)\]', text, re.S
+    )
+    if not match:
+        raise ValueError(
+            f'could not find variable "zip_excludes" default in {tf_variables_file}'
+        )
+    return re.findall(r'"([^"]+)"', match.group(1))
+
+
+def doublestar_match(pattern_parts: list[str], path_parts: list[str]) -> bool:
+    if not pattern_parts:
+        return not path_parts
+    head, rest = pattern_parts[0], pattern_parts[1:]
+    if head == "**":
+        if doublestar_match(rest, path_parts):
+            return True
+        if path_parts and doublestar_match(pattern_parts, path_parts[1:]):
+            return True
+        return False
+    if not path_parts:
+        return False
+    if fnmatch.fnmatchcase(path_parts[0], head):
+        return doublestar_match(rest, path_parts[1:])
+    return False
+
+
+def is_excluded(rel_path: Path, excludes: list[str]) -> bool:
+    parts = rel_path.as_posix().split("/")
+    return any(doublestar_match(pattern.split("/"), parts) for pattern in excludes)
+
+
+def sum_dir_size(root: Path, excludes: list[str] | None = None) -> int:
+    total = 0
+    for path in root.rglob("*"):
+        if path.is_dir():
+            continue
+        rel = path.relative_to(root)
+        if excludes and is_excluded(rel, excludes):
+            continue
+        total += path.stat().st_size
+    return total
+
+
+def install_dependencies(requirements: Path, target: Path) -> int:
+    subprocess.run(
+        [
+            "pip",
+            "install",
+            "-r",
+            str(requirements),
+            "-t",
+            str(target),
+            "--platform",
+            "manylinux2014_x86_64",
+            "--implementation",
+            "cp",
+            "--python-version",
+            "3.11",
+            "--only-binary=:all:",
+        ],
+        check=True,
+    )
+    return sum_dir_size(target)
+
+
+def human(n: int) -> str:
+    return f"{n:,} bytes ({n / 1024 / 1024:.1f} MiB)"
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--source-dir", type=Path, default=REPO_ROOT)
+    parser.add_argument("--requirements", type=Path, default=DEFAULT_REQUIREMENTS)
+    parser.add_argument(
+        "--skip-deps",
+        action="store_true",
+        help="Skip the pip install step and only measure source files.",
+    )
+    parser.add_argument(
+        "--exclude",
+        action="append",
+        default=None,
+        dest="excludes",
+        help="Repeatable. Defaults to zip_excludes read from --tf-variables-file.",
+    )
+    parser.add_argument("--tf-variables-file", type=Path, default=DEFAULT_TF_VARIABLES_FILE)
+    parser.add_argument("--limit", type=int, default=LAMBDA_UNZIPPED_LIMIT_BYTES)
+    parser.add_argument(
+        "--warn-pct",
+        type=float,
+        default=85.0,
+        help="Print a warning once projected size crosses this %% of the limit.",
+    )
+    parser.add_argument(
+        "--strict",
+        action="store_true",
+        help="Exit non-zero on the warning threshold too, not just the hard limit.",
+    )
+    args = parser.parse_args()
+
+    excludes = args.excludes
+    if excludes is None:
+        excludes = parse_zip_excludes_from_tf(args.tf_variables_file)
+        print(f"excludes (from {args.tf_variables_file.relative_to(REPO_ROOT)}): {excludes}")
+
+    source_size = sum_dir_size(args.source_dir, excludes)
+
+    deps_size = 0
+    if not args.skip_deps:
+        with tempfile.TemporaryDirectory(prefix="lambda-deps-") as tmp:
+            deps_size = install_dependencies(args.requirements, Path(tmp))
+
+    total = source_size + deps_size
+    pct = total / args.limit * 100
+
+    print(f"source files (post-exclude): {human(source_size)}")
+    if not args.skip_deps:
+        print(f"pip dependencies:            {human(deps_size)}")
+    print(f"projected unzipped total:     {human(total)}  [{pct:.1f}% of limit]")
+    print(f"AWS limit:                    {human(args.limit)}")
+
+    if total > args.limit:
+        print(
+            f"\nFAIL: projected unzipped size exceeds the Lambda limit by "
+            f"{human(total - args.limit)}.",
+            file=sys.stderr,
+        )
+        return 1
+
+    if pct >= args.warn_pct:
+        msg = (
+            f"\nWARNING: projected unzipped size is at {pct:.1f}% of the "
+            f"{human(args.limit)} limit."
+        )
+        print(msg, file=sys.stderr)
+        if args.strict:
+            return 1
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

deployment/terraform/lambda/fast-api/main.tf

@@ -86,6 +86,7 @@ module "fastapi" {
 
   environment = {
     ENVIRONMENT = var.stage
+    GITHUB_SHA  = var.github_sha
     API_KEY     = var.api_key
     SECRET_KEY  = var.secret_key
     # DOMAIN_NAME    = var.domain_name
@@ -110,11 +111,11 @@ module "fastapi" {
     CARBON_BASELINE_PREDICTIONS_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_carbon_baseline_predictions_bucket_name
     HEAT_BASELINE_PREDICTIONS_BUCKET   = data.terraform_remote_state.shared.outputs.retrofit_heat_baseline_predictions_bucket_name
 
-    ENGINE_SQS_URL              = data.terraform_remote_state.engine.outputs.ara_engine_queue_url
-    CATEGORISATION_SQS_URL      = data.terraform_remote_state.categorisation.outputs.categorisation_queue_url
-    POSTCODE_SPLITTER_SQS_URL   = data.terraform_remote_state.postcode_splitter.outputs.postcode_splitter_queue_url
-    COMBINER_SQS_URL            = data.terraform_remote_state.bulk_address2uprn_combiner.outputs.bulk_address2uprn_combiner_queue_url
-    FINALISER_SQS_URL           = data.terraform_remote_state.bulk_upload_finaliser.outputs.bulk_upload_finaliser_queue_url
+    ENGINE_SQS_URL            = data.terraform_remote_state.engine.outputs.ara_engine_queue_url
+    CATEGORISATION_SQS_URL    = data.terraform_remote_state.categorisation.outputs.categorisation_queue_url
+    POSTCODE_SPLITTER_SQS_URL = data.terraform_remote_state.postcode_splitter.outputs.postcode_splitter_queue_url
+    COMBINER_SQS_URL          = data.terraform_remote_state.bulk_address2uprn_combiner.outputs.bulk_address2uprn_combiner_queue_url
+    FINALISER_SQS_URL         = data.terraform_remote_state.bulk_upload_finaliser.outputs.bulk_upload_finaliser_queue_url
   }
 }
 

deployment/terraform/lambda/fast-api/variables.tf

@@ -41,4 +41,10 @@ variable "epc_auth_token" {
 variable "google_solar_api_key" {
   type      = string
   sensitive = true
-}
+}
+
+variable "github_sha" {
+  type        = string
+  description = "Commit SHA being deployed, surfaced via /health so cloud logs can be tied back to a deploy."
+  default     = "unknown"
+}

deployment/terraform/modules/lambda_with_api_gateway/variables.tf

@@ -11,7 +11,10 @@ variable "zip_excludes" {
     "**/*.pyc",
     "**/.pytest_cache/**",
     "**/tests/**",
-    "**/deployment/**"
+    "**/deployment/**",
+    "**/.git/**",
+    "**/json_samples/**",
+    "**/docs/specs/**"
   ]
 }