diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 8f2d405f4..9907ebee3 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -547,18 +547,7 @@ jobs: python-version: "3.11" - name: Check projected unzipped Lambda size - run: | - python3 scripts/check_lambda_zip_size.py \ - --source-dir . \ - --requirements backend/app/requirements/requirements.txt \ - --exclude "**/__pycache__/**" \ - --exclude "**/*.pyc" \ - --exclude "**/.pytest_cache/**" \ - --exclude "**/tests/**" \ - --exclude "**/deployment/**" \ - --exclude "**/.git/**" \ - --exclude "**/json_samples/**" \ - --exclude "**/docs/specs/**" + run: python3 backend/app/requirements/check_lambda_zip_size.py # ============================================================ # Deploy FastAPI Lambda diff --git a/scripts/check_lambda_zip_size.py b/backend/app/requirements/check_lambda_zip_size.py similarity index 63% rename from scripts/check_lambda_zip_size.py rename to backend/app/requirements/check_lambda_zip_size.py index 160ac4b63..2ccd9a07a 100755 --- a/scripts/check_lambda_zip_size.py +++ b/backend/app/requirements/check_lambda_zip_size.py @@ -1,18 +1,22 @@ #!/usr/bin/env python3 -"""Estimate the unzipped size of a Terraform-packaged Lambda before deploying. +"""Estimate the unzipped size of the fastapi Lambda before deploying. -Mirrors what `deployment/terraform/modules/lambda_with_api_gateway/main.tf` does: -`pip install -r -t ...` followed by zipping +Mirrors what `deployment/terraform/modules/lambda_with_api_gateway/main.tf` does +for the `ara_fast_api` lambda (`deployment/terraform/lambda/fast-api`): +`pip install -r requirements.txt -t ...` followed by zipping `source_dir` minus `zip_excludes`. AWS Lambda's hard limit is 262144000 bytes (250 MiB) unzipped. We install the dependencies into a scratch directory instead of the real source tree so this can run locally or in CI without mutating the repo. -Usage: - python3 scripts/check_lambda_zip_size.py \ - --source-dir . \ - --requirements backend/app/requirements/requirements.txt \ - --exclude "**/tests/**" --exclude "**/.git/**" ... +Usage (from the repo root): + python3 backend/app/requirements/check_lambda_zip_size.py --source-dir . + +--requirements defaults to the requirements.txt next to this script, and +--exclude defaults to whatever `zip_excludes` is set to in +`deployment/terraform/modules/lambda_with_api_gateway/variables.tf` — read +straight from that file, so this can never silently drift from what +Terraform actually excludes. Pass --exclude explicitly to override. Exits non-zero if the projected unzipped size exceeds --limit (or exceeds --warn-pct of the limit, when --strict is not passed the warning is just @@ -22,6 +26,7 @@ from __future__ import annotations import argparse import fnmatch +import re import subprocess import sys import tempfile @@ -29,6 +34,24 @@ from pathlib import Path LAMBDA_UNZIPPED_LIMIT_BYTES = 262_144_000 +REPO_ROOT = Path(__file__).resolve().parents[3] +DEFAULT_REQUIREMENTS = Path(__file__).resolve().parent / "requirements.txt" +DEFAULT_TF_VARIABLES_FILE = ( + REPO_ROOT / "deployment/terraform/modules/lambda_with_api_gateway/variables.tf" +) + + +def parse_zip_excludes_from_tf(tf_variables_file: Path) -> list[str]: + text = tf_variables_file.read_text() + match = re.search( + r'variable\s+"zip_excludes"\s*{.*?default\s*=\s*\[(.*?)\]', text, re.S + ) + if not match: + raise ValueError( + f'could not find variable "zip_excludes" default in {tf_variables_file}' + ) + return re.findall(r'"([^"]+)"', match.group(1)) + def doublestar_match(pattern_parts: list[str], path_parts: list[str]) -> bool: if not pattern_parts: @@ -92,9 +115,21 @@ def human(n: int) -> str: def main() -> int: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--source-dir", type=Path, default=Path(".")) - parser.add_argument("--requirements", type=Path, default=None) - parser.add_argument("--exclude", action="append", default=[], dest="excludes") + parser.add_argument("--source-dir", type=Path, default=REPO_ROOT) + parser.add_argument("--requirements", type=Path, default=DEFAULT_REQUIREMENTS) + parser.add_argument( + "--skip-deps", + action="store_true", + help="Skip the pip install step and only measure source files.", + ) + parser.add_argument( + "--exclude", + action="append", + default=None, + dest="excludes", + help="Repeatable. Defaults to zip_excludes read from --tf-variables-file.", + ) + parser.add_argument("--tf-variables-file", type=Path, default=DEFAULT_TF_VARIABLES_FILE) parser.add_argument("--limit", type=int, default=LAMBDA_UNZIPPED_LIMIT_BYTES) parser.add_argument( "--warn-pct", @@ -109,10 +144,15 @@ def main() -> int: ) args = parser.parse_args() - source_size = sum_dir_size(args.source_dir, args.excludes) + excludes = args.excludes + if excludes is None: + excludes = parse_zip_excludes_from_tf(args.tf_variables_file) + print(f"excludes (from {args.tf_variables_file.relative_to(REPO_ROOT)}): {excludes}") + + source_size = sum_dir_size(args.source_dir, excludes) deps_size = 0 - if args.requirements is not None: + if not args.skip_deps: with tempfile.TemporaryDirectory(prefix="lambda-deps-") as tmp: deps_size = install_dependencies(args.requirements, Path(tmp)) @@ -120,7 +160,7 @@ def main() -> int: pct = total / args.limit * 100 print(f"source files (post-exclude): {human(source_size)}") - if args.requirements is not None: + if not args.skip_deps: print(f"pip dependencies: {human(deps_size)}") print(f"projected unzipped total: {human(total)} [{pct:.1f}% of limit]") print(f"AWS limit: {human(args.limit)}")