From d5f1c61689ef513df7ab2c97aa3d5db039b11ad0 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 6 Jul 2026 13:00:59 +0000 Subject: [PATCH] Move zip-size checker next to the fastapi requirements it checks Relocate scripts/check_lambda_zip_size.py to backend/app/requirements/, right beside the requirements.txt it's built to measure, since it's specific to the fastapi lambda rather than a generic repo script. While moving it, also make the excludes list self-updating: the script now reads zip_excludes straight out of deployment/terraform/modules/lambda_with_api_gateway/variables.tf by default instead of duplicating it as CLI flags, so the CI check and the actual Terraform packaging can't drift apart. requirements.txt defaults to the sibling file too, so both the CI step and local runs now collapse to a bare `check_lambda_zip_size.py` invocation. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/deploy_terraform.yml | 13 +--- .../requirements}/check_lambda_zip_size.py | 68 +++++++++++++++---- 2 files changed, 55 insertions(+), 26 deletions(-) rename {scripts => backend/app/requirements}/check_lambda_zip_size.py (63%) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 8f2d405f4..9907ebee3 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -547,18 +547,7 @@ jobs: python-version: "3.11" - name: Check projected unzipped Lambda size - run: | - python3 scripts/check_lambda_zip_size.py \ - --source-dir . \ - --requirements backend/app/requirements/requirements.txt \ - --exclude "**/__pycache__/**" \ - --exclude "**/*.pyc" \ - --exclude "**/.pytest_cache/**" \ - --exclude "**/tests/**" \ - --exclude "**/deployment/**" \ - --exclude "**/.git/**" \ - --exclude "**/json_samples/**" \ - --exclude "**/docs/specs/**" + run: python3 backend/app/requirements/check_lambda_zip_size.py # ============================================================ # Deploy FastAPI Lambda diff --git a/scripts/check_lambda_zip_size.py b/backend/app/requirements/check_lambda_zip_size.py similarity index 63% rename from scripts/check_lambda_zip_size.py rename to backend/app/requirements/check_lambda_zip_size.py index 160ac4b63..2ccd9a07a 100755 --- a/scripts/check_lambda_zip_size.py +++ b/backend/app/requirements/check_lambda_zip_size.py @@ -1,18 +1,22 @@ #!/usr/bin/env python3 -"""Estimate the unzipped size of a Terraform-packaged Lambda before deploying. +"""Estimate the unzipped size of the fastapi Lambda before deploying. -Mirrors what `deployment/terraform/modules/lambda_with_api_gateway/main.tf` does: -`pip install -r -t ...` followed by zipping +Mirrors what `deployment/terraform/modules/lambda_with_api_gateway/main.tf` does +for the `ara_fast_api` lambda (`deployment/terraform/lambda/fast-api`): +`pip install -r requirements.txt -t ...` followed by zipping `source_dir` minus `zip_excludes`. AWS Lambda's hard limit is 262144000 bytes (250 MiB) unzipped. We install the dependencies into a scratch directory instead of the real source tree so this can run locally or in CI without mutating the repo. -Usage: - python3 scripts/check_lambda_zip_size.py \ - --source-dir . \ - --requirements backend/app/requirements/requirements.txt \ - --exclude "**/tests/**" --exclude "**/.git/**" ... +Usage (from the repo root): + python3 backend/app/requirements/check_lambda_zip_size.py --source-dir . + +--requirements defaults to the requirements.txt next to this script, and +--exclude defaults to whatever `zip_excludes` is set to in +`deployment/terraform/modules/lambda_with_api_gateway/variables.tf` — read +straight from that file, so this can never silently drift from what +Terraform actually excludes. Pass --exclude explicitly to override. Exits non-zero if the projected unzipped size exceeds --limit (or exceeds --warn-pct of the limit, when --strict is not passed the warning is just @@ -22,6 +26,7 @@ from __future__ import annotations import argparse import fnmatch +import re import subprocess import sys import tempfile @@ -29,6 +34,24 @@ from pathlib import Path LAMBDA_UNZIPPED_LIMIT_BYTES = 262_144_000 +REPO_ROOT = Path(__file__).resolve().parents[3] +DEFAULT_REQUIREMENTS = Path(__file__).resolve().parent / "requirements.txt" +DEFAULT_TF_VARIABLES_FILE = ( + REPO_ROOT / "deployment/terraform/modules/lambda_with_api_gateway/variables.tf" +) + + +def parse_zip_excludes_from_tf(tf_variables_file: Path) -> list[str]: + text = tf_variables_file.read_text() + match = re.search( + r'variable\s+"zip_excludes"\s*{.*?default\s*=\s*\[(.*?)\]', text, re.S + ) + if not match: + raise ValueError( + f'could not find variable "zip_excludes" default in {tf_variables_file}' + ) + return re.findall(r'"([^"]+)"', match.group(1)) + def doublestar_match(pattern_parts: list[str], path_parts: list[str]) -> bool: if not pattern_parts: @@ -92,9 +115,21 @@ def human(n: int) -> str: def main() -> int: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--source-dir", type=Path, default=Path(".")) - parser.add_argument("--requirements", type=Path, default=None) - parser.add_argument("--exclude", action="append", default=[], dest="excludes") + parser.add_argument("--source-dir", type=Path, default=REPO_ROOT) + parser.add_argument("--requirements", type=Path, default=DEFAULT_REQUIREMENTS) + parser.add_argument( + "--skip-deps", + action="store_true", + help="Skip the pip install step and only measure source files.", + ) + parser.add_argument( + "--exclude", + action="append", + default=None, + dest="excludes", + help="Repeatable. Defaults to zip_excludes read from --tf-variables-file.", + ) + parser.add_argument("--tf-variables-file", type=Path, default=DEFAULT_TF_VARIABLES_FILE) parser.add_argument("--limit", type=int, default=LAMBDA_UNZIPPED_LIMIT_BYTES) parser.add_argument( "--warn-pct", @@ -109,10 +144,15 @@ def main() -> int: ) args = parser.parse_args() - source_size = sum_dir_size(args.source_dir, args.excludes) + excludes = args.excludes + if excludes is None: + excludes = parse_zip_excludes_from_tf(args.tf_variables_file) + print(f"excludes (from {args.tf_variables_file.relative_to(REPO_ROOT)}): {excludes}") + + source_size = sum_dir_size(args.source_dir, excludes) deps_size = 0 - if args.requirements is not None: + if not args.skip_deps: with tempfile.TemporaryDirectory(prefix="lambda-deps-") as tmp: deps_size = install_dependencies(args.requirements, Path(tmp)) @@ -120,7 +160,7 @@ def main() -> int: pct = total / args.limit * 100 print(f"source files (post-exclude): {human(source_size)}") - if args.requirements is not None: + if not args.skip_deps: print(f"pip dependencies: {human(deps_size)}") print(f"projected unzipped total: {human(total)} [{pct:.1f}% of limit]") print(f"AWS limit: {human(args.limit)}")