Pace Solar calls per container to stay under the 600 QPM fleet cap 🟥

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

infrastructure/solar/google_solar_api_client.py

@@ -1,3 +1,4 @@
+import time
 from typing import Any, Literal, Optional
 
 import requests
@@ -20,8 +21,23 @@ class GoogleSolarApiClient:
     MAX_BACKOFF_SECONDS: float = 60.0
     ENTITY_NOT_FOUND_ERROR: str = "Requested entity was not found."
 
-    def __init__(self, api_key: str) -> None:
+    def __init__(
+        self, api_key: str, min_request_interval_seconds: float = 0.0
+    ) -> None:
+        # Per-instance min-interval throttle. The Solar API caps the whole GCP
+        # project at a hard 600 QPM shared across Building Insights + Data Layers
+        # (https://developers.google.com/maps/documentation/solar/usage-and-billing).
+        # The modelling_e2e fleet runs up to `maximum_concurrency` (32) containers,
+        # each looping its ~50-property batch sequentially — so without pacing the
+        # fleet offers multiples of the quota and a steady fraction of calls 429
+        # for a sustained window that #1327's jittered backoff can't ride out.
+        # Each container holding >= quota/N seconds between its OWN consecutive
+        # calls keeps the fleet sum under the ceiling. Default 0.0 = no throttle,
+        # so the interactive backend path and tests are unchanged; the scaled
+        # caller (the Lambda) opts in via env.
         self._api_key = api_key
+        self._min_request_interval_seconds = min_request_interval_seconds
+        self._last_request_monotonic: Optional[float] = None
 
     @staticmethod
     def _parse_retry_after(response: requests.Response) -> Optional[float]:

tests/infrastructure/solar/test_google_solar_api_client.py

@@ -95,6 +95,59 @@ def test_get_building_insights_does_not_retry_a_400() -> None:
     assert mock_get.call_count == 1
 
 
+# ---------------------------------------------------------------------------
+# Throttle: per-instance min-interval keeps the fleet under the 600 QPM cap
+# ---------------------------------------------------------------------------
+
+
+def test_throttle_spaces_consecutive_calls_by_min_interval() -> None:
+    # Arrange — a client paced at a 4s minimum gap between its OWN Solar calls
+    # (quota/N at the 32-wide fleet: 0.8 * 600 QPM / 32 ≈ one call / 4s). The
+    # first call fires immediately; the second must wait out the remainder of the
+    # interval before hitting the API, so the fleet stays under the hard 600 QPM
+    # ceiling instead of sustaining the over-quota burst that 429'd portfolio 796.
+    client = GoogleSolarApiClient(
+        api_key="test-key", min_request_interval_seconds=4.0
+    )
+    ok = MagicMock()
+    ok.json.return_value = {"solarPotential": {}}
+    ok.raise_for_status.return_value = None
+
+    with patch("requests.get", return_value=ok):
+        with patch(
+            "infrastructure.solar.google_solar_api_client.time.sleep"
+        ) as sleep:
+            # Act — two back-to-back calls with no real work between them.
+            client.get_building_insights(longitude=-0.1278, latitude=51.5074)
+            client.get_building_insights(longitude=-0.1278, latitude=51.5074)
+
+    # Assert — first call doesn't wait; second waits ~the full interval (no time
+    # elapsed between them, so the remaining gap is essentially the whole 4s).
+    assert sleep.call_count == 1
+    (waited,), _ = sleep.call_args
+    assert 3.5 <= waited <= 4.0
+
+
+def test_throttle_off_by_default_does_not_sleep() -> None:
+    # Arrange — no interval configured (the interactive backend path): the client
+    # must not pace, preserving today's behaviour.
+    client = GoogleSolarApiClient(api_key="test-key")
+    ok = MagicMock()
+    ok.json.return_value = {"solarPotential": {}}
+    ok.raise_for_status.return_value = None
+
+    with patch("requests.get", return_value=ok):
+        with patch(
+            "infrastructure.solar.google_solar_api_client.time.sleep"
+        ) as sleep:
+            # Act
+            client.get_building_insights(longitude=-0.1278, latitude=51.5074)
+            client.get_building_insights(longitude=-0.1278, latitude=51.5074)
+
+    # Assert
+    sleep.assert_not_called()
+
+
 # ---------------------------------------------------------------------------
 # Slice 1: Successful response returns parsed JSON
 # ---------------------------------------------------------------------------