fix: overall request deadline (180s) + upstream timeout 180->60s — prevent multi-key retry chain hanging 15min (2026-08-15 incident)

gateway/scripts/ocg_router.py

@@ -48,6 +48,11 @@ UPSTREAM_COOLDOWN_SEC = 30  # 上游模型级故障（Router.Unavailable）短
 USAGE_REFRESH_INTERVAL = 120  # 2 分钟
 # 最大重试次数（所有 key 耗尽）
 MAX_RETRIES = 6
+# 单次上游请求超时（秒）：翻译等小请求正常 2-5s，60s 绰绰有余；
+# 从 180s 下调，避免"单次 180s × 多 key"叠加成长时间挂死（2026-08-15 事故）
+UPSTREAM_REQUEST_TIMEOUT = 60
+# 整个请求（含所有 key 重试）的总截止时间：3 key × 60s = 180s，保证 3 分钟内一定结束
+REQUEST_DEADLINE_SEC = 180
 
 # ── 日志 ────────────────────────────────────────────────────
 LOG_DIR = _GATEWAY_DIR / "logs"
@@ -543,6 +548,12 @@ class RouterHandler(BaseHTTPRequestHandler):
         client_error = False  # True=客户端请求错误（400/ModelError），应返回 4xx 而非 502
 
         for attempt, k in enumerate(ranked_keys):
+            # 总超时保护：整个请求（含所有 key 重试）最迟在 REQUEST_DEADLINE_SEC 内结束，
+            # 防止"单次 60s × 多 key"叠加成 15 分钟挂死（2026-08-15 事故根因）
+            if time.time() - start > REQUEST_DEADLINE_SEC:
+                last_error = f"overall request deadline ({REQUEST_DEADLINE_SEC}s) exceeded after {attempt} key attempts"
+                log.warning("⚠ %s overall deadline exceeded after %d attempts", model_name, attempt)
+                break
             kid = k["key_id"]
             api_key = k.get("api_key", "")
             if not api_key:
@@ -641,7 +652,7 @@ class RouterHandler(BaseHTTPRequestHandler):
                 },
                 method="POST",
             )
-            with urlopen(req, timeout=180) as resp:
+            with urlopen(req, timeout=UPSTREAM_REQUEST_TIMEOUT) as resp:
                 status = resp.status
                 if status >= 400:
                     err_body = resp.read().decode("utf-8", errors="replace")[:500]
@@ -711,7 +722,7 @@ class RouterHandler(BaseHTTPRequestHandler):
 
         try:
             req = Request(url, data=body or None, headers=headers, method=method)
-            with urlopen(req, timeout=120) as resp:
+            with urlopen(req, timeout=UPSTREAM_REQUEST_TIMEOUT) as resp:
                 resp_body = resp.read()
                 self._send_response(resp.status, dict(resp.headers), resp_body)
         except HTTPError as e: