diff --git a/gateway/scripts/ocg_router.py b/gateway/scripts/ocg_router.py index f82afb2..0509cf6 100644 --- a/gateway/scripts/ocg_router.py +++ b/gateway/scripts/ocg_router.py @@ -48,6 +48,11 @@ UPSTREAM_COOLDOWN_SEC = 30 # 上游模型级故障(Router.Unavailable)短 USAGE_REFRESH_INTERVAL = 120 # 2 分钟 # 最大重试次数(所有 key 耗尽) MAX_RETRIES = 6 +# 单次上游请求超时(秒):翻译等小请求正常 2-5s,60s 绰绰有余; +# 从 180s 下调,避免"单次 180s × 多 key"叠加成长时间挂死(2026-08-15 事故) +UPSTREAM_REQUEST_TIMEOUT = 60 +# 整个请求(含所有 key 重试)的总截止时间:3 key × 60s = 180s,保证 3 分钟内一定结束 +REQUEST_DEADLINE_SEC = 180 # ── 日志 ──────────────────────────────────────────────────── LOG_DIR = _GATEWAY_DIR / "logs" @@ -543,6 +548,12 @@ class RouterHandler(BaseHTTPRequestHandler): client_error = False # True=客户端请求错误(400/ModelError),应返回 4xx 而非 502 for attempt, k in enumerate(ranked_keys): + # 总超时保护:整个请求(含所有 key 重试)最迟在 REQUEST_DEADLINE_SEC 内结束, + # 防止"单次 60s × 多 key"叠加成 15 分钟挂死(2026-08-15 事故根因) + if time.time() - start > REQUEST_DEADLINE_SEC: + last_error = f"overall request deadline ({REQUEST_DEADLINE_SEC}s) exceeded after {attempt} key attempts" + log.warning("⚠ %s overall deadline exceeded after %d attempts", model_name, attempt) + break kid = k["key_id"] api_key = k.get("api_key", "") if not api_key: @@ -641,7 +652,7 @@ class RouterHandler(BaseHTTPRequestHandler): }, method="POST", ) - with urlopen(req, timeout=180) as resp: + with urlopen(req, timeout=UPSTREAM_REQUEST_TIMEOUT) as resp: status = resp.status if status >= 400: err_body = resp.read().decode("utf-8", errors="replace")[:500] @@ -711,7 +722,7 @@ class RouterHandler(BaseHTTPRequestHandler): try: req = Request(url, data=body or None, headers=headers, method=method) - with urlopen(req, timeout=120) as resp: + with urlopen(req, timeout=UPSTREAM_REQUEST_TIMEOUT) as resp: resp_body = resp.read() self._send_response(resp.status, dict(resp.headers), resp_body) except HTTPError as e: