diff --git a/gateway/scripts/ocg_router.py b/gateway/scripts/ocg_router.py index 7af2cd3..d0f401f 100644 --- a/gateway/scripts/ocg_router.py +++ b/gateway/scripts/ocg_router.py @@ -42,7 +42,8 @@ USAGE_STATS_FILE = _GATEWAY_DIR / "temp" / "usage_stats.json" ACCOUNTS_FILE = _SCRIPT_DIR / "usage_monitor" / "accounts.json" # 故障切换:同一个 key 失败后冷却时间(秒) -FAIL_COOLDOWN_SEC = 300 # 5 分钟 +FAIL_COOLDOWN_SEC = 300 # 5 分钟(key 自身故障) +UPSTREAM_COOLDOWN_SEC = 30 # 上游模型级故障(Router.Unavailable)短冷却:换 key 无意义 # 用量数据刷新间隔 USAGE_REFRESH_INTERVAL = 120 # 2 分钟 # 最大重试次数(所有 key 耗尽) @@ -65,6 +66,7 @@ log = logging.getLogger("ocg_router") _keys: list[dict] = [] # [{key_id, label, api_key, workspace_id}] _key_usage: dict[str, dict] = {} # key_id → {rolling_pct, weekly_pct, monthly_pct} _key_failures: dict[str, float] = {} # key_id → fail_until_timestamp +_key_failure_reason: dict[str, str] = {} # key_id → 最近失败原因(冷却跳过时用于报错) _route_stats: dict[str, int] = {} # key_id → hit_count _total_hits = 0 _current_key_id = "" @@ -200,13 +202,15 @@ def pick_key(): return best -def mark_key_failed(kid): - """标记 key 故障,进入冷却期。""" - until = time.time() + FAIL_COOLDOWN_SEC +def mark_key_failed(kid, reason="", cooldown=FAIL_COOLDOWN_SEC): + """标记 key 故障,进入冷却期。记录失败原因供冷却期报错。""" + until = time.time() + cooldown with _state_lock: _key_failures[kid] = until - log.warning("key %s marked failed until %s", kid, - datetime.fromtimestamp(until).strftime("%H:%M:%S")) + if reason: + _key_failure_reason[kid] = reason + log.warning("key %s marked failed until %s (%ds): %s", kid, + datetime.fromtimestamp(until).strftime("%H:%M:%S"), cooldown, reason) def record_route(kid): @@ -332,6 +336,7 @@ class RouterHandler(BaseHTTPRequestHandler): "monthly_pct": usage.get("monthly_pct", None), "healthy": usage.get("subscribed", True) and (time.time() >= fail_until), "in_cooldown": time.time() < fail_until, + "fail_reason": _key_failure_reason.get(kid, ""), "error": usage.get("error", False), "hits": _route_stats.get(kid, 0), "last_update": usage.get("last_update", ""), @@ -459,14 +464,35 @@ class RouterHandler(BaseHTTPRequestHandler): # 401(未授权/CreditsError)/402/403/429/5xx → 标记 key 故障 status = result.get("status", 0) if status in (401, 402, 403, 429) or status >= 500: - mark_key_failed(kid) + if "Router.Unavailable" in last_error: + # 上游模型级故障:换 key 无意义 → 全体短冷却并立即失败返回 + with _state_lock: + until = time.time() + UPSTREAM_COOLDOWN_SEC + for k2 in _keys: + _key_failures[k2["key_id"]] = until + _key_failure_reason[k2["key_id"]] = last_error + log.warning("upstream Router.Unavailable → all keys short-cooldown %ds", UPSTREAM_COOLDOWN_SEC) + success = False + break + mark_key_failed(kid, last_error) except Exception as e: last_error = str(e) log.warning("⚠ %s via %s exception: %s", model_name, kid, e) - mark_key_failed(kid) + mark_key_failed(kid, last_error) if not success: elapsed = time.time() - start + if not last_error: + # 所有 key 在冷却中被跳过 → 组装原因+最早到期时间 + now = time.time() + cooling = [(kid, until) for kid, until in failures_snapshot.items() if now < until] + if cooling: + earliest = min(until for _, until in cooling) + reasons = {_key_failure_reason.get(kid, "unknown") for kid, _ in cooling} + reason_str = "; ".join(sorted(r for r in reasons if r and r != "unknown")) + last_error = (f"all keys in cooldown until " + f"{datetime.fromtimestamp(earliest).strftime('%H:%M:%S')}" + f"{' (' + reason_str + ')' if reason_str else ''}") log.error("✗ %s ALL KEYS EXHAUSTED (%.2fs): %s", model_name, elapsed, last_error) self._error_response(502, f"All OCG keys exhausted: {last_error}") @@ -597,6 +623,7 @@ def _bg_refresh(): expired = [k for k, v in _key_failures.items() if now >= v] for k in expired: del _key_failures[k] + _key_failure_reason.pop(k, None) log.info("key %s cooldown expired, restored", k) log.info("bg refresh: %d keys, healthy pool", len(_keys)) except Exception as e: