fix: Router.Unavailable -> all keys short-cooldown 30s + cooldown error shows cause/expiry

gateway/scripts/ocg_router.py

@@ -42,7 +42,8 @@ USAGE_STATS_FILE = _GATEWAY_DIR / "temp" / "usage_stats.json"
 ACCOUNTS_FILE = _SCRIPT_DIR / "usage_monitor" / "accounts.json"
 
 # 故障切换：同一个 key 失败后冷却时间（秒）
-FAIL_COOLDOWN_SEC = 300  # 5 分钟
+FAIL_COOLDOWN_SEC = 300  # 5 分钟（key 自身故障）
+UPSTREAM_COOLDOWN_SEC = 30  # 上游模型级故障（Router.Unavailable）短冷却：换 key 无意义
 # 用量数据刷新间隔
 USAGE_REFRESH_INTERVAL = 120  # 2 分钟
 # 最大重试次数（所有 key 耗尽）
@@ -65,6 +66,7 @@ log = logging.getLogger("ocg_router")
 _keys: list[dict] = []  # [{key_id, label, api_key, workspace_id}]
 _key_usage: dict[str, dict] = {}  # key_id → {rolling_pct, weekly_pct, monthly_pct}
 _key_failures: dict[str, float] = {}  # key_id → fail_until_timestamp
+_key_failure_reason: dict[str, str] = {}  # key_id → 最近失败原因（冷却跳过时用于报错）
 _route_stats: dict[str, int] = {}  # key_id → hit_count
 _total_hits = 0
 _current_key_id = ""
@@ -200,13 +202,15 @@ def pick_key():
     return best
 
 
-def mark_key_failed(kid):
-    """标记 key 故障，进入冷却期。"""
-    until = time.time() + FAIL_COOLDOWN_SEC
+def mark_key_failed(kid, reason="", cooldown=FAIL_COOLDOWN_SEC):
+    """标记 key 故障，进入冷却期。记录失败原因供冷却期报错。"""
+    until = time.time() + cooldown
     with _state_lock:
         _key_failures[kid] = until
-    log.warning("key %s marked failed until %s", kid,
-                datetime.fromtimestamp(until).strftime("%H:%M:%S"))
+        if reason:
+            _key_failure_reason[kid] = reason
+    log.warning("key %s marked failed until %s (%ds): %s", kid,
+                datetime.fromtimestamp(until).strftime("%H:%M:%S"), cooldown, reason)
 
 
 def record_route(kid):
@@ -332,6 +336,7 @@ class RouterHandler(BaseHTTPRequestHandler):
                     "monthly_pct": usage.get("monthly_pct", None),
                     "healthy": usage.get("subscribed", True) and (time.time() >= fail_until),
                     "in_cooldown": time.time() < fail_until,
+                    "fail_reason": _key_failure_reason.get(kid, ""),
                     "error": usage.get("error", False),
                     "hits": _route_stats.get(kid, 0),
                     "last_update": usage.get("last_update", ""),
@@ -459,14 +464,35 @@ class RouterHandler(BaseHTTPRequestHandler):
                     # 401(未授权/CreditsError)/402/403/429/5xx → 标记 key 故障
                     status = result.get("status", 0)
                     if status in (401, 402, 403, 429) or status >= 500:
-                        mark_key_failed(kid)
+                        if "Router.Unavailable" in last_error:
+                            # 上游模型级故障：换 key 无意义 → 全体短冷却并立即失败返回
+                            with _state_lock:
+                                until = time.time() + UPSTREAM_COOLDOWN_SEC
+                                for k2 in _keys:
+                                    _key_failures[k2["key_id"]] = until
+                                    _key_failure_reason[k2["key_id"]] = last_error
+                            log.warning("upstream Router.Unavailable → all keys short-cooldown %ds", UPSTREAM_COOLDOWN_SEC)
+                            success = False
+                            break
+                        mark_key_failed(kid, last_error)
             except Exception as e:
                 last_error = str(e)
                 log.warning("⚠ %s via %s exception: %s", model_name, kid, e)
-                mark_key_failed(kid)
+                mark_key_failed(kid, last_error)
 
         if not success:
             elapsed = time.time() - start
+            if not last_error:
+                # 所有 key 在冷却中被跳过 → 组装原因+最早到期时间
+                now = time.time()
+                cooling = [(kid, until) for kid, until in failures_snapshot.items() if now < until]
+                if cooling:
+                    earliest = min(until for _, until in cooling)
+                    reasons = {_key_failure_reason.get(kid, "unknown") for kid, _ in cooling}
+                    reason_str = "; ".join(sorted(r for r in reasons if r and r != "unknown"))
+                    last_error = (f"all keys in cooldown until "
+                                  f"{datetime.fromtimestamp(earliest).strftime('%H:%M:%S')}"
+                                  f"{' (' + reason_str + ')' if reason_str else ''}")
             log.error("✗ %s ALL KEYS EXHAUSTED (%.2fs): %s", model_name, elapsed, last_error)
             self._error_response(502, f"All OCG keys exhausted: {last_error}")
 
@@ -597,6 +623,7 @@ def _bg_refresh():
                 expired = [k for k, v in _key_failures.items() if now >= v]
                 for k in expired:
                     del _key_failures[k]
+                    _key_failure_reason.pop(k, None)
                     log.info("key %s cooldown expired, restored", k)
             log.info("bg refresh: %d keys, healthy pool", len(_keys))
         except Exception as e: