diff --git a/gateway/scripts/ocg_router.py b/gateway/scripts/ocg_router.py index 29a5e50..dbea9e0 100644 --- a/gateway/scripts/ocg_router.py +++ b/gateway/scripts/ocg_router.py @@ -67,6 +67,7 @@ _keys: list[dict] = [] # [{key_id, label, api_key, workspace_id}] _key_usage: dict[str, dict] = {} # key_id → {rolling_pct, weekly_pct, monthly_pct} _key_failures: dict[str, float] = {} # key_id → fail_until_timestamp _key_failure_reason: dict[str, str] = {} # key_id → 最近失败原因(冷却跳过时用于报错) +_key_fail_count: dict[str, int] = {} # key_id → 连续失败次数(指数退避基数,2026-08-14 老莫) _route_stats: dict[str, int] = {} # key_id → hit_count _total_hits = 0 _current_key_id = "" @@ -233,24 +234,44 @@ def pick_key_round_robin(): return _keys[0] -def mark_key_failed(kid, reason="", cooldown=FAIL_COOLDOWN_SEC): - """标记 key 故障,进入冷却期。记录失败原因供冷却期报错。""" - until = time.time() + cooldown +def mark_key_failed(kid, reason="", cooldown=None): + """标记 key 故障,进入指数退避冷却期(按 key 独立统计失败次数)。 + 2026-08-14 老莫设计:失败冷却时长指数增加(base × 2^count),上限 600s(10分钟)。 + 空输出/报错 → 立即换下一个 key,同时该 key 进入指数退避冷却。""" + now = time.time() with _state_lock: + # 失败计数 +1(独立统计) + _key_fail_count[kid] = _key_fail_count.get(kid, 0) + 1 + count = _key_fail_count[kid] + # 指数退避:base=2s, 每次失败 ×2,上限 600s(10分钟) + base = 2 + cool = min(base * (2 ** (count - 1)), 600) + until = now + cool _key_failures[kid] = until if reason: _key_failure_reason[kid] = reason - log.warning("key %s marked failed until %s (%ds): %s", kid, - datetime.fromtimestamp(until).strftime("%H:%M:%S"), cooldown, reason) + log.warning("key %s marked failed (count=%d, cooldown=%ds): %s", kid, count, cool, reason) + + +def reset_key_fail(kid): + """key 成功 → 重置失败计数(清除指数退避)""" + with _state_lock: + if kid in _key_fail_count: + del _key_fail_count[kid] + if kid in _key_failures: + del _key_failures[kid] + if kid in _key_failure_reason: + del _key_failure_reason[kid] def record_route(kid): - """记录一次成功路由。""" + """记录一次成功路由 + 重置失败计数(指数退避清除)。""" global _total_hits, _current_key_id with _state_lock: _route_stats[kid] = _route_stats.get(kid, 0) + 1 _total_hits += 1 _current_key_id = kid + reset_key_fail(kid) # 成功重置失败计数 # ═══════════════════════════════════════════════════════════════