feat: ocg_router key降级改指数退避——按key独立统计失败次数,失败冷却base2s×2^count指数增加(上限600s/10分钟),成功后重置计数; 空输出/报错立即换key+该key进入指数退避
This commit is contained in:
@@ -67,6 +67,7 @@ _keys: list[dict] = [] # [{key_id, label, api_key, workspace_id}]
|
||||
_key_usage: dict[str, dict] = {} # key_id → {rolling_pct, weekly_pct, monthly_pct}
|
||||
_key_failures: dict[str, float] = {} # key_id → fail_until_timestamp
|
||||
_key_failure_reason: dict[str, str] = {} # key_id → 最近失败原因(冷却跳过时用于报错)
|
||||
_key_fail_count: dict[str, int] = {} # key_id → 连续失败次数(指数退避基数,2026-08-14 老莫)
|
||||
_route_stats: dict[str, int] = {} # key_id → hit_count
|
||||
_total_hits = 0
|
||||
_current_key_id = ""
|
||||
@@ -233,24 +234,44 @@ def pick_key_round_robin():
|
||||
return _keys[0]
|
||||
|
||||
|
||||
def mark_key_failed(kid, reason="", cooldown=FAIL_COOLDOWN_SEC):
|
||||
"""标记 key 故障,进入冷却期。记录失败原因供冷却期报错。"""
|
||||
until = time.time() + cooldown
|
||||
def mark_key_failed(kid, reason="", cooldown=None):
|
||||
"""标记 key 故障,进入指数退避冷却期(按 key 独立统计失败次数)。
|
||||
2026-08-14 老莫设计:失败冷却时长指数增加(base × 2^count),上限 600s(10分钟)。
|
||||
空输出/报错 → 立即换下一个 key,同时该 key 进入指数退避冷却。"""
|
||||
now = time.time()
|
||||
with _state_lock:
|
||||
# 失败计数 +1(独立统计)
|
||||
_key_fail_count[kid] = _key_fail_count.get(kid, 0) + 1
|
||||
count = _key_fail_count[kid]
|
||||
# 指数退避:base=2s, 每次失败 ×2,上限 600s(10分钟)
|
||||
base = 2
|
||||
cool = min(base * (2 ** (count - 1)), 600)
|
||||
until = now + cool
|
||||
_key_failures[kid] = until
|
||||
if reason:
|
||||
_key_failure_reason[kid] = reason
|
||||
log.warning("key %s marked failed until %s (%ds): %s", kid,
|
||||
datetime.fromtimestamp(until).strftime("%H:%M:%S"), cooldown, reason)
|
||||
log.warning("key %s marked failed (count=%d, cooldown=%ds): %s", kid, count, cool, reason)
|
||||
|
||||
|
||||
def reset_key_fail(kid):
|
||||
"""key 成功 → 重置失败计数(清除指数退避)"""
|
||||
with _state_lock:
|
||||
if kid in _key_fail_count:
|
||||
del _key_fail_count[kid]
|
||||
if kid in _key_failures:
|
||||
del _key_failures[kid]
|
||||
if kid in _key_failure_reason:
|
||||
del _key_failure_reason[kid]
|
||||
|
||||
|
||||
def record_route(kid):
|
||||
"""记录一次成功路由。"""
|
||||
"""记录一次成功路由 + 重置失败计数(指数退避清除)。"""
|
||||
global _total_hits, _current_key_id
|
||||
with _state_lock:
|
||||
_route_stats[kid] = _route_stats.get(kid, 0) + 1
|
||||
_total_hits += 1
|
||||
_current_key_id = kid
|
||||
reset_key_fail(kid) # 成功重置失败计数
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
|
||||
Reference in New Issue
Block a user