feat: ocg_router key降级改指数退避——按key独立统计失败次数,失败冷却base2s×2^count指数增加(上限600s/10分钟),成功后重置计数; 空输出/报错立即换key+该key进入指数退避

This commit is contained in:
2026-08-14 10:08:51 +08:00
parent ef4ead6a03
commit 85ef30f896
+27 -6
View File
@@ -67,6 +67,7 @@ _keys: list[dict] = [] # [{key_id, label, api_key, workspace_id}]
_key_usage: dict[str, dict] = {} # key_id → {rolling_pct, weekly_pct, monthly_pct}
_key_failures: dict[str, float] = {} # key_id → fail_until_timestamp
_key_failure_reason: dict[str, str] = {} # key_id → 最近失败原因(冷却跳过时用于报错)
_key_fail_count: dict[str, int] = {} # key_id → 连续失败次数(指数退避基数,2026-08-14 老莫)
_route_stats: dict[str, int] = {} # key_id → hit_count
_total_hits = 0
_current_key_id = ""
@@ -233,24 +234,44 @@ def pick_key_round_robin():
return _keys[0]
def mark_key_failed(kid, reason="", cooldown=FAIL_COOLDOWN_SEC):
"""标记 key 故障,进入冷却期。记录失败原因供冷却期报错。"""
until = time.time() + cooldown
def mark_key_failed(kid, reason="", cooldown=None):
"""标记 key 故障,进入指数退避冷却期(按 key 独立统计失败次数)。
2026-08-14 老莫设计:失败冷却时长指数增加(base × 2^count),上限 600s10分钟)。
空输出/报错 → 立即换下一个 key,同时该 key 进入指数退避冷却。"""
now = time.time()
with _state_lock:
# 失败计数 +1(独立统计)
_key_fail_count[kid] = _key_fail_count.get(kid, 0) + 1
count = _key_fail_count[kid]
# 指数退避:base=2s, 每次失败 ×2,上限 600s(10分钟)
base = 2
cool = min(base * (2 ** (count - 1)), 600)
until = now + cool
_key_failures[kid] = until
if reason:
_key_failure_reason[kid] = reason
log.warning("key %s marked failed until %s (%ds): %s", kid,
datetime.fromtimestamp(until).strftime("%H:%M:%S"), cooldown, reason)
log.warning("key %s marked failed (count=%d, cooldown=%ds): %s", kid, count, cool, reason)
def reset_key_fail(kid):
"""key 成功 → 重置失败计数(清除指数退避)"""
with _state_lock:
if kid in _key_fail_count:
del _key_fail_count[kid]
if kid in _key_failures:
del _key_failures[kid]
if kid in _key_failure_reason:
del _key_failure_reason[kid]
def record_route(kid):
"""记录一次成功路由。"""
"""记录一次成功路由 + 重置失败计数(指数退避清除)"""
global _total_hits, _current_key_id
with _state_lock:
_route_stats[kid] = _route_stats.get(kid, 0) + 1
_total_hits += 1
_current_key_id = kid
reset_key_fail(kid) # 成功重置失败计数
# ═══════════════════════════════════════════════════════════════