fix(router/dashboard): 429按打满档位精确冷却+limitName兜底; mark_key_failed尊重显式cooldown(原死参数上限600s); weekly>=100不派单且healthy对齐; /api/usage三来源统一stale标记+前端置灰
This commit is contained in:
hmo
@@ -9,9 +9,10 @@ Flask app on :5803. Monitors agents across platforms via:
|
||||
|
||||
Auto-recovery: restarts local Windows agents after 3 consecutive offline checks.
|
||||
"""
|
||||
import random
|
||||
import os, sys, re, json, time, socket, subprocess, logging, urllib.request, sqlite3, shutil, threading
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timedelta
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from flask import Flask, jsonify, request, send_from_directory
|
||||
|
||||
# ---- Paths (platform-agnostic: auto-detect from script location) ----
|
||||
@@ -611,14 +612,21 @@ def _augment_wechat_status(entry):
|
||||
entry["qr_timestamp"] = None
|
||||
entry["session_age_hours"] = 0
|
||||
|
||||
# 1. Fetch QR code (always available)
|
||||
# 二维码【不再自动拉取】：避免 dashboard 每10秒轮询时自动刷新二维码（用户要求改手动获取）。
|
||||
# 只从"手动获取缓存"读取——用户点"获取登录二维码"后由 /api/wechat/status 写入；
|
||||
# 且仅当缓存未过期（二维码约5分钟有效）才返回，过期视为无 → 前端显示"获取登录二维码"按钮。
|
||||
try:
|
||||
req = urllib.request.Request("http://localhost:3001/login?token=mowechat_fixed_token_001")
|
||||
html = urllib.request.urlopen(req, timeout=5).read().decode("utf-8", errors="replace")
|
||||
m = re.search(r'qrcode\.makeCode\("([^"]+)"\)', html)
|
||||
if m:
|
||||
entry["qr_url"] = m.group(1)
|
||||
entry["qr_timestamp"] = now.isoformat()
|
||||
cache_file = str(_GATEWAY_DIR / "temp" / "wechat_qr_cache.json")
|
||||
if os.path.exists(cache_file):
|
||||
with open(cache_file, encoding="utf-8") as f:
|
||||
qc = json.load(f)
|
||||
qurl = qc.get("qr_url")
|
||||
qts = qc.get("ts", "")
|
||||
if qurl and qts:
|
||||
age_sec = (now - datetime.fromisoformat(qts)).total_seconds()
|
||||
if age_sec < 300: # 二维码约5分钟有效
|
||||
entry["qr_url"] = qurl
|
||||
entry["qr_timestamp"] = qts
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@@ -1376,6 +1384,27 @@ def api_rdp_toggle():
|
||||
return jsonify({"ok": False, "error": str(e)})
|
||||
|
||||
|
||||
@app.route("/api/rdp/progress")
|
||||
def api_rdp_progress():
|
||||
"""RDP enable 实时进度 — proxy to xmpp_bot on Windows (polls rdp_progress.json)."""
|
||||
try:
|
||||
data = _bridge_post("/rdp", {"action": "progress"}, timeout=8)
|
||||
return jsonify(data)
|
||||
except Exception as e:
|
||||
return jsonify({"ok": False, "state": "unknown", "message": "progress 查询失败", "error": str(e), "steps": []})
|
||||
|
||||
|
||||
@app.route("/api/rdp/enable_log")
|
||||
def api_rdp_enable_log():
|
||||
"""RDP enable 结构化日志 — proxy to xmpp_bot on Windows (for post-mortem debug)."""
|
||||
try:
|
||||
n = request.args.get("lines", 80, type=int)
|
||||
data = _bridge_post("/rdp", {"action": "enable_log", "lines": n}, timeout=8)
|
||||
return jsonify(data)
|
||||
except Exception as e:
|
||||
return jsonify({"ok": False, "error": str(e), "lines": []})
|
||||
|
||||
|
||||
# ════════════════════════════════════════════════════════════
|
||||
# OpenCode Go Usage Monitor — 4个账号用量配额监控 (246 本地采集)
|
||||
# See gateway/scripts/specs/usage_monitor.json
|
||||
@@ -1429,6 +1458,23 @@ def api_usage():
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 新鲜度统一标记（2026-08-26）：三种来源一律按 last_update_iso 算年龄
|
||||
try:
|
||||
_now_dt = datetime.now(timezone.utc)
|
||||
for a in accounts:
|
||||
lu = a.get("last_update_iso")
|
||||
a["age_minutes"] = None
|
||||
a["stale"] = True
|
||||
if lu:
|
||||
dt = datetime.fromisoformat(lu)
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
age_min = (_now_dt - dt).total_seconds() / 60.0
|
||||
a["age_minutes"] = int(round(age_min))
|
||||
a["stale"] = age_min > 60
|
||||
except Exception as _e:
|
||||
log.warning("usage: stale-mark failed: %s", _e)
|
||||
|
||||
if not accounts:
|
||||
return jsonify({"ok": False, "error": "no data yet",
|
||||
"last_refresh_iso": None, "accounts": [], "providers": {}})
|
||||
@@ -1462,17 +1508,17 @@ def api_usage_refresh():
|
||||
if proc.returncode != 0 and proc.stderr:
|
||||
log.warning(f"usage_collector stderr:\n{proc.stderr[-1000:]}")
|
||||
|
||||
# 2. Kimi 远程采集（SSH 到 Windows）+ scp 回 246
|
||||
try:
|
||||
_collect_kimi_usage()
|
||||
except Exception as e:
|
||||
log.warning(f"usage: Kimi collection failed: {e}")
|
||||
# 2. Kimi 远程采集（已禁用 by xxm 2026-08-22，CDP弹窗骚扰）
|
||||
# try:
|
||||
# _collect_kimi_usage()
|
||||
# except Exception as e:
|
||||
# log.warning(f"usage: Kimi collection failed: {e}")
|
||||
|
||||
# 3. SenseNova 远程采集（SSH 到 Windows，CDP 刮 console 余量）
|
||||
try:
|
||||
_collect_sensenova_usage()
|
||||
except Exception as e:
|
||||
log.warning(f"usage: SenseNova collection failed: {e}")
|
||||
# 3. SenseNova 远程采集（已禁用 by xxm 2026-08-22）
|
||||
# try:
|
||||
# _collect_sensenova_usage()
|
||||
# except Exception as e:
|
||||
# log.warning(f"usage: SenseNova collection failed: {e}")
|
||||
except Exception as e:
|
||||
log.error(f"usage: collector crash: {e}")
|
||||
finally:
|
||||
@@ -1646,7 +1692,7 @@ def api_keys():
|
||||
|
||||
|
||||
# ── Auto-collect timer (every 5 min, keeps cookies alive) ──
|
||||
_USAGE_AUTO_INTERVAL = 300
|
||||
_USAGE_AUTO_INTERVAL = 1800
|
||||
|
||||
def _start_usage_auto_timer():
|
||||
"""后台线程：OCG 每 5 分钟自动采集；Kimi/SenseNova 每 30 分钟采集（SSH 到 Windows）。"""
|
||||
@@ -1666,22 +1712,16 @@ def _start_usage_auto_timer():
|
||||
if proc.returncode != 0 and proc.stderr:
|
||||
log.warning(f"usage auto-timer stderr:\n{proc.stderr[-800:]}")
|
||||
|
||||
# Kimi/SenseNova：每 30 分钟刷新一次（SSH 到 Windows + CDP，较重）
|
||||
tick += 1
|
||||
if tick % _KIMI_INTERVAL_MULT == 0:
|
||||
try:
|
||||
_collect_kimi_usage()
|
||||
except Exception as e:
|
||||
log.warning(f"usage auto-timer: Kimi failed: {e}")
|
||||
try:
|
||||
_collect_sensenova_usage()
|
||||
except Exception as e:
|
||||
log.warning(f"usage auto-timer: SenseNova failed: {e}")
|
||||
# Kimi/SenseNova auto-timer disabled by xxm 2026-08-22 (CDP弹窗骚扰)
|
||||
# tick += 1
|
||||
# if tick % _KIMI_INTERVAL_MULT == 0:
|
||||
# _collect_kimi_usage()
|
||||
# _collect_sensenova_usage()
|
||||
_usage_collector_running = False
|
||||
except Exception as e:
|
||||
log.error(f"usage auto-timer error: {e}")
|
||||
_usage_collector_running = False
|
||||
time.sleep(_USAGE_AUTO_INTERVAL)
|
||||
time.sleep(_USAGE_AUTO_INTERVAL + random.randint(-240, 240))
|
||||
|
||||
t = threading.Thread(target=_loop, name="usage_auto_timer", daemon=True)
|
||||
t.start()
|
||||
@@ -1861,6 +1901,13 @@ def api_wechat_status():
|
||||
if m:
|
||||
result["qr_url"] = m.group(1)
|
||||
result["qr_timestamp"] = now.isoformat()
|
||||
# 手动获取到二维码 → 写入缓存，dashboard infra status 据此显示（避免自动拉取刷新）
|
||||
try:
|
||||
cache_file = str(_GATEWAY_DIR / "temp" / "wechat_qr_cache.json")
|
||||
with open(cache_file, "w", encoding="utf-8") as _cf:
|
||||
json.dump({"qr_url": m.group(1), "ts": now.isoformat()}, _cf, ensure_ascii=False)
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as e:
|
||||
result["message"] = f"QR page fetch failed: {e}"
|
||||
|
||||
@@ -2006,7 +2053,7 @@ def api_wechat_trigger_login():
|
||||
import urllib.request as _ur
|
||||
try:
|
||||
req = _ur.Request(
|
||||
"http://127.0.0.1:3001/api/bot/login",
|
||||
"http://127.0.0.1:3001/api/bot/login?token=mowechat_fixed_token_001",
|
||||
data=b"",
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST",
|
||||
|
||||
@@ -45,7 +45,7 @@ ACCOUNTS_FILE = _SCRIPT_DIR / "usage_monitor" / "accounts.json"
|
||||
FAIL_COOLDOWN_SEC = 300 # 5 分钟（key 自身故障）
|
||||
UPSTREAM_COOLDOWN_SEC = 30 # 上游模型级故障（Router.Unavailable）短冷却：换 key 无意义
|
||||
# 用量数据刷新间隔
|
||||
USAGE_REFRESH_INTERVAL = 120 # 2 分钟
|
||||
USAGE_REFRESH_INTERVAL = 1800 # 统一30分钟节奏（2026-08-26）
|
||||
# 最大重试次数（所有 key 耗尽）
|
||||
MAX_RETRIES = 6
|
||||
# 单次上游请求保底超时（秒）：上游处理大请求（1MB+）可能要 60-90s+，
|
||||
@@ -154,12 +154,13 @@ def refresh_state():
|
||||
global _keys, _key_usage
|
||||
_keys = load_accounts()
|
||||
_key_usage = load_usage()
|
||||
# 合并 accounts.json 的订阅标记：采集失败时 usage 缺失该信息，
|
||||
# 以 accounts.json 的 renewal/subscribed 为准
|
||||
# 合并 accounts.json 的订阅标记（仅兜底）：usage_stats.json 是 CDP 实时检测，
|
||||
# 有该 key 的实时数据时以实时数据为准；只有用量数据缺失该 key 时才回退
|
||||
# 到 accounts.json 的 renewal/subscribed 静态标记（2026-08-20 修复：不再覆盖实时检测）
|
||||
for k in _keys:
|
||||
kid = k["key_id"]
|
||||
if not k.get("subscribed", True):
|
||||
_key_usage.setdefault(kid, {})["subscribed"] = False
|
||||
if kid not in _key_usage and not k.get("subscribed", True):
|
||||
_key_usage[kid] = {"subscribed": False}
|
||||
log.info("state refreshed: %d keys, %d with usage data", len(_keys), len(_key_usage))
|
||||
|
||||
|
||||
@@ -181,6 +182,10 @@ def _key_health_score(kid):
|
||||
if not usage.get("subscribed", True):
|
||||
return 999
|
||||
|
||||
# weekly 打满：沉底与冷却同级，reset 前不派单（2026-08-26）
|
||||
if (usage.get("weekly_pct", 0) or 0) >= 100:
|
||||
return 888
|
||||
|
||||
# 用量采集失败 → 降权但可用（不因采集问题误杀 key）
|
||||
if usage.get("error"):
|
||||
return 500
|
||||
@@ -233,6 +238,8 @@ def pick_key_round_robin():
|
||||
usage = _key_usage.get(kid, {})
|
||||
if not usage.get("subscribed", True):
|
||||
continue
|
||||
if (usage.get("weekly_pct", 0) or 0) >= 100:
|
||||
continue # weekly 打满（2026-08-26）
|
||||
# 选中：更新索引到下一个
|
||||
_rr_index = (idx + 1) % n
|
||||
return k
|
||||
@@ -251,7 +258,11 @@ def mark_key_failed(kid, reason="", cooldown=None):
|
||||
count = _key_fail_count[kid]
|
||||
# 指数退避：base=2s, 每次失败 ×2，上限 600s（10分钟）
|
||||
base = 2
|
||||
cool = min(base * (2 ** (count - 1)), 600)
|
||||
if cooldown and cooldown > 0:
|
||||
# explicit cooldown wins over exponential-backoff cap (2026-08-26)
|
||||
cool = max(int(cooldown), 1)
|
||||
else:
|
||||
cool = min(base * (2 ** (count - 1)), 600)
|
||||
until = now + cool
|
||||
_key_failures[kid] = until
|
||||
if reason:
|
||||
@@ -392,7 +403,9 @@ class RouterHandler(BaseHTTPRequestHandler):
|
||||
"rolling_pct": usage.get("rolling_pct", None),
|
||||
"weekly_pct": usage.get("weekly_pct", None),
|
||||
"monthly_pct": usage.get("monthly_pct", None),
|
||||
"healthy": usage.get("subscribed", True) and (time.time() >= fail_until),
|
||||
"healthy": (usage.get("subscribed", True)
|
||||
and (time.time() >= fail_until)
|
||||
and not ((usage.get("weekly_pct", 0) or 0) >= 100)),
|
||||
"in_cooldown": time.time() < fail_until,
|
||||
"fail_reason": _key_failure_reason.get(kid, ""),
|
||||
"error": usage.get("error", False),
|
||||
@@ -602,7 +615,34 @@ class RouterHandler(BaseHTTPRequestHandler):
|
||||
break
|
||||
mark_key_failed(kid, last_error, cooldown=FAIL_COOLDOWN_SEC)
|
||||
elif status in (402, 429):
|
||||
mark_key_failed(kid, last_error, cooldown=60)
|
||||
# 用 usage_stats 的 reset_in_sec 设定精确冷却时间
|
||||
# 2026-08-26 定稿：只在打满档位取最远 reset；空集合按 limitName 分级兜底。
|
||||
kid_usage = usage_snapshot.get(kid, {})
|
||||
tiers = [kid_usage.get('rolling') or {},
|
||||
kid_usage.get('weekly') or {},
|
||||
kid_usage.get('monthly') or {}]
|
||||
exhausted = [t['reset_in_sec'] for t in tiers
|
||||
if t.get('usage_percent', 0) >= 100 and t.get('reset_in_sec', 0) > 0]
|
||||
limit_name = ''
|
||||
try:
|
||||
eidx = last_error.find("{")
|
||||
if eidx >= 0:
|
||||
meta = (json.loads(last_error[eidx:]).get('error') or {}).get('metadata') or {}
|
||||
limit_name = str(meta.get('limitName', '')).strip().lower()
|
||||
except Exception:
|
||||
limit_name = ''
|
||||
if exhausted:
|
||||
cooldown_sec = max(exhausted) + 1800 # 快照年龄补偿30分钟
|
||||
basis = 'maxed-tiers'
|
||||
elif limit_name in ('weekly', 'monthly'):
|
||||
cooldown_sec = 86400
|
||||
basis = 'limitName:' + limit_name
|
||||
else:
|
||||
cooldown_sec = 3600
|
||||
basis = 'default'
|
||||
log.warning("key %s 429, cooldown %.0fs (%s; limitName=%s)",
|
||||
kid, cooldown_sec, basis, limit_name or 'unknown')
|
||||
mark_key_failed(kid, last_error, cooldown=cooldown_sec)
|
||||
elif status >= 500:
|
||||
# 上游服务端错误（503/500/Internal/Router.Unavailable）：
|
||||
# 只是当前 key 的上游实例可能有问题，换下一个 key 重试（不冷却）。
|
||||
@@ -633,7 +673,7 @@ class RouterHandler(BaseHTTPRequestHandler):
|
||||
self._error_response(400, last_error)
|
||||
else:
|
||||
log.error("✗ %s ALL KEYS EXHAUSTED (%.2fs): %s", model_name, elapsed, last_error)
|
||||
self._error_response(502, f"All OCG keys exhausted: {last_error}")
|
||||
self._error_response(429, f"Rate limited, retry later: {last_error}")
|
||||
|
||||
def _forward_request(self, api_key, body, is_stream):
|
||||
"""转发单次请求到指定 key。返回 {ok, status, error}。
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user