From 8c0c0d08bf0593122e6bf29d0d0a8078246716e9 Mon Sep 17 00:00:00 2001 From: hmo Date: Wed, 26 Aug 2026 12:20:47 +0800 Subject: [PATCH] =?UTF-8?q?fix(router/dashboard):=20429=E6=8C=89=E6=89=93?= =?UTF-8?q?=E6=BB=A1=E6=A1=A3=E4=BD=8D=E7=B2=BE=E7=A1=AE=E5=86=B7=E5=8D=B4?= =?UTF-8?q?+limitName=E5=85=9C=E5=BA=95;=20mark=5Fkey=5Ffailed=E5=B0=8A?= =?UTF-8?q?=E9=87=8D=E6=98=BE=E5=BC=8Fcooldown(=E5=8E=9F=E6=AD=BB=E5=8F=82?= =?UTF-8?q?=E6=95=B0=E4=B8=8A=E9=99=90600s);=20weekly>=3D100=E4=B8=8D?= =?UTF-8?q?=E6=B4=BE=E5=8D=95=E4=B8=94healthy=E5=AF=B9=E9=BD=90;=20/api/us?= =?UTF-8?q?age=E4=B8=89=E6=9D=A5=E6=BA=90=E7=BB=9F=E4=B8=80stale=E6=A0=87?= =?UTF-8?q?=E8=AE=B0+=E5=89=8D=E7=AB=AF=E7=BD=AE=E7=81=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- gateway/scripts/dashboard.py | 111 +- gateway/scripts/ocg_router.py | 58 +- gateway/scripts/templates/dashboard.html | 1414 +++++++++++----------- 3 files changed, 859 insertions(+), 724 deletions(-) diff --git a/gateway/scripts/dashboard.py b/gateway/scripts/dashboard.py index b3c57d0..3c0abde 100644 --- a/gateway/scripts/dashboard.py +++ b/gateway/scripts/dashboard.py @@ -9,9 +9,10 @@ Flask app on :5803. Monitors agents across platforms via: Auto-recovery: restarts local Windows agents after 3 consecutive offline checks. """ +import random import os, sys, re, json, time, socket, subprocess, logging, urllib.request, sqlite3, shutil, threading from pathlib import Path -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from flask import Flask, jsonify, request, send_from_directory # ---- Paths (platform-agnostic: auto-detect from script location) ---- @@ -611,14 +612,21 @@ def _augment_wechat_status(entry): entry["qr_timestamp"] = None entry["session_age_hours"] = 0 - # 1. Fetch QR code (always available) + # 二维码【不再自动拉取】：避免 dashboard 每10秒轮询时自动刷新二维码（用户要求改手动获取）。 + # 只从"手动获取缓存"读取——用户点"获取登录二维码"后由 /api/wechat/status 写入； + # 且仅当缓存未过期（二维码约5分钟有效）才返回，过期视为无 → 前端显示"获取登录二维码"按钮。 try: - req = urllib.request.Request("http://localhost:3001/login?token=mowechat_fixed_token_001") - html = urllib.request.urlopen(req, timeout=5).read().decode("utf-8", errors="replace") - m = re.search(r'qrcode\.makeCode\("([^"]+)"\)', html) - if m: - entry["qr_url"] = m.group(1) - entry["qr_timestamp"] = now.isoformat() + cache_file = str(_GATEWAY_DIR / "temp" / "wechat_qr_cache.json") + if os.path.exists(cache_file): + with open(cache_file, encoding="utf-8") as f: + qc = json.load(f) + qurl = qc.get("qr_url") + qts = qc.get("ts", "") + if qurl and qts: + age_sec = (now - datetime.fromisoformat(qts)).total_seconds() + if age_sec < 300: # 二维码约5分钟有效 + entry["qr_url"] = qurl + entry["qr_timestamp"] = qts except Exception: pass @@ -1376,6 +1384,27 @@ def api_rdp_toggle(): return jsonify({"ok": False, "error": str(e)}) +@app.route("/api/rdp/progress") +def api_rdp_progress(): + """RDP enable 实时进度 — proxy to xmpp_bot on Windows (polls rdp_progress.json).""" + try: + data = _bridge_post("/rdp", {"action": "progress"}, timeout=8) + return jsonify(data) + except Exception as e: + return jsonify({"ok": False, "state": "unknown", "message": "progress 查询失败", "error": str(e), "steps": []}) + + +@app.route("/api/rdp/enable_log") +def api_rdp_enable_log(): + """RDP enable 结构化日志 — proxy to xmpp_bot on Windows (for post-mortem debug).""" + try: + n = request.args.get("lines", 80, type=int) + data = _bridge_post("/rdp", {"action": "enable_log", "lines": n}, timeout=8) + return jsonify(data) + except Exception as e: + return jsonify({"ok": False, "error": str(e), "lines": []}) + + # ════════════════════════════════════════════════════════════ # OpenCode Go Usage Monitor — 4个账号用量配额监控 (246 本地采集) # See gateway/scripts/specs/usage_monitor.json @@ -1429,6 +1458,23 @@ def api_usage(): except Exception: pass + # 新鲜度统一标记（2026-08-26）：三种来源一律按 last_update_iso 算年龄 + try: + _now_dt = datetime.now(timezone.utc) + for a in accounts: + lu = a.get("last_update_iso") + a["age_minutes"] = None + a["stale"] = True + if lu: + dt = datetime.fromisoformat(lu) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + age_min = (_now_dt - dt).total_seconds() / 60.0 + a["age_minutes"] = int(round(age_min)) + a["stale"] = age_min > 60 + except Exception as _e: + log.warning("usage: stale-mark failed: %s", _e) + if not accounts: return jsonify({"ok": False, "error": "no data yet", "last_refresh_iso": None, "accounts": [], "providers": {}}) @@ -1462,17 +1508,17 @@ def api_usage_refresh(): if proc.returncode != 0 and proc.stderr: log.warning(f"usage_collector stderr:\n{proc.stderr[-1000:]}") - # 2. Kimi 远程采集（SSH 到 Windows）+ scp 回 246 - try: - _collect_kimi_usage() - except Exception as e: - log.warning(f"usage: Kimi collection failed: {e}") + # 2. Kimi 远程采集（已禁用 by xxm 2026-08-22，CDP弹窗骚扰） + # try: + # _collect_kimi_usage() + # except Exception as e: + # log.warning(f"usage: Kimi collection failed: {e}") - # 3. SenseNova 远程采集（SSH 到 Windows，CDP 刮 console 余量） - try: - _collect_sensenova_usage() - except Exception as e: - log.warning(f"usage: SenseNova collection failed: {e}") + # 3. SenseNova 远程采集（已禁用 by xxm 2026-08-22） + # try: + # _collect_sensenova_usage() + # except Exception as e: + # log.warning(f"usage: SenseNova collection failed: {e}") except Exception as e: log.error(f"usage: collector crash: {e}") finally: @@ -1646,7 +1692,7 @@ def api_keys(): # ── Auto-collect timer (every 5 min, keeps cookies alive) ── -_USAGE_AUTO_INTERVAL = 300 +_USAGE_AUTO_INTERVAL = 1800 def _start_usage_auto_timer(): """后台线程：OCG 每 5 分钟自动采集；Kimi/SenseNova 每 30 分钟采集（SSH 到 Windows）。""" @@ -1666,22 +1712,16 @@ def _start_usage_auto_timer(): if proc.returncode != 0 and proc.stderr: log.warning(f"usage auto-timer stderr:\n{proc.stderr[-800:]}") - # Kimi/SenseNova：每 30 分钟刷新一次（SSH 到 Windows + CDP，较重） - tick += 1 - if tick % _KIMI_INTERVAL_MULT == 0: - try: - _collect_kimi_usage() - except Exception as e: - log.warning(f"usage auto-timer: Kimi failed: {e}") - try: - _collect_sensenova_usage() - except Exception as e: - log.warning(f"usage auto-timer: SenseNova failed: {e}") + # Kimi/SenseNova auto-timer disabled by xxm 2026-08-22 (CDP弹窗骚扰) + # tick += 1 + # if tick % _KIMI_INTERVAL_MULT == 0: + # _collect_kimi_usage() + # _collect_sensenova_usage() _usage_collector_running = False except Exception as e: log.error(f"usage auto-timer error: {e}") _usage_collector_running = False - time.sleep(_USAGE_AUTO_INTERVAL) + time.sleep(_USAGE_AUTO_INTERVAL + random.randint(-240, 240)) t = threading.Thread(target=_loop, name="usage_auto_timer", daemon=True) t.start() @@ -1861,6 +1901,13 @@ def api_wechat_status(): if m: result["qr_url"] = m.group(1) result["qr_timestamp"] = now.isoformat() + # 手动获取到二维码 → 写入缓存，dashboard infra status 据此显示（避免自动拉取刷新） + try: + cache_file = str(_GATEWAY_DIR / "temp" / "wechat_qr_cache.json") + with open(cache_file, "w", encoding="utf-8") as _cf: + json.dump({"qr_url": m.group(1), "ts": now.isoformat()}, _cf, ensure_ascii=False) + except Exception: + pass except Exception as e: result["message"] = f"QR page fetch failed: {e}" @@ -2006,7 +2053,7 @@ def api_wechat_trigger_login(): import urllib.request as _ur try: req = _ur.Request( - "http://127.0.0.1:3001/api/bot/login", + "http://127.0.0.1:3001/api/bot/login?token=mowechat_fixed_token_001", data=b"", headers={"Content-Type": "application/json"}, method="POST", diff --git a/gateway/scripts/ocg_router.py b/gateway/scripts/ocg_router.py index fec9646..dd285de 100644 --- a/gateway/scripts/ocg_router.py +++ b/gateway/scripts/ocg_router.py @@ -45,7 +45,7 @@ ACCOUNTS_FILE = _SCRIPT_DIR / "usage_monitor" / "accounts.json" FAIL_COOLDOWN_SEC = 300 # 5 分钟（key 自身故障） UPSTREAM_COOLDOWN_SEC = 30 # 上游模型级故障（Router.Unavailable）短冷却：换 key 无意义 # 用量数据刷新间隔 -USAGE_REFRESH_INTERVAL = 120 # 2 分钟 +USAGE_REFRESH_INTERVAL = 1800 # 统一30分钟节奏（2026-08-26） # 最大重试次数（所有 key 耗尽） MAX_RETRIES = 6 # 单次上游请求保底超时（秒）：上游处理大请求（1MB+）可能要 60-90s+， @@ -154,12 +154,13 @@ def refresh_state(): global _keys, _key_usage _keys = load_accounts() _key_usage = load_usage() - # 合并 accounts.json 的订阅标记：采集失败时 usage 缺失该信息， - # 以 accounts.json 的 renewal/subscribed 为准 + # 合并 accounts.json 的订阅标记（仅兜底）：usage_stats.json 是 CDP 实时检测， + # 有该 key 的实时数据时以实时数据为准；只有用量数据缺失该 key 时才回退 + # 到 accounts.json 的 renewal/subscribed 静态标记（2026-08-20 修复：不再覆盖实时检测） for k in _keys: kid = k["key_id"] - if not k.get("subscribed", True): - _key_usage.setdefault(kid, {})["subscribed"] = False + if kid not in _key_usage and not k.get("subscribed", True): + _key_usage[kid] = {"subscribed": False} log.info("state refreshed: %d keys, %d with usage data", len(_keys), len(_key_usage)) @@ -181,6 +182,10 @@ def _key_health_score(kid): if not usage.get("subscribed", True): return 999 + # weekly 打满：沉底与冷却同级，reset 前不派单（2026-08-26） + if (usage.get("weekly_pct", 0) or 0) >= 100: + return 888 + # 用量采集失败 → 降权但可用（不因采集问题误杀 key） if usage.get("error"): return 500 @@ -233,6 +238,8 @@ def pick_key_round_robin(): usage = _key_usage.get(kid, {}) if not usage.get("subscribed", True): continue + if (usage.get("weekly_pct", 0) or 0) >= 100: + continue # weekly 打满（2026-08-26） # 选中：更新索引到下一个 _rr_index = (idx + 1) % n return k @@ -251,7 +258,11 @@ def mark_key_failed(kid, reason="", cooldown=None): count = _key_fail_count[kid] # 指数退避：base=2s, 每次失败 ×2，上限 600s（10分钟） base = 2 - cool = min(base * (2 ** (count - 1)), 600) + if cooldown and cooldown > 0: + # explicit cooldown wins over exponential-backoff cap (2026-08-26) + cool = max(int(cooldown), 1) + else: + cool = min(base * (2 ** (count - 1)), 600) until = now + cool _key_failures[kid] = until if reason: @@ -392,7 +403,9 @@ class RouterHandler(BaseHTTPRequestHandler): "rolling_pct": usage.get("rolling_pct", None), "weekly_pct": usage.get("weekly_pct", None), "monthly_pct": usage.get("monthly_pct", None), - "healthy": usage.get("subscribed", True) and (time.time() >= fail_until), + "healthy": (usage.get("subscribed", True) + and (time.time() >= fail_until) + and not ((usage.get("weekly_pct", 0) or 0) >= 100)), "in_cooldown": time.time() < fail_until, "fail_reason": _key_failure_reason.get(kid, ""), "error": usage.get("error", False), @@ -602,7 +615,34 @@ class RouterHandler(BaseHTTPRequestHandler): break mark_key_failed(kid, last_error, cooldown=FAIL_COOLDOWN_SEC) elif status in (402, 429): - mark_key_failed(kid, last_error, cooldown=60) + # 用 usage_stats 的 reset_in_sec 设定精确冷却时间 + # 2026-08-26 定稿：只在打满档位取最远 reset；空集合按 limitName 分级兜底。 + kid_usage = usage_snapshot.get(kid, {}) + tiers = [kid_usage.get('rolling') or {}, + kid_usage.get('weekly') or {}, + kid_usage.get('monthly') or {}] + exhausted = [t['reset_in_sec'] for t in tiers + if t.get('usage_percent', 0) >= 100 and t.get('reset_in_sec', 0) > 0] + limit_name = '' + try: + eidx = last_error.find("{") + if eidx >= 0: + meta = (json.loads(last_error[eidx:]).get('error') or {}).get('metadata') or {} + limit_name = str(meta.get('limitName', '')).strip().lower() + except Exception: + limit_name = '' + if exhausted: + cooldown_sec = max(exhausted) + 1800 # 快照年龄补偿30分钟 + basis = 'maxed-tiers' + elif limit_name in ('weekly', 'monthly'): + cooldown_sec = 86400 + basis = 'limitName:' + limit_name + else: + cooldown_sec = 3600 + basis = 'default' + log.warning("key %s 429, cooldown %.0fs (%s; limitName=%s)", + kid, cooldown_sec, basis, limit_name or 'unknown') + mark_key_failed(kid, last_error, cooldown=cooldown_sec) elif status >= 500: # 上游服务端错误（503/500/Internal/Router.Unavailable）： # 只是当前 key 的上游实例可能有问题，换下一个 key 重试（不冷却）。 @@ -633,7 +673,7 @@ class RouterHandler(BaseHTTPRequestHandler): self._error_response(400, last_error) else: log.error("✗ %s ALL KEYS EXHAUSTED (%.2fs): %s", model_name, elapsed, last_error) - self._error_response(502, f"All OCG keys exhausted: {last_error}") + self._error_response(429, f"Rate limited, retry later: {last_error}") def _forward_request(self, api_key, body, is_stream): """转发单次请求到指定 key。返回 {ok, status, error}。 diff --git a/gateway/scripts/templates/dashboard.html b/gateway/scripts/templates/dashboard.html index 2f2020f..b6a63e8 100644 --- a/gateway/scripts/templates/dashboard.html +++ b/gateway/scripts/templates/dashboard.html @@ -1,684 +1,732 @@ - -AgentsMeeting Dashboard - -

AgentsMeeting

Dashboard
-
-
- \ No newline at end of file