From 7a43fec2b1a75b2f9c71ec7a9241b8092dc33ac8 Mon Sep 17 00:00:00 2001 From: hmo Date: Sun, 19 Jul 2026 19:46:08 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20multi-provider=20usage=20=E2=80=94=20Ki?= =?UTF-8?q?mi=20collector=20via=20CDP,=20unified=20/api/usage,=20frontend?= =?UTF-8?q?=20grouping?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- gateway/scripts/dashboard.py | 49 ++++-- gateway/scripts/templates/dashboard.html | 73 +++++---- gateway/scripts/usage_collector_kimi.py | 163 ++++++++++++++++++++ gateway/scripts/usage_collector_master.py | 173 ++++++++++++++++++++++ 4 files changed, 420 insertions(+), 38 deletions(-) create mode 100644 gateway/scripts/usage_collector_kimi.py create mode 100644 gateway/scripts/usage_collector_master.py diff --git a/gateway/scripts/dashboard.py b/gateway/scripts/dashboard.py index 0e26ef3..195cc14 100644 --- a/gateway/scripts/dashboard.py +++ b/gateway/scripts/dashboard.py @@ -1195,15 +1195,46 @@ _usage_collector_running = False @app.route("/api/usage") def api_usage(): - """直读本地 usage_stats.json（秒级响应，不依赖 Windows bot）""" - if not _USAGE_STATS_FILE.exists(): - return jsonify({"ok": False, "error": "no data yet (collect never ran)", - "last_refresh_iso": None, "accounts": []}) - try: - with open(str(_USAGE_STATS_FILE), "r", encoding="utf-8") as f: - return jsonify(json.load(f)) - except Exception as e: - return jsonify({"ok": False, "error": f"read failed: {e}", "accounts": []}) + """读取 usage_stats.json，合并 Kimi 数据（如果存在）。""" + accounts = [] + providers = {} + + # 1. 主 OCG 数据 + if _USAGE_STATS_FILE.exists(): + try: + with open(str(_USAGE_STATS_FILE), "r", encoding="utf-8") as f: + data = json.load(f) + for a in data.get("accounts", []): + a["provider"] = a.get("provider", "opencode_go") + accounts.append(a) + providers["opencode_go"] = providers.get("opencode_go", 0) + 1 + except Exception as e: + log.warning(f"usage: read failed: {e}") + + # 2. Kimi 数据 + kimi_file = TEMP_DIR / "usage_stats_kimi.json" + if kimi_file.exists(): + try: + with open(str(kimi_file), "r", encoding="utf-8") as f: + kimi_data = json.load(f) + for a in kimi_data.get("accounts", []): + a["provider"] = a.get("provider", "kimi") + accounts.append(a) + providers["kimi"] = providers.get("kimi", 0) + 1 + except Exception: + pass + + if not accounts: + return jsonify({"ok": False, "error": "no data yet", + "last_refresh_iso": None, "accounts": [], "providers": {}}) + return jsonify({ + "ok": True, + "last_refresh_iso": max((a.get("last_update_iso","") for a in accounts), default=None), + "account_count": len(accounts), + "collected_count": sum(1 for a in accounts if a.get("error") is None), + "providers": providers, + "accounts": accounts, + }) @app.route("/api/usage/refresh", methods=["POST"]) diff --git a/gateway/scripts/templates/dashboard.html b/gateway/scripts/templates/dashboard.html index cf4dea8..e33568f 100644 --- a/gateway/scripts/templates/dashboard.html +++ b/gateway/scripts/templates/dashboard.html @@ -368,7 +368,7 @@ document.getElementById('rdp-status-txt').textContent=fOn?'Connected':e4.rdp_ena var us=document.getElementById('usage-section'); if(!us){ us=document.createElement('div');us.id='usage-section';us.className='ps';us.style.marginTop='16px'; - us.innerHTML='

OpenCode Go Usage?§

' + us.innerHTML='

API 用量?§

' +'
' +'
' +'' @@ -404,39 +404,54 @@ document.getElementById('rdp-status-txt').textContent=fOn?'Connected':e4.rdp_ena lu.textContent='上次采集: '+d.getFullYear()+'-'+pad(d.getMonth()+1)+'-'+pad(d.getDate()) +' '+pad(d.getHours())+':'+pad(d.getMinutes())+':'+pad(d.getSeconds())+' CST'; } - // Render cards via innerHTML rebuild (simple data cards, no flicker risk) + // Render cards grouped by provider var html=''; var accts=e5.accounts||[]; + // Group by provider + var groups={},order=[]; for(var i=0;i=95){edge='#e53935';barColor='#e53935';} - else if(maxPct>=80){edge='#fb8c00';barColor='#fb8c00';} - html+='
' - +'
' - +''+esc(a.label||a.key_id||a.workspace_id||'?')+''; - if(a.session_expired){ - html+='SESSION EXPIRED'; - }else if(a.error){ - html+='⚠'; - }else{ - html+=''+maxPct+'% max'; - } - html+='
'; - if(a.session_expired||a.error){ - html+='
'+esc(a.error||'请重新登录此账号')+'
'; - }else{ - html+=_renderUsageMetric('5h rolling',a.rolling,barColor); - html+=_renderUsageMetric('Weekly',a.weekly,barColor); - html+=_renderUsageMetric('Monthly',a.monthly,barColor); - } - html+='
'; + var p=a.provider||'other'; + if(!groups[p]){groups[p]=[];order.push(p);} + groups[p].push(a); } + // Provider names + var pn={opencode_go:'OpenCode Go',kimi:'Kimi'}; + for(var gi=0;gi0){ + html+='
'+esc(pn[p]||p)+'
'; + html+='
'; + var ag=groups[p]; + for(var i=0;i=95){edge='#e53935';barColor='#e53935';} + else if(maxPct>=80){edge='#fb8c00';barColor='#fb8c00';} + html+='
' + +'
' + +''+esc(a.label||a.key_id||a.workspace_id||'?')+''; + if(a.session_expired){ + html+='SESSION EXPIRED'; + }else if(a.error){ + html+='⚠'; + }else{ + html+=''+maxPct+'% max'; + } + html+='
'; + if(a.session_expired||a.error){ + html+='
'+esc(a.error||'请重新登录此账号')+'
'; + }else{ + html+=_renderUsageMetric('5h rolling',a.rolling,barColor); + html+=_renderUsageMetric('Weekly',a.weekly,barColor); + html+=_renderUsageMetric('Monthly',a.monthly,barColor); + } + html+='
';} + html+='
';}} if(!accts.length){html='
未配置账号 (accounts.json 未就绪，或 cookies 未提取)
';} if(cards)cards.innerHTML=html; } diff --git a/gateway/scripts/usage_collector_kimi.py b/gateway/scripts/usage_collector_kimi.py new file mode 100644 index 0000000..159d80e --- /dev/null +++ b/gateway/scripts/usage_collector_kimi.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +""" +usage_collector_kimi.py — Kimi 用量采集（CDP 浏览器自动化） +========================================================== +连接 Chrome CDP proxy (localhost:3456)，打开 Kimi 控制台， +提取用量数据，保存到 usage_stats_kimi.json。 + +调用方式: python usage_collector_kimi.py +""" +import json, os, sys, time, urllib.request, urllib.error +from pathlib import Path +from datetime import datetime, timezone + +CDP = "http://localhost:3456" +KIMI_CONSOLE = "https://www.kimi.com/code/console" + +SCRIPT_DIR = Path(__file__).resolve().parent +GATEWAY_DIR = SCRIPT_DIR.parent +TEMP_DIR = GATEWAY_DIR / "temp" +os.makedirs(str(TEMP_DIR), exist_ok=True) + +OUTPUT_FILE = TEMP_DIR / "usage_stats_kimi.json" +KEY_ID = "key7" +PROVIDER = "kimi" + + +def cdp_get(path, timeout=10): + url = f"{CDP}{path}" + try: + with urllib.request.urlopen(url, timeout=timeout) as r: + return json.loads(r.read().decode()) + except Exception as e: + return {"error": str(e)} + + +def cdp_post(path, body="", timeout=15): + url = f"{CDP}{path}" + try: + data = body.encode("utf-8") if isinstance(body, str) else body + req = urllib.request.Request(url, data=data, method="POST") + req.add_header("Content-Type", "text/plain") + with urllib.request.urlopen(req, timeout=timeout) as r: + return r.read().decode() + except Exception as e: + return f"ERROR: {e}" + + +def cdp_eval(target_id, js, timeout=10): + return cdp_post(f"/eval?target={target_id}", js, timeout) + + +def main(): + print("[kimi] Starting collection...") + + # 1. Find or create a Kimi tab + targets = cdp_get("/targets") or [] + kimi_tab = None + for t in targets: + url = t.get("url", "") + if "kimi.com" in url: + kimi_tab = t + break + + if kimi_tab: + target_id = kimi_tab.get("targetId") or kimi_tab.get("id") + print(f"[kimi] Found existing tab: {target_id}") + # Navigate to console if needed + cur_url = kimi_tab.get("url", "") + if "code/console" not in cur_url: + print(f"[kimi] Navigating to console...") + cdp_get(f"/navigate?target={target_id}&url={KIMI_CONSOLE}") + time.sleep(3) + else: + print(f"[kimi] Opening new tab...") + result = cdp_get(f"/new?url={KIMI_CONSOLE}") + if isinstance(result, dict): + target_id = result.get("targetId") or result.get("id", "") + else: + target_id = str(result) + print(f"[kimi] New tab: {target_id}") + time.sleep(4) + + if not target_id: + print("[kimi] ERROR: no target ID") + return + + # 2. Wait for page load + extract data + print("[kimi] Extracting usage data...") + + # Simple extraction: get all text, parse with Python + js_get_text = "document.body ? document.body.innerText : ''" + raw_text = cdp_eval(target_id, js_get_text) + + # CDP eval returns JSON: {"value":"text content"} — unwrap it + try: + parsed = json.loads(raw_text) if isinstance(raw_text, str) else raw_text + text = parsed.get("value", "") if isinstance(parsed, dict) else str(raw_text) + except (json.JSONDecodeError, AttributeError): + text = str(raw_text) + + # Parse the text in Python + import re + + # Find all "N%" followed by "N 小时后重置" patterns + # The page has two: first = 本周用量, second = 频率明细 + pct_pattern = re.findall(r'(\d+)%', text) + hour_pattern = re.findall(r'(\d+)\s*小时后重置', text) + + weekly_pct = int(pct_pattern[0]) if len(pct_pattern) > 0 else 0 + weekly_hours = int(hour_pattern[0]) if len(hour_pattern) > 0 else 0 + rate_pct = int(pct_pattern[1]) if len(pct_pattern) > 1 else 0 + rate_hours = int(hour_pattern[1]) if len(hour_pattern) > 1 else 0 + + print(f"[kimi] Found {len(pct_pattern)} pct values, {len(hour_pattern)} hour values") + + print(f"[kimi] Weekly: {weekly_pct}% / {weekly_hours}h reset | Rate: {rate_pct}% / {rate_hours}h reset") + + # Also take a screenshot for debugging + try: + cdp_get(f"/screenshot?target={target_id}&file={TEMP_DIR}/kimi_console.png") + print("[kimi] Screenshot saved") + except: + pass + + # Build output with Python-parsed data + wp = weekly_pct or 0 + wh = weekly_hours or 0 + rp = rate_pct or 0 + rh = rate_hours or 0 + + output = { + "ok": True, + "last_refresh_iso": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "provider": PROVIDER, + "accounts": [{ + "key_id": KEY_ID, + "label": "key7 (Kimi)", + "provider": PROVIDER, + "workspace_id": "kimi", + "rolling": { + "usage_percent": rp, + "reset_in_sec": rh * 3600, + "status": "ok" if rp < 80 else ("warn" if rp < 95 else "rate-limited"), + }, + "weekly": { + "usage_percent": wp, + "reset_in_sec": wh * 3600, + "status": "ok" if wp < 80 else ("warn" if wp < 95 else "rate-limited"), + }, + "monthly": None, + "session_expired": False, + "last_update_iso": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "error": None, + }], + } + + with open(str(OUTPUT_FILE), "w", encoding="utf-8") as f: + json.dump(output, f, ensure_ascii=False, indent=2) + print(f"[kimi] Output saved to {OUTPUT_FILE}") + + +if __name__ == "__main__": + main() diff --git a/gateway/scripts/usage_collector_master.py b/gateway/scripts/usage_collector_master.py new file mode 100644 index 0000000..219c6d8 --- /dev/null +++ b/gateway/scripts/usage_collector_master.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +""" +usage_collector.py — 多 Provider 用量采集主控 +============================================= +编排所有 provider 的采集，合并到统一的 usage_stats.json。 + +Provider: + - opencode_go: 本地 HTTP/CDP 采集（usage_collector_ocg.py） + - kimi: SSH 远程调用 Windows 采集器（usage_collector_kimi.py） + +输出: gateway/temp/usage_stats.json（Dashboard /api/usage 读取） +""" +import json, os, sys, subprocess +from pathlib import Path +from datetime import datetime, timezone + +SCRIPT_DIR = Path(__file__).resolve().parent +GATEWAY_DIR = SCRIPT_DIR.parent +TEMP_DIR = GATEWAY_DIR / "temp" +LOGS_DIR = GATEWAY_DIR / "logs" +os.makedirs(str(TEMP_DIR), exist_ok=True) +os.makedirs(str(LOGS_DIR), exist_ok=True) + +OUTPUT_FILE = TEMP_DIR / "usage_stats.json" +WINDOWS_HOST = "192.168.1.16" +WINDOWS_PYTHON = r"C:\Users\hmo\AppData\Local\Programs\Python\Python310\python.exe" +WINDOWS_SCRIPT = r"D:\F\NewI\opencode\daily-workspace\projects\AgentsMeeting\gateway\scripts\usage_collector_kimi.py" + + +def log(msg): + ts = datetime.now().strftime("%H:%M:%S") + line = f"[{ts}] {msg}" + print(line, flush=True) + + +def collect_opencode_go(): + """运行本地 OCG 采集器。""" + log("OCG: starting...") + ocg_script = SCRIPT_DIR / "usage_collector_ocg.py" + if not ocg_script.exists(): + # Fallback: old name + ocg_script = SCRIPT_DIR / "usage_collector.py" + + try: + r = subprocess.run( + [sys.executable, str(ocg_script)], + capture_output=True, text=True, timeout=120, + cwd=str(SCRIPT_DIR), + ) + if r.returncode != 0 and r.stderr: + log(f"OCG: FAILED (rc={r.returncode})") + log(f"OCG stderr: {r.stderr[-500:]}") + else: + log(f"OCG: done (rc={r.returncode})") + except Exception as e: + log(f"OCG: EXCEPTION: {e}") + + +def collect_kimi(): + """SSH 到 Windows 运行 Kimi 采集器，scp 结果回来。""" + log("Kimi: SSH to Windows...") + try: + r = subprocess.run( + ["ssh", "-o", "StrictHostKeyChecking=no", "-o", "ConnectTimeout=10", + "-o", "BatchMode=yes", f"hmo@{WINDOWS_HOST}", + f'"{WINDOWS_PYTHON}" "{WINDOWS_SCRIPT}"'], + capture_output=True, text=True, timeout=45, + ) + for line in r.stdout.splitlines()[-5:]: + log(f"Kimi: {line.strip()}") + if r.returncode != 0: + log(f"Kimi: SSH FAILED (rc={r.returncode}): {r.stderr[-200:]}") + # Try scp alternative + r2 = subprocess.run( + ["scp", "-o", "StrictHostKeyChecking=no", "-o", "ConnectTimeout=10", + f"hmo@{WINDOWS_HOST}:{WINDOWS_SCRIPT}", "/dev/null"], + capture_output=True, timeout=10, + ) + if r2.returncode == 0: + log("Kimi: SSH failed but scp works — check script path") + except subprocess.TimeoutExpired: + log("Kimi: SSH TIMEOUT") + except Exception as e: + log(f"Kimi: SSH EXCEPTION: {e}") + + # Try to scp the output file from Windows + kimi_output = TEMP_DIR / "usage_stats_kimi.json" + try: + r = subprocess.run( + ["scp", "-o", "StrictHostKeyChecking=no", "-o", "ConnectTimeout=10", + f"hmo@{WINDOWS_HOST}:" + WINDOWS_SCRIPT.replace("usage_collector_kimi.py", "") + "../temp/usage_stats_kimi.json", + str(kimi_output)], + capture_output=True, text=True, timeout=15, + ) + if kimi_output.exists(): + log(f"Kimi: output scp'd ({kimi_output.stat().st_size} bytes)") + else: + log("Kimi: scp failed or file empty") + except Exception as e: + log(f"Kimi: scp EXCEPTION: {e}") + + +def merge_all(): + """合并所有 provider 的数据到 usage_stats.json。""" + accounts = [] + + # 1. Read existing OCG cache (from usage_collector_ocg output) + # The OCG collector writes directly to usage_stats.json or a cache + cache_file = TEMP_DIR / "usage_stats.json" + if cache_file.exists(): + try: + with open(str(cache_file), "r", encoding="utf-8") as f: + cached = json.load(f) + for a in cached.get("accounts", []): + a["provider"] = a.get("provider", "opencode_go") + accounts.append(a) + log(f"Merge: {len(accounts)} OCG accounts from cache") + except Exception as e: + log(f"Merge: failed to read cache: {e}") + + # 2. Read Kimi output + kimi_file = TEMP_DIR / "usage_stats_kimi.json" + if kimi_file.exists(): + try: + with open(str(kimi_file), "r", encoding="utf-8") as f: + kimi_data = json.load(f) + for a in kimi_data.get("accounts", []): + a["provider"] = a.get("provider", "kimi") + # Check if this key already exists (update instead of duplicate) + existing = [i for i, x in enumerate(accounts) if x.get("key_id") == a.get("key_id")] + if existing: + accounts[existing[0]] = a + else: + accounts.append(a) + log(f"Merge: +{len(kimi_data.get('accounts',[]))} Kimi accounts") + except Exception as e: + log(f"Merge: failed to read Kimi data: {e}") + + # 3. Count by provider + providers = {} + for a in accounts: + p = a.get("provider", "unknown") + providers[p] = providers.get(p, 0) + 1 + + output = { + "ok": True, + "last_refresh_iso": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "account_count": len(accounts), + "collected_count": sum(1 for a in accounts if a.get("error") is None), + "providers": providers, + "accounts": accounts, + } + + with open(str(OUTPUT_FILE), "w", encoding="utf-8") as f: + json.dump(output, f, ensure_ascii=False, indent=2) + log(f"Merge: saved {len(accounts)} accounts ({providers}) to {OUTPUT_FILE}") + + +def main(): + log("=== Multi-provider collection started ===") + + # Run collectors in sequence (OCG takes longest, Kimi needs CDP sync) + collect_opencode_go() + collect_kimi() + + # Merge + merge_all() + + log("=== Collection complete ===") + + +if __name__ == "__main__": + main()