From 0802046d5c1e2dafd564a5c23ac483105f929cb1 Mon Sep 17 00:00:00 2001 From: hmo Date: Sun, 19 Jul 2026 13:25:37 +0800 Subject: [PATCH] feat: Hermes Gateway + LLM provider monitoring, root cause visible in health tab --- static/index.html | 33 ++++++++++----- xmpp_logger.py | 101 ++++++++++++++++++++++++++-------------------- 2 files changed, 81 insertions(+), 53 deletions(-) diff --git a/static/index.html b/static/index.html index a63dd9de..fc5709a0 100644 --- a/static/index.html +++ b/static/index.html @@ -1526,7 +1526,7 @@ async function renderHealth() { html += '
状态
' + xmpp.status + '
'; html += ''; - // Bot 活动（知微） + // Bot 活动 const ba = xmpp.bot_activity || {}; html += '
'; html += '
📩 入站
' + (ba.inbound || 0) + '
'; @@ -1536,6 +1536,19 @@ async function renderHealth() { if (ba.last_error) { html += '
' + ba.last_error + '
'; } + + // Hermes Gateway + LLM Provider + const gw = xmpp.gateways || {}; + const llm = xmpp.llm_provider || {}; + html += '
'; + html += '
🔌 Hermes Gateway
'; + Object.entries(gw).forEach(([k,v]) => { + html += '' + k + ':' + v.port + (v.alive ? ' ✅' : ' ❌') + ' '; + }); + html += '
'; + html += '
🧠 LLM Provider
' + (llm.status || '?') + ''; + if (llm.error) html += '
' + llm.error + ''; + html += '
'; html += '
'; } el.innerHTML = html; @@ -1578,7 +1591,7 @@ async function refreshHealth() { } }); - // 更新 XMPP 详情 + // 更新 XMPP 详情 + Bot 活动 if (xmpp) { const d_age = document.getElementById('hXmppDetail_age'); const d_err = document.getElementById('hXmppDetail_err'); @@ -1588,22 +1601,24 @@ async function refreshHealth() { if (d_err) d_err.textContent = xmpp.error_rate_1h + '%'; if (d_eja) d_eja.textContent = xmpp.ejabberd || '?'; if (d_st) d_st.textContent = xmpp.status; - // 更新卡片颜色 if (ageEl && xmpp) ageEl.style.color = xmpp.last_message_age_sec > 600 ? '#f85149' : '#3fb950'; if (labelEl && xmpp) { const labels = {ok:'XMPP 正常', degraded:'XMPP 降级', critical:'XMPP 断联'}; labelEl.textContent = labels[xmpp.status] || 'XMPP 无数据'; } - // 更新 Bot 活动 + // Bot 活动 const ba = xmpp.bot_activity || {}; - const bi = document.getElementById('hBot_in'); - const bo = document.getElementById('hBot_out'); - const be = document.getElementById('hBot_err'); - const bl = document.getElementById('hBot_lastErr'); + const bi = document.getElementById('hBot_in'), bo = document.getElementById('hBot_out'); + const be = document.getElementById('hBot_err'), bl = document.getElementById('hBot_lastErr'); if (bi) bi.textContent = ba.inbound || 0; if (bo) { bo.textContent = ba.outbound || 0; bo.className = 'font-mono ' + (ba.outbound ? 'text-[#3fb950]' : (ba.inbound ? 'text-[#f85149]' : 'text-slate-500')); } if (be) { be.textContent = ba.errors || 0; be.className = 'font-mono ' + (ba.errors ? 'text-[#f85149]' : 'text-slate-500'); } - if (bl && ba.last_error) bl.textContent = ba.last_error; + if (bl) bl.textContent = ba.last_error || ''; + // LLM Provider + const llm = xmpp.llm_provider || {}; + const ls = document.getElementById('hLlmStatus'), le = document.getElementById('hLlmErr'); + if (ls) { ls.textContent = llm.status || '?'; ls.className = 'font-mono ' + (llm.status === 'ok' ? 'text-[#3fb950]' : 'text-[#f85149]'); } + if (le) le.textContent = llm.error || ''; } // 更新时间戳 diff --git a/xmpp_logger.py b/xmpp_logger.py index 9d4c9e07..aadc1bb8 100644 --- a/xmpp_logger.py +++ b/xmpp_logger.py @@ -142,78 +142,91 @@ def stats(): def health(): - """快速健康检查：返回最后消息年龄 + 最近1h错误率 + bot 日志状态""" - result = {"status": "no_data", "last_message_age_sec": -1, "error_rate_1h": 0, "bot_activity": {}} + """快速健康检查：返回最后消息年龄 + 最近1h错误率 + bot 日志状态 + Hermes Gateway 状态""" + result = {"status": "no_data", "last_message_age_sec": -1, "error_rate_1h": 0, + "bot_activity": {}, "gateways": {}, "llm_provider": {}} # 1. 检查 xmpp_messages.jsonl + now_epoch = _time.time() if LOG_FILE.exists(): - now_epoch = _time.time() last_epoch = 0 recent_total = 0 recent_errors = 0 one_hour_ago = now_epoch - 3600 - try: with open(LOG_FILE, "r", encoding="utf-8") as f: for line in f: try: e = json.loads(line) ep = e.get("epoch", 0) - if ep > last_epoch: - last_epoch = ep + if ep > last_epoch: last_epoch = ep if ep > one_hour_ago: recent_total += 1 - if e["status"] != "ok": - recent_errors += 1 - except Exception: - continue - except Exception: - pass + if e["status"] != "ok": recent_errors += 1 + except Exception: continue + except Exception: pass + result["last_message_age_sec"] = int(now_epoch - last_epoch) if last_epoch else -1 + result["error_rate_1h"] = round(recent_errors / recent_total * 100, 1) if recent_total else 0 - last_age = int(now_epoch - last_epoch) if last_epoch else -1 - err_rate = round(recent_errors / recent_total * 100, 1) if recent_total else 0 - result["last_message_age_sec"] = last_age - result["error_rate_1h"] = err_rate - - # 2. 检查知微 Bot systemd journal（最近5分钟） + # 2. 检查知微 Bot systemd journal + 持久化状态 try: import subprocess - r = subprocess.run( - ["journalctl", "-u", "xmpp-zhiwei", "--no-pager", "--since", "5 min ago", "-o", "cat"], - capture_output=True, timeout=5, text=True - ) + r = subprocess.run(["journalctl", "-u", "xmpp-zhiwei", "--no-pager", "--since", "30 min ago", "-o", "cat"], + capture_output=True, timeout=5, text=True) lines = [l for l in r.stdout.split("\n") if l.strip()] inbound = [l for l in lines if "📩 收到" in l] outbound = [l for l in lines if "📤 发送" in l or "📤" in l] - errors = [l for l in lines if "ERROR" in l or "error" in l or "timeout" in l or "Timeout" in l] + errors = [l for l in lines if "ERROR" in l or "TimeoutError" in l or "timed out" in l] result["bot_activity"] = { - "inbound": len(inbound), - "outbound": len(outbound), - "errors": len(errors), - "last_error": errors[-1][:200] if errors else None, - "last_inbound": inbound[-1][:200] if inbound else None, + "inbound": len(inbound), "outbound": len(outbound), "errors": len(errors), + "last_error": errors[-1][:250] if errors else None, + "last_inbound": inbound[-1][:250] if inbound else None, + "last_outbound": outbound[-1][:200] if outbound else None, } - # 有错误且无出站 → degraded - if errors and not outbound: - if result.get("status") != "critical": - result["status"] = "degraded" - # 有入站无出站 → 可能卡住了 - if inbound and not outbound: - result["status"] = "degraded" + if errors and not outbound: result["status"] = "degraded" + if inbound and not outbound: result["status"] = "degraded" except Exception: result["bot_activity"] = {"error": "journalctl failed"} - # 3. 综合判定 + # 3. 检查 Hermes Gateway 各 profile 状态（端口检测） + try: + import socket + profiles = {"zhiwei": 8643, "mohe": 8642, "xiaoguo": 8645} + for name, port in profiles.items(): + ok = False + try: + s = socket.create_connection(("127.0.0.1", port), timeout=2) + s.close() + ok = True + except Exception: pass + result["gateways"][name] = {"port": port, "alive": ok} + + # 测试知微的 LLM 调用是否可达 + if result["gateways"].get("zhiwei", {}).get("alive"): + try: + import urllib.request + payload = json.dumps({"model": "gpt-4", "messages": [{"role": "user", "content": "ping"}], + "max_tokens": 5, "stream": False}).encode() + req = urllib.request.Request("http://127.0.0.1:8643/v1/chat/completions", + data=payload, headers={"Content-Type": "application/json", + "Authorization": "Bearer hermes123"}) + urllib.request.urlopen(req, timeout=8) + result["llm_provider"] = {"status": "ok", "latency": "fast"} + except Exception as e: + result["llm_provider"] = {"status": "timeout" if "timeout" in str(e).lower() else "error", + "error": str(e)[:150]} + except Exception as e: + result["gateways"] = {"error": str(e)[:100]} + + # 4. 综合判定 if result.get("status") != "degraded": la = result.get("last_message_age_sec", -1) er = result.get("error_rate_1h", 0) - if la < 0: - result["status"] = "no_data" - elif la > 600: - result["status"] = "critical" - elif er > 50: - result["status"] = "degraded" - elif la > 0: - result["status"] = "ok" + llm = result.get("llm_provider", {}).get("status", "") + if llm in ("timeout", "error"): result["status"] = "degraded" + elif la < 0: result["status"] = "no_data" + elif la > 600: result["status"] = "critical" + elif er > 50: result["status"] = "degraded" + elif la > 0: result["status"] = "ok" return result