feat: Hermes Gateway + LLM provider monitoring, root cause visible in health tab
This commit is contained in:
+24
-9
@@ -1526,7 +1526,7 @@ async function renderHealth() {
|
||||
html += '<div><span class="text-slate-500">状态</span><br><span class="font-mono" id="hXmppDetail_st">' + xmpp.status + '</span></div>';
|
||||
html += '</div>';
|
||||
|
||||
// Bot 活动（知微）
|
||||
// Bot 活动
|
||||
const ba = xmpp.bot_activity || {};
|
||||
html += '<div class="grid grid-cols-3 gap-2 text-xs mt-3 pt-3 border-t border-slate-800/50">';
|
||||
html += '<div><span class="text-slate-500">📩 入站</span><br><span class="font-mono text-[#58a6ff]" id="hBot_in">' + (ba.inbound || 0) + '</span></div>';
|
||||
@@ -1536,6 +1536,19 @@ async function renderHealth() {
|
||||
if (ba.last_error) {
|
||||
html += '<div class="text-xs text-[#f85149] mt-2 p-2 bg-red-900/20 rounded" id="hBot_lastErr">' + ba.last_error + '</div>';
|
||||
}
|
||||
|
||||
// Hermes Gateway + LLM Provider
|
||||
const gw = xmpp.gateways || {};
|
||||
const llm = xmpp.llm_provider || {};
|
||||
html += '<div class="grid grid-cols-2 gap-2 text-xs mt-3 pt-3 border-t border-slate-800/50">';
|
||||
html += '<div><span class="text-slate-500">🔌 Hermes Gateway</span><br>';
|
||||
Object.entries(gw).forEach(([k,v]) => {
|
||||
html += '<span class="font-mono ' + (v.alive ? 'text-[#3fb950]' : 'text-[#f85149]') + '">' + k + ':' + v.port + (v.alive ? ' ✅' : ' ❌') + '</span> ';
|
||||
});
|
||||
html += '</div>';
|
||||
html += '<div><span class="text-slate-500">🧠 LLM Provider</span><br><span class="font-mono ' + (llm.status === 'ok' ? 'text-[#3fb950]' : 'text-[#f85149]') + '" id="hLlmStatus">' + (llm.status || '?') + '</span>';
|
||||
if (llm.error) html += '<br><span class="text-[#f85149]" id="hLlmErr">' + llm.error + '</span>';
|
||||
html += '</div></div>';
|
||||
html += '</div>';
|
||||
}
|
||||
el.innerHTML = html;
|
||||
@@ -1578,7 +1591,7 @@ async function refreshHealth() {
|
||||
}
|
||||
});
|
||||
|
||||
// 更新 XMPP 详情
|
||||
// 更新 XMPP 详情 + Bot 活动
|
||||
if (xmpp) {
|
||||
const d_age = document.getElementById('hXmppDetail_age');
|
||||
const d_err = document.getElementById('hXmppDetail_err');
|
||||
@@ -1588,22 +1601,24 @@ async function refreshHealth() {
|
||||
if (d_err) d_err.textContent = xmpp.error_rate_1h + '%';
|
||||
if (d_eja) d_eja.textContent = xmpp.ejabberd || '?';
|
||||
if (d_st) d_st.textContent = xmpp.status;
|
||||
// 更新卡片颜色
|
||||
if (ageEl && xmpp) ageEl.style.color = xmpp.last_message_age_sec > 600 ? '#f85149' : '#3fb950';
|
||||
if (labelEl && xmpp) {
|
||||
const labels = {ok:'XMPP 正常', degraded:'XMPP 降级', critical:'XMPP 断联'};
|
||||
labelEl.textContent = labels[xmpp.status] || 'XMPP 无数据';
|
||||
}
|
||||
// 更新 Bot 活动
|
||||
// Bot 活动
|
||||
const ba = xmpp.bot_activity || {};
|
||||
const bi = document.getElementById('hBot_in');
|
||||
const bo = document.getElementById('hBot_out');
|
||||
const be = document.getElementById('hBot_err');
|
||||
const bl = document.getElementById('hBot_lastErr');
|
||||
const bi = document.getElementById('hBot_in'), bo = document.getElementById('hBot_out');
|
||||
const be = document.getElementById('hBot_err'), bl = document.getElementById('hBot_lastErr');
|
||||
if (bi) bi.textContent = ba.inbound || 0;
|
||||
if (bo) { bo.textContent = ba.outbound || 0; bo.className = 'font-mono ' + (ba.outbound ? 'text-[#3fb950]' : (ba.inbound ? 'text-[#f85149]' : 'text-slate-500')); }
|
||||
if (be) { be.textContent = ba.errors || 0; be.className = 'font-mono ' + (ba.errors ? 'text-[#f85149]' : 'text-slate-500'); }
|
||||
if (bl && ba.last_error) bl.textContent = ba.last_error;
|
||||
if (bl) bl.textContent = ba.last_error || '';
|
||||
// LLM Provider
|
||||
const llm = xmpp.llm_provider || {};
|
||||
const ls = document.getElementById('hLlmStatus'), le = document.getElementById('hLlmErr');
|
||||
if (ls) { ls.textContent = llm.status || '?'; ls.className = 'font-mono ' + (llm.status === 'ok' ? 'text-[#3fb950]' : 'text-[#f85149]'); }
|
||||
if (le) le.textContent = llm.error || '';
|
||||
}
|
||||
|
||||
// 更新时间戳
|
||||
|
||||
+57
-44
@@ -142,78 +142,91 @@ def stats():
|
||||
|
||||
|
||||
def health():
|
||||
"""快速健康检查：返回最后消息年龄 + 最近1h错误率 + bot 日志状态"""
|
||||
result = {"status": "no_data", "last_message_age_sec": -1, "error_rate_1h": 0, "bot_activity": {}}
|
||||
"""快速健康检查：返回最后消息年龄 + 最近1h错误率 + bot 日志状态 + Hermes Gateway 状态"""
|
||||
result = {"status": "no_data", "last_message_age_sec": -1, "error_rate_1h": 0,
|
||||
"bot_activity": {}, "gateways": {}, "llm_provider": {}}
|
||||
|
||||
# 1. 检查 xmpp_messages.jsonl
|
||||
now_epoch = _time.time()
|
||||
if LOG_FILE.exists():
|
||||
now_epoch = _time.time()
|
||||
last_epoch = 0
|
||||
recent_total = 0
|
||||
recent_errors = 0
|
||||
one_hour_ago = now_epoch - 3600
|
||||
|
||||
try:
|
||||
with open(LOG_FILE, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
e = json.loads(line)
|
||||
ep = e.get("epoch", 0)
|
||||
if ep > last_epoch:
|
||||
last_epoch = ep
|
||||
if ep > last_epoch: last_epoch = ep
|
||||
if ep > one_hour_ago:
|
||||
recent_total += 1
|
||||
if e["status"] != "ok":
|
||||
recent_errors += 1
|
||||
except Exception:
|
||||
continue
|
||||
except Exception:
|
||||
pass
|
||||
if e["status"] != "ok": recent_errors += 1
|
||||
except Exception: continue
|
||||
except Exception: pass
|
||||
result["last_message_age_sec"] = int(now_epoch - last_epoch) if last_epoch else -1
|
||||
result["error_rate_1h"] = round(recent_errors / recent_total * 100, 1) if recent_total else 0
|
||||
|
||||
last_age = int(now_epoch - last_epoch) if last_epoch else -1
|
||||
err_rate = round(recent_errors / recent_total * 100, 1) if recent_total else 0
|
||||
result["last_message_age_sec"] = last_age
|
||||
result["error_rate_1h"] = err_rate
|
||||
|
||||
# 2. 检查知微 Bot systemd journal（最近5分钟）
|
||||
# 2. 检查知微 Bot systemd journal + 持久化状态
|
||||
try:
|
||||
import subprocess
|
||||
r = subprocess.run(
|
||||
["journalctl", "-u", "xmpp-zhiwei", "--no-pager", "--since", "5 min ago", "-o", "cat"],
|
||||
capture_output=True, timeout=5, text=True
|
||||
)
|
||||
r = subprocess.run(["journalctl", "-u", "xmpp-zhiwei", "--no-pager", "--since", "30 min ago", "-o", "cat"],
|
||||
capture_output=True, timeout=5, text=True)
|
||||
lines = [l for l in r.stdout.split("\n") if l.strip()]
|
||||
inbound = [l for l in lines if "📩 收到" in l]
|
||||
outbound = [l for l in lines if "📤 发送" in l or "📤" in l]
|
||||
errors = [l for l in lines if "ERROR" in l or "error" in l or "timeout" in l or "Timeout" in l]
|
||||
errors = [l for l in lines if "ERROR" in l or "TimeoutError" in l or "timed out" in l]
|
||||
result["bot_activity"] = {
|
||||
"inbound": len(inbound),
|
||||
"outbound": len(outbound),
|
||||
"errors": len(errors),
|
||||
"last_error": errors[-1][:200] if errors else None,
|
||||
"last_inbound": inbound[-1][:200] if inbound else None,
|
||||
"inbound": len(inbound), "outbound": len(outbound), "errors": len(errors),
|
||||
"last_error": errors[-1][:250] if errors else None,
|
||||
"last_inbound": inbound[-1][:250] if inbound else None,
|
||||
"last_outbound": outbound[-1][:200] if outbound else None,
|
||||
}
|
||||
# 有错误且无出站 → degraded
|
||||
if errors and not outbound:
|
||||
if result.get("status") != "critical":
|
||||
result["status"] = "degraded"
|
||||
# 有入站无出站 → 可能卡住了
|
||||
if inbound and not outbound:
|
||||
result["status"] = "degraded"
|
||||
if errors and not outbound: result["status"] = "degraded"
|
||||
if inbound and not outbound: result["status"] = "degraded"
|
||||
except Exception:
|
||||
result["bot_activity"] = {"error": "journalctl failed"}
|
||||
|
||||
# 3. 综合判定
|
||||
# 3. 检查 Hermes Gateway 各 profile 状态（端口检测）
|
||||
try:
|
||||
import socket
|
||||
profiles = {"zhiwei": 8643, "mohe": 8642, "xiaoguo": 8645}
|
||||
for name, port in profiles.items():
|
||||
ok = False
|
||||
try:
|
||||
s = socket.create_connection(("127.0.0.1", port), timeout=2)
|
||||
s.close()
|
||||
ok = True
|
||||
except Exception: pass
|
||||
result["gateways"][name] = {"port": port, "alive": ok}
|
||||
|
||||
# 测试知微的 LLM 调用是否可达
|
||||
if result["gateways"].get("zhiwei", {}).get("alive"):
|
||||
try:
|
||||
import urllib.request
|
||||
payload = json.dumps({"model": "gpt-4", "messages": [{"role": "user", "content": "ping"}],
|
||||
"max_tokens": 5, "stream": False}).encode()
|
||||
req = urllib.request.Request("http://127.0.0.1:8643/v1/chat/completions",
|
||||
data=payload, headers={"Content-Type": "application/json",
|
||||
"Authorization": "Bearer hermes123"})
|
||||
urllib.request.urlopen(req, timeout=8)
|
||||
result["llm_provider"] = {"status": "ok", "latency": "fast"}
|
||||
except Exception as e:
|
||||
result["llm_provider"] = {"status": "timeout" if "timeout" in str(e).lower() else "error",
|
||||
"error": str(e)[:150]}
|
||||
except Exception as e:
|
||||
result["gateways"] = {"error": str(e)[:100]}
|
||||
|
||||
# 4. 综合判定
|
||||
if result.get("status") != "degraded":
|
||||
la = result.get("last_message_age_sec", -1)
|
||||
er = result.get("error_rate_1h", 0)
|
||||
if la < 0:
|
||||
result["status"] = "no_data"
|
||||
elif la > 600:
|
||||
result["status"] = "critical"
|
||||
elif er > 50:
|
||||
result["status"] = "degraded"
|
||||
elif la > 0:
|
||||
result["status"] = "ok"
|
||||
llm = result.get("llm_provider", {}).get("status", "")
|
||||
if llm in ("timeout", "error"): result["status"] = "degraded"
|
||||
elif la < 0: result["status"] = "no_data"
|
||||
elif la > 600: result["status"] = "critical"
|
||||
elif er > 50: result["status"] = "degraded"
|
||||
elif la > 0: result["status"] = "ok"
|
||||
|
||||
return result
|
||||
|
||||
Reference in New Issue
Block a user