hmo/MoFin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: XMPP bot journal monitoring — detect inbound/outbound/errors, smooth health refresh

Browse Source
This commit is contained in:
hmo
2026-07-19 13:15:37 +08:00
parent 9f3d1d8106
commit 50ae4e1cc9
2 changed files with 167 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
static/index.html
+98 -14
View File
@@ -154,9 +154,11 @@ document.addEventListener('DOMContentLoaded', function() {
if (name !== 'health' && healthRefreshInterval) {
clearInterval(healthRefreshInterval);
healthRefreshInterval = null;
const he = document.getElementById('tab-health');
if (he) delete he.dataset.rendered;
}
if (name === 'health' && !healthRefreshInterval) {
healthRefreshInterval = setInterval(renderHealth, 10000);
healthRefreshInterval = setInterval(refreshHealth, 10000);
}
document.querySelectorAll('.tab-content').forEach(el => el.classList.add('hidden'));
document.querySelectorAll('.tab-btn').forEach(el => el.classList.remove('active'));
@@ -174,10 +176,12 @@ function switchTab(name) {
if (name !== 'health' && healthRefreshInterval) {
clearInterval(healthRefreshInterval);
healthRefreshInterval = null;
const he = document.getElementById('tab-health');
if (he) delete he.dataset.rendered;
}
// 进入 health 时确保定时器运行
if (name === 'health' && !healthRefreshInterval) {
healthRefreshInterval = setInterval(renderHealth, 10000);
healthRefreshInterval = setInterval(refreshHealth, 10000);
}
const btn = document.querySelector(`[data-tab="${name}"]`);
if (btn) {
@@ -1465,13 +1469,13 @@ async function renderHealth() {
// Summary cards — 包含 XMPP 状态
let html = '<div class="grid grid-cols-2 sm:grid-cols-4 gap-3 mb-4">';
html += '<div class="card p-3 text-center"><div class="text-lg font-bold text-[#3fb950] font-mono">' + (summary.ok || 0) + '</div><div class="text-xs text-slate-500">正常服务</div></div>';
html += '<div class="card p-3 text-center"><div class="text-lg font-bold text-white font-mono">' + (summary.total || 0) + '</div><div class="text-xs text-slate-500">总服务数</div></div>';
html += '<div class="card p-3 text-center"><div class="text-lg font-bold text-[#3fb950] font-mono" id="hSvcOk">' + (summary.ok || 0) + '</div><div class="text-xs text-slate-500">正常服务</div></div>';
html += '<div class="card p-3 text-center"><div class="text-lg font-bold text-white font-mono" id="hSvcTotal">' + (summary.total || 0) + '</div><div class="text-xs text-slate-500">总服务数</div></div>';
const xmppStatus = xmpp ? xmpp.status : 'unknown';
const xmppColor = xmppStatus === 'ok' ? '#3fb950' : xmppStatus === 'degraded' ? '#d29922' : '#f85149';
const xmppLabel = xmppStatus === 'ok' ? 'XMPP 正常' : xmppStatus === 'degraded' ? 'XMPP 降级' : xmppStatus === 'critical' ? 'XMPP 断联' : 'XMPP 无数据';
html += '<div class="card p-3 text-center"><div class="text-lg font-bold font-mono" style="color:' + xmppColor + '">' + (xmpp ? (xmpp.last_message_age_sec > 0 ? Math.round(xmpp.last_message_age_sec/60) + 'm' : '--') : '--') + '</div><div class="text-xs text-slate-500">' + xmppLabel + '</div></div>';
html += '<div class="card p-3 text-center"><div class="text-lg font-bold text-[#d29922] font-mono">' + (xmpp ? xmpp.error_rate_1h + '%' : '--') + '</div><div class="text-xs text-slate-500">XMPP 错误率(1h)</div></div>';
html += '<div class="card p-3 text-center"><div class="text-lg font-bold font-mono" id="hXmppAge" style="color:' + xmppColor + '">' + (xmpp ? (xmpp.last_message_age_sec > 0 ? Math.round(xmpp.last_message_age_sec/60) + 'm' : '--') : '--') + '</div><div class="text-xs text-slate-500" id="hXmppLabel">' + xmppLabel + '</div></div>';
html += '<div class="card p-3 text-center"><div class="text-lg font-bold text-[#d29922] font-mono" id="hXmppErr">' + (xmpp ? xmpp.error_rate_1h + '%' : '--') + '</div><div class="text-xs text-slate-500">XMPP 错误率(1h)</div></div>';
html += '</div>';
// Services by layer
@@ -1491,9 +1495,9 @@ async function renderHealth() {
const label = s.label || s.name || s.type || '';
const port = s.port || '-';
if (i === 0) {
html += '<tr class="border-b border-slate-800/30"><td class="p-3 font-semibold text-slate-300" rowspan="' + layers[layer].length + '">' + layer + '</td><td class="p-3">' + s.name + ' <span class="text-slate-500">' + label + '</span></td><td class="p-3 font-mono">' + port + '</td><td class="p-3 ' + statusColor + '">' + statusIcon + (ok ? ' 正常' : (ok === false ? ' 异常' : ' 未知')) + '</td></tr>';
html += '<tr class="border-b border-slate-800/30"><td class="p-3 font-semibold text-slate-300" rowspan="' + layers[layer].length + '">' + layer + '</td><td class="p-3">' + s.name + ' <span class="text-slate-500">' + label + '</span></td><td class="p-3 font-mono">' + port + '</td><td class="p-3 ' + statusColor + '" id="hSvc_' + s.name + '">' + statusIcon + (ok ? ' 正常' : (ok === false ? ' 异常' : ' 未知')) + '</td></tr>';
} else {
html += '<tr class="border-b border-slate-800/30"><td class="p-3">' + s.name + ' <span class="text-slate-500">' + label + '</span></td><td class="p-3 font-mono">' + port + '</td><td class="p-3 ' + statusColor + '">' + statusIcon + (ok ? ' 正常' : (ok === false ? ' 异常' : ' 未知')) + '</td></tr>';
html += '<tr class="border-b border-slate-800/30"><td class="p-3">' + s.name + ' <span class="text-slate-500">' + label + '</span></td><td class="p-3 font-mono">' + port + '</td><td class="p-3 ' + statusColor + '" id="hSvc_' + s.name + '">' + statusIcon + (ok ? ' 正常' : (ok === false ? ' 异常' : ' 未知')) + '</td></tr>';
}
});
});
@@ -1510,24 +1514,104 @@ async function renderHealth() {
html += '</tbody></table></div>';
}
html += '<div class="text-xs text-slate-600 mt-2">更新于 ' + (monitor.generated_at || new Date().toLocaleTimeString()) + ' · 每10秒刷新</div>';
html += '<div class="text-xs text-slate-600 mt-2" id="hUpdated">更新于 ' + (monitor.generated_at || new Date().toLocaleTimeString()) + '</div>';
// XMPP 通道详情
if (xmpp) {
html += '<div class="card p-4 mt-4"><h3 class="text-sm font-semibold text-[#58a6ff] mb-2">📡 XMPP 通道</h3>';
html += '<div class="grid grid-cols-2 sm:grid-cols-4 gap-2 text-xs">';
html += '<div><span class="text-slate-500">最后消息</span><br><span class="font-mono ' + (xmpp.last_message_age_sec > 600 ? 'text-[#f85149]' : 'text-[#3fb950]') + '">' + (xmpp.last_message_age_sec > 0 ? Math.round(xmpp.last_message_age_sec) + '秒前' : '无数据') + '</span></div>';
html += '<div><span class="text-slate-500">1h错误率</span><br><span class="font-mono ' + (xmpp.error_rate_1h > 50 ? 'text-[#f85149]' : xmpp.error_rate_1h > 10 ? 'text-[#d29922]' : 'text-[#3fb950]') + '">' + xmpp.error_rate_1h + '%</span></div>';
html += '<div><span class="text-slate-500">ejabberd</span><br><span class="font-mono ' + ((xmpp.ejabberd || '').includes('Up') ? 'text-[#3fb950]' : 'text-[#f85149]') + '">' + (xmpp.ejabberd || '?') + '</span></div>';
html += '<div><span class="text-slate-500">状态</span><br><span class="font-mono ' + (xmpp.status === 'ok' ? 'text-[#3fb950]' : xmpp.status === 'degraded' ? 'text-[#d29922]' : 'text-[#f85149]') + '">' + xmpp.status + '</span></div>';
html += '</div></div>';
html += '<div><span class="text-slate-500">最后消息</span><br><span class="font-mono" id="hXmppDetail_age">' + (xmpp.last_message_age_sec > 0 ? Math.round(xmpp.last_message_age_sec) + '秒前' : '无数据') + '</span></div>';
html += '<div><span class="text-slate-500">1h错误率</span><br><span class="font-mono" id="hXmppDetail_err">' + xmpp.error_rate_1h + '%</span></div>';
html += '<div><span class="text-slate-500">ejabberd</span><br><span class="font-mono" id="hXmppDetail_eja">' + (xmpp.ejabberd || '?') + '</span></div>';
html += '<div><span class="text-slate-500">状态</span><br><span class="font-mono" id="hXmppDetail_st">' + xmpp.status + '</span></div>';
html += '</div>';
// Bot 活动（知微）
const ba = xmpp.bot_activity || {};
html += '<div class="grid grid-cols-3 gap-2 text-xs mt-3 pt-3 border-t border-slate-800/50">';
html += '<div><span class="text-slate-500">📩 入站</span><br><span class="font-mono text-[#58a6ff]" id="hBot_in">' + (ba.inbound || 0) + '</span></div>';
html += '<div><span class="text-slate-500">📤 出站</span><br><span class="font-mono ' + (ba.outbound ? 'text-[#3fb950]' : (ba.inbound ? 'text-[#f85149]' : 'text-slate-500')) + '" id="hBot_out">' + (ba.outbound || 0) + '</span></div>';
html += '<div><span class="text-slate-500">⚠️ 错误</span><br><span class="font-mono ' + (ba.errors ? 'text-[#f85149]' : 'text-slate-500') + '" id="hBot_err">' + (ba.errors || 0) + '</span></div>';
html += '</div>';
if (ba.last_error) {
html += '<div class="text-xs text-[#f85149] mt-2 p-2 bg-red-900/20 rounded" id="hBot_lastErr">' + ba.last_error + '</div>';
}
html += '</div>';
}
el.innerHTML = html;
el.dataset.rendered = '1'; // 标记已首渲
} catch(e) {
el.innerHTML = '<div class="text-red-400">加载健康状态失败: ' + e.message + '</div>';
}
}
// 增量刷新健康数据（不重建 DOM，无闪烁）
async function refreshHealth() {
const el = document.getElementById('tab-health');
if (!el || el.classList.contains('hidden') || !el.dataset.rendered) return;
try {
const [services, xmpp] = await Promise.all([
fetchJSON('/api/services'),
fetchJSON('/api/xmpp/health')
]);
const svcs = (services.services || []);
const sum = services.summary || {};
// 更新卡片数字
const okEl = document.getElementById('hSvcOk');
const totalEl = document.getElementById('hSvcTotal');
const ageEl = document.getElementById('hXmppAge');
const errEl = document.getElementById('hXmppErr');
const labelEl = document.getElementById('hXmppLabel');
if (okEl) okEl.textContent = sum.ok || 0;
if (totalEl) totalEl.textContent = sum.total || 0;
if (ageEl && xmpp) ageEl.textContent = xmpp.last_message_age_sec > 0 ? Math.round(xmpp.last_message_age_sec/60) + 'm' : '--';
if (errEl && xmpp) errEl.textContent = xmpp.error_rate_1h + '%';
// 更新服务行状态
svcs.forEach(s => {
const cell = document.getElementById('hSvc_' + s.name);
if (cell) {
const ok = s.health?.ok;
cell.textContent = (ok ? '🟢' : (ok === false ? '🔴' : '🟡')) + (ok ? ' 正常' : (ok === false ? ' 异常' : ' 未知'));
cell.className = 'p-3 ' + (ok ? 'text-[#3fb950]' : (ok === false ? 'text-[#f85149]' : 'text-[#d29922]'));
}
});
// 更新 XMPP 详情
if (xmpp) {
const d_age = document.getElementById('hXmppDetail_age');
const d_err = document.getElementById('hXmppDetail_err');
const d_eja = document.getElementById('hXmppDetail_eja');
const d_st = document.getElementById('hXmppDetail_st');
if (d_age) d_age.textContent = xmpp.last_message_age_sec > 0 ? Math.round(xmpp.last_message_age_sec) + '秒前' : '无数据';
if (d_err) d_err.textContent = xmpp.error_rate_1h + '%';
if (d_eja) d_eja.textContent = xmpp.ejabberd || '?';
if (d_st) d_st.textContent = xmpp.status;
// 更新卡片颜色
if (ageEl && xmpp) ageEl.style.color = xmpp.last_message_age_sec > 600 ? '#f85149' : '#3fb950';
if (labelEl && xmpp) {
const labels = {ok:'XMPP 正常', degraded:'XMPP 降级', critical:'XMPP 断联'};
labelEl.textContent = labels[xmpp.status] || 'XMPP 无数据';
}
// 更新 Bot 活动
const ba = xmpp.bot_activity || {};
const bi = document.getElementById('hBot_in');
const bo = document.getElementById('hBot_out');
const be = document.getElementById('hBot_err');
const bl = document.getElementById('hBot_lastErr');
if (bi) bi.textContent = ba.inbound || 0;
if (bo) { bo.textContent = ba.outbound || 0; bo.className = 'font-mono ' + (ba.outbound ? 'text-[#3fb950]' : (ba.inbound ? 'text-[#f85149]' : 'text-slate-500')); }
if (be) { be.textContent = ba.errors || 0; be.className = 'font-mono ' + (ba.errors ? 'text-[#f85149]' : 'text-slate-500'); }
if (bl && ba.last_error) bl.textContent = ba.last_error;
}
// 更新时间戳
const upd = document.getElementById('hUpdated');
if (upd) upd.textContent = '更新于 ' + new Date().toLocaleTimeString();
} catch(e) { /* 静默失败，保留旧数据 */ }
}
// ── 盯盘 ──
let watchTimer = null;
async function renderWatch() {
xmpp_logger.py
+69 -35
View File
@@ -142,44 +142,78 @@ def stats():
def health():
"""快速健康检查：返回最后消息年龄 + 最近1h错误率"""
if not LOG_FILE.exists():
return {"status": "no_data", "last_message_age_sec": -1, "error_rate_1h": 0}
"""快速健康检查：返回最后消息年龄 + 最近1h错误率 + bot 日志状态"""
result = {"status": "no_data", "last_message_age_sec": -1, "error_rate_1h": 0, "bot_activity": {}}
now_epoch = _time.time()
last_epoch = 0
recent_total = 0
recent_errors = 0
one_hour_ago = now_epoch - 3600
# 1. 检查 xmpp_messages.jsonl
if LOG_FILE.exists():
now_epoch = _time.time()
last_epoch = 0
recent_total = 0
recent_errors = 0
one_hour_ago = now_epoch - 3600
try:
with open(LOG_FILE, "r", encoding="utf-8") as f:
for line in f:
try:
e = json.loads(line)
ep = e.get("epoch", 0)
if ep > last_epoch:
last_epoch = ep
if ep > one_hour_ago:
recent_total += 1
if e["status"] != "ok":
recent_errors += 1
except Exception:
continue
except Exception:
pass
last_age = int(now_epoch - last_epoch) if last_epoch else -1
err_rate = round(recent_errors / recent_total * 100, 1) if recent_total else 0
result["last_message_age_sec"] = last_age
result["error_rate_1h"] = err_rate
# 2. 检查知微 Bot systemd journal（最近5分钟）
try:
with open(LOG_FILE, "r", encoding="utf-8") as f:
for line in f:
try:
e = json.loads(line)
ep = e.get("epoch", 0)
if ep > last_epoch:
last_epoch = ep
if ep > one_hour_ago:
recent_total += 1
if e["status"] != "ok":
recent_errors += 1
except Exception:
continue
import subprocess
r = subprocess.run(
["journalctl", "-u", "xmpp-zhiwei", "--no-pager", "--since", "5 min ago", "-o", "cat"],
capture_output=True, timeout=5, text=True
)
lines = [l for l in r.stdout.split("\n") if l.strip()]
inbound = [l for l in lines if "📩 收到" in l]
outbound = [l for l in lines if "📤 发送" in l or "📤" in l]
errors = [l for l in lines if "ERROR" in l or "error" in l or "timeout" in l or "Timeout" in l]
result["bot_activity"] = {
"inbound": len(inbound),
"outbound": len(outbound),
"errors": len(errors),
"last_error": errors[-1][:200] if errors else None,
"last_inbound": inbound[-1][:200] if inbound else None,
}
# 有错误且无出站 → degraded
if errors and not outbound:
if result.get("status") != "critical":
result["status"] = "degraded"
# 有入站无出站 → 可能卡住了
if inbound and not outbound:
result["status"] = "degraded"
except Exception:
pass
result["bot_activity"] = {"error": "journalctl failed"}
last_age = int(now_epoch - last_epoch) if last_epoch else -1
err_rate = round(recent_errors / recent_total * 100, 1) if recent_total else 0
# 3. 综合判定
if result.get("status") != "degraded":
la = result.get("last_message_age_sec", -1)
er = result.get("error_rate_1h", 0)
if la < 0:
result["status"] = "no_data"
elif la > 600:
result["status"] = "critical"
elif er > 50:
result["status"] = "degraded"
elif la > 0:
result["status"] = "ok"
# 状态判断
if last_age < 0:
st = "no_data"
elif last_age > 600: # >10min no message
st = "critical"
elif err_rate > 50:
st = "degraded"
else:
st = "ok"
return {"status": st, "last_message_age_sec": last_age, "error_rate_1h": err_rate}
return result