From 50ae4e1cc9f46be63b1ec7fb216ed703812bdf3a Mon Sep 17 00:00:00 2001 From: hmo Date: Sun, 19 Jul 2026 13:15:37 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20XMPP=20bot=20journal=20monitoring=20?= =?UTF-8?q?=E2=80=94=20detect=20inbound/outbound/errors,=20smooth=20health?= =?UTF-8?q?=20refresh?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- static/index.html | 112 ++++++++++++++++++++++++++++++++++++++++------ xmpp_logger.py | 104 +++++++++++++++++++++++++++--------------- 2 files changed, 167 insertions(+), 49 deletions(-) diff --git a/static/index.html b/static/index.html index 5e13ad61..a63dd9de 100644 --- a/static/index.html +++ b/static/index.html @@ -154,9 +154,11 @@ document.addEventListener('DOMContentLoaded', function() { if (name !== 'health' && healthRefreshInterval) { clearInterval(healthRefreshInterval); healthRefreshInterval = null; + const he = document.getElementById('tab-health'); + if (he) delete he.dataset.rendered; } if (name === 'health' && !healthRefreshInterval) { - healthRefreshInterval = setInterval(renderHealth, 10000); + healthRefreshInterval = setInterval(refreshHealth, 10000); } document.querySelectorAll('.tab-content').forEach(el => el.classList.add('hidden')); document.querySelectorAll('.tab-btn').forEach(el => el.classList.remove('active')); @@ -174,10 +176,12 @@ function switchTab(name) { if (name !== 'health' && healthRefreshInterval) { clearInterval(healthRefreshInterval); healthRefreshInterval = null; + const he = document.getElementById('tab-health'); + if (he) delete he.dataset.rendered; } // 进入 health 时确保定时器运行 if (name === 'health' && !healthRefreshInterval) { - healthRefreshInterval = setInterval(renderHealth, 10000); + healthRefreshInterval = setInterval(refreshHealth, 10000); } const btn = document.querySelector(`[data-tab="${name}"]`); if (btn) { @@ -1465,13 +1469,13 @@ async function renderHealth() { // Summary cards — 包含 XMPP 状态 let html = '
'; - html += '
' + (summary.ok || 0) + '
正常服务
'; - html += '
' + (summary.total || 0) + '
总服务数
'; + html += '
' + (summary.ok || 0) + '
正常服务
'; + html += '
' + (summary.total || 0) + '
总服务数
'; const xmppStatus = xmpp ? xmpp.status : 'unknown'; const xmppColor = xmppStatus === 'ok' ? '#3fb950' : xmppStatus === 'degraded' ? '#d29922' : '#f85149'; const xmppLabel = xmppStatus === 'ok' ? 'XMPP 正常' : xmppStatus === 'degraded' ? 'XMPP 降级' : xmppStatus === 'critical' ? 'XMPP 断联' : 'XMPP 无数据'; - html += '
' + (xmpp ? (xmpp.last_message_age_sec > 0 ? Math.round(xmpp.last_message_age_sec/60) + 'm' : '--') : '--') + '
' + xmppLabel + '
'; - html += '
' + (xmpp ? xmpp.error_rate_1h + '%' : '--') + '
XMPP 错误率(1h)
'; + html += '
' + (xmpp ? (xmpp.last_message_age_sec > 0 ? Math.round(xmpp.last_message_age_sec/60) + 'm' : '--') : '--') + '
' + xmppLabel + '
'; + html += '
' + (xmpp ? xmpp.error_rate_1h + '%' : '--') + '
XMPP 错误率(1h)
'; html += '
'; // Services by layer @@ -1491,9 +1495,9 @@ async function renderHealth() { const label = s.label || s.name || s.type || ''; const port = s.port || '-'; if (i === 0) { - html += '' + layer + '' + s.name + ' ' + label + '' + port + '' + statusIcon + (ok ? ' 正常' : (ok === false ? ' 异常' : ' 未知')) + ''; + html += '' + layer + '' + s.name + ' ' + label + '' + port + '' + statusIcon + (ok ? ' 正常' : (ok === false ? ' 异常' : ' 未知')) + ''; } else { - html += '' + s.name + ' ' + label + '' + port + '' + statusIcon + (ok ? ' 正常' : (ok === false ? ' 异常' : ' 未知')) + ''; + html += '' + s.name + ' ' + label + '' + port + '' + statusIcon + (ok ? ' 正常' : (ok === false ? ' 异常' : ' 未知')) + ''; } }); }); @@ -1510,24 +1514,104 @@ async function renderHealth() { html += ''; } - html += '
更新于 ' + (monitor.generated_at || new Date().toLocaleTimeString()) + ' · 每10秒刷新
'; + html += '
更新于 ' + (monitor.generated_at || new Date().toLocaleTimeString()) + '
'; // XMPP 通道详情 if (xmpp) { html += '

📡 XMPP 通道

'; html += '
'; - html += '
最后消息
' + (xmpp.last_message_age_sec > 0 ? Math.round(xmpp.last_message_age_sec) + '秒前' : '无数据') + '
'; - html += '
1h错误率
' + xmpp.error_rate_1h + '%
'; - html += '
ejabberd
' + (xmpp.ejabberd || '?') + '
'; - html += '
状态
' + xmpp.status + '
'; - html += '
'; + html += '
最后消息
' + (xmpp.last_message_age_sec > 0 ? Math.round(xmpp.last_message_age_sec) + '秒前' : '无数据') + '
'; + html += '
1h错误率
' + xmpp.error_rate_1h + '%
'; + html += '
ejabberd
' + (xmpp.ejabberd || '?') + '
'; + html += '
状态
' + xmpp.status + '
'; + html += ''; + + // Bot 活动（知微） + const ba = xmpp.bot_activity || {}; + html += '
'; + html += '
📩 入站
' + (ba.inbound || 0) + '
'; + html += '
📤 出站
' + (ba.outbound || 0) + '
'; + html += '
⚠️ 错误
' + (ba.errors || 0) + '
'; + html += '
'; + if (ba.last_error) { + html += '
' + ba.last_error + '
'; + } + html += ''; } el.innerHTML = html; + el.dataset.rendered = '1'; // 标记已首渲 } catch(e) { el.innerHTML = '
加载健康状态失败: ' + e.message + '
'; } } +// 增量刷新健康数据（不重建 DOM，无闪烁） +async function refreshHealth() { + const el = document.getElementById('tab-health'); + if (!el || el.classList.contains('hidden') || !el.dataset.rendered) return; + try { + const [services, xmpp] = await Promise.all([ + fetchJSON('/api/services'), + fetchJSON('/api/xmpp/health') + ]); + const svcs = (services.services || []); + const sum = services.summary || {}; + + // 更新卡片数字 + const okEl = document.getElementById('hSvcOk'); + const totalEl = document.getElementById('hSvcTotal'); + const ageEl = document.getElementById('hXmppAge'); + const errEl = document.getElementById('hXmppErr'); + const labelEl = document.getElementById('hXmppLabel'); + if (okEl) okEl.textContent = sum.ok || 0; + if (totalEl) totalEl.textContent = sum.total || 0; + if (ageEl && xmpp) ageEl.textContent = xmpp.last_message_age_sec > 0 ? Math.round(xmpp.last_message_age_sec/60) + 'm' : '--'; + if (errEl && xmpp) errEl.textContent = xmpp.error_rate_1h + '%'; + + // 更新服务行状态 + svcs.forEach(s => { + const cell = document.getElementById('hSvc_' + s.name); + if (cell) { + const ok = s.health?.ok; + cell.textContent = (ok ? '🟢' : (ok === false ? '🔴' : '🟡')) + (ok ? ' 正常' : (ok === false ? ' 异常' : ' 未知')); + cell.className = 'p-3 ' + (ok ? 'text-[#3fb950]' : (ok === false ? 'text-[#f85149]' : 'text-[#d29922]')); + } + }); + + // 更新 XMPP 详情 + if (xmpp) { + const d_age = document.getElementById('hXmppDetail_age'); + const d_err = document.getElementById('hXmppDetail_err'); + const d_eja = document.getElementById('hXmppDetail_eja'); + const d_st = document.getElementById('hXmppDetail_st'); + if (d_age) d_age.textContent = xmpp.last_message_age_sec > 0 ? Math.round(xmpp.last_message_age_sec) + '秒前' : '无数据'; + if (d_err) d_err.textContent = xmpp.error_rate_1h + '%'; + if (d_eja) d_eja.textContent = xmpp.ejabberd || '?'; + if (d_st) d_st.textContent = xmpp.status; + // 更新卡片颜色 + if (ageEl && xmpp) ageEl.style.color = xmpp.last_message_age_sec > 600 ? '#f85149' : '#3fb950'; + if (labelEl && xmpp) { + const labels = {ok:'XMPP 正常', degraded:'XMPP 降级', critical:'XMPP 断联'}; + labelEl.textContent = labels[xmpp.status] || 'XMPP 无数据'; + } + // 更新 Bot 活动 + const ba = xmpp.bot_activity || {}; + const bi = document.getElementById('hBot_in'); + const bo = document.getElementById('hBot_out'); + const be = document.getElementById('hBot_err'); + const bl = document.getElementById('hBot_lastErr'); + if (bi) bi.textContent = ba.inbound || 0; + if (bo) { bo.textContent = ba.outbound || 0; bo.className = 'font-mono ' + (ba.outbound ? 'text-[#3fb950]' : (ba.inbound ? 'text-[#f85149]' : 'text-slate-500')); } + if (be) { be.textContent = ba.errors || 0; be.className = 'font-mono ' + (ba.errors ? 'text-[#f85149]' : 'text-slate-500'); } + if (bl && ba.last_error) bl.textContent = ba.last_error; + } + + // 更新时间戳 + const upd = document.getElementById('hUpdated'); + if (upd) upd.textContent = '更新于 ' + new Date().toLocaleTimeString(); + } catch(e) { /* 静默失败，保留旧数据 */ } +} + // ── 盯盘 ── let watchTimer = null; async function renderWatch() { diff --git a/xmpp_logger.py b/xmpp_logger.py index 73233cd7..9d4c9e07 100644 --- a/xmpp_logger.py +++ b/xmpp_logger.py @@ -142,44 +142,78 @@ def stats(): def health(): - """快速健康检查：返回最后消息年龄 + 最近1h错误率""" - if not LOG_FILE.exists(): - return {"status": "no_data", "last_message_age_sec": -1, "error_rate_1h": 0} + """快速健康检查：返回最后消息年龄 + 最近1h错误率 + bot 日志状态""" + result = {"status": "no_data", "last_message_age_sec": -1, "error_rate_1h": 0, "bot_activity": {}} - now_epoch = _time.time() - last_epoch = 0 - recent_total = 0 - recent_errors = 0 - one_hour_ago = now_epoch - 3600 + # 1. 检查 xmpp_messages.jsonl + if LOG_FILE.exists(): + now_epoch = _time.time() + last_epoch = 0 + recent_total = 0 + recent_errors = 0 + one_hour_ago = now_epoch - 3600 + try: + with open(LOG_FILE, "r", encoding="utf-8") as f: + for line in f: + try: + e = json.loads(line) + ep = e.get("epoch", 0) + if ep > last_epoch: + last_epoch = ep + if ep > one_hour_ago: + recent_total += 1 + if e["status"] != "ok": + recent_errors += 1 + except Exception: + continue + except Exception: + pass + + last_age = int(now_epoch - last_epoch) if last_epoch else -1 + err_rate = round(recent_errors / recent_total * 100, 1) if recent_total else 0 + result["last_message_age_sec"] = last_age + result["error_rate_1h"] = err_rate + + # 2. 检查知微 Bot systemd journal（最近5分钟） try: - with open(LOG_FILE, "r", encoding="utf-8") as f: - for line in f: - try: - e = json.loads(line) - ep = e.get("epoch", 0) - if ep > last_epoch: - last_epoch = ep - if ep > one_hour_ago: - recent_total += 1 - if e["status"] != "ok": - recent_errors += 1 - except Exception: - continue + import subprocess + r = subprocess.run( + ["journalctl", "-u", "xmpp-zhiwei", "--no-pager", "--since", "5 min ago", "-o", "cat"], + capture_output=True, timeout=5, text=True + ) + lines = [l for l in r.stdout.split("\n") if l.strip()] + inbound = [l for l in lines if "📩 收到" in l] + outbound = [l for l in lines if "📤 发送" in l or "📤" in l] + errors = [l for l in lines if "ERROR" in l or "error" in l or "timeout" in l or "Timeout" in l] + result["bot_activity"] = { + "inbound": len(inbound), + "outbound": len(outbound), + "errors": len(errors), + "last_error": errors[-1][:200] if errors else None, + "last_inbound": inbound[-1][:200] if inbound else None, + } + # 有错误且无出站 → degraded + if errors and not outbound: + if result.get("status") != "critical": + result["status"] = "degraded" + # 有入站无出站 → 可能卡住了 + if inbound and not outbound: + result["status"] = "degraded" except Exception: - pass + result["bot_activity"] = {"error": "journalctl failed"} - last_age = int(now_epoch - last_epoch) if last_epoch else -1 - err_rate = round(recent_errors / recent_total * 100, 1) if recent_total else 0 + # 3. 综合判定 + if result.get("status") != "degraded": + la = result.get("last_message_age_sec", -1) + er = result.get("error_rate_1h", 0) + if la < 0: + result["status"] = "no_data" + elif la > 600: + result["status"] = "critical" + elif er > 50: + result["status"] = "degraded" + elif la > 0: + result["status"] = "ok" - # 状态判断 - if last_age < 0: - st = "no_data" - elif last_age > 600: # >10min no message - st = "critical" - elif err_rate > 50: - st = "degraded" - else: - st = "ok" - - return {"status": st, "last_message_age_sec": last_age, "error_rate_1h": err_rate} + return result