Files
MoFin/static/mofin_health.html
T
hmo 08eef1e181 feat(self-check): L0-L4 layered self-check architecture with LLM auto-repair
User directive: daily not weekly; clear responsibilities per layer with no
overlap; functional criteria (does the function WORK) not process liveness;
problems get FIXED via LLM with file-and-report discipline (act first,
report after); plus a meta-layer watching the watchers; deeply integrated
into F健康.

Architecture (responsibility matrix in dev-spec.md):
- L0 agents_health_check (5min): port/HTTP/DB liveness + auto_heal executor
- L1 functional_health_check (15min trading): per-module FUNCTIONAL
  criteria — output freshness/validity per REGISTRY (live_prices/market_
  snapshots/mtf_cache/macro_context/bot/LLM/cron engine), not process alive
- L2 system_hygiene_audit (daily 08:20, was weekly): divergence/hardlink/
  zombie/orphan/dead-cron/db-freshness
- L3 self_repair (30min): reads L1/L2 failures -> LLM diagnoses -> executes
  WHITELISTED repair actions directly (rerun_script/restart_service/
  sync_links/switch_llm_key/none) -> repair_log.jsonl + XMPP report.
  Max 2 repairs/module/day anti-loop. LLM unavailable -> rule fallback.
- L4 meta_watchdog (hourly): checks L0-L3 output freshness + L3 cron
  registration + XMPP bridge; direct XMPP alert as last resort

Retired (overlap): Cron监护-高频 (cron_watchdog -> L3), 全局cron健康监控
(cron_health_monitor -> L1).

Dashboard: mofin_health.py now emits self_check section (functional/meta/
hygiene/recent_repairs); mofin_health.html new '🩺 自检体系' tab rendering
L4 layers, L1 module checks, L2 issues, L3 repair history.

E2E verified: stopped xmpp bot -> L1 flagged fail -> systemd recovered ->
L3 LLM correctly diagnosed 'none needed' and logged; rerun_script whitelist
path executes real scripts successfully; meta_watchdog all-green after fix.
2026-07-20 19:39:58 +08:00

391 lines
20 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html>
<html lang="zh-CN">
<head><meta charset="utf-8"><title>MoFin 健康监控</title>
<style>
*{margin:0;padding:0;box-sizing:border-box;font-family:system-ui,-apple-system,sans-serif}
body{background:#0d1117;color:#c9d1d9;padding:20px}
h1{font-size:22px;margin-bottom:16px;color:#58a6ff}
.tabs{display:flex;gap:4px;margin-bottom:16px;border-bottom:1px solid #30363d;flex-wrap:wrap}
.tab{padding:8px 20px;cursor:pointer;border:1px solid transparent;border-radius:4px 4px 0 0;color:#8b949e;font-size:14px}
.tab.active{background:#161b22;border-color:#30363d #30363d #161b22;color:#c9d1d9}
.tab:hover{color:#58a6ff}
.panel{display:none}
.panel.active{display:block}
.badge{display:inline-block;width:10px;height:10px;border-radius:50%;margin-right:6px}
.badge.ok{background:#3fb950}
.badge.warn{background:#d29922}
.badge.fail{background:#f85149}
table{width:100%;border-collapse:collapse;font-size:13px;margin-bottom:20px}
th,td{padding:5px 8px;text-align:left;border-bottom:1px solid #21262d;white-space:nowrap}
th{background:#161b22;color:#8b949e;font-weight:500;position:sticky;top:0}
tr:hover{background:#1c2128}
td.desc{color:#8b949e;font-size:11px;white-space:normal;max-width:260px}
td.wrap{white-space:normal;font-size:11px;max-width:200px}
.tree-node{padding:2px 0;font-size:13px}
.tree-children{padding-left:22px;border-left:1px solid #30363d;margin-left:8px}
.tree-node.ok{color:#3fb950}
.tree-node.warn{color:#d29922}
.tree-node.fail{color:#f85149}
.pipeline-ok{color:#3fb950}
.pipeline-warn{color:#d29922}
.pipeline-error{color:#f85149}
.pipeline-ever{color:#8b949e}
.orphan{background:#2d1517!important}
.orphan td{color:#f85149}
.summary{display:flex;gap:20px;margin-bottom:16px;font-size:13px;color:#8b949e;flex-wrap:wrap}
.summary span strong{color:#c9d1d9}
.toggle{color:#58a6ff;cursor:pointer;font-size:11px;margin-left:8px}
.search-box{background:#0d1117;border:1px solid #30363d;color:#c9d1d9;padding:6px 12px;border-radius:4px;margin-bottom:10px;width:300px;font-size:13px}
.cron-detail{font-size:11px;color:#8b949e;padding-left:20px}
.cron-tag{display:inline-block;padding:1px 6px;border-radius:3px;font-size:10px;margin:0 2px}
.cron-tag.ok{background:#1a3a1a;color:#3fb950;border:1px solid #3fb950}
.cron-tag.warn{background:#3a2a1a;color:#d29922;border:1px solid #d29922}
.cron-tag.error{background:#3a1a1a;color:#f85149;border:1px solid #f85149}
.cron-tag.new{background:#1a1a3a;color:#58a6ff;border:1px solid #58a6ff}
</style></head>
<body>
<h1>📊 MoFin 系统健康监控</h1>
<div class="summary" id="summary">加载中...</div>
<div class="tabs">
<div class="tab active" onclick="switchTab(0)">🌳 功能树</div>
<div class="tab" onclick="switchTab(1)">🔧 全部流程/Cron</div>
<div class="tab" onclick="switchTab(2)">🗃️ 数据实体</div>
<div class="tab" onclick="switchTab(3)">🔀 数据流</div>
<div class="tab" onclick="switchTab(4)">🩺 自检体系</div>
</div>
<div id="panel0" class="panel active"></div>
<div id="panel1" class="panel"></div>
<div id="panel2" class="panel"></div>
<div id="panel3" class="panel"></div>
<div id="panel4" class="panel"></div>
<script>
let data = null;
function renderSelfCheck(sc) {
if (!sc) return '<div style="color:#8b949e">自检体系数据未生成(等待 L1/L4 cron 首次运行)</div>';
let html = '';
const icon = s => s === 'ok' ? '✅' : s === 'warn' ? '🟡' : s === 'fail' ? '❌' : '⏭';
// L4 元监控(自检系统的自检)
if (sc.meta_watchdog) {
const mw = sc.meta_watchdog;
html += `<div style="background:#161b22;border:1px solid #30363d;border-radius:6px;padding:10px;margin-bottom:10px">`;
html += `<div style="font-weight:bold;color:#58a6ff;margin-bottom:6px">🐕 L4 元监控 — 自检系统的自检 <span style="color:${mw.status === 'ok' ? '#3fb950' : '#f85149'}">${icon(mw.status)} ${mw.status}</span> <span style="color:#8b949e;font-size:11px">${mw.generated_at || ''}</span></div>`;
html += '<table><thead><tr><th>层</th><th>状态</th><th>说明</th></tr></thead>';
(mw.layers || []).forEach(l => {
html += `<tr><td>${l.layer}</td><td class="pipeline-${l.status === 'ok' ? 'ok' : 'error'}">${icon(l.status)} ${l.status}</td><td style="font-size:11px;color:#8b949e">${l.reason || ''}</td></tr>`;
});
html += '</table></div>';
}
// L1 功能健康
if (sc.functional) {
const fh = sc.functional;
const s = fh.summary || {};
html += `<div style="background:#161b22;border:1px solid #30363d;border-radius:6px;padding:10px;margin-bottom:10px">`;
html += `<div style="font-weight:bold;color:#58a6ff;margin-bottom:6px">⚙️ L1 功能健康(功能是否达成,非进程存活) <span style="color:${fh.status === 'ok' ? '#3fb950' : fh.status === 'warn' ? '#d29922' : '#f85149'}">${icon(fh.status)} ${fh.status}</span> <span style="color:#8b949e;font-size:11px">ok:${s.ok} warn:${s.warn} fail:${s.fail} skip:${s.skip} · ${fh.generated_at || ''}</span></div>`;
html += '<table><thead><tr><th>模块</th><th>功能判据</th><th>状态</th><th>说明</th></tr></thead>';
(fh.checks || []).forEach(c => {
html += `<tr><td>${c.module}</td><td style="font-size:11px;color:#8b949e">${c.function}</td><td class="pipeline-${c.status === 'ok' ? 'ok' : c.status === 'warn' ? 'warn' : c.status === 'fail' ? 'error' : 'ever'}">${icon(c.status)} ${c.status}</td><td style="font-size:11px;color:#8b949e">${c.reason || ''}</td></tr>`;
});
html += '</table></div>';
}
// L2 卫生审计
if (sc.hygiene) {
const hy = sc.hygiene;
html += `<div style="background:#161b22;border:1px solid #30363d;border-radius:6px;padding:10px;margin-bottom:10px">`;
html += `<div style="font-weight:bold;color:#58a6ff;margin-bottom:6px">🧹 L2 系统卫生审计(每日08:20 <span style="color:${hy.status === 'ok' ? '#3fb950' : '#d29922'}">${icon(hy.status)} ${hy.status}</span> <span style="color:#8b949e;font-size:11px">${hy.issue_count} 个问题 · ${hy.generated_at || ''}</span></div>`;
(hy.issues || []).forEach(i => {
html += `<div style="font-size:11px;color:#d29922">• [${i.type}] ${i.file || i.job || i.table || ''}${i.action || ''}</div>`;
});
html += '</div>';
}
// L3 修复记录
html += `<div style="background:#161b22;border:1px solid #30363d;border-radius:6px;padding:10px;margin-bottom:10px">`;
html += `<div style="font-weight:bold;color:#58a6ff;margin-bottom:6px">🔧 L3 LLM 修复循环 — 最近修复动作(报备制)</div>`;
const reps = sc.recent_repairs || [];
if (!reps.length) {
html += '<div style="font-size:11px;color:#8b949e">暂无修复记录(系统健康或 L3 未触发)</div>';
} else {
reps.forEach(r => {
const okIcon = r.ok === true ? '✅' : r.ok === false ? '❌' : '⏭';
html += `<div style="font-size:11px;margin:3px 0"><span style="color:#8b949e">${(r.ts || '').slice(5, 16)}</span> ${okIcon} [${r.module}] ${(r.action || {}).action || r.action}: <span style="color:#8b949e">${(r.detail || r.reason || '').slice(0, 80)}</span></div>`;
});
}
html += '</div>';
return html;
}
function switchTab(idx) {
document.querySelectorAll('.tab').forEach((t,i)=>t.classList.toggle('active',i==idx));
document.querySelectorAll('.panel').forEach((p,i)=>p.classList.toggle('active',i==idx));
}
function renderFeatureTree(tree) {
let grp = (function(){let i=0;return function(){return 'ftg-'+(++i);}})();
let html = '<table><thead><tr><th style="width:32%">功能模块</th><th style="width:35%">说明</th><th>来源</th><th>类型</th><th>调度</th><th>状态</th><th>最后运行</th></tr></thead>';
function walk(node, depth, groupId) {
const indent = 'padding-left:'+(depth*20+8)+'px';
const cls = node.status || 'ok';
const hasKids = node.children && node.children.length > 0;
const pipes = node.pipes || [];
const isCat = hasKids && pipes.length === 0;
if (isCat) {
// 分类节点——生成一个可折叠组
const g = grp();
const desc = node.desc || '';
html += `<tr id="${g}-h" onclick="toggleFt('${g}')" style="cursor:pointer" data-group="${groupId||''}">`;
html += `<td style="${indent}"><span id="${g}-ico">▼</span> <span class="badge ${cls}"></span><strong>${node.label}</strong></td>`;
html += `<td style="font-size:11px;color:#8b949e">${desc}</td>`;
html += `<td>-</td><td>-</td><td>-</td><td class="pipeline-${cls}">${cls}</td><td>-</td></tr>`;
// 子节点都带上这个组的data-sub
node.children.forEach(c => walk(c, depth+1, g));
return;
}
if (pipes.length > 0) {
pipes.forEach((p, i) => {
const indent2 = i===0 ? indent : 'padding-left:'+(depth*20+32)+'px';
const tagCls = p.status==='ok'?'ok':p.status==='error'?'error':'warn';
const pBadge = p.profile==='position-analyst' ? '📋' : '📦';
const descText = i===0 ? (node.desc || '') : '';
const groupAttr = groupId ? `data-group="${groupId}"` : '';
if (i===0) {
html += `<tr ${groupAttr}><td style="${indent2}"><span class="badge ${cls}"></span><strong>${node.label}</strong></td>`;
} else {
html += `<tr ${groupAttr}><td style="${indent2}"><span style="color:#30363d">└</span> ${p.name||p.script||'LLM'}</td>`;
}
html += `<td style="font-size:11px;color:#8b949e">${descText}</td>`;
html += `<td><span class="cron-tag ${tagCls}">${p.type||'LLM'}</span></td>`;
html += `<td style="font-size:11px">${p.schedule||'-'}</td>`;
html += `<td class="pipeline-${tagCls}">${p.status}</td>`;
html += `<td style="font-size:11px">${p.last_run||'-'}</td></tr>`;
});
} else {
// 无cron的叶子
const groupAttr = groupId ? `data-group="${groupId}"` : '';
html += `<tr ${groupAttr}><td style="${indent}"><span class="badge ${cls}"></span><strong>${node.label}</strong></td>`;
html += `<td>-</td><td>-</td><td>-</td><td class="pipeline-${cls}">${cls}</td><td>-</td></tr>`;
}
if (hasKids && pipes.length > 0) {
node.children.forEach(c => walk(c, depth+1, groupId));
}
}
walk(tree, 0, null);
html += '</table>';
return html;
}
// 折叠切换:通过data-group控制
let ftState = {};
function toggleFt(g) {
const ico = document.getElementById(g+'-ico');
const isHidden = ftState[g];
ftState[g] = !isHidden;
ico.textContent = isHidden ? '▼' : '▶';
// 隐藏/显示所有带data-group={g}的行
document.querySelectorAll(`[data-group="${g}"]`).forEach(el => {
el.style.display = isHidden ? '' : 'none';
});
}
function renderDataFlow(entities, jsonFiles, architecture) {
// 只显示有写入方或读取方的实体
const filtered = entities.filter(e => (e.writers||[]).length > 0 || (e.readers||[]).length > 0);
// 只显示有读写关系的json
const jf = jsonFiles.filter(j => (j.readers||[]).length > 0);
let html = '<div style="margin-bottom:10px;color:#8b949e;font-size:13px">';
// 架构违规告警
const violations = (architecture||{}).price_api_violations || [];
if (violations.length > 0) {
const scripts = [...new Set(violations.map(v => v.script))];
html += `<div style="background:#3d1c02;border:1px solid #d29922;border-radius:6px;padding:10px;margin-bottom:12px">`;
html += `<div style="font-weight:bold;color:#d29922;margin-bottom:4px">⚠️ 架构违规: ${violations.length}处, ${scripts.length}个脚本自拉API不走live_prices</div>`;
html += `<div style="font-size:11px;color:#8b949e;margin-bottom:6px">预期: price_monitor→live_prices→其他脚本读。以下脚本直接调腾讯API:</div>`;
html += `<div style="font-size:11px">`;
scripts.forEach(s => {
const lines = violations.filter(v => v.script === s).map(v => `L${v.line}`).join(',');
html += `<span class="cron-tag" style="border-color:#d29922;color:#d29922;margin-right:4px;margin-bottom:3px;display:inline-block">${s} (${lines})</span>`;
});
html += `</div></div>`;
}
html += `显示 ${filtered.length} 个数据表 + ${jf.length} 个JSON文件的读写流向。绿色→写入,蓝色→读取。</div>`;
// 按写入方数量排序,最活跃的排前面
filtered.sort((a,b) => (b.writers||[]).length - (a.writers||[]).length);
filtered.forEach(e => {
const w = e.writers || [];
const r = e.readers || [];
if (w.length === 0 && r.length === 0) return;
const fd = e.flow_detail || {};
const wDesc = fd.writers || {};
const rDesc = fd.readers || {};
html += '<div style="background:#161b22;border:1px solid #30363d;border-radius:6px;padding:10px;margin-bottom:8px">';
html += `<div style="font-weight:bold;font-size:14px;color:#58a6ff;margin-bottom:4px">🗄️ ${e.name}</div>`;
html += `<div style="font-size:11px;color:#8b949e;margin-bottom:4px">${e.desc||''}</div>`;
// 综合总结
if (fd.summary) {
html += `<div style="font-size:11px;color:#c9d1d9;margin-bottom:6px;padding:4px 8px;background:#0d1117;border-radius:4px">📌 ${fd.summary}</div>`;
}
// 写入方(带详细说明)
if (w.length > 0) {
html += '<div style="margin-bottom:4px">';
html += '<span style="color:#3fb950;font-size:11px;font-weight:bold">✏️ 写入:</span> ';
w.forEach(wn => {
const desc = wDesc[wn] || '';
html += `<div style="display:inline-block;margin:2px 4px 2px 0">`;
html += `<span class="cron-tag ok" style="font-size:11px">${wn}</span>`;
if (desc) html += `<span style="font-size:10px;color:#8b949e;margin-left:2px">— ${desc}</span>`;
html += `</div>`;
});
html += '</div>';
}
// 读取方(带详细说明)
if (r.length > 0) {
html += '<div>';
html += '<span style="color:#58a6ff;font-size:11px;font-weight:bold">📖 读取:</span> ';
r.forEach(rn => {
const desc = rDesc[rn] || '';
html += `<div style="display:inline-block;margin:2px 4px 2px 0">`;
html += `<span class="cron-tag" style="border-color:#58a6ff;color:#58a6ff;font-size:11px">${rn}</span>`;
if (desc) html += `<span style="font-size:10px;color:#8b949e;margin-left:2px">— ${desc}</span>`;
html += `</div>`;
});
html += '</div>';
}
html += '</div>';
});
// JSON文件
jf.forEach(j => {
const r = j.readers || [];
html += '<div style="background:#161b22;border:1px solid #30363d;border-radius:6px;padding:10px;margin-bottom:8px">';
html += `<div style="font-weight:bold;font-size:14px;color:#d29922;margin-bottom:6px">📄 ${j.name}</div>`;
html += `<div style="font-size:11px;color:#8b949e;margin-bottom:6px">${j.desc||''} · ${j.size_kb}KB</div>`;
if (r.length > 0) {
html += '<div>';
html += '<span style="color:#58a6ff;font-size:11px">📖 读取:</span> ';
html += r.map(s => `<span class="cron-tag" style="border-color:#58a6ff;color:#58a6ff;font-size:11px">${s}</span>`).join(' ');
html += '</div>';
}
html += '</div>';
});
return html;
}
function renderPipelineTable(pipelines) {
let html = '<div style="display:flex;gap:8px;margin-bottom:8px;align-items:center">';
html += '<input class="search-box" id="pipeSearch" placeholder="搜索流程..." oninput="filterPipes()" style="flex:1">';
html += '<select id="pipeProfile" onchange="filterPipes()" style="background:#21262d;color:#c9d1d9;border:1px solid #30363d;border-radius:4px;padding:5px 8px;font-size:12px">';
html += '<option value="position-analyst">📋 知微</option>';
html += '<option value="all">📋+📦 全部</option>';
html += '</select></div>';
html += '<table><thead><tr><th>名称</th><th>来源</th><th>脚本/LLM</th><th>类型</th><th>调度</th><th>状态</th><th>最后运行</th></tr></thead><tbody id="pipeBody">';
pipelines.forEach(p => {
const tagCls = p.status==='ok'?'ok':p.status==='error'?'error':'warn';
const statusDisplay = p.last_run ? p.status : '待首次运行';
const profileBadge = p.profile==='position-analyst' ? '📋' : '📦';
html += `<tr class="pipe-row" data-profile="${p.profile||'default'}" data-name="${(p.name||'').toLowerCase()}"><td>${p.name||p.script||'LLM'}</td>`;
html += `<td style="font-size:11px">${profileBadge} ${p.profile||'?'}</td>`;
html += `<td style="font-size:11px">${p.script||'LLM'}</td>`;
html += `<td><span class="cron-tag ${tagCls}">${p.type||'cron'}</span></td>`;
html += `<td style="font-size:11px">${p.schedule||'-'}</td>`;
html += `<td class="pipeline-${tagCls}">${statusDisplay}</td>`;
html += `<td style="font-size:11px">${p.last_run||'-'}</td></tr>`;
});
html += '</tbody></table>';
return html;
}
function filterPipes() {
const q = document.getElementById('pipeSearch').value.toLowerCase();
const profile = document.getElementById('pipeProfile').value;
document.querySelectorAll('.pipe-row').forEach(r => {
const name = r.getAttribute('data-name') || '';
const rp = r.getAttribute('data-profile') || 'default';
const matchName = !q || name.includes(q);
const matchProfile = profile === 'all' || rp === profile;
r.style.display = matchName && matchProfile ? '' : 'none';
});
}
function loadData() {
fetch('/mofin_health.json?_='+Date.now())
.then(r=>r.json())
.then(d=>{
data = d;
// 统计
const cnt = {ok:0,warn:0,fail:0};
function countStatus(n) { cnt[n.status]++; if(n.children) n.children.forEach(countStatus); }
countStatus(d.feature_tree);
const pipeOk = d.pipelines.filter(p=>p.status==='ok').length;
const pipeErr = d.pipelines.filter(p=>p.status==='error'||p.status==='fail').length;
document.getElementById('summary').innerHTML =
`<span>生成: ${d.generated_at}</span>` +
`<span>功能: ✅${cnt.ok} ⚠️${cnt.warn}${cnt.fail}</span>` +
`<span>Cron: ✅${pipeOk}${pipeErr}${d.pipelines.length}</span>` +
`<span>数据表: ${d.entities.length} | ` +
`<span style="color:#f85149">孤立${d.entities.filter(e=>e.flow_status==='orphan').length}</span> ` +
`<span style="color:#d29922">写无读${d.entities.filter(e=>e.flow_status==='write_only').length}</span> ` +
`<span style="color:#58a6ff">读无写${d.entities.filter(e=>e.flow_status==='read_only').length}</span>` +
`<span style="color:#d29922;margin-left:8px">架构⚠️${(d.architecture||{}).violation_count||0}</span></span>`;
// Tab 0: 功能树
document.getElementById('panel0').innerHTML = renderFeatureTree(d.feature_tree);
// Tab 1: 全部流程表
document.getElementById('panel1').innerHTML = renderPipelineTable(d.pipelines);
// Tab 2: 数据实体
let h2 = '<table><thead><tr><th>表名</th><th>作用</th><th>行数</th><th>写入方</th><th>读取方</th><th>数据流</th></tr></thead><tbody>';
d.entities.forEach(e => {
const flowMap = {healthy:'✅ 正常', write_only:'✏️ 写无读', read_only:'📖 读无写', orphan:'🔴 孤立'};
const flowColors = {healthy:'#3fb950', write_only:'#d29922', read_only:'#58a6ff', orphan:'#f85149'};
const badge = flowMap[e.flow_status] || '?';
const color = flowColors[e.flow_status] || '#8b949e';
const cls = e.orphan ? 'orphan' : '';
h2 += `<tr class="${cls}"><td><strong>${e.name}</strong></td><td class="desc">${e.desc||''}</td><td>${e.rows}</td>`;
h2 += `<td class="wrap">${(e.writers||[]).slice(0,4).join(', ')||'<span style="color:#f85149">无</span>'}</td>`;
h2 += `<td class="wrap">${(e.readers||[]).slice(0,4).join(', ')||'<span style="color:#f85149">无</span>'}</td>`;
h2 += `<td style="color:${color}">${badge}</td></tr>`;
});
(d.json_files||[]).forEach(j => {
h2 += `<tr><td>📄 ${j.name}</td><td class="desc">${j.desc||'JSON文件'}</td><td>${j.size_kb}KB</td>`;
h2 += `<td class="wrap">JSON写入</td><td class="wrap">${(j.readers||[]).slice(0,4).join(', ')||'<span style="color:#f85149">无</span>'}</td>`;
h2 += `<td>${j.warn?'⚠️':'✅'}</td></tr>`;
});
h2 += '</tbody></table>';
document.getElementById('panel2').innerHTML = h2;
// Tab 3: 数据流
document.getElementById('panel3').innerHTML = renderDataFlow(d.entities, d.json_files||[], d.architecture||{});
// Tab 4: 自检体系
document.getElementById('panel4').innerHTML = renderSelfCheck(d.self_check);
})
.catch(e => document.getElementById('summary').innerHTML = '❌ 加载失败: ' + e.message);
}
loadData();
setInterval(loadData, 60000);
</script>
</body></html>