feat(self-check): L0-L4 layered self-check architecture with LLM auto-repair
User directive: daily not weekly; clear responsibilities per layer with no
overlap; functional criteria (does the function WORK) not process liveness;
problems get FIXED via LLM with file-and-report discipline (act first,
report after); plus a meta-layer watching the watchers; deeply integrated
into F健康.
Architecture (responsibility matrix in dev-spec.md):
- L0 agents_health_check (5min): port/HTTP/DB liveness + auto_heal executor
- L1 functional_health_check (15min trading): per-module FUNCTIONAL
criteria — output freshness/validity per REGISTRY (live_prices/market_
snapshots/mtf_cache/macro_context/bot/LLM/cron engine), not process alive
- L2 system_hygiene_audit (daily 08:20, was weekly): divergence/hardlink/
zombie/orphan/dead-cron/db-freshness
- L3 self_repair (30min): reads L1/L2 failures -> LLM diagnoses -> executes
WHITELISTED repair actions directly (rerun_script/restart_service/
sync_links/switch_llm_key/none) -> repair_log.jsonl + XMPP report.
Max 2 repairs/module/day anti-loop. LLM unavailable -> rule fallback.
- L4 meta_watchdog (hourly): checks L0-L3 output freshness + L3 cron
registration + XMPP bridge; direct XMPP alert as last resort
Retired (overlap): Cron监护-高频 (cron_watchdog -> L3), 全局cron健康监控
(cron_health_monitor -> L1).
Dashboard: mofin_health.py now emits self_check section (functional/meta/
hygiene/recent_repairs); mofin_health.html new '🩺 自检体系' tab rendering
L4 layers, L1 module checks, L2 issues, L3 repair history.
E2E verified: stopped xmpp bot -> L1 flagged fail -> systemd recovered ->
L3 LLM correctly diagnosed 'none needed' and logged; rerun_script whitelist
path executes real scripts successfully; meta_watchdog all-green after fix.
This commit is contained in:
@@ -51,16 +51,75 @@ td.wrap{white-space:normal;font-size:11px;max-width:200px}
|
||||
<div class="tab" onclick="switchTab(1)">🔧 全部流程/Cron</div>
|
||||
<div class="tab" onclick="switchTab(2)">🗃️ 数据实体</div>
|
||||
<div class="tab" onclick="switchTab(3)">🔀 数据流</div>
|
||||
<div class="tab" onclick="switchTab(4)">🩺 自检体系</div>
|
||||
</div>
|
||||
|
||||
<div id="panel0" class="panel active"></div>
|
||||
<div id="panel1" class="panel"></div>
|
||||
<div id="panel2" class="panel"></div>
|
||||
<div id="panel3" class="panel"></div>
|
||||
<div id="panel4" class="panel"></div>
|
||||
|
||||
<script>
|
||||
let data = null;
|
||||
|
||||
function renderSelfCheck(sc) {
|
||||
if (!sc) return '<div style="color:#8b949e">自检体系数据未生成(等待 L1/L4 cron 首次运行)</div>';
|
||||
let html = '';
|
||||
const icon = s => s === 'ok' ? '✅' : s === 'warn' ? '🟡' : s === 'fail' ? '❌' : '⏭';
|
||||
|
||||
// L4 元监控(自检系统的自检)
|
||||
if (sc.meta_watchdog) {
|
||||
const mw = sc.meta_watchdog;
|
||||
html += `<div style="background:#161b22;border:1px solid #30363d;border-radius:6px;padding:10px;margin-bottom:10px">`;
|
||||
html += `<div style="font-weight:bold;color:#58a6ff;margin-bottom:6px">🐕 L4 元监控 — 自检系统的自检 <span style="color:${mw.status === 'ok' ? '#3fb950' : '#f85149'}">${icon(mw.status)} ${mw.status}</span> <span style="color:#8b949e;font-size:11px">${mw.generated_at || ''}</span></div>`;
|
||||
html += '<table><thead><tr><th>层</th><th>状态</th><th>说明</th></tr></thead>';
|
||||
(mw.layers || []).forEach(l => {
|
||||
html += `<tr><td>${l.layer}</td><td class="pipeline-${l.status === 'ok' ? 'ok' : 'error'}">${icon(l.status)} ${l.status}</td><td style="font-size:11px;color:#8b949e">${l.reason || ''}</td></tr>`;
|
||||
});
|
||||
html += '</table></div>';
|
||||
}
|
||||
|
||||
// L1 功能健康
|
||||
if (sc.functional) {
|
||||
const fh = sc.functional;
|
||||
const s = fh.summary || {};
|
||||
html += `<div style="background:#161b22;border:1px solid #30363d;border-radius:6px;padding:10px;margin-bottom:10px">`;
|
||||
html += `<div style="font-weight:bold;color:#58a6ff;margin-bottom:6px">⚙️ L1 功能健康(功能是否达成,非进程存活) <span style="color:${fh.status === 'ok' ? '#3fb950' : fh.status === 'warn' ? '#d29922' : '#f85149'}">${icon(fh.status)} ${fh.status}</span> <span style="color:#8b949e;font-size:11px">ok:${s.ok} warn:${s.warn} fail:${s.fail} skip:${s.skip} · ${fh.generated_at || ''}</span></div>`;
|
||||
html += '<table><thead><tr><th>模块</th><th>功能判据</th><th>状态</th><th>说明</th></tr></thead>';
|
||||
(fh.checks || []).forEach(c => {
|
||||
html += `<tr><td>${c.module}</td><td style="font-size:11px;color:#8b949e">${c.function}</td><td class="pipeline-${c.status === 'ok' ? 'ok' : c.status === 'warn' ? 'warn' : c.status === 'fail' ? 'error' : 'ever'}">${icon(c.status)} ${c.status}</td><td style="font-size:11px;color:#8b949e">${c.reason || ''}</td></tr>`;
|
||||
});
|
||||
html += '</table></div>';
|
||||
}
|
||||
|
||||
// L2 卫生审计
|
||||
if (sc.hygiene) {
|
||||
const hy = sc.hygiene;
|
||||
html += `<div style="background:#161b22;border:1px solid #30363d;border-radius:6px;padding:10px;margin-bottom:10px">`;
|
||||
html += `<div style="font-weight:bold;color:#58a6ff;margin-bottom:6px">🧹 L2 系统卫生审计(每日08:20) <span style="color:${hy.status === 'ok' ? '#3fb950' : '#d29922'}">${icon(hy.status)} ${hy.status}</span> <span style="color:#8b949e;font-size:11px">${hy.issue_count} 个问题 · ${hy.generated_at || ''}</span></div>`;
|
||||
(hy.issues || []).forEach(i => {
|
||||
html += `<div style="font-size:11px;color:#d29922">• [${i.type}] ${i.file || i.job || i.table || ''} → ${i.action || ''}</div>`;
|
||||
});
|
||||
html += '</div>';
|
||||
}
|
||||
|
||||
// L3 修复记录
|
||||
html += `<div style="background:#161b22;border:1px solid #30363d;border-radius:6px;padding:10px;margin-bottom:10px">`;
|
||||
html += `<div style="font-weight:bold;color:#58a6ff;margin-bottom:6px">🔧 L3 LLM 修复循环 — 最近修复动作(报备制)</div>`;
|
||||
const reps = sc.recent_repairs || [];
|
||||
if (!reps.length) {
|
||||
html += '<div style="font-size:11px;color:#8b949e">暂无修复记录(系统健康或 L3 未触发)</div>';
|
||||
} else {
|
||||
reps.forEach(r => {
|
||||
const okIcon = r.ok === true ? '✅' : r.ok === false ? '❌' : '⏭';
|
||||
html += `<div style="font-size:11px;margin:3px 0"><span style="color:#8b949e">${(r.ts || '').slice(5, 16)}</span> ${okIcon} [${r.module}] ${(r.action || {}).action || r.action}: <span style="color:#8b949e">${(r.detail || r.reason || '').slice(0, 80)}</span></div>`;
|
||||
});
|
||||
}
|
||||
html += '</div>';
|
||||
return html;
|
||||
}
|
||||
|
||||
function switchTab(idx) {
|
||||
document.querySelectorAll('.tab').forEach((t,i)=>t.classList.toggle('active',i==idx));
|
||||
document.querySelectorAll('.panel').forEach((p,i)=>p.classList.toggle('active',i==idx));
|
||||
@@ -318,6 +377,9 @@ function loadData() {
|
||||
|
||||
// Tab 3: 数据流
|
||||
document.getElementById('panel3').innerHTML = renderDataFlow(d.entities, d.json_files||[], d.architecture||{});
|
||||
|
||||
// Tab 4: 自检体系
|
||||
document.getElementById('panel4').innerHTML = renderSelfCheck(d.self_check);
|
||||
})
|
||||
.catch(e => document.getElementById('summary').innerHTML = '❌ 加载失败: ' + e.message);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user