feat(self-check): L0-L4 layered self-check architecture with LLM auto-repair

User directive: daily not weekly; clear responsibilities per layer with no
overlap; functional criteria (does the function WORK) not process liveness;
problems get FIXED via LLM with file-and-report discipline (act first,
report after); plus a meta-layer watching the watchers; deeply integrated
into F健康.

Architecture (responsibility matrix in dev-spec.md):
- L0 agents_health_check (5min): port/HTTP/DB liveness + auto_heal executor
- L1 functional_health_check (15min trading): per-module FUNCTIONAL
  criteria — output freshness/validity per REGISTRY (live_prices/market_
  snapshots/mtf_cache/macro_context/bot/LLM/cron engine), not process alive
- L2 system_hygiene_audit (daily 08:20, was weekly): divergence/hardlink/
  zombie/orphan/dead-cron/db-freshness
- L3 self_repair (30min): reads L1/L2 failures -> LLM diagnoses -> executes
  WHITELISTED repair actions directly (rerun_script/restart_service/
  sync_links/switch_llm_key/none) -> repair_log.jsonl + XMPP report.
  Max 2 repairs/module/day anti-loop. LLM unavailable -> rule fallback.
- L4 meta_watchdog (hourly): checks L0-L3 output freshness + L3 cron
  registration + XMPP bridge; direct XMPP alert as last resort

Retired (overlap): Cron监护-高频 (cron_watchdog -> L3), 全局cron健康监控
(cron_health_monitor -> L1).

Dashboard: mofin_health.py now emits self_check section (functional/meta/
hygiene/recent_repairs); mofin_health.html new '🩺 自检体系' tab rendering
L4 layers, L1 module checks, L2 issues, L3 repair history.

E2E verified: stopped xmpp bot -> L1 flagged fail -> systemd recovered ->
L3 LLM correctly diagnosed 'none needed' and logged; rerun_script whitelist
path executes real scripts successfully; meta_watchdog all-green after fix.
This commit is contained in:
hmo
2026-07-20 19:39:58 +08:00
parent 4f83ee8a01
commit 08eef1e181
11 changed files with 874 additions and 1 deletions
+62
View File
@@ -51,16 +51,75 @@ td.wrap{white-space:normal;font-size:11px;max-width:200px}
<div class="tab" onclick="switchTab(1)">🔧 全部流程/Cron</div>
<div class="tab" onclick="switchTab(2)">🗃️ 数据实体</div>
<div class="tab" onclick="switchTab(3)">🔀 数据流</div>
<div class="tab" onclick="switchTab(4)">🩺 自检体系</div>
</div>
<div id="panel0" class="panel active"></div>
<div id="panel1" class="panel"></div>
<div id="panel2" class="panel"></div>
<div id="panel3" class="panel"></div>
<div id="panel4" class="panel"></div>
<script>
let data = null;
function renderSelfCheck(sc) {
if (!sc) return '<div style="color:#8b949e">自检体系数据未生成(等待 L1/L4 cron 首次运行)</div>';
let html = '';
const icon = s => s === 'ok' ? '✅' : s === 'warn' ? '🟡' : s === 'fail' ? '❌' : '⏭';
// L4 元监控(自检系统的自检)
if (sc.meta_watchdog) {
const mw = sc.meta_watchdog;
html += `<div style="background:#161b22;border:1px solid #30363d;border-radius:6px;padding:10px;margin-bottom:10px">`;
html += `<div style="font-weight:bold;color:#58a6ff;margin-bottom:6px">🐕 L4 元监控 — 自检系统的自检 <span style="color:${mw.status === 'ok' ? '#3fb950' : '#f85149'}">${icon(mw.status)} ${mw.status}</span> <span style="color:#8b949e;font-size:11px">${mw.generated_at || ''}</span></div>`;
html += '<table><thead><tr><th>层</th><th>状态</th><th>说明</th></tr></thead>';
(mw.layers || []).forEach(l => {
html += `<tr><td>${l.layer}</td><td class="pipeline-${l.status === 'ok' ? 'ok' : 'error'}">${icon(l.status)} ${l.status}</td><td style="font-size:11px;color:#8b949e">${l.reason || ''}</td></tr>`;
});
html += '</table></div>';
}
// L1 功能健康
if (sc.functional) {
const fh = sc.functional;
const s = fh.summary || {};
html += `<div style="background:#161b22;border:1px solid #30363d;border-radius:6px;padding:10px;margin-bottom:10px">`;
html += `<div style="font-weight:bold;color:#58a6ff;margin-bottom:6px">⚙️ L1 功能健康(功能是否达成,非进程存活) <span style="color:${fh.status === 'ok' ? '#3fb950' : fh.status === 'warn' ? '#d29922' : '#f85149'}">${icon(fh.status)} ${fh.status}</span> <span style="color:#8b949e;font-size:11px">ok:${s.ok} warn:${s.warn} fail:${s.fail} skip:${s.skip} · ${fh.generated_at || ''}</span></div>`;
html += '<table><thead><tr><th>模块</th><th>功能判据</th><th>状态</th><th>说明</th></tr></thead>';
(fh.checks || []).forEach(c => {
html += `<tr><td>${c.module}</td><td style="font-size:11px;color:#8b949e">${c.function}</td><td class="pipeline-${c.status === 'ok' ? 'ok' : c.status === 'warn' ? 'warn' : c.status === 'fail' ? 'error' : 'ever'}">${icon(c.status)} ${c.status}</td><td style="font-size:11px;color:#8b949e">${c.reason || ''}</td></tr>`;
});
html += '</table></div>';
}
// L2 卫生审计
if (sc.hygiene) {
const hy = sc.hygiene;
html += `<div style="background:#161b22;border:1px solid #30363d;border-radius:6px;padding:10px;margin-bottom:10px">`;
html += `<div style="font-weight:bold;color:#58a6ff;margin-bottom:6px">🧹 L2 系统卫生审计(每日08:20 <span style="color:${hy.status === 'ok' ? '#3fb950' : '#d29922'}">${icon(hy.status)} ${hy.status}</span> <span style="color:#8b949e;font-size:11px">${hy.issue_count} 个问题 · ${hy.generated_at || ''}</span></div>`;
(hy.issues || []).forEach(i => {
html += `<div style="font-size:11px;color:#d29922">• [${i.type}] ${i.file || i.job || i.table || ''}${i.action || ''}</div>`;
});
html += '</div>';
}
// L3 修复记录
html += `<div style="background:#161b22;border:1px solid #30363d;border-radius:6px;padding:10px;margin-bottom:10px">`;
html += `<div style="font-weight:bold;color:#58a6ff;margin-bottom:6px">🔧 L3 LLM 修复循环 — 最近修复动作(报备制)</div>`;
const reps = sc.recent_repairs || [];
if (!reps.length) {
html += '<div style="font-size:11px;color:#8b949e">暂无修复记录(系统健康或 L3 未触发)</div>';
} else {
reps.forEach(r => {
const okIcon = r.ok === true ? '✅' : r.ok === false ? '❌' : '⏭';
html += `<div style="font-size:11px;margin:3px 0"><span style="color:#8b949e">${(r.ts || '').slice(5, 16)}</span> ${okIcon} [${r.module}] ${(r.action || {}).action || r.action}: <span style="color:#8b949e">${(r.detail || r.reason || '').slice(0, 80)}</span></div>`;
});
}
html += '</div>';
return html;
}
function switchTab(idx) {
document.querySelectorAll('.tab').forEach((t,i)=>t.classList.toggle('active',i==idx));
document.querySelectorAll('.panel').forEach((p,i)=>p.classList.toggle('active',i==idx));
@@ -318,6 +377,9 @@ function loadData() {
// Tab 3: 数据流
document.getElementById('panel3').innerHTML = renderDataFlow(d.entities, d.json_files||[], d.architecture||{});
// Tab 4: 自检体系
document.getElementById('panel4').innerHTML = renderSelfCheck(d.self_check);
})
.catch(e => document.getElementById('summary').innerHTML = '❌ 加载失败: ' + e.message);
}