feat: mofin_health 加 LLM 端点错误日志监控(规范5.5)

This commit is contained in:
xxm
2026-08-10 18:30:35 +08:00
parent 4ad9923020
commit 1b997a6d6f
+58
View File
@@ -996,6 +996,64 @@ def build_report():
except Exception:
self_check['recent_repairs'] = []
# ── LLM端点健康检查(2026-08-10新增:监控LLM调用失败,防静默故障)──
# 背景:opencode.ai deepseek-v4-flash 端点故障时,gateway进程活着但LLM全挂,
# 监控只查端口/进程发现不了。扫描hermes错误日志弥补。
llm_health = {"ok": True, "errors_recent": [], "scan_window_min": 60}
_LLM_ERR_PATTERNS = [
"RemoteProtocolError", "Stream stale", "empty stream",
"peer closed connection", "finish_reason", "API call failed",
]
try:
_err_logs = [
Path('/home/hmo/.hermes/profiles/position-analyst/logs/errors.log'),
Path('/home/hmo/.hermes/profiles/position-analyst/logs/agent.log'),
]
_recent_errs = []
_scan_cutoff = (now.timestamp() - 60 * llm_health["scan_window_min"])
for _lf in _err_logs:
if not _lf.exists():
continue
try:
_lines = _lf.read_text(encoding='utf-8', errors='ignore').splitlines()[-5000:]
for _line in _lines:
# 提取时间戳(行首 YYYY-MM-DD HH:MM:SS
if len(_line) < 19:
continue
try:
_ts = datetime.strptime(_line[:19], "%Y-%m-%d %H:%M:%S")
except Exception:
continue
if _ts.timestamp() < _scan_cutoff:
continue
if any(_p in _line for _p in _LLM_ERR_PATTERNS):
_recent_errs.append({
"log": _lf.name, "time": _ts.strftime("%H:%M"),
"msg": _line[20:180],
})
except Exception:
pass
# 去重+限量
_seen = set()
_uniq = []
for _e in _recent_errs:
_k = _e["time"] + _e["msg"][:50]
if _k not in _seen:
_seen.add(_k)
_uniq.append(_e)
llm_health["errors_recent"] = _uniq[:10]
llm_health["error_count"] = len(_uniq)
llm_health["ok"] = len(_uniq) == 0
self_check["llm_health"] = llm_health
# 若LLM故障,功能树根节点标 fail
if not llm_health["ok"]:
feature_tree["status"] = "fail"
feature_tree.setdefault("warn_detail", {})["llm"] = (
f"LLM端点最近{llm_health['scan_window_min']}分钟{llm_health['error_count']}次错误"
f"{llm_health['errors_recent'][0]['log'] if llm_health['errors_recent'] else '?'}")
except Exception as _e:
self_check["llm_health"] = {"ok": True, "error": str(_e)[:100]}
# ── 写JSON ──
report = {
"generated_at": now.strftime("%Y-%m-%d %H:%M:%S"),