feat: mofin_health 加 LLM 端点错误日志监控(规范5.5)
This commit is contained in:
@@ -996,6 +996,64 @@ def build_report():
|
||||
except Exception:
|
||||
self_check['recent_repairs'] = []
|
||||
|
||||
# ── LLM端点健康检查(2026-08-10新增:监控LLM调用失败,防静默故障)──
|
||||
# 背景:opencode.ai deepseek-v4-flash 端点故障时,gateway进程活着但LLM全挂,
|
||||
# 监控只查端口/进程发现不了。扫描hermes错误日志弥补。
|
||||
llm_health = {"ok": True, "errors_recent": [], "scan_window_min": 60}
|
||||
_LLM_ERR_PATTERNS = [
|
||||
"RemoteProtocolError", "Stream stale", "empty stream",
|
||||
"peer closed connection", "finish_reason", "API call failed",
|
||||
]
|
||||
try:
|
||||
_err_logs = [
|
||||
Path('/home/hmo/.hermes/profiles/position-analyst/logs/errors.log'),
|
||||
Path('/home/hmo/.hermes/profiles/position-analyst/logs/agent.log'),
|
||||
]
|
||||
_recent_errs = []
|
||||
_scan_cutoff = (now.timestamp() - 60 * llm_health["scan_window_min"])
|
||||
for _lf in _err_logs:
|
||||
if not _lf.exists():
|
||||
continue
|
||||
try:
|
||||
_lines = _lf.read_text(encoding='utf-8', errors='ignore').splitlines()[-5000:]
|
||||
for _line in _lines:
|
||||
# 提取时间戳(行首 YYYY-MM-DD HH:MM:SS)
|
||||
if len(_line) < 19:
|
||||
continue
|
||||
try:
|
||||
_ts = datetime.strptime(_line[:19], "%Y-%m-%d %H:%M:%S")
|
||||
except Exception:
|
||||
continue
|
||||
if _ts.timestamp() < _scan_cutoff:
|
||||
continue
|
||||
if any(_p in _line for _p in _LLM_ERR_PATTERNS):
|
||||
_recent_errs.append({
|
||||
"log": _lf.name, "time": _ts.strftime("%H:%M"),
|
||||
"msg": _line[20:180],
|
||||
})
|
||||
except Exception:
|
||||
pass
|
||||
# 去重+限量
|
||||
_seen = set()
|
||||
_uniq = []
|
||||
for _e in _recent_errs:
|
||||
_k = _e["time"] + _e["msg"][:50]
|
||||
if _k not in _seen:
|
||||
_seen.add(_k)
|
||||
_uniq.append(_e)
|
||||
llm_health["errors_recent"] = _uniq[:10]
|
||||
llm_health["error_count"] = len(_uniq)
|
||||
llm_health["ok"] = len(_uniq) == 0
|
||||
self_check["llm_health"] = llm_health
|
||||
# 若LLM故障,功能树根节点标 fail
|
||||
if not llm_health["ok"]:
|
||||
feature_tree["status"] = "fail"
|
||||
feature_tree.setdefault("warn_detail", {})["llm"] = (
|
||||
f"LLM端点最近{llm_health['scan_window_min']}分钟{llm_health['error_count']}次错误"
|
||||
f"({llm_health['errors_recent'][0]['log'] if llm_health['errors_recent'] else '?'})")
|
||||
except Exception as _e:
|
||||
self_check["llm_health"] = {"ok": True, "error": str(_e)[:100]}
|
||||
|
||||
# ── 写JSON ──
|
||||
report = {
|
||||
"generated_at": now.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
|
||||
Reference in New Issue
Block a user