151 lines
5.6 KiB
Python
151 lines
5.6 KiB
Python
#!/usr/bin/env python3
|
||
"""agent_spiral_watchdog.py — agent 会话螺旋检测（每 10 分钟 cron）
|
||
|
||
盲区①的系统性补丁（2026-07-21 603288 事件：一次重评请求在 hermes agent
|
||
运行时里螺旋 35 分钟/37 轮工具/153k token，无任何组件察觉）。
|
||
|
||
检测：各 profile state.db 中仍开放的 api_server session，
|
||
启动超 15 分钟且 message_count > 80（正常调用 <20 条）→ 螺旋嫌疑。
|
||
|
||
v1 动作：XMPP 告警 + 日志（不自动 kill——kill 需要 hermes 会话取消 API，
|
||
待确认安全机制后升级 v2）。同一 session 只告警一次（state 文件去重）。
|
||
"""
|
||
import json, os, sqlite3, glob
|
||
from datetime import datetime
|
||
|
||
# ── 消息通道统一路由(broadcast/xmpp by delivery) ──
|
||
try:
|
||
from messenger import install_stdio_hook as _msh
|
||
_msh()
|
||
except Exception:
|
||
pass
|
||
|
||
STATE_FILE = "/home/hmo/MoFin/gateway/logs/spiral_watchdog_state.json"
|
||
# 2026-08-19 僵尸会话检测：记录每 session 上次 msg 数，停滞不增长 = 僵尸（非螺旋）
|
||
ZOMBIE_GROWTH = 0.001 # 若本轮 msgs 相对上次增长 < 0.1%（停滞）→ 僵尸跳过
|
||
MIN_GROW_TO_ALERT = 20 # 真螺旋：本次 msgs 比上次增长 >= 20 条才 XMPP 告警（飞速跑）
|
||
LOG = "/home/hmo/MoFin/gateway/logs/spiral_watchdog.log"
|
||
MIN_AGE_SEC = 15 * 60
|
||
MIN_MESSAGES = 80
|
||
|
||
|
||
def log(msg):
|
||
line = f"[{datetime.now().isoformat(timespec='seconds')}] {msg}"
|
||
print(line, flush=True)
|
||
try:
|
||
os.makedirs(os.path.dirname(LOG), exist_ok=True)
|
||
with open(LOG, "a", encoding="utf-8") as f:
|
||
f.write(line + "\n")
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
def xmpp(msg):
|
||
try:
|
||
import sys
|
||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||
from alert_helper import notify, INFO
|
||
notify("螺旋监控", msg, INFO)
|
||
except Exception as e:
|
||
log(f"XMPP发送失败: {e}")
|
||
|
||
|
||
def load_state():
|
||
try:
|
||
with open(STATE_FILE, encoding="utf-8") as f:
|
||
return json.load(f)
|
||
except Exception:
|
||
return {"alerted": []}
|
||
|
||
|
||
def save_state(st):
|
||
try:
|
||
st["alerted"] = st.get("alerted", [])[-50:]
|
||
with open(STATE_FILE, "w", encoding="utf-8") as f:
|
||
json.dump(st, f)
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
def _to_ms(ts):
|
||
"""started_at 单位自适应：>1e12 视为毫秒，否则视为秒。"""
|
||
if not ts:
|
||
return 0
|
||
return ts if ts > 1e12 else ts * 1000
|
||
|
||
|
||
def main():
|
||
st = load_state()
|
||
alerted = set(st.get("alerted", []))
|
||
prev_msgs = st.get("prev_msgs", {}) # {sid: last_msg_count}
|
||
now_ms = datetime.now().timestamp() * 1000
|
||
found = 0
|
||
|
||
for db_path in glob.glob("/home/hmo/.hermes/profiles/*/state.db"):
|
||
profile = db_path.split("/")[-2]
|
||
try:
|
||
conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True, timeout=10)
|
||
conn.row_factory = sqlite3.Row
|
||
tables = {r[0] for r in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")}
|
||
if "sessions" not in tables:
|
||
conn.close()
|
||
continue
|
||
rows = conn.execute("""
|
||
SELECT id, started_at, message_count, input_tokens, tool_call_count
|
||
FROM sessions
|
||
WHERE source = 'api_server'
|
||
AND (ended_at IS NULL OR ended_at = 0)
|
||
ORDER BY started_at DESC LIMIT 30
|
||
""").fetchall()
|
||
conn.close()
|
||
except Exception as e:
|
||
log(f"[{profile}] state.db 读取失败: {str(e)[:80]}")
|
||
continue
|
||
|
||
for r in rows:
|
||
sa = _to_ms(r["started_at"])
|
||
age_sec = (now_ms - sa) / 1000
|
||
if age_sec < MIN_AGE_SEC:
|
||
continue
|
||
if (r["message_count"] or 0) < MIN_MESSAGES:
|
||
continue
|
||
sid = str(r["id"])
|
||
mc = r["message_count"] or 0
|
||
# 僵尸会话检测：同 session msgs 停滞不增长 → 不是螺旋（7/31 旧会话 bug 教训）
|
||
prev = prev_msgs.get(sid)
|
||
if prev is not None and abs(mc - prev) <= 1:
|
||
# 停滞 → 僵尸，仅记录不再告警
|
||
log(f"ZOMBIE: [{profile}] {sid} msgs={mc}（停滞，非螺旋，跳过）")
|
||
continue
|
||
if sid in alerted:
|
||
continue
|
||
found += 1
|
||
alerted.add(sid)
|
||
prev_msgs[sid] = mc
|
||
# 真螺旋判定：首次发现（prev 为 None）→ 只写日志
|
||
# 之后 msgs 快速增长（>=MIN_GROW_TO_ALERT）→ XMPP 告警（跑飞了）
|
||
if prev is not None and mc - prev >= MIN_GROW_TO_ALERT:
|
||
msg = (f"🚨 agent 螺旋确认（msgs 快速增长）\n"
|
||
f"profile: {profile}\n"
|
||
f"session: {sid}\n"
|
||
f"已运行: {age_sec/60:.0f} 分钟\n"
|
||
f"消息数: {mc}（上轮 {prev}）| 工具: {r['tool_call_count']}\n"
|
||
f"⚠️ 疑似跑飞，需人工处理。")
|
||
xmpp(msg) # 真螺旋才 XMPP
|
||
log(f"SPIRAL-CONFIRMED: [{profile}] {sid} msgs {prev}→{mc}")
|
||
else:
|
||
if prev is not None:
|
||
log(f"SPIRAL: [{profile}] {sid} age={age_sec/60:.0f}min msgs={mc} (较上轮 {mc-prev:+d})")
|
||
else:
|
||
log(f"SPIRAL: [{profile}] {sid} age={age_sec/60:.0f}min msgs={mc}")
|
||
|
||
st["alerted"] = sorted(alerted)[-200:] # 去重窗口扩大（2026-08-19 防僵尸被挤掉）
|
||
st["prev_msgs"] = dict(list(prev_msgs.items())[-500:])
|
||
save_state(st)
|
||
log(f"── 结束: 新告警 {found} ──")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|