127 lines
4.3 KiB
Python
127 lines
4.3 KiB
Python
#!/usr/bin/env python3
|
||
"""agent_spiral_watchdog.py — agent 会话螺旋检测（每 10 分钟 cron）
|
||
|
||
盲区①的系统性补丁（2026-07-21 603288 事件：一次重评请求在 hermes agent
|
||
运行时里螺旋 35 分钟/37 轮工具/153k token，无任何组件察觉）。
|
||
|
||
检测：各 profile state.db 中仍开放的 api_server session，
|
||
启动超 15 分钟且 message_count > 80（正常调用 <20 条）→ 螺旋嫌疑。
|
||
|
||
v1 动作：XMPP 告警 + 日志（不自动 kill——kill 需要 hermes 会话取消 API，
|
||
待确认安全机制后升级 v2）。同一 session 只告警一次（state 文件去重）。
|
||
"""
|
||
import json, os, sqlite3, glob
|
||
from datetime import datetime
|
||
|
||
STATE_FILE = "/home/hmo/MoFin/gateway/logs/spiral_watchdog_state.json"
|
||
LOG = "/home/hmo/MoFin/gateway/logs/spiral_watchdog.log"
|
||
MIN_AGE_SEC = 15 * 60
|
||
MIN_MESSAGES = 80
|
||
|
||
|
||
def log(msg):
|
||
line = f"[{datetime.now().isoformat(timespec='seconds')}] {msg}"
|
||
print(line, flush=True)
|
||
try:
|
||
os.makedirs(os.path.dirname(LOG), exist_ok=True)
|
||
with open(LOG, "a", encoding="utf-8") as f:
|
||
f.write(line + "\n")
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
def xmpp(msg):
|
||
try:
|
||
import urllib.request
|
||
req = urllib.request.Request(
|
||
"http://127.0.0.1:5805/",
|
||
data=json.dumps({"body": msg, "to": "hmo@yoin.fun", "type": "chat"}).encode(),
|
||
headers={"Content-Type": "application/json"})
|
||
urllib.request.urlopen(req, timeout=5)
|
||
except Exception as e:
|
||
log(f"XMPP发送失败: {e}")
|
||
|
||
|
||
def load_state():
|
||
try:
|
||
with open(STATE_FILE, encoding="utf-8") as f:
|
||
return json.load(f)
|
||
except Exception:
|
||
return {"alerted": []}
|
||
|
||
|
||
def save_state(st):
|
||
try:
|
||
st["alerted"] = st.get("alerted", [])[-50:]
|
||
with open(STATE_FILE, "w", encoding="utf-8") as f:
|
||
json.dump(st, f)
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
def _to_ms(ts):
|
||
"""started_at 单位自适应：>1e12 视为毫秒，否则视为秒。"""
|
||
if not ts:
|
||
return 0
|
||
return ts if ts > 1e12 else ts * 1000
|
||
|
||
|
||
def main():
|
||
st = load_state()
|
||
alerted = set(st.get("alerted", []))
|
||
now_ms = datetime.now().timestamp() * 1000
|
||
found = 0
|
||
|
||
for db_path in glob.glob("/home/hmo/.hermes/profiles/*/state.db"):
|
||
profile = db_path.split("/")[-2]
|
||
try:
|
||
conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True, timeout=10)
|
||
conn.row_factory = sqlite3.Row
|
||
tables = {r[0] for r in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")}
|
||
if "sessions" not in tables:
|
||
conn.close()
|
||
continue
|
||
rows = conn.execute("""
|
||
SELECT id, started_at, message_count, input_tokens, tool_call_count
|
||
FROM sessions
|
||
WHERE source = 'api_server'
|
||
AND (ended_at IS NULL OR ended_at = 0)
|
||
ORDER BY started_at DESC LIMIT 30
|
||
""").fetchall()
|
||
conn.close()
|
||
except Exception as e:
|
||
log(f"[{profile}] state.db 读取失败: {str(e)[:80]}")
|
||
continue
|
||
|
||
for r in rows:
|
||
sa = _to_ms(r["started_at"])
|
||
age_sec = (now_ms - sa) / 1000
|
||
if age_sec < MIN_AGE_SEC:
|
||
continue
|
||
if (r["message_count"] or 0) < MIN_MESSAGES:
|
||
continue
|
||
sid = r["id"]
|
||
if sid in alerted:
|
||
continue
|
||
found += 1
|
||
alerted.add(sid)
|
||
msg = (f"🌀 agent 螺旋嫌疑\n"
|
||
f"profile: {profile}\n"
|
||
f"session: {sid}\n"
|
||
f"已运行: {age_sec/60:.0f} 分钟\n"
|
||
f"消息数: {r['message_count']} | 工具调用: {r['tool_call_count']} | "
|
||
f"输入token: {r['input_tokens']}\n"
|
||
f"特征类似 603288 事件（gateway agent 运行时螺旋）。"
|
||
f"如是正常长任务可忽略；否则需人工检查 gateway。")
|
||
log(f"SPIRAL: [{profile}] {sid} age={age_sec/60:.0f}min msgs={r['message_count']} tools={r['tool_call_count']}")
|
||
xmpp(msg)
|
||
|
||
st["alerted"] = sorted(alerted)
|
||
save_state(st)
|
||
log(f"── 结束: 新告警 {found} ──")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|