fix: 螺旋监控僵尸会话反复告警——①僵尸(msgs停滞)跳过②去重窗口50→200③真螺旋(msgs快速增)才XMPP,普通嫌疑只日志(老莫:3分钟前还在收螺旋嫌疑)

This commit is contained in:
xxm
2026-08-19 11:59:28 +08:00
parent 6989a8f8b7
commit 2aa5e41616
+31 -11
View File
@@ -14,6 +14,9 @@ import json, os, sqlite3, glob
from datetime import datetime
STATE_FILE = "/home/hmo/MoFin/gateway/logs/spiral_watchdog_state.json"
# 2026-08-19 僵尸会话检测:记录每 session 上次 msg 数,停滞不增长 = 僵尸(非螺旋)
ZOMBIE_GROWTH = 0.001 # 若本轮 msgs 相对上次增长 < 0.1%(停滞)→ 僵尸跳过
MIN_GROW_TO_ALERT = 20 # 真螺旋:本次 msgs 比上次增长 >= 20 条才 XMPP 告警(飞速跑)
LOG = "/home/hmo/MoFin/gateway/logs/spiral_watchdog.log"
MIN_AGE_SEC = 15 * 60
MIN_MESSAGES = 80
@@ -67,6 +70,7 @@ def _to_ms(ts):
def main():
st = load_state()
alerted = set(st.get("alerted", []))
prev_msgs = st.get("prev_msgs", {}) # {sid: last_msg_count}
now_ms = datetime.now().timestamp() * 1000
found = 0
@@ -98,22 +102,38 @@ def main():
continue
if (r["message_count"] or 0) < MIN_MESSAGES:
continue
sid = r["id"]
sid = str(r["id"])
mc = r["message_count"] or 0
# 僵尸会话检测:同 session msgs 停滞不增长 → 不是螺旋(7/31 旧会话 bug 教训)
prev = prev_msgs.get(sid)
if prev is not None and abs(mc - prev) <= 1:
# 停滞 → 僵尸,仅记录不再告警
log(f"ZOMBIE: [{profile}] {sid} msgs={mc}(停滞,非螺旋,跳过)")
continue
if sid in alerted:
continue
found += 1
alerted.add(sid)
msg = (f"agent 螺旋嫌疑\n"
f"profile: {profile}\n"
f"session: {sid}\n"
f"已运行: {age_sec/60:.0f} 分钟\n"
f"消息数: {r['message_count']} | 工具调用: {r['tool_call_count']} | "
f"输入token: {r['input_tokens']}\n"
f"特征类似 603288 事件。正常长任务可忽略;否则需人工检查。")
log(f"SPIRAL: [{profile}] {sid} age={age_sec/60:.0f}min msgs={r['message_count']} tools={r['tool_call_count']}")
xmpp(msg)
prev_msgs[sid] = mc
# 真螺旋判定:首次发现(prev 为 None)→ 只写日志
# 之后 msgs 快速增长(>=MIN_GROW_TO_ALERT)→ XMPP 告警(跑飞了)
if prev is not None and mc - prev >= MIN_GROW_TO_ALERT:
msg = (f"🚨 agent 螺旋确认(msgs 快速增长)\n"
f"profile: {profile}\n"
f"session: {sid}\n"
f"已运行: {age_sec/60:.0f} 分钟\n"
f"消息数: {mc}(上轮 {prev}| 工具: {r['tool_call_count']}\n"
f"⚠️ 疑似跑飞,需人工处理。")
xmpp(msg) # 真螺旋才 XMPP
log(f"SPIRAL-CONFIRMED: [{profile}] {sid} msgs {prev}{mc}")
else:
if prev is not None:
log(f"SPIRAL: [{profile}] {sid} age={age_sec/60:.0f}min msgs={mc} (较上轮 {mc-prev:+d})")
else:
log(f"SPIRAL: [{profile}] {sid} age={age_sec/60:.0f}min msgs={mc}")
st["alerted"] = sorted(alerted)
st["alerted"] = sorted(alerted)[-200:] # 去重窗口扩大(2026-08-19 防僵尸被挤掉)
st["prev_msgs"] = dict(list(prev_msgs.items())[-500:])
save_state(st)
log(f"── 结束: 新告警 {found} ──")
return 0