fix: 螺旋监控僵尸会话反复告警——①僵尸(msgs停滞)跳过②去重窗口50→200③真螺旋(msgs快速增)才XMPP,普通嫌疑只日志(老莫:3分钟前还在收螺旋嫌疑)
This commit is contained in:
@@ -14,6 +14,9 @@ import json, os, sqlite3, glob
|
||||
from datetime import datetime
|
||||
|
||||
STATE_FILE = "/home/hmo/MoFin/gateway/logs/spiral_watchdog_state.json"
|
||||
# 2026-08-19 僵尸会话检测:记录每 session 上次 msg 数,停滞不增长 = 僵尸(非螺旋)
|
||||
ZOMBIE_GROWTH = 0.001 # 若本轮 msgs 相对上次增长 < 0.1%(停滞)→ 僵尸跳过
|
||||
MIN_GROW_TO_ALERT = 20 # 真螺旋:本次 msgs 比上次增长 >= 20 条才 XMPP 告警(飞速跑)
|
||||
LOG = "/home/hmo/MoFin/gateway/logs/spiral_watchdog.log"
|
||||
MIN_AGE_SEC = 15 * 60
|
||||
MIN_MESSAGES = 80
|
||||
@@ -67,6 +70,7 @@ def _to_ms(ts):
|
||||
def main():
|
||||
st = load_state()
|
||||
alerted = set(st.get("alerted", []))
|
||||
prev_msgs = st.get("prev_msgs", {}) # {sid: last_msg_count}
|
||||
now_ms = datetime.now().timestamp() * 1000
|
||||
found = 0
|
||||
|
||||
@@ -98,22 +102,38 @@ def main():
|
||||
continue
|
||||
if (r["message_count"] or 0) < MIN_MESSAGES:
|
||||
continue
|
||||
sid = r["id"]
|
||||
sid = str(r["id"])
|
||||
mc = r["message_count"] or 0
|
||||
# 僵尸会话检测:同 session msgs 停滞不增长 → 不是螺旋(7/31 旧会话 bug 教训)
|
||||
prev = prev_msgs.get(sid)
|
||||
if prev is not None and abs(mc - prev) <= 1:
|
||||
# 停滞 → 僵尸,仅记录不再告警
|
||||
log(f"ZOMBIE: [{profile}] {sid} msgs={mc}(停滞,非螺旋,跳过)")
|
||||
continue
|
||||
if sid in alerted:
|
||||
continue
|
||||
found += 1
|
||||
alerted.add(sid)
|
||||
msg = (f"agent 螺旋嫌疑\n"
|
||||
f"profile: {profile}\n"
|
||||
f"session: {sid}\n"
|
||||
f"已运行: {age_sec/60:.0f} 分钟\n"
|
||||
f"消息数: {r['message_count']} | 工具调用: {r['tool_call_count']} | "
|
||||
f"输入token: {r['input_tokens']}\n"
|
||||
f"特征类似 603288 事件。正常长任务可忽略;否则需人工检查。")
|
||||
log(f"SPIRAL: [{profile}] {sid} age={age_sec/60:.0f}min msgs={r['message_count']} tools={r['tool_call_count']}")
|
||||
xmpp(msg)
|
||||
prev_msgs[sid] = mc
|
||||
# 真螺旋判定:首次发现(prev 为 None)→ 只写日志
|
||||
# 之后 msgs 快速增长(>=MIN_GROW_TO_ALERT)→ XMPP 告警(跑飞了)
|
||||
if prev is not None and mc - prev >= MIN_GROW_TO_ALERT:
|
||||
msg = (f"🚨 agent 螺旋确认(msgs 快速增长)\n"
|
||||
f"profile: {profile}\n"
|
||||
f"session: {sid}\n"
|
||||
f"已运行: {age_sec/60:.0f} 分钟\n"
|
||||
f"消息数: {mc}(上轮 {prev})| 工具: {r['tool_call_count']}\n"
|
||||
f"⚠️ 疑似跑飞,需人工处理。")
|
||||
xmpp(msg) # 真螺旋才 XMPP
|
||||
log(f"SPIRAL-CONFIRMED: [{profile}] {sid} msgs {prev}→{mc}")
|
||||
else:
|
||||
if prev is not None:
|
||||
log(f"SPIRAL: [{profile}] {sid} age={age_sec/60:.0f}min msgs={mc} (较上轮 {mc-prev:+d})")
|
||||
else:
|
||||
log(f"SPIRAL: [{profile}] {sid} age={age_sec/60:.0f}min msgs={mc}")
|
||||
|
||||
st["alerted"] = sorted(alerted)
|
||||
st["alerted"] = sorted(alerted)[-200:] # 去重窗口扩大(2026-08-19 防僵尸被挤掉)
|
||||
st["prev_msgs"] = dict(list(prev_msgs.items())[-500:])
|
||||
save_state(st)
|
||||
log(f"── 结束: 新告警 {found} ──")
|
||||
return 0
|
||||
|
||||
Reference in New Issue
Block a user