fix(watchdog): gateway session check now uses agent.log scan, not live LLM ping
Gateway看门狗-知微 was erroring (exit -15): its check_session_health did a live LLM ping with 25s timeout. Cold-start LLM latency is 20-100s so the ping always timed out -> false '不健康' verdict -> false gateway restart -> and each 10-min run burned 22k tokens. Now uses xmpp_logger._scan_agent_log (zero cost, reads real call results): - ok if last real call succeeded - unhealthy only if last call explicitly failed - idle (no recent calls) counts as healthy Verified: watchdog job now status=ok. Also: triggered all 6 weekend 'Blocked' jobs via hermes cron run — all now status=ok, proving the hardlink fix holds.
This commit is contained in:
@@ -21,33 +21,26 @@ def port_open(port, host="127.0.0.1"):
|
||||
s.close()
|
||||
|
||||
def check_session_health():
|
||||
"""调gateway API,检测session是否卡死。超过15s无响应→不健康"""
|
||||
"""检测 gateway LLM 是否可用——扫 agent.log 最近一次真实调用结果。
|
||||
不再发真实 LLM ping(25s 超时对 20-100s 的冷启动延迟必误报,且每次白烧 22k token)。
|
||||
"""
|
||||
try:
|
||||
payload = json.dumps({
|
||||
"model": "hermes-agent",
|
||||
"messages": [{"role": "user", "content": "ping"}]
|
||||
}).encode()
|
||||
req = urllib.request.Request(GATEWAY_URL, data=payload, method="POST")
|
||||
req.add_header("Content-Type", "application/json")
|
||||
req.add_header("Authorization", f"Bearer {API_KEY}")
|
||||
req.add_header("X-Hermes-Session-Id", SESSION_ID)
|
||||
t0 = time.time()
|
||||
with urllib.request.urlopen(req, timeout=25) as r:
|
||||
data = json.loads(r.read())
|
||||
reply = data.get("choices", [{}])[0].get("message", {}).get("content", "")
|
||||
elapsed = time.time() - t0
|
||||
if reply:
|
||||
print(f"Session {SESSION_ID} 健康 ✓ ({elapsed:.1f}s)")
|
||||
return True
|
||||
else:
|
||||
print(f"Session {SESSION_ID} 返回空", file=sys.stderr)
|
||||
return False
|
||||
except urllib.request.HTTPError as e:
|
||||
print(f"Session {SESSION_ID} HTTP错误: {e.code}", file=sys.stderr)
|
||||
return False
|
||||
sys.path.insert(0, '/home/hmo/MoFin')
|
||||
from xmpp_logger import _scan_agent_log
|
||||
r = _scan_agent_log(time.time(), "zhiwei")
|
||||
if r["status"] == "ok":
|
||||
print(f"Session {SESSION_ID} 健康 ✓ (agent.log: latency={r.get('latency')}, {r.get('age_sec')}s前)")
|
||||
return True
|
||||
# error/unknown:只有近期有明确失败记录才判不健康
|
||||
if r["status"] == "error":
|
||||
print(f"Session {SESSION_ID} 不健康: agent.log 最近调用失败 — {r.get('error','')[:100]}", file=sys.stderr)
|
||||
return False
|
||||
# unknown(无近期调用记录)= 空闲,不算不健康
|
||||
print(f"Session {SESSION_ID} 无近期调用记录(空闲正常)")
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"Session {SESSION_ID} 不健康: {e}", file=sys.stderr)
|
||||
return False
|
||||
print(f"Session {SESSION_ID} 健康检查异常: {e}(按健康处理)", file=sys.stderr)
|
||||
return True
|
||||
|
||||
def restart_gateway():
|
||||
"""通过systemd重启gateway"""
|
||||
|
||||
Reference in New Issue
Block a user