merge: watchdog agent.log check
This commit is contained in:
知微
@@ -1,93 +1,93 @@
|
||||
#!/usr/bin/env python3
|
||||
"""fix_gateway_port.py — 自愈系统调用的网关/XMPP Bot修复脚本
|
||||
v2: 新增 session 健康检查，检测卡死的 session 自动重启
|
||||
"""
|
||||
import subprocess, sys, time, socket, json, urllib.request
|
||||
|
||||
GATEWAY_PORT = 8643
|
||||
BOT_PORT = 5805
|
||||
BOT_SCRIPT = "/home/hmo/xmpp_zhiwei_bot.py"
|
||||
API_KEY = "hermes123"
|
||||
SESSION_ID = "xmpp-zhiwei"
|
||||
GATEWAY_URL = f"http://127.0.0.1:{GATEWAY_PORT}/v1/chat/completions"
|
||||
|
||||
def port_open(port, host="127.0.0.1"):
|
||||
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
s.settimeout(2)
|
||||
try:
|
||||
r = s.connect_ex((host, port))
|
||||
return r == 0
|
||||
finally:
|
||||
s.close()
|
||||
|
||||
def check_session_health():
|
||||
"""检测 gateway LLM 是否可用——扫 agent.log 最近一次真实调用结果。
|
||||
不再发真实 LLM ping（25s 超时对 20-100s 的冷启动延迟必误报，且每次白烧 22k token）。
|
||||
"""
|
||||
try:
|
||||
sys.path.insert(0, '/home/hmo/MoFin')
|
||||
from xmpp_logger import _scan_agent_log
|
||||
r = _scan_agent_log(time.time(), "zhiwei")
|
||||
if r["status"] == "ok":
|
||||
print(f"Session {SESSION_ID} 健康 ✓ (agent.log: latency={r.get('latency')}, {r.get('age_sec')}s前)")
|
||||
return True
|
||||
# error/unknown：只有近期有明确失败记录才判不健康
|
||||
if r["status"] == "error":
|
||||
print(f"Session {SESSION_ID} 不健康: agent.log 最近调用失败 — {r.get('error','')[:100]}", file=sys.stderr)
|
||||
return False
|
||||
# unknown（无近期调用记录）= 空闲，不算不健康
|
||||
print(f"Session {SESSION_ID} 无近期调用记录（空闲正常）")
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"Session {SESSION_ID} 健康检查异常: {e}（按健康处理）", file=sys.stderr)
|
||||
return True
|
||||
|
||||
def restart_gateway():
|
||||
"""通过systemd重启gateway"""
|
||||
print(f"Gateway 端口{GATEWAY_PORT} 异常 → 重启中...")
|
||||
subprocess.run(["sudo", "systemctl", "restart", "hermes-gateway-zhiwei.service"],
|
||||
timeout=30, capture_output=True)
|
||||
time.sleep(5)
|
||||
if port_open(GATEWAY_PORT):
|
||||
print(f"Gateway 已恢复 ✓")
|
||||
return True
|
||||
else:
|
||||
print(f"Gateway 重启后仍不可达", file=sys.stderr)
|
||||
return False
|
||||
|
||||
target = sys.argv[1] if len(sys.argv) > 1 else "all"
|
||||
|
||||
if target in ("all", "bot", "xmpp_bot"):
|
||||
if not port_open(BOT_PORT):
|
||||
print(f"XMPP bot port {BOT_PORT} CLOSED → 启动")
|
||||
subprocess.run(["pkill", "-f", "xmpp_zhiwei_bot.py"], timeout=5, capture_output=True)
|
||||
time.sleep(1)
|
||||
subprocess.Popen(["python3", BOT_SCRIPT], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
time.sleep(5)
|
||||
if port_open(BOT_PORT):
|
||||
print(f"XMPP bot 端口{BOT_PORT} 已打开 ✓")
|
||||
else:
|
||||
print(f"XMPP bot 修复后仍不可达", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
else:
|
||||
print(f"XMPP bot 端口{BOT_PORT} 正常 ✓")
|
||||
|
||||
if target in ("all", "gateway", "session"):
|
||||
if not port_open(GATEWAY_PORT):
|
||||
restart_gateway()
|
||||
elif target in ("all", "gateway") or target == "session":
|
||||
# 端口通了 → 进一步检查session健康
|
||||
if not check_session_health():
|
||||
print(f"Session {SESSION_ID} 不健康 → 重启gateway")
|
||||
restart_gateway()
|
||||
else:
|
||||
print(f"Gateway 端口{GATEWAY_PORT} 正常 ✓")
|
||||
|
||||
if target in ("all", "session"):
|
||||
# 仅session检查
|
||||
if port_open(GATEWAY_PORT):
|
||||
if not check_session_health():
|
||||
print(f"Session {SESSION_ID} 不健康 → 重启gateway")
|
||||
restart_gateway()
|
||||
|
||||
sys.exit(0)
|
||||
#!/usr/bin/env python3
|
||||
"""fix_gateway_port.py — 自愈系统调用的网关/XMPP Bot修复脚本
|
||||
v2: 新增 session 健康检查，检测卡死的 session 自动重启
|
||||
"""
|
||||
import subprocess, sys, time, socket, json, urllib.request
|
||||
|
||||
GATEWAY_PORT = 8643
|
||||
BOT_PORT = 5805
|
||||
BOT_SCRIPT = "/home/hmo/xmpp_zhiwei_bot.py"
|
||||
API_KEY = "hermes123"
|
||||
SESSION_ID = "xmpp-zhiwei"
|
||||
GATEWAY_URL = f"http://127.0.0.1:{GATEWAY_PORT}/v1/chat/completions"
|
||||
|
||||
def port_open(port, host="127.0.0.1"):
|
||||
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
s.settimeout(2)
|
||||
try:
|
||||
r = s.connect_ex((host, port))
|
||||
return r == 0
|
||||
finally:
|
||||
s.close()
|
||||
|
||||
def check_session_health():
|
||||
"""检测 gateway LLM 是否可用——扫 agent.log 最近一次真实调用结果。
|
||||
不再发真实 LLM ping（25s 超时对 20-100s 的冷启动延迟必误报，且每次白烧 22k token）。
|
||||
"""
|
||||
try:
|
||||
sys.path.insert(0, '/home/hmo/MoFin')
|
||||
from xmpp_logger import _scan_agent_log
|
||||
r = _scan_agent_log(time.time(), "zhiwei")
|
||||
if r["status"] == "ok":
|
||||
print(f"Session {SESSION_ID} 健康 ✓ (agent.log: latency={r.get('latency')}, {r.get('age_sec')}s前)")
|
||||
return True
|
||||
# error/unknown：只有近期有明确失败记录才判不健康
|
||||
if r["status"] == "error":
|
||||
print(f"Session {SESSION_ID} 不健康: agent.log 最近调用失败 — {r.get('error','')[:100]}", file=sys.stderr)
|
||||
return False
|
||||
# unknown（无近期调用记录）= 空闲，不算不健康
|
||||
print(f"Session {SESSION_ID} 无近期调用记录（空闲正常）")
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"Session {SESSION_ID} 健康检查异常: {e}（按健康处理）", file=sys.stderr)
|
||||
return True
|
||||
|
||||
def restart_gateway():
|
||||
"""通过systemd重启gateway"""
|
||||
print(f"Gateway 端口{GATEWAY_PORT} 异常 → 重启中...")
|
||||
subprocess.run(["sudo", "systemctl", "restart", "hermes-gateway-zhiwei.service"],
|
||||
timeout=30, capture_output=True)
|
||||
time.sleep(5)
|
||||
if port_open(GATEWAY_PORT):
|
||||
print(f"Gateway 已恢复 ✓")
|
||||
return True
|
||||
else:
|
||||
print(f"Gateway 重启后仍不可达", file=sys.stderr)
|
||||
return False
|
||||
|
||||
target = sys.argv[1] if len(sys.argv) > 1 else "all"
|
||||
|
||||
if target in ("all", "bot", "xmpp_bot"):
|
||||
if not port_open(BOT_PORT):
|
||||
print(f"XMPP bot port {BOT_PORT} CLOSED → 启动")
|
||||
subprocess.run(["pkill", "-f", "xmpp_zhiwei_bot.py"], timeout=5, capture_output=True)
|
||||
time.sleep(1)
|
||||
subprocess.Popen(["python3", BOT_SCRIPT], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
time.sleep(5)
|
||||
if port_open(BOT_PORT):
|
||||
print(f"XMPP bot 端口{BOT_PORT} 已打开 ✓")
|
||||
else:
|
||||
print(f"XMPP bot 修复后仍不可达", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
else:
|
||||
print(f"XMPP bot 端口{BOT_PORT} 正常 ✓")
|
||||
|
||||
if target in ("all", "gateway", "session"):
|
||||
if not port_open(GATEWAY_PORT):
|
||||
restart_gateway()
|
||||
elif target in ("all", "gateway") or target == "session":
|
||||
# 端口通了 → 进一步检查session健康
|
||||
if not check_session_health():
|
||||
print(f"Session {SESSION_ID} 不健康 → 重启gateway")
|
||||
restart_gateway()
|
||||
else:
|
||||
print(f"Gateway 端口{GATEWAY_PORT} 正常 ✓")
|
||||
|
||||
if target in ("all", "session"):
|
||||
# 仅session检查
|
||||
if port_open(GATEWAY_PORT):
|
||||
if not check_session_health():
|
||||
print(f"Session {SESSION_ID} 不健康 → 重启gateway")
|
||||
restart_gateway()
|
||||
|
||||
sys.exit(0)
|
||||
|
||||
Reference in New Issue
Block a user