165 lines
6.5 KiB
Python
165 lines
6.5 KiB
Python
#!/usr/bin/env python3
|
||
"""fix_gateway_port.py — 自愈系统调用的网关/XMPP Bot修复脚本
|
||
v3: 2026-08-10 修复自愈反噬死循环（8/3起gateway每5分钟被重启→所有LLM cron停摆）
|
||
1. 重启冷却：30分钟内不重复 restart（防死循环）
|
||
2. 等待120s：覆盖LLM冷启动20-100s（原5s必误判）
|
||
3. 区分故障类型：LLM端点/网络故障不重启（重启gateway无用），只告警
|
||
"""
|
||
import subprocess, sys, time, socket, json, urllib.request, os
|
||
from datetime import datetime
|
||
|
||
# ── 消息通道统一路由(broadcast/xmpp by delivery) ──
|
||
try:
|
||
from messenger import install_stdio_hook as _msh
|
||
_msh()
|
||
except Exception:
|
||
pass
|
||
|
||
GATEWAY_PORT = 8643
|
||
BOT_PORT = 5805
|
||
BOT_SCRIPT = "/home/hmo/xmpp_zhiwei_bot.py"
|
||
API_KEY = "hermes123"
|
||
SESSION_ID = "xmpp-zhiwei"
|
||
GATEWAY_URL = f"http://127.0.0.1:{GATEWAY_PORT}/v1/chat/completions"
|
||
|
||
# v3 新增：重启冷却与等待
|
||
RESTART_COOLDOWN_SEC = 1800 # 30分钟内最多重启1次
|
||
RESTART_WAIT_SEC = 120 # 重启后等待120s（冷启动20-100s）
|
||
LOCK_FILE = "/tmp/fix_gateway_port.last_restart"
|
||
|
||
# LLM端点/网络类故障关键词（这些不是gateway自身问题，重启无用）
|
||
LLM_ENDPOINT_ERR = [
|
||
"RemoteProtocolError", "empty stream", "Stream stale", "peer closed",
|
||
"Connection", "connection", "timeout", "Timeout", "HTTP 429", "429",
|
||
"RemoteDisconnected", "ConnectionError",
|
||
]
|
||
|
||
def port_open(port, host="127.0.0.1"):
|
||
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||
s.settimeout(2)
|
||
try:
|
||
r = s.connect_ex((host, port))
|
||
return r == 0
|
||
finally:
|
||
s.close()
|
||
|
||
def check_session_health():
|
||
"""检测 gateway LLM 是否可用——扫 agent.log 最近一次真实调用结果。
|
||
返回 (healthy: bool, error_type: str|None)
|
||
error_type: "endpoint"(LLM端点故障,不重启) / "other"(其他,可重启) / None(健康)
|
||
"""
|
||
try:
|
||
sys.path.insert(0, '/home/hmo/MoFin')
|
||
from xmpp_logger import _scan_agent_log
|
||
r = _scan_agent_log(time.time(), "zhiwei")
|
||
if r["status"] == "ok":
|
||
print(f"Session {SESSION_ID} 健康 ✓ (agent.log: latency={r.get('latency')}, {r.get('age_sec')}s前)")
|
||
return True, None
|
||
if r["status"] == "error":
|
||
err = str(r.get("error", ""))
|
||
# v3: 区分LLM端点故障 vs session卡死
|
||
is_endpoint = any(k in err for k in LLM_ENDPOINT_ERR)
|
||
print(f"Session {SESSION_ID} 不健康: {err[:100]} → "
|
||
f"{'LLM端点故障(不重启,只告警)' if is_endpoint else '其他故障(可重启)'}", file=sys.stderr)
|
||
return False, ("endpoint" if is_endpoint else "other")
|
||
print(f"Session {SESSION_ID} 无近期调用记录（空闲正常）")
|
||
return True, None
|
||
except Exception as e:
|
||
print(f"Session {SESSION_ID} 健康检查异常: {e}（按健康处理）", file=sys.stderr)
|
||
return True, None
|
||
|
||
def check_cooldown():
|
||
"""v3: 重启冷却检查。距上次重启<30分钟则拒绝再次重启。"""
|
||
try:
|
||
if os.path.exists(LOCK_FILE):
|
||
last = float(open(LOCK_FILE).read().strip())
|
||
elapsed = time.time() - last
|
||
if elapsed < RESTART_COOLDOWN_SEC:
|
||
print(f"⚠️ 距上次重启仅{int(elapsed)}s (<{RESTART_COOLDOWN_SEC}s冷却)，跳过重启防止循环", file=sys.stderr)
|
||
return False
|
||
except Exception:
|
||
pass
|
||
return True
|
||
|
||
def mark_restarted():
|
||
try:
|
||
with open(LOCK_FILE, "w") as f:
|
||
f.write(str(time.time()))
|
||
except Exception:
|
||
pass
|
||
|
||
def restart_gateway():
|
||
"""通过systemd重启gateway（v3: 冷却检查 + 等待120s）"""
|
||
if not check_cooldown():
|
||
return False
|
||
print(f"Gateway 端口{GATEWAY_PORT} 异常 → 重启中...")
|
||
subprocess.run(["sudo", "systemctl", "restart", "hermes-gateway-zhiwei.service"],
|
||
timeout=30, capture_output=True)
|
||
mark_restarted()
|
||
print(f" 等待 {RESTART_WAIT_SEC}s 让gateway冷启动...")
|
||
time.sleep(RESTART_WAIT_SEC)
|
||
if port_open(GATEWAY_PORT):
|
||
print(f"Gateway 已恢复 ✓")
|
||
return True
|
||
else:
|
||
print(f"Gateway 重启后{int(RESTART_WAIT_SEC)}s仍不可达（可能冷启动更慢或真故障）", file=sys.stderr)
|
||
return False
|
||
|
||
def notify_llm_endpoint():
|
||
"""v3: LLM端点故障告警（不重启）"""
|
||
try:
|
||
sys.path.insert(0, '/home/hmo/.hermes/profiles/position-analyst/scripts')
|
||
from alert_helper import notify, INFO
|
||
notify("LLM端点故障",
|
||
"fix_gateway_port检测到LLM端点异常（非gateway自身问题），不重启gateway。"
|
||
"请检查opencode.ai/deepseek-v4-flash可用性。",
|
||
INFO)
|
||
print("LLM端点故障已告警（未重启gateway）")
|
||
except Exception as e:
|
||
print(f"告警失败: {e}", file=sys.stderr)
|
||
|
||
target = sys.argv[1] if len(sys.argv) > 1 else "all"
|
||
|
||
if target in ("all", "bot", "xmpp_bot"):
|
||
if not port_open(BOT_PORT):
|
||
print(f"XMPP bot port {BOT_PORT} CLOSED → 启动")
|
||
subprocess.run(["pkill", "-f", "xmpp_zhiwei_bot.py"], timeout=5, capture_output=True)
|
||
time.sleep(1)
|
||
subprocess.Popen(["python3", BOT_SCRIPT], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||
time.sleep(5)
|
||
if port_open(BOT_PORT):
|
||
print(f"XMPP bot 端口{BOT_PORT} 已打开 ✓")
|
||
else:
|
||
print(f"XMPP bot 修复后仍不可达", file=sys.stderr)
|
||
sys.exit(1)
|
||
else:
|
||
print(f"XMPP bot 端口{BOT_PORT} 正常 ✓")
|
||
|
||
if target in ("all", "gateway", "session"):
|
||
if not port_open(GATEWAY_PORT):
|
||
restart_gateway()
|
||
elif target in ("all", "gateway") or target == "session":
|
||
# 端口通了 → 进一步检查session健康
|
||
healthy, err_type = check_session_health()
|
||
if not healthy:
|
||
if err_type == "endpoint":
|
||
# v3: LLM端点故障不重启（重启无用且导致死循环），只告警
|
||
notify_llm_endpoint()
|
||
else:
|
||
print(f"Session {SESSION_ID} 不健康({err_type}) → 重启gateway")
|
||
restart_gateway()
|
||
else:
|
||
print(f"Gateway 端口{GATEWAY_PORT} 正常 ✓")
|
||
|
||
if target in ("all", "session"):
|
||
if port_open(GATEWAY_PORT):
|
||
healthy, err_type = check_session_health()
|
||
if not healthy:
|
||
if err_type == "endpoint":
|
||
notify_llm_endpoint()
|
||
else:
|
||
print(f"Session {SESSION_ID} 不健康({err_type}) → 重启gateway")
|
||
restart_gateway()
|
||
|
||
sys.exit(0)
|