diff --git a/deploy/profile-scripts/fix_gateway_port.py b/deploy/profile-scripts/fix_gateway_port.py index 283cc1b7..89e2dcd9 100644 --- a/deploy/profile-scripts/fix_gateway_port.py +++ b/deploy/profile-scripts/fix_gateway_port.py @@ -1,8 +1,12 @@ #!/usr/bin/env python3 """fix_gateway_port.py — 自愈系统调用的网关/XMPP Bot修复脚本 -v2: 新增 session 健康检查,检测卡死的 session 自动重启 +v3: 2026-08-10 修复自愈反噬死循环(8/3起gateway每5分钟被重启→所有LLM cron停摆) +1. 重启冷却:30分钟内不重复 restart(防死循环) +2. 等待120s:覆盖LLM冷启动20-100s(原5s必误判) +3. 区分故障类型:LLM端点/网络故障不重启(重启gateway无用),只告警 """ -import subprocess, sys, time, socket, json, urllib.request +import subprocess, sys, time, socket, json, urllib.request, os +from datetime import datetime GATEWAY_PORT = 8643 BOT_PORT = 5805 @@ -11,6 +15,18 @@ API_KEY = "hermes123" SESSION_ID = "xmpp-zhiwei" GATEWAY_URL = f"http://127.0.0.1:{GATEWAY_PORT}/v1/chat/completions" +# v3 新增:重启冷却与等待 +RESTART_COOLDOWN_SEC = 1800 # 30分钟内最多重启1次 +RESTART_WAIT_SEC = 120 # 重启后等待120s(冷启动20-100s) +LOCK_FILE = "/tmp/fix_gateway_port.last_restart" + +# LLM端点/网络类故障关键词(这些不是gateway自身问题,重启无用) +LLM_ENDPOINT_ERR = [ + "RemoteProtocolError", "empty stream", "Stream stale", "peer closed", + "Connection", "connection", "timeout", "Timeout", "HTTP 429", "429", + "RemoteDisconnected", "ConnectionError", +] + def port_open(port, host="127.0.0.1"): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.settimeout(2) @@ -22,7 +38,8 @@ def port_open(port, host="127.0.0.1"): def check_session_health(): """检测 gateway LLM 是否可用——扫 agent.log 最近一次真实调用结果。 - 不再发真实 LLM ping(25s 超时对 20-100s 的冷启动延迟必误报,且每次白烧 22k token)。 + 返回 (healthy: bool, error_type: str|None) + error_type: "endpoint"(LLM端点故障,不重启) / "other"(其他,可重启) / None(健康) """ try: sys.path.insert(0, '/home/hmo/MoFin') @@ -30,31 +47,70 @@ def check_session_health(): r = _scan_agent_log(time.time(), "zhiwei") if r["status"] == "ok": print(f"Session {SESSION_ID} 健康 ✓ (agent.log: latency={r.get('latency')}, {r.get('age_sec')}s前)") - return True - # error/unknown:只有近期有明确失败记录才判不健康 + return True, None if r["status"] == "error": - print(f"Session {SESSION_ID} 不健康: agent.log 最近调用失败 — {r.get('error','')[:100]}", file=sys.stderr) - return False - # unknown(无近期调用记录)= 空闲,不算不健康 + err = str(r.get("error", "")) + # v3: 区分LLM端点故障 vs session卡死 + is_endpoint = any(k in err for k in LLM_ENDPOINT_ERR) + print(f"Session {SESSION_ID} 不健康: {err[:100]} → " + f"{'LLM端点故障(不重启,只告警)' if is_endpoint else '其他故障(可重启)'}", file=sys.stderr) + return False, ("endpoint" if is_endpoint else "other") print(f"Session {SESSION_ID} 无近期调用记录(空闲正常)") - return True + return True, None except Exception as e: print(f"Session {SESSION_ID} 健康检查异常: {e}(按健康处理)", file=sys.stderr) - return True + return True, None + +def check_cooldown(): + """v3: 重启冷却检查。距上次重启<30分钟则拒绝再次重启。""" + try: + if os.path.exists(LOCK_FILE): + last = float(open(LOCK_FILE).read().strip()) + elapsed = time.time() - last + if elapsed < RESTART_COOLDOWN_SEC: + print(f"⚠️ 距上次重启仅{int(elapsed)}s (<{RESTART_COOLDOWN_SEC}s冷却),跳过重启防止循环", file=sys.stderr) + return False + except Exception: + pass + return True + +def mark_restarted(): + try: + with open(LOCK_FILE, "w") as f: + f.write(str(time.time())) + except Exception: + pass def restart_gateway(): - """通过systemd重启gateway""" + """通过systemd重启gateway(v3: 冷却检查 + 等待120s)""" + if not check_cooldown(): + return False print(f"Gateway 端口{GATEWAY_PORT} 异常 → 重启中...") subprocess.run(["sudo", "systemctl", "restart", "hermes-gateway-zhiwei.service"], timeout=30, capture_output=True) - time.sleep(5) + mark_restarted() + print(f" 等待 {RESTART_WAIT_SEC}s 让gateway冷启动...") + time.sleep(RESTART_WAIT_SEC) if port_open(GATEWAY_PORT): print(f"Gateway 已恢复 ✓") return True else: - print(f"Gateway 重启后仍不可达", file=sys.stderr) + print(f"Gateway 重启后{int(RESTART_WAIT_SEC)}s仍不可达(可能冷启动更慢或真故障)", file=sys.stderr) return False +def notify_llm_endpoint(): + """v3: LLM端点故障告警(不重启)""" + try: + sys.path.insert(0, '/home/hmo/.hermes/profiles/position-analyst/scripts') + from alert_helper import notify, INFO + notify("LLM端点故障", + "fix_gateway_port检测到LLM端点异常(非gateway自身问题),不重启gateway。" + "请检查opencode.ai/deepseek-v4-flash可用性。", + INFO) + print("LLM端点故障已告警(未重启gateway)") + except Exception as e: + print(f"告警失败: {e}", file=sys.stderr) + target = sys.argv[1] if len(sys.argv) > 1 else "all" if target in ("all", "bot", "xmpp_bot"): @@ -77,17 +133,25 @@ if target in ("all", "gateway", "session"): restart_gateway() elif target in ("all", "gateway") or target == "session": # 端口通了 → 进一步检查session健康 - if not check_session_health(): - print(f"Session {SESSION_ID} 不健康 → 重启gateway") - restart_gateway() + healthy, err_type = check_session_health() + if not healthy: + if err_type == "endpoint": + # v3: LLM端点故障不重启(重启无用且导致死循环),只告警 + notify_llm_endpoint() + else: + print(f"Session {SESSION_ID} 不健康({err_type}) → 重启gateway") + restart_gateway() else: print(f"Gateway 端口{GATEWAY_PORT} 正常 ✓") if target in ("all", "session"): - # 仅session检查 if port_open(GATEWAY_PORT): - if not check_session_health(): - print(f"Session {SESSION_ID} 不健康 → 重启gateway") - restart_gateway() + healthy, err_type = check_session_health() + if not healthy: + if err_type == "endpoint": + notify_llm_endpoint() + else: + print(f"Session {SESSION_ID} 不健康({err_type}) → 重启gateway") + restart_gateway() sys.exit(0)