fix: fix_gateway_port v3 修复自愈反噬死循环（冷却30min+等待120s+LLM端点故障不重启只告警）

deploy/profile-scripts/fix_gateway_port.py

@@ -1,8 +1,12 @@
 #!/usr/bin/env python3
 """fix_gateway_port.py — 自愈系统调用的网关/XMPP Bot修复脚本
-v2: 新增 session 健康检查，检测卡死的 session 自动重启
+v3: 2026-08-10 修复自愈反噬死循环（8/3起gateway每5分钟被重启→所有LLM cron停摆）
+1. 重启冷却：30分钟内不重复 restart（防死循环）
+2. 等待120s：覆盖LLM冷启动20-100s（原5s必误判）
+3. 区分故障类型：LLM端点/网络故障不重启（重启gateway无用），只告警
 """
-import subprocess, sys, time, socket, json, urllib.request
+import subprocess, sys, time, socket, json, urllib.request, os
+from datetime import datetime
 
 GATEWAY_PORT = 8643
 BOT_PORT = 5805
@@ -11,6 +15,18 @@ API_KEY = "hermes123"
 SESSION_ID = "xmpp-zhiwei"
 GATEWAY_URL = f"http://127.0.0.1:{GATEWAY_PORT}/v1/chat/completions"
 
+# v3 新增：重启冷却与等待
+RESTART_COOLDOWN_SEC = 1800   # 30分钟内最多重启1次
+RESTART_WAIT_SEC = 120        # 重启后等待120s（冷启动20-100s）
+LOCK_FILE = "/tmp/fix_gateway_port.last_restart"
+
+# LLM端点/网络类故障关键词（这些不是gateway自身问题，重启无用）
+LLM_ENDPOINT_ERR = [
+    "RemoteProtocolError", "empty stream", "Stream stale", "peer closed",
+    "Connection", "connection", "timeout", "Timeout", "HTTP 429", "429",
+    "RemoteDisconnected", "ConnectionError",
+]
+
 def port_open(port, host="127.0.0.1"):
     s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
     s.settimeout(2)
@@ -22,7 +38,8 @@ def port_open(port, host="127.0.0.1"):
 
 def check_session_health():
     """检测 gateway LLM 是否可用——扫 agent.log 最近一次真实调用结果。
-    不再发真实 LLM ping（25s 超时对 20-100s 的冷启动延迟必误报，且每次白烧 22k token）。
+    返回 (healthy: bool, error_type: str|None)
+    error_type: "endpoint"(LLM端点故障,不重启) / "other"(其他,可重启) / None(健康)
     """
     try:
         sys.path.insert(0, '/home/hmo/MoFin')
@@ -30,31 +47,70 @@ def check_session_health():
         r = _scan_agent_log(time.time(), "zhiwei")
         if r["status"] == "ok":
             print(f"Session {SESSION_ID} 健康 ✓ (agent.log: latency={r.get('latency')}, {r.get('age_sec')}s前)")
-            return True
-        # error/unknown：只有近期有明确失败记录才判不健康
+            return True, None
         if r["status"] == "error":
-            print(f"Session {SESSION_ID} 不健康: agent.log 最近调用失败 — {r.get('error','')[:100]}", file=sys.stderr)
-            return False
-        # unknown（无近期调用记录）= 空闲，不算不健康
+            err = str(r.get("error", ""))
+            # v3: 区分LLM端点故障 vs session卡死
+            is_endpoint = any(k in err for k in LLM_ENDPOINT_ERR)
+            print(f"Session {SESSION_ID} 不健康: {err[:100]} → "
+                  f"{'LLM端点故障(不重启,只告警)' if is_endpoint else '其他故障(可重启)'}", file=sys.stderr)
+            return False, ("endpoint" if is_endpoint else "other")
         print(f"Session {SESSION_ID} 无近期调用记录（空闲正常）")
-        return True
+        return True, None
     except Exception as e:
         print(f"Session {SESSION_ID} 健康检查异常: {e}（按健康处理）", file=sys.stderr)
-        return True
+        return True, None
+
+def check_cooldown():
+    """v3: 重启冷却检查。距上次重启<30分钟则拒绝再次重启。"""
+    try:
+        if os.path.exists(LOCK_FILE):
+            last = float(open(LOCK_FILE).read().strip())
+            elapsed = time.time() - last
+            if elapsed < RESTART_COOLDOWN_SEC:
+                print(f"⚠️ 距上次重启仅{int(elapsed)}s (<{RESTART_COOLDOWN_SEC}s冷却)，跳过重启防止循环", file=sys.stderr)
+                return False
+    except Exception:
+        pass
+    return True
+
+def mark_restarted():
+    try:
+        with open(LOCK_FILE, "w") as f:
+            f.write(str(time.time()))
+    except Exception:
+        pass
 
 def restart_gateway():
-    """通过systemd重启gateway"""
+    """通过systemd重启gateway（v3: 冷却检查 + 等待120s）"""
+    if not check_cooldown():
+        return False
     print(f"Gateway 端口{GATEWAY_PORT} 异常 → 重启中...")
     subprocess.run(["sudo", "systemctl", "restart", "hermes-gateway-zhiwei.service"],
                    timeout=30, capture_output=True)
-    time.sleep(5)
+    mark_restarted()
+    print(f"  等待 {RESTART_WAIT_SEC}s 让gateway冷启动...")
+    time.sleep(RESTART_WAIT_SEC)
     if port_open(GATEWAY_PORT):
         print(f"Gateway 已恢复 ✓")
         return True
     else:
-        print(f"Gateway 重启后仍不可达", file=sys.stderr)
+        print(f"Gateway 重启后{int(RESTART_WAIT_SEC)}s仍不可达（可能冷启动更慢或真故障）", file=sys.stderr)
         return False
 
+def notify_llm_endpoint():
+    """v3: LLM端点故障告警（不重启）"""
+    try:
+        sys.path.insert(0, '/home/hmo/.hermes/profiles/position-analyst/scripts')
+        from alert_helper import notify, INFO
+        notify("LLM端点故障",
+               "fix_gateway_port检测到LLM端点异常（非gateway自身问题），不重启gateway。"
+               "请检查opencode.ai/deepseek-v4-flash可用性。",
+               INFO)
+        print("LLM端点故障已告警（未重启gateway）")
+    except Exception as e:
+        print(f"告警失败: {e}", file=sys.stderr)
+
 target = sys.argv[1] if len(sys.argv) > 1 else "all"
 
 if target in ("all", "bot", "xmpp_bot"):
@@ -77,17 +133,25 @@ if target in ("all", "gateway", "session"):
         restart_gateway()
     elif target in ("all", "gateway") or target == "session":
         # 端口通了 → 进一步检查session健康
-        if not check_session_health():
-            print(f"Session {SESSION_ID} 不健康 → 重启gateway")
-            restart_gateway()
+        healthy, err_type = check_session_health()
+        if not healthy:
+            if err_type == "endpoint":
+                # v3: LLM端点故障不重启（重启无用且导致死循环），只告警
+                notify_llm_endpoint()
+            else:
+                print(f"Session {SESSION_ID} 不健康({err_type}) → 重启gateway")
+                restart_gateway()
         else:
             print(f"Gateway 端口{GATEWAY_PORT} 正常 ✓")
 
 if target in ("all", "session"):
-    # 仅session检查
     if port_open(GATEWAY_PORT):
-        if not check_session_health():
-            print(f"Session {SESSION_ID} 不健康 → 重启gateway")
-            restart_gateway()
+        healthy, err_type = check_session_health()
+        if not healthy:
+            if err_type == "endpoint":
+                notify_llm_endpoint()
+            else:
+                print(f"Session {SESSION_ID} 不健康({err_type}) → 重启gateway")
+                restart_gateway()
 
 sys.exit(0)