#!/usr/bin/env python3
"""fix_gateway_port.py — 自愈系统调用的网关/XMPP Bot修复脚本
v3: 2026-08-10 修复自愈反噬死循环（8/3起gateway每5分钟被重启→所有LLM cron停摆）
1. 重启冷却：30分钟内不重复 restart（防死循环）
2. 等待120s：覆盖LLM冷启动20-100s（原5s必误判）
3. 区分故障类型：LLM端点/网络故障不重启（重启gateway无用），只告警
"""
import subprocess, sys, time, socket, json, urllib.request, os
from datetime import datetime

# ── 消息通道统一路由(broadcast/xmpp by delivery) ──
try:
    from messenger import install_stdio_hook as _msh
    _msh()
except Exception:
    pass

GATEWAY_PORT = 8643
BOT_PORT = 5805
BOT_SCRIPT = "/home/hmo/xmpp_zhiwei_bot.py"
API_KEY = "hermes123"
SESSION_ID = "xmpp-zhiwei"
GATEWAY_URL = f"http://127.0.0.1:{GATEWAY_PORT}/v1/chat/completions"

# v3 新增：重启冷却与等待
RESTART_COOLDOWN_SEC = 1800   # 30分钟内最多重启1次
RESTART_WAIT_SEC = 120        # 重启后等待120s（冷启动20-100s）
LOCK_FILE = "/tmp/fix_gateway_port.last_restart"

# LLM端点/网络类故障关键词（这些不是gateway自身问题，重启无用）
LLM_ENDPOINT_ERR = [
    "RemoteProtocolError", "empty stream", "Stream stale", "peer closed",
    "Connection", "connection", "timeout", "Timeout", "HTTP 429", "429",
    "RemoteDisconnected", "ConnectionError",
]

def port_open(port, host="127.0.0.1"):
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.settimeout(2)
    try:
        r = s.connect_ex((host, port))
        return r == 0
    finally:
        s.close()

def check_session_health():
    """检测 gateway LLM 是否可用——扫 agent.log 最近一次真实调用结果。
    返回 (healthy: bool, error_type: str|None)
    error_type: "endpoint"(LLM端点故障,不重启) / "other"(其他,可重启) / None(健康)
    """
    try:
        sys.path.insert(0, '/home/hmo/MoFin')
        from xmpp_logger import _scan_agent_log
        r = _scan_agent_log(time.time(), "zhiwei")
        if r["status"] == "ok":
            print(f"Session {SESSION_ID} 健康 ✓ (agent.log: latency={r.get('latency')}, {r.get('age_sec')}s前)")
            return True, None
        if r["status"] == "error":
            err = str(r.get("error", ""))
            # v3: 区分LLM端点故障 vs session卡死
            is_endpoint = any(k in err for k in LLM_ENDPOINT_ERR)
            print(f"Session {SESSION_ID} 不健康: {err[:100]} → "
                  f"{'LLM端点故障(不重启,只告警)' if is_endpoint else '其他故障(可重启)'}", file=sys.stderr)
            return False, ("endpoint" if is_endpoint else "other")
        print(f"Session {SESSION_ID} 无近期调用记录（空闲正常）")
        return True, None
    except Exception as e:
        print(f"Session {SESSION_ID} 健康检查异常: {e}（按健康处理）", file=sys.stderr)
        return True, None

def check_cooldown():
    """v3: 重启冷却检查。距上次重启<30分钟则拒绝再次重启。"""
    try:
        if os.path.exists(LOCK_FILE):
            last = float(open(LOCK_FILE).read().strip())
            elapsed = time.time() - last
            if elapsed < RESTART_COOLDOWN_SEC:
                print(f"⚠️ 距上次重启仅{int(elapsed)}s (<{RESTART_COOLDOWN_SEC}s冷却)，跳过重启防止循环", file=sys.stderr)
                return False
    except Exception:
        pass
    return True

def mark_restarted():
    try:
        with open(LOCK_FILE, "w") as f:
            f.write(str(time.time()))
    except Exception:
        pass

def restart_gateway():
    """通过systemd重启gateway（v3: 冷却检查 + 等待120s）"""
    if not check_cooldown():
        return False
    print(f"Gateway 端口{GATEWAY_PORT} 异常 → 重启中...")
    subprocess.run(["sudo", "systemctl", "restart", "hermes-gateway-zhiwei.service"],
                   timeout=30, capture_output=True)
    mark_restarted()
    print(f"  等待 {RESTART_WAIT_SEC}s 让gateway冷启动...")
    time.sleep(RESTART_WAIT_SEC)
    if port_open(GATEWAY_PORT):
        print(f"Gateway 已恢复 ✓")
        return True
    else:
        print(f"Gateway 重启后{int(RESTART_WAIT_SEC)}s仍不可达（可能冷启动更慢或真故障）", file=sys.stderr)
        return False

def notify_llm_endpoint():
    """v3: LLM端点故障告警（不重启）"""
    try:
        sys.path.insert(0, '/home/hmo/.hermes/profiles/position-analyst/scripts')
        from alert_helper import notify, INFO
        notify("LLM端点故障",
               "fix_gateway_port检测到LLM端点异常（非gateway自身问题），不重启gateway。"
               "请检查opencode.ai/deepseek-v4-flash可用性。",
               INFO)
        print("LLM端点故障已告警（未重启gateway）")
    except Exception as e:
        print(f"告警失败: {e}", file=sys.stderr)

target = sys.argv[1] if len(sys.argv) > 1 else "all"

if target in ("all", "bot", "xmpp_bot"):
    if not port_open(BOT_PORT):
        print(f"XMPP bot port {BOT_PORT} CLOSED → 启动")
        subprocess.run(["pkill", "-f", "xmpp_zhiwei_bot.py"], timeout=5, capture_output=True)
        time.sleep(1)
        subprocess.Popen(["python3", BOT_SCRIPT], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
        time.sleep(5)
        if port_open(BOT_PORT):
            print(f"XMPP bot 端口{BOT_PORT} 已打开 ✓")
        else:
            print(f"XMPP bot 修复后仍不可达", file=sys.stderr)
            sys.exit(1)
    else:
        print(f"XMPP bot 端口{BOT_PORT} 正常 ✓")

if target in ("all", "gateway", "session"):
    if not port_open(GATEWAY_PORT):
        restart_gateway()
    elif target in ("all", "gateway") or target == "session":
        # 端口通了 → 进一步检查session健康
        healthy, err_type = check_session_health()
        if not healthy:
            if err_type == "endpoint":
                # v3: LLM端点故障不重启（重启无用且导致死循环），只告警
                notify_llm_endpoint()
            else:
                print(f"Session {SESSION_ID} 不健康({err_type}) → 重启gateway")
                restart_gateway()
        else:
            print(f"Gateway 端口{GATEWAY_PORT} 正常 ✓")

if target in ("all", "session"):
    if port_open(GATEWAY_PORT):
        healthy, err_type = check_session_health()
        if not healthy:
            if err_type == "endpoint":
                notify_llm_endpoint()
            else:
                print(f"Session {SESSION_ID} 不健康({err_type}) → 重启gateway")
                restart_gateway()

sys.exit(0)