MoFin/agents_health_check.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
agents_health_check.py — MoFin Tier1 快速健康检查
====================================================
每 5 分钟运行一次（crontab）。检查关键服务的端口/HTTP/DB 可用性。
全正常时静默（不输出）。异常时写入 TODO 文件和 JSON 报告。
同时采集 XMPP 通道健康数据到时间序列日志。

部署: crontab */5 * * * * cd /home/hmo/MoFin && python3 agents_health_check.py
"""
import json, os, sys, socket, sqlite3, urllib.request
from datetime import datetime
from pathlib import Path

# ---- Config ----
SCRIPT_DIR = Path(__file__).resolve().parent
TEMP_DIR = SCRIPT_DIR / "gateway" / "temp"
LOGS_DIR = SCRIPT_DIR / "gateway" / "logs"

# Ensure dirs
TEMP_DIR.mkdir(parents=True, exist_ok=True)
LOGS_DIR.mkdir(parents=True, exist_ok=True)

REPORT_FILE = TEMP_DIR / "last_health_check.json"
TODO_FILE = TEMP_DIR / "health_todos.jsonl"
LOG_FILE = LOGS_DIR / "health_check.log"

# ---- Service List ----
SERVICES = [
    {"name": "mofin_api", "label": "MoFin API", "host": "127.0.0.1", "port": 8899, "type": "http", "check": "/api/health"},
    {"name": "zhiwei_gateway", "label": "知微 Gateway", "host": "127.0.0.1", "port": 8643, "type": "http", "check": "/v1/health"},
    {"name": "ejabberd", "label": "ejabberd XMPP", "host": "127.0.0.1", "port": 5222, "type": "tcp", "check": None},
    {"name": "mofin_db", "label": "MoFin 数据库", "host": "127.0.0.1", "port": 0, "type": "db", "check": "/home/hmo/web-dashboard/data/mofin.db"},
]

# ── XMPP 通道健康采集 ──────────────────────────────

XMPP_HEALTH_LOG = LOGS_DIR / "xmpp_health_log.jsonl"


def _collect_xmpp_health(now):
    """采集 XMPP 通道健康数据，追加到时间序列日志。"""
    try:
        from xmpp_logger import health as xmpp_health
        h = xmpp_health()
    except Exception as e:
        h = {"status": "collector_error", "error": str(e)[:200]}
    entry = {"timestamp": now.strftime("%Y-%m-%d %H:%M:%S"), **h}
    XMPP_HEALTH_LOG.parent.mkdir(parents=True, exist_ok=True)
    with open(XMPP_HEALTH_LOG, "a", encoding="utf-8") as f:
        f.write(json.dumps(entry, ensure_ascii=False) + "\n")


# ---- Checkers ----

def check_tcp(host, port, timeout=3):
    try:
        sock = socket.create_connection((host, port), timeout=timeout)
        sock.close()
        return True, "ok"
    except Exception as e:
        return False, str(e)


def check_http(host, port, path, timeout=3):
    try:
        url = f"http://{host}:{port}{path}"
        req = urllib.request.Request(url)
        resp = urllib.request.urlopen(req, timeout=timeout)
        return 200 <= resp.status < 300, f"HTTP {resp.status}"
    except Exception as e:
        return False, str(e)


def check_db(db_path):
    try:
        conn = sqlite3.connect(db_path)
        conn.execute("SELECT 1")
        conn.close()
        return True, "ok"
    except Exception as e:
        return False, str(e)


# ---- Main ----

def run():
    now = datetime.now()
    results = []
    issues = []

    for svc in SERVICES:
        if svc["type"] == "tcp":
            ok, detail = check_tcp(svc["host"], svc["port"])
        elif svc["type"] == "http":
            ok, detail = check_http(svc["host"], svc["port"], svc["check"])
        elif svc["type"] == "db":
            ok, detail = check_db(svc["check"])
        else:
            ok, detail = False, "unknown type"

        results.append({
            "name": svc["name"], "label": svc["label"],
            "type": svc["type"], "port": svc["port"],
            "health": {"ok": ok}, "detail": detail,
        })
        if not ok:
            issues.append(svc)

    # Write report
    report = {
        "services": results,
        "summary": {"ok": sum(1 for r in results if r["health"]["ok"]), "total": len(results)},
        "generated_at": now.strftime("%Y-%m-%d %H:%M:%S"),
    }
    with open(REPORT_FILE, "w", encoding="utf-8") as f:
        json.dump(report, f, ensure_ascii=False, indent=2)

# ── XMPP 通道健康采集 ──
    _collect_xmpp_health(now)

    # ── 自愈：检测 rate-limit → 切 key；Gateway 异常 → systemctl restart ──
    try:
        from xmpp_logger import auto_heal as _xmpp_auto_heal
        heal_result = _xmpp_auto_heal()
        actions = heal_result.get("actions", [])
        if actions:
            with open(LOG_FILE, "a", encoding="utf-8") as f:
                f.write(f"[{now.strftime('%Y-%m-%d %H:%M:%S')}] AUTO-HEAL actions:\n")
                for a in actions:
                    f.write(f"  {json.dumps(a, ensure_ascii=False)}\n")
            print(f"[{now.strftime('%H:%M')}] Auto-heal: {len(actions)} action(s) — {heal_result.get('status')}")
            for a in actions:
                print(f"  → {a.get('action', '?')}: success={a.get('success', a.get('switched', '?'))}")
    except Exception as e:
        print(f"[{now.strftime('%H:%M')}] Auto-heal error: {e}")
        with open(LOG_FILE, "a", encoding="utf-8") as f:
            f.write(f"[{now.strftime('%Y-%m-%d %H:%M:%S')}] AUTO-HEAL ERROR: {e}\n")

    # ── Gateway 端口宕机兜底：异步 systemctl 重启（带 cooldown 防频繁触发）──
    gateway_ok = any(r["name"] == "zhiwei_gateway" and r["health"]["ok"] for r in results)
    if not gateway_ok:
        import time as _t
        import subprocess as _sp
        cooldown_file = LOGS_DIR / "last_restart.txt"
        cooldown_sec = 180
        try:
            last_ts = float(cooldown_file.read_text().strip()) if cooldown_file.exists() else 0
        except Exception:
            last_ts = 0
        elapsed = _t.time() - last_ts
        if elapsed < cooldown_sec:
            with open(LOG_FILE, "a", encoding="utf-8") as f:
                f.write(f"[{now.strftime('%Y-%m-%d %H:%M:%S')}] Gateway DOWN but cooldown {int(elapsed)}s < {cooldown_sec}s — skip restart\n")
            print(f"[{now.strftime('%H:%M')}] Gateway DOWN but in cooldown ({int(elapsed)}s)")
        else:
            try:
                _sp.Popen(["sudo", "-n", "systemctl", "restart", "hermes-gateway-zhiwei.service"],
                          stdout=_sp.DEVNULL, stderr=_sp.DEVNULL, start_new_session=True)
                cooldown_file.write_text(str(_t.time()))
                with open(LOG_FILE, "a", encoding="utf-8") as f:
                    f.write(f"[{now.strftime('%Y-%m-%d %H:%M:%S')}] Gateway DOWN → systemctl restart triggered (async, cooldown set)\n")
                print(f"[{now.strftime('%H:%M')}] Gateway DOWN → systemctl restart triggered")
            except Exception as e:
                print(f"[{now.strftime('%H:%M')}] Gateway restart error: {e}")

    # Handle issues
    if issues:
        with open(TODO_FILE, "a", encoding="utf-8") as f:
            for svc in issues:
                entry = {
                    "service": svc["name"], "label": svc["label"],
                    "reason": next((r["detail"] for r in results if r["name"] == svc["name"]), "unknown"),
                    "timestamp": now.isoformat(),
                }
                f.write(json.dumps(entry, ensure_ascii=False) + "\n")
        with open(LOG_FILE, "a", encoding="utf-8") as f:
            f.write(f"[{now.strftime('%Y-%m-%d %H:%M:%S')}] ISSUES: {len(issues)} failed\n")
            for svc in issues:
                detail = next((r["detail"] for r in results if r["name"] == svc["name"]), "")
                f.write(f"  - {svc['label']}: {detail}\n")
        print(f"[{now.strftime('%H:%M')}] Health check: {len(issues)}/{len(SERVICES)} services failed")
        for svc in issues:
            detail = next((r["detail"] for r in results if r["name"] == svc["name"]), "")
            print(f"  FAIL: {svc['label']} ({svc['name']}) — {detail}")


if __name__ == "__main__":
    run()