fix(bot): 防递归重连 guard - 防止 on_disconnect → reconnect → disconnect 无限递归导致栈溢出
This commit is contained in:
知微
@@ -0,0 +1,190 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
agents_health_check.py — MoFin Tier1 快速健康检查
|
||||
====================================================
|
||||
每 5 分钟运行一次（crontab）。检查关键服务的端口/HTTP/DB 可用性。
|
||||
全正常时静默（不输出）。异常时写入 TODO 文件和 JSON 报告。
|
||||
同时采集 XMPP 通道健康数据到时间序列日志。
|
||||
|
||||
部署: crontab */5 * * * * cd /home/hmo/MoFin && python3 agents_health_check.py
|
||||
"""
|
||||
import json, os, sys, socket, sqlite3, urllib.request
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
# ---- Config ----
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
TEMP_DIR = SCRIPT_DIR / "gateway" / "temp"
|
||||
LOGS_DIR = SCRIPT_DIR / "gateway" / "logs"
|
||||
|
||||
# Ensure dirs
|
||||
TEMP_DIR.mkdir(parents=True, exist_ok=True)
|
||||
LOGS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
REPORT_FILE = TEMP_DIR / "last_health_check.json"
|
||||
TODO_FILE = TEMP_DIR / "health_todos.jsonl"
|
||||
LOG_FILE = LOGS_DIR / "health_check.log"
|
||||
|
||||
# ---- Service List ----
|
||||
SERVICES = [
|
||||
{"name": "mofin_api", "label": "MoFin API", "host": "127.0.0.1", "port": 8899, "type": "http", "check": "/api/health"},
|
||||
{"name": "zhiwei_gateway", "label": "知微 Gateway", "host": "127.0.0.1", "port": 8643, "type": "http", "check": "/v1/health"},
|
||||
{"name": "ejabberd", "label": "ejabberd XMPP", "host": "127.0.0.1", "port": 5222, "type": "tcp", "check": None},
|
||||
{"name": "mofin_db", "label": "MoFin 数据库", "host": "127.0.0.1", "port": 0, "type": "db", "check": "/home/hmo/web-dashboard/data/mofin.db"},
|
||||
]
|
||||
|
||||
# ── XMPP 通道健康采集 ──────────────────────────────
|
||||
|
||||
XMPP_HEALTH_LOG = LOGS_DIR / "xmpp_health_log.jsonl"
|
||||
|
||||
|
||||
def _collect_xmpp_health(now):
|
||||
"""采集 XMPP 通道健康数据，追加到时间序列日志。"""
|
||||
try:
|
||||
from xmpp_logger import health as xmpp_health
|
||||
h = xmpp_health()
|
||||
except Exception as e:
|
||||
h = {"status": "collector_error", "error": str(e)[:200]}
|
||||
entry = {"timestamp": now.strftime("%Y-%m-%d %H:%M:%S"), **h}
|
||||
XMPP_HEALTH_LOG.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(XMPP_HEALTH_LOG, "a", encoding="utf-8") as f:
|
||||
f.write(json.dumps(entry, ensure_ascii=False) + "\n")
|
||||
|
||||
|
||||
# ---- Checkers ----
|
||||
|
||||
def check_tcp(host, port, timeout=3):
|
||||
try:
|
||||
sock = socket.create_connection((host, port), timeout=timeout)
|
||||
sock.close()
|
||||
return True, "ok"
|
||||
except Exception as e:
|
||||
return False, str(e)
|
||||
|
||||
|
||||
def check_http(host, port, path, timeout=3):
|
||||
try:
|
||||
url = f"http://{host}:{port}{path}"
|
||||
req = urllib.request.Request(url)
|
||||
resp = urllib.request.urlopen(req, timeout=timeout)
|
||||
return 200 <= resp.status < 300, f"HTTP {resp.status}"
|
||||
except Exception as e:
|
||||
return False, str(e)
|
||||
|
||||
|
||||
def check_db(db_path):
|
||||
try:
|
||||
conn = sqlite3.connect(db_path)
|
||||
conn.execute("SELECT 1")
|
||||
conn.close()
|
||||
return True, "ok"
|
||||
except Exception as e:
|
||||
return False, str(e)
|
||||
|
||||
|
||||
# ---- Main ----
|
||||
|
||||
def run():
|
||||
now = datetime.now()
|
||||
results = []
|
||||
issues = []
|
||||
|
||||
for svc in SERVICES:
|
||||
if svc["type"] == "tcp":
|
||||
ok, detail = check_tcp(svc["host"], svc["port"])
|
||||
elif svc["type"] == "http":
|
||||
ok, detail = check_http(svc["host"], svc["port"], svc["check"])
|
||||
elif svc["type"] == "db":
|
||||
ok, detail = check_db(svc["check"])
|
||||
else:
|
||||
ok, detail = False, "unknown type"
|
||||
|
||||
results.append({
|
||||
"name": svc["name"], "label": svc["label"],
|
||||
"type": svc["type"], "port": svc["port"],
|
||||
"health": {"ok": ok}, "detail": detail,
|
||||
})
|
||||
if not ok:
|
||||
issues.append(svc)
|
||||
|
||||
# Write report
|
||||
report = {
|
||||
"services": results,
|
||||
"summary": {"ok": sum(1 for r in results if r["health"]["ok"]), "total": len(results)},
|
||||
"generated_at": now.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
}
|
||||
with open(REPORT_FILE, "w", encoding="utf-8") as f:
|
||||
json.dump(report, f, ensure_ascii=False, indent=2)
|
||||
|
||||
# ── XMPP 通道健康采集 ──
|
||||
_collect_xmpp_health(now)
|
||||
|
||||
# ── 自愈：检测 rate-limit → 切 key；Gateway 异常 → systemctl restart ──
|
||||
try:
|
||||
from xmpp_logger import auto_heal as _xmpp_auto_heal
|
||||
heal_result = _xmpp_auto_heal()
|
||||
actions = heal_result.get("actions", [])
|
||||
if actions:
|
||||
with open(LOG_FILE, "a", encoding="utf-8") as f:
|
||||
f.write(f"[{now.strftime('%Y-%m-%d %H:%M:%S')}] AUTO-HEAL actions:\n")
|
||||
for a in actions:
|
||||
f.write(f" {json.dumps(a, ensure_ascii=False)}\n")
|
||||
print(f"[{now.strftime('%H:%M')}] Auto-heal: {len(actions)} action(s) — {heal_result.get('status')}")
|
||||
for a in actions:
|
||||
print(f" → {a.get('action', '?')}: success={a.get('success', a.get('switched', '?'))}")
|
||||
except Exception as e:
|
||||
print(f"[{now.strftime('%H:%M')}] Auto-heal error: {e}")
|
||||
with open(LOG_FILE, "a", encoding="utf-8") as f:
|
||||
f.write(f"[{now.strftime('%Y-%m-%d %H:%M:%S')}] AUTO-HEAL ERROR: {e}\n")
|
||||
|
||||
# ── Gateway 端口宕机兜底：异步 systemctl 重启（带 cooldown 防频繁触发）──
|
||||
gateway_ok = any(r["name"] == "zhiwei_gateway" and r["health"]["ok"] for r in results)
|
||||
if not gateway_ok:
|
||||
import time as _t
|
||||
import subprocess as _sp
|
||||
cooldown_file = LOGS_DIR / "last_restart.txt"
|
||||
cooldown_sec = 180
|
||||
try:
|
||||
last_ts = float(cooldown_file.read_text().strip()) if cooldown_file.exists() else 0
|
||||
except Exception:
|
||||
last_ts = 0
|
||||
elapsed = _t.time() - last_ts
|
||||
if elapsed < cooldown_sec:
|
||||
with open(LOG_FILE, "a", encoding="utf-8") as f:
|
||||
f.write(f"[{now.strftime('%Y-%m-%d %H:%M:%S')}] Gateway DOWN but cooldown {int(elapsed)}s < {cooldown_sec}s — skip restart\n")
|
||||
print(f"[{now.strftime('%H:%M')}] Gateway DOWN but in cooldown ({int(elapsed)}s)")
|
||||
else:
|
||||
try:
|
||||
_sp.Popen(["sudo", "-n", "systemctl", "restart", "hermes-gateway-zhiwei.service"],
|
||||
stdout=_sp.DEVNULL, stderr=_sp.DEVNULL, start_new_session=True)
|
||||
cooldown_file.write_text(str(_t.time()))
|
||||
with open(LOG_FILE, "a", encoding="utf-8") as f:
|
||||
f.write(f"[{now.strftime('%Y-%m-%d %H:%M:%S')}] Gateway DOWN → systemctl restart triggered (async, cooldown set)\n")
|
||||
print(f"[{now.strftime('%H:%M')}] Gateway DOWN → systemctl restart triggered")
|
||||
except Exception as e:
|
||||
print(f"[{now.strftime('%H:%M')}] Gateway restart error: {e}")
|
||||
|
||||
# Handle issues
|
||||
if issues:
|
||||
with open(TODO_FILE, "a", encoding="utf-8") as f:
|
||||
for svc in issues:
|
||||
entry = {
|
||||
"service": svc["name"], "label": svc["label"],
|
||||
"reason": next((r["detail"] for r in results if r["name"] == svc["name"]), "unknown"),
|
||||
"timestamp": now.isoformat(),
|
||||
}
|
||||
f.write(json.dumps(entry, ensure_ascii=False) + "\n")
|
||||
with open(LOG_FILE, "a", encoding="utf-8") as f:
|
||||
f.write(f"[{now.strftime('%Y-%m-%d %H:%M:%S')}] ISSUES: {len(issues)} failed\n")
|
||||
for svc in issues:
|
||||
detail = next((r["detail"] for r in results if r["name"] == svc["name"]), "")
|
||||
f.write(f" - {svc['label']}: {detail}\n")
|
||||
print(f"[{now.strftime('%H:%M')}] Health check: {len(issues)}/{len(SERVICES)} services failed")
|
||||
for svc in issues:
|
||||
detail = next((r["detail"] for r in results if r["name"] == svc["name"]), "")
|
||||
print(f" FAIL: {svc['label']} ({svc['name']}) — {detail}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run()
|
||||
Reference in New Issue
Block a user