191 lines
7.7 KiB
Python
191 lines
7.7 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
"""
|
||
agents_health_check.py — MoFin Tier1 快速健康检查
|
||
====================================================
|
||
每 5 分钟运行一次（crontab）。检查关键服务的端口/HTTP/DB 可用性。
|
||
全正常时静默（不输出）。异常时写入 TODO 文件和 JSON 报告。
|
||
同时采集 XMPP 通道健康数据到时间序列日志。
|
||
|
||
部署: crontab */5 * * * * cd /home/hmo/MoFin && python3 agents_health_check.py
|
||
"""
|
||
import json, os, sys, socket, sqlite3, urllib.request
|
||
from datetime import datetime
|
||
from pathlib import Path
|
||
|
||
# ---- Config ----
|
||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||
TEMP_DIR = SCRIPT_DIR / "gateway" / "temp"
|
||
LOGS_DIR = SCRIPT_DIR / "gateway" / "logs"
|
||
|
||
# Ensure dirs
|
||
TEMP_DIR.mkdir(parents=True, exist_ok=True)
|
||
LOGS_DIR.mkdir(parents=True, exist_ok=True)
|
||
|
||
REPORT_FILE = TEMP_DIR / "last_health_check.json"
|
||
TODO_FILE = TEMP_DIR / "health_todos.jsonl"
|
||
LOG_FILE = LOGS_DIR / "health_check.log"
|
||
|
||
# ---- Service List ----
|
||
SERVICES = [
|
||
{"name": "mofin_api", "label": "MoFin API", "host": "127.0.0.1", "port": 8899, "type": "http", "check": "/api/health"},
|
||
{"name": "zhiwei_gateway", "label": "知微 Gateway", "host": "127.0.0.1", "port": 8643, "type": "http", "check": "/v1/health"},
|
||
{"name": "ejabberd", "label": "ejabberd XMPP", "host": "127.0.0.1", "port": 5222, "type": "tcp", "check": None},
|
||
{"name": "mofin_db", "label": "MoFin 数据库", "host": "127.0.0.1", "port": 0, "type": "db", "check": "/home/hmo/web-dashboard/data/mofin.db"},
|
||
]
|
||
|
||
# ── XMPP 通道健康采集 ──────────────────────────────
|
||
|
||
XMPP_HEALTH_LOG = LOGS_DIR / "xmpp_health_log.jsonl"
|
||
|
||
|
||
def _collect_xmpp_health(now):
|
||
"""采集 XMPP 通道健康数据，追加到时间序列日志。"""
|
||
try:
|
||
from xmpp_logger import health as xmpp_health
|
||
h = xmpp_health()
|
||
except Exception as e:
|
||
h = {"status": "collector_error", "error": str(e)[:200]}
|
||
entry = {"timestamp": now.strftime("%Y-%m-%d %H:%M:%S"), **h}
|
||
XMPP_HEALTH_LOG.parent.mkdir(parents=True, exist_ok=True)
|
||
with open(XMPP_HEALTH_LOG, "a", encoding="utf-8") as f:
|
||
f.write(json.dumps(entry, ensure_ascii=False) + "\n")
|
||
|
||
|
||
# ---- Checkers ----
|
||
|
||
def check_tcp(host, port, timeout=3):
|
||
try:
|
||
sock = socket.create_connection((host, port), timeout=timeout)
|
||
sock.close()
|
||
return True, "ok"
|
||
except Exception as e:
|
||
return False, str(e)
|
||
|
||
|
||
def check_http(host, port, path, timeout=3):
|
||
try:
|
||
url = f"http://{host}:{port}{path}"
|
||
req = urllib.request.Request(url)
|
||
resp = urllib.request.urlopen(req, timeout=timeout)
|
||
return 200 <= resp.status < 300, f"HTTP {resp.status}"
|
||
except Exception as e:
|
||
return False, str(e)
|
||
|
||
|
||
def check_db(db_path):
|
||
try:
|
||
conn = sqlite3.connect(db_path)
|
||
conn.execute("SELECT 1")
|
||
conn.close()
|
||
return True, "ok"
|
||
except Exception as e:
|
||
return False, str(e)
|
||
|
||
|
||
# ---- Main ----
|
||
|
||
def run():
|
||
now = datetime.now()
|
||
results = []
|
||
issues = []
|
||
|
||
for svc in SERVICES:
|
||
if svc["type"] == "tcp":
|
||
ok, detail = check_tcp(svc["host"], svc["port"])
|
||
elif svc["type"] == "http":
|
||
ok, detail = check_http(svc["host"], svc["port"], svc["check"])
|
||
elif svc["type"] == "db":
|
||
ok, detail = check_db(svc["check"])
|
||
else:
|
||
ok, detail = False, "unknown type"
|
||
|
||
results.append({
|
||
"name": svc["name"], "label": svc["label"],
|
||
"type": svc["type"], "port": svc["port"],
|
||
"health": {"ok": ok}, "detail": detail,
|
||
})
|
||
if not ok:
|
||
issues.append(svc)
|
||
|
||
# Write report
|
||
report = {
|
||
"services": results,
|
||
"summary": {"ok": sum(1 for r in results if r["health"]["ok"]), "total": len(results)},
|
||
"generated_at": now.strftime("%Y-%m-%d %H:%M:%S"),
|
||
}
|
||
with open(REPORT_FILE, "w", encoding="utf-8") as f:
|
||
json.dump(report, f, ensure_ascii=False, indent=2)
|
||
|
||
# ── XMPP 通道健康采集 ──
|
||
_collect_xmpp_health(now)
|
||
|
||
# ── 自愈：检测 rate-limit → 切 key；Gateway 异常 → systemctl restart ──
|
||
try:
|
||
from xmpp_logger import auto_heal as _xmpp_auto_heal
|
||
heal_result = _xmpp_auto_heal()
|
||
actions = heal_result.get("actions", [])
|
||
if actions:
|
||
with open(LOG_FILE, "a", encoding="utf-8") as f:
|
||
f.write(f"[{now.strftime('%Y-%m-%d %H:%M:%S')}] AUTO-HEAL actions:\n")
|
||
for a in actions:
|
||
f.write(f" {json.dumps(a, ensure_ascii=False)}\n")
|
||
print(f"[{now.strftime('%H:%M')}] Auto-heal: {len(actions)} action(s) — {heal_result.get('status')}")
|
||
for a in actions:
|
||
print(f" → {a.get('action', '?')}: success={a.get('success', a.get('switched', '?'))}")
|
||
except Exception as e:
|
||
print(f"[{now.strftime('%H:%M')}] Auto-heal error: {e}")
|
||
with open(LOG_FILE, "a", encoding="utf-8") as f:
|
||
f.write(f"[{now.strftime('%Y-%m-%d %H:%M:%S')}] AUTO-HEAL ERROR: {e}\n")
|
||
|
||
# ── Gateway 端口宕机兜底：异步 systemctl 重启（带 cooldown 防频繁触发）──
|
||
gateway_ok = any(r["name"] == "zhiwei_gateway" and r["health"]["ok"] for r in results)
|
||
if not gateway_ok:
|
||
import time as _t
|
||
import subprocess as _sp
|
||
cooldown_file = LOGS_DIR / "last_restart.txt"
|
||
cooldown_sec = 180
|
||
try:
|
||
last_ts = float(cooldown_file.read_text().strip()) if cooldown_file.exists() else 0
|
||
except Exception:
|
||
last_ts = 0
|
||
elapsed = _t.time() - last_ts
|
||
if elapsed < cooldown_sec:
|
||
with open(LOG_FILE, "a", encoding="utf-8") as f:
|
||
f.write(f"[{now.strftime('%Y-%m-%d %H:%M:%S')}] Gateway DOWN but cooldown {int(elapsed)}s < {cooldown_sec}s — skip restart\n")
|
||
print(f"[{now.strftime('%H:%M')}] Gateway DOWN but in cooldown ({int(elapsed)}s)")
|
||
else:
|
||
try:
|
||
_sp.Popen(["sudo", "-n", "systemctl", "restart", "hermes-gateway-zhiwei.service"],
|
||
stdout=_sp.DEVNULL, stderr=_sp.DEVNULL, start_new_session=True)
|
||
cooldown_file.write_text(str(_t.time()))
|
||
with open(LOG_FILE, "a", encoding="utf-8") as f:
|
||
f.write(f"[{now.strftime('%Y-%m-%d %H:%M:%S')}] Gateway DOWN → systemctl restart triggered (async, cooldown set)\n")
|
||
print(f"[{now.strftime('%H:%M')}] Gateway DOWN → systemctl restart triggered")
|
||
except Exception as e:
|
||
print(f"[{now.strftime('%H:%M')}] Gateway restart error: {e}")
|
||
|
||
# Handle issues
|
||
if issues:
|
||
with open(TODO_FILE, "a", encoding="utf-8") as f:
|
||
for svc in issues:
|
||
entry = {
|
||
"service": svc["name"], "label": svc["label"],
|
||
"reason": next((r["detail"] for r in results if r["name"] == svc["name"]), "unknown"),
|
||
"timestamp": now.isoformat(),
|
||
}
|
||
f.write(json.dumps(entry, ensure_ascii=False) + "\n")
|
||
with open(LOG_FILE, "a", encoding="utf-8") as f:
|
||
f.write(f"[{now.strftime('%Y-%m-%d %H:%M:%S')}] ISSUES: {len(issues)} failed\n")
|
||
for svc in issues:
|
||
detail = next((r["detail"] for r in results if r["name"] == svc["name"]), "")
|
||
f.write(f" - {svc['label']}: {detail}\n")
|
||
print(f"[{now.strftime('%H:%M')}] Health check: {len(issues)}/{len(SERVICES)} services failed")
|
||
for svc in issues:
|
||
detail = next((r["detail"] for r in results if r["name"] == svc["name"]), "")
|
||
print(f" FAIL: {svc['label']} ({svc['name']}) — {detail}")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
run()
|