247 lines
8.3 KiB
Python
247 lines
8.3 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
agents_health_check.py — Tier 1 快速健康检查（Linux/246 部署版）
|
||
============================================================
|
||
每 5 分钟触发。对所有注册服务做检查：PID存活 → 端口监听 → HTTP /health。
|
||
异常时写 TODO 给 self_todo_executor 消费。正常时静默。
|
||
|
||
部署：Linux crontab: */5 * * * * cd /home/hmo/AgentsMeeting/gateway/scripts && /usr/bin/python3 agents_health_check.py >> ../logs/health_check.log 2>&1
|
||
"""
|
||
import json, os, sys, socket, subprocess
|
||
from datetime import datetime
|
||
from urllib.request import urlopen, Request
|
||
from urllib.error import HTTPError, URLError
|
||
|
||
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
||
GATEWAY_DIR = os.path.dirname(SCRIPT_DIR)
|
||
BASE = os.path.dirname(GATEWAY_DIR)
|
||
TEMP = os.path.join(GATEWAY_DIR, "temp")
|
||
LOGS = os.path.join(GATEWAY_DIR, "logs")
|
||
os.makedirs(TEMP, exist_ok=True)
|
||
os.makedirs(LOGS, exist_ok=True)
|
||
|
||
NOW = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||
TODO_FILE = os.path.join(TEMP, "health_todos.jsonl")
|
||
REPORT_FILE = os.path.join(TEMP, "last_health_check.json")
|
||
HEALTH_LOG = os.path.join(LOGS, "health_check_report.log")
|
||
|
||
# ── 服务注册表（Linux 246 视角）──────────────────────────────
|
||
# 本机服务用 127.0.0.1，远程服务用实际 IP + HTTP only
|
||
SERVICES = [
|
||
# === 本机服务（246）===
|
||
{
|
||
"name": "dashboard",
|
||
"host": "127.0.0.1",
|
||
"port": 5803,
|
||
"health_url": "http://127.0.0.1:5803/api/health",
|
||
"fix_cmd": ["sudo", "systemctl", "restart", "agentsmeeting-dashboard"],
|
||
"remote": False,
|
||
"critical": True,
|
||
},
|
||
{
|
||
"name": "hermes_gateway_mohe",
|
||
"host": "127.0.0.1",
|
||
"port": 8646,
|
||
"health_url": "http://127.0.0.1:8646/v1/health",
|
||
"fix_cmd": ["sudo", "systemctl", "restart", "hermes-gateway@mohe"],
|
||
"remote": False,
|
||
"critical": True,
|
||
},
|
||
{
|
||
"name": "hermes_gateway_zhiwei",
|
||
"host": "127.0.0.1",
|
||
"port": 8643,
|
||
"health_url": "http://127.0.0.1:8643/v1/health",
|
||
"fix_cmd": ["sudo", "systemctl", "restart", "hermes-gateway@zhiwei"],
|
||
"remote": False,
|
||
"critical": True,
|
||
},
|
||
{
|
||
"name": "wechat_bridge",
|
||
"host": "127.0.0.1",
|
||
"port": 3001,
|
||
"health_url": None, # TCP only
|
||
"fix_cmd": ["docker", "restart", "wxBotWebhook"],
|
||
"remote": False,
|
||
"critical": True,
|
||
"docker_name": "wxBotWebhook",
|
||
"login_check": True, # 额外检查微信登录态
|
||
},
|
||
{
|
||
"name": "wechat_webhook",
|
||
"host": "127.0.0.1",
|
||
"port": 5804,
|
||
"health_url": None, # TCP only
|
||
"fix_cmd": ["sudo", "systemctl", "restart", "wechat-webhook"],
|
||
"remote": False,
|
||
"critical": True,
|
||
},
|
||
# === 远程服务（跨网 HTTP 检查）===
|
||
{
|
||
"name": "xmpp_bot_xxm",
|
||
"host": "192.168.1.16",
|
||
"port": 5807,
|
||
"health_url": "http://192.168.1.16:5807/health",
|
||
"fix_cmd": None, # 远程服务无法自动修复
|
||
"remote": True,
|
||
"critical": True,
|
||
},
|
||
{
|
||
"name": "article_processor",
|
||
"host": "192.168.1.16",
|
||
"port": 5810,
|
||
"health_url": "http://192.168.1.16:5810/health",
|
||
"fix_cmd": None,
|
||
"remote": True,
|
||
"critical": False,
|
||
},
|
||
]
|
||
|
||
|
||
# ── 检查基元（Linux 原生）────────────────────────────────
|
||
def log(msg):
|
||
ts = datetime.now().strftime("%H:%M:%S")
|
||
line = f"[{ts}] {msg}"
|
||
with open(HEALTH_LOG, "a", encoding="utf-8") as f:
|
||
f.write(line + "\n")
|
||
|
||
|
||
def pid_alive(pid):
|
||
"""发送信号 0 测试进程存活。"""
|
||
try:
|
||
os.kill(pid, 0)
|
||
return True
|
||
except (ProcessLookupError, PermissionError):
|
||
return False
|
||
|
||
|
||
def port_open(host, port, timeout=3):
|
||
"""socket connect 测试端口是否监听。"""
|
||
try:
|
||
s = socket.create_connection((host, port), timeout=timeout)
|
||
s.close()
|
||
return True
|
||
except (socket.timeout, ConnectionRefusedError, OSError):
|
||
return False
|
||
|
||
|
||
def http_check(url, timeout=5):
|
||
"""HTTP /health 检查。返回 (ok, detail)。"""
|
||
if not url:
|
||
return (True, "no_health_url")
|
||
try:
|
||
req = Request(url)
|
||
with urlopen(req, timeout=timeout) as r:
|
||
return (r.status == 200, f"HTTP {r.status}")
|
||
except HTTPError as e:
|
||
return (e.code in (401, 403), f"HTTP {e.code}")
|
||
except URLError as e:
|
||
return (False, str(e.reason)[:60])
|
||
except Exception as e:
|
||
return (False, str(e)[:60])
|
||
|
||
|
||
def docker_log_check(container_name, pattern, minutes=10):
|
||
"""检查 Docker 日志中最近 N 分钟是否出现指定模式。
|
||
返回 True = 模式出现（如二维码提示 = 未登录），False = 未出现。"""
|
||
try:
|
||
result = subprocess.run(
|
||
["docker", "logs", container_name, "--since", f"{minutes}m"],
|
||
capture_output=True, text=True, timeout=10
|
||
)
|
||
logs = result.stdout + result.stderr
|
||
return pattern in logs
|
||
except Exception:
|
||
return False
|
||
|
||
|
||
def write_todo(name, issue):
|
||
"""写 TODO 供 executor 消费。"""
|
||
entry = {
|
||
"created": NOW,
|
||
"service": name,
|
||
"issue": issue,
|
||
"status": "pending",
|
||
}
|
||
with open(TODO_FILE, "a", encoding="utf-8") as f:
|
||
f.write(json.dumps(entry, ensure_ascii=False) + "\n")
|
||
log(f"TODO: {name} — {issue}")
|
||
|
||
|
||
# ── 主检查 ────────────────────────────────────────────
|
||
def main():
|
||
report = {"time": NOW, "services": [], "summary": {"total": 0, "ok": 0, "fail": 0}}
|
||
dirty = False
|
||
|
||
for svc in SERVICES:
|
||
name = svc["name"]
|
||
host = svc["host"]
|
||
port = svc["port"]
|
||
url = svc.get("health_url")
|
||
is_remote = svc.get("remote", False)
|
||
critical = svc.get("critical", True)
|
||
|
||
entry = {"name": name, "critical": critical}
|
||
report["services"].append(entry)
|
||
report["summary"]["total"] += 1
|
||
|
||
# 1. Port check（socket connect，跨平台）
|
||
port_ok = port_open(host, port)
|
||
entry["port_ok"] = port_ok
|
||
|
||
# 2. HTTP check
|
||
if port_ok:
|
||
ok, detail = http_check(url, timeout=5 if is_remote else 8)
|
||
entry["http_ok"] = ok
|
||
entry["http_detail"] = detail
|
||
else:
|
||
entry["http_ok"] = False
|
||
entry["http_detail"] = "port_closed"
|
||
|
||
# 3. Docker 登录态检查（如果配置了 login_check）
|
||
login_ok = True
|
||
if port_ok and svc.get("login_check") and svc.get("docker_name"):
|
||
qr_showing = docker_log_check(svc["docker_name"], "扫码", minutes=10)
|
||
if qr_showing:
|
||
login_ok = False
|
||
entry["login_ok"] = False
|
||
entry["login_detail"] = "not_logged_in"
|
||
else:
|
||
entry["login_ok"] = True
|
||
|
||
# 4. 判定：Port+HTTP OK + 登录态 = 健康
|
||
primary_ok = port_ok and entry.get("http_ok", False) and login_ok
|
||
entry["status"] = "ok" if primary_ok else "fail"
|
||
|
||
if primary_ok:
|
||
report["summary"]["ok"] += 1
|
||
else:
|
||
report["summary"]["fail"] += 1
|
||
dirty = True
|
||
reasons = []
|
||
if not port_ok:
|
||
reasons.append("port_closed")
|
||
if port_ok and not entry.get("http_ok"):
|
||
reasons.append(f"http_{entry.get('http_detail','?')}")
|
||
if port_ok and not login_ok:
|
||
reasons.append("not_logged_in")
|
||
issue = f"异常: {' + '.join(reasons)}"
|
||
write_todo(name, issue)
|
||
|
||
# 输出报告
|
||
summary = report["summary"]
|
||
if dirty:
|
||
log(f"=== 健康检查: {summary['ok']}/{summary['total']} OK, {summary['fail']} FAIL ===")
|
||
for s in report["services"]:
|
||
if s["status"] == "fail":
|
||
print(f"[FAIL] {s['name']}: PORT={s['port_ok']} HTTP={s.get('http_detail','?')}")
|
||
else:
|
||
log(f"=== 健康检查: {summary['ok']}/{summary['total']} 全部正常 ===")
|
||
|
||
with open(REPORT_FILE, "w", encoding="utf-8") as f:
|
||
json.dump(report, f, ensure_ascii=False, indent=2)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|