196 lines
7.4 KiB
Python
196 lines
7.4 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
agents_daily_health.py — Tier 2 每日全面健康检查（Linux/246 部署版）
|
||
==================================================================
|
||
每天 08:00 触发。检查范围：服务状态、磁盘空间、crontab 存活、看门狗日志新鲜度。
|
||
|
||
部署：Linux crontab: 0 8 * * * cd /home/hmo/AgentsMeeting/gateway/scripts && /usr/bin/python3 agents_daily_health.py >> ../logs/daily_health.log 2>&1
|
||
"""
|
||
import json, os, sys, time, subprocess, shutil
|
||
from datetime import datetime
|
||
from urllib.request import urlopen, Request
|
||
from urllib.error import HTTPError, URLError
|
||
|
||
try:
|
||
sys.stdout.reconfigure(encoding="utf-8")
|
||
sys.stderr.reconfigure(encoding="utf-8")
|
||
except Exception:
|
||
pass
|
||
|
||
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
||
GATEWAY_DIR = os.path.dirname(SCRIPT_DIR)
|
||
TEMP = os.path.join(GATEWAY_DIR, "temp")
|
||
LOGS = os.path.join(GATEWAY_DIR, "logs")
|
||
os.makedirs(LOGS, exist_ok=True)
|
||
os.makedirs(TEMP, exist_ok=True)
|
||
|
||
REPORT_LOG = os.path.join(LOGS, "daily_health_report.log")
|
||
REPORT_FILE = os.path.join(TEMP, "last_daily_health.json")
|
||
|
||
# ── 阈值 ──────────────────────────────────────────────
|
||
DISK_WARN_GB = 10
|
||
DISK_CRIT_GB = 2
|
||
|
||
# ── 服务列表（同 Tier1，但增加 depends_on / description）──
|
||
SERVICES = [
|
||
{"name": "dashboard", "host": "127.0.0.1", "port": 5803, "health_url": "http://127.0.0.1:5803/api/health", "depends_on": [], "desc": "管理门户"},
|
||
{"name": "hermes_gateway_mohe", "host": "127.0.0.1", "port": 8642, "health_url": "http://127.0.0.1:8642/v1/health", "depends_on": [], "desc": "莫荷 AI Gateway"},
|
||
{"name": "hermes_gateway_zhiwei","host": "127.0.0.1", "port": 8643, "health_url": "http://127.0.0.1:8643/v1/health", "depends_on": [], "desc": "知微 AI Gateway"},
|
||
{"name": "wechat_bridge", "host": "127.0.0.1", "port": 3001, "health_url": None, "depends_on": [], "desc": "微信桥接 Docker"},
|
||
{"name": "xmpp_bot_xxm", "host": "192.168.1.248", "port": 5807, "health_url": "http://192.168.1.248:5807/health", "depends_on": ["ejabberd"],"desc": "小小莫 XMPP Bot"},
|
||
{"name": "article_processor", "host": "192.168.1.248", "port": 5810, "health_url": "http://192.168.1.248:5810/health", "depends_on": [], "desc": "文章抓取服务"},
|
||
]
|
||
|
||
# ── 期望的定时任务（crontab 条目关键字）──────────────────
|
||
EXPECTED_CRON = [
|
||
"auto_heal.py",
|
||
"agents_health_check.py",
|
||
]
|
||
|
||
|
||
def log(msg):
|
||
ts = datetime.now().strftime("%H:%M:%S")
|
||
line = f"[{ts}] {msg}"
|
||
print(line)
|
||
with open(REPORT_LOG, "a", encoding="utf-8") as f:
|
||
f.write(line + "\n")
|
||
|
||
|
||
def port_open(host, port, timeout=3):
|
||
import socket
|
||
try:
|
||
s = socket.create_connection((host, port), timeout=timeout)
|
||
s.close()
|
||
return True
|
||
except:
|
||
return False
|
||
|
||
|
||
def http_check(url, timeout=8):
|
||
if not url:
|
||
return (True, "no_health_url")
|
||
try:
|
||
req = Request(url)
|
||
with urlopen(req, timeout=timeout) as r:
|
||
return (r.status == 200, f"HTTP {r.status}")
|
||
except HTTPError as e:
|
||
return (e.code in (401, 403), f"HTTP {e.code}")
|
||
except Exception as e:
|
||
return (False, str(e)[:60])
|
||
|
||
|
||
def check_disk(path="/"):
|
||
"""Linux df 磁盘检查。"""
|
||
try:
|
||
usage = shutil.disk_usage(path)
|
||
free_gb = usage.free / (1024 ** 3)
|
||
if free_gb < DISK_CRIT_GB:
|
||
return ("critical", f"{free_gb:.1f}GB free (threshold: {DISK_CRIT_GB}GB)")
|
||
elif free_gb < DISK_WARN_GB:
|
||
return ("warn", f"{free_gb:.1f}GB free (threshold: {DISK_WARN_GB}GB)")
|
||
else:
|
||
return ("ok", f"{free_gb:.1f}GB free")
|
||
except Exception as e:
|
||
return ("unknown", f"disk check failed: {e}")
|
||
|
||
|
||
def check_cron_tasks():
|
||
"""检查 crontab 中是否包含期望的定时任务。"""
|
||
results = []
|
||
try:
|
||
r = subprocess.run(["crontab", "-l"], capture_output=True, text=True, timeout=5)
|
||
cron_text = r.stdout
|
||
for name in EXPECTED_CRON:
|
||
if name in cron_text:
|
||
results.append({"name": name, "status": "ok"})
|
||
else:
|
||
results.append({"name": name, "status": "missing"})
|
||
except Exception as e:
|
||
results.append({"name": "check_failed", "status": f"error: {e}"})
|
||
return results
|
||
|
||
|
||
def main():
|
||
report = {
|
||
"time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||
"services": [],
|
||
"disk": {},
|
||
"cron_tasks": [],
|
||
"summary": {"total": 0, "ok": 0, "warn": 0, "fail": 0},
|
||
"recommendations": [],
|
||
}
|
||
|
||
# 0. 磁盘
|
||
disk_level, disk_msg = check_disk("/")
|
||
report["disk"] = {"level": disk_level, "message": disk_msg}
|
||
if disk_level in ("warn", "critical"):
|
||
report["recommendations"].append(f"磁盘空间不足: {disk_msg}")
|
||
|
||
# 1. 服务检查
|
||
for svc in SERVICES:
|
||
name = svc["name"]
|
||
host = svc["host"]
|
||
port = svc["port"]
|
||
url = svc.get("health_url")
|
||
entry = {"name": name, "desc": svc.get("desc", "")}
|
||
report["services"].append(entry)
|
||
report["summary"]["total"] += 1
|
||
|
||
port_ok = port_open(host, port)
|
||
http_ok = False
|
||
http_msg = ""
|
||
if port_ok:
|
||
http_ok, http_msg = http_check(url, timeout=8)
|
||
|
||
entry["port_ok"] = port_ok
|
||
entry["http_ok"] = http_ok
|
||
entry["http_detail"] = http_msg
|
||
|
||
if port_ok and http_ok:
|
||
entry["status"] = "ok"
|
||
report["summary"]["ok"] += 1
|
||
elif port_ok and not http_ok:
|
||
entry["status"] = "degraded"
|
||
report["summary"]["warn"] += 1
|
||
report["recommendations"].append(f"{name}: 端口通但 /health 异常（{http_msg}）")
|
||
else:
|
||
entry["status"] = "fail"
|
||
report["summary"]["fail"] += 1
|
||
report["recommendations"].append(f"{name}: 离线（Port={port_ok}, HTTP={http_msg}）")
|
||
|
||
# 2. Crontab 检查
|
||
report["cron_tasks"] = check_cron_tasks()
|
||
for t in report["cron_tasks"]:
|
||
if t["status"] == "missing":
|
||
report["recommendations"].append(f"定时任务 {t['name']} 缺失")
|
||
|
||
# 3. 看门狗日志新鲜度
|
||
wd_log = os.path.join(LOGS, "watchdog.log")
|
||
if os.path.exists(wd_log):
|
||
mtime = os.path.getmtime(wd_log)
|
||
age_hours = (time.time() - mtime) / 3600
|
||
report["watchdog_log_age_hours"] = round(age_hours, 1)
|
||
if age_hours > 1:
|
||
report["recommendations"].append(f"看门狗日志 {age_hours:.1f}h 未更新")
|
||
else:
|
||
report["watchdog_log_age_hours"] = None
|
||
|
||
# 4. 输出
|
||
s = report["summary"]
|
||
log(f"=== 每日健康: {s['ok']}/{s['total']} OK, {s['warn']} degraded, {s['fail']} fail ===")
|
||
for svc in report["services"]:
|
||
emoji = {"ok": "[OK]", "degraded": "[WARN]", "fail": "[DOWN]"}.get(svc["status"], "[?]")
|
||
log(f" {emoji} {svc['name']} | Port={svc['port_ok']} HTTP={svc.get('http_detail','')}")
|
||
log(f" [DISK] {report['disk']['message']}")
|
||
for t in report["cron_tasks"]:
|
||
log(f" [CRON] {t['name']}: {t['status']}")
|
||
for r in report["recommendations"]:
|
||
log(f" -> {r}")
|
||
|
||
with open(REPORT_FILE, "w", encoding="utf-8") as f:
|
||
json.dump(report, f, ensure_ascii=False, indent=2)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|