#!/usr/bin/env python3 """ agents_daily_health.py — Tier 2 每日全面健康检查（Linux/246 部署版） ================================================================== 每天 08:00 触发。检查范围：服务状态、磁盘空间、crontab 存活、看门狗日志新鲜度。 部署：Linux crontab: 0 8 * * * cd /home/hmo/AgentsMeeting/gateway/scripts && /usr/bin/python3 agents_daily_health.py >> ../logs/daily_health.log 2>&1 """ import json, os, sys, time, subprocess, shutil from datetime import datetime from urllib.request import urlopen, Request from urllib.error import HTTPError, URLError try: sys.stdout.reconfigure(encoding="utf-8") sys.stderr.reconfigure(encoding="utf-8") except Exception: pass SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) GATEWAY_DIR = os.path.dirname(SCRIPT_DIR) TEMP = os.path.join(GATEWAY_DIR, "temp") LOGS = os.path.join(GATEWAY_DIR, "logs") os.makedirs(LOGS, exist_ok=True) os.makedirs(TEMP, exist_ok=True) REPORT_LOG = os.path.join(LOGS, "daily_health_report.log") REPORT_FILE = os.path.join(TEMP, "last_daily_health.json") # ── 阈值 ────────────────────────────────────────────── DISK_WARN_GB = 10 DISK_CRIT_GB = 2 # ── 服务列表（同 Tier1，但增加 depends_on / description）── SERVICES = [ {"name": "dashboard", "host": "127.0.0.1", "port": 5803, "health_url": "http://127.0.0.1:5803/api/health", "depends_on": [], "desc": "管理门户"}, {"name": "hermes_gateway_mohe", "host": "127.0.0.1", "port": 8642, "health_url": "http://127.0.0.1:8642/v1/health", "depends_on": [], "desc": "莫荷 AI Gateway"}, {"name": "hermes_gateway_zhiwei","host": "127.0.0.1", "port": 8643, "health_url": "http://127.0.0.1:8643/v1/health", "depends_on": [], "desc": "知微 AI Gateway"}, {"name": "wechat_bridge", "host": "127.0.0.1", "port": 3001, "health_url": None, "depends_on": [], "desc": "微信桥接 Docker"}, {"name": "xmpp_bot_xxm", "host": "192.168.1.16", "port": 5807, "health_url": "http://192.168.1.16:5807/health", "depends_on": ["ejabberd"],"desc": "小小莫 XMPP Bot"}, {"name": "article_processor", "host": "192.168.1.16", "port": 5810, "health_url": "http://192.168.1.16:5810/health", "depends_on": [], "desc": "文章抓取服务"}, ] # ── 期望的定时任务（crontab 条目关键字）────────────────── EXPECTED_CRON = [ "auto_heal.py", "agents_health_check.py", ] def log(msg): ts = datetime.now().strftime("%H:%M:%S") line = f"[{ts}] {msg}" print(line) with open(REPORT_LOG, "a", encoding="utf-8") as f: f.write(line + "\n") def port_open(host, port, timeout=3): import socket try: s = socket.create_connection((host, port), timeout=timeout) s.close() return True except: return False def http_check(url, timeout=8): if not url: return (True, "no_health_url") try: req = Request(url) with urlopen(req, timeout=timeout) as r: return (r.status == 200, f"HTTP {r.status}") except HTTPError as e: return (e.code in (401, 403), f"HTTP {e.code}") except Exception as e: return (False, str(e)[:60]) def check_disk(path="/"): """Linux df 磁盘检查。""" try: usage = shutil.disk_usage(path) free_gb = usage.free / (1024 ** 3) if free_gb < DISK_CRIT_GB: return ("critical", f"{free_gb:.1f}GB free (threshold: {DISK_CRIT_GB}GB)") elif free_gb < DISK_WARN_GB: return ("warn", f"{free_gb:.1f}GB free (threshold: {DISK_WARN_GB}GB)") else: return ("ok", f"{free_gb:.1f}GB free") except Exception as e: return ("unknown", f"disk check failed: {e}") def check_cron_tasks(): """检查 crontab 中是否包含期望的定时任务。""" results = [] try: r = subprocess.run(["crontab", "-l"], capture_output=True, text=True, timeout=5) cron_text = r.stdout for name in EXPECTED_CRON: if name in cron_text: results.append({"name": name, "status": "ok"}) else: results.append({"name": name, "status": "missing"}) except Exception as e: results.append({"name": "check_failed", "status": f"error: {e}"}) return results def main(): report = { "time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), "services": [], "disk": {}, "cron_tasks": [], "summary": {"total": 0, "ok": 0, "warn": 0, "fail": 0}, "recommendations": [], } # 0. 磁盘 disk_level, disk_msg = check_disk("/") report["disk"] = {"level": disk_level, "message": disk_msg} if disk_level in ("warn", "critical"): report["recommendations"].append(f"磁盘空间不足: {disk_msg}") # 1. 服务检查 for svc in SERVICES: name = svc["name"] host = svc["host"] port = svc["port"] url = svc.get("health_url") entry = {"name": name, "desc": svc.get("desc", "")} report["services"].append(entry) report["summary"]["total"] += 1 port_ok = port_open(host, port) http_ok = False http_msg = "" if port_ok: http_ok, http_msg = http_check(url, timeout=8) entry["port_ok"] = port_ok entry["http_ok"] = http_ok entry["http_detail"] = http_msg if port_ok and http_ok: entry["status"] = "ok" report["summary"]["ok"] += 1 elif port_ok and not http_ok: entry["status"] = "degraded" report["summary"]["warn"] += 1 report["recommendations"].append(f"{name}: 端口通但 /health 异常（{http_msg}）") else: entry["status"] = "fail" report["summary"]["fail"] += 1 report["recommendations"].append(f"{name}: 离线（Port={port_ok}, HTTP={http_msg}）") # 2. Crontab 检查 report["cron_tasks"] = check_cron_tasks() for t in report["cron_tasks"]: if t["status"] == "missing": report["recommendations"].append(f"定时任务 {t['name']} 缺失") # 3. 看门狗日志新鲜度 wd_log = os.path.join(LOGS, "watchdog.log") if os.path.exists(wd_log): mtime = os.path.getmtime(wd_log) age_hours = (time.time() - mtime) / 3600 report["watchdog_log_age_hours"] = round(age_hours, 1) if age_hours > 1: report["recommendations"].append(f"看门狗日志 {age_hours:.1f}h 未更新") else: report["watchdog_log_age_hours"] = None # 4. 输出 s = report["summary"] log(f"=== 每日健康: {s['ok']}/{s['total']} OK, {s['warn']} degraded, {s['fail']} fail ===") for svc in report["services"]: emoji = {"ok": "[OK]", "degraded": "[WARN]", "fail": "[DOWN]"}.get(svc["status"], "[?]") log(f" {emoji} {svc['name']} | Port={svc['port_ok']} HTTP={svc.get('http_detail','')}") log(f" [DISK] {report['disk']['message']}") for t in report["cron_tasks"]: log(f" [CRON] {t['name']}: {t['status']}") for r in report["recommendations"]: log(f" -> {r}") with open(REPORT_FILE, "w", encoding="utf-8") as f: json.dump(report, f, ensure_ascii=False, indent=2) if __name__ == "__main__": main()