hmo
69fa67e24f
- A: git log + dirty status - B: 服务架构矩阵 (/health + watchdog覆盖 + 拓扑依赖) - C: Tier1/Tier2 监控结果 + 定时任务状态 - D: TODO执行记录 + escalation日志 - F: 期望状态 vs 实际状态合规矩阵 - Kanban: dashboard内嵌版块 - 弹窗修复: 计划任务改用pythonw.exe + subprocess加CREATE_NO_WINDOW - 修复ast_grep_replace导致的
258 lines
9.1 KiB
Python
258 lines
9.1 KiB
Python
"""
|
||
agents_daily_health.py — Tier 2 全面健康检查（每日 08:00 触发）
|
||
在 Tier 1（进程/端口/HTTP）基础上增加：
|
||
- 磁盘空间检查
|
||
- 定时任务存活检查
|
||
- 数据新鲜度（各服务最后响应时间）
|
||
- 跨服务依赖链检查
|
||
- XMPP 异常报告
|
||
|
||
基于 MoFin morning_health_check.py 的模式。
|
||
"""
|
||
import json, os, sys, time, subprocess, urllib.request, urllib.error
|
||
from datetime import datetime
|
||
|
||
# 强制 stdout/stderr 为 UTF-8，避免 Windows GBK 控制台对中文/emoji 编码失败
|
||
try:
|
||
sys.stdout.reconfigure(encoding="utf-8")
|
||
sys.stderr.reconfigure(encoding="utf-8")
|
||
except Exception:
|
||
# Python <3.7 不支持 reconfigure，忽略
|
||
pass
|
||
|
||
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
||
GATEWAY_DIR = os.path.dirname(SCRIPT_DIR)
|
||
BASE = os.path.dirname(GATEWAY_DIR)
|
||
TEMP = os.path.join(BASE, "gateway", "temp")
|
||
LOGS = os.path.join(BASE, "gateway", "logs")
|
||
PYTHON = r"C:\Users\hmo\AppData\Local\Programs\Python\Python310\python.exe"
|
||
|
||
os.makedirs(LOGS, exist_ok=True)
|
||
os.makedirs(TEMP, exist_ok=True)
|
||
|
||
REPORT_LOG = os.path.join(LOGS, "daily_health_report.log")
|
||
REPORT_FILE = os.path.join(TEMP, "last_daily_health.json")
|
||
|
||
# ── 磁盘阈值 ─────────────────────────────────
|
||
DISK_WARN_GB = 10 # 剩余 <10G → 警告
|
||
DISK_CRIT_GB = 2 # 剩余 <2G → 严重
|
||
|
||
# ── 服务注册表（从共享 registry 导入）─────────────
|
||
sys.path.insert(0, SCRIPT_DIR)
|
||
from service_registry import SERVICES as _SR, TEMP as _TEMP, GATEWAY_DIR as _GD
|
||
|
||
TEMP = _TEMP
|
||
LOGS = os.path.join(_GD, "logs")
|
||
os.makedirs(LOGS, exist_ok=True)
|
||
os.makedirs(TEMP, exist_ok=True)
|
||
|
||
SERVICES = []
|
||
for svc in _SR:
|
||
SERVICES.append({
|
||
"name": svc["name"],
|
||
"pid_file": svc["pid_file"],
|
||
"port": svc["port"],
|
||
"health_url": svc["health_url"],
|
||
"accept_401": svc.get("accept_401", False),
|
||
"depends_on": svc.get("depends_on", []),
|
||
})
|
||
|
||
|
||
def log(msg):
|
||
ts = datetime.now().strftime("%H:%M:%S")
|
||
line = f"[{ts}] {msg}"
|
||
print(line)
|
||
with open(REPORT_LOG, "a", encoding="utf-8") as f:
|
||
f.write(line + "\n")
|
||
|
||
|
||
def sizeof_fmt(n):
|
||
for unit in ("B", "K", "M", "G"):
|
||
if n < 1024:
|
||
return f"{n:.1f}{unit}"
|
||
n /= 1024
|
||
return f"{n:.1f}T"
|
||
|
||
|
||
# ── 检查基元 ──────────────────────────────────
|
||
def pid_alive(pid):
|
||
try:
|
||
r = subprocess.run(["tasklist", "/FI", f"PID eq {pid}", "/NH"],
|
||
capture_output=True, text=True, timeout=5,
|
||
creationflags=subprocess.CREATE_NO_WINDOW)
|
||
return str(pid) in r.stdout
|
||
except:
|
||
return False
|
||
|
||
|
||
def port_open(port):
|
||
try:
|
||
r = subprocess.run(["netstat", "-ano"], capture_output=True,
|
||
text=True, timeout=5,
|
||
creationflags=subprocess.CREATE_NO_WINDOW)
|
||
return any(f":{port} " in line and "LISTENING" in line
|
||
for line in r.stdout.splitlines())
|
||
except:
|
||
return False
|
||
|
||
|
||
def http_check(url, timeout=5, accept_401=False):
|
||
try:
|
||
r = urllib.request.urlopen(url, timeout=timeout)
|
||
return (r.status == 200, f"HTTP {r.status}")
|
||
except urllib.error.HTTPError as e:
|
||
if accept_401 and e.code in (401, 403):
|
||
return (True, "auth")
|
||
return (False, f"HTTP {e.code}")
|
||
except urllib.error.URLError as e:
|
||
return (False, str(e.reason)[:60])
|
||
except Exception as e:
|
||
return (False, str(e)[:60])
|
||
|
||
|
||
def check_disk(path="C:"):
|
||
"""Windows 磁盘剩余空间检查。"""
|
||
try:
|
||
r = subprocess.run(
|
||
["wmic", "LogicalDisk", "where", f"DeviceID='{path}'",
|
||
"get", "FreeSpace,Size", "/format:csv"],
|
||
capture_output=True, text=True, timeout=10
|
||
)
|
||
for line in r.stdout.splitlines():
|
||
parts = [p.strip() for p in line.split(",")]
|
||
if len(parts) >= 3 and parts[1] == path:
|
||
free = int(parts[2])
|
||
free_gb = free / (1024**3)
|
||
if free_gb < DISK_CRIT_GB:
|
||
return ("critical", f"{free_gb:.1f}GB free (threshold: {DISK_CRIT_GB}GB)")
|
||
elif free_gb < DISK_WARN_GB:
|
||
return ("warn", f"{free_gb:.1f}GB free (threshold: {DISK_WARN_GB}GB)")
|
||
else:
|
||
return ("ok", f"{free_gb:.1f}GB free")
|
||
except:
|
||
pass
|
||
return ("unknown", "disk check failed")
|
||
|
||
|
||
def check_cron_tasks():
|
||
"""检查关键定时任务是否启用。"""
|
||
expected = ["agents-health-check"]
|
||
results = []
|
||
try:
|
||
for name in expected:
|
||
r = subprocess.run(
|
||
["schtasks", "/Query", "/TN", name, "/FO", "CSV", "/NH"],
|
||
capture_output=True, text=True, timeout=10
|
||
)
|
||
if name in r.stdout:
|
||
status = "ok" if "Ready" in r.stdout or "Running" in r.stdout else "warn"
|
||
results.append({"name": name, "status": status})
|
||
else:
|
||
results.append({"name": name, "status": "missing"})
|
||
except:
|
||
results.append({"name": "check_failed", "status": "error"})
|
||
return results
|
||
|
||
|
||
# ── 主检查 ────────────────────────────────────
|
||
def main():
|
||
report = {
|
||
"time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||
"services": [],
|
||
"disk": {},
|
||
"cron_tasks": [],
|
||
"summary": {"total": 0, "ok": 0, "warn": 0, "fail": 0},
|
||
"recommendations": [],
|
||
}
|
||
|
||
# 0. 磁盘
|
||
disk_level, disk_msg = check_disk("C:")
|
||
report["disk"] = {"level": disk_level, "message": disk_msg}
|
||
if disk_level in ("warn", "critical"):
|
||
report["summary"][disk_level] += 1
|
||
report["recommendations"].append(f"磁盘空间不足: {disk_msg}")
|
||
|
||
# 1. 服务检查
|
||
for svc in SERVICES:
|
||
name = svc["name"]
|
||
entry = {"name": name}
|
||
report["services"].append(entry)
|
||
report["summary"]["total"] += 1
|
||
|
||
pid = 0
|
||
pid_file = svc.get("pid_file")
|
||
if pid_file:
|
||
try:
|
||
with open(pid_file) as f:
|
||
pid = int(f.read().strip())
|
||
except:
|
||
pid = 0
|
||
|
||
pid_ok = (pid > 0 and pid_alive(pid)) if pid_file else True # 无 PID 文件不视为异常
|
||
port_ok = port_open(svc["port"])
|
||
http_ok = False
|
||
http_msg = ""
|
||
if port_ok:
|
||
http_ok, http_msg = http_check(svc["health_url"], accept_401=svc.get("accept_401", False))
|
||
|
||
entry["pid"] = pid if pid_ok else 0
|
||
entry["pid_ok"] = pid_ok
|
||
entry["port_ok"] = port_ok
|
||
entry["http_ok"] = http_ok
|
||
entry["http_detail"] = http_msg
|
||
|
||
if pid_ok and port_ok and http_ok:
|
||
entry["status"] = "ok"
|
||
report["summary"]["ok"] += 1
|
||
elif pid_ok and port_ok and not http_ok:
|
||
entry["status"] = "degraded"
|
||
report["summary"]["warn"] += 1
|
||
report["recommendations"].append(f"{name}: PID/Port 正常但 /health 异常（{http_msg}）")
|
||
else:
|
||
entry["status"] = "fail"
|
||
report["summary"]["fail"] += 1
|
||
report["recommendations"].append(f"{name}: 离线（PID={pid_ok}, Port={port_ok}, HTTP={http_msg}）")
|
||
|
||
# 2. 定时任务检查
|
||
report["cron_tasks"] = check_cron_tasks()
|
||
for t in report["cron_tasks"]:
|
||
if t["status"] == "missing":
|
||
report["recommendations"].append(f"定时任务 {t['name']} 缺失")
|
||
|
||
# 3. 看门狗日志新鲜度（最后修改时间）
|
||
watchdog_log = os.path.join(LOGS, "watchdog.log")
|
||
if os.path.exists(watchdog_log):
|
||
mtime = os.path.getmtime(watchdog_log)
|
||
age_hours = (time.time() - mtime) / 3600
|
||
report["watchdog_log_age_hours"] = round(age_hours, 1)
|
||
if age_hours > 1:
|
||
report["recommendations"].append(f"看门狗日志 {age_hours:.1f} 小时未更新，可能看门狗已死")
|
||
else:
|
||
report["watchdog_log_age_hours"] = None
|
||
report["recommendations"].append("看门狗日志不存在")
|
||
|
||
# 4. 输出报告
|
||
summary = report["summary"]
|
||
log(f"=== 每日健康检查: {summary['ok']}/{summary['total']} 正常, {summary['warn']} 降级, {summary['fail']} 离线 ===")
|
||
for svc in report["services"]:
|
||
emoji = {"ok": "[OK]", "degraded": "[WARN]", "fail": "[DOWN]"}.get(svc["status"], "[?]")
|
||
detail = svc.get("http_detail", "")
|
||
log(f" {emoji} {svc['name']} | PID={svc['pid_ok']} Port={svc['port_ok']} HTTP={detail}")
|
||
log(f" [DISK] {report['disk']['message']}")
|
||
for t in report["cron_tasks"]:
|
||
log(f" [CRON] {t['name']}: {t['status']}")
|
||
log(f" [WATCHDOG] 日志: {report.get('watchdog_log_age_hours', 'N/A')}h old")
|
||
|
||
if report["recommendations"]:
|
||
log("=== 建议动作 ===")
|
||
for r in report["recommendations"]:
|
||
log(f" - {r}")
|
||
|
||
# 保存 JSON 报告
|
||
with open(REPORT_FILE, "w", encoding="utf-8") as f:
|
||
json.dump(report, f, ensure_ascii=False, indent=2)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|