From b56b1b3b47761a174bdfc630d458621a08c5cadd Mon Sep 17 00:00:00 2001 From: hmo Date: Sun, 19 Jul 2026 09:05:16 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20health=20check=20PID=20now=20advisory-on?= =?UTF-8?q?ly,=20remote=20services=20skip=20local=20port,=20add=20=3F?= =?UTF-8?q?=C2=A7=20to=20task=20entries?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- gateway/scripts/agents_health_check.py | 32 ++++++++++------ gateway/scripts/service_registry.py | 2 + gateway/scripts/specs/daily_health.json | 46 +++++++++++++++++++++++ gateway/scripts/specs/health_check.json | 48 ++++++++++++++++++++++++ gateway/scripts/specs/todo_executor.json | 47 +++++++++++++++++++++++ gateway/scripts/templates/dashboard.html | 8 +++- 6 files changed, 169 insertions(+), 14 deletions(-) create mode 100644 gateway/scripts/specs/daily_health.json create mode 100644 gateway/scripts/specs/health_check.json create mode 100644 gateway/scripts/specs/todo_executor.json diff --git a/gateway/scripts/agents_health_check.py b/gateway/scripts/agents_health_check.py index 2bdc121..4f250cc 100644 --- a/gateway/scripts/agents_health_check.py +++ b/gateway/scripts/agents_health_check.py @@ -30,6 +30,7 @@ for svc in _SR: "fix_script": svc["script"], "fix_args": svc["args"], "fix_cwd": svc["workdir"], + "remote": svc.get("remote", False), }) TODO_FILE = os.path.join(TEMP, "health_todos.jsonl") @@ -126,13 +127,17 @@ def main(): entry["pid"] = pid if pid_ok else 0 entry["pid_ok"] = pid_ok - # 2. Port check - port_ok = port_open(svc["port"]) + # 2. Port check（远程服务跳过本地端口检测） + is_remote = svc.get("remote", False) + if is_remote: + port_ok = True # 远程服务不检查本地端口 + else: + port_ok = port_open(svc["port"]) entry["port_ok"] = port_ok - # 3. HTTP health check + # 3. HTTP health check（远程服务直接用 HTTP 判断） http_msg = "" - if port_ok: + if port_ok or is_remote: ok, detail = http_check(svc["health_url"]) # accept 401/403 as healthy if configured if not ok and svc.get("accept_401") and ("401" in detail or "403" in detail): @@ -146,20 +151,21 @@ def main(): entry["http_ok"] = False entry["http_detail"] = "port_closed" - healthy = (pid_ok or not pid_file) and port_ok and entry.get("http_ok", False) + # 健康判定：port+HTTP 都 OK → 健康；PID 仅作为辅助信号，不阻塞判定 + primary_ok = port_ok and entry.get("http_ok", False) + entry["status"] = "ok" if primary_ok else "fail" - if healthy: - entry["status"] = "ok" + if primary_ok: + if not pid_ok: + entry["pid_warning"] = True # PID 异常但服务在正常运行 report["summary"]["ok"] += 1 else: - entry["status"] = "fail" report["summary"]["fail"] += 1 dirty = True # 收集具体失败原因 reasons = [] - if not pid_ok: reasons.append("pid_dead") - if not port_ok: reasons.append("port_closed") - if not entry.get("http_ok") and port_ok: reasons.append(f"http_{http_msg}") + if not port_ok and not is_remote: reasons.append("port_closed") + if not entry.get("http_ok"): reasons.append(f"http_{http_msg}") issue = f"异常: {' + '.join(reasons)}" write_todo(name, issue, svc["fix_script"], svc.get("fix_args", []), svc["fix_cwd"]) @@ -170,7 +176,9 @@ def main(): # 只输出异常服务到 stdout（silent-by-default） for svc in report["services"]: if svc["status"] != "ok": - print(f"[FAIL] {svc['name']}: PID={svc['pid_ok']} PORT={svc['port_ok']} HTTP={svc.get('http_detail','?')}") + print(f"[FAIL] {svc['name']}: PORT={svc['port_ok']} HTTP={svc.get('http_detail','?')}") + elif svc.get("pid_warning"): + print(f"[WARN] {svc['name']}: PID异常但服务正常") else: # 全正常→完全静默（silent-by-default） pass diff --git a/gateway/scripts/service_registry.py b/gateway/scripts/service_registry.py index 2633c24..e78231d 100644 --- a/gateway/scripts/service_registry.py +++ b/gateway/scripts/service_registry.py @@ -63,6 +63,7 @@ SERVICES = [ "health_url": "http://192.168.1.246:8642/v1/health", "accept_401": True, # may need auth "depends_on": ["ejabberd"], + "remote": True, # 服务运行在 Linux 246，不检查本地端口 "log_files": lambda: [], "description": "Hermes Gateway (mohe@yoin.fun)", }, @@ -76,6 +77,7 @@ SERVICES = [ "health_url": "http://192.168.1.246:8643/v1/health", "accept_401": True, "depends_on": ["ejabberd"], + "remote": True, # 服务运行在 Linux 246，不检查本地端口 "log_files": lambda: [], "description": "Hermes Gateway (zhiwei@yoin.fun)", }, diff --git a/gateway/scripts/specs/daily_health.json b/gateway/scripts/specs/daily_health.json new file mode 100644 index 0000000..6a119f0 --- /dev/null +++ b/gateway/scripts/specs/daily_health.json @@ -0,0 +1,46 @@ +{ + "module": "daily_health", + "version": "1.0", + "purpose": "Tier 2 全面健康检查（每日 08:00）。在 Tier 1 基础上增加：磁盘空间检查、定时任务存活检查、数据新鲜度、跨服务依赖链检查、看门狗日志新鲜度。", + "ui_location": "F Tab → 定时任务 → 日报健康检查(每日)", + + "human_help": { + "title": "Tier 2 每日健康检查", + "description": [ + "每天早上 8:00 由 Windows Task Scheduler 触发，生成全面系统健康日报。", + "检查范围：服务状态（PID+端口+HTTP）、磁盘空间（C盘）、定时任务存活、看门狗日志新鲜度。", + "输出结构化 JSON 报告供 Dashboard 和人工审阅。", + "发现异常时生成推荐动作清单。" + ], + "usage": [ + "报告文件：gateway/temp/last_daily_health.json", + "日志：gateway/logs/daily_health_report.log", + "Dashboard 通过 /api/monitor 读取报告显示 Tier2 统计" + ], + "troubleshooting": [ + "如果定时任务未运行：检查 Windows Task Scheduler → agents-daily-health", + "磁盘警告阈值：剩余 < 10GB 警告，< 2GB 严重", + "看门狗日志超过 1 小时未更新 → 可能看门狗已死" + ], + "related": "依赖 service_registry.py（服务定义）; 参考 agents_health_check.py（Tier1 检查逻辑）" + }, + + "ai_spec": { + "dependencies": [ + "service_registry.py — 服务定义", + "Python 3.10", + "Windows Task Scheduler — 每天 08:00 触发", + "wmic — Windows 磁盘空间查询" + ], + "constraints": [ + "只在 Windows 上运行（使用 tasklist/netstat/wmic）", + "磁盘检查仅针对 C: 盘", + "定时任务检查通过 schtasks /Query 验证" + ], + "related_files": [ + "gateway/scripts/agents_daily_health.py — 本脚本", + "gateway/scripts/service_registry.py — 服务注册表", + "gateway/scripts/specs/daily_health.json — 本 spec" + ] + } +} diff --git a/gateway/scripts/specs/health_check.json b/gateway/scripts/specs/health_check.json new file mode 100644 index 0000000..f0088a6 --- /dev/null +++ b/gateway/scripts/specs/health_check.json @@ -0,0 +1,48 @@ +{ + "module": "health_check", + "version": "1.0", + "purpose": "Tier 1 快速健康检查（每5分钟）。对 service_registry 中所有注册服务做三合一检查：PID存活 → 端口监听 → HTTP /health。异常时写 TODO 给 self_todo_executor 消费。正常时静默。", + "ui_location": "F Tab → 定时任务 → 健康检查(5min)", + + "human_help": { + "title": "Tier 1 健康检查", + "description": [ + "每 5 分钟由 Windows Task Scheduler 触发，检查所有注册服务的运行状态。", + "检查项：端口监听（netstat）+ HTTP /health 端点响应。远程服务跳过本地端口检测。", + "PID 检查仅作为辅助信号——端口+HTTP 都正常即判定为健康。", + "异常服务会写入 health_todos.jsonl，由自修复执行器消费。", + "全正常时静默运行，不输出任何噪音。" + ], + "usage": [ + "报告文件：gateway/temp/last_health_check.json", + "TODO 文件：gateway/temp/health_todos.jsonl", + "日志：gateway/logs/health_check_report.log", + "Dashboard 通过 /api/monitor 读取报告显示 Tier1 统计" + ], + "troubleshooting": [ + "如果定时任务未运行：检查 Windows Task Scheduler → agents-health-check", + "如果报告未更新：检查 C:\Users\hmo\AppData\Local\Programs\Python\Python310\python.exe 是否可用", + "如果 TODO 堆积：检查 self_todo_executor 是否正常消费" + ], + "related": "依赖 service_registry.py（服务定义）; 输出给 self_todo_executor.py（修复消费）; 报告被 dashboard.py 读取" + }, + + "ai_spec": { + "dependencies": [ + "service_registry.py — 服务定义（名称/端口/health URL/fix_script）", + "Python 3.10 (C:\\Users\\hmo\\AppData\\Local\\Programs\\Python\\Python310\\python.exe)", + "Windows Task Scheduler — 每5分钟触发" + ], + "constraints": [ + "PID 检查不是硬性失败条件——端口+HTTP OK 即视为健康", + "远程服务（remote:true）跳过本地 netstat 端口检测，直接用 HTTP 判断", + "静默原则：全正常时零输出；仅异常时写 TODO + 打印报告" + ], + "related_files": [ + "gateway/scripts/agents_health_check.py — 本脚本", + "gateway/scripts/service_registry.py — 服务注册表", + "gateway/scripts/self_todo_executor.py — TODO 消费者", + "gateway/scripts/specs/health_check.json — 本 spec" + ] + } +} diff --git a/gateway/scripts/specs/todo_executor.json b/gateway/scripts/specs/todo_executor.json new file mode 100644 index 0000000..51237fa --- /dev/null +++ b/gateway/scripts/specs/todo_executor.json @@ -0,0 +1,47 @@ +{ + "module": "todo_executor", + "version": "1.0", + "purpose": "TODO 自修复执行器（每10分钟）。轮询 health_todos.jsonl 中 status=pending 的条目，执行 fix_script 重启异常服务。成功标记 completed，失败 escalation 到 XMPP 群聊。", + "ui_location": "F Tab → 定时任务 → TODO自修复", + + "human_help": { + "title": "TODO 自修复执行器", + "description": [ + "每 10 分钟由 Windows Task Scheduler 触发，自动修复健康检查发现的异常。", + "读取 health_todos.jsonl 中的 pending 条目，执行对应的 fix_script。", + "修复成功 → 标记 completed。", + "修复失败 → 通过 XMPP HTTP 桥（:5802/send）发送告警到群聊。", + "特殊情况：如果错误信息为 'another instance is already running'，说明服务已在正常运行，标记 completed 跳过。" + ], + "usage": [ + "TODO 文件：gateway/temp/health_todos.jsonl（输入+输出）", + "日志：gateway/logs/todo_executor.log", + "告警通道：HTTP POST → 127.0.0.1:5802/send → XMPP 群聊" + ], + "troubleshooting": [ + "如果 TODO 堆积不减少：检查 agents-todo-executor 定时任务是否运行", + "如果修复失败：检查 fix_script 路径是否存在（远程服务的 fix_script 为 null，无法自动修复）", + "如果告警未发出：检查 xmpp_bot HTTP 桥 :5802/send 是否可达" + ], + "related": "消费 agents_health_check.py 的 TODO 输出; 通过 xmpp_bot HTTP 桥发送告警" + }, + + "ai_spec": { + "dependencies": [ + "health_todos.jsonl — TODO 输入（由 agents_health_check.py 写入）", + "xmpp_bot HTTP 桥 (:5802/send) — 告警输出通道", + "Python 3.10", + "Windows Task Scheduler — 每10分钟触发" + ], + "constraints": [ + "只修复有 fix_script 的服务（远程服务无本地脚本，无法自动修复）", + "already_running 错误视为成功（服务已在运行）", + "单次执行超时 30s — 大脚本可能被截断" + ], + "related_files": [ + "gateway/scripts/self_todo_executor.py — 本脚本", + "gateway/scripts/agents_health_check.py — TODO 生产者", + "gateway/scripts/specs/todo_executor.json — 本 spec" + ] + } +} diff --git a/gateway/scripts/templates/dashboard.html b/gateway/scripts/templates/dashboard.html index cd76590..cf4dea8 100644 --- a/gateway/scripts/templates/dashboard.html +++ b/gateway/scripts/templates/dashboard.html @@ -176,8 +176,9 @@ async function fHealth(){try{ M.dashboard={n:'管理门户',l:'辅助服务',i:'Dashboard无法访问',spec:'dashboard'}; M.watchdog={n:'进程看门狗',l:'辅助服务',i:'异常进程无自动恢复'}; M.ejabberd={n:'XMPP服务器',l:'通信层',i:'所有XMPP连接不可用',spec:'ejabberd'}; - M['agents-health-check']={n:'健康检查(5min)',l:'定时任务',i:'5分钟级健康检测未运行'}; - M['agents-daily-health']={n:'日报健康检查(每日)',l:'定时任务',i:'日报统计未运行'}; + M['agents-health-check']={n:'健康检查(5min)',l:'定时任务',i:'5分钟级健康检测未运行',spec:'health_check'}; + M['agents-daily-health']={n:'日报健康检查(每日)',l:'定时任务',i:'日报统计未运行',spec:'daily_health'}; + M['agents-todo-executor']={n:'TODO自修复(10min)',l:'定时任务',i:'服务异常无法自动恢复',spec:'todo_executor'}; var layerOrder=['通信层','AI网关','辅助服务','定时任务']; var h='

F 系统健康?§

'; @@ -263,11 +264,14 @@ async function fHealth(){try{ h+='

定时任务

'; for(var i=0;i'; h+=''+esc(m.n)+''; + if(ms){h+='?'; + h+='§';} h+=''+(tk?'正常':tn?'未部署':'异常')+'
';} h+='';}