Phase 1-2: 监控体系全面就绪
- 多进程看门狗: xmpp_watchdog.py (守护 bot + article_processor) - Tier 1 快速健康检查: agents_health_check.py (5min) - Tier 2 全面健康检查: agents_daily_health.py (每日08:00) - TODO 自修复执行器: self_todo_executor.py (10min) - 共享服务注册表: service_registry.py - xmpp_agent_core: /health 免认证 + /send 端点 - Dashboard: SSH检测修复 + kanban版块 + 路径跨平台兼容 - 修复kanban-api-reference.md误导内容 - 修复R02: frp隧道heartbeat保活
This commit is contained in:
+215
-128
@@ -1,174 +1,261 @@
|
||||
"""
|
||||
xmpp_watchdog.py — monitors xmpp_bot, auto-restarts on crash, reports status.
|
||||
Runs alongside xmpp_bot.py as a separate process.
|
||||
xmpp_watchdog.py — monitors xmpp_agent_core + article_processor,
|
||||
auto-restarts on crash, reports status.
|
||||
Runs alongside gateway processes as a separate 30s-loop daemon.
|
||||
"""
|
||||
import os, sys, time, subprocess, json, threading
|
||||
|
||||
PROJECT_ROOT = os.path.dirname(os.path.abspath(__file__))
|
||||
BOT_SCRIPT = os.path.join(PROJECT_ROOT, "xmpp_bot.py")
|
||||
LOG_DIR = os.path.join(os.path.dirname(PROJECT_ROOT), "logs")
|
||||
WATCHDOG_LOG = os.path.join(LOG_DIR, "watchdog.log")
|
||||
PID_FILE = os.path.join(os.path.dirname(PROJECT_ROOT), "temp", ".xmpp_watchdog.pid")
|
||||
BOT_PID_FILE = os.path.join(os.path.dirname(PROJECT_ROOT), "temp", ".xmpp_bot.pid")
|
||||
PYTHON = r"C:\Users\hmo\AppData\Local\Programs\Python\Python310\python.exe"
|
||||
CHECK_INTERVAL = 30 # seconds between health checks
|
||||
import os, sys, time, subprocess, json, urllib.request, urllib.error
|
||||
|
||||
# ── Paths ────────────────────────────────────────────────────
|
||||
PROJECT_ROOT = os.path.dirname(os.path.abspath(__file__)) # gateway/scripts/
|
||||
GATEWAY_DIR = os.path.dirname(PROJECT_ROOT) # gateway/
|
||||
TEMP_DIR = os.path.join(GATEWAY_DIR, "temp")
|
||||
LOG_DIR = os.path.join(GATEWAY_DIR, "logs")
|
||||
os.makedirs(LOG_DIR, exist_ok=True)
|
||||
os.makedirs(TEMP_DIR, exist_ok=True)
|
||||
|
||||
PYTHON = r"C:\Users\hmo\AppData\Local\Programs\Python\Python310\python.exe"
|
||||
CHECK_INTERVAL = 30 # seconds between health checks
|
||||
HEALTH_TIMEOUT = 5 # HTTP health check timeout
|
||||
|
||||
WATCHDOG_LOG = os.path.join(LOG_DIR, "watchdog.log")
|
||||
PID_FILE = os.path.join(TEMP_DIR, ".multi_watchdog.pid")
|
||||
|
||||
|
||||
# ── Service registry（从共享 registry 导入）───────────────────
|
||||
sys.path.insert(0, PROJECT_ROOT)
|
||||
from service_registry import SERVICES as _SR
|
||||
|
||||
SERVICES = {}
|
||||
for svc in _SR:
|
||||
name = svc["name"]
|
||||
SERVICES[name] = {
|
||||
"script": svc["script"],
|
||||
"args": svc["args"],
|
||||
"workdir": svc["workdir"],
|
||||
"pid_file": svc["pid_file"],
|
||||
"port": svc["port"],
|
||||
"health_url": svc["health_url"],
|
||||
"log_files": svc.get("log_files", lambda: [])(),
|
||||
}
|
||||
|
||||
|
||||
# ── Logging ───────────────────────────────────────────────────
|
||||
def wlog(msg: str):
|
||||
ts = time.strftime("%Y-%m-%d %H:%M:%S")
|
||||
line = f"{ts} [watchdog] {msg}\n"
|
||||
with open(WATCHDOG_LOG, "a", encoding="utf-8") as f:
|
||||
f.write(f"{ts} [watchdog] {msg}\n")
|
||||
print(f"[watchdog] {msg}", flush=True)
|
||||
f.write(line)
|
||||
print(line, end="", flush=True)
|
||||
|
||||
|
||||
# ── Log rotation ──────────────────────────────────────────────
|
||||
def rotate_log(path: str, max_bytes: int = 5 * 1024 * 1024):
|
||||
"""Rotate log file if it exceeds max_bytes. Keeps last 3 backups."""
|
||||
try:
|
||||
if not os.path.exists(path):
|
||||
return
|
||||
if os.path.getsize(path) > max_bytes:
|
||||
# shift .2→.tmp, .1→.2, file→.1
|
||||
bak2 = f"{path}.2"
|
||||
bak1 = f"{path}.1"
|
||||
if os.path.exists(bak2): os.remove(bak2)
|
||||
if os.path.exists(bak1): os.rename(bak1, bak2)
|
||||
if os.path.exists(bak2):
|
||||
os.remove(bak2)
|
||||
if os.path.exists(bak1):
|
||||
os.rename(bak1, bak2)
|
||||
os.rename(path, bak1)
|
||||
wlog(f"Rotated: {os.path.basename(path)}")
|
||||
except:
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
# ── Process helpers ───────────────────────────────────────────
|
||||
def is_process_alive(pid: int) -> bool:
|
||||
"""Check if a process with given PID is alive."""
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
['tasklist', '/FI', f'PID eq {pid}', '/NH'],
|
||||
r = subprocess.run(
|
||||
["tasklist", "/FI", f"PID eq {pid}", "/NH"],
|
||||
capture_output=True, text=True, timeout=5
|
||||
)
|
||||
return str(pid) in proc.stdout
|
||||
return str(pid) in r.stdout
|
||||
except:
|
||||
return False
|
||||
|
||||
|
||||
def kill_bot():
|
||||
"""Kill ALL existing xmpp_bot.py processes before starting a new one."""
|
||||
killed = 0
|
||||
def get_pid_from_file(path: str) -> int:
|
||||
try:
|
||||
r = subprocess.run(
|
||||
['tasklist', '/FO', 'CSV', '/NH', '/FI', 'IMAGENAME eq python.exe'],
|
||||
capture_output=True, text=True, timeout=10
|
||||
)
|
||||
for line in r.stdout.splitlines():
|
||||
parts = line.strip('"').split('","')
|
||||
if len(parts) >= 2 and parts[0] == 'python.exe':
|
||||
pid_str = parts[1].strip()
|
||||
try:
|
||||
wmi = subprocess.run(
|
||||
['wmic', 'process', 'where', f'ProcessId={pid_str}',
|
||||
'get', 'CommandLine', '/format:list'],
|
||||
capture_output=True, text=True, timeout=5
|
||||
)
|
||||
if 'xmpp_bot' in wmi.stdout and 'watchdog' not in wmi.stdout:
|
||||
subprocess.run(['taskkill', '/f', '/pid', pid_str],
|
||||
capture_output=True, timeout=5)
|
||||
killed += 1
|
||||
wlog(f"Killed old bot (PID {pid_str})")
|
||||
except:
|
||||
pass
|
||||
except:
|
||||
pass
|
||||
if killed > 0:
|
||||
time.sleep(3) # wait for process cleanup
|
||||
|
||||
def start_bot() -> int:
|
||||
"""Start xmpp_bot.py and return its PID. Kills old instances first."""
|
||||
kill_bot()
|
||||
wlog("Starting xmpp_bot...")
|
||||
proc = subprocess.Popen(
|
||||
[PYTHON, BOT_SCRIPT],
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
creationflags=subprocess.CREATE_NO_WINDOW
|
||||
)
|
||||
pid = proc.pid
|
||||
with open(BOT_PID_FILE, "w") as f:
|
||||
f.write(str(pid))
|
||||
wlog(f"xmpp_bot started (PID {pid})")
|
||||
return pid
|
||||
|
||||
|
||||
def get_last_log_activity() -> float:
|
||||
"""Get timestamp of last xmpp_bot.log modification."""
|
||||
log_file = os.path.join(LOG_DIR, "xmpp_bot.log")
|
||||
try:
|
||||
return os.path.getmtime(log_file)
|
||||
with open(path) as f:
|
||||
return int(f.read().strip())
|
||||
except:
|
||||
return 0
|
||||
|
||||
|
||||
def health_check(bot_pid: int, last_activity: float) -> tuple[bool, int, float]:
|
||||
def is_port_listening(port: int) -> bool:
|
||||
"""Quick port liveness check via netstat."""
|
||||
try:
|
||||
r = subprocess.run(
|
||||
["netstat", "-ano"],
|
||||
capture_output=True, text=True, timeout=5
|
||||
)
|
||||
return any(f":{port} " in line and "LISTENING" in line
|
||||
for line in r.stdout.splitlines())
|
||||
except:
|
||||
return False
|
||||
|
||||
|
||||
def check_http_health(url: str, timeout: int = HEALTH_TIMEOUT) -> tuple[bool, str]:
|
||||
"""Returns (ok, detail)."""
|
||||
try:
|
||||
r = urllib.request.urlopen(url, timeout=timeout)
|
||||
body = r.read().decode("utf-8", errors="replace")[:200]
|
||||
return (r.status == 200, body)
|
||||
except urllib.error.HTTPError as e:
|
||||
return (False, f"HTTP {e.code}")
|
||||
except urllib.error.URLError as e:
|
||||
return (False, str(e.reason)[:60])
|
||||
except Exception as e:
|
||||
return (False, str(e)[:60])
|
||||
|
||||
|
||||
# ── Process lifecycle ────────────────────────────────────────
|
||||
def kill_service(name: str, script_match: str):
|
||||
"""Kill ALL processes whose command line contains script_match."""
|
||||
killed = 0
|
||||
try:
|
||||
r = subprocess.run(
|
||||
["tasklist", "/FO", "CSV", "/NH", "/FI", "IMAGENAME eq python.exe"],
|
||||
capture_output=True, text=True, timeout=10
|
||||
)
|
||||
for line in r.stdout.splitlines():
|
||||
parts = line.strip('"').split('","')
|
||||
if len(parts) < 2 or parts[0] != "python.exe":
|
||||
continue
|
||||
pid_str = parts[1].strip()
|
||||
try:
|
||||
wmi = subprocess.run(
|
||||
["wmic", "process", "where", f"ProcessId={pid_str}",
|
||||
"get", "CommandLine", "/format:list"],
|
||||
capture_output=True, text=True, timeout=5
|
||||
)
|
||||
if script_match in wmi.stdout:
|
||||
subprocess.run(["taskkill", "/f", "/pid", pid_str],
|
||||
capture_output=True, timeout=5)
|
||||
killed += 1
|
||||
wlog(f"Killed old {name} (PID {pid_str})")
|
||||
except:
|
||||
pass
|
||||
if killed > 0:
|
||||
time.sleep(3)
|
||||
except:
|
||||
pass
|
||||
return killed
|
||||
|
||||
|
||||
def start_service(name: str, cfg: dict) -> int:
|
||||
"""Start service and return PID. Kills old instances first."""
|
||||
kill_service(name, os.path.basename(cfg["script"]))
|
||||
wlog(f"Starting {name}...")
|
||||
proc = subprocess.Popen(
|
||||
[PYTHON, cfg["script"]] + cfg["args"],
|
||||
cwd=cfg["workdir"],
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
creationflags=subprocess.CREATE_NO_WINDOW,
|
||||
)
|
||||
pid = proc.pid
|
||||
with open(cfg["pid_file"], "w") as f:
|
||||
f.write(str(pid))
|
||||
wlog(f"{name} started (PID {pid})")
|
||||
return pid
|
||||
|
||||
|
||||
# ── Service health check ─────────────────────────────────────
|
||||
def check_service(name: str, cfg: dict, pid: int) -> tuple[bool, int, str]:
|
||||
"""
|
||||
Check bot health.
|
||||
Returns (is_alive, pid, last_activity).
|
||||
If dead, restarts bot.
|
||||
Returns (is_healthy, new_pid, status_msg).
|
||||
If dead, auto-restarts.
|
||||
"""
|
||||
alive = is_process_alive(bot_pid)
|
||||
|
||||
alive = is_process_alive(pid)
|
||||
|
||||
if not alive:
|
||||
wlog(f"Bot PID {bot_pid} is DEAD. Restarting...")
|
||||
bot_pid = start_bot()
|
||||
wlog(f"{name} PID {pid} is DEAD. Restarting...")
|
||||
new_pid = start_service(name, cfg)
|
||||
time.sleep(5)
|
||||
last_activity = get_last_log_activity()
|
||||
return (True, bot_pid, last_activity)
|
||||
|
||||
# Check if bot has been active recently (last 5 minutes)
|
||||
current_activity = get_last_log_activity()
|
||||
if current_activity > last_activity:
|
||||
last_activity = current_activity
|
||||
|
||||
# If no activity for 5 minutes but bot is alive, warn
|
||||
if time.time() - last_activity > 300:
|
||||
wlog(f"WARNING: Bot PID {bot_pid} alive but no activity for 5+ min")
|
||||
|
||||
return (True, bot_pid, last_activity)
|
||||
return (True, new_pid, "restarted")
|
||||
|
||||
# Port-level check
|
||||
port_ok = is_port_listening(cfg["port"])
|
||||
if not port_ok:
|
||||
wlog(f"{name} PID {pid} alive but port {cfg['port']} not listening. Restarting...")
|
||||
new_pid = start_service(name, cfg)
|
||||
time.sleep(5)
|
||||
return (True, new_pid, "restarted (port dead)")
|
||||
|
||||
# HTTP health check
|
||||
http_ok, detail = check_http_health(cfg["health_url"])
|
||||
if not http_ok:
|
||||
# 401/403 = service alive but requires auth → treat as healthy
|
||||
if "HTTP 401" in detail or "HTTP 403" in detail:
|
||||
wlog(f"{name} PID {pid} /health {detail} (auth required, service alive)")
|
||||
else:
|
||||
wlog(f"{name} PID {pid} /health FAILED: {detail}. Restarting...")
|
||||
new_pid = start_service(name, cfg)
|
||||
time.sleep(5)
|
||||
return (True, new_pid, f"restarted (/health: {detail})")
|
||||
|
||||
return (True, pid, "ok")
|
||||
|
||||
|
||||
# ── Main ──────────────────────────────────────────────────────
|
||||
if __name__ == "__main__":
|
||||
wlog("Watchdog started")
|
||||
|
||||
# Start bot if not already running
|
||||
bot_pid = 0
|
||||
if os.path.exists(BOT_PID_FILE):
|
||||
try:
|
||||
with open(BOT_PID_FILE) as f:
|
||||
bot_pid = int(f.read().strip())
|
||||
if not is_process_alive(bot_pid):
|
||||
bot_pid = 0
|
||||
except:
|
||||
bot_pid = 0
|
||||
|
||||
if bot_pid == 0:
|
||||
bot_pid = start_bot()
|
||||
|
||||
last_activity = get_last_log_activity()
|
||||
wlog(f"Initial: bot PID {bot_pid}, log last activity: {time.ctime(last_activity)}")
|
||||
|
||||
wlog("=== Multi-Watchdog started ===")
|
||||
|
||||
# Discover / start all services
|
||||
pids = {}
|
||||
statuses = {}
|
||||
for name, cfg in SERVICES.items():
|
||||
pid = get_pid_from_file(cfg["pid_file"])
|
||||
if pid and is_process_alive(pid) and is_port_listening(cfg["port"]):
|
||||
wlog(f"{name}: found running (PID {pid})")
|
||||
else:
|
||||
pid = start_service(name, cfg)
|
||||
pids[name] = pid
|
||||
statuses[name] = "starting"
|
||||
time.sleep(2)
|
||||
|
||||
# Track "last good" per-service for activity monitoring
|
||||
last_http_ok = {name: time.time() for name in SERVICES}
|
||||
|
||||
log_rotate_counter = 0
|
||||
|
||||
# Main monitoring loop
|
||||
cycle = 0
|
||||
|
||||
while True:
|
||||
time.sleep(CHECK_INTERVAL)
|
||||
alive, bot_pid, last_activity = health_check(bot_pid, last_activity)
|
||||
|
||||
# Log rotation (every 30 checks ≈ 15 min)
|
||||
cycle += 1
|
||||
log_rotate_counter += 1
|
||||
|
||||
any_restart = False
|
||||
|
||||
for name, cfg in SERVICES.items():
|
||||
ok, new_pid, msg = check_service(name, cfg, pids[name])
|
||||
if new_pid != pids[name]:
|
||||
any_restart = True
|
||||
pids[name] = new_pid
|
||||
statuses[name] = msg
|
||||
else:
|
||||
statuses[name] = msg
|
||||
if ok:
|
||||
last_http_ok[name] = time.time()
|
||||
|
||||
# Log rotation every 30 checks ≈ 15 min
|
||||
if log_rotate_counter >= 30:
|
||||
log_rotate_counter = 0
|
||||
bot_log = os.path.join(LOG_DIR, "xmpp_bot.log")
|
||||
bridge_log = os.path.join(LOG_DIR, "bridge.log")
|
||||
rotate_log(bot_log)
|
||||
rotate_log(bridge_log)
|
||||
|
||||
# Every 5 minutes, report status
|
||||
if int(time.time()) % 300 < CHECK_INTERVAL:
|
||||
alive_str = "ALIVE" if alive else "RESTARTED"
|
||||
wlog(f"Status: bot PID {bot_pid} [{alive_str}]")
|
||||
for name, cfg in SERVICES.items():
|
||||
for log_path in cfg["log_files"]:
|
||||
rotate_log(log_path)
|
||||
rotate_log(WATCHDOG_LOG)
|
||||
|
||||
# Status report every 10 cycles (5 min)
|
||||
if cycle % 10 == 0 or any_restart:
|
||||
parts = []
|
||||
for name in SERVICES:
|
||||
p = pids[name]
|
||||
s = statuses[name]
|
||||
parts.append(f"{name} PID {p} [{s}]")
|
||||
wlog(" | ".join(parts))
|
||||
|
||||
Reference in New Issue
Block a user