hmo/MoFin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

merge: watchdog agent.log check

Browse Source
This commit is contained in:
知微
2026-07-20 20:56:05 +08:00
parent c971d6bced 10a37f10f9
commit ad6a416ef4
4 changed files with 141 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deploy/profile-scripts/fix_gateway_port.py
+93 -93
View File
@@ -1,93 +1,93 @@
#!/usr/bin/env python3
"""fix_gateway_port.py — 自愈系统调用的网关/XMPP Bot修复脚本
v2: 新增 session 健康检查，检测卡死的 session 自动重启
"""
import subprocess, sys, time, socket, json, urllib.request
GATEWAY_PORT = 8643
BOT_PORT = 5805
BOT_SCRIPT = "/home/hmo/xmpp_zhiwei_bot.py"
API_KEY = "hermes123"
SESSION_ID = "xmpp-zhiwei"
GATEWAY_URL = f"http://127.0.0.1:{GATEWAY_PORT}/v1/chat/completions"
def port_open(port, host="127.0.0.1"):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(2)
try:
r = s.connect_ex((host, port))
return r == 0
finally:
s.close()
def check_session_health():
"""检测 gateway LLM 是否可用——扫 agent.log 最近一次真实调用结果。
不再发真实 LLM ping（25s 超时对 20-100s 的冷启动延迟必误报，且每次白烧 22k token）。
"""
try:
sys.path.insert(0, '/home/hmo/MoFin')
from xmpp_logger import _scan_agent_log
r = _scan_agent_log(time.time(), "zhiwei")
if r["status"] == "ok":
print(f"Session {SESSION_ID} 健康 ✓ (agent.log: latency={r.get('latency')}, {r.get('age_sec')}s前)")
return True
# error/unknown：只有近期有明确失败记录才判不健康
if r["status"] == "error":
print(f"Session {SESSION_ID} 不健康: agent.log 最近调用失败 — {r.get('error','')[:100]}", file=sys.stderr)
return False
# unknown（无近期调用记录）= 空闲，不算不健康
print(f"Session {SESSION_ID} 无近期调用记录（空闲正常）")
return True
except Exception as e:
print(f"Session {SESSION_ID} 健康检查异常: {e}（按健康处理）", file=sys.stderr)
return True
def restart_gateway():
"""通过systemd重启gateway"""
print(f"Gateway 端口{GATEWAY_PORT} 异常 → 重启中...")
subprocess.run(["sudo", "systemctl", "restart", "hermes-gateway-zhiwei.service"],
timeout=30, capture_output=True)
time.sleep(5)
if port_open(GATEWAY_PORT):
print(f"Gateway 已恢复 ✓")
return True
else:
print(f"Gateway 重启后仍不可达", file=sys.stderr)
return False
target = sys.argv[1] if len(sys.argv) > 1 else "all"
if target in ("all", "bot", "xmpp_bot"):
if not port_open(BOT_PORT):
print(f"XMPP bot port {BOT_PORT} CLOSED → 启动")
subprocess.run(["pkill", "-f", "xmpp_zhiwei_bot.py"], timeout=5, capture_output=True)
time.sleep(1)
subprocess.Popen(["python3", BOT_SCRIPT], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
time.sleep(5)
if port_open(BOT_PORT):
print(f"XMPP bot 端口{BOT_PORT} 已打开 ✓")
else:
print(f"XMPP bot 修复后仍不可达", file=sys.stderr)
sys.exit(1)
else:
print(f"XMPP bot 端口{BOT_PORT} 正常 ✓")
if target in ("all", "gateway", "session"):
if not port_open(GATEWAY_PORT):
restart_gateway()
elif target in ("all", "gateway") or target == "session":
# 端口通了 → 进一步检查session健康
if not check_session_health():
print(f"Session {SESSION_ID} 不健康 → 重启gateway")
restart_gateway()
else:
print(f"Gateway 端口{GATEWAY_PORT} 正常 ✓")
if target in ("all", "session"):
# 仅session检查
if port_open(GATEWAY_PORT):
if not check_session_health():
print(f"Session {SESSION_ID} 不健康 → 重启gateway")
restart_gateway()
sys.exit(0)
#!/usr/bin/env python3
"""fix_gateway_port.py — 自愈系统调用的网关/XMPP Bot修复脚本
v2: 新增 session 健康检查，检测卡死的 session 自动重启
"""
import subprocess, sys, time, socket, json, urllib.request
GATEWAY_PORT = 8643
BOT_PORT = 5805
BOT_SCRIPT = "/home/hmo/xmpp_zhiwei_bot.py"
API_KEY = "hermes123"
SESSION_ID = "xmpp-zhiwei"
GATEWAY_URL = f"http://127.0.0.1:{GATEWAY_PORT}/v1/chat/completions"
def port_open(port, host="127.0.0.1"):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(2)
try:
r = s.connect_ex((host, port))
return r == 0
finally:
s.close()
def check_session_health():
"""检测 gateway LLM 是否可用——扫 agent.log 最近一次真实调用结果。
不再发真实 LLM ping（25s 超时对 20-100s 的冷启动延迟必误报，且每次白烧 22k token）。
"""
try:
sys.path.insert(0, '/home/hmo/MoFin')
from xmpp_logger import _scan_agent_log
r = _scan_agent_log(time.time(), "zhiwei")
if r["status"] == "ok":
print(f"Session {SESSION_ID} 健康 ✓ (agent.log: latency={r.get('latency')}, {r.get('age_sec')}s前)")
return True
# error/unknown：只有近期有明确失败记录才判不健康
if r["status"] == "error":
print(f"Session {SESSION_ID} 不健康: agent.log 最近调用失败 — {r.get('error','')[:100]}", file=sys.stderr)
return False
# unknown（无近期调用记录）= 空闲，不算不健康
print(f"Session {SESSION_ID} 无近期调用记录（空闲正常）")
return True
except Exception as e:
print(f"Session {SESSION_ID} 健康检查异常: {e}（按健康处理）", file=sys.stderr)
return True
def restart_gateway():
"""通过systemd重启gateway"""
print(f"Gateway 端口{GATEWAY_PORT} 异常 → 重启中...")
subprocess.run(["sudo", "systemctl", "restart", "hermes-gateway-zhiwei.service"],
timeout=30, capture_output=True)
time.sleep(5)
if port_open(GATEWAY_PORT):
print(f"Gateway 已恢复 ✓")
return True
else:
print(f"Gateway 重启后仍不可达", file=sys.stderr)
return False
target = sys.argv[1] if len(sys.argv) > 1 else "all"
if target in ("all", "bot", "xmpp_bot"):
if not port_open(BOT_PORT):
print(f"XMPP bot port {BOT_PORT} CLOSED → 启动")
subprocess.run(["pkill", "-f", "xmpp_zhiwei_bot.py"], timeout=5, capture_output=True)
time.sleep(1)
subprocess.Popen(["python3", BOT_SCRIPT], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
time.sleep(5)
if port_open(BOT_PORT):
print(f"XMPP bot 端口{BOT_PORT} 已打开 ✓")
else:
print(f"XMPP bot 修复后仍不可达", file=sys.stderr)
sys.exit(1)
else:
print(f"XMPP bot 端口{BOT_PORT} 正常 ✓")
if target in ("all", "gateway", "session"):
if not port_open(GATEWAY_PORT):
restart_gateway()
elif target in ("all", "gateway") or target == "session":
# 端口通了 → 进一步检查session健康
if not check_session_health():
print(f"Session {SESSION_ID} 不健康 → 重启gateway")
restart_gateway()
else:
print(f"Gateway 端口{GATEWAY_PORT} 正常 ✓")
if target in ("all", "session"):
# 仅session检查
if port_open(GATEWAY_PORT):
if not check_session_health():
print(f"Session {SESSION_ID} 不健康 → 重启gateway")
restart_gateway()
sys.exit(0)
scripts/check_error_freshness.py
+25
View File
@@ -0,0 +1,25 @@
import json
from datetime import datetime
print('=== pa profile error jobs ===')
d = json.load(open('/home/hmo/.hermes/profiles/position-analyst/cron/jobs.json'))
jobs = d if isinstance(d, list) else d.get('jobs', [])
for j in jobs:
if j.get('last_status') == 'error':
lr = str(j.get('last_run_at') or '?')[:19]
err = str(j.get('last_error') or '')[:120].replace('\n', ' ')
print(f"{j.get('name')} | last={lr} | {err}")
print()
print('=== default profile error jobs ===')
d2 = json.load(open('/home/hmo/.hermes/cron/jobs.json'))
jobs2 = d2 if isinstance(d2, list) else d2.get('jobs', [])
for j in jobs2:
if j.get('last_status') == 'error':
lr = str(j.get('last_run_at') or '?')[:19]
err = str(j.get('last_error') or '')[:120].replace('\n', ' ')
print(f"{j.get('name')} | last={lr} | {err}")
print()
print('当前时间:', datetime.now().strftime('%Y-%m-%d %H:%M'))
print('硬链接修复时间: 2026-07-20 00:53 (周一凌晨)')
scripts/check_triggered.py
+8
View File
@@ -0,0 +1,8 @@
import json
d = json.load(open('/home/hmo/.hermes/profiles/position-analyst/cron/jobs.json'))
jobs = d if isinstance(d, list) else d.get('jobs', [])
targets = ['策略评估-每周', '建议对账-每周', '数据治理-每周', '跨市场背离检测-周末',
'自选股自动重评-周末', 'state.db真空整理-每周', 'Gateway看门狗-知微']
for j in jobs:
if j.get('name') in targets:
print(f"{j['name']}: status={j.get('last_status')} last={str(j.get('last_run_at'))[:19]} err={str(j.get('last_error'))[:90]}")
scripts/trigger_weekend_jobs.py
+15
View File
@@ -0,0 +1,15 @@
import json, subprocess
d = json.load(open('/home/hmo/.hermes/profiles/position-analyst/cron/jobs.json'))
jobs = d if isinstance(d, list) else d.get('jobs', [])
targets = ['策略评估-每周', '建议对账-每周', '数据治理-每周',
'跨市场背离检测-周末', '自选股自动重评-周末', 'state.db真空整理-每周']
for j in jobs:
if j.get('name') in targets:
jid = j.get('id')
print(f"triggering: {j['name']} (id={jid})")
r = subprocess.run(['/home/hmo/hermes-agent/.venv/bin/python', '-m', 'hermes_cli.main',
'-p', 'position-analyst', 'cron', 'run', jid],
capture_output=True, text=True, timeout=30)
out = (r.stdout + r.stderr).strip()[:150]
print(f' -> {out}')