fix(watchdog): gateway session check now uses agent.log scan, not live LLM ping

Gateway看门狗-知微 was erroring (exit -15): its check_session_health did a
live LLM ping with 25s timeout. Cold-start LLM latency is 20-100s so the
ping always timed out -> false '不健康' verdict -> false gateway restart
-> and each 10-min run burned 22k tokens.

Now uses xmpp_logger._scan_agent_log (zero cost, reads real call results):
- ok if last real call succeeded
- unhealthy only if last call explicitly failed
- idle (no recent calls) counts as healthy
Verified: watchdog job now status=ok.

Also: triggered all 6 weekend 'Blocked' jobs via hermes cron run — all
now status=ok, proving the hardlink fix holds.
This commit is contained in:
hmo
2026-07-20 20:55:30 +08:00
parent c42769f293
commit 10a37f10f9
4 changed files with 66 additions and 25 deletions
+18 -25
View File
@@ -21,33 +21,26 @@ def port_open(port, host="127.0.0.1"):
s.close()
def check_session_health():
"""gateway API,检测session是否卡死。超过15s无响应→不健康"""
"""检测 gateway LLM 是否可用——扫 agent.log 最近一次真实调用结果。
不再发真实 LLM ping25s 超时对 20-100s 的冷启动延迟必误报,且每次白烧 22k token)。
"""
try:
payload = json.dumps({
"model": "hermes-agent",
"messages": [{"role": "user", "content": "ping"}]
}).encode()
req = urllib.request.Request(GATEWAY_URL, data=payload, method="POST")
req.add_header("Content-Type", "application/json")
req.add_header("Authorization", f"Bearer {API_KEY}")
req.add_header("X-Hermes-Session-Id", SESSION_ID)
t0 = time.time()
with urllib.request.urlopen(req, timeout=25) as r:
data = json.loads(r.read())
reply = data.get("choices", [{}])[0].get("message", {}).get("content", "")
elapsed = time.time() - t0
if reply:
print(f"Session {SESSION_ID} 健康 ✓ ({elapsed:.1f}s)")
return True
else:
print(f"Session {SESSION_ID} 返回空", file=sys.stderr)
return False
except urllib.request.HTTPError as e:
print(f"Session {SESSION_ID} HTTP错误: {e.code}", file=sys.stderr)
return False
sys.path.insert(0, '/home/hmo/MoFin')
from xmpp_logger import _scan_agent_log
r = _scan_agent_log(time.time(), "zhiwei")
if r["status"] == "ok":
print(f"Session {SESSION_ID} 健康 ✓ (agent.log: latency={r.get('latency')}, {r.get('age_sec')}s前)")
return True
# error/unknown:只有近期有明确失败记录才判不健康
if r["status"] == "error":
print(f"Session {SESSION_ID} 不健康: agent.log 最近调用失败 — {r.get('error','')[:100]}", file=sys.stderr)
return False
# unknown(无近期调用记录)= 空闲,不算不健康
print(f"Session {SESSION_ID} 无近期调用记录(空闲正常)")
return True
except Exception as e:
print(f"Session {SESSION_ID} 健康: {e}", file=sys.stderr)
return False
print(f"Session {SESSION_ID} 健康检查异常: {e}(按健康处理)", file=sys.stderr)
return True
def restart_gateway():
"""通过systemd重启gateway"""
+25
View File
@@ -0,0 +1,25 @@
import json
from datetime import datetime
print('=== pa profile error jobs ===')
d = json.load(open('/home/hmo/.hermes/profiles/position-analyst/cron/jobs.json'))
jobs = d if isinstance(d, list) else d.get('jobs', [])
for j in jobs:
if j.get('last_status') == 'error':
lr = str(j.get('last_run_at') or '?')[:19]
err = str(j.get('last_error') or '')[:120].replace('\n', ' ')
print(f"{j.get('name')} | last={lr} | {err}")
print()
print('=== default profile error jobs ===')
d2 = json.load(open('/home/hmo/.hermes/cron/jobs.json'))
jobs2 = d2 if isinstance(d2, list) else d2.get('jobs', [])
for j in jobs2:
if j.get('last_status') == 'error':
lr = str(j.get('last_run_at') or '?')[:19]
err = str(j.get('last_error') or '')[:120].replace('\n', ' ')
print(f"{j.get('name')} | last={lr} | {err}")
print()
print('当前时间:', datetime.now().strftime('%Y-%m-%d %H:%M'))
print('硬链接修复时间: 2026-07-20 00:53 (周一凌晨)')
+8
View File
@@ -0,0 +1,8 @@
import json
d = json.load(open('/home/hmo/.hermes/profiles/position-analyst/cron/jobs.json'))
jobs = d if isinstance(d, list) else d.get('jobs', [])
targets = ['策略评估-每周', '建议对账-每周', '数据治理-每周', '跨市场背离检测-周末',
'自选股自动重评-周末', 'state.db真空整理-每周', 'Gateway看门狗-知微']
for j in jobs:
if j.get('name') in targets:
print(f"{j['name']}: status={j.get('last_status')} last={str(j.get('last_run_at'))[:19]} err={str(j.get('last_error'))[:90]}")
+15
View File
@@ -0,0 +1,15 @@
import json, subprocess
d = json.load(open('/home/hmo/.hermes/profiles/position-analyst/cron/jobs.json'))
jobs = d if isinstance(d, list) else d.get('jobs', [])
targets = ['策略评估-每周', '建议对账-每周', '数据治理-每周',
'跨市场背离检测-周末', '自选股自动重评-周末', 'state.db真空整理-每周']
for j in jobs:
if j.get('name') in targets:
jid = j.get('id')
print(f"triggering: {j['name']} (id={jid})")
r = subprocess.run(['/home/hmo/hermes-agent/.venv/bin/python', '-m', 'hermes_cli.main',
'-p', 'position-analyst', 'cron', 'run', jid],
capture_output=True, text=True, timeout=30)
out = (r.stdout + r.stderr).strip()[:150]
print(f' -> {out}')