From c971d6bcedc0fd4ccc9ce7c84176738ac3a47447 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=9F=A5=E5=BE=AE?= Date: Mon, 20 Jul 2026 20:55:34 +0800 Subject: [PATCH] chore: watchdog fix --- data/price_history.json | 10 + deploy/profile-scripts/fix_gateway_port.py | 193 ++++++++-------- static/mofin_health.json | 253 +++++++++++---------- 3 files changed, 230 insertions(+), 226 deletions(-) diff --git a/data/price_history.json b/data/price_history.json index 70b2cc94..ef626a12 100644 --- a/data/price_history.json +++ b/data/price_history.json @@ -1844,5 +1844,15 @@ "volume": 2832553.0, "amount": 9286.0 } + ], + "000711": [ + { + "date": "2026-07-20", + "high": 4.55, + "low": 4.21, + "close": 4.32, + "volume": 728146.0, + "amount": 31883.0 + } ] } \ No newline at end of file diff --git a/deploy/profile-scripts/fix_gateway_port.py b/deploy/profile-scripts/fix_gateway_port.py index 2d7b0996..63e9122d 100644 --- a/deploy/profile-scripts/fix_gateway_port.py +++ b/deploy/profile-scripts/fix_gateway_port.py @@ -1,100 +1,93 @@ -#!/usr/bin/env python3 -"""fix_gateway_port.py — 自愈系统调用的网关/XMPP Bot修复脚本 -v2: 新增 session 健康检查，检测卡死的 session 自动重启 -""" -import subprocess, sys, time, socket, json, urllib.request - -GATEWAY_PORT = 8643 -BOT_PORT = 5805 -BOT_SCRIPT = "/home/hmo/xmpp_zhiwei_bot.py" -API_KEY = "hermes123" -SESSION_ID = "xmpp-zhiwei" -GATEWAY_URL = f"http://127.0.0.1:{GATEWAY_PORT}/v1/chat/completions" - -def port_open(port, host="127.0.0.1"): - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.settimeout(2) - try: - r = s.connect_ex((host, port)) - return r == 0 - finally: - s.close() - -def check_session_health(): - """调gateway API，检测session是否卡死。超过15s无响应→不健康""" - try: - payload = json.dumps({ - "model": "hermes-agent", - "messages": [{"role": "user", "content": "ping"}] - }).encode() - req = urllib.request.Request(GATEWAY_URL, data=payload, method="POST") - req.add_header("Content-Type", "application/json") - req.add_header("Authorization", f"Bearer {API_KEY}") - req.add_header("X-Hermes-Session-Id", SESSION_ID) - t0 = time.time() - with urllib.request.urlopen(req, timeout=25) as r: - data = json.loads(r.read()) - reply = data.get("choices", [{}])[0].get("message", {}).get("content", "") - elapsed = time.time() - t0 - if reply: - print(f"Session {SESSION_ID} 健康 ✓ ({elapsed:.1f}s)") - return True - else: - print(f"Session {SESSION_ID} 返回空", file=sys.stderr) - return False - except urllib.request.HTTPError as e: - print(f"Session {SESSION_ID} HTTP错误: {e.code}", file=sys.stderr) - return False - except Exception as e: - print(f"Session {SESSION_ID} 不健康: {e}", file=sys.stderr) - return False - -def restart_gateway(): - """通过systemd重启gateway""" - print(f"Gateway 端口{GATEWAY_PORT} 异常 → 重启中...") - subprocess.run(["sudo", "systemctl", "restart", "hermes-gateway-zhiwei.service"], - timeout=30, capture_output=True) - time.sleep(5) - if port_open(GATEWAY_PORT): - print(f"Gateway 已恢复 ✓") - return True - else: - print(f"Gateway 重启后仍不可达", file=sys.stderr) - return False - -target = sys.argv[1] if len(sys.argv) > 1 else "all" - -if target in ("all", "bot", "xmpp_bot"): - if not port_open(BOT_PORT): - print(f"XMPP bot port {BOT_PORT} CLOSED → 启动") - subprocess.run(["pkill", "-f", "xmpp_zhiwei_bot.py"], timeout=5, capture_output=True) - time.sleep(1) - subprocess.Popen(["python3", BOT_SCRIPT], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - time.sleep(5) - if port_open(BOT_PORT): - print(f"XMPP bot 端口{BOT_PORT} 已打开 ✓") - else: - print(f"XMPP bot 修复后仍不可达", file=sys.stderr) - sys.exit(1) - else: - print(f"XMPP bot 端口{BOT_PORT} 正常 ✓") - -if target in ("all", "gateway", "session"): - if not port_open(GATEWAY_PORT): - restart_gateway() - elif target in ("all", "gateway") or target == "session": - # 端口通了 → 进一步检查session健康 - if not check_session_health(): - print(f"Session {SESSION_ID} 不健康 → 重启gateway") - restart_gateway() - else: - print(f"Gateway 端口{GATEWAY_PORT} 正常 ✓") - -if target in ("all", "session"): - # 仅session检查 - if port_open(GATEWAY_PORT): - if not check_session_health(): - print(f"Session {SESSION_ID} 不健康 → 重启gateway") - restart_gateway() - -sys.exit(0) +#!/usr/bin/env python3 +"""fix_gateway_port.py — 自愈系统调用的网关/XMPP Bot修复脚本 +v2: 新增 session 健康检查，检测卡死的 session 自动重启 +""" +import subprocess, sys, time, socket, json, urllib.request + +GATEWAY_PORT = 8643 +BOT_PORT = 5805 +BOT_SCRIPT = "/home/hmo/xmpp_zhiwei_bot.py" +API_KEY = "hermes123" +SESSION_ID = "xmpp-zhiwei" +GATEWAY_URL = f"http://127.0.0.1:{GATEWAY_PORT}/v1/chat/completions" + +def port_open(port, host="127.0.0.1"): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(2) + try: + r = s.connect_ex((host, port)) + return r == 0 + finally: + s.close() + +def check_session_health(): + """检测 gateway LLM 是否可用——扫 agent.log 最近一次真实调用结果。 + 不再发真实 LLM ping（25s 超时对 20-100s 的冷启动延迟必误报，且每次白烧 22k token）。 + """ + try: + sys.path.insert(0, '/home/hmo/MoFin') + from xmpp_logger import _scan_agent_log + r = _scan_agent_log(time.time(), "zhiwei") + if r["status"] == "ok": + print(f"Session {SESSION_ID} 健康 ✓ (agent.log: latency={r.get('latency')}, {r.get('age_sec')}s前)") + return True + # error/unknown：只有近期有明确失败记录才判不健康 + if r["status"] == "error": + print(f"Session {SESSION_ID} 不健康: agent.log 最近调用失败 — {r.get('error','')[:100]}", file=sys.stderr) + return False + # unknown（无近期调用记录）= 空闲，不算不健康 + print(f"Session {SESSION_ID} 无近期调用记录（空闲正常）") + return True + except Exception as e: + print(f"Session {SESSION_ID} 健康检查异常: {e}（按健康处理）", file=sys.stderr) + return True + +def restart_gateway(): + """通过systemd重启gateway""" + print(f"Gateway 端口{GATEWAY_PORT} 异常 → 重启中...") + subprocess.run(["sudo", "systemctl", "restart", "hermes-gateway-zhiwei.service"], + timeout=30, capture_output=True) + time.sleep(5) + if port_open(GATEWAY_PORT): + print(f"Gateway 已恢复 ✓") + return True + else: + print(f"Gateway 重启后仍不可达", file=sys.stderr) + return False + +target = sys.argv[1] if len(sys.argv) > 1 else "all" + +if target in ("all", "bot", "xmpp_bot"): + if not port_open(BOT_PORT): + print(f"XMPP bot port {BOT_PORT} CLOSED → 启动") + subprocess.run(["pkill", "-f", "xmpp_zhiwei_bot.py"], timeout=5, capture_output=True) + time.sleep(1) + subprocess.Popen(["python3", BOT_SCRIPT], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + time.sleep(5) + if port_open(BOT_PORT): + print(f"XMPP bot 端口{BOT_PORT} 已打开 ✓") + else: + print(f"XMPP bot 修复后仍不可达", file=sys.stderr) + sys.exit(1) + else: + print(f"XMPP bot 端口{BOT_PORT} 正常 ✓") + +if target in ("all", "gateway", "session"): + if not port_open(GATEWAY_PORT): + restart_gateway() + elif target in ("all", "gateway") or target == "session": + # 端口通了 → 进一步检查session健康 + if not check_session_health(): + print(f"Session {SESSION_ID} 不健康 → 重启gateway") + restart_gateway() + else: + print(f"Gateway 端口{GATEWAY_PORT} 正常 ✓") + +if target in ("all", "session"): + # 仅session检查 + if port_open(GATEWAY_PORT): + if not check_session_health(): + print(f"Session {SESSION_ID} 不健康 → 重启gateway") + restart_gateway() + +sys.exit(0) diff --git a/static/mofin_health.json b/static/mofin_health.json index 9ccb1d84..9960566d 100644 --- a/static/mofin_health.json +++ b/static/mofin_health.json @@ -1,5 +1,5 @@ { - "generated_at": "2026-07-20 20:30:35", + "generated_at": "2026-07-20 20:45:45", "feature_tree": { "label": "MoFin 系统", "status": "ok", @@ -170,7 +170,7 @@ "script": "collect_evaluation_data.py", "schedule": "30 20 * * 1-5", "status": "ok", - "last_run": "2026-07-17T20:30", + "last_run": "2026-07-20T20:30", "type": "no_agent", "profile": "position-analyst" } @@ -537,7 +537,7 @@ "script": "mofin_health.py", "schedule": "*/15 9-16,20-22 * * 1-5", "status": "ok", - "last_run": "2026-07-20T20:15", + "last_run": "2026-07-20T20:30", "type": "no_agent", "profile": "position-analyst" } @@ -624,7 +624,7 @@ "script": "self_todo_executor.py", "schedule": "*/10 8-22 * * 1-5", "status": "ok", - "last_run": "2026-07-20T20:20", + "last_run": "2026-07-20T20:40", "type": "no_agent", "profile": "position-analyst" } @@ -885,7 +885,7 @@ "script": "fix_gateway_port.py", "schedule": "every 10m", "status": "ok", - "last_run": "2026-07-20T20:25", + "last_run": "2026-07-20T20:36", "type": "no_agent", "profile": "position-analyst" } @@ -910,14 +910,14 @@ { "label": "功能健康检查-L1 (functional_health_check.py)", "desc": "", - "status": null, + "status": "ok", "pipes": [ { "name": "功能健康检查-L1", "script": "functional_health_check.py", "schedule": "*/15 9-16,20-22 * * 1-5", - "status": null, - "last_run": "", + "status": "ok", + "last_run": "2026-07-20T20:30", "type": "no_agent", "profile": "position-analyst" } @@ -926,14 +926,14 @@ { "label": "LLM修复循环-L3 (self_repair.py)", "desc": "", - "status": null, + "status": "ok", "pipes": [ { "name": "LLM修复循环-L3", "script": "self_repair.py", "schedule": "*/30 9-16,20-22 * * 1-5", - "status": null, - "last_run": "", + "status": "ok", + "last_run": "2026-07-20T20:30", "type": "no_agent", "profile": "position-analyst" } @@ -1028,8 +1028,8 @@ "desc": "建议执行时间线", "rows": 12735, "readers": [ - "advice_reconciliation", - "mofin_db" + "mofin_db", + "advice_reconciliation" ], "writers": [ "advice_reconciliation" @@ -1086,10 +1086,10 @@ "mofin_db" ], "writers": [ - "market_scanner", "promote_candidates", "candidate_filter", - "accumulation_scanner" + "accumulation_scanner", + "market_scanner" ], "has_input": true, "has_output": true, @@ -1114,8 +1114,8 @@ "readers": [ "per_stock_reassess", "merge_third_db", - "mofin_db", - "batch_reassess" + "batch_reassess", + "mofin_db" ], "writers": [ "merge_third_db", @@ -1141,8 +1141,8 @@ "desc": "资金变动记录", "rows": 20, "readers": [ - "mofin_db", - "prepare_report_data" + "prepare_report_data", + "mofin_db" ], "writers": [ "mo_data", @@ -1195,25 +1195,25 @@ "desc": "每只股票的完整策略参数", "rows": 227, "readers": [ + "promote_candidates", + "candidate_filter", + "pre-flight-check", + "strategy_lifecycle", + "stale_push_wlin", + "verify_300308", + "mofin_collect", "capital_flow_collector", "data_governance", - "batch_reassess", - "mofin_db", - "accumulation_scanner", - "run_all_tests", - "generate_report", - "promote_candidates", - "price_monitor", - "sync_decisions_to_db" + "per_stock_reassess" ], "writers": [ - "per_stock_reassess", - "watchlist_auto_exit", - "data_governance", "promote_candidates", + "mofin_db", "sync_decisions_to_db", "batch_reassess", - "mofin_db" + "data_governance", + "per_stock_reassess", + "watchlist_auto_exit" ], "has_input": true, "has_output": true, @@ -1241,21 +1241,21 @@ "desc": "当前持仓（权威源）", "rows": 14, "readers": [ - "capital_flow_collector", - "server", - "xiaoguo_signal_consumer", "prepare_report_data", - "mofin_db", + "candidate_filter", + "strategy_lifecycle", + "trend_detector", + "stale_push_wlin", "refresh_mtf_cache", - "accumulation_scanner", - "run_all_tests", + "mofin_collect", "mo_alphasift_bridge", - "price_monitor" + "capital_flow_collector", + "xiaoguo_signal_consumer" ], "writers": [ "price_monitor", - "import_holding_xls", - "mofin_db" + "mofin_db", + "import_holding_xls" ], "has_input": true, "has_output": true, @@ -1285,12 +1285,12 @@ "readers": [ "candidate_filter", "cron_health_monitor", - "generate_report", - "system_audit", - "stale_push_wlin", + "mofin_db", "verify_reassess_pipeline", "mo_data", - "mofin_db" + "generate_report", + "system_audit", + "stale_push_wlin" ], "writers": [ "mo_data", @@ -1321,15 +1321,15 @@ "desc": "宏观上下文（大盘偏向/指数）", "rows": 81, "readers": [ - "strategy_lifecycle", - "per_stock_reassess", - "stock_profile", "divergence_detector", - "system_audit", - "stale_push_wlin", + "strategy_tree", + "stock_profile", "batch_reassess", "xiaoguo_signal_consumer", - "strategy_tree" + "per_stock_reassess", + "strategy_lifecycle", + "system_audit", + "stale_push_wlin" ], "writers": [ "refresh_macro_context" @@ -1384,14 +1384,14 @@ "desc": "大盘指数快照（每10分）", "rows": 989, "readers": [ - "mofin_query", - "mofin_db", "prepare_report_data", - "market_screener", + "mofin_db", "market_scanner", "system_audit", + "merge_third_db", "trend_detector", - "merge_third_db" + "mofin_query", + "market_screener" ], "writers": [ "merge_third_db", @@ -1420,8 +1420,8 @@ "desc": "多周期均线缓存", "rows": 64, "readers": [ - "technical_analysis", "multi_timeframe", + "technical_analysis", "mofin_db" ], "writers": [ @@ -1451,18 +1451,18 @@ "desc": "总资产/现金/仓位汇总", "rows": 1, "readers": [ - "mofin_db", "prepare_report_data", - "import_holding_xls", - "price_monitor", + "mofin_db", "mo_data", "preflight_verify", - "check_db_state" + "check_db_state", + "import_holding_xls", + "price_monitor" ], "writers": [ "price_monitor", - "import_holding_xls", - "mofin_db" + "mofin_db", + "import_holding_xls" ], "has_input": true, "has_output": true, @@ -1489,21 +1489,21 @@ "desc": "价格区间突破事件日志", "rows": 6353, "readers": [ - "check_price_events", - "test_raw_insert", - "mofin_db", "test_dual_write", - "test_db_only", + "mofin_db", "test_db_write", - "backfill_price_events" + "backfill_price_events", + "check_price_events", + "test_db_only", + "test_raw_insert" ], "writers": [ - "test_raw_insert", "test_dual_write", "mofin_db", - "test_db_only", "test_db_write", - "backfill_price_events" + "backfill_price_events", + "test_db_only", + "test_raw_insert" ], "has_input": true, "has_output": true, @@ -1525,15 +1525,15 @@ "desc": "行业信号（趋势检测产出）", "rows": 653, "readers": [ - "mofin_news", "xiaoguo_news_processor", + "server", "trend_detector", - "server" + "mofin_news" ], "writers": [ - "mofin_news", "xiaoguo_news_processor", - "trend_detector" + "trend_detector", + "mofin_news" ], "has_input": true, "has_output": true, @@ -1560,13 +1560,13 @@ "desc": "行业板块数据", "rows": 68408, "readers": [ - "strategy_lifecycle", "mofin_db", - "market_screener", "market_scanner", + "strategy_lifecycle", + "merge_third_db", "trend_detector", "inspect_third_db", - "merge_third_db" + "market_screener" ], "writers": [ "merge_third_db", @@ -1597,20 +1597,20 @@ "readers": [ "intraday_health_check", "server", - "per_stock_reassess", - "system_audit", "batch_reassess", "xiaoguo_signal_consumer", + "per_stock_reassess", + "system_audit", "macro_signal_consumer" ], "writers": [ + "divergence_detector", "mofin_news", "xiaoguo_news_processor", - "divergence_detector", - "xiaoguo_signal_consumer", "macro_context_collector", - "macro_signal_consumer", - "xiaoguo_scanner" + "xiaoguo_signal_consumer", + "xiaoguo_scanner", + "macro_signal_consumer" ], "has_input": true, "has_output": true, @@ -1726,12 +1726,12 @@ "desc": "股票行业映射", "rows": 64, "readers": [ - "strategy_lifecycle", "mofin_news", - "per_stock_reassess", + "mofin_db", "xiaoguo_news_processor", - "trend_detector", - "mofin_db" + "per_stock_reassess", + "strategy_lifecycle", + "trend_detector" ], "writers": [ "mofin_db" @@ -1776,19 +1776,19 @@ "desc": "全量股票代码", "rows": 5575, "readers": [ - "accumulation_scanner", - "mofin_db", "mofin_news", + "mofin_db", "xiaoguo_news_processor", - "import_full_stocks", + "accumulation_scanner", "trend_detector", + "import_full_stocks", "check_stocks_table" ], "writers": [ "price_monitor", - "backfill_price_events", + "import_full_stocks", "mofin_db", - "import_full_stocks" + "backfill_price_events" ], "has_input": true, "has_output": true, @@ -1813,8 +1813,8 @@ "desc": "策略重评历史记录", "rows": 217, "readers": [ - "mofin_db", - "verify_reassess_pipeline" + "verify_reassess_pipeline", + "mofin_db" ], "writers": [ "mofin_collect" @@ -1868,20 +1868,20 @@ "desc": "自愈任务队列", "rows": 96, "readers": [ - "strategy-staleness-check", "intraday_health_check", - "self_todo_executor", + "strategy-staleness-check", "merge_third_db", + "self_todo_executor", "morning_health_check" ], "writers": [ "intraday_health_check", - "strategy-staleness-check", "cron_health_monitor", - "self_todo_executor", "mofin_collect", + "strategy-staleness-check", "preflight_verify", "merge_third_db", + "self_todo_executor", "morning_health_check" ], "has_input": true, @@ -1930,15 +1930,15 @@ "rows": 69, "readers": [ "refresh_mtf_cache", - "run_all_tests", - "mo_alphasift_bridge", - "per_stock_reassess", - "stock_quote", - "price_monitor", - "trend_detector", - "stale_push_wlin", "mofin_collect", - "xiaoguo_signal_consumer" + "mofin_db", + "mo_alphasift_bridge", + "stock_quote", + "run_all_tests", + "per_stock_reassess", + "xiaoguo_signal_consumer", + "xiaoguo_scanner", + "trend_detector" ], "writers": [ "per_stock_reassess", @@ -2078,7 +2078,7 @@ "size_kb": 0.3, "readers": [], "writers": [], - "last_modified": "07-20 15:31", + "last_modified": "07-20 20:45", "warn": true, "migrated_to_db": null }, @@ -2105,14 +2105,15 @@ { "name": "mofin_health.json", "desc": "健康监控数据", - "size_kb": 89.2, + "size_kb": 89.3, "readers": [ "analyze_health", "verify_health_json", - "verify_self_check_section" + "verify_self_check_section", + "verify_deployment" ], "writers": [], - "last_modified": "07-20 20:18", + "last_modified": "07-20 20:30", "warn": false, "migrated_to_db": null }, @@ -2244,7 +2245,7 @@ "script": "fix_gateway_port.py", "schedule": "every 10m", "status": "ok", - "last_run": "2026-07-20T20:25:58", + "last_run": "2026-07-20T20:36:59", "profile": "position-analyst" }, { @@ -2252,8 +2253,8 @@ "type": "no_agent", "script": "self_repair.py", "schedule": "*/30 9-16,20-22 * * 1-5", - "status": null, - "last_run": "None", + "status": "ok", + "last_run": "2026-07-20T20:30:36", "profile": "position-analyst" }, { @@ -2361,7 +2362,7 @@ "script": "mofin_health.py", "schedule": "*/15 9-16,20-22 * * 1-5", "status": "ok", - "last_run": "2026-07-20T20:15:38", + "last_run": "2026-07-20T20:30:46", "profile": "position-analyst" }, { @@ -2414,8 +2415,8 @@ "type": "no_agent", "script": "functional_health_check.py", "schedule": "*/15 9-16,20-22 * * 1-5", - "status": null, - "last_run": "None", + "status": "ok", + "last_run": "2026-07-20T20:30:37", "profile": "position-analyst" }, { @@ -2433,7 +2434,7 @@ "script": "", "schedule": "{'kind': 'cron', 'expr': '*/10 * * * *'}", "status": "ok", - "last_run": "2026-07-20T20:21:30", + "last_run": "2026-07-20T20:41:32", "profile": "default" }, { @@ -2595,7 +2596,7 @@ "script": "collect_evaluation_data.py", "schedule": "30 20 * * 1-5", "status": "ok", - "last_run": "2026-07-17T20:30:47", + "last_run": "2026-07-20T20:30:38", "profile": "position-analyst" }, { @@ -2748,7 +2749,7 @@ "script": "self_todo_executor.py", "schedule": "*/10 8-22 * * 1-5", "status": "ok", - "last_run": "2026-07-20T20:20:32", + "last_run": "2026-07-20T20:40:41", "profile": "position-analyst" }, { @@ -2865,41 +2866,41 @@ "table": "mtf_cache", "label": "多周期均线缓存", "last_record": "07-20 17:08", - "age_hours": 3.4, + "age_hours": 3.6, "warn": false }, { "table": "macro_context_log", "label": "宏观上下文", "last_record": "07-20 15:31", - "age_hours": 5.0, + "age_hours": 5.2, "warn": false }, { "table": "market_snapshots", "label": "市场快照", "last_record": "07-20 15:50", - "age_hours": 4.7, + "age_hours": 4.9, "warn": false }, { "table": "live_prices", "label": "实时价格", "last_record": "07-20 18:12", - "age_hours": 2.3, + "age_hours": 2.6, "warn": false }, { "table": "price_events", "label": "价格事件", "last_record": "07-20 17:06", - "age_hours": 3.4, + "age_hours": 3.7, "warn": false } ], "self_check": { "functional": { - "generated_at": "2026-07-20 20:30:35", + "generated_at": "2026-07-20 20:45:45", "status": "ok", "summary": { "total": 9, @@ -2943,7 +2944,7 @@ "module": "premarket", "function": "盘前全量重评(premarket summary)", "status": "ok", - "reason": "739min前", + "reason": "754min前", "repair": { "action": "rerun_script", "script": "premarket_full_review.py" @@ -2953,7 +2954,7 @@ "module": "gateway_llm", "function": "LLM调用链可用(gateway agent.log)", "status": "ok", - "reason": "latency=2.9s", + "reason": "latency=14.0s", "repair": { "action": "llm_diagnose" }