hmo/MoFin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: watchdog fix

Browse Source
This commit is contained in:
知微
2026-07-20 20:55:34 +08:00
parent e5b0335f93
commit c971d6bced
3 changed files with 230 additions and 226 deletions
Download Patch File Download Diff File Expand all files Collapse all files
data/price_history.json
+10
View File
@@ -1844,5 +1844,15 @@
"volume": 2832553.0,
"amount": 9286.0
}
],
"000711": [
{
"date": "2026-07-20",
"high": 4.55,
"low": 4.21,
"close": 4.32,
"volume": 728146.0,
"amount": 31883.0
}
]
}
deploy/profile-scripts/fix_gateway_port.py
+93 -100
View File
@@ -1,100 +1,93 @@
#!/usr/bin/env python3
"""fix_gateway_port.py — 自愈系统调用的网关/XMPP Bot修复脚本
v2: 新增 session 健康检查，检测卡死的 session 自动重启
"""
import subprocess, sys, time, socket, json, urllib.request
GATEWAY_PORT = 8643
BOT_PORT = 5805
BOT_SCRIPT = "/home/hmo/xmpp_zhiwei_bot.py"
API_KEY = "hermes123"
SESSION_ID = "xmpp-zhiwei"
GATEWAY_URL = f"http://127.0.0.1:{GATEWAY_PORT}/v1/chat/completions"
def port_open(port, host="127.0.0.1"):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(2)
try:
r = s.connect_ex((host, port))
return r == 0
finally:
s.close()
def check_session_health():
"""调gateway API，检测session是否卡死。超过15s无响应→不健康"""
try:
payload = json.dumps({
"model": "hermes-agent",
"messages": [{"role": "user", "content": "ping"}]
}).encode()
req = urllib.request.Request(GATEWAY_URL, data=payload, method="POST")
req.add_header("Content-Type", "application/json")
req.add_header("Authorization", f"Bearer {API_KEY}")
req.add_header("X-Hermes-Session-Id", SESSION_ID)
t0 = time.time()
with urllib.request.urlopen(req, timeout=25) as r:
data = json.loads(r.read())
reply = data.get("choices", [{}])[0].get("message", {}).get("content", "")
elapsed = time.time() - t0
if reply:
print(f"Session {SESSION_ID} 健康 ✓ ({elapsed:.1f}s)")
return True
else:
print(f"Session {SESSION_ID} 返回空", file=sys.stderr)
return False
except urllib.request.HTTPError as e:
print(f"Session {SESSION_ID} HTTP错误: {e.code}", file=sys.stderr)
return False
except Exception as e:
print(f"Session {SESSION_ID} 不健康: {e}", file=sys.stderr)
return False
def restart_gateway():
"""通过systemd重启gateway"""
print(f"Gateway 端口{GATEWAY_PORT} 异常 → 重启中...")
subprocess.run(["sudo", "systemctl", "restart", "hermes-gateway-zhiwei.service"],
timeout=30, capture_output=True)
time.sleep(5)
if port_open(GATEWAY_PORT):
print(f"Gateway 已恢复 ✓")
return True
else:
print(f"Gateway 重启后仍不可达", file=sys.stderr)
return False
target = sys.argv[1] if len(sys.argv) > 1 else "all"
if target in ("all", "bot", "xmpp_bot"):
if not port_open(BOT_PORT):
print(f"XMPP bot port {BOT_PORT} CLOSED → 启动")
subprocess.run(["pkill", "-f", "xmpp_zhiwei_bot.py"], timeout=5, capture_output=True)
time.sleep(1)
subprocess.Popen(["python3", BOT_SCRIPT], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
time.sleep(5)
if port_open(BOT_PORT):
print(f"XMPP bot 端口{BOT_PORT} 已打开 ✓")
else:
print(f"XMPP bot 修复后仍不可达", file=sys.stderr)
sys.exit(1)
else:
print(f"XMPP bot 端口{BOT_PORT} 正常 ✓")
if target in ("all", "gateway", "session"):
if not port_open(GATEWAY_PORT):
restart_gateway()
elif target in ("all", "gateway") or target == "session":
# 端口通了 → 进一步检查session健康
if not check_session_health():
print(f"Session {SESSION_ID} 不健康 → 重启gateway")
restart_gateway()
else:
print(f"Gateway 端口{GATEWAY_PORT} 正常 ✓")
if target in ("all", "session"):
# 仅session检查
if port_open(GATEWAY_PORT):
if not check_session_health():
print(f"Session {SESSION_ID} 不健康 → 重启gateway")
restart_gateway()
sys.exit(0)
#!/usr/bin/env python3
"""fix_gateway_port.py — 自愈系统调用的网关/XMPP Bot修复脚本
v2: 新增 session 健康检查，检测卡死的 session 自动重启
"""
import subprocess, sys, time, socket, json, urllib.request
GATEWAY_PORT = 8643
BOT_PORT = 5805
BOT_SCRIPT = "/home/hmo/xmpp_zhiwei_bot.py"
API_KEY = "hermes123"
SESSION_ID = "xmpp-zhiwei"
GATEWAY_URL = f"http://127.0.0.1:{GATEWAY_PORT}/v1/chat/completions"
def port_open(port, host="127.0.0.1"):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(2)
try:
r = s.connect_ex((host, port))
return r == 0
finally:
s.close()
def check_session_health():
"""检测 gateway LLM 是否可用——扫 agent.log 最近一次真实调用结果。
不再发真实 LLM ping（25s 超时对 20-100s 的冷启动延迟必误报，且每次白烧 22k token）。
"""
try:
sys.path.insert(0, '/home/hmo/MoFin')
from xmpp_logger import _scan_agent_log
r = _scan_agent_log(time.time(), "zhiwei")
if r["status"] == "ok":
print(f"Session {SESSION_ID} 健康 ✓ (agent.log: latency={r.get('latency')}, {r.get('age_sec')}s前)")
return True
# error/unknown：只有近期有明确失败记录才判不健康
if r["status"] == "error":
print(f"Session {SESSION_ID} 不健康: agent.log 最近调用失败 — {r.get('error','')[:100]}", file=sys.stderr)
return False
# unknown（无近期调用记录）= 空闲，不算不健康
print(f"Session {SESSION_ID} 无近期调用记录（空闲正常）")
return True
except Exception as e:
print(f"Session {SESSION_ID} 健康检查异常: {e}（按健康处理）", file=sys.stderr)
return True
def restart_gateway():
"""通过systemd重启gateway"""
print(f"Gateway 端口{GATEWAY_PORT} 异常 → 重启中...")
subprocess.run(["sudo", "systemctl", "restart", "hermes-gateway-zhiwei.service"],
timeout=30, capture_output=True)
time.sleep(5)
if port_open(GATEWAY_PORT):
print(f"Gateway 已恢复 ✓")
return True
else:
print(f"Gateway 重启后仍不可达", file=sys.stderr)
return False
target = sys.argv[1] if len(sys.argv) > 1 else "all"
if target in ("all", "bot", "xmpp_bot"):
if not port_open(BOT_PORT):
print(f"XMPP bot port {BOT_PORT} CLOSED → 启动")
subprocess.run(["pkill", "-f", "xmpp_zhiwei_bot.py"], timeout=5, capture_output=True)
time.sleep(1)
subprocess.Popen(["python3", BOT_SCRIPT], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
time.sleep(5)
if port_open(BOT_PORT):
print(f"XMPP bot 端口{BOT_PORT} 已打开 ✓")
else:
print(f"XMPP bot 修复后仍不可达", file=sys.stderr)
sys.exit(1)
else:
print(f"XMPP bot 端口{BOT_PORT} 正常 ✓")
if target in ("all", "gateway", "session"):
if not port_open(GATEWAY_PORT):
restart_gateway()
elif target in ("all", "gateway") or target == "session":
# 端口通了 → 进一步检查session健康
if not check_session_health():
print(f"Session {SESSION_ID} 不健康 → 重启gateway")
restart_gateway()
else:
print(f"Gateway 端口{GATEWAY_PORT} 正常 ✓")
if target in ("all", "session"):
# 仅session检查
if port_open(GATEWAY_PORT):
if not check_session_health():
print(f"Session {SESSION_ID} 不健康 → 重启gateway")
restart_gateway()
sys.exit(0)
static/mofin_health.json
+127 -126
View File
@@ -1,5 +1,5 @@
{
"generated_at": "2026-07-20 20:30:35",
"generated_at": "2026-07-20 20:45:45",
"feature_tree": {
"label": "MoFin 系统",
"status": "ok",
@@ -170,7 +170,7 @@
"script": "collect_evaluation_data.py",
"schedule": "30 20 * * 1-5",
"status": "ok",
"last_run": "2026-07-17T20:30",
"last_run": "2026-07-20T20:30",
"type": "no_agent",
"profile": "position-analyst"
}
@@ -537,7 +537,7 @@
"script": "mofin_health.py",
"schedule": "*/15 9-16,20-22 * * 1-5",
"status": "ok",
"last_run": "2026-07-20T20:15",
"last_run": "2026-07-20T20:30",
"type": "no_agent",
"profile": "position-analyst"
}
@@ -624,7 +624,7 @@
"script": "self_todo_executor.py",
"schedule": "*/10 8-22 * * 1-5",
"status": "ok",
"last_run": "2026-07-20T20:20",
"last_run": "2026-07-20T20:40",
"type": "no_agent",
"profile": "position-analyst"
}
@@ -885,7 +885,7 @@
"script": "fix_gateway_port.py",
"schedule": "every 10m",
"status": "ok",
"last_run": "2026-07-20T20:25",
"last_run": "2026-07-20T20:36",
"type": "no_agent",
"profile": "position-analyst"
}
@@ -910,14 +910,14 @@
{
"label": "功能健康检查-L1 (functional_health_check.py)",
"desc": "",
"status": null,
"status": "ok",
"pipes": [
{
"name": "功能健康检查-L1",
"script": "functional_health_check.py",
"schedule": "*/15 9-16,20-22 * * 1-5",
"status": null,
"last_run": "",
"status": "ok",
"last_run": "2026-07-20T20:30",
"type": "no_agent",
"profile": "position-analyst"
}
@@ -926,14 +926,14 @@
{
"label": "LLM修复循环-L3 (self_repair.py)",
"desc": "",
"status": null,
"status": "ok",
"pipes": [
{
"name": "LLM修复循环-L3",
"script": "self_repair.py",
"schedule": "*/30 9-16,20-22 * * 1-5",
"status": null,
"last_run": "",
"status": "ok",
"last_run": "2026-07-20T20:30",
"type": "no_agent",
"profile": "position-analyst"
}
@@ -1028,8 +1028,8 @@
"desc": "建议执行时间线",
"rows": 12735,
"readers": [
"advice_reconciliation",
"mofin_db"
"mofin_db",
"advice_reconciliation"
],
"writers": [
"advice_reconciliation"
@@ -1086,10 +1086,10 @@
"mofin_db"
],
"writers": [
"market_scanner",
"promote_candidates",
"candidate_filter",
"accumulation_scanner"
"accumulation_scanner",
"market_scanner"
],
"has_input": true,
"has_output": true,
@@ -1114,8 +1114,8 @@
"readers": [
"per_stock_reassess",
"merge_third_db",
"mofin_db",
"batch_reassess"
"batch_reassess",
"mofin_db"
],
"writers": [
"merge_third_db",
@@ -1141,8 +1141,8 @@
"desc": "资金变动记录",
"rows": 20,
"readers": [
"mofin_db",
"prepare_report_data"
"prepare_report_data",
"mofin_db"
],
"writers": [
"mo_data",
@@ -1195,25 +1195,25 @@
"desc": "每只股票的完整策略参数",
"rows": 227,
"readers": [
"promote_candidates",
"candidate_filter",
"pre-flight-check",
"strategy_lifecycle",
"stale_push_wlin",
"verify_300308",
"mofin_collect",
"capital_flow_collector",
"data_governance",
"batch_reassess",
"mofin_db",
"accumulation_scanner",
"run_all_tests",
"generate_report",
"promote_candidates",
"price_monitor",
"sync_decisions_to_db"
"per_stock_reassess"
],
"writers": [
"per_stock_reassess",
"watchlist_auto_exit",
"data_governance",
"promote_candidates",
"mofin_db",
"sync_decisions_to_db",
"batch_reassess",
"mofin_db"
"data_governance",
"per_stock_reassess",
"watchlist_auto_exit"
],
"has_input": true,
"has_output": true,
@@ -1241,21 +1241,21 @@
"desc": "当前持仓（权威源）",
"rows": 14,
"readers": [
"capital_flow_collector",
"server",
"xiaoguo_signal_consumer",
"prepare_report_data",
"mofin_db",
"candidate_filter",
"strategy_lifecycle",
"trend_detector",
"stale_push_wlin",
"refresh_mtf_cache",
"accumulation_scanner",
"run_all_tests",
"mofin_collect",
"mo_alphasift_bridge",
"price_monitor"
"capital_flow_collector",
"xiaoguo_signal_consumer"
],
"writers": [
"price_monitor",
"import_holding_xls",
"mofin_db"
"mofin_db",
"import_holding_xls"
],
"has_input": true,
"has_output": true,
@@ -1285,12 +1285,12 @@
"readers": [
"candidate_filter",
"cron_health_monitor",
"generate_report",
"system_audit",
"stale_push_wlin",
"mofin_db",
"verify_reassess_pipeline",
"mo_data",
"mofin_db"
"generate_report",
"system_audit",
"stale_push_wlin"
],
"writers": [
"mo_data",
@@ -1321,15 +1321,15 @@
"desc": "宏观上下文（大盘偏向/指数）",
"rows": 81,
"readers": [
"strategy_lifecycle",
"per_stock_reassess",
"stock_profile",
"divergence_detector",
"system_audit",
"stale_push_wlin",
"strategy_tree",
"stock_profile",
"batch_reassess",
"xiaoguo_signal_consumer",
"strategy_tree"
"per_stock_reassess",
"strategy_lifecycle",
"system_audit",
"stale_push_wlin"
],
"writers": [
"refresh_macro_context"
@@ -1384,14 +1384,14 @@
"desc": "大盘指数快照（每10分）",
"rows": 989,
"readers": [
"mofin_query",
"mofin_db",
"prepare_report_data",
"market_screener",
"mofin_db",
"market_scanner",
"system_audit",
"merge_third_db",
"trend_detector",
"merge_third_db"
"mofin_query",
"market_screener"
],
"writers": [
"merge_third_db",
@@ -1420,8 +1420,8 @@
"desc": "多周期均线缓存",
"rows": 64,
"readers": [
"technical_analysis",
"multi_timeframe",
"technical_analysis",
"mofin_db"
],
"writers": [
@@ -1451,18 +1451,18 @@
"desc": "总资产/现金/仓位汇总",
"rows": 1,
"readers": [
"mofin_db",
"prepare_report_data",
"import_holding_xls",
"price_monitor",
"mofin_db",
"mo_data",
"preflight_verify",
"check_db_state"
"check_db_state",
"import_holding_xls",
"price_monitor"
],
"writers": [
"price_monitor",
"import_holding_xls",
"mofin_db"
"mofin_db",
"import_holding_xls"
],
"has_input": true,
"has_output": true,
@@ -1489,21 +1489,21 @@
"desc": "价格区间突破事件日志",
"rows": 6353,
"readers": [
"check_price_events",
"test_raw_insert",
"mofin_db",
"test_dual_write",
"test_db_only",
"mofin_db",
"test_db_write",
"backfill_price_events"
"backfill_price_events",
"check_price_events",
"test_db_only",
"test_raw_insert"
],
"writers": [
"test_raw_insert",
"test_dual_write",
"mofin_db",
"test_db_only",
"test_db_write",
"backfill_price_events"
"backfill_price_events",
"test_db_only",
"test_raw_insert"
],
"has_input": true,
"has_output": true,
@@ -1525,15 +1525,15 @@
"desc": "行业信号（趋势检测产出）",
"rows": 653,
"readers": [
"mofin_news",
"xiaoguo_news_processor",
"server",
"trend_detector",
"server"
"mofin_news"
],
"writers": [
"mofin_news",
"xiaoguo_news_processor",
"trend_detector"
"trend_detector",
"mofin_news"
],
"has_input": true,
"has_output": true,
@@ -1560,13 +1560,13 @@
"desc": "行业板块数据",
"rows": 68408,
"readers": [
"strategy_lifecycle",
"mofin_db",
"market_screener",
"market_scanner",
"strategy_lifecycle",
"merge_third_db",
"trend_detector",
"inspect_third_db",
"merge_third_db"
"market_screener"
],
"writers": [
"merge_third_db",
@@ -1597,20 +1597,20 @@
"readers": [
"intraday_health_check",
"server",
"per_stock_reassess",
"system_audit",
"batch_reassess",
"xiaoguo_signal_consumer",
"per_stock_reassess",
"system_audit",
"macro_signal_consumer"
],
"writers": [
"divergence_detector",
"mofin_news",
"xiaoguo_news_processor",
"divergence_detector",
"xiaoguo_signal_consumer",
"macro_context_collector",
"macro_signal_consumer",
"xiaoguo_scanner"
"xiaoguo_signal_consumer",
"xiaoguo_scanner",
"macro_signal_consumer"
],
"has_input": true,
"has_output": true,
@@ -1726,12 +1726,12 @@
"desc": "股票行业映射",
"rows": 64,
"readers": [
"strategy_lifecycle",
"mofin_news",
"per_stock_reassess",
"mofin_db",
"xiaoguo_news_processor",
"trend_detector",
"mofin_db"
"per_stock_reassess",
"strategy_lifecycle",
"trend_detector"
],
"writers": [
"mofin_db"
@@ -1776,19 +1776,19 @@
"desc": "全量股票代码",
"rows": 5575,
"readers": [
"accumulation_scanner",
"mofin_db",
"mofin_news",
"mofin_db",
"xiaoguo_news_processor",
"import_full_stocks",
"accumulation_scanner",
"trend_detector",
"import_full_stocks",
"check_stocks_table"
],
"writers": [
"price_monitor",
"backfill_price_events",
"import_full_stocks",
"mofin_db",
"import_full_stocks"
"backfill_price_events"
],
"has_input": true,
"has_output": true,
@@ -1813,8 +1813,8 @@
"desc": "策略重评历史记录",
"rows": 217,
"readers": [
"mofin_db",
"verify_reassess_pipeline"
"verify_reassess_pipeline",
"mofin_db"
],
"writers": [
"mofin_collect"
@@ -1868,20 +1868,20 @@
"desc": "自愈任务队列",
"rows": 96,
"readers": [
"strategy-staleness-check",
"intraday_health_check",
"self_todo_executor",
"strategy-staleness-check",
"merge_third_db",
"self_todo_executor",
"morning_health_check"
],
"writers": [
"intraday_health_check",
"strategy-staleness-check",
"cron_health_monitor",
"self_todo_executor",
"mofin_collect",
"strategy-staleness-check",
"preflight_verify",
"merge_third_db",
"self_todo_executor",
"morning_health_check"
],
"has_input": true,
@@ -1930,15 +1930,15 @@
"rows": 69,
"readers": [
"refresh_mtf_cache",
"run_all_tests",
"mo_alphasift_bridge",
"per_stock_reassess",
"stock_quote",
"price_monitor",
"trend_detector",
"stale_push_wlin",
"mofin_collect",
"xiaoguo_signal_consumer"
"mofin_db",
"mo_alphasift_bridge",
"stock_quote",
"run_all_tests",
"per_stock_reassess",
"xiaoguo_signal_consumer",
"xiaoguo_scanner",
"trend_detector"
],
"writers": [
"per_stock_reassess",
@@ -2078,7 +2078,7 @@
"size_kb": 0.3,
"readers": [],
"writers": [],
"last_modified": "07-20 15:31",
"last_modified": "07-20 20:45",
"warn": true,
"migrated_to_db": null
},
@@ -2105,14 +2105,15 @@
{
"name": "mofin_health.json",
"desc": "健康监控数据",
"size_kb": 89.2,
"size_kb": 89.3,
"readers": [
"analyze_health",
"verify_health_json",
"verify_self_check_section"
"verify_self_check_section",
"verify_deployment"
],
"writers": [],
"last_modified": "07-20 20:18",
"last_modified": "07-20 20:30",
"warn": false,
"migrated_to_db": null
},
@@ -2244,7 +2245,7 @@
"script": "fix_gateway_port.py",
"schedule": "every 10m",
"status": "ok",
"last_run": "2026-07-20T20:25:58",
"last_run": "2026-07-20T20:36:59",
"profile": "position-analyst"
},
{
@@ -2252,8 +2253,8 @@
"type": "no_agent",
"script": "self_repair.py",
"schedule": "*/30 9-16,20-22 * * 1-5",
"status": null,
"last_run": "None",
"status": "ok",
"last_run": "2026-07-20T20:30:36",
"profile": "position-analyst"
},
{
@@ -2361,7 +2362,7 @@
"script": "mofin_health.py",
"schedule": "*/15 9-16,20-22 * * 1-5",
"status": "ok",
"last_run": "2026-07-20T20:15:38",
"last_run": "2026-07-20T20:30:46",
"profile": "position-analyst"
},
{
@@ -2414,8 +2415,8 @@
"type": "no_agent",
"script": "functional_health_check.py",
"schedule": "*/15 9-16,20-22 * * 1-5",
"status": null,
"last_run": "None",
"status": "ok",
"last_run": "2026-07-20T20:30:37",
"profile": "position-analyst"
},
{
@@ -2433,7 +2434,7 @@
"script": "",
"schedule": "{'kind': 'cron', 'expr': '*/10 * * * *'}",
"status": "ok",
"last_run": "2026-07-20T20:21:30",
"last_run": "2026-07-20T20:41:32",
"profile": "default"
},
{
@@ -2595,7 +2596,7 @@
"script": "collect_evaluation_data.py",
"schedule": "30 20 * * 1-5",
"status": "ok",
"last_run": "2026-07-17T20:30:47",
"last_run": "2026-07-20T20:30:38",
"profile": "position-analyst"
},
{
@@ -2748,7 +2749,7 @@
"script": "self_todo_executor.py",
"schedule": "*/10 8-22 * * 1-5",
"status": "ok",
"last_run": "2026-07-20T20:20:32",
"last_run": "2026-07-20T20:40:41",
"profile": "position-analyst"
},
{
@@ -2865,41 +2866,41 @@
"table": "mtf_cache",
"label": "多周期均线缓存",
"last_record": "07-20 17:08",
"age_hours": 3.4,
"age_hours": 3.6,
"warn": false
},
{
"table": "macro_context_log",
"label": "宏观上下文",
"last_record": "07-20 15:31",
"age_hours": 5.0,
"age_hours": 5.2,
"warn": false
},
{
"table": "market_snapshots",
"label": "市场快照",
"last_record": "07-20 15:50",
"age_hours": 4.7,
"age_hours": 4.9,
"warn": false
},
{
"table": "live_prices",
"label": "实时价格",
"last_record": "07-20 18:12",
"age_hours": 2.3,
"age_hours": 2.6,
"warn": false
},
{
"table": "price_events",
"label": "价格事件",
"last_record": "07-20 17:06",
"age_hours": 3.4,
"age_hours": 3.7,
"warn": false
}
],
"self_check": {
"functional": {
"generated_at": "2026-07-20 20:30:35",
"generated_at": "2026-07-20 20:45:45",
"status": "ok",
"summary": {
"total": 9,
@@ -2943,7 +2944,7 @@
"module": "premarket",
"function": "盘前全量重评(premarket summary)",
"status": "ok",
"reason": "739min前",
"reason": "754min前",
"repair": {
"action": "rerun_script",
"script": "premarket_full_review.py"
@@ -2953,7 +2954,7 @@
"module": "gateway_llm",
"function": "LLM调用链可用(gateway agent.log)",
"status": "ok",
"reason": "latency=2.9s",
"reason": "latency=14.0s",
"repair": {
"action": "llm_diagnose"
}