chore: deployed L0-L4 self-check system
This commit is contained in:
知微
@@ -0,0 +1,216 @@
|
||||
#!/usr/bin/env python3
|
||||
"""functional_health_check.py — L1 功能健康检查
|
||||
|
||||
判据不是"进程活着"，而是"功能是否达成"：每个核心模块检查其
|
||||
**输出物的新鲜度和有效性**。输出物不新鲜 = 功能未达成 = 异常。
|
||||
|
||||
频率：交易时段每 15 分钟（cron），输出 gateway/logs/functional_health.json。
|
||||
责任矩阵 L1。修复由 L3 self_repair 读取本报告执行。
|
||||
"""
|
||||
import os, sys, json, sqlite3, subprocess
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
sys.path.insert(0, '/home/hmo/MoFin')
|
||||
|
||||
DB = '/home/hmo/MoFin/data/mofin.db'
|
||||
OUT = '/home/hmo/MoFin/gateway/logs/functional_health.json'
|
||||
|
||||
# ── 功能注册表：模块 → 功能判据 ──────────────────────────────
|
||||
# type:
|
||||
# db_freshness: DB 表 MAX(col) 距今不超过 max_age（trading=交易时段才检查, daily=每日, always=总是）
|
||||
# file_freshness: 文件 mtime 距今不超过 max_age
|
||||
# bot_activity: XMPP bot 活动检查（journal）
|
||||
# agent_log: gateway agent.log 最近有成功 LLM 调用
|
||||
# soft=True: 无输出可能属正常（如区间未触发），降级为 warn 而非 fail
|
||||
REGISTRY = [
|
||||
{"module": "price_monitor", "function": "实时价格写入DB(live_prices)",
|
||||
"check": {"type": "db_freshness", "table": "live_prices", "col": "updated_at",
|
||||
"max_age_min": 6, "when": "trading"},
|
||||
"repair": {"action": "rerun_script", "script": "price_monitor.py"}},
|
||||
{"module": "market_watch", "function": "市场快照采集(market_snapshots)",
|
||||
"check": {"type": "db_freshness", "table": "market_snapshots", "col": "created_at",
|
||||
"max_age_min": 40, "when": "trading"},
|
||||
"repair": {"action": "rerun_script", "script": "market_watch.py"}},
|
||||
{"module": "mtf_cache", "function": "多周期均线缓存刷新(mtf_cache)",
|
||||
"check": {"type": "db_freshness", "table": "mtf_cache", "col": "updated_at",
|
||||
"max_age_min": 75, "when": "trading"},
|
||||
"repair": {"action": "rerun_script", "script": "refresh_mtf_cache.py"}},
|
||||
{"module": "macro_context", "function": "宏观上下文刷新(macro_context_log)",
|
||||
"check": {"type": "db_freshness", "table": "macro_context_log", "col": "created_at",
|
||||
"max_age_min": 45, "when": "trading"},
|
||||
"repair": {"action": "rerun_script", "script": "refresh_macro_context.py"}},
|
||||
{"module": "health_collector", "function": "健康数据采集(mofin_health.json)",
|
||||
"check": {"type": "file_freshness", "path": "/home/hmo/web-dashboard/static/mofin_health.json",
|
||||
"max_age_min": 20, "when": "trading"},
|
||||
"repair": {"action": "rerun_script", "script": "mofin_health.py"}},
|
||||
{"module": "premarket", "function": "盘前全量重评(premarket summary)",
|
||||
"check": {"type": "file_freshness", "path": "/tmp/mofin_premarket/summary.json",
|
||||
"max_age_h": 26, "when": "daily"},
|
||||
"repair": {"action": "rerun_script", "script": "premarket_full_review.py"}},
|
||||
{"module": "gateway_llm", "function": "LLM调用链可用(gateway agent.log)",
|
||||
"check": {"type": "agent_log", "max_age_min": 30, "when": "always"},
|
||||
"repair": {"action": "llm_diagnose"}},
|
||||
{"module": "xmpp_bot", "function": "XMPP消息收发(bot journal)",
|
||||
"check": {"type": "bot_activity", "when": "always"},
|
||||
"repair": {"action": "llm_diagnose"}},
|
||||
{"module": "cron_engine", "function": "cron调度引擎本身(有job在最近10min运行)",
|
||||
"check": {"type": "cron_engine", "max_age_min": 12, "when": "trading"},
|
||||
"repair": {"action": "llm_diagnose"}},
|
||||
]
|
||||
|
||||
|
||||
def is_trading_now(now):
|
||||
return now.weekday() < 5 and 9 <= now.hour <= 16
|
||||
|
||||
|
||||
def check_db_freshness(conn, chk, now):
|
||||
try:
|
||||
row = conn.execute(f"SELECT MAX({chk['col']}) FROM {chk['table']}").fetchone()
|
||||
if not row or not row[0]:
|
||||
return "fail", f"表 {chk['table']} 无数据"
|
||||
last = datetime.fromisoformat(str(row[0]).replace("Z", ""))
|
||||
age_min = (now - last).total_seconds() / 60
|
||||
limit = chk.get("max_age_min", chk.get("max_age_h", 24) * 60)
|
||||
if age_min > limit:
|
||||
return ("warn" if chk.get("soft") else "fail"), \
|
||||
f"{chk['table']} 最新记录 {last.strftime('%m-%d %H:%M')}（{age_min/60:.1f}h前，阈值{limit}min）"
|
||||
return "ok", f"{age_min:.0f}min前"
|
||||
except Exception as e:
|
||||
return "fail", f"查询失败: {str(e)[:80]}"
|
||||
|
||||
|
||||
def check_file_freshness(chk, now):
|
||||
p = chk["path"]
|
||||
if not os.path.exists(p):
|
||||
return "fail", f"文件不存在: {p}"
|
||||
age_min = (now.timestamp() - os.path.getmtime(p)) / 60
|
||||
limit = chk.get("max_age_min", chk.get("max_age_h", 24) * 60)
|
||||
if age_min > limit:
|
||||
return ("warn" if chk.get("soft") else "fail"), \
|
||||
f"{os.path.basename(p)} {age_min/60:.1f}h 未更新（阈值{limit}min）"
|
||||
return "ok", f"{age_min:.0f}min前"
|
||||
|
||||
|
||||
def check_agent_log(chk, now):
|
||||
try:
|
||||
sys.path.insert(0, '/home/hmo/MoFin')
|
||||
from xmpp_logger import _scan_agent_log
|
||||
r = _scan_agent_log(now.timestamp(), "zhiwei")
|
||||
if r["status"] == "ok":
|
||||
age = r.get("age_sec", -1)
|
||||
if age > chk["max_age_min"] * 60:
|
||||
return "warn", f"最近成功LLM调用在 {age//60}min 前（无新流量，可能正常）"
|
||||
return "ok", f"latency={r.get('latency')}"
|
||||
return "fail", f"agent.log 最近调用失败: {r.get('error','?')[:100]}"
|
||||
except Exception as e:
|
||||
return "fail", f"检查失败: {e}"
|
||||
|
||||
|
||||
def check_bot_activity(chk, now):
|
||||
try:
|
||||
# 第一判据：服务当前是否 active（权威"现在在不在跑"）
|
||||
r2 = subprocess.run(["systemctl", "is-active", "xmpp-zhiwei"],
|
||||
capture_output=True, text=True, timeout=5)
|
||||
if r2.stdout.strip() != "active":
|
||||
return "fail", "bot 服务非 active（已停止）"
|
||||
# 第二判据：最近 journal 是否处于断线循环
|
||||
r = subprocess.run(
|
||||
["journalctl", "-u", "xmpp-zhiwei", "--no-pager", "--since", "30 min ago", "-o", "cat"],
|
||||
capture_output=True, timeout=8, text=True)
|
||||
lines = r.stdout
|
||||
if "连接超时" in lines and "就绪" not in lines:
|
||||
return "fail", "bot 处于断线重连循环"
|
||||
if "XMPP 就绪" in lines or "已发送" in lines or "收到" in lines:
|
||||
return "ok", "bot 活动正常"
|
||||
return "ok", "bot 在线空闲（无新消息）"
|
||||
except Exception as e:
|
||||
return "fail", f"检查失败: {e}"
|
||||
|
||||
|
||||
def check_cron_engine(chk, now):
|
||||
"""cron 引擎：最近 max_age_min 内是否有任何 job 运行过"""
|
||||
try:
|
||||
d = json.load(open('/home/hmo/.hermes/profiles/position-analyst/cron/jobs.json'))
|
||||
jobs = d if isinstance(d, list) else d.get('jobs', [])
|
||||
latest = None
|
||||
for j in jobs:
|
||||
lr = j.get('last_run_at')
|
||||
if lr:
|
||||
try:
|
||||
t = datetime.fromisoformat(lr.replace('Z', '+00:00')).replace(tzinfo=None)
|
||||
if latest is None or t > latest:
|
||||
latest = t
|
||||
except Exception:
|
||||
pass
|
||||
if not latest:
|
||||
return "fail", "所有 job 均无运行记录"
|
||||
age_min = (now - latest).total_seconds() / 60
|
||||
if age_min > chk["max_age_min"]:
|
||||
return "fail", f"最近 job 运行在 {age_min:.0f}min 前，调度引擎疑似停摆"
|
||||
return "ok", f"最近 job 运行于 {age_min:.0f}min 前"
|
||||
except Exception as e:
|
||||
return "fail", f"检查失败: {e}"
|
||||
|
||||
|
||||
def main():
|
||||
now = datetime.now()
|
||||
trading = is_trading_now(now)
|
||||
results = []
|
||||
conn = sqlite3.connect(DB, timeout=10)
|
||||
|
||||
for item in REGISTRY:
|
||||
chk = item["check"]
|
||||
when = chk.get("when", "always")
|
||||
if when == "trading" and not trading:
|
||||
results.append({"module": item["module"], "function": item["function"],
|
||||
"status": "skip", "reason": "非交易时段"})
|
||||
continue
|
||||
if when == "daily":
|
||||
# 每日类：只 fail 不 warn，且非交易日放宽到 72h
|
||||
if now.weekday() >= 5:
|
||||
chk = dict(chk)
|
||||
chk["max_age_h"] = 72
|
||||
|
||||
t = chk["type"]
|
||||
if t == "db_freshness":
|
||||
status, reason = check_db_freshness(conn, chk, now)
|
||||
elif t == "file_freshness":
|
||||
status, reason = check_file_freshness(chk, now)
|
||||
elif t == "agent_log":
|
||||
status, reason = check_agent_log(chk, now)
|
||||
elif t == "bot_activity":
|
||||
status, reason = check_bot_activity(chk, now)
|
||||
elif t == "cron_engine":
|
||||
status, reason = check_cron_engine(chk, now)
|
||||
else:
|
||||
status, reason = "fail", f"未知检查类型 {t}"
|
||||
|
||||
results.append({"module": item["module"], "function": item["function"],
|
||||
"status": status, "reason": reason,
|
||||
"repair": item.get("repair")})
|
||||
|
||||
conn.close()
|
||||
|
||||
fails = [r for r in results if r["status"] == "fail"]
|
||||
warns = [r for r in results if r["status"] == "warn"]
|
||||
report = {
|
||||
"generated_at": now.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"trading_hours": trading,
|
||||
"summary": {"total": len(results), "ok": sum(1 for r in results if r["status"] == "ok"),
|
||||
"warn": len(warns), "fail": len(fails),
|
||||
"skip": sum(1 for r in results if r["status"] == "skip")},
|
||||
"status": "fail" if fails else ("warn" if warns else "ok"),
|
||||
"checks": results,
|
||||
}
|
||||
os.makedirs(os.path.dirname(OUT), exist_ok=True)
|
||||
with open(OUT, "w", encoding="utf-8") as f:
|
||||
json.dump(report, f, ensure_ascii=False, indent=2)
|
||||
|
||||
print(f"功能健康: {report['summary']} status={report['status']}")
|
||||
for r in results:
|
||||
if r["status"] in ("fail", "warn"):
|
||||
print(f" {r['status'].upper()} {r['module']}: {r['reason']}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,123 @@
|
||||
#!/usr/bin/env python3
|
||||
"""meta_watchdog.py — L4 自检系统的自检（看门狗的看门狗）
|
||||
|
||||
检查 L0-L3 各自检组件本身是否在正常运转：
|
||||
- L0 agents_health_check: last_health_check.json 是否 <10min
|
||||
- L1 functional_health_check: functional_health.json 是否 <20min（交易时段）
|
||||
- L2 system_hygiene_audit: hygiene_report.json 是否 <26h（每日）
|
||||
- L3 self_repair: repair_state.json 存在性 + cron 是否注册
|
||||
- mofin_health 采集: mofin_health.json 是否 <20min（交易时段）
|
||||
- XMPP 桥: :5805 是否可发（self_repair 的报备通道）
|
||||
|
||||
任何一层死了 → 推 XMPP 点名（这是最后的兜底，必须直达用户）。
|
||||
频率：每小时（cron）。输出 gateway/logs/meta_watchdog.json。
|
||||
"""
|
||||
import os, sys, json, subprocess
|
||||
from datetime import datetime
|
||||
|
||||
OUT = '/home/hmo/MoFin/gateway/logs/meta_watchdog.json'
|
||||
|
||||
LAYERS = [
|
||||
{"layer": "L0 agents_health_check", "file": "/home/hmo/MoFin/gateway/temp/last_health_check.json",
|
||||
"max_age_min": 10, "when": "always",
|
||||
"repair": "crontab */5 agents_health_check.py 停摆，检查系统 crontab"},
|
||||
{"layer": "L1 functional_health", "file": "/home/hmo/MoFin/gateway/logs/functional_health.json",
|
||||
"max_age_min": 25, "when": "trading",
|
||||
"repair": "L1 cron 停摆，检查 hermes cron 引擎"},
|
||||
{"layer": "L2 hygiene_audit", "file": "/home/hmo/MoFin/gateway/logs/hygiene_report.json",
|
||||
"max_age_min": 26 * 60, "when": "always",
|
||||
"repair": "L2 每日审计未跑，检查 hermes cron"},
|
||||
{"layer": "L1.5 mofin_health采集", "file": "/home/hmo/web-dashboard/static/mofin_health.json",
|
||||
"max_age_min": 25, "when": "trading",
|
||||
"repair": "mofin_health.py 采集停摆"},
|
||||
{"layer": "L3 self_repair", "file": "/home/hmo/MoFin/gateway/logs/repair_log.jsonl",
|
||||
"max_age_min": None, "when": "meta",
|
||||
"repair": "self_repair cron 未注册"},
|
||||
]
|
||||
|
||||
|
||||
def is_trading(now):
|
||||
return now.weekday() < 5 and 9 <= now.hour <= 16
|
||||
|
||||
|
||||
def main():
|
||||
now = datetime.now()
|
||||
trading = is_trading(now)
|
||||
results = []
|
||||
|
||||
for L in LAYERS:
|
||||
if L["when"] == "trading" and not trading:
|
||||
results.append({"layer": L["layer"], "status": "skip", "reason": "非交易时段"})
|
||||
continue
|
||||
if L["when"] == "meta":
|
||||
# 检查 self_repair 是否注册在 cron
|
||||
try:
|
||||
d = json.load(open('/home/hmo/.hermes/profiles/position-analyst/cron/jobs.json'))
|
||||
jobs = d if isinstance(d, list) else d.get('jobs', [])
|
||||
registered = any(j.get('script') == 'self_repair.py' and j.get('enabled', True) for j in jobs)
|
||||
results.append({"layer": L["layer"],
|
||||
"status": "ok" if registered else "fail",
|
||||
"reason": "已注册" if registered else "未在 cron 注册"})
|
||||
except Exception as e:
|
||||
results.append({"layer": L["layer"], "status": "fail", "reason": str(e)[:60]})
|
||||
continue
|
||||
|
||||
f = L["file"]
|
||||
if not os.path.exists(f):
|
||||
results.append({"layer": L["layer"], "status": "fail",
|
||||
"reason": f"输出物不存在", "repair": L["repair"]})
|
||||
continue
|
||||
age_min = (now.timestamp() - os.path.getmtime(f)) / 60
|
||||
if L["max_age_min"] and age_min > L["max_age_min"]:
|
||||
results.append({"layer": L["layer"], "status": "fail",
|
||||
"reason": f"输出物 {age_min/60:.1f}h 未更新（阈值 {L['max_age_min']}min）",
|
||||
"repair": L["repair"]})
|
||||
else:
|
||||
results.append({"layer": L["layer"], "status": "ok",
|
||||
"reason": f"{age_min:.0f}min 前"})
|
||||
|
||||
# XMPP 桥（报备通道）：只收 POST，GET 会 501，但任何 HTTP 响应都说明进程活着
|
||||
try:
|
||||
import urllib.request
|
||||
urllib.request.urlopen('http://127.0.0.1:5805/', timeout=3)
|
||||
results.append({"layer": "XMPP桥 :5805", "status": "ok", "reason": "可达"})
|
||||
except urllib.error.HTTPError as e:
|
||||
results.append({"layer": "XMPP桥 :5805", "status": "ok", "reason": f"可达(HTTP {e.code})"})
|
||||
except Exception:
|
||||
results.append({"layer": "XMPP桥 :5805", "status": "fail",
|
||||
"reason": "不可达", "repair": "重启 xmpp-zhiwei"})
|
||||
|
||||
fails = [r for r in results if r["status"] == "fail"]
|
||||
report = {
|
||||
"generated_at": now.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"status": "fail" if fails else "ok",
|
||||
"layers": results,
|
||||
}
|
||||
os.makedirs(os.path.dirname(OUT), exist_ok=True)
|
||||
with open(OUT, 'w', encoding='utf-8') as f:
|
||||
json.dump(report, f, ensure_ascii=False, indent=2)
|
||||
|
||||
print(f"meta_watchdog: {report['status']}")
|
||||
for r in results:
|
||||
icon = {"ok": "✅", "fail": "❌", "skip": "⏭"}[r["status"]]
|
||||
print(f" {icon} {r['layer']}: {r['reason']}")
|
||||
|
||||
if fails:
|
||||
try:
|
||||
import urllib.request
|
||||
lines = [f"🚨 自检系统自检（L4兜底）发现 {len(fails)} 层异常："]
|
||||
for r in fails:
|
||||
lines.append(f"❌ {r['layer']}: {r['reason']}")
|
||||
if r.get('repair'):
|
||||
lines.append(f" → 处置建议: {r['repair']}")
|
||||
payload = json.dumps({'to': 'hmo@yoin.fun', 'body': '\n'.join(lines), 'type': 'chat'}).encode()
|
||||
req = urllib.request.Request('http://127.0.0.1:5805/', data=payload,
|
||||
headers={'Content-Type': 'application/json'})
|
||||
urllib.request.urlopen(req, timeout=5)
|
||||
print(' 📨 已推 XMPP（兜底直达）')
|
||||
except Exception as e:
|
||||
print(f' XMPP 失败: {e}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
+1018
-976
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,255 @@
|
||||
#!/usr/bin/env python3
|
||||
"""self_repair.py — L3 LLM 修复循环（报备制）
|
||||
|
||||
读取 L1 functional_health.json + L2 hygiene_report.json 的失败项，
|
||||
调用 LLM 诊断并**直接执行修复**（先斩后奏），然后记录日志并 XMPP 报备。
|
||||
|
||||
安全设计：
|
||||
- LLM 只能从白名单动作中选择（不许任意执行代码）
|
||||
- 每个模块每天最多自动修复 2 次（防修复循环）
|
||||
- LLM 不可用时降级为规则默认动作（rerun_script）
|
||||
- 所有动作记录 repair_log.jsonl
|
||||
|
||||
频率：每 30 分钟（cron）。
|
||||
"""
|
||||
import os, sys, json, subprocess, sqlite3, time
|
||||
from datetime import datetime
|
||||
|
||||
sys.path.insert(0, '/home/hmo/MoFin')
|
||||
|
||||
FUNCTIONAL_REPORT = '/home/hmo/MoFin/gateway/logs/functional_health.json'
|
||||
HYGIENE_REPORT = '/home/hmo/MoFin/gateway/logs/hygiene_report.json'
|
||||
REPAIR_LOG = '/home/hmo/MoFin/gateway/logs/repair_log.jsonl'
|
||||
REPAIR_STATE = '/home/hmo/MoFin/gateway/logs/repair_state.json'
|
||||
GATEWAY = 'http://127.0.0.1:8643/v1/chat/completions'
|
||||
SCRIPTS_DIR = '/home/hmo/.hermes/profiles/position-analyst/scripts'
|
||||
MAX_REPAIRS_PER_MODULE_PER_DAY = 2
|
||||
|
||||
WHITELIST_ACTIONS = """
|
||||
可执行的白名单动作（只能选其一，不许自创）：
|
||||
1. {"action": "rerun_script", "script": "<脚本名>"} — 立即重跑指定 cron 脚本（限注册表内的脚本）
|
||||
2. {"action": "restart_service", "service": "<服务名>"} — systemctl 重启（限: hermes-gateway-zhiwei, xmpp-zhiwei, mofin-dashboard）
|
||||
3. {"action": "sync_links"} — 重建脚本硬链接（修复 cron 跑旧代码）
|
||||
4. {"action": "switch_llm_key"} — 切换 LLM key（429 限流时）
|
||||
5. {"action": "none", "reason": "<原因>"} — 判断为无需动作（如非交易时段的正常空闲）
|
||||
"""
|
||||
|
||||
|
||||
def log_repair(entry):
|
||||
os.makedirs(os.path.dirname(REPAIR_LOG), exist_ok=True)
|
||||
with open(REPAIR_LOG, 'a', encoding='utf-8') as f:
|
||||
f.write(json.dumps(entry, ensure_ascii=False) + '\n')
|
||||
|
||||
|
||||
def load_state():
|
||||
try:
|
||||
return json.load(open(REPAIR_STATE))
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def save_state(s):
|
||||
with open(REPAIR_STATE, 'w') as f:
|
||||
json.dump(s, f)
|
||||
|
||||
|
||||
def repairs_today(state, module):
|
||||
today = datetime.now().strftime('%Y-%m-%d')
|
||||
return state.get(today, {}).get(module, 0)
|
||||
|
||||
|
||||
def bump_repair(state, module):
|
||||
today = datetime.now().strftime('%Y-%m-%d')
|
||||
state.setdefault(today, {})
|
||||
state[today][module] = state[today].get(module, 0) + 1
|
||||
# 只保留最近 3 天
|
||||
for k in list(state.keys()):
|
||||
if k < (datetime.now().strftime('%Y-%m-%d'))[:8] + '17':
|
||||
del state[k]
|
||||
save_state(state)
|
||||
|
||||
|
||||
def collect_failures():
|
||||
failures = []
|
||||
try:
|
||||
fh = json.load(open(FUNCTIONAL_REPORT))
|
||||
for c in fh.get('checks', []):
|
||||
if c.get('status') == 'fail':
|
||||
failures.append({
|
||||
'source': 'functional', 'module': c['module'],
|
||||
'function': c['function'], 'reason': c['reason'],
|
||||
'repair_hint': c.get('repair'),
|
||||
})
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
hy = json.load(open(HYGIENE_REPORT))
|
||||
for i in hy.get('issues', []):
|
||||
failures.append({
|
||||
'source': 'hygiene', 'module': i.get('type'),
|
||||
'function': i.get('file') or i.get('job') or i.get('table', '?'),
|
||||
'reason': json.dumps(i, ensure_ascii=False)[:200],
|
||||
'repair_hint': {'action': 'llm_diagnose'},
|
||||
})
|
||||
except Exception:
|
||||
pass
|
||||
return failures
|
||||
|
||||
|
||||
def run_action(action, module):
|
||||
"""执行白名单动作，返回 (ok, detail)"""
|
||||
act = action.get('action')
|
||||
try:
|
||||
if act == 'rerun_script':
|
||||
script = action.get('script', '')
|
||||
# 只允许注册表内脚本
|
||||
if not script.endswith('.py') or '/' in script or '..' in script:
|
||||
return False, f'非法脚本名: {script}'
|
||||
path = os.path.join(SCRIPTS_DIR, script)
|
||||
if not os.path.exists(path):
|
||||
return False, f'脚本不存在: {script}'
|
||||
r = subprocess.run(['python3', path], capture_output=True, text=True,
|
||||
timeout=600, cwd=SCRIPTS_DIR)
|
||||
tail = (r.stdout or r.stderr)[-150:]
|
||||
return r.returncode == 0, f'exit={r.returncode} {tail}'
|
||||
|
||||
elif act == 'restart_service':
|
||||
svc = action.get('service', '')
|
||||
allowed = {'hermes-gateway-zhiwei': ['sudo', '-n', 'systemctl', 'restart', svc],
|
||||
'xmpp-zhiwei': ['sudo', '-n', 'systemctl', 'restart', svc],
|
||||
'mofin-dashboard': ['sudo', '-n', 'systemctl', 'restart', svc]}
|
||||
if svc not in allowed:
|
||||
return False, f'服务不在白名单: {svc}'
|
||||
subprocess.Popen(allowed[svc], stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL, start_new_session=True)
|
||||
return True, f'{svc} 重启已触发(异步)'
|
||||
|
||||
elif act == 'sync_links':
|
||||
r = subprocess.run(['bash', '/home/hmo/MoFin/deploy/profile-scripts/sync_profile_scripts.sh'],
|
||||
capture_output=True, text=True, timeout=60)
|
||||
return True, r.stdout.strip()
|
||||
|
||||
elif act == 'switch_llm_key':
|
||||
sys.path.insert(0, '/home/hmo/MoFin')
|
||||
from xmpp_logger import best_key, switch_key, KEY_TO_PROVIDER, current_provider
|
||||
bk = best_key()
|
||||
if not bk:
|
||||
return False, '无可用 key'
|
||||
target = KEY_TO_PROVIDER.get(bk['key_id'])
|
||||
if not target or target == current_provider():
|
||||
return False, f'已在最优 key ({bk["key_id"]})'
|
||||
r = switch_key(bk['key_id'])
|
||||
return r.get('switched', False), json.dumps(r, ensure_ascii=False)[:150]
|
||||
|
||||
elif act == 'none':
|
||||
return True, f'无需动作: {action.get("reason", "")}'
|
||||
|
||||
return False, f'未知动作: {act}'
|
||||
except subprocess.TimeoutExpired:
|
||||
return False, '动作超时(600s)'
|
||||
except Exception as e:
|
||||
return False, f'动作异常: {str(e)[:120]}'
|
||||
|
||||
|
||||
def llm_diagnose(failure):
|
||||
"""调 LLM 诊断并返回白名单动作。LLM 不可用 → 返回 None（走规则默认）"""
|
||||
hint = failure.get('repair_hint') or {}
|
||||
default_action = hint if hint.get('action') and hint['action'] != 'llm_diagnose' else None
|
||||
|
||||
prompt = f"""你是 MoFin 系统的自动修复工程师。一个健康检查发现了功能异常，请诊断并给出一个修复动作。
|
||||
|
||||
【异常信息】
|
||||
模块: {failure['module']}
|
||||
功能: {failure['function']}
|
||||
异常表现: {failure['reason']}
|
||||
来源: {failure['source']}
|
||||
|
||||
【背景】
|
||||
- 系统: Linux 246, MoFin 股票分析系统
|
||||
- cron 脚本目录: {SCRIPTS_DIR}
|
||||
- 数据库: /home/hmo/MoFin/data/mofin.db
|
||||
- 当前时间: {datetime.now().strftime('%Y-%m-%d %H:%M')}（注意是否交易时段，非交易时段部分管道停跑属正常）
|
||||
|
||||
{WHITELIST_ACTIONS}
|
||||
|
||||
请只输出一个 JSON 动作（不要其他文字）。如果是非交易时段的正常空闲，选 none。"""
|
||||
|
||||
try:
|
||||
payload = json.dumps({
|
||||
'model': 'deepseek-v4-flash',
|
||||
'messages': [{'role': 'user', 'content': prompt}],
|
||||
'max_tokens': 300, 'stream': False,
|
||||
}).encode()
|
||||
import urllib.request
|
||||
req = urllib.request.Request(GATEWAY, data=payload, headers={
|
||||
'Content-Type': 'application/json', 'Authorization': 'Bearer hermes123',
|
||||
'X-Hermes-Session-Id': 'self-repair'})
|
||||
resp = urllib.request.urlopen(req, timeout=120)
|
||||
text = json.loads(resp.read())['choices'][0]['message']['content']
|
||||
# 提取 JSON
|
||||
import re
|
||||
m = re.search(r'\{[^{}]*"action"[^{}]*\}', text)
|
||||
if m:
|
||||
action = json.loads(m.group(0))
|
||||
if action.get('action') in ('rerun_script', 'restart_service', 'sync_links',
|
||||
'switch_llm_key', 'none'):
|
||||
return action, text[:200]
|
||||
except Exception as e:
|
||||
print(f' LLM 诊断不可用: {str(e)[:80]}')
|
||||
return (default_action, None) if default_action else (None, None)
|
||||
|
||||
|
||||
def xmpp_report(lines):
|
||||
try:
|
||||
import urllib.request
|
||||
payload = json.dumps({'to': 'hmo@yoin.fun', 'body': '\n'.join(lines), 'type': 'chat'}).encode()
|
||||
req = urllib.request.Request('http://127.0.0.1:5805/', data=payload,
|
||||
headers={'Content-Type': 'application/json'})
|
||||
urllib.request.urlopen(req, timeout=5)
|
||||
except Exception as e:
|
||||
print(f'XMPP 失败: {e}')
|
||||
|
||||
|
||||
def main():
|
||||
now = datetime.now()
|
||||
print('🔧 self_repair', now.strftime('%H:%M'))
|
||||
failures = collect_failures()
|
||||
if not failures:
|
||||
print(' ✅ 无失败项')
|
||||
return
|
||||
|
||||
state = load_state()
|
||||
reports = []
|
||||
for f in failures:
|
||||
module = f['module']
|
||||
if repairs_today(state, module) >= MAX_REPAIRS_PER_MODULE_PER_DAY:
|
||||
print(f' ⏭ {module}: 今日已修复 {MAX_REPAIRS_PER_MODULE_PER_DAY} 次，跳过（防循环）')
|
||||
continue
|
||||
|
||||
print(f' 🔍 {module}: {f["reason"][:60]}')
|
||||
action, llm_note = llm_diagnose(f)
|
||||
if action is None:
|
||||
print(f' 无可用动作，跳过')
|
||||
log_repair({'ts': now.isoformat(), 'module': module, 'reason': f['reason'],
|
||||
'action': 'skip_no_action', 'ok': None})
|
||||
continue
|
||||
|
||||
ok, detail = run_action(action, module)
|
||||
bump_repair(state, module)
|
||||
entry = {
|
||||
'ts': now.isoformat(), 'module': module, 'function': f['function'],
|
||||
'reason': f['reason'], 'action': action, 'ok': ok, 'detail': detail[:300],
|
||||
'llm_note': llm_note,
|
||||
}
|
||||
log_repair(entry)
|
||||
icon = '✅' if ok else '❌'
|
||||
print(f' {icon} {action.get("action")} -> {detail[:80]}')
|
||||
reports.append(f'{icon} [{module}] {action.get("action")}: {detail[:80]}')
|
||||
|
||||
if reports:
|
||||
xmpp_report(['🔧 自检系统自动修复报备：', f'发现 {len(failures)} 个功能异常，已直接处理：'] + reports +
|
||||
['', '详情: gateway/logs/repair_log.jsonl'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user