265 lines
10 KiB
Python
265 lines
10 KiB
Python
#!/usr/bin/env python3
|
||
"""self_repair.py — L3 LLM 修复循环（报备制）
|
||
|
||
读取 L1 functional_health.json + L2 hygiene_report.json 的失败项，
|
||
调用 LLM 诊断并**直接执行修复**（先斩后奏），然后记录日志并 XMPP 报备。
|
||
|
||
安全设计：
|
||
- LLM 只能从白名单动作中选择（不许任意执行代码）
|
||
- 每个模块每天最多自动修复 2 次（防修复循环）
|
||
- LLM 不可用时降级为规则默认动作（rerun_script）
|
||
- 所有动作记录 repair_log.jsonl
|
||
|
||
频率：每 30 分钟（cron）。
|
||
"""
|
||
import os, sys, json, subprocess, sqlite3, time
|
||
from datetime import datetime
|
||
|
||
# ── 消息通道统一路由(broadcast/xmpp by delivery) ──
|
||
try:
|
||
from messenger import install_stdio_hook as _msh
|
||
_msh()
|
||
except Exception:
|
||
pass
|
||
|
||
sys.path.insert(0, '/home/hmo/MoFin')
|
||
|
||
FUNCTIONAL_REPORT = '/home/hmo/MoFin/gateway/logs/functional_health.json'
|
||
HYGIENE_REPORT = '/home/hmo/MoFin/gateway/logs/hygiene_report.json'
|
||
REPAIR_LOG = '/home/hmo/MoFin/gateway/logs/repair_log.jsonl'
|
||
REPAIR_STATE = '/home/hmo/MoFin/gateway/logs/repair_state.json'
|
||
GATEWAY = 'http://127.0.0.1:8643/v1/chat/completions'
|
||
SCRIPTS_DIR = '/home/hmo/.hermes/profiles/position-analyst/scripts'
|
||
MAX_REPAIRS_PER_MODULE_PER_DAY = 2
|
||
|
||
WHITELIST_ACTIONS = """
|
||
可执行的白名单动作（只能选其一，不许自创）：
|
||
1. {"action": "rerun_script", "script": "<脚本名>"} — 立即重跑指定 cron 脚本（限注册表内的脚本）
|
||
2. {"action": "restart_service", "service": "<服务名>"} — systemctl 重启（限: hermes-gateway-zhiwei, xmpp-zhiwei, mofin-dashboard）
|
||
3. {"action": "sync_links"} — 重建脚本硬链接（修复 cron 跑旧代码）
|
||
4. {"action": "switch_llm_key"} — 切换 LLM key（429 限流时）
|
||
5. {"action": "none", "reason": "<原因>"} — 判断为无需动作（如非交易时段的正常空闲）
|
||
"""
|
||
|
||
|
||
def log_repair(entry):
|
||
os.makedirs(os.path.dirname(REPAIR_LOG), exist_ok=True)
|
||
with open(REPAIR_LOG, 'a', encoding='utf-8') as f:
|
||
f.write(json.dumps(entry, ensure_ascii=False) + '\n')
|
||
|
||
|
||
def load_state():
|
||
try:
|
||
return json.load(open(REPAIR_STATE))
|
||
except Exception:
|
||
return {}
|
||
|
||
|
||
def save_state(s):
|
||
with open(REPAIR_STATE, 'w') as f:
|
||
json.dump(s, f)
|
||
|
||
|
||
def repairs_today(state, module):
|
||
today = datetime.now().strftime('%Y-%m-%d')
|
||
return state.get(today, {}).get(module, 0)
|
||
|
||
|
||
def bump_repair(state, module):
|
||
today = datetime.now().strftime('%Y-%m-%d')
|
||
state.setdefault(today, {})
|
||
state[today][module] = state[today].get(module, 0) + 1
|
||
# 只保留最近 3 天
|
||
for k in list(state.keys()):
|
||
if k < (datetime.now().strftime('%Y-%m-%d'))[:8] + '17':
|
||
del state[k]
|
||
save_state(state)
|
||
|
||
|
||
def collect_failures():
|
||
failures = []
|
||
try:
|
||
fh = json.load(open(FUNCTIONAL_REPORT))
|
||
for c in fh.get('checks', []):
|
||
if c.get('status') == 'fail':
|
||
failures.append({
|
||
'source': 'functional', 'module': c['module'],
|
||
'function': c['function'], 'reason': c['reason'],
|
||
'repair_hint': c.get('repair'),
|
||
})
|
||
except Exception:
|
||
pass
|
||
try:
|
||
hy = json.load(open(HYGIENE_REPORT))
|
||
for i in hy.get('issues', []):
|
||
failures.append({
|
||
'source': 'hygiene', 'module': i.get('type'),
|
||
'function': i.get('file') or i.get('job') or i.get('table', '?'),
|
||
'reason': json.dumps(i, ensure_ascii=False)[:200],
|
||
'repair_hint': {'action': 'llm_diagnose'},
|
||
})
|
||
except Exception:
|
||
pass
|
||
return failures
|
||
|
||
|
||
def run_action(action, module):
|
||
"""执行白名单动作，返回 (ok, detail)"""
|
||
act = action.get('action')
|
||
try:
|
||
if act == 'rerun_script':
|
||
script = action.get('script', '')
|
||
# 只允许注册表内脚本
|
||
if not script.endswith('.py') or '/' in script or '..' in script:
|
||
return False, f'非法脚本名: {script}'
|
||
path = os.path.join(SCRIPTS_DIR, script)
|
||
if not os.path.exists(path):
|
||
return False, f'脚本不存在: {script}'
|
||
r = subprocess.run(['python3', path], capture_output=True, text=True,
|
||
timeout=600, cwd=SCRIPTS_DIR)
|
||
tail = (r.stdout or r.stderr)[-150:]
|
||
return r.returncode == 0, f'exit={r.returncode} {tail}'
|
||
|
||
elif act == 'restart_service':
|
||
svc = action.get('service', '')
|
||
allowed = {'hermes-gateway-zhiwei': ['sudo', '-n', 'systemctl', 'restart', svc],
|
||
'xmpp-zhiwei': ['sudo', '-n', 'systemctl', 'restart', svc],
|
||
'mofin-dashboard': ['sudo', '-n', 'systemctl', 'restart', svc]}
|
||
if svc not in allowed:
|
||
return False, f'服务不在白名单: {svc}'
|
||
subprocess.Popen(allowed[svc], stdout=subprocess.DEVNULL,
|
||
stderr=subprocess.DEVNULL, start_new_session=True)
|
||
return True, f'{svc} 重启已触发(异步)'
|
||
|
||
elif act == 'sync_links':
|
||
r = subprocess.run(['bash', '/home/hmo/MoFin/deploy/profile-scripts/sync_profile_scripts.sh'],
|
||
capture_output=True, text=True, timeout=60)
|
||
return True, r.stdout.strip()
|
||
|
||
elif act == 'switch_llm_key':
|
||
sys.path.insert(0, '/home/hmo/MoFin')
|
||
from xmpp_logger import best_key, switch_key, KEY_TO_PROVIDER, current_provider
|
||
bk = best_key()
|
||
if not bk:
|
||
return False, '无可用 key'
|
||
target = KEY_TO_PROVIDER.get(bk['key_id'])
|
||
if not target or target == current_provider():
|
||
return False, f'已在最优 key ({bk["key_id"]})'
|
||
r = switch_key(bk['key_id'])
|
||
return r.get('switched', False), json.dumps(r, ensure_ascii=False)[:150]
|
||
|
||
elif act == 'none':
|
||
return True, f'无需动作: {action.get("reason", "")}'
|
||
|
||
return False, f'未知动作: {act}'
|
||
except subprocess.TimeoutExpired:
|
||
return False, '动作超时(600s)'
|
||
except Exception as e:
|
||
return False, f'动作异常: {str(e)[:120]}'
|
||
|
||
|
||
def llm_diagnose(failure):
|
||
"""调 LLM 诊断并返回白名单动作。LLM 不可用 → 返回 None（走规则默认）"""
|
||
hint = failure.get('repair_hint') or {}
|
||
default_action = hint if hint.get('action') and hint['action'] != 'llm_diagnose' else None
|
||
|
||
prompt = f"""你是 MoFin 系统的自动修复工程师。一个健康检查发现了功能异常，请诊断并给出一个修复动作。
|
||
|
||
【异常信息】
|
||
模块: {failure['module']}
|
||
功能: {failure['function']}
|
||
异常表现: {failure['reason']}
|
||
来源: {failure['source']}
|
||
|
||
【背景】
|
||
- 系统: Linux 246, MoFin 股票分析系统
|
||
- cron 脚本目录: {SCRIPTS_DIR}
|
||
- 数据库: /home/hmo/MoFin/data/mofin.db
|
||
- 当前时间: {datetime.now().strftime('%Y-%m-%d %H:%M')}（注意是否交易时段，非交易时段部分管道停跑属正常）
|
||
|
||
【注册表推荐修复动作】{json.dumps(hint, ensure_ascii=False) if hint else '无'}
|
||
（判断为真实异常时，rerun_script 的 script 名必须严格用推荐动作里的，禁止自编）
|
||
|
||
{WHITELIST_ACTIONS}
|
||
|
||
请只输出一个 JSON 动作（不要其他文字）。如果是非交易时段的正常空闲，选 none。"""
|
||
|
||
try:
|
||
# llm_client：OCG 直连，无 agent 无常驻 session（原 gateway 'self-repair'
|
||
# session 指令冻结+上下文无限累积，2026-07-21 废弃）
|
||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||
from llm_client import call_llm
|
||
res = call_llm(prompt, max_tokens=None, timeout=90, retries=0, backoff=0, concurrent=True) # 文档: 不指定max_tokens
|
||
if not res["ok"]:
|
||
raise RuntimeError(res["error"][:120])
|
||
text = res["content"]
|
||
# 提取 JSON
|
||
import re
|
||
m = re.search(r'\{[^{}]*"action"[^{}]*\}', text)
|
||
if m:
|
||
action = json.loads(m.group(0))
|
||
if action.get('action') in ('rerun_script', 'restart_service', 'sync_links',
|
||
'switch_llm_key', 'none'):
|
||
# 2026-08-24 script名校正：LLM常把模块名当脚本名编(如mtf_cache→非法)，
|
||
# 注册表 hint 有标准脚本名时强制使用（空转修复循环根治）
|
||
if action.get('action') == 'rerun_script' and hint.get('script'):
|
||
action['script'] = hint['script']
|
||
return action, text[:200]
|
||
except Exception as e:
|
||
print(f' LLM 诊断不可用: {str(e)[:80]}')
|
||
return (default_action, None) if default_action else (None, None)
|
||
|
||
|
||
def xmpp_report(lines):
|
||
try:
|
||
import sys as _s, os as _o
|
||
_s.path.insert(0, _o.path.dirname(_o.path.abspath(__file__)))
|
||
from alert_helper import notify, INFO
|
||
notify("L3自愈", '\n'.join(lines), INFO)
|
||
except Exception as e:
|
||
print(f'XMPP 失败: {e}')
|
||
|
||
|
||
def main():
|
||
now = datetime.now()
|
||
print('🔧 self_repair', now.strftime('%H:%M'))
|
||
failures = collect_failures()
|
||
if not failures:
|
||
print(' ✅ 无失败项')
|
||
return
|
||
|
||
state = load_state()
|
||
reports = []
|
||
for f in failures:
|
||
module = f['module']
|
||
if repairs_today(state, module) >= MAX_REPAIRS_PER_MODULE_PER_DAY:
|
||
print(f' ⏭ {module}: 今日已修复 {MAX_REPAIRS_PER_MODULE_PER_DAY} 次，跳过（防循环）')
|
||
continue
|
||
|
||
print(f' 🔍 {module}: {f["reason"][:60]}')
|
||
action, llm_note = llm_diagnose(f)
|
||
if action is None:
|
||
print(f' 无可用动作，跳过')
|
||
log_repair({'ts': now.isoformat(), 'module': module, 'reason': f['reason'],
|
||
'action': 'skip_no_action', 'ok': None})
|
||
continue
|
||
|
||
ok, detail = run_action(action, module)
|
||
bump_repair(state, module)
|
||
entry = {
|
||
'ts': now.isoformat(), 'module': module, 'function': f['function'],
|
||
'reason': f['reason'], 'action': action, 'ok': ok, 'detail': detail[:300],
|
||
'llm_note': llm_note,
|
||
}
|
||
log_repair(entry)
|
||
icon = '✅' if ok else '❌'
|
||
print(f' {icon} {action.get("action")} -> {detail[:80]}')
|
||
reports.append(f'{icon} [{module}] {action.get("action")}: {detail[:80]}')
|
||
|
||
if reports:
|
||
xmpp_report(['📟【MoFin系统·L3自愈】（非知微本人）自动修复报备：', f'发现 {len(failures)} 个功能异常，已直接处理：'] + reports +
|
||
['', '详情: gateway/logs/repair_log.jsonl'])
|
||
|
||
|
||
if __name__ == '__main__':
|
||
main() |