feat(auto_heal): multi-profile key switching + fix two error classes

Error investigation (from restored monitoring) found 2 root causes:

1. 'Blocked: script path resolves outside scripts dir' (5+ cron jobs):
   Jul 17 symlink refactor replaced real scripts with symlinks; the hermes
   cron scheduler's security check (Path.resolve + relative_to) rejects
   symlink escape. ALL no_agent script jobs blocked since Jul 17 23:12.
   FIX: converted 102 symlinks to hardlinks (same inode, resolve() stays
   inside scripts_dir, single-source still works). Permanent structural fix.

2. HTTP 429 on default profile (知识研究/梦境循环/wiki-self-growth/
   evolution-pulse/大脑任务执行): default gateway used ocg-key1 (weekly
   100%). FIX: switched default profile to ocg-key6 + added missing
   provider block. LLM verified working (3.1s).

auto_heal extended to actually cover these automatically next time:
- PROFILES registry: zhiwei (8643, system svc) + default (8642, user svc)
- current_provider/switch_key/_scan_agent_log parameterized by profile
- health() now reports llm_provider_default (agent.log scan)
- auto_heal: per-profile 429 detection -> best_key -> switch_key(profile)
- _ensure_provider_block: injects missing provider credentials from
  zhiwei config (single source of truth) into target config

scripts/analyze_health.py

@@ -0,0 +1,45 @@
+import json
+
+d = json.load(open('/tmp/mofin_health.json'))
+
+print("generated_at:", d.get('generated_at'))
+print()
+
+# 1. Pipelines with error/warn status
+print("=== PIPELINES (error/warn) ===")
+for p in d.get('pipelines', []):
+    if p.get('status') in ('error', 'fail', 'warn'):
+        print(f"[{p['status']}] {p.get('name') or p.get('script')} | profile={p.get('profile')} | type={p.get('type')} | schedule={p.get('schedule')} | last_run={p.get('last_run')}")
+
+print()
+print("=== FEATURE TREE (non-ok nodes) ===")
+def walk(n, path=''):
+    label = n.get('label', '?')
+    p = f"{path}/{label}"
+    if n.get('status') not in ('ok', None):
+        print(f"[{n.get('status')}] {p}")
+        for pipe in n.get('pipes', []):
+            if pipe.get('status') != 'ok':
+                print(f"    pipe: [{pipe.get('status')}] {pipe.get('name') or pipe.get('script')} | last_run={pipe.get('last_run')}")
+    for c in n.get('children', []):
+        walk(c, p)
+walk(d.get('feature_tree', {}))
+
+print()
+print("=== DATA ENTITIES (orphan/write_only) ===")
+for e in d.get('entities', []):
+    if e.get('flow_status') in ('orphan', 'write_only'):
+        print(f"[{e['flow_status']}] {e['name']} | rows={e.get('rows')} | writers={e.get('writers')} | readers={e.get('readers')} | {e.get('desc','')[:60]}")
+
+print()
+print("=== ARCHITECTURE violations ===")
+arch = d.get('architecture', {})
+print("violation_count:", arch.get('violation_count'))
+for v in (arch.get('price_api_violations') or [])[:10]:
+    print(f"  {v.get('script')} L{v.get('line')}")
+
+print()
+print("=== JSON files with warn ===")
+for j in d.get('json_files', []):
+    if j.get('warn'):
+        print(f"[warn] {j['name']} | {j.get('desc','')[:60]} | readers={j.get('readers')}")

scripts/check_missing_scripts.py

@@ -0,0 +1,31 @@
+import os
+
+files = ['meta_growth.py', 'macro_context_collector.py', 'divergence_detector.py',
+         'memory_guardian.py', 'fix_gateway_port.py']
+d = '/home/hmo/.hermes/profiles/position-analyst/scripts/'
+for f in files:
+    p = os.path.join(d, f)
+    if os.path.islink(p):
+        target = os.readlink(p)
+        ok = os.path.exists(p)
+        print(f"{f} -> symlink -> {target} [{'OK' if ok else 'BROKEN'}]")
+    elif os.path.exists(p):
+        print(f"{f} -> real file")
+    else:
+        print(f"{f} -> MISSING")
+
+print()
+print('=== deploy/profile-scripts candidates ===')
+deploy = '/home/hmo/MoFin/deploy/profile-scripts/'
+for f in os.listdir(deploy):
+    if any(k in f for k in ['meta_growth', 'macro_context', 'divergence', 'memory_guardian', 'fix_gateway']):
+        print(' ', f)
+
+print()
+print('=== search whole MoFin for the 5 scripts ===')
+import subprocess
+for f in files:
+    r = subprocess.run(['find', '/home/hmo/MoFin', '/home/hmo/projects', '-name', f,
+                        '-not', '-path', '*/venv/*'], capture_output=True, text=True, timeout=30)
+    found = [l for l in r.stdout.splitlines() if l.strip()]
+    print(f"{f}: {found if found else 'NOT FOUND'}")

scripts/cron_status.py

@@ -0,0 +1,31 @@
+import json
+from datetime import datetime, timezone, timedelta
+
+d = json.load(open('/home/hmo/.hermes/profiles/position-analyst/cron/jobs.json'))
+jobs = d if isinstance(d, list) else d.get('jobs', [])
+now = datetime.now(timezone.utc)
+print(f"{'name':32} {'last_run':20} {'status':8} {'next_run':20} {'en'}")
+print('-' * 95)
+for j in sorted(jobs, key=lambda x: x.get('last_run_at') or ''):
+    if not j.get('enabled', True):
+        continue
+    lr = (j.get('last_run_at') or '?')[:19]
+    nr = (j.get('next_run_at') or '?')[:19]
+    st = str(j.get('last_status', '?'))
+    print(f"{j.get('name','?')[:32]:32} {lr:20} {st:8} {nr:20} {j.get('enabled')}")
+
+# also check gateway-side cron (default profile)
+print()
+print('=== default profile cron ===')
+try:
+    d2 = json.load(open('/home/hmo/.hermes/cron/jobs.json'))
+    jobs2 = d2 if isinstance(d2, list) else d2.get('jobs', [])
+    for j in sorted(jobs2, key=lambda x: x.get('last_run_at') or ''):
+        if not j.get('enabled', True):
+            continue
+        lr = (j.get('last_run_at') or '?')[:19]
+        nr = (j.get('next_run_at') or '?')[:19]
+        st = str(j.get('last_status', '?'))
+        print(f"{j.get('name','?')[:32]:32} {lr:20} {st:8} {nr:20} {j.get('enabled')}")
+except Exception as e:
+    print('err:', e)

scripts/find_cron_errors.py

@@ -0,0 +1,15 @@
+import json, os, glob
+
+# find job ids for the failing jobs
+targets = {
+    'pa': ('/home/hmo/.hermes/profiles/position-analyst/cron/jobs.json',
+           ['大脑任务执行', 'Gateway看门狗-知微', '元自成长-每日', '记忆守卫-每日', '宏观新闻采集', '跨市场背离检测']),
+    'default': ('/home/hmo/.hermes/cron/jobs.json',
+                ['大脑任务执行', 'evolution-pulse', 'wiki-self-growth', '知识研究-日常', '梦境循环-知识库归并']),
+}
+for label, (jf, names) in targets.items():
+    d = json.load(open(jf))
+    jobs = d if isinstance(d, list) else d.get('jobs', [])
+    for j in jobs:
+        if j.get('name') in names:
+            print(f"{label} | {j['name']} | id={j.get('id')} | status={j.get('last_status')} | error={str(j.get('last_error'))[:200]}")

scripts/fix_symlinks.py

@@ -0,0 +1,33 @@
+import os, sys
+
+scripts_dir = '/home/hmo/.hermes/profiles/position-analyst/scripts'
+fixed, skipped, errors = 0, 0, []
+for name in os.listdir(scripts_dir):
+    p = os.path.join(scripts_dir, name)
+    if not os.path.islink(p):
+        skipped += 1
+        continue
+    target = os.path.realpath(p)
+    if not os.path.exists(target):
+        errors.append(f"{name}: broken symlink -> {target}")
+        continue
+    try:
+        os.unlink(p)
+        os.link(target, p)  # hardlink: same inode, .resolve() stays in scripts_dir
+        fixed += 1
+    except Exception as e:
+        errors.append(f"{name}: {e}")
+
+print(f"fixed: {fixed}, skipped (non-symlink): {skipped}, errors: {len(errors)}")
+for e in errors:
+    print(' ', e)
+
+# verify with the scheduler's own check
+from pathlib import Path
+scripts_resolved = Path(scripts_dir).resolve()
+test = (Path(scripts_dir) / 'meta_growth.py').resolve()
+try:
+    test.relative_to(scripts_resolved)
+    print('VERIFY OK: meta_growth.py now passes relative_to check')
+except ValueError:
+    print('VERIFY FAIL: still resolves outside')

scripts/test_multi_profile.py

@@ -0,0 +1,22 @@
+import ast, sys, json
+p = '/home/hmo/MoFin/xmpp_logger.py'
+try:
+    ast.parse(open(p).read())
+    print('SYNTAX OK')
+except SyntaxError as e:
+    print('SYNTAX ERROR:', e)
+    sys.exit(1)
+
+sys.path.insert(0, '/home/hmo/MoFin')
+import importlib, xmpp_logger
+importlib.reload(xmpp_logger)
+x = xmpp_logger
+import time
+
+print('== current_provider(zhiwei):', x.current_provider('zhiwei'))
+print('== current_provider(default):', x.current_provider('default'))
+print('== scan zhiwei:', json.dumps(x._scan_agent_log(time.time(), 'zhiwei'), ensure_ascii=False))
+print('== scan default:', json.dumps(x._scan_agent_log(time.time(), 'default'), ensure_ascii=False))
+print()
+print('== auto_heal ==')
+print(json.dumps(x.auto_heal(), ensure_ascii=False, indent=2))

scripts/validate_fixes.py

@@ -0,0 +1,37 @@
+import json, urllib.request
+from pathlib import Path
+
+print("=== 验证1: 5个被 Blocked 的脚本现在是否通过调度器路径检查 ===")
+scripts_dir = Path('/home/hmo/.hermes/profiles/position-analyst/scripts')
+resolved = scripts_dir.resolve()
+files = ['meta_growth.py', 'macro_context_collector.py', 'divergence_detector.py',
+         'memory_guardian.py', 'fix_gateway_port.py']
+all_ok = True
+for f in files:
+    p = (scripts_dir / f).resolve()
+    try:
+        p.relative_to(resolved)
+        exists = p.exists()
+        print(f"  PASS {f} (exists={exists})")
+        if not exists:
+            all_ok = False
+    except ValueError:
+        all_ok = False
+        print(f"  FAIL {f} still blocked")
+print('  =>', 'ALL PASS' if all_ok else 'STILL FAILING')
+
+print()
+print("=== 验证2: default gateway (8642) LLM 调用（原 key1 429，现 key6）===")
+payload = json.dumps({'model': 'deepseek-v4-flash',
+                      'messages': [{'role': 'user', 'content': 'reply with one word: ok'}],
+                      'max_tokens': 10}).encode()
+req = urllib.request.Request('http://127.0.0.1:8642/v1/chat/completions', data=payload,
+                             headers={'Content-Type': 'application/json',
+                                      'Authorization': 'Bearer hermes123'})
+try:
+    resp = urllib.request.urlopen(req, timeout=90)
+    d = json.loads(resp.read().decode())
+    content = d.get('choices', [{}])[0].get('message', {}).get('content', '')
+    print('  LLM OK:', content[:100])
+except Exception as e:
+    print('  LLM FAIL:', e)

xmpp_logger.py

@@ -19,9 +19,26 @@ LOG_DIR = Path(__file__).resolve().parent / "gateway" / "logs"
 LOG_FILE = LOG_DIR / "xmpp_messages.jsonl"
 MAX_AGE_DAYS = 7
 
-# Position-analyst profile config path (where model.provider is set)
-HERMES_CONFIG = Path("/home/hmo/.hermes/profiles/position-analyst/config.yaml")
-GATEWAY_SERVICE = "hermes-gateway-zhiwei.service"
+# 监控的 Hermes Gateway profiles
+PROFILES = {
+    "zhiwei": {
+        "config": Path("/home/hmo/.hermes/profiles/position-analyst/config.yaml"),
+        "agent_log": Path("/home/hmo/.hermes/profiles/position-analyst/logs/agent.log"),
+        "service": "hermes-gateway-zhiwei.service",
+        "user_service": False,
+        "gateway_port": 8643,
+    },
+    "default": {
+        "config": Path("/home/hmo/.hermes/config.yaml"),
+        "agent_log": Path("/home/hmo/.hermes/logs/agent.log"),
+        "service": "hermes-gateway.service",
+        "user_service": True,
+        "gateway_port": 8642,
+    },
+}
+# 向后兼容（旧引用）
+HERMES_CONFIG = PROFILES["zhiwei"]["config"]
+GATEWAY_SERVICE = PROFILES["zhiwei"]["service"]
 RESTART_COOLDOWN_FILE = LOG_DIR / "last_restart.txt"
 RESTART_COOLDOWN_SEC = 180  # 3 min: don't trigger another restart within 3 min of last
 
@@ -38,15 +55,16 @@ KEY_TO_PROVIDER = {
 }
 
 
-def current_provider() -> str | None:
+def current_provider(profile="zhiwei") -> str | None:
     """Read current model.provider from Hermes config.
 
     Walks the file tracking the model: block to find its nested `provider:`
     (avoiding other blocks like `agent.alerts[0].provider:`).
     """
+    cfg_path = PROFILES.get(profile, PROFILES["zhiwei"])["config"]
     try:
         in_model = False
-        for line in HERMES_CONFIG.read_text().splitlines():
+        for line in cfg_path.read_text().splitlines():
             stripped = line.rstrip()
             # Detect top-level "model:" at column 0
             if stripped == "model:" or stripped.startswith("model:") and not line.startswith(" "):
@@ -65,50 +83,74 @@ def current_provider() -> str | None:
     return None
 
 
-def switch_key(key_id: str) -> dict:
+def _ensure_provider_block(target_cfg: Path, provider: str) -> bool:
+    """若目标 config 缺少 provider 凭据块，从 zhiwei config（凭据单一事实源）复制注入。"""
+    txt = target_cfg.read_text()
+    if _re.search(rf"^  {provider}:\s*$", txt, flags=_re.MULTILINE):
+        return True
+    src = PROFILES["zhiwei"]["config"].read_text()
+    m = _re.search(rf"(^  {provider}:\n(?:    .+\n)+)", src, flags=_re.MULTILINE)
+    if not m:
+        return False
+    block = m.group(1)
+    # 插到 providers: 区块第一个 provider 之前
+    m2 = _re.search(r"^providers:\s*$", txt, flags=_re.MULTILINE)
+    if not m2:
+        return False
+    txt = txt[:m2.end()] + "\n" + block + txt[m2.end():]
+    target_cfg.write_text(txt)
+    return True
+
+
+def _restart_gateway(profile: str) -> dict:
+    """异步重启指定 profile 的 gateway。"""
+    p = PROFILES[profile]
+    try:
+        if p["user_service"]:
+            _sp.Popen(["systemctl", "--user", "restart", p["service"]],
+                      stdout=_sp.DEVNULL, stderr=_sp.DEVNULL, start_new_session=True)
+        else:
+            _sp.Popen(["sudo", "-n", "systemctl", "restart", p["service"]],
+                      stdout=_sp.DEVNULL, stderr=_sp.DEVNULL, start_new_session=True)
+        return {"success": True, "detail": "restart triggered (async)"}
+    except Exception as e:
+        return {"success": False, "detail": str(e)[:100]}
+
+
+def switch_key(key_id: str, profile: str = "zhiwei") -> dict:
     """Switch Hermes config to use the given key_id's provider, then restart Gateway."""
     provider = KEY_TO_PROVIDER.get(key_id)
     if not provider:
         return {"switched": False, "detail": f"unknown key_id {key_id}"}
 
-    old = current_provider()
+    cfg_path = PROFILES[profile]["config"]
+    old = current_provider(profile)
     if old == provider:
         return {"switched": False, "old": old, "new": provider, "detail": "already on this key"}
 
     try:
-        txt = HERMES_CONFIG.read_text()
+        if not _ensure_provider_block(cfg_path, provider):
+            return {"switched": False, "detail": f"provider {provider} 凭据块缺失且无法注入"}
+        txt = cfg_path.read_text()
         new_txt, n = _re.subn(r"^  provider:\s*\S+\s*$",
                               f"  provider: {provider}", txt, count=1, flags=_re.MULTILINE)
         if n == 0:
             return {"switched": False, "detail": "no provider: line found"}
-        HERMES_CONFIG.write_text(new_txt)
+        cfg_path.write_text(new_txt)
     except Exception as e:
         return {"switched": False, "detail": f"config edit failed: {e}"}
 
-    # systemctl restart blocks until gateway drain completes (60+s).
-    # Fire asynchronously via Popen to avoid blocking cron; verify later.
-    try:
-        _sp.Popen(["sudo", "-n", "systemctl", "restart", GATEWAY_SERVICE],
-                  stdout=_sp.DEVNULL, stderr=_sp.DEVNULL, start_new_session=True)
+    r = _restart_gateway(profile)
     RESTART_COOLDOWN_FILE.write_text(str(_time.time()))
-        ok = True
-        detail = "restart triggered (async)"
-    except Exception as e:
-        ok = False
-        detail = f"restart trigger failed: {e}"
-
-    # Wait long enough for systemd to drain + restart the gateway (max 90s)
-    _time.sleep(45)
-    verify = _verify_llm()
 
     return {
         "switched": True,
         "old": old,
         "new": provider,
         "key_id": key_id,
-        "restart": ok,
-        "verify": verify,
-        "detail": detail,
+        "profile": profile,
+        "restart": r["success"],
+        "detail": r["detail"],
     }
 
 
@@ -340,7 +382,9 @@ def health():
 
         # LLM 健康：扫 gateway agent.log 最近一次真实调用结果（零成本，不发 LLM 请求）
         # agent.log 里每次真实调用都有记录：成功 "API call #N: ... latency=Xs" / 失败 "HTTP 429..."
-        result["llm_provider"] = _scan_agent_log(now_epoch)
+        result["llm_provider"] = _scan_agent_log(now_epoch, "zhiwei")
+        # default profile（莫荷 8642）同样监控——它的 429 曾导致整组 cron 失败
+        result["llm_provider_default"] = _scan_agent_log(now_epoch, "default")
     except Exception as e:
         result["gateways"] = {"error": str(e)[:100]}
 
@@ -365,22 +409,20 @@ def health():
     return result
 
 
-AGENT_LOG = Path("/home/hmo/.hermes/profiles/position-analyst/logs/agent.log")
-
-
-def _scan_agent_log(now_epoch, tail_lines=300):
+def _scan_agent_log(now_epoch, profile="zhiwei", tail_lines=300):
     """扫 gateway agent.log 尾部，取最近一次真实 LLM 调用的结果。
 
     成功行: "2026-07-19 20:04:42,880 INFO ... API call #4: ... latency=16.0s ..."
     失败行: "2026-07-19 19:35:59 WARNING ... API call failed ... HTTP 429: Weekly usage limit reached ..."
     返回 {"status": "ok"|"error"|"unknown", "latency"/"error", "age_sec"}
     """
-    if not AGENT_LOG.exists():
+    log_path = PROFILES.get(profile, PROFILES["zhiwei"])["agent_log"]
+    if not log_path.exists():
         return {"status": "unknown", "error": "agent.log not found"}
     try:
         # 只读尾部（大文件不全读）
-        size = AGENT_LOG.stat().st_size
-        with open(AGENT_LOG, "rb") as f:
+        size = log_path.stat().st_size
+        with open(log_path, "rb") as f:
             f.seek(max(0, size - 64 * 1024))
             chunk = f.read().decode("utf-8", errors="replace")
         lines = [l for l in chunk.splitlines() if l.strip()][-tail_lines:]
@@ -484,53 +526,62 @@ def auto_heal():
     h = health()
     actions = []
 
-    # 0. LLM Provider 异常 → 先查是否有更好的 Key 可切换
-    llm_status = h.get("llm_provider", {}).get("status")
-    llm_error = h.get("llm_provider", {}).get("error", "")
+    # 0. 检查每个 profile 的 LLM 状态（zhiwei + default），429 → 切 key
+    bk_cache = None
+    for profile in ("zhiwei", "default"):
+        llm = h.get("llm_provider" if profile == "zhiwei" else "llm_provider_default", {})
+        llm_status = llm.get("status")
+        llm_error = llm.get("error", "") or ""
 
-    # Detect rate-limit / timeout / error
         is_rate_limited = (llm_status == "rate_limited" or
                            "429" in llm_error or "Weekly usage" in llm_error or
                            "RateLimit" in llm_error)
-    is_timeout_or_error = llm_status in ("timeout", "error")
+        if not is_rate_limited:
+            continue
 
-    if is_rate_limited or is_timeout_or_error:
-        bk = best_key()
-        if bk:
+        if bk_cache is None:
+            bk_cache = best_key()
+        bk = bk_cache
+        if not bk:
+            continue
         actions.append({
             "action": "check_keys",
+            "profile": profile,
             "best_key": bk["key_id"],
-                "current_provider": current_provider(),
+            "current_provider": current_provider(profile),
             "best_provider": KEY_TO_PROVIDER.get(bk["key_id"]),
-                "rolling_pct": bk["rolling"]["usage_percent"],
             "weekly_pct": bk["weekly"]["usage_percent"],
-                "issues": bk["issues"],
         })
 
-            # 如果 best key 对应的 provider 跟当前不同 → 切换
         target_provider = KEY_TO_PROVIDER.get(bk["key_id"])
-            current = current_provider()
+        current = current_provider(profile)
         if target_provider is None:
-                # best key is a non-OCG provider (e.g. key7=kimi) — never switch zhiwei there
             actions.append({
                 "action": "skip_switch",
-                    "reason": f"best key {bk['key_id']} is non-OCG (not in KEY_TO_PROVIDER), zhiwei must stay on deepseek",
-                    "best_key": bk["key_id"],
+                "profile": profile,
+                "reason": f"best key {bk['key_id']} is non-OCG (kimi), stay on deepseek",
             })
         elif target_provider != current:
-                # 只有当 best key 自身至少 weekly ok 才切，否则切过去也白搭
             if bk["weekly"]["status"] == "ok":
-                    action = switch_key(bk["key_id"])
+                action = switch_key(bk["key_id"], profile=profile)
                 action["action_group"] = "switch_key"
+                action["profile"] = profile
                 actions.append(action)
             else:
                 actions.append({
                     "action": "skip_switch",
+                    "profile": profile,
                     "reason": f"best key {bk['key_id']} weekly status {bk['weekly']['status']}, no improvement",
                 })
 
+    # zhiwei LLM timeout/error（非429）→ 重启 Gateway
+    llm_status = h.get("llm_provider", {}).get("status")
+    llm_error = h.get("llm_provider", {}).get("error", "") or ""
+    is_rate_limited_zw = ("429" in llm_error or "Weekly usage" in llm_error or "RateLimit" in llm_error)
+    is_timeout_or_error = llm_status in ("timeout", "error")
+
     # 1. LLM Provider timeout/error（非429）→ 重启 Gateway（不是切 key）
-    if is_timeout_or_error and not is_rate_limited:
+    if is_timeout_or_error and not is_rate_limited_zw:
         gw = h.get("gateways", {}).get("zhiwei", {})
         if gw.get("alive"):
             # Check cooldown to avoid restart loops