MoFin/deploy/profile-scripts/meta_watchdog.py

#!/usr/bin/env python3
"""meta_watchdog.py — L4 自检系统的自检（看门狗的看门狗）

检查 L0-L3 各自检组件本身是否在正常运转：
- L0 agents_health_check: last_health_check.json 是否 <10min
- L1 functional_health_check: functional_health.json 是否 <20min（交易时段）
- L2 system_hygiene_audit: hygiene_report.json 是否 <26h（每日）
- L3 self_repair: repair_state.json 存在性 + cron 是否注册
- mofin_health 采集: mofin_health.json 是否 <20min（交易时段）
- XMPP 桥: :5805 是否可发（self_repair 的报备通道）

任何一层死了 → 推 XMPP 点名（这是最后的兜底，必须直达用户）。
频率：每小时（cron）。输出 gateway/logs/meta_watchdog.json。
"""
import os, sys, json, subprocess
from datetime import datetime

OUT = '/home/hmo/MoFin/gateway/logs/meta_watchdog.json'

LAYERS = [
    {"layer": "L0 agents_health_check", "file": "/home/hmo/MoFin/gateway/temp/last_health_check.json",
     "max_age_min": 10, "when": "always",
     "repair": "crontab */5 agents_health_check.py 停摆，检查系统 crontab"},
    {"layer": "L1 functional_health", "file": "/home/hmo/MoFin/gateway/logs/functional_health.json",
     "max_age_min": 25, "when": "trading",
     "repair": "L1 cron 停摆，检查 hermes cron 引擎"},
    {"layer": "L2 hygiene_audit", "file": "/home/hmo/MoFin/gateway/logs/hygiene_report.json",
     "max_age_min": 26 * 60, "when": "always",
     "repair": "L2 每日审计未跑，检查 hermes cron"},
    {"layer": "L1.5 mofin_health采集", "file": "/home/hmo/web-dashboard/static/mofin_health.json",
     "max_age_min": 25, "when": "trading",
     "repair": "mofin_health.py 采集停摆"},
    {"layer": "L3 self_repair", "file": "/home/hmo/MoFin/gateway/logs/repair_log.jsonl",
     "max_age_min": None, "when": "meta",
     "repair": "self_repair cron 未注册"},
]


def is_trading(now):
    return now.weekday() < 5 and 9 <= now.hour <= 16


def main():
    now = datetime.now()
    trading = is_trading(now)
    results = []

    for L in LAYERS:
        if L["when"] == "trading" and not trading:
            results.append({"layer": L["layer"], "status": "skip", "reason": "非交易时段"})
            continue
        if L["when"] == "meta":
            # 检查 self_repair 是否注册在 cron
            try:
                d = json.load(open('/home/hmo/.hermes/profiles/position-analyst/cron/jobs.json'))
                jobs = d if isinstance(d, list) else d.get('jobs', [])
                registered = any(j.get('script') == 'self_repair.py' and j.get('enabled', True) for j in jobs)
                results.append({"layer": L["layer"],
                                "status": "ok" if registered else "fail",
                                "reason": "已注册" if registered else "未在 cron 注册"})
            except Exception as e:
                results.append({"layer": L["layer"], "status": "fail", "reason": str(e)[:60]})
            continue

        f = L["file"]
        if not os.path.exists(f):
            results.append({"layer": L["layer"], "status": "fail",
                            "reason": f"输出物不存在", "repair": L["repair"]})
            continue
        age_min = (now.timestamp() - os.path.getmtime(f)) / 60
        if L["max_age_min"] and age_min > L["max_age_min"]:
            results.append({"layer": L["layer"], "status": "fail",
                            "reason": f"输出物 {age_min/60:.1f}h 未更新（阈值 {L['max_age_min']}min）",
                            "repair": L["repair"]})
        else:
            results.append({"layer": L["layer"], "status": "ok",
                            "reason": f"{age_min:.0f}min 前"})

    # XMPP 桥（报备通道）：只收 POST，GET 会 501，但任何 HTTP 响应都说明进程活着
    try:
        import urllib.request
        urllib.request.urlopen('http://127.0.0.1:5805/', timeout=3)
        results.append({"layer": "XMPP桥 :5805", "status": "ok", "reason": "可达"})
    except urllib.error.HTTPError as e:
        results.append({"layer": "XMPP桥 :5805", "status": "ok", "reason": f"可达(HTTP {e.code})"})
    except Exception:
        results.append({"layer": "XMPP桥 :5805", "status": "fail",
                        "reason": "不可达", "repair": "重启 xmpp-zhiwei"})

    fails = [r for r in results if r["status"] == "fail"]
    report = {
        "generated_at": now.strftime("%Y-%m-%d %H:%M:%S"),
        "status": "fail" if fails else "ok",
        "layers": results,
    }
    os.makedirs(os.path.dirname(OUT), exist_ok=True)
    with open(OUT, 'w', encoding='utf-8') as f:
        json.dump(report, f, ensure_ascii=False, indent=2)

    print(f"meta_watchdog: {report['status']}")
    for r in results:
        icon = {"ok": "✅", "fail": "❌", "skip": "⏭"}[r["status"]]
        print(f"  {icon} {r['layer']}: {r['reason']}")

    if fails:
        try:
            import urllib.request
            lines = [f"🚨 自检系统自检（L4兜底）发现 {len(fails)} 层异常："]
            for r in fails:
                lines.append(f"❌ {r['layer']}: {r['reason']}")
                if r.get('repair'):
                    lines.append(f"   → 处置建议: {r['repair']}")
            payload = json.dumps({'to': 'hmo@yoin.fun', 'body': '\n'.join(lines), 'type': 'chat'}).encode()
            req = urllib.request.Request('http://127.0.0.1:5805/', data=payload,
                                         headers={'Content-Type': 'application/json'})
            urllib.request.urlopen(req, timeout=5)
            print('  📨 已推 XMPP（兜底直达）')
        except Exception as e:
            print(f'  XMPP 失败: {e}')


if __name__ == '__main__':
    main()