123 lines
5.6 KiB
Python
123 lines
5.6 KiB
Python
#!/usr/bin/env python3
|
||
"""meta_watchdog.py — L4 自检系统的自检（看门狗的看门狗）
|
||
|
||
检查 L0-L3 各自检组件本身是否在正常运转：
|
||
- L0 agents_health_check: last_health_check.json 是否 <10min
|
||
- L1 functional_health_check: functional_health.json 是否 <20min（交易时段）
|
||
- L2 system_hygiene_audit: hygiene_report.json 是否 <26h（每日）
|
||
- L3 self_repair: repair_state.json 存在性 + cron 是否注册
|
||
- mofin_health 采集: mofin_health.json 是否 <20min（交易时段）
|
||
- XMPP 桥: :5805 是否可发（self_repair 的报备通道）
|
||
|
||
任何一层死了 → 推 XMPP 点名（这是最后的兜底，必须直达用户）。
|
||
频率：每小时（cron）。输出 gateway/logs/meta_watchdog.json。
|
||
"""
|
||
import os, sys, json, subprocess
|
||
from datetime import datetime
|
||
|
||
OUT = '/home/hmo/MoFin/gateway/logs/meta_watchdog.json'
|
||
|
||
LAYERS = [
|
||
{"layer": "L0 agents_health_check", "file": "/home/hmo/MoFin/gateway/temp/last_health_check.json",
|
||
"max_age_min": 10, "when": "always",
|
||
"repair": "crontab */5 agents_health_check.py 停摆，检查系统 crontab"},
|
||
{"layer": "L1 functional_health", "file": "/home/hmo/MoFin/gateway/logs/functional_health.json",
|
||
"max_age_min": 25, "when": "trading",
|
||
"repair": "L1 cron 停摆，检查 hermes cron 引擎"},
|
||
{"layer": "L2 hygiene_audit", "file": "/home/hmo/MoFin/gateway/logs/hygiene_report.json",
|
||
"max_age_min": 26 * 60, "when": "always",
|
||
"repair": "L2 每日审计未跑，检查 hermes cron"},
|
||
{"layer": "L1.5 mofin_health采集", "file": "/home/hmo/web-dashboard/static/mofin_health.json",
|
||
"max_age_min": 25, "when": "trading",
|
||
"repair": "mofin_health.py 采集停摆"},
|
||
{"layer": "L3 self_repair", "file": "/home/hmo/MoFin/gateway/logs/repair_log.jsonl",
|
||
"max_age_min": None, "when": "meta",
|
||
"repair": "self_repair cron 未注册"},
|
||
]
|
||
|
||
|
||
def is_trading(now):
|
||
return now.weekday() < 5 and 9 <= now.hour <= 16
|
||
|
||
|
||
def main():
|
||
now = datetime.now()
|
||
trading = is_trading(now)
|
||
results = []
|
||
|
||
for L in LAYERS:
|
||
if L["when"] == "trading" and not trading:
|
||
results.append({"layer": L["layer"], "status": "skip", "reason": "非交易时段"})
|
||
continue
|
||
if L["when"] == "meta":
|
||
# 检查 self_repair 是否注册在 cron
|
||
try:
|
||
d = json.load(open('/home/hmo/.hermes/profiles/position-analyst/cron/jobs.json'))
|
||
jobs = d if isinstance(d, list) else d.get('jobs', [])
|
||
registered = any(j.get('script') == 'self_repair.py' and j.get('enabled', True) for j in jobs)
|
||
results.append({"layer": L["layer"],
|
||
"status": "ok" if registered else "fail",
|
||
"reason": "已注册" if registered else "未在 cron 注册"})
|
||
except Exception as e:
|
||
results.append({"layer": L["layer"], "status": "fail", "reason": str(e)[:60]})
|
||
continue
|
||
|
||
f = L["file"]
|
||
if not os.path.exists(f):
|
||
results.append({"layer": L["layer"], "status": "fail",
|
||
"reason": f"输出物不存在", "repair": L["repair"]})
|
||
continue
|
||
age_min = (now.timestamp() - os.path.getmtime(f)) / 60
|
||
if L["max_age_min"] and age_min > L["max_age_min"]:
|
||
results.append({"layer": L["layer"], "status": "fail",
|
||
"reason": f"输出物 {age_min/60:.1f}h 未更新（阈值 {L['max_age_min']}min）",
|
||
"repair": L["repair"]})
|
||
else:
|
||
results.append({"layer": L["layer"], "status": "ok",
|
||
"reason": f"{age_min:.0f}min 前"})
|
||
|
||
# XMPP 桥（报备通道）：只收 POST，GET 会 501，但任何 HTTP 响应都说明进程活着
|
||
try:
|
||
import urllib.request
|
||
urllib.request.urlopen('http://127.0.0.1:5805/', timeout=3)
|
||
results.append({"layer": "XMPP桥 :5805", "status": "ok", "reason": "可达"})
|
||
except urllib.error.HTTPError as e:
|
||
results.append({"layer": "XMPP桥 :5805", "status": "ok", "reason": f"可达(HTTP {e.code})"})
|
||
except Exception:
|
||
results.append({"layer": "XMPP桥 :5805", "status": "fail",
|
||
"reason": "不可达", "repair": "重启 xmpp-zhiwei"})
|
||
|
||
fails = [r for r in results if r["status"] == "fail"]
|
||
report = {
|
||
"generated_at": now.strftime("%Y-%m-%d %H:%M:%S"),
|
||
"status": "fail" if fails else "ok",
|
||
"layers": results,
|
||
}
|
||
os.makedirs(os.path.dirname(OUT), exist_ok=True)
|
||
with open(OUT, 'w', encoding='utf-8') as f:
|
||
json.dump(report, f, ensure_ascii=False, indent=2)
|
||
|
||
print(f"meta_watchdog: {report['status']}")
|
||
for r in results:
|
||
icon = {"ok": "✅", "fail": "❌", "skip": "⏭"}[r["status"]]
|
||
print(f" {icon} {r['layer']}: {r['reason']}")
|
||
|
||
if fails:
|
||
try:
|
||
import urllib.request
|
||
lines = [f"🚨 自检系统自检（L4兜底）发现 {len(fails)} 层异常："]
|
||
for r in fails:
|
||
lines.append(f"❌ {r['layer']}: {r['reason']}")
|
||
if r.get('repair'):
|
||
lines.append(f" → 处置建议: {r['repair']}")
|
||
payload = json.dumps({'to': 'hmo@yoin.fun', 'body': '\n'.join(lines), 'type': 'chat'}).encode()
|
||
req = urllib.request.Request('http://127.0.0.1:5805/', data=payload,
|
||
headers={'Content-Type': 'application/json'})
|
||
urllib.request.urlopen(req, timeout=5)
|
||
print(' 📨 已推 XMPP（兜底直达）')
|
||
except Exception as e:
|
||
print(f' XMPP 失败: {e}')
|
||
|
||
|
||
if __name__ == '__main__':
|
||
main() |