hmo
9a359f49bd
User insight: hardlink breakage only happens at deploy time (scp file replacement / git checkout-merge), so detection must be welded INTO the deploy pipeline, not left to daily audit. Three automatic layers, no reliance on discipline: 1. systemd path watcher (profile-scripts-sync.path): watches deploy/profile-scripts/ directory, auto-fires sync_profile_scripts.sh on any change. Verified: fires within 4s of file replacement, logs to gateway/logs/link_sync.log (runs as hmo user) 2. git hooks (.git/hooks/post-merge + post-checkout on 246 repo): auto re-link after git operations 3. Manual fallback: sync_profile_scripts.sh (now self-logging) dev-spec red line #6 updated: SSOT rule now documents the three layers and states breakage only happens at deploy time.
118 lines
5.1 KiB
Python
118 lines
5.1 KiB
Python
import json, os, subprocess, hashlib
|
||
from datetime import datetime
|
||
|
||
now = datetime.now()
|
||
ok_count = 0
|
||
fail_items = []
|
||
|
||
def check(name, ok, detail=''):
|
||
global ok_count
|
||
icon = '✅' if ok else '❌'
|
||
print(f'{icon} {name}: {detail}')
|
||
if ok:
|
||
ok_count += 1
|
||
else:
|
||
fail_items.append(name)
|
||
|
||
def md5(p):
|
||
try:
|
||
return hashlib.md5(open(p, 'rb').read()).hexdigest()[:10]
|
||
except Exception:
|
||
return 'ERR'
|
||
|
||
print('=== 1. 新脚本文件部署（deploy + profile scripts 硬链接一致）===')
|
||
NEW_SCRIPTS = ['functional_health_check.py', 'self_repair.py', 'meta_watchdog.py',
|
||
'system_hygiene_audit.py', 'sync_profile_scripts.sh', 'mofin_health.py']
|
||
for f in NEW_SCRIPTS:
|
||
dp = f'/home/hmo/MoFin/deploy/profile-scripts/{f}'
|
||
pp = f'/home/hmo/.hermes/profiles/position-analyst/scripts/{f}'
|
||
d_ok = os.path.exists(dp)
|
||
p_ok = os.path.exists(pp)
|
||
same = d_ok and p_ok and md5(dp) == md5(pp)
|
||
check(f, same, f'deploy={d_ok} profile={p_ok} 内容一致={same}')
|
||
|
||
print()
|
||
print('=== 2. cron 注册（L1/L2/L3/L4 jobs）===')
|
||
d = json.load(open('/home/hmo/.hermes/profiles/position-analyst/cron/jobs.json'))
|
||
jobs = d if isinstance(d, list) else d.get('jobs', [])
|
||
expect = {
|
||
'functional_health_check.py': 'L1功能健康(15min)',
|
||
'self_repair.py': 'L3修复循环(30min)',
|
||
'meta_watchdog.py': 'L4元监控(每小时)',
|
||
'system_hygiene_audit.py': 'L2卫生(每日)',
|
||
}
|
||
for script, label in expect.items():
|
||
j = next((x for x in jobs if x.get('script') == script), None)
|
||
if not j:
|
||
check(label, False, '未注册')
|
||
else:
|
||
sched = j.get('schedule_display') or str(j.get('schedule'))
|
||
en = j.get('enabled', True)
|
||
check(label, bool(j and en), f'registered sched={sched} enabled={en}')
|
||
|
||
# 退休的 job 不在
|
||
for dead in ['Cron监护-高频', '全局cron健康监控-每10分']:
|
||
j = next((x for x in jobs if x.get('name') == dead), None)
|
||
check(f'退休job[{dead}]已删除', j is None, '已删除' if j is None else '仍存在!')
|
||
|
||
print()
|
||
print('=== 3. 输出物生成 ===')
|
||
for name, path, max_h in [
|
||
('functional_health.json', '/home/hmo/MoFin/gateway/logs/functional_health.json', 1),
|
||
('hygiene_report.json', '/home/hmo/MoFin/gateway/logs/hygiene_report.json', 26),
|
||
('meta_watchdog.json', '/home/hmo/MoFin/gateway/logs/meta_watchdog.json', 2),
|
||
('mofin_health.json', '/home/hmo/web-dashboard/static/mofin_health.json', 1),
|
||
]:
|
||
if not os.path.exists(path):
|
||
check(name, False, '不存在')
|
||
else:
|
||
age_h = (now.timestamp() - os.path.getmtime(path)) / 3600
|
||
check(name, age_h < max_h, f'{age_h:.1f}h前生成（阈值{max_h}h）')
|
||
|
||
print()
|
||
print('=== 4. mofin_health.json 含 self_check 区块 ===')
|
||
try:
|
||
mh = json.load(open('/home/hmo/web-dashboard/static/mofin_health.json'))
|
||
sc = mh.get('self_check', {})
|
||
check('self_check.functional', bool(sc.get('functional')), str(bool(sc.get('functional'))))
|
||
check('self_check.meta_watchdog', bool(sc.get('meta_watchdog')), str(bool(sc.get('meta_watchdog'))))
|
||
check('self_check.hygiene', bool(sc.get('hygiene')), str(bool(sc.get('hygiene'))))
|
||
except Exception as e:
|
||
check('mofin_health.json', False, str(e))
|
||
|
||
print()
|
||
print('=== 5. Dashboard 页面含自检体系 Tab ===')
|
||
html = open('/home/hmo/web-dashboard/static/mofin_health.html', encoding='utf-8').read()
|
||
check('🩺自检体系 tab', '自检体系' in html and 'panel4' in html, '')
|
||
r = subprocess.run(['curl', '-s', '-o', '/dev/null', '-w', '%{http_code}',
|
||
'http://127.0.0.1:8899/mofin_health.html'], capture_output=True, text=True, timeout=10)
|
||
check('dashboard 页面 200', r.stdout.strip() == '200', f'HTTP {r.stdout.strip()}')
|
||
|
||
print()
|
||
print('=== 6. systemd 超时配置 ===')
|
||
r1 = subprocess.run(['sudo', '-n', 'systemctl', 'show', 'hermes-gateway-zhiwei', '-p', 'Environment'],
|
||
capture_output=True, text=True, timeout=10)
|
||
check('zhiwei HERMES_CRON_SCRIPT_TIMEOUT=600', 'HERMES_CRON_SCRIPT_TIMEOUT=600' in r1.stdout, '')
|
||
r2 = subprocess.run(['systemctl', '--user', 'show', 'hermes-gateway', '-p', 'Environment'],
|
||
capture_output=True, text=True, timeout=10)
|
||
check('default HERMES_CRON_SCRIPT_TIMEOUT=600', 'HERMES_CRON_SCRIPT_TIMEOUT=600' in r2.stdout, '')
|
||
|
||
print()
|
||
print('=== 7. 小果残留 ===')
|
||
r = subprocess.run(['ps', 'aux'], capture_output=True, text=True, timeout=10)
|
||
xg = [l for l in r.stdout.splitlines() if 'xiaoguo' in l.lower() and 'grep' not in l]
|
||
check('小果进程清零', len(xg) == 0, f'{len(xg)}个残留' if xg else '无')
|
||
r = subprocess.run(['ss', '-tlnp'], capture_output=True, text=True, timeout=10)
|
||
check('8645 已关闭', '8645' not in r.stdout, '')
|
||
|
||
print()
|
||
print('=== 8. 服务全景 ===')
|
||
r = subprocess.run(['ss', '-tlnp'], capture_output=True, text=True, timeout=10)
|
||
for port, label in [('8642', 'default gw'), ('8643', 'zhiwei gw'), ('8646', 'mohe gw'),
|
||
('8899', 'dashboard'), ('5222', 'ejabberd'), ('5805', 'xmpp桥')]:
|
||
check(f'{label}:{port}', f':{port}' in r.stdout, '')
|
||
|
||
print()
|
||
print(f'=== 汇总: {ok_count} 项通过, {len(fail_items)} 项失败 ===')
|
||
if fail_items:
|
||
print('失败项:', fail_items) |