hmo
4f83ee8a01
Root cause analysis of the 2026-07-20 redundancy incident: 1. No single-source-of-truth rule -> same file legitimately lived in 4+ locations, diverging silently 2. Relative path resolution (Path(__file__).parent/'data') -> each hardlinked copy of mofin_db.py pointed to a DIFFERENT database 3. 'Backup habit' left .bak/legacy files in production dirs, which monitoring then scanned and reported as false alarms 4. Half-done migrations: DB tables created but old JSON writers/readers stayed (price_events), old files stayed 5. Dead modules never got buried: xiaoguo 'dead' but bot ran 8 days as root eating 2.5GB 6. Monitoring checked 'does it exist' not 'is it alive' -> stale file mtime reported as 'pipeline stalled 14 days' (false alarm) 7. No 'system hygiene' as a check category at all Prevention implemented: - dev-spec.md v2.0: 五条红线 -> 十条红线 #6 single source of truth (hardlink only, no independent copies) #7 absolute data paths only (no __file__-relative data resolution) #8 no backups/legacy in production data dirs (archive immediately) #9 dead module burial checklist (6 mandatory steps) #10 monitor liveness (DB table freshness) not existence - File Location Constitution: canonical location per content type - NEW system_hygiene_audit.py: weekly Monday 07:30 cron checking diverged copies / broken hardlinks / zombie processes / orphan data files / dead cron scripts / DB freshness -> hygiene_report.json + XMPP - specs/hygiene.json: module spec per red line #1 - Verified: audit found 5 real issues on first run, all fixed, re-run clean
84 lines
3.6 KiB
Python
84 lines
3.6 KiB
Python
#!/usr/bin/env python3
|
|
"""audit_data.py — 数据存储碎片化审计：三个数据目录 + 遗留JSON + 旧库"""
|
|
import os, json, sqlite3
|
|
from datetime import datetime
|
|
|
|
DATA_DIRS = [
|
|
'/home/hmo/MoFin/data',
|
|
'/home/hmo/web-dashboard/data',
|
|
'/home/hmo/.hermes/profiles/position-analyst/scripts/data',
|
|
'/home/hmo/.hermes/profiles/position-analyst/data',
|
|
'/home/hmo/.hermes/data',
|
|
]
|
|
|
|
now = datetime.now()
|
|
|
|
print("=== 1. 各数据目录内容与时效 ===")
|
|
for dd in DATA_DIRS:
|
|
if not os.path.isdir(dd):
|
|
print(f"\n{dd}: 不存在")
|
|
continue
|
|
print(f"\n{dd}:")
|
|
try:
|
|
entries = []
|
|
for f in sorted(os.listdir(dd)):
|
|
p = os.path.join(dd, f)
|
|
if os.path.isfile(p):
|
|
mt = datetime.fromtimestamp(os.path.getmtime(p))
|
|
age_h = (now - mt).total_seconds() / 3600
|
|
entries.append((f, os.path.getsize(p), mt, age_h))
|
|
# 只显示 >7天 或 >100KB 的
|
|
for f, size, mt, age_h in entries:
|
|
flag = '🔴' if age_h > 24*14 else ('🟡' if age_h > 24*3 else '🟢')
|
|
print(f" {flag} {f:45s} {size//1024:6d}KB {mt.strftime('%m-%d %H:%M')} ({age_h/24:.0f}d)")
|
|
except Exception as e:
|
|
print(f" ERR: {e}")
|
|
|
|
print("\n=== 2. 三个 mofin.db 对比 ===")
|
|
for label, path in [('canonical', '/home/hmo/MoFin/data/mofin.db'),
|
|
('profile-local', '/home/hmo/.hermes/profiles/position-analyst/scripts/data/mofin.db'),
|
|
('profile-data', '/home/hmo/.hermes/profiles/position-analyst/data/mofin.db')]:
|
|
if not os.path.exists(path):
|
|
print(f" {label}: 不存在")
|
|
continue
|
|
c = sqlite3.connect(path, timeout=5)
|
|
tables = {r[0]: r[1] for r in c.execute(
|
|
"SELECT name, (SELECT COUNT(*) FROM sqlite_master m2 WHERE m2.name=m1.name) FROM sqlite_master m1 WHERE type='table'")}
|
|
cnts = {}
|
|
for t in tables:
|
|
try:
|
|
cnts[t] = c.execute(f"SELECT COUNT(*) FROM {t}").fetchone()[0]
|
|
except Exception:
|
|
cnts[t] = -1
|
|
nonempty = {k: v for k, v in cnts.items() if v > 0}
|
|
print(f" {label} ({os.path.getsize(path)//1024}KB): {len(nonempty)} 张非空表")
|
|
for t, n in sorted(nonempty.items(), key=lambda x: -x[1])[:8]:
|
|
print(f" {t}: {n}")
|
|
c.close()
|
|
|
|
print("\n=== 3. 遗留 JSON 文件（web-dashboard/data，按最后修改排序）===")
|
|
wd = '/home/hmo/web-dashboard/data'
|
|
jfiles = []
|
|
for f in os.listdir(wd):
|
|
if f.endswith('.json') and os.path.isfile(os.path.join(wd, f)):
|
|
p = os.path.join(wd, f)
|
|
mt = datetime.fromtimestamp(os.path.getmtime(p))
|
|
jfiles.append((f, os.path.getsize(p), mt))
|
|
jfiles.sort(key=lambda x: x[2])
|
|
for f, size, mt in jfiles:
|
|
age_d = (now - mt).total_seconds() / 86400
|
|
flag = '🔴' if age_d > 14 else ('🟡' if age_d > 3 else '🟢')
|
|
print(f" {flag} {f:45s} {size//1024:5d}KB {mt.strftime('%m-%d')} ({age_d:.0f}d前)")
|
|
|
|
print("\n=== 4. trashbox / archive / 旧日志 ===")
|
|
for d in ['/home/hmo/trashbox', '/home/hmo/MoFin/archive', '/home/hmo/MoFin/data/archive']:
|
|
if os.path.isdir(d):
|
|
total = sum(os.path.getsize(os.path.join(r, f)) for r, _, fs in os.walk(d) for f in fs if os.path.isfile(os.path.join(r, f)))
|
|
cnt = sum(len(fs) for _, _, fs in os.walk(d))
|
|
print(f" {d}: {cnt} 个文件, {total//1024//1024}MB")
|
|
logs = [f for f in os.listdir('/home/hmo') if f.endswith('.log') and os.path.isfile(f'/home/hmo/{f}')]
|
|
print(f"\n /home/hmo 根目录 .log 文件: {len(logs)} 个")
|
|
for f in sorted(logs):
|
|
p = f'/home/hmo/{f}'
|
|
mt = datetime.fromtimestamp(os.path.getmtime(p))
|
|
print(f" {f} {os.path.getsize(p)//1024}KB {mt.strftime('%m-%d')}") |