hmo
4f83ee8a01
Root cause analysis of the 2026-07-20 redundancy incident: 1. No single-source-of-truth rule -> same file legitimately lived in 4+ locations, diverging silently 2. Relative path resolution (Path(__file__).parent/'data') -> each hardlinked copy of mofin_db.py pointed to a DIFFERENT database 3. 'Backup habit' left .bak/legacy files in production dirs, which monitoring then scanned and reported as false alarms 4. Half-done migrations: DB tables created but old JSON writers/readers stayed (price_events), old files stayed 5. Dead modules never got buried: xiaoguo 'dead' but bot ran 8 days as root eating 2.5GB 6. Monitoring checked 'does it exist' not 'is it alive' -> stale file mtime reported as 'pipeline stalled 14 days' (false alarm) 7. No 'system hygiene' as a check category at all Prevention implemented: - dev-spec.md v2.0: 五条红线 -> 十条红线 #6 single source of truth (hardlink only, no independent copies) #7 absolute data paths only (no __file__-relative data resolution) #8 no backups/legacy in production data dirs (archive immediately) #9 dead module burial checklist (6 mandatory steps) #10 monitor liveness (DB table freshness) not existence - File Location Constitution: canonical location per content type - NEW system_hygiene_audit.py: weekly Monday 07:30 cron checking diverged copies / broken hardlinks / zombie processes / orphan data files / dead cron scripts / DB freshness -> hygiene_report.json + XMPP - specs/hygiene.json: module spec per red line #1 - Verified: audit found 5 real issues on first run, all fixed, re-run clean
84 lines
2.7 KiB
Python
84 lines
2.7 KiB
Python
#!/usr/bin/env python3
|
|
"""audit_duplication.py — 扫描 MoFin 相关目录的文件重复情况"""
|
|
import os, hashlib, json
|
|
from collections import defaultdict
|
|
|
|
LOCATIONS = [
|
|
'/home/hmo/MoFin',
|
|
'/home/hmo/MoFin/scripts',
|
|
'/home/hmo/MoFin/deploy/profile-scripts',
|
|
'/home/hmo/MoFin/deploy/bot',
|
|
'/home/hmo/.hermes/profiles/position-analyst/scripts',
|
|
'/home/hmo/.hermes/scripts',
|
|
'/home/hmo/web-dashboard',
|
|
]
|
|
|
|
def md5(p):
|
|
try:
|
|
with open(p, 'rb') as f:
|
|
return hashlib.md5(f.read()).hexdigest()[:10]
|
|
except Exception:
|
|
return 'ERR'
|
|
|
|
def ino(p):
|
|
try:
|
|
return os.stat(p).st_ino
|
|
except Exception:
|
|
return 0
|
|
|
|
# 收集所有 .py 文件
|
|
files = defaultdict(list) # name -> [(loc, path, md5, inode, is_link)]
|
|
for loc in LOCATIONS:
|
|
if not os.path.isdir(loc):
|
|
continue
|
|
for f in os.listdir(loc):
|
|
if not f.endswith('.py'):
|
|
continue
|
|
p = os.path.join(loc, f)
|
|
if not os.path.isfile(p):
|
|
continue
|
|
files[f].append({
|
|
'loc': loc, 'path': p, 'md5': md5(p), 'inode': ino(p),
|
|
'is_link': os.path.islink(p),
|
|
'size': os.path.getsize(p),
|
|
'mtime': int(os.path.getmtime(p)),
|
|
})
|
|
|
|
print("=== 重复文件（同名出现在2+位置）===")
|
|
dups = {k: v for k, v in files.items() if len(v) > 1}
|
|
identical = 0
|
|
hardlinked = 0
|
|
diverged = 0
|
|
for name in sorted(dups):
|
|
entries = dups[name]
|
|
md5s = set(e['md5'] for e in entries)
|
|
inodes = set(e['inode'] for e in entries)
|
|
if len(inodes) == 1:
|
|
status = 'HARDLINK(同一文件)'
|
|
hardlinked += 1
|
|
elif len(md5s) == 1:
|
|
status = 'COPY(内容相同,多份独立)'
|
|
identical += 1
|
|
else:
|
|
status = 'DIVERGED(内容不同!)'
|
|
diverged += 1
|
|
print(f"{status} {name}")
|
|
for e in entries:
|
|
print(f" {e['path']} md5={e['md5']} ino={e['inode']} size={e['size']}")
|
|
|
|
print(f"\n汇总: {len(dups)} 个重复文件名 | hardlink={hardlinked} 内容相同副本={identical} 内容分叉={diverged}")
|
|
|
|
print("\n=== deploy/profile-scripts 中有但 position-analyst/scripts 中缺失的 ===")
|
|
pa_dir = '/home/hmo/.hermes/profiles/position-analyst/scripts'
|
|
deploy_dir = '/home/hmo/MoFin/deploy/profile-scripts'
|
|
pa_files = set(os.listdir(pa_dir)) if os.path.isdir(pa_dir) else set()
|
|
for f in sorted(os.listdir(deploy_dir)):
|
|
if f.endswith('.py') and f not in pa_files:
|
|
print(f" {f}")
|
|
|
|
print("\n=== position-analyst/scripts 中有但 deploy 中没有的（可能孤儿）===")
|
|
deploy_files = set(os.listdir(deploy_dir))
|
|
for f in sorted(pa_files):
|
|
if f.endswith('.py') and f not in deploy_files:
|
|
p = os.path.join(pa_dir, f)
|
|
print(f" {f} size={os.path.getsize(p)}") |