Files
MoFin/scripts/unify_sources.py
T
hmo 4f83ee8a01 feat(hygiene): anti-redundancy enforcement — spec rules + weekly audit
Root cause analysis of the 2026-07-20 redundancy incident:
1. No single-source-of-truth rule -> same file legitimately lived in 4+
   locations, diverging silently
2. Relative path resolution (Path(__file__).parent/'data') -> each
   hardlinked copy of mofin_db.py pointed to a DIFFERENT database
3. 'Backup habit' left .bak/legacy files in production dirs, which
   monitoring then scanned and reported as false alarms
4. Half-done migrations: DB tables created but old JSON writers/readers
   stayed (price_events), old files stayed
5. Dead modules never got buried: xiaoguo 'dead' but bot ran 8 days
   as root eating 2.5GB
6. Monitoring checked 'does it exist' not 'is it alive' -> stale file
   mtime reported as 'pipeline stalled 14 days' (false alarm)
7. No 'system hygiene' as a check category at all

Prevention implemented:
- dev-spec.md v2.0: 五条红线 -> 十条红线
  #6 single source of truth (hardlink only, no independent copies)
  #7 absolute data paths only (no __file__-relative data resolution)
  #8 no backups/legacy in production data dirs (archive immediately)
  #9 dead module burial checklist (6 mandatory steps)
  #10 monitor liveness (DB table freshness) not existence
- File Location Constitution: canonical location per content type
- NEW system_hygiene_audit.py: weekly Monday 07:30 cron checking
  diverged copies / broken hardlinks / zombie processes / orphan data
  files / dead cron scripts / DB freshness -> hygiene_report.json + XMPP
- specs/hygiene.json: module spec per red line #1
- Verified: audit found 5 real issues on first run, all fixed, re-run clean
2026-07-20 19:04:05 +08:00

106 lines
5.1 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import os, shutil, glob
from datetime import datetime
ARCHIVE = f'/home/hmo/MoFin/archive/dedup-20260720-{datetime.now().strftime("%H%M")}'
os.makedirs(ARCHIVE, exist_ok=True)
def hardlink_to(src, targets):
"""把 targets 全部硬链接到 src(内容统一为 src"""
done = []
for t in targets:
if not os.path.exists(t) and not os.path.islink(t):
continue
if os.path.isfile(t) and os.path.samefile(src, t):
continue
try:
if os.path.islink(t) or os.path.isfile(t):
os.unlink(t)
os.link(src, t)
done.append(t)
except Exception as e:
print(f' LINK FAIL {t}: {e}')
return done
def archive(path, note=''):
if os.path.exists(path) or os.path.islink(path):
dst = os.path.join(ARCHIVE, os.path.basename(path))
if os.path.exists(dst):
dst = dst + '.' + datetime.now().strftime('%H%M%S')
shutil.move(path, dst)
print(f' archived: {path} {note}')
# ── A. LIBRARY 类:root 是活的(cron 经 sys.path 导入),统一所有副本到 root ──
LIBRARY = ['strategy_lifecycle.py', 'strategy_tree.py', 'technical_analysis.py',
'multi_timeframe.py', 'mofin_news.py']
print('=== A. LIBRARY 统一到 MoFin root ===')
for f in LIBRARY:
root = f'/home/hmo/MoFin/{f}'
if not os.path.exists(root):
print(f' {f}: root 不存在,跳过!')
continue
targets = [
f'/home/hmo/MoFin/deploy/profile-scripts/{f}',
f'/home/hmo/.hermes/profiles/position-analyst/scripts/{f}',
f'/home/hmo/MoFin/scripts/{f}',
f'/home/hmo/web-dashboard/{f}',
]
done = hardlink_to(root, targets)
print(f' {f}: unified {len(done)} copies -> root')
# ── B. CRON 类:deploy 是唯一源,归档其他副本 ──
CRON_FILES = ['advice_reconciliation.py', 'branch_scanner.py', 'collect_evaluation_data.py',
'cron_to_xmpp.py', 'market_insight.py', 'memory_guardian.py', 'mofin_health.py',
'premarket_full_review.py', 'promote_candidates.py', 'prune_branches.py',
'strategy_evaluator.py', 'system_health_check.py', 'system_audit.py']
print('\n=== B. CRON 类归档非 deploy 副本 ===')
for f in CRON_FILES:
for loc in ['/home/hmo/MoFin', '/home/hmo/MoFin/scripts',
'/home/hmo/web-dashboard', '/home/hmo/.hermes/scripts']:
p = os.path.join(loc, f)
# 不碰 deploy 和 pa/scripts(硬链接到 deploy
if os.path.exists(p) or os.path.islink(p):
# 跳过与 deploy 同 inode 的(那些是合法硬链接)
deploy_p = f'/home/hmo/MoFin/deploy/profile-scripts/{f}'
if os.path.exists(deploy_p) and os.path.isfile(p) and not os.path.islink(p):
try:
if os.path.samefile(p, deploy_p):
continue
except Exception:
pass
archive(p, f'(cron file {f})')
# ── C. market_watch / market_screener 特殊:crontab 正在跑 root 旧版 ──
print('\n=== C. market_watch/market_screener 统一 ===')
# 暂时只统一内容到 deploydeploy 是 hermes cron 维护的版本),crontab 稍后改指向
for f in ['market_watch.py', 'market_screener.py']:
deploy_p = f'/home/hmo/MoFin/deploy/profile-scripts/{f}'
if os.path.exists(deploy_p):
done = hardlink_to(deploy_p, [f'/home/hmo/MoFin/{f}', f'/home/hmo/MoFin/scripts/{f}'])
print(f' {f}: unified {len(done)} -> deploy (crontab 路径仍有效但内容已统一)')
# ── D. 待查类 ──
print('\n=== D. 待查类 ===')
# server.py: web-dashboard + root 是活的 dashboard,归档 deploy 旧拷贝
archive('/home/hmo/MoFin/deploy/profile-scripts/server.py', '(live=web-dashboard hardlink)')
# xmpp_agent_core.py: deploy/bot 是活的
for p in ['/home/hmo/MoFin/xmpp_agent_core.py', '/home/hmo/MoFin/scripts/xmpp_agent_core.py',
'/home/hmo/.hermes/scripts/xmpp_agent_core.py', '/home/hmo/web-dashboard/xmpp_agent_core.py']:
archive(p, '(live=deploy/bot)')
# 手动工具类:deploy 为准,归档 root 旧副本
for f in ['mo_provider.py', 'mofin_collect.py', 'stock_profile.py', 'stock_sector_enrich.py',
'trend_detector.py', 'bulk_strategy_regenerate.py', 'strategy_feedback.py']:
deploy_p = f'/home/hmo/MoFin/deploy/profile-scripts/{f}'
if os.path.exists(deploy_p):
done = hardlink_to(deploy_p, [f'/home/hmo/MoFin/{f}', f'/home/hmo/MoFin/scripts/{f}'])
print(f' {f}: unified {len(done)} -> deploy')
else:
for loc in ['/home/hmo/MoFin', '/home/hmo/MoFin/scripts']:
archive(os.path.join(loc, f), '(no deploy version)')
print('\n=== 验证:关键 import 测试 ===')
import subprocess
r = subprocess.run(
"cd /home/hmo/.hermes/profiles/position-analyst/scripts && python3 -c \"import sys; sys.path.insert(0,'/home/hmo/MoFin'); import strategy_lifecycle, technical_analysis, multi_timeframe, mofin_news, strategy_tree; print('imports OK', strategy_lifecycle.__file__)\"",
shell=True, capture_output=True, text=True, timeout=30)
print(r.stdout.strip() or r.stderr.strip()[:200])
print('\nARCHIVE dir:', ARCHIVE)