From 4f83ee8a0175e176574f9ef8ced9c89d53bd0f50 Mon Sep 17 00:00:00 2001 From: hmo Date: Mon, 20 Jul 2026 19:04:05 +0800 Subject: [PATCH] =?UTF-8?q?feat(hygiene):=20anti-redundancy=20enforcement?= =?UTF-8?q?=20=E2=80=94=20spec=20rules=20+=20weekly=20audit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause analysis of the 2026-07-20 redundancy incident: 1. No single-source-of-truth rule -> same file legitimately lived in 4+ locations, diverging silently 2. Relative path resolution (Path(__file__).parent/'data') -> each hardlinked copy of mofin_db.py pointed to a DIFFERENT database 3. 'Backup habit' left .bak/legacy files in production dirs, which monitoring then scanned and reported as false alarms 4. Half-done migrations: DB tables created but old JSON writers/readers stayed (price_events), old files stayed 5. Dead modules never got buried: xiaoguo 'dead' but bot ran 8 days as root eating 2.5GB 6. Monitoring checked 'does it exist' not 'is it alive' -> stale file mtime reported as 'pipeline stalled 14 days' (false alarm) 7. No 'system hygiene' as a check category at all Prevention implemented: - dev-spec.md v2.0: 五条红线 -> 十条红线 #6 single source of truth (hardlink only, no independent copies) #7 absolute data paths only (no __file__-relative data resolution) #8 no backups/legacy in production data dirs (archive immediately) #9 dead module burial checklist (6 mandatory steps) #10 monitor liveness (DB table freshness) not existence - File Location Constitution: canonical location per content type - NEW system_hygiene_audit.py: weekly Monday 07:30 cron checking diverged copies / broken hardlinks / zombie processes / orphan data files / dead cron scripts / DB freshness -> hygiene_report.json + XMPP - specs/hygiene.json: module spec per red line #1 - Verified: audit found 5 real issues on first run, all fixed, re-run clean --- .../profile-scripts/system_hygiene_audit.py | 266 ++++++++++++++++++ docs/dev-spec.md | 41 ++- scripts/add_hygiene_cron.py | 45 +++ scripts/archive_legacy_json.py | 58 ++++ scripts/archive_old_files.py | 30 ++ scripts/audit_data.py | 84 ++++++ scripts/audit_deadcode.py | 92 ++++++ scripts/audit_duplication.py | 84 ++++++ scripts/audit_runtime.py | 46 +++ scripts/classify_diverged.py | 47 ++++ scripts/cleanup_disabled_cron.py | 30 ++ scripts/fix_crontab_cron_to_xmpp.py | 27 ++ scripts/fix_crontab_market.py | 17 ++ scripts/inspect_third_db.py | 33 +++ scripts/merge_third_db.py | 94 +++++++ scripts/notify_user4.py | 32 +++ scripts/notify_user5.py | 38 +++ scripts/remove_xiaoguo.py | 52 ++++ scripts/remove_xiaoguo2.py | 18 ++ scripts/unify_sources.py | 106 +++++++ specs/hygiene.json | 49 ++++ 21 files changed, 1286 insertions(+), 3 deletions(-) create mode 100644 deploy/profile-scripts/system_hygiene_audit.py create mode 100644 scripts/add_hygiene_cron.py create mode 100644 scripts/archive_legacy_json.py create mode 100644 scripts/archive_old_files.py create mode 100644 scripts/audit_data.py create mode 100644 scripts/audit_deadcode.py create mode 100644 scripts/audit_duplication.py create mode 100644 scripts/audit_runtime.py create mode 100644 scripts/classify_diverged.py create mode 100644 scripts/cleanup_disabled_cron.py create mode 100644 scripts/fix_crontab_cron_to_xmpp.py create mode 100644 scripts/fix_crontab_market.py create mode 100644 scripts/inspect_third_db.py create mode 100644 scripts/merge_third_db.py create mode 100644 scripts/notify_user4.py create mode 100644 scripts/notify_user5.py create mode 100644 scripts/remove_xiaoguo.py create mode 100644 scripts/remove_xiaoguo2.py create mode 100644 scripts/unify_sources.py create mode 100644 specs/hygiene.json diff --git a/deploy/profile-scripts/system_hygiene_audit.py b/deploy/profile-scripts/system_hygiene_audit.py new file mode 100644 index 00000000..05ba20f8 --- /dev/null +++ b/deploy/profile-scripts/system_hygiene_audit.py @@ -0,0 +1,266 @@ +#!/usr/bin/env python3 +"""system_hygiene_audit.py — 系统卫生审计（防冗余复发） + +每周运行。检查六类问题，输出 hygiene_report.json，有问题时推 XMPP。 +红线6/7/8/9/10 的自动化 enforcement。 + +检查项： +1. 分叉副本：同名 .py 在不同权威位置内容不一致 +2. 断裂硬链接：deploy/profile-scripts vs profile scripts 内容不一致（cron 会跑旧代码！） +3. 僵尸进程：>3 天的 python/node 进程不在白名单 +4. 孤儿数据文件：生产数据目录 >14 天未修改且不在活文件注册表 +5. 死 cron：jobs.json 中 script 不存在 +6. DB 表新鲜度：核心表 >24h 无新记录（交易时间） +""" +import os, sys, json, glob, hashlib, sqlite3, subprocess +from datetime import datetime, timedelta +from pathlib import Path + +sys.path.insert(0, '/home/hmo/MoFin') + +DEPLOY = '/home/hmo/MoFin/deploy/profile-scripts' +PA_SCRIPTS = '/home/hmo/.hermes/profiles/position-analyst/scripts' +MOFIN_ROOT = '/home/hmo/MoFin' +DATA_DIR = '/home/hmo/MoFin/data' +REPORT = '/home/hmo/MoFin/gateway/logs/hygiene_report.json' + +# 活文件注册表（生产数据目录允许存在的非数据文件） +LIVE_DATA_FILES = { + 'mofin.db', 'mofin.db-shm', 'mofin.db-wal', 'mofin_health.json', 'portfolio.json', + 'preflight_result.json', 'growth_registry.json', 'hardcode_audit.json', + 'health_checklist.json', 'macro_divergence_state.json', 'macro_risk_state.json', + 'market.json', 'scanner_state.json', 'state.db', 'strategy_staleness_report.json', + 'system_audit_report.json', 'price_history.json', 'evaluation.json', + 'accuracy_stats.json', 'format_error_library.json', 'candidate_pool.json', + 'pipeline_registry.json', 'analyst-knowledge-log.md', 'mofin_health.html', + 'evaluation_input.json', 'push_cooldown.json', 'system_audit.json', + 'system_inventory.json', 'stocks', +} + +PROCESS_WHITELIST = [ + 'hermes_cli.main', 'server.py', 'xmpp_zhiwei_bot.py', 'xmpp_mohe_bot.py', + 'shadowsocks', 'unattended-upgrades', 'dashboard.py', 'kanban_api.py', + 'main_dsa.py', 'vc-webhook.py', 'obsidian-api.py', 'http.server', + 'wechat_webhook.py', 'qq-poll', 'afw', 'mcp_server', 'uvicorn', + 'agentmemory', 'kimi_collect', 'mohe_knowledge_relay', 'wechat_watchdog', + 'todo_scanner', 'miner.py', +] + +CORE_TABLES = [ + ('live_prices', 'updated_at', '实时价格'), + ('market_snapshots', 'created_at', '市场快照'), + ('mtf_cache', 'updated_at', '多周期缓存'), + ('macro_context_log', 'created_at', '宏观上下文'), + ('price_events', 'created_at', '价格事件'), +] + + +def md5(p): + try: + return hashlib.md5(open(p, 'rb').read()).hexdigest() + except Exception: + return 'ERR' + + +def check_diverged(): + """检查 deploy vs MoFin/scripts vs MoFin根 的分叉副本""" + issues = [] + compare_dirs = [f'{MOFIN_ROOT}/scripts', MOFIN_ROOT, '/home/hmo/web-dashboard'] + for f in os.listdir(DEPLOY): + if not f.endswith('.py'): + continue + dp = os.path.join(DEPLOY, f) + d_md5 = md5(dp) + for d in compare_dirs: + p = os.path.join(d, f) + if os.path.exists(p) and not os.path.islink(p): + try: + if os.path.samefile(p, dp): + continue + except Exception: + pass + if md5(p) != d_md5: + issues.append({ + 'type': 'diverged_copy', 'file': f, + 'canonical': dp, 'stale_copy': p, + 'action': f'归档 {p} 或硬链接到权威版', + }) + return issues + + +def check_broken_hardlinks(): + """deploy vs pa/scripts 内容不一致 = cron 跑旧代码""" + issues = [] + for f in os.listdir(DEPLOY): + if not f.endswith('.py'): + continue + dp = os.path.join(DEPLOY, f) + pp = os.path.join(PA_SCRIPTS, f) + if os.path.exists(pp): + try: + if os.path.samefile(dp, pp): + continue + except Exception: + pass + if md5(dp) != md5(pp): + issues.append({ + 'type': 'broken_hardlink', 'file': f, + 'action': 'bash deploy/profile-scripts/sync_profile_scripts.sh', + }) + else: + issues.append({ + 'type': 'missing_profile_link', 'file': f, + 'action': 'bash deploy/profile-scripts/sync_profile_scripts.sh', + }) + return issues + + +def check_zombies(): + """>3 天的 python/node 进程不在白名单（docker 容器内进程豁免）""" + issues = [] + try: + r = subprocess.run(['ps', '-eo', 'pid,etime,args'], capture_output=True, text=True, timeout=10) + for line in r.stdout.splitlines()[1:]: + parts = line.split(None, 2) + if len(parts) < 3: + continue + pid, etime, cmd = parts + if 'python' not in cmd and 'node' not in cmd: + continue + # docker 容器内进程豁免（cgroup 含 docker） + try: + cg = open(f'/proc/{pid}/cgroup').read() + if 'docker' in cg: + continue + except Exception: + pass + # etime 格式: dd-hh:mm:ss 或 hh:mm:ss + days = 0 + if '-' in etime: + days = int(etime.split('-')[0]) + if days >= 3: + if not any(w in cmd for w in PROCESS_WHITELIST): + issues.append({ + 'type': 'zombie_process', 'pid': pid, 'days': days, + 'cmd': cmd[:120], + 'action': f'确认后 kill {pid}（红线9 收尸流程）', + }) + except Exception: + pass + return issues + + +def check_orphan_files(): + """生产数据目录的孤儿文件""" + issues = [] + now = datetime.now() + for f in os.listdir(DATA_DIR): + p = os.path.join(DATA_DIR, f) + if not os.path.isfile(p) or f.startswith('.'): + continue + if f in LIVE_DATA_FILES: + continue + age_d = (now.timestamp() - os.path.getmtime(p)) / 86400 + if age_d > 14: + issues.append({ + 'type': 'orphan_data_file', 'file': f, + 'age_days': round(age_d), + 'action': f'归档到 archive/（红线8）', + }) + return issues + + +def check_dead_cron(): + """cron job 指向不存在的脚本""" + issues = [] + for jf, sdir in [('/home/hmo/.hermes/profiles/position-analyst/cron/jobs.json', PA_SCRIPTS), + ('/home/hmo/.hermes/cron/jobs.json', '/home/hmo/.hermes/scripts')]: + try: + d = json.load(open(jf)) + jobs = d if isinstance(d, list) else d.get('jobs', []) + for j in jobs: + s = j.get('script') + if s and j.get('enabled', True) and not os.path.exists(os.path.join(sdir, s)): + issues.append({ + 'type': 'dead_cron_script', 'job': j.get('name'), 'script': s, + 'action': '删除 job 或补齐脚本', + }) + except Exception: + pass + return issues + + +def check_db_freshness(): + """核心表新鲜度（红线10）""" + issues = [] + try: + c = sqlite3.connect(os.path.join(DATA_DIR, 'mofin.db'), timeout=10) + now = datetime.now() + is_trading_time = now.weekday() < 5 and 9 <= now.hour <= 16 + for table, col, label in CORE_TABLES: + try: + row = c.execute(f"SELECT MAX({col}) FROM {table}").fetchone() + if row and row[0]: + last = datetime.fromisoformat(str(row[0]).replace('Z', '')) + age_h = (now - last).total_seconds() / 3600 + threshold = 4 if is_trading_time else 48 + if age_h > threshold: + issues.append({ + 'type': 'stale_table', 'table': table, 'label': label, + 'age_hours': round(age_h, 1), 'threshold': threshold, + 'action': '查对应采集脚本的 cron 状态', + }) + else: + issues.append({'type': 'empty_table', 'table': table, 'label': label, + 'action': '查采集链路'}) + except Exception: + pass + c.close() + except Exception as e: + issues.append({'type': 'db_error', 'error': str(e)[:100]}) + return issues + + +def main(): + print('🧹 系统卫生审计', datetime.now().strftime('%Y-%m-%d %H:%M')) + all_issues = [] + for name, fn in [('分叉副本', check_diverged), ('断裂硬链接', check_broken_hardlinks), + ('僵尸进程', check_zombies), ('孤儿文件', check_orphan_files), + ('死cron', check_dead_cron), ('DB新鲜度', check_db_freshness)]: + found = fn() + status = f'❌ {len(found)}' if found else '✅' + print(f' {status} {name}') + all_issues.extend(found) + + report = { + 'generated_at': datetime.now().strftime('%Y-%m-%d %H:%M:%S'), + 'issue_count': len(all_issues), + 'issues': all_issues, + 'status': 'warn' if all_issues else 'ok', + } + os.makedirs(os.path.dirname(REPORT), exist_ok=True) + with open(REPORT, 'w', encoding='utf-8') as f: + json.dump(report, f, ensure_ascii=False, indent=2) + + if all_issues: + # 推 XMPP + try: + import urllib.request + lines = [f"🧹 系统卫生审计发现 {len(all_issues)} 个问题："] + for i in all_issues[:8]: + lines.append(f"• [{i['type']}] {i.get('file') or i.get('job') or i.get('table') or i.get('pid')}: {i.get('action','')[:60]}") + if len(all_issues) > 8: + lines.append(f'… 共 {len(all_issues)} 个，详见 hygiene_report.json') + payload = json.dumps({'to': 'hmo@yoin.fun', 'body': '\n'.join(lines), 'type': 'chat'}).encode() + req = urllib.request.Request('http://127.0.0.1:5805/', data=payload, + headers={'Content-Type': 'application/json'}) + urllib.request.urlopen(req, timeout=5) + print(' 📨 已推 XMPP') + except Exception as e: + print(f' XMPP 推送失败: {e}') + else: + print(' ✅ 系统卫生良好') + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/docs/dev-spec.md b/docs/dev-spec.md index db0b237f..c986f5f6 100644 --- a/docs/dev-spec.md +++ b/docs/dev-spec.md @@ -1,18 +1,23 @@ # MoFin 开发规范 -> 版本: v1.0 | 更新: 2026-07-19 | 基于 AgentsMeeting 样板重构 +> 版本: v2.0 | 更新: 2026-07-20 | 基于 AgentsMeeting 样板重构 + 冗余事件复盘 > > 📋 样板参考: [AgentsMeeting TEMPLATE-GUIDE.md](../AgentsMeeting/docs/TEMPLATE-GUIDE.md) --- -## 五条红线 +## 十条红线 1. **先读/写 Spec，再写代码** — 新增功能先写 spec 再实现；修改已有功能先读对应 spec 了解架构和约束再动手。没有 spec 的模块在 Dashboard 不可见，视为未完成 2. **部署必验** — 部署后不打开 Dashboard F Tab 验证 = 部署未完成 3. **不可见即不存在** — 组件不在 Dashboard 中显示 = 等于没部署。离线不告警 = 监控缺陷 4. **实现后同步 Spec** — 每轮开发完毕后，必须将 `specs/{module}.json` 更新为与实际实现一致的状态。文档过期 = 等于没写 5. **部署目标即验收标准** — 所有代码必须以部署目标环境（Linux 246）为基准编写和测试。禁止使用 Windows 专属 API（`tasklist`、`netstat`、`schtasks`、`wmic`）在 246 部署的代码中 +6. **单一事实源（SSOT）** — 每个文件全系统只有一个权威位置，其他位置只允许硬链接（同 inode），**禁止独立副本**。权威位置：`deploy/profile-scripts/`（cron 脚本）、`/home/hmo/MoFin/`（被 import 的库）、`deploy/bot/`（XMPP bot）。修改任何文件后若存在硬链接关系被破坏（scp/编辑器换 inode），必须立即跑 `deploy/profile-scripts/sync_profile_scripts.sh` 重建 +7. **数据路径必须绝对** — 引用数据文件/数据库时，必须写绝对路径并指向权威位置（`/home/hmo/MoFin/data/`）。**禁止**用 `Path(__file__).parent / "data"` 这类相对解析——同一个模块被硬链接到不同位置时会解析出不同的数据库（2026-07-20 三库事件的根因） +8. **备份/遗留物禁止留在生产数据目录** — `.bak`、`decisions_backup_*`、迁移残留 JSON、废弃 DB，必须在迁移/变更完成时移到 `archive/`。生产数据目录（`MoFin/data` = `web-dashboard/data`）只放活文件。监控脚本扫描生产目录时，遗留物就是未来的假警报 +9. **死模块必须收尸** — 宣布模块废弃时，必须在同一轮操作中完成收尸六步：①杀进程 ②stop+disable systemd 服务 ③删 cron job ④归档脚本到 `archive/` ⑤归档数据文件 ⑥从期望矩阵/监控中移除。只说"已废弃"不收尸 = 没废弃（小果 bot 以 root 白跑 8 天 2.5GB 的教训） +10. **监控查"活"不查"在"** — 健康检查必须验证**数据新鲜度**（DB 表 MAX(时间列)）而非"文件存在/进程存在"。文件 mtime、进程存活都不构成健康证据——数据 24h 不更新才是事故。禁止拿遗留文件的 mtime 当管道健康指标（"数据管道停滞14天"假警报的根因） --- @@ -80,6 +85,7 @@ specs/{module}.json | dashboard | `specs/dashboard.json` | Dashboard 自身 | ✅ | | health | `specs/health.json` | 健康监控管线 | ✅ | | xmpp_monitor | `specs/xmpp_monitor.json` | XMPP 通信可观测性 | ✅ | +| hygiene | `specs/hygiene.json` | 系统卫生审计（防冗余） | ✅ | | price_monitor | `specs/price_monitor.json` | 价格监控 cron | 📋 | | strategy_lifecycle | `specs/strategy_lifecycle.json` | 策略生命周期 | 📋 | @@ -87,7 +93,36 @@ specs/{module}.json --- -## 二、验证闭环 +## 二、文件位置宪法（2026-07-20 冗余事件后确立） + +| 内容类型 | 唯一权威位置 | 其他位置的合法形态 | +|---------|-------------|------------------| +| cron 脚本（被调度直接执行） | `deploy/profile-scripts/` | profile scripts 目录硬链接（经 `sync_profile_scripts.sh` 同步） | +| 被 import 的库（mo_*/mofin_*/strategy_*/technical_*） | `/home/hmo/MoFin/`（根目录） | 禁止副本；deploy/profile-scripts 中的同名库文件只能是对 root 的硬链接 | +| XMPP bot | `deploy/bot/` | `/home/hmo/xmpp_zhiwei_bot.py` 符号链接 | +| Dashboard 服务 | `web-dashboard/server.py`（= `/home/hmo/MoFin/server.py` 硬链接） | — | +| 数据文件/数据库 | `/home/hmo/MoFin/data/`（= `web-dashboard/data` 硬链接） | **禁止**任何第二个数据目录 | +| 归档 | `archive/<主题>-<日期>/` | — | +| 待销毁 | `trashbox/` | 定期人工清空 | + +**禁止出现的位置**：`MoFin/scripts/*.py`、`MoFin/` 根目录的 cron 脚本副本、`.hermes/*/scripts/data/`（任何 profile 本地 data 目录存业务数据）、`projects/` 下与生产同名的项目副本。 + +### 死模块收尸清单（红线9 的执行版） + +``` +宣布模块 X 废弃时，同一轮操作内必须完成： +□ 杀进程：pkill 或 systemctl stop（确认 ps 无残留） +□ 服务：systemctl disable + rm unit 文件 + daemon-reload +□ cron：两个 jobs.json 中删除 X 的 job，确认无残留 +□ 脚本：移到 archive/<模块>-retired-<日期>/ +□ 数据文件：同上 +□ 监控：从期望矩阵/健康检查注册表中移除 X +□ 记录：CHANGELOG 写明收尸动作 +``` + +--- + +## 三、验证闭环 ``` ┌────────────┐ ┌──────────┐ ┌──────────┐ diff --git a/scripts/add_hygiene_cron.py b/scripts/add_hygiene_cron.py new file mode 100644 index 00000000..a295b497 --- /dev/null +++ b/scripts/add_hygiene_cron.py @@ -0,0 +1,45 @@ +import json, shutil, uuid +from datetime import datetime + +jf = '/home/hmo/.hermes/profiles/position-analyst/cron/jobs.json' +shutil.copy(jf, jf + '.bak-20260720-hygiene') +d = json.load(open(jf)) +is_list = isinstance(d, list) +jobs = d if is_list else d.get('jobs', []) + +# 防重复 +if any(j.get('script') == 'system_hygiene_audit.py' for j in jobs): + print('job already exists') +else: + job = { + "id": uuid.uuid4().hex[:12], + "name": "系统卫生审计-每周", + "prompt": "", + "skills": [], + "skill": None, + "model": None, + "provider": None, + "base_url": None, + "script": "system_hygiene_audit.py", + "no_agent": True, + "context_from": None, + "schedule": {"kind": "cron", "expr": "30 7 * * 1", "display": "30 7 * * 1"}, + "schedule_display": "30 7 * * 1", + "repeat": {"times": None, "completed": 0}, + "enabled": True, + "state": "scheduled", + "paused_at": None, + "paused_reason": None, + "created_at": datetime.now().isoformat(), + "next_run_at": "2026-07-21T07:30:00+08:00", + "last_run_at": None, + "last_status": None, + } + jobs.append(job) + if is_list: + json.dump(jobs, open(jf, 'w'), ensure_ascii=False, indent=2) + else: + d['jobs'] = jobs + json.dump(d, open(jf, 'w'), ensure_ascii=False, indent=2) + print('added job: 系统卫生审计-每周 (Mon 07:30)') +print('total jobs:', len(jobs)) \ No newline at end of file diff --git a/scripts/archive_legacy_json.py b/scripts/archive_legacy_json.py new file mode 100644 index 00000000..cfee9c4e --- /dev/null +++ b/scripts/archive_legacy_json.py @@ -0,0 +1,58 @@ +import os, shutil, glob +from datetime import datetime + +ARCHIVE = f'/home/hmo/MoFin/archive/json-legacy-20260720' +os.makedirs(ARCHIVE, exist_ok=True) + +# 保留的活文件（在用，不动） +KEEP = { + 'evaluation.json', # evaluator 在读 + 'accuracy_stats.json', # strategy_feedback 在读 + 'format_error_library.json', # 在用 (9d) + 'candidate_pool.json', # 在用 (3d) + 'pipeline_registry.json', # 在用 (3d) + # 今天还在写的 + 'growth_registry.json', 'hardcode_audit.json', 'health_checklist.json', + 'macro_divergence_state.json', 'macro_risk_state.json', 'market.json', + 'mofin_health.json', 'portfolio.json', 'preflight_result.json', + 'price_history.json', 'scanner_state.json', 'state.db', + 'strategy_staleness_report.json', 'system_audit_report.json', + 'mofin.db', 'mofin.db-shm', 'mofin.db-wal', + 'analyst-knowledge-log.md', 'mofin_health.html', 'evaluation_input.json', + 'push_cooldown.json', 'system_audit.json', 'system_inventory.json', + 'watchlist.json.bak2', +} + +DATA = '/home/hmo/web-dashboard/data' # = MoFin/data 硬链接 +moved = 0 +for f in sorted(os.listdir(DATA)): + p = os.path.join(DATA, f) + if not os.path.isfile(p): + continue + if f in KEEP: + continue + if f.startswith('.'): + continue + # 只归档 json/db/txt/log/md 类数据文件，别的不动 + if not any(f.endswith(ext) for ext in ('.json', '.db', '.txt')): + continue + # 归档目标明确为遗留：>7天未修改 + age_d = (datetime.now().timestamp() - os.path.getmtime(p)) / 86400 + if age_d < 7: + continue + shutil.move(p, os.path.join(ARCHIVE, f)) + moved += 1 + print(f' archived: {f} ({age_d:.0f}d)') + +print(f'\narchived {moved} files -> {ARCHIVE}') + +# 废弃小库 +print('\n=== 废弃小库 ===') +for f in ['market.db', 'market_data.db', 'stock_analysis.db']: + p = os.path.join(DATA, f) + if os.path.exists(p): + print(f' {f}: 已在归档中' if not os.path.exists(p) else '') +# 上面已经按 >7d 规则归档了（27d/21d 的 db 文件），确认： +for f in ['market.db', 'market_data.db', 'stock_analysis.db']: + archived = os.path.join(ARCHIVE, f) + print(f' {f}: {"ARCHIVED" if os.path.exists(archived) else "still present!"}') \ No newline at end of file diff --git a/scripts/archive_old_files.py b/scripts/archive_old_files.py new file mode 100644 index 00000000..fe72880e --- /dev/null +++ b/scripts/archive_old_files.py @@ -0,0 +1,30 @@ +import os, shutil, glob +from datetime import datetime + +ARCHIVE = '/home/hmo/MoFin/archive/old-files-20260720' +os.makedirs(ARCHIVE, exist_ok=True) + +# 1. /home/hmo 根目录旧日志（>10天） +count = 0 +for f in glob.glob('/home/hmo/*.log'): + age_d = (datetime.now().timestamp() - os.path.getmtime(f)) / 86400 + if age_d > 10: + shutil.move(f, os.path.join(ARCHIVE, os.path.basename(f))) + count += 1 + print(f' log: {os.path.basename(f)} ({age_d:.0f}d)') +print(f'logs archived: {count}') + +# 2. 旧 bot 文件 +for f in ['/home/hmo/xmpp_zhiwei_bot.py.bak', '/home/hmo/run_zhiwei_bot.py', '/home/hmo/xmpp_bot_rest.py']: + if os.path.exists(f): + shutil.move(f, os.path.join(ARCHIVE, os.path.basename(f))) + print(f' bot: {os.path.basename(f)}') + +# 3. 数据目录 bak 文件 +for f in ['/home/hmo/web-dashboard/data/watchlist.json.bak2', + '/home/hmo/.hermes/profiles/position-analyst/data/mofin.db.bak']: + if os.path.exists(f): + shutil.move(f, os.path.join(ARCHIVE, os.path.basename(f))) + print(f' bak: {os.path.basename(f)}') + +print('DONE ->', ARCHIVE) \ No newline at end of file diff --git a/scripts/audit_data.py b/scripts/audit_data.py new file mode 100644 index 00000000..3ff6cb3d --- /dev/null +++ b/scripts/audit_data.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""audit_data.py — 数据存储碎片化审计：三个数据目录 + 遗留JSON + 旧库""" +import os, json, sqlite3 +from datetime import datetime + +DATA_DIRS = [ + '/home/hmo/MoFin/data', + '/home/hmo/web-dashboard/data', + '/home/hmo/.hermes/profiles/position-analyst/scripts/data', + '/home/hmo/.hermes/profiles/position-analyst/data', + '/home/hmo/.hermes/data', +] + +now = datetime.now() + +print("=== 1. 各数据目录内容与时效 ===") +for dd in DATA_DIRS: + if not os.path.isdir(dd): + print(f"\n{dd}: 不存在") + continue + print(f"\n{dd}:") + try: + entries = [] + for f in sorted(os.listdir(dd)): + p = os.path.join(dd, f) + if os.path.isfile(p): + mt = datetime.fromtimestamp(os.path.getmtime(p)) + age_h = (now - mt).total_seconds() / 3600 + entries.append((f, os.path.getsize(p), mt, age_h)) + # 只显示 >7天 或 >100KB 的 + for f, size, mt, age_h in entries: + flag = '🔴' if age_h > 24*14 else ('🟡' if age_h > 24*3 else '🟢') + print(f" {flag} {f:45s} {size//1024:6d}KB {mt.strftime('%m-%d %H:%M')} ({age_h/24:.0f}d)") + except Exception as e: + print(f" ERR: {e}") + +print("\n=== 2. 三个 mofin.db 对比 ===") +for label, path in [('canonical', '/home/hmo/MoFin/data/mofin.db'), + ('profile-local', '/home/hmo/.hermes/profiles/position-analyst/scripts/data/mofin.db'), + ('profile-data', '/home/hmo/.hermes/profiles/position-analyst/data/mofin.db')]: + if not os.path.exists(path): + print(f" {label}: 不存在") + continue + c = sqlite3.connect(path, timeout=5) + tables = {r[0]: r[1] for r in c.execute( + "SELECT name, (SELECT COUNT(*) FROM sqlite_master m2 WHERE m2.name=m1.name) FROM sqlite_master m1 WHERE type='table'")} + cnts = {} + for t in tables: + try: + cnts[t] = c.execute(f"SELECT COUNT(*) FROM {t}").fetchone()[0] + except Exception: + cnts[t] = -1 + nonempty = {k: v for k, v in cnts.items() if v > 0} + print(f" {label} ({os.path.getsize(path)//1024}KB): {len(nonempty)} 张非空表") + for t, n in sorted(nonempty.items(), key=lambda x: -x[1])[:8]: + print(f" {t}: {n}") + c.close() + +print("\n=== 3. 遗留 JSON 文件（web-dashboard/data，按最后修改排序）===") +wd = '/home/hmo/web-dashboard/data' +jfiles = [] +for f in os.listdir(wd): + if f.endswith('.json') and os.path.isfile(os.path.join(wd, f)): + p = os.path.join(wd, f) + mt = datetime.fromtimestamp(os.path.getmtime(p)) + jfiles.append((f, os.path.getsize(p), mt)) +jfiles.sort(key=lambda x: x[2]) +for f, size, mt in jfiles: + age_d = (now - mt).total_seconds() / 86400 + flag = '🔴' if age_d > 14 else ('🟡' if age_d > 3 else '🟢') + print(f" {flag} {f:45s} {size//1024:5d}KB {mt.strftime('%m-%d')} ({age_d:.0f}d前)") + +print("\n=== 4. trashbox / archive / 旧日志 ===") +for d in ['/home/hmo/trashbox', '/home/hmo/MoFin/archive', '/home/hmo/MoFin/data/archive']: + if os.path.isdir(d): + total = sum(os.path.getsize(os.path.join(r, f)) for r, _, fs in os.walk(d) for f in fs if os.path.isfile(os.path.join(r, f))) + cnt = sum(len(fs) for _, _, fs in os.walk(d)) + print(f" {d}: {cnt} 个文件, {total//1024//1024}MB") +logs = [f for f in os.listdir('/home/hmo') if f.endswith('.log') and os.path.isfile(f'/home/hmo/{f}')] +print(f"\n /home/hmo 根目录 .log 文件: {len(logs)} 个") +for f in sorted(logs): + p = f'/home/hmo/{f}' + mt = datetime.fromtimestamp(os.path.getmtime(p)) + print(f" {f} {os.path.getsize(p)//1024}KB {mt.strftime('%m-%d')}") \ No newline at end of file diff --git a/scripts/audit_deadcode.py b/scripts/audit_deadcode.py new file mode 100644 index 00000000..a747c946 --- /dev/null +++ b/scripts/audit_deadcode.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""audit_deadcode.py — 废弃模块/死代码审计：小果生态、旧bot、疑似废弃脚本""" +import os, subprocess, json +from datetime import datetime + +def sh(cmd, timeout=10): + try: + r = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout) + return r.stdout.strip() + except Exception as e: + return f"ERR: {e}" + +print("=== 1. 小果(xiaoguo)生态残留 ===") +print("--- 运行中的 xiaoguo 进程 ---") +print(sh("ps aux | grep -i xiaoguo | grep -v grep")) +print("\n--- xiaoguo 相关文件 ---") +print(sh("ls -la /home/hmo/MoFin/deploy/profile-scripts/ 2>/dev/null | grep -i xiaoguo; ls /home/hmo/MoFin/ | grep -i xiaoguo; ls /home/hmo/MoFin/scripts/ 2>/dev/null | grep -i xiaoguo; ls /home/hmo/AgentsMeeting/ 2>/dev/null | grep -i xiaoguo | head -5")) +print("\n--- xiaoguo 相关数据文件 ---") +print(sh("ls -la /home/hmo/web-dashboard/data/ | grep -i xiaoguo; ls -la /home/hmo/MoFin/data/ 2>/dev/null | grep -i xiaoguo")) +print("\n--- xiaoguo 相关 cron ---") +for jf, prof in [('/home/hmo/.hermes/profiles/position-analyst/cron/jobs.json', 'pa'), ('/home/hmo/.hermes/cron/jobs.json', 'default')]: + try: + d = json.load(open(jf)) + jobs = d if isinstance(d, list) else d.get('jobs', []) + for j in jobs: + if 'xiaoguo' in str(j.get('script', '')).lower() or '小果' in str(j.get('name', '')): + print(f" [{prof}] {j.get('name')} script={j.get('script')} enabled={j.get('enabled')} last={str(j.get('last_run_at'))[:10]} status={j.get('last_status')}") + except Exception as e: + print(f" {prof}: {e}") + +print("\n=== 2. 旧 bot 文件 ===") +for f in ['/home/hmo/xmpp_zhiwei_bot.py.bak', '/home/hmo/run_zhiwei_bot.py', '/home/hmo/xmpp_bot_rest.py', + '/home/hmo/xmpp_xiaoguo_bot.py', '/home/hmo/xmpp_mohe_bot.py', '/home/hmo/xmpp_zhiwei_bot.py']: + if os.path.exists(f): + p = f + mt = datetime.fromtimestamp(os.path.getmtime(p)) + link = os.path.islink(p) + tgt = os.readlink(p) if link else '' + print(f" {f} {'symlink->'+tgt if link else str(os.path.getsize(p)//1024)+'KB'} {mt.strftime('%m-%d')}") + +print("\n=== 3. AgentsMeeting 目录与 MoFin 的关系（是否重复项目）===") +print(sh("du -sh /home/hmo/AgentsMeeting /home/hmo/MoFin /home/hmo/web-dashboard /home/hmo/projects/AgentsMeeting /home/hmo/projects/MoFin 2>/dev/null")) +print("\n--- AgentsMeeting 里和 MoFin 同名的脚本 ---") +am = '/home/hmo/AgentsMeeting' +if os.path.isdir(am): + am_scripts = set() + for r, _, fs in os.walk(am): + if 'venv' in r or 'node_modules' in r or '.git' in r: + continue + for f in fs: + if f.endswith('.py'): + am_scripts.add(f) + mofin_scripts = set() + for d in ['/home/hmo/MoFin', '/home/hmo/MoFin/scripts', '/home/hmo/MoFin/deploy/profile-scripts']: + if os.path.isdir(d): + mofin_scripts.update(f for f in os.listdir(d) if f.endswith('.py')) + common = sorted(am_scripts & mofin_scripts) + print(f" 同名 .py 文件: {len(common)} 个") + for f in common[:20]: + print(f" {f}") + +print("\n=== 4. 超过30天未修改的可疑废弃脚本（deploy/profile-scripts 中）===") +now = datetime.now() +dd = '/home/hmo/MoFin/deploy/profile-scripts' +for f in sorted(os.listdir(dd)): + if not f.endswith('.py'): + continue + p = os.path.join(dd, f) + mt = datetime.fromtimestamp(os.path.getmtime(p)) + age = (now - mt).total_seconds() / 86400 + if age > 30: + print(f" {f:45s} {mt.strftime('%m-%d')} ({age:.0f}d)") + +print("\n=== 5. scripts/ 下的工具脚本（非 cron 调度，疑似一次性）===") +# 哪些脚本不在 cron jobs 里 +cron_scripts = set() +for jf in ['/home/hmo/.hermes/profiles/position-analyst/cron/jobs.json', '/home/hmo/.hermes/cron/jobs.json']: + try: + d = json.load(open(jf)) + jobs = d if isinstance(d, list) else d.get('jobs', []) + for j in jobs: + if j.get('script'): + cron_scripts.add(j['script']) + except Exception: + pass +for d in ['/home/hmo/MoFin/deploy/profile-scripts']: + for f in sorted(os.listdir(d)): + if f.endswith('.py') and f not in cron_scripts: + p = os.path.join(d, f) + mt = datetime.fromtimestamp(os.path.getmtime(p)) + age = (now - mt).total_seconds() / 86400 + print(f" {f:45s} 不在cron中 ({age:.0f}d)") \ No newline at end of file diff --git a/scripts/audit_duplication.py b/scripts/audit_duplication.py new file mode 100644 index 00000000..4c6ae50c --- /dev/null +++ b/scripts/audit_duplication.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""audit_duplication.py — 扫描 MoFin 相关目录的文件重复情况""" +import os, hashlib, json +from collections import defaultdict + +LOCATIONS = [ + '/home/hmo/MoFin', + '/home/hmo/MoFin/scripts', + '/home/hmo/MoFin/deploy/profile-scripts', + '/home/hmo/MoFin/deploy/bot', + '/home/hmo/.hermes/profiles/position-analyst/scripts', + '/home/hmo/.hermes/scripts', + '/home/hmo/web-dashboard', +] + +def md5(p): + try: + with open(p, 'rb') as f: + return hashlib.md5(f.read()).hexdigest()[:10] + except Exception: + return 'ERR' + +def ino(p): + try: + return os.stat(p).st_ino + except Exception: + return 0 + +# 收集所有 .py 文件 +files = defaultdict(list) # name -> [(loc, path, md5, inode, is_link)] +for loc in LOCATIONS: + if not os.path.isdir(loc): + continue + for f in os.listdir(loc): + if not f.endswith('.py'): + continue + p = os.path.join(loc, f) + if not os.path.isfile(p): + continue + files[f].append({ + 'loc': loc, 'path': p, 'md5': md5(p), 'inode': ino(p), + 'is_link': os.path.islink(p), + 'size': os.path.getsize(p), + 'mtime': int(os.path.getmtime(p)), + }) + +print("=== 重复文件（同名出现在2+位置）===") +dups = {k: v for k, v in files.items() if len(v) > 1} +identical = 0 +hardlinked = 0 +diverged = 0 +for name in sorted(dups): + entries = dups[name] + md5s = set(e['md5'] for e in entries) + inodes = set(e['inode'] for e in entries) + if len(inodes) == 1: + status = 'HARDLINK(同一文件)' + hardlinked += 1 + elif len(md5s) == 1: + status = 'COPY(内容相同,多份独立)' + identical += 1 + else: + status = 'DIVERGED(内容不同!)' + diverged += 1 + print(f"{status} {name}") + for e in entries: + print(f" {e['path']} md5={e['md5']} ino={e['inode']} size={e['size']}") + +print(f"\n汇总: {len(dups)} 个重复文件名 | hardlink={hardlinked} 内容相同副本={identical} 内容分叉={diverged}") + +print("\n=== deploy/profile-scripts 中有但 position-analyst/scripts 中缺失的 ===") +pa_dir = '/home/hmo/.hermes/profiles/position-analyst/scripts' +deploy_dir = '/home/hmo/MoFin/deploy/profile-scripts' +pa_files = set(os.listdir(pa_dir)) if os.path.isdir(pa_dir) else set() +for f in sorted(os.listdir(deploy_dir)): + if f.endswith('.py') and f not in pa_files: + print(f" {f}") + +print("\n=== position-analyst/scripts 中有但 deploy 中没有的（可能孤儿）===") +deploy_files = set(os.listdir(deploy_dir)) +for f in sorted(pa_files): + if f.endswith('.py') and f not in deploy_files: + p = os.path.join(pa_dir, f) + print(f" {f} size={os.path.getsize(p)}") \ No newline at end of file diff --git a/scripts/audit_runtime.py b/scripts/audit_runtime.py new file mode 100644 index 00000000..7d7a9831 --- /dev/null +++ b/scripts/audit_runtime.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""audit_runtime.py — 运行中的进程/服务/cron 审计""" +import subprocess, json, os + +def sh(cmd, timeout=10): + try: + r = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout) + return r.stdout.strip() + except Exception as e: + return f"ERR: {e}" + +print("=== 1. 长期运行的 Python 进程（>1天）===") +out = sh("ps -eo pid,etime,user,args --sort=-etime | grep -E 'python|node' | grep -v grep | head -25") +print(out) + +print("\n=== 2. systemd 服务状态（全部 mofin/hermes/xmpp/ejabberd 相关）===") +out = sh("systemctl list-units --all --type=service 2>/dev/null | grep -iE 'mofin|hermes|xmpp|ejabberd|wechat|gitea|kanban|wiki|stock|xiaoguo|zhiwei|mohe' | head -25") +print(out) +print("\n--- user services ---") +out = sh("systemctl --user list-units --all --type=service 2>/dev/null | grep -iE 'hermes|xmpp|mofin|wiki' | head -10") +print(out) + +print("\n=== 3. DISABLED cron jobs（hermes 两个 profile）===") +for jf, prof in [('/home/hmo/.hermes/profiles/position-analyst/cron/jobs.json', 'position-analyst'), + ('/home/hmo/.hermes/cron/jobs.json', 'default')]: + try: + d = json.load(open(jf)) + jobs = d if isinstance(d, list) else d.get('jobs', []) + for j in jobs: + if not j.get('enabled', True): + lr = str(j.get('last_run_at') or '?')[:10] + print(f" [{prof}] {j.get('name')} | script={j.get('script')} | last={lr} | paused_reason={j.get('paused_reason')}") + except Exception as e: + print(f" {prof}: {e}") + +print("\n=== 4. 系统 crontab 全文（非注释行）===") +out = sh("crontab -l | grep -vE '^\\s*#' | grep -vE '^\\s*$'") +print(out) + +print("\n=== 5. 僵尸/异常进程（root跑的python、很老的进程）===") +out = sh("ps -eo pid,etime,user,args | grep -E '^\\s*\\S+\\s+\\S+\\s+root' | grep python | grep -v grep") +print(out if out else "(none)") + +print("\n=== 6. 监听端口清单 ===") +out = sh("ss -tlnp 2>/dev/null | grep -E '8642|8643|8645|8646|8899|5801|5802|5803|5804|5805|5807|5808|5810|9090|9580|5222|5443|5280|19088|19099|9877|9878' | awk '{print $4, $6}'") +print(out) \ No newline at end of file diff --git a/scripts/classify_diverged.py b/scripts/classify_diverged.py new file mode 100644 index 00000000..29db40ab --- /dev/null +++ b/scripts/classify_diverged.py @@ -0,0 +1,47 @@ +import subprocess, os, re +from datetime import datetime + +DIVERGED = """advice_reconciliation.py branch_scanner.py bulk_strategy_regenerate.py +collect_evaluation_data.py cron_to_xmpp.py market_insight.py market_screener.py market_watch.py +memory_guardian.py mo_provider.py mofin_collect.py mofin_health.py mofin_news.py multi_timeframe.py +premarket_full_review.py promote_candidates.py prune_branches.py server.py stock_profile.py +stock_sector_enrich.py strategy_evaluator.py strategy_feedback.py strategy_lifecycle.py strategy_tree.py +system_audit.py system_health_check.py technical_analysis.py trend_detector.py +xiaoguo_news_processor.py xiaoguo_scanner.py xmpp_agent_core.py""".split() + +SEARCH_DIRS = ['/home/hmo/MoFin/deploy/profile-scripts', '/home/hmo/.hermes/profiles/position-analyst/scripts', '/home/hmo/.hermes/scripts'] + +now = datetime.now() + +print(f"{'file':35s} {'imported_by':40s} {'cron?':5s} {'root_mtime':12s} {'deploy_mtime':12s} verdict") +print('-' * 130) + +for f in DIVERGED: + stem = f[:-3] + # 谁 import 它 + r = subprocess.run( + f"grep -rln -E '(^|\\s)(from|import)\\s+{stem}(\\s|$|\\.)' /home/hmo/MoFin/deploy/profile-scripts /home/hmo/.hermes/profiles/position-analyst/scripts /home/hmo/MoFin --include='*.py' 2>/dev/null | grep -v venv | grep -v '{f}' | head -3", + shell=True, capture_output=True, text=True, timeout=15) + importers = [os.path.basename(x) for x in r.stdout.splitlines() if x.strip() and f not in x] + imported = ','.join(importers[:3]) if importers else '-' + + # 是否在 cron 里被直接执行 + in_cron = subprocess.run( + f"grep -l '\"script\": \"{f}\"' /home/hmo/.hermes/profiles/position-analyst/cron/jobs.json /home/hmo/.hermes/cron/jobs.json 2>/dev/null", + shell=True, capture_output=True, text=True, timeout=5) + is_cron = 'Y' if in_cron.stdout.strip() else '-' + + # mtimes + root_p = f'/home/hmo/MoFin/{f}' + deploy_p = f'/home/hmo/MoFin/deploy/profile-scripts/{f}' + rm = datetime.fromtimestamp(os.path.getmtime(root_p)).strftime('%m-%d') if os.path.exists(root_p) else '-' + dm = datetime.fromtimestamp(os.path.getmtime(deploy_p)).strftime('%m-%d') if os.path.exists(deploy_p) else '-' + + # verdict + if importers: + verdict = 'LIBRARY->root为准' + elif is_cron == 'Y': + verdict = 'CRON->deploy为准' + else: + verdict = '待查' + print(f"{f:35s} {imported:40s} {is_cron:5s} {rm:12s} {dm:12s} {verdict}") \ No newline at end of file diff --git a/scripts/cleanup_disabled_cron.py b/scripts/cleanup_disabled_cron.py new file mode 100644 index 00000000..c0d58685 --- /dev/null +++ b/scripts/cleanup_disabled_cron.py @@ -0,0 +1,30 @@ +import json, shutil + +REMOVE = { + 'pa': ['MoFin 盘前中监控', 'MoFin 午后监控', '自选买入区提醒', '候选股自动推广-盘中'], + 'default': ['持仓情报-盘中', '快速盯盘-15分钟', '价格监控-1分钟', 'PM-项目跟进'], +} +FILES = { + 'pa': '/home/hmo/.hermes/profiles/position-analyst/cron/jobs.json', + 'default': '/home/hmo/.hermes/cron/jobs.json', +} + +for prof, jf in FILES.items(): + shutil.copy(jf, jf + '.bak-20260720-disabled-cleanup') + d = json.load(open(jf)) + is_list = isinstance(d, list) + jobs = d if is_list else d.get('jobs', []) + removed = [j.get('name') for j in jobs if j.get('name') in REMOVE[prof]] + kept = [j for j in jobs if j.get('name') not in REMOVE[prof]] + if removed: + if is_list: + json.dump(kept, open(jf, 'w'), ensure_ascii=False, indent=2) + else: + d['jobs'] = kept + json.dump(d, open(jf, 'w'), ensure_ascii=False, indent=2) + print(f'{prof}: removed {removed}') + # 剩余禁用任务 + still_disabled = [j.get('name') for j in kept if not j.get('enabled', True)] + print(f' remaining disabled: {still_disabled}') + total = len(kept) + print(f' total jobs now: {total}') \ No newline at end of file diff --git a/scripts/fix_crontab_cron_to_xmpp.py b/scripts/fix_crontab_cron_to_xmpp.py new file mode 100644 index 00000000..10ebac9b --- /dev/null +++ b/scripts/fix_crontab_cron_to_xmpp.py @@ -0,0 +1,27 @@ +import subprocess + +# 更新 crontab：cron_to_xmpp.py 改指 deploy 路径（web-dashboard 副本已归档） +r = subprocess.run(['crontab', '-l'], capture_output=True, text=True) +lines = r.stdout.splitlines() +new_lines = [] +for line in lines: + if 'cron_to_xmpp.py' in line and 'web-dashboard' in line: + new_line = line.replace( + 'cd /home/hmo/web-dashboard && python3 cron_to_xmpp.py', + 'cd /home/hmo/MoFin/deploy/profile-scripts && python3 cron_to_xmpp.py') + new_lines.append(new_line) + print('CHANGED:') + print(' old:', line) + print(' new:', new_line) + else: + new_lines.append(line) + +p = subprocess.run(['crontab', '-'], input='\n'.join(new_lines) + '\n', + capture_output=True, text=True) +print('crontab updated, rc=', p.returncode) + +# 验证 deploy 版本能跑（dry check: import + 语法） +r2 = subprocess.run( + "cd /home/hmo/MoFin/deploy/profile-scripts && python3 -c 'import ast; ast.parse(open(\"cron_to_xmpp.py\").read()); print(\"syntax OK\")'", + shell=True, capture_output=True, text=True, timeout=10) +print(r2.stdout.strip() or r2.stderr.strip()[:200]) \ No newline at end of file diff --git a/scripts/fix_crontab_market.py b/scripts/fix_crontab_market.py new file mode 100644 index 00000000..d7c4de80 --- /dev/null +++ b/scripts/fix_crontab_market.py @@ -0,0 +1,17 @@ +import subprocess + +r = subprocess.run(['crontab', '-l'], capture_output=True, text=True) +lines = r.stdout.splitlines() +new_lines = [] +for line in lines: + nl = line + if 'cd /home/hmo/MoFin &&' in line and ('market_watch.py' in line or 'market_screener.py' in line): + nl = line.replace('cd /home/hmo/MoFin &&', 'cd /home/hmo/MoFin/deploy/profile-scripts &&') + print('CHANGED:') + print(' old:', line) + print(' new:', nl) + new_lines.append(nl) + +p = subprocess.run(['crontab', '-'], input='\n'.join(new_lines) + '\n', + capture_output=True, text=True) +print('rc=', p.returncode) \ No newline at end of file diff --git a/scripts/inspect_third_db.py b/scripts/inspect_third_db.py new file mode 100644 index 00000000..894dbc0c --- /dev/null +++ b/scripts/inspect_third_db.py @@ -0,0 +1,33 @@ +import sqlite3 +from datetime import datetime + +THIRD = '/home/hmo/.hermes/profiles/position-analyst/scripts/data/mofin.db' +MAIN = '/home/hmo/MoFin/data/mofin.db' + +t = sqlite3.connect(THIRD, timeout=10) +m = sqlite3.connect(MAIN, timeout=30) +m.execute('PRAGMA busy_timeout=30000') + +for table in ['sector_snapshots', 'market_snapshots', 'todos', 'capital_flow_cache']: + try: + tcols = [r[1] for r in t.execute(f"PRAGMA table_info({table})")] + mcols = [r[1] for r in m.execute(f"PRAGMA table_info({table})")] + print(f'{table}: third_cols={tcols}') + print(f' main_cols={mcols}') + tcnt = t.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] + mcnt = m.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] + print(f' third={tcnt} rows, main={mcnt} rows') + except Exception as e: + print(f'{table}: ERR {e}') + print() + +# sector_snapshots schema in both +print('sector_snapshots sample (third):') +for r in t.execute("SELECT * FROM sector_snapshots ORDER BY rowid DESC LIMIT 2"): + print(' ', str(r)[:200]) +print('sector_snapshots sample (main):') +for r in m.execute("SELECT * FROM sector_snapshots ORDER BY rowid DESC LIMIT 2"): + print(' ', str(r)[:200]) + +t.close() +m.close() \ No newline at end of file diff --git a/scripts/merge_third_db.py b/scripts/merge_third_db.py new file mode 100644 index 00000000..6af1e289 --- /dev/null +++ b/scripts/merge_third_db.py @@ -0,0 +1,94 @@ +import sqlite3, os, shutil +from datetime import datetime + +THIRD = '/home/hmo/.hermes/profiles/position-analyst/scripts/data/mofin.db' +MAIN = '/home/hmo/MoFin/data/mofin.db' +BACKUP = f'/home/hmo/MoFin/archive/third-db-backup-{datetime.now().strftime("%Y%m%d-%H%M")}' +os.makedirs(BACKUP, exist_ok=True) +shutil.copy(THIRD, os.path.join(BACKUP, 'mofin.db')) +print('backup:', BACKUP) + +t = sqlite3.connect(THIRD, timeout=10) +m = sqlite3.connect(MAIN, timeout=30) +m.execute('PRAGMA busy_timeout=30000') + +# 1. sector_snapshots: dedup by (snapshot_id, name) +existing = set() +for r in m.execute("SELECT snapshot_id, name FROM sector_snapshots"): + existing.add((r[0], r[1])) +print(f'main sector_snapshots keys: {len(existing)}') + +cols = ['snapshot_id', 'name', 'change_pct', 'up_count', 'down_count', 'net_inflow', + 'lead_stock', 'lead_stock_change', 'volume', 'turnover'] +ins = 0 +skip = 0 +for r in t.execute(f"SELECT {', '.join(cols)} FROM sector_snapshots"): + if (r[0], r[1]) in existing: + skip += 1 + continue + m.execute(f"INSERT INTO sector_snapshots ({', '.join(cols)}) VALUES ({','.join('?'*len(cols))})", r) + existing.add((r[0], r[1])) + ins += 1 +m.commit() +print(f'sector_snapshots: inserted {ins}, skipped {skip}') + +# 2. market_snapshots: dedup by (timestamp, source) +existing2 = set() +for r in m.execute("SELECT timestamp, source FROM market_snapshots"): + existing2.add((r[0], r[1])) +cols2 = ['timestamp', 'source', 'up_ratio', 'mood', 'created_at'] +ins2 = 0 +skip2 = 0 +for r in t.execute(f"SELECT {', '.join(cols2)} FROM market_snapshots"): + if (r[0], r[1]) in existing2: + skip2 += 1 + continue + m.execute(f"INSERT INTO market_snapshots ({', '.join(cols2)}) VALUES ({','.join('?'*len(cols2))})", r) + existing2.add((r[0], r[1])) + ins2 += 1 +m.commit() +print(f'market_snapshots: inserted {ins2}, skipped {skip2}') + +# 3. todos: third has 4 rows, main has 94 — check by title, insert missing (map to main schema) +todos_t = t.execute("SELECT title, description, status, priority, source, fix_action, retry_count, note, created_at, updated_at FROM todos").fetchall() +ins3 = 0 +for r in todos_t: + ex = m.execute("SELECT id FROM todos WHERE title=?", (r[0],)).fetchone() + if ex: + continue + m.execute("INSERT INTO todos (title, description, status, priority, source, fix_action, retry_count, note, created_at, updated_at) VALUES (?,?,?,?,?,?,?,?,?,?)", r) + ins3 += 1 +m.commit() +print(f'todos: inserted {ins3}, skipped {len(todos_t)-ins3}') + +# 4. capital_flow_cache: keep whichever is newer +t_row = t.execute("SELECT cache_json, updated_at FROM capital_flow_cache ORDER BY id DESC LIMIT 1").fetchone() +m_row = m.execute("SELECT cache_json, updated_at FROM capital_flow_cache ORDER BY id DESC LIMIT 1").fetchone() +if t_row and (not m_row or (t_row[1] or '') > (m_row[1] or '')): + m.execute("DELETE FROM capital_flow_cache") + m.execute("INSERT INTO capital_flow_cache (cache_json, updated_at) VALUES (?,?)", t_row) + m.commit() + print('capital_flow_cache: replaced with third (newer)') +else: + print('capital_flow_cache: main is newer/equal, kept') + +# verify +total = m.execute("SELECT COUNT(*) FROM sector_snapshots").fetchone()[0] +total2 = m.execute("SELECT COUNT(*) FROM market_snapshots").fetchone()[0] +print(f'after merge: sector_snapshots={total}, market_snapshots={total2}') + +m.close() +t.close() + +# 5. 删除第三库（连同 data 目录里的其他残留） +third_dir = os.path.dirname(THIRD) +trash = '/home/hmo/trashbox/third-db-retired-20260720' +os.makedirs(trash, exist_ok=True) +shutil.move(THIRD, os.path.join(trash, 'mofin.db')) +print('third db moved to', trash) +for f in os.listdir(third_dir): + src = os.path.join(third_dir, f) + if os.path.isfile(src): + shutil.move(src, os.path.join(trash, f)) + print(' also moved:', f) +print('DONE') \ No newline at end of file diff --git a/scripts/notify_user4.py b/scripts/notify_user4.py new file mode 100644 index 00000000..3f2df39e --- /dev/null +++ b/scripts/notify_user4.py @@ -0,0 +1,32 @@ +import json, urllib.request + +msg = """[笑笑] price_events.json 已彻底退役 — 全系统只认 DB + +按你的指示，所有和 price_events.json 相关的代码已全部改完： + +【代码修改（6个文件）】 +1. price_monitor.py — record_event 只写 DB，EVENTS_PATH/load_events/save_events 全部删除 +2. strategy_feedback.py — 价格事件只从 DB 读，JSON fallback 删除 +3. system_health_check.py — 文件检查清单移除 price_events.json，事件统计只读 DB +4. mo_config.py — 删除无调用的 price_events_path 死属性 +5. mofin_health.py — 新鲜度检查改回读 DB 表（现在它是权威了） +6. mofin_db.py — 修了一个更大的坑 ⬇️ + +【顺藤摸出的更大的坑：三个数据库】 +mofin_db.py 的 DB_PATH 原来是相对路径（__file__.parent/data），导致每个硬链接副本解析到不同的库：主库、web-dashboard 库、还有 profile 本地一个几乎全空的第三库。健康检查之前读到 0 条事件就是因为读到了那个空库。已改为绝对路径 /home/hmo/MoFin/data，4 份 mofin_db 副本已统一硬链接。 + +【验证结果】 +✅ record_event 只写 DB，JSON 不再被重建 +✅ system_health_check：历史事件 6353 条 / 今日事件 2965 条（之前误报 0/0） +✅ strategy_feedback / price_monitor 完整跑通 +✅ price_events.json 已归档 trashbox（数据早已全量回填 DB） + +JSON 时代结束。""" + +payload = json.dumps({"to": "hmo@yoin.fun", "body": msg, "type": "chat"}).encode() +req = urllib.request.Request("http://127.0.0.1:5805/", data=payload, + headers={"Content-Type": "application/json"}) +try: + print("XMPP:", urllib.request.urlopen(req, timeout=10).read().decode()[:80]) +except Exception as e: + print("XMPP fail:", e) \ No newline at end of file diff --git a/scripts/notify_user5.py b/scripts/notify_user5.py new file mode 100644 index 00000000..3546a0db --- /dev/null +++ b/scripts/notify_user5.py @@ -0,0 +1,38 @@ +import json, urllib.request + +msg = """[笑笑] 系统冗余全面审计完成 — 报告已生成 + +按你的要求只审计不动手。报告位置： +📄 本地 vault: daily/调研笔记/系统冗余审计-20260720.md +📄 246 Obsidian: daily/系统冗余审计-20260720.md + +核心发现（9 类问题）： + +🔴 高危×3： +1. 31 个同名 .py 文件内容分叉（deploy vs MoFin/scripts vs MoFin根 vs .hermes/scripts）——今天"修了还报错"的病根就是这个 +2. 数据库碎片化：除主库外还有一个"第三库"（含 17340 行 sector 数据需合并）+ 3 个废弃小库 +3. 僵尸进程：xmpp_xiaoguo_bot 以 root 跑了 8 天吃 2.5GB 内存；xiaoguo-tunnel 对着不可达的 Mac Mini 循环重连 + +🟡 中危×4： +4. 三套 dashboard（8899/5803/9090）+ 两套 health check + 两套 auto_heal +5. 小果生态残留：2 进程 + 4 脚本 + 2 服务 + 2 cron（扫描死了但其余还活着） +6. 26 个 >14 天的遗留 JSON 文件 +7. projects/ 与根目录两对重复目录（1.4G + 2.2M） + +🟢 低危×2： +8. 10 个被禁用的 cron 任务 +9. 18 个旧日志 + 旧 .bak 文件 + +报告末尾有 7 个需要你拍板的问题（Q1-Q7），最关键的两个： +- Q1: xiaoguo gateway (8645) 的"压缩策略"还在用吗？ +- Q2: xiaoguo-quick-scan（今天还在跑）和 market_scanner 什么关系？ + +你看完报告定方向，我再按决定执行。""" + +payload = json.dumps({"to": "hmo@yoin.fun", "body": msg, "type": "chat"}).encode() +req = urllib.request.Request("http://127.0.0.1:5805/", data=payload, + headers={"Content-Type": "application/json"}) +try: + print("XMPP:", urllib.request.urlopen(req, timeout=10).read().decode()[:80]) +except Exception as e: + print("XMPP fail:", e) \ No newline at end of file diff --git a/scripts/remove_xiaoguo.py b/scripts/remove_xiaoguo.py new file mode 100644 index 00000000..f2febf89 --- /dev/null +++ b/scripts/remove_xiaoguo.py @@ -0,0 +1,52 @@ +import json, shutil, os +from datetime import datetime + +# 1. 删小果相关 cron job（quick-scan / tunnel-watchdog / 情感分析 / 独立扫描） +REMOVE_JOBS = {'xiaoguo-quick-scan', 'xiaoguo-tunnel-watchdog', '小果情感分析', '小果独立扫描'} +for jf in ['/home/hmo/.hermes/profiles/position-analyst/cron/jobs.json', + '/home/hmo/.hermes/cron/jobs.json']: + shutil.copy(jf, jf + '.bak-20260720-xiaoguo') + d = json.load(open(jf)) + is_list = isinstance(d, list) + jobs = d if is_list else d.get('jobs', []) + kept = [j for j in jobs if j.get('name') not in REMOVE_JOBS] + removed = [j.get('name') for j in jobs if j.get('name') in REMOVE_JOBS] + if removed: + print(f'{jf}: removed {removed}') + if is_list: + json.dump(kept, open(jf, 'w'), ensure_ascii=False, indent=2) + else: + d['jobs'] = kept + json.dump(d, open(jf, 'w'), ensure_ascii=False, indent=2) + else: + print(f'{jf}: nothing to remove') + +# 2. 归档小果脚本和数据文件 +ARCHIVE = '/home/hmo/MoFin/archive/xiaoguo-retired-20260720' +os.makedirs(ARCHIVE, exist_ok=True) +candidates = [ + '/home/hmo/MoFin/deploy/profile-scripts/xiaoguo_scanner.py', + '/home/hmo/MoFin/deploy/profile-scripts/xiaoguo_news_processor.py', + '/home/hmo/MoFin/deploy/profile-scripts/xiaoguo_sentiment_bridge.py', + '/home/hmo/MoFin/deploy/profile-scripts/xiaoguo_signal_consumer.py', + '/home/hmo/MoFin/scripts/inject_xiaoguo_insight.py', + '/home/hmo/web-dashboard/data/xiaoguo_insights.json', + '/home/hmo/web-dashboard/data/xiaoguo_sentiment.json', + '/home/hmo/MoFin/data/xiaoguo_insights.json', + '/home/hmo/MoFin/data/xiaoguo_sentiment.json', +] +for f in candidates: + if os.path.exists(f): + shutil.move(f, os.path.join(ARCHIVE, os.path.basename(f))) + print('archived:', f) + +# 3. 处理 profile scripts 里残留的 xiaoguo 硬链接 +import glob +for f in glob.glob('/home/hmo/.hermes/profiles/position-analyst/scripts/xiaoguo_*.py'): + os.unlink(f) + print('unlinked:', f) + +# 4. 检查 default profile scripts 的 xiaoguo 文件 +for f in glob.glob('/home/hmo/.hermes/scripts/xiaoguo_*.py') + glob.glob('/home/hmo/.hermes/scripts/xiaoguo_*.sh'): + print('default profile xiaoguo file:', f) +print('DONE') \ No newline at end of file diff --git a/scripts/remove_xiaoguo2.py b/scripts/remove_xiaoguo2.py new file mode 100644 index 00000000..0a7efaed --- /dev/null +++ b/scripts/remove_xiaoguo2.py @@ -0,0 +1,18 @@ +import shutil, os, glob +ARCHIVE = '/home/hmo/MoFin/archive/xiaoguo-retired-20260720' +os.makedirs(ARCHIVE, exist_ok=True) +for f in ['/home/hmo/.hermes/scripts/xiaoguo_quick_scan.py', + '/home/hmo/.hermes/scripts/xiaoguo_scanner.py', + '/home/hmo/.hermes/scripts/xiaoguo_news_processor.py', + '/home/hmo/.hermes/scripts/xiaoguo_tunnel_watchdog.sh', + '/home/hmo/xmpp_xiaoguo_bot.py']: + if os.path.exists(f): + shutil.move(f, os.path.join(ARCHIVE, os.path.basename(f))) + print('archived:', f) +# xiaoguo profile dir +p = '/home/hmo/.hermes/profiles/xiaoguo' +if os.path.isdir(p): + print('xiaoguo profile dir exists:', os.listdir(p)[:8]) +else: + print('no xiaoguo profile dir') +print('DONE') \ No newline at end of file diff --git a/scripts/unify_sources.py b/scripts/unify_sources.py new file mode 100644 index 00000000..45477149 --- /dev/null +++ b/scripts/unify_sources.py @@ -0,0 +1,106 @@ +import os, shutil, glob +from datetime import datetime + +ARCHIVE = f'/home/hmo/MoFin/archive/dedup-20260720-{datetime.now().strftime("%H%M")}' +os.makedirs(ARCHIVE, exist_ok=True) + +def hardlink_to(src, targets): + """把 targets 全部硬链接到 src（内容统一为 src）""" + done = [] + for t in targets: + if not os.path.exists(t) and not os.path.islink(t): + continue + if os.path.isfile(t) and os.path.samefile(src, t): + continue + try: + if os.path.islink(t) or os.path.isfile(t): + os.unlink(t) + os.link(src, t) + done.append(t) + except Exception as e: + print(f' LINK FAIL {t}: {e}') + return done + +def archive(path, note=''): + if os.path.exists(path) or os.path.islink(path): + dst = os.path.join(ARCHIVE, os.path.basename(path)) + if os.path.exists(dst): + dst = dst + '.' + datetime.now().strftime('%H%M%S') + shutil.move(path, dst) + print(f' archived: {path} {note}') + +# ── A. LIBRARY 类：root 是活的（cron 经 sys.path 导入），统一所有副本到 root ── +LIBRARY = ['strategy_lifecycle.py', 'strategy_tree.py', 'technical_analysis.py', + 'multi_timeframe.py', 'mofin_news.py'] +print('=== A. LIBRARY 统一到 MoFin root ===') +for f in LIBRARY: + root = f'/home/hmo/MoFin/{f}' + if not os.path.exists(root): + print(f' {f}: root 不存在，跳过!') + continue + targets = [ + f'/home/hmo/MoFin/deploy/profile-scripts/{f}', + f'/home/hmo/.hermes/profiles/position-analyst/scripts/{f}', + f'/home/hmo/MoFin/scripts/{f}', + f'/home/hmo/web-dashboard/{f}', + ] + done = hardlink_to(root, targets) + print(f' {f}: unified {len(done)} copies -> root') + +# ── B. CRON 类：deploy 是唯一源，归档其他副本 ── +CRON_FILES = ['advice_reconciliation.py', 'branch_scanner.py', 'collect_evaluation_data.py', + 'cron_to_xmpp.py', 'market_insight.py', 'memory_guardian.py', 'mofin_health.py', + 'premarket_full_review.py', 'promote_candidates.py', 'prune_branches.py', + 'strategy_evaluator.py', 'system_health_check.py', 'system_audit.py'] +print('\n=== B. CRON 类归档非 deploy 副本 ===') +for f in CRON_FILES: + for loc in ['/home/hmo/MoFin', '/home/hmo/MoFin/scripts', + '/home/hmo/web-dashboard', '/home/hmo/.hermes/scripts']: + p = os.path.join(loc, f) + # 不碰 deploy 和 pa/scripts（硬链接到 deploy） + if os.path.exists(p) or os.path.islink(p): + # 跳过与 deploy 同 inode 的（那些是合法硬链接） + deploy_p = f'/home/hmo/MoFin/deploy/profile-scripts/{f}' + if os.path.exists(deploy_p) and os.path.isfile(p) and not os.path.islink(p): + try: + if os.path.samefile(p, deploy_p): + continue + except Exception: + pass + archive(p, f'(cron file {f})') + +# ── C. market_watch / market_screener 特殊：crontab 正在跑 root 旧版 ── +print('\n=== C. market_watch/market_screener 统一 ===') +# 暂时只统一内容到 deploy（deploy 是 hermes cron 维护的版本），crontab 稍后改指向 +for f in ['market_watch.py', 'market_screener.py']: + deploy_p = f'/home/hmo/MoFin/deploy/profile-scripts/{f}' + if os.path.exists(deploy_p): + done = hardlink_to(deploy_p, [f'/home/hmo/MoFin/{f}', f'/home/hmo/MoFin/scripts/{f}']) + print(f' {f}: unified {len(done)} -> deploy (crontab 路径仍有效但内容已统一)') + +# ── D. 待查类 ── +print('\n=== D. 待查类 ===') +# server.py: web-dashboard + root 是活的 dashboard，归档 deploy 旧拷贝 +archive('/home/hmo/MoFin/deploy/profile-scripts/server.py', '(live=web-dashboard hardlink)') +# xmpp_agent_core.py: deploy/bot 是活的 +for p in ['/home/hmo/MoFin/xmpp_agent_core.py', '/home/hmo/MoFin/scripts/xmpp_agent_core.py', + '/home/hmo/.hermes/scripts/xmpp_agent_core.py', '/home/hmo/web-dashboard/xmpp_agent_core.py']: + archive(p, '(live=deploy/bot)') +# 手动工具类：deploy 为准，归档 root 旧副本 +for f in ['mo_provider.py', 'mofin_collect.py', 'stock_profile.py', 'stock_sector_enrich.py', + 'trend_detector.py', 'bulk_strategy_regenerate.py', 'strategy_feedback.py']: + deploy_p = f'/home/hmo/MoFin/deploy/profile-scripts/{f}' + if os.path.exists(deploy_p): + done = hardlink_to(deploy_p, [f'/home/hmo/MoFin/{f}', f'/home/hmo/MoFin/scripts/{f}']) + print(f' {f}: unified {len(done)} -> deploy') + else: + for loc in ['/home/hmo/MoFin', '/home/hmo/MoFin/scripts']: + archive(os.path.join(loc, f), '(no deploy version)') + +print('\n=== 验证：关键 import 测试 ===') +import subprocess +r = subprocess.run( + "cd /home/hmo/.hermes/profiles/position-analyst/scripts && python3 -c \"import sys; sys.path.insert(0,'/home/hmo/MoFin'); import strategy_lifecycle, technical_analysis, multi_timeframe, mofin_news, strategy_tree; print('imports OK', strategy_lifecycle.__file__)\"", + shell=True, capture_output=True, text=True, timeout=30) +print(r.stdout.strip() or r.stderr.strip()[:200]) +print('\nARCHIVE dir:', ARCHIVE) \ No newline at end of file diff --git a/specs/hygiene.json b/specs/hygiene.json new file mode 100644 index 00000000..91b077b9 --- /dev/null +++ b/specs/hygiene.json @@ -0,0 +1,49 @@ +{ + "module": "hygiene", + "version": "1.0", + "purpose": "系统卫生审计——防止冗余/废弃/僵尸问题复发的每周自动检查（红线6-10 的 enforcement）", + "human_help": { + "title": "系统卫生审计说明", + "description": [ + "每周一 07:30 自动运行 system_hygiene_audit.py，检查六类系统卫生问题并推送 XMPP 报告。", + "这是 2026-07-20 冗余事件（三个数据库、31个分叉文件、小果僵尸进程）后建立的防复发机制。" + ], + "usage": [ + "手动运行: cd /home/hmo/.hermes/profiles/position-analyst/scripts && python3 system_hygiene_audit.py", + "查看报告: /home/hmo/MoFin/gateway/logs/hygiene_report.json" + ], + "troubleshooting": [ + "分叉副本(diverged_copy): 同名 .py 在不同位置内容不同 → 归档旧副本或硬链接到权威版", + "断裂硬链接(broken_hardlink): deploy 和 profile scripts 内容不一致 → 跑 sync_profile_scripts.sh", + "孤儿文件(orphan_data_file): 生产数据目录 >14 天的文件 → 归档到 archive/（红线8）" + ] + }, + "ai_spec": { + "apis": [], + "dependencies": [ + "deploy/profile-scripts/system_hygiene_audit.py", + "deploy/profile-scripts/sync_profile_scripts.sh", + "docs/dev-spec.md 红线 6-10" + ], + "constraints": [ + "检查项: 分叉副本/断裂硬链接/僵尸进程/孤儿文件/死cron/DB新鲜度", + "docker 容器内进程豁免僵尸检测", + "报告写入 gateway/logs/hygiene_report.json，有问题推 XMPP(:5805)", + "活文件注册表 LIVE_DATA_FILES 需随新数据文件引入而更新" + ], + "must_not": [ + "禁止只报告不修复——发现问题必须在当轮处理或创建 TODO", + "禁止将生产数据目录下 >14 天文件默认视为正常" + ], + "tests": [ + {"id": "H1", "name": "人为制造分叉副本，下一轮审计能发现"}, + {"id": "H2", "name": "sync 断裂后审计报 broken_hardlink"}, + {"id": "H3", "name": "全部检查通过时 status=ok 且不推 XMPP"} + ], + "related_files": [ + "deploy/profile-scripts/system_hygiene_audit.py", + "deploy/profile-scripts/sync_profile_scripts.sh", + "gateway/logs/hygiene_report.json" + ] + } +}