hmo
4f83ee8a01
Root cause analysis of the 2026-07-20 redundancy incident: 1. No single-source-of-truth rule -> same file legitimately lived in 4+ locations, diverging silently 2. Relative path resolution (Path(__file__).parent/'data') -> each hardlinked copy of mofin_db.py pointed to a DIFFERENT database 3. 'Backup habit' left .bak/legacy files in production dirs, which monitoring then scanned and reported as false alarms 4. Half-done migrations: DB tables created but old JSON writers/readers stayed (price_events), old files stayed 5. Dead modules never got buried: xiaoguo 'dead' but bot ran 8 days as root eating 2.5GB 6. Monitoring checked 'does it exist' not 'is it alive' -> stale file mtime reported as 'pipeline stalled 14 days' (false alarm) 7. No 'system hygiene' as a check category at all Prevention implemented: - dev-spec.md v2.0: 五条红线 -> 十条红线 #6 single source of truth (hardlink only, no independent copies) #7 absolute data paths only (no __file__-relative data resolution) #8 no backups/legacy in production data dirs (archive immediately) #9 dead module burial checklist (6 mandatory steps) #10 monitor liveness (DB table freshness) not existence - File Location Constitution: canonical location per content type - NEW system_hygiene_audit.py: weekly Monday 07:30 cron checking diverged copies / broken hardlinks / zombie processes / orphan data files / dead cron scripts / DB freshness -> hygiene_report.json + XMPP - specs/hygiene.json: module spec per red line #1 - Verified: audit found 5 real issues on first run, all fixed, re-run clean
94 lines
3.8 KiB
Python
94 lines
3.8 KiB
Python
import sqlite3, os, shutil
|
|
from datetime import datetime
|
|
|
|
THIRD = '/home/hmo/.hermes/profiles/position-analyst/scripts/data/mofin.db'
|
|
MAIN = '/home/hmo/MoFin/data/mofin.db'
|
|
BACKUP = f'/home/hmo/MoFin/archive/third-db-backup-{datetime.now().strftime("%Y%m%d-%H%M")}'
|
|
os.makedirs(BACKUP, exist_ok=True)
|
|
shutil.copy(THIRD, os.path.join(BACKUP, 'mofin.db'))
|
|
print('backup:', BACKUP)
|
|
|
|
t = sqlite3.connect(THIRD, timeout=10)
|
|
m = sqlite3.connect(MAIN, timeout=30)
|
|
m.execute('PRAGMA busy_timeout=30000')
|
|
|
|
# 1. sector_snapshots: dedup by (snapshot_id, name)
|
|
existing = set()
|
|
for r in m.execute("SELECT snapshot_id, name FROM sector_snapshots"):
|
|
existing.add((r[0], r[1]))
|
|
print(f'main sector_snapshots keys: {len(existing)}')
|
|
|
|
cols = ['snapshot_id', 'name', 'change_pct', 'up_count', 'down_count', 'net_inflow',
|
|
'lead_stock', 'lead_stock_change', 'volume', 'turnover']
|
|
ins = 0
|
|
skip = 0
|
|
for r in t.execute(f"SELECT {', '.join(cols)} FROM sector_snapshots"):
|
|
if (r[0], r[1]) in existing:
|
|
skip += 1
|
|
continue
|
|
m.execute(f"INSERT INTO sector_snapshots ({', '.join(cols)}) VALUES ({','.join('?'*len(cols))})", r)
|
|
existing.add((r[0], r[1]))
|
|
ins += 1
|
|
m.commit()
|
|
print(f'sector_snapshots: inserted {ins}, skipped {skip}')
|
|
|
|
# 2. market_snapshots: dedup by (timestamp, source)
|
|
existing2 = set()
|
|
for r in m.execute("SELECT timestamp, source FROM market_snapshots"):
|
|
existing2.add((r[0], r[1]))
|
|
cols2 = ['timestamp', 'source', 'up_ratio', 'mood', 'created_at']
|
|
ins2 = 0
|
|
skip2 = 0
|
|
for r in t.execute(f"SELECT {', '.join(cols2)} FROM market_snapshots"):
|
|
if (r[0], r[1]) in existing2:
|
|
skip2 += 1
|
|
continue
|
|
m.execute(f"INSERT INTO market_snapshots ({', '.join(cols2)}) VALUES ({','.join('?'*len(cols2))})", r)
|
|
existing2.add((r[0], r[1]))
|
|
ins2 += 1
|
|
m.commit()
|
|
print(f'market_snapshots: inserted {ins2}, skipped {skip2}')
|
|
|
|
# 3. todos: third has 4 rows, main has 94 — check by title, insert missing (map to main schema)
|
|
todos_t = t.execute("SELECT title, description, status, priority, source, fix_action, retry_count, note, created_at, updated_at FROM todos").fetchall()
|
|
ins3 = 0
|
|
for r in todos_t:
|
|
ex = m.execute("SELECT id FROM todos WHERE title=?", (r[0],)).fetchone()
|
|
if ex:
|
|
continue
|
|
m.execute("INSERT INTO todos (title, description, status, priority, source, fix_action, retry_count, note, created_at, updated_at) VALUES (?,?,?,?,?,?,?,?,?,?)", r)
|
|
ins3 += 1
|
|
m.commit()
|
|
print(f'todos: inserted {ins3}, skipped {len(todos_t)-ins3}')
|
|
|
|
# 4. capital_flow_cache: keep whichever is newer
|
|
t_row = t.execute("SELECT cache_json, updated_at FROM capital_flow_cache ORDER BY id DESC LIMIT 1").fetchone()
|
|
m_row = m.execute("SELECT cache_json, updated_at FROM capital_flow_cache ORDER BY id DESC LIMIT 1").fetchone()
|
|
if t_row and (not m_row or (t_row[1] or '') > (m_row[1] or '')):
|
|
m.execute("DELETE FROM capital_flow_cache")
|
|
m.execute("INSERT INTO capital_flow_cache (cache_json, updated_at) VALUES (?,?)", t_row)
|
|
m.commit()
|
|
print('capital_flow_cache: replaced with third (newer)')
|
|
else:
|
|
print('capital_flow_cache: main is newer/equal, kept')
|
|
|
|
# verify
|
|
total = m.execute("SELECT COUNT(*) FROM sector_snapshots").fetchone()[0]
|
|
total2 = m.execute("SELECT COUNT(*) FROM market_snapshots").fetchone()[0]
|
|
print(f'after merge: sector_snapshots={total}, market_snapshots={total2}')
|
|
|
|
m.close()
|
|
t.close()
|
|
|
|
# 5. 删除第三库（连同 data 目录里的其他残留）
|
|
third_dir = os.path.dirname(THIRD)
|
|
trash = '/home/hmo/trashbox/third-db-retired-20260720'
|
|
os.makedirs(trash, exist_ok=True)
|
|
shutil.move(THIRD, os.path.join(trash, 'mofin.db'))
|
|
print('third db moved to', trash)
|
|
for f in os.listdir(third_dir):
|
|
src = os.path.join(third_dir, f)
|
|
if os.path.isfile(src):
|
|
shutil.move(src, os.path.join(trash, f))
|
|
print(' also moved:', f)
|
|
print('DONE') |