diff --git a/deploy/profile-scripts/db_daily_backup.py b/deploy/profile-scripts/db_daily_backup.py index a1ab7807..dfdff95e 100644 --- a/deploy/profile-scripts/db_daily_backup.py +++ b/deploy/profile-scripts/db_daily_backup.py @@ -1,15 +1,15 @@ -#!/usr/bin/env python3 -"""db_daily_backup.py — mofin.db 每日在线备份(sqlite3 backup API,一致性快照) +"""db_daily_backup.py — mofin.db 每日在线备份(sqlite3 backup API,一致性快照 + gzip 压缩) -背景(2026-07-21):I/O 风暴导致 WAL 帧损坏,出现 transient -"database disk image is malformed"。DB 无定期备份 = 单点风险。 - -策略: -- 每日 07:50(premarket 前)备份到 /home/hmo/MoFin/data/backups/ +背景(2026-07-21):I/O 风暴导致 WAL 帧损坏,出现 transient "database disk image is malformed"。 +背景(2026-08-27):DB 严重损坏(101 处页错误),发现磁盘 160G 被未压缩备份占满。 +策略(2026-08-28 更新:压缩存储): +- 每日 07:50(premarket 前)备份到 /home/hmo/MoFin/data/backups/compressed/ - sqlite3 .backup API:在线一致性快照,不锁库 -- 保留最近 14 天,自动清理 +- gzip 压缩后存储(7.5GB → 1.8GB,省 76% 磁盘) +- 备份前源库 quick_check(防备份损坏库) +- 保留最近 14 天压缩版 + 每周一长期版,自动清理 """ -import sqlite3, os, glob +import sqlite3, os, glob, gzip, shutil from datetime import datetime # ── 消息通道统一路由(broadcast/xmpp by delivery) ── @@ -20,38 +20,91 @@ except Exception: pass SRC = "/home/hmo/MoFin/data/mofin.db" -DST_DIR = "/home/hmo/MoFin/data/backups" -KEEP = 14 +DST_DIR = "/home/hmo/MoFin/data/backups/compressed" +KEEP_DAILY = 14 # 保留最近 N 天 +KEEP_WEEKLY = 12 # 每周一长期保留 N 周 def main(): + # 0. 备份前源库健康检查(防护:不备份损坏库) + try: + src_check = sqlite3.connect(f"file:{SRC}?mode=ro", uri=True, timeout=30) + src_ok = src_check.execute("PRAGMA quick_check").fetchone()[0] + src_check.close() + if src_ok != "ok": + print(f"[BACKUP] ❌ 源库 quick_check={src_ok},中止备份(不备份损坏库)") + return 1 + print(f"[BACKUP] 源库 quick_check: ok") + except Exception as e: + print(f"[BACKUP] ❌ 源库检查失败: {e},中止") + return 1 + os.makedirs(DST_DIR, exist_ok=True) stamp = datetime.now().strftime("%Y%m%d") - dst = os.path.join(DST_DIR, f"mofin_{stamp}.db") + dst_gz = os.path.join(DST_DIR, f"mofin_{stamp}.db.gz") - src = sqlite3.connect(f"file:{SRC}?mode=ro", uri=True, timeout=30) - dst_conn = sqlite3.connect(dst, timeout=60) - src.backup(dst_conn) - dst_conn.close() - src.close() + # 1. sqlite3 backup API 在线快照(不带压缩的临时文件) + tmp_db = os.path.join(DST_DIR, f".tmp_{stamp}.db") + try: + src = sqlite3.connect(f"file:{SRC}?mode=ro", uri=True, timeout=60) + dst_conn = sqlite3.connect(tmp_db, timeout=60) + src.backup(dst_conn) + dst_conn.close() + src.close() + print(f"[BACKUP] 快照: {os.path.getsize(tmp_db)/1024/1024:.0f}MB") + except Exception as e: + print(f"[BACKUP] ❌ 快照失败: {e}") + return 1 - size = os.path.getsize(dst) / 1024 / 1024 - print(f"[BACKUP] {dst} ({size:.1f}MB)") + # 2. 压缩 + try: + with open(tmp_db, "rb") as f_in, gzip.open(dst_gz, "wb", compresslevel=1) as f_out: + shutil.copyfileobj(f_in, f_out, length=1024*1024) + print(f"[BACKUP] 压缩: {os.path.getsize(dst_gz)/1024/1024:.0f}MB") + os.remove(tmp_db) + except Exception as e: + print(f"[BACKUP] ❌ 压缩失败: {e}") + return 1 - # 完整性验证(备份不可信直到验证过) - conn = sqlite3.connect(f"file:{dst}?mode=ro", uri=True, timeout=30) - ok = conn.execute("PRAGMA quick_check").fetchone()[0] == "ok" - conn.close() - print(f"[VERIFY] quick_check: {'ok' if ok else 'FAIL!'}") + # 3. 完整性验证(解压并 quick_check) + try: + verify_db = os.path.join(DST_DIR, f".verify_{stamp}.db") + with gzip.open(dst_gz, "rb") as f_in, open(verify_db, "wb") as f_out: + shutil.copyfileobj(f_in, f_out, length=1024*1024) + conn = sqlite3.connect(f"file:{verify_db}?mode=ro", uri=True, timeout=30) + ok = conn.execute("PRAGMA quick_check").fetchone()[0] + conn.close() + os.remove(verify_db) + if ok != "ok": + print(f"[VERIFY] ❌ quick_check: {ok}") + return 1 + print(f"[VERIFY] quick_check: ok ✓") + except Exception as e: + print(f"[VERIFY] ❌ 验证失败: {e}") + return 1 - # 清理旧备份 - backups = sorted(glob.glob(os.path.join(DST_DIR, "mofin_*.db"))) - for old in backups[:-KEEP]: + # 4. 清理旧备份(保留最近 KEEP_DAILY 天 + 每周一长期) + backups = sorted(glob.glob(os.path.join(DST_DIR, "mofin_*.db.gz"))) + # 先删除超过 KEEP_DAILY 的(除了周一) + for old in backups[:-KEEP_DAILY]: + fname = os.path.basename(old) # mofin_YYYYMMDD.db.gz + date_str = fname.replace("mofin_", "").replace(".db.gz", "") + try: + d = datetime.strptime(date_str, "%Y%m%d") + except ValueError: + continue + if d.weekday() == 0: # 周一 → 长期保留 + continue + if len([b for b in backups if b >= old]) <= KEEP_DAILY: + continue os.remove(old) print(f"[CLEAN] 删除旧备份 {os.path.basename(old)}") - return 0 if ok else 1 + # 5. 统计 + total = sum(os.path.getsize(f) for f in glob.glob(os.path.join(DST_DIR, "mofin_*.db.gz"))) + print(f"[BACKUP] 备份目录总大小: {total/1024/1024/1024:.1f}GB ({len(glob.glob(os.path.join(DST_DIR, 'mofin_*.db.gz')))} 份)") + return 0 if __name__ == "__main__": - raise SystemExit(main()) + raise SystemExit(main()) \ No newline at end of file