106 lines
4.4 KiB
Python
106 lines
4.4 KiB
Python
"""db_daily_backup.py — mofin.db 每日在线备份（sqlite3 backup API，一致性快照 + gzip 压缩）
|
||
|
||
背景（2026-07-21）：I/O 风暴导致 WAL 帧损坏，出现 transient "database disk image is malformed"。
|
||
背景（2026-08-27）：DB 严重损坏（101 处页错误），发现磁盘 160G 被未压缩备份占满。
|
||
策略（2026-08-28 更新：压缩存储）：
|
||
- 每日 07:50（premarket 前）备份到 /home/hmo/MoFin/data/backups/compressed/
|
||
- sqlite3 .backup API：在线一致性快照，不锁库
|
||
- gzip 压缩后存储（7.5GB → 1.8GB，省 76% 磁盘）
|
||
- 备份前源库 quick_check（防备份损坏库）
|
||
- 保留最近 14 天压缩版 + 每周一长期版，自动清理
|
||
"""
|
||
import sqlite3, os, glob, gzip, shutil
|
||
from datetime import datetime
|
||
|
||
# ── 消息通道统一路由(broadcast/xmpp by delivery) ──
|
||
# 注：备份脚本不用 messenger hook（stdout 会被块化缓冲，导致备份进度卡住不输出）
|
||
|
||
SRC = "/home/hmo/MoFin/data/mofin.db"
|
||
DST_DIR = "/home/hmo/MoFin/data/backups/compressed"
|
||
KEEP_DAILY = 14 # 保留最近 N 天
|
||
KEEP_WEEKLY = 12 # 每周一长期保留 N 周
|
||
|
||
|
||
def main():
|
||
# 0. 备份前源库健康检查（防护：不备份损坏库）
|
||
try:
|
||
src_check = sqlite3.connect(f"file:{SRC}?mode=ro", uri=True, timeout=30)
|
||
src_ok = src_check.execute("PRAGMA quick_check").fetchone()[0]
|
||
src_check.close()
|
||
if src_ok != "ok":
|
||
print(f"[BACKUP] ❌ 源库 quick_check={src_ok}，中止备份（不备份损坏库）")
|
||
return 1
|
||
print(f"[BACKUP] 源库 quick_check: ok")
|
||
except Exception as e:
|
||
print(f"[BACKUP] ❌ 源库检查失败: {e}，中止")
|
||
return 1
|
||
|
||
os.makedirs(DST_DIR, exist_ok=True)
|
||
stamp = datetime.now().strftime("%Y%m%d")
|
||
dst_gz = os.path.join(DST_DIR, f"mofin_{stamp}.db.gz")
|
||
|
||
# 1. sqlite3 backup API 在线快照（不带压缩的临时文件）
|
||
tmp_db = os.path.join(DST_DIR, f".tmp_{stamp}.db")
|
||
try:
|
||
src = sqlite3.connect(f"file:{SRC}?mode=ro", uri=True, timeout=60)
|
||
dst_conn = sqlite3.connect(tmp_db, timeout=60)
|
||
src.backup(dst_conn)
|
||
dst_conn.close()
|
||
src.close()
|
||
print(f"[BACKUP] 快照: {os.path.getsize(tmp_db)/1024/1024:.0f}MB")
|
||
except Exception as e:
|
||
print(f"[BACKUP] ❌ 快照失败: {e}")
|
||
return 1
|
||
|
||
# 2. 压缩
|
||
try:
|
||
with open(tmp_db, "rb") as f_in, gzip.open(dst_gz, "wb", compresslevel=1) as f_out:
|
||
shutil.copyfileobj(f_in, f_out, length=1024*1024)
|
||
print(f"[BACKUP] 压缩: {os.path.getsize(dst_gz)/1024/1024:.0f}MB")
|
||
os.remove(tmp_db)
|
||
except Exception as e:
|
||
print(f"[BACKUP] ❌ 压缩失败: {e}")
|
||
return 1
|
||
|
||
# 3. 完整性验证（解压并 quick_check）
|
||
try:
|
||
verify_db = os.path.join(DST_DIR, f".verify_{stamp}.db")
|
||
with gzip.open(dst_gz, "rb") as f_in, open(verify_db, "wb") as f_out:
|
||
shutil.copyfileobj(f_in, f_out, length=1024*1024)
|
||
conn = sqlite3.connect(f"file:{verify_db}?mode=ro", uri=True, timeout=30)
|
||
ok = conn.execute("PRAGMA quick_check").fetchone()[0]
|
||
conn.close()
|
||
os.remove(verify_db)
|
||
if ok != "ok":
|
||
print(f"[VERIFY] ❌ quick_check: {ok}")
|
||
return 1
|
||
print(f"[VERIFY] quick_check: ok ✓")
|
||
except Exception as e:
|
||
print(f"[VERIFY] ❌ 验证失败: {e}")
|
||
return 1
|
||
|
||
# 4. 清理旧备份（保留最近 KEEP_DAILY 天 + 每周一长期）
|
||
backups = sorted(glob.glob(os.path.join(DST_DIR, "mofin_*.db.gz")))
|
||
# 先删除超过 KEEP_DAILY 的（除了周一）
|
||
for old in backups[:-KEEP_DAILY]:
|
||
fname = os.path.basename(old) # mofin_YYYYMMDD.db.gz
|
||
date_str = fname.replace("mofin_", "").replace(".db.gz", "")
|
||
try:
|
||
d = datetime.strptime(date_str, "%Y%m%d")
|
||
except ValueError:
|
||
continue
|
||
if d.weekday() == 0: # 周一 → 长期保留
|
||
continue
|
||
if len([b for b in backups if b >= old]) <= KEEP_DAILY:
|
||
continue
|
||
os.remove(old)
|
||
print(f"[CLEAN] 删除旧备份 {os.path.basename(old)}")
|
||
|
||
# 5. 统计
|
||
total = sum(os.path.getsize(f) for f in glob.glob(os.path.join(DST_DIR, "mofin_*.db.gz")))
|
||
print(f"[BACKUP] 备份目录总大小: {total/1024/1024/1024:.1f}GB ({len(glob.glob(os.path.join(DST_DIR, 'mofin_*.db.gz')))} 份)")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main()) |