feat: 港股通个股长历史回填脚本(每只2000根约8年,腾讯hk前缀,INSERT OR IGNORE幂等)

2026-08-14 21:03:31 +08:00
parent 95202ab4ca
commit ddd2e12b38
1 changed files with 93 additions and 0 deletions
@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""backfill_hk_stocks.py — 港股通个股长历史日K回填（周末·阶段4前置）
+
+背景：stock_daily 港股个股只有约2年（2024-07起，522条），回测样本不足。
+本脚本对港股通名单（hk_connect_stocks 613只）每只拉 2000 根（腾讯接口单次上限，约8年），
+回填入 stock_daily，供港股策略回测 + 实盘技术分析。
+
+数据源：腾讯日K接口 hk 前缀（单次上限2000根，恒指回填已验证）。
+  http://ifzq.gtimg.cn/appstock/app/fqkline/get?param=hk{code},day,,,2000,qfq
+
+跑法（heavy_run 受控，约10分钟）：
+  flock + nice19 + ionice，见 heavy_run.sh 等效约束
+幂等：INSERT OR IGNORE（code,date 唯一键），可重复执行。
+"""
+import json
+import sqlite3
+import sys
+import time
+import urllib.request
+from pathlib import Path
+
+DB_PATH = "/home/hmo/MoFin/data/mofin.db"
+UA = "Mozilla/5.0"
+COUNT = 2000       # 腾讯单次上限
+SLEEP = 0.3        # 限速防封（铁律）
+
+
+def fetch_klines(code):
+    """拉单只港股 2000 根日K（腾讯 hk 前缀）"""
+    url = (f"http://ifzq.gtimg.cn/appstock/app/fqkline/get?"
+           f"param=hk{code},day,,,{COUNT},qfq")
+    req = urllib.request.Request(url, headers={"User-Agent": UA})
+    opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
+    with opener.open(req, timeout=20) as r:
+        text = r.read().decode("utf-8", errors="replace")
+    data = json.loads(text)
+    node = (data.get("data") or {}).get(f"hk{code}", {})
+    return node.get("qfqday") or node.get("day") or []
+
+
+def main():
+    conn = sqlite3.connect(DB_PATH, timeout=30)
+    conn.execute("PRAGMA busy_timeout=30000")
+    codes = [r[0] for r in conn.execute(
+        "SELECT code FROM hk_connect_stocks WHERE is_active=1 ORDER BY code").fetchall()]
+    print(f"港股通个股长历史回填：{len(codes)} 只，每只 {COUNT} 根", flush=True)
+
+    total_new = 0
+    ok = empty = fail = 0
+    t0 = time.time()
+    for idx, code in enumerate(codes):
+        try:
+            bars = fetch_klines(code)
+            if not bars:
+                empty += 1
+                continue
+            new = 0
+            for b in bars:
+                try:
+                    date, open_, close, high, low = b[0], b[1], b[2], b[3], b[4]
+                    volume = float(b[5]) if len(b) > 5 and b[5] not in (None, "") else 0
+                    amount = float(b[6]) if len(b) > 6 and b[6] not in (None, "") else 0
+                except (IndexError, ValueError):
+                    continue
+                cur = conn.execute(
+                    "INSERT OR IGNORE INTO stock_daily (code, date, open, close, high, low, volume, amount) "
+                    "VALUES (?,?,?,?,?,?,?,?)",
+                    (code, date, float(open_), float(close), float(high), float(low), volume, amount))
+                new += cur.rowcount
+            total_new += new
+            ok += 1
+            if (idx + 1) % 50 == 0:
+                conn.commit()
+                print(f"  [{idx+1}/{len(codes)}] ok={ok} empty={empty} fail={fail} 新增{total_new}根 | {time.time()-t0:.0f}s", flush=True)
+        except Exception as e:
+            fail += 1
+            if fail <= 5:
+                print(f"  FAIL {code}: {str(e)[:60]}", flush=True)
+        time.sleep(SLEEP)
+    conn.commit()
+
+    # 验证
+    stat = conn.execute(
+        "SELECT COUNT(DISTINCT code), MIN(date), MAX(date) FROM stock_daily WHERE length(code)=5").fetchone()
+    conn.close()
+    print(f"\n完成：ok={ok} empty={empty} fail={fail} 新增{total_new}根 耗时{time.time()-t0:.0f}s", flush=True)
+    print(f"stock_daily 港股: {stat[0]} 只, {stat[1]} ~ {stat[2]}", flush=True)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())