98 lines
3.9 KiB
Python
98 lines
3.9 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
"""backfill_hk_stocks.py — 港股通个股长历史日K回填（周末·阶段4前置）
|
||
|
||
背景：stock_daily 港股个股只有约2年（2024-07起，522条），回测样本不足。
|
||
本脚本对港股通名单（hk_connect_stocks 613只）每只拉 2000 根（腾讯接口单次上限，约8年），
|
||
回填入 stock_daily，供港股策略回测 + 实盘技术分析。
|
||
|
||
数据源：腾讯日K接口 hk 前缀（单次上限2000根，恒指回填已验证）。
|
||
http://ifzq.gtimg.cn/appstock/app/fqkline/get?param=hk{code},day,,,2000,qfq
|
||
|
||
跑法（heavy_run 受控，约10分钟）：
|
||
flock + nice19 + ionice，见 heavy_run.sh 等效约束
|
||
幂等：INSERT OR IGNORE（code,date 唯一键），可重复执行。
|
||
"""
|
||
import json
|
||
import sqlite3
|
||
import sys
|
||
import time
|
||
import urllib.request
|
||
from pathlib import Path
|
||
|
||
DB_PATH = "/home/hmo/MoFin/data/mofin.db"
|
||
UA = "Mozilla/5.0"
|
||
COUNT = 2000 # 腾讯单次上限
|
||
SLEEP = 0.3 # 限速防封（铁律）
|
||
|
||
|
||
def fetch_klines(code):
|
||
"""拉单只港股 2000 根日K（腾讯 hk 前缀）"""
|
||
url = (f"http://ifzq.gtimg.cn/appstock/app/fqkline/get?"
|
||
f"param=hk{code},day,,,{COUNT},qfq")
|
||
req = urllib.request.Request(url, headers={"User-Agent": UA})
|
||
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
||
with opener.open(req, timeout=20) as r:
|
||
text = r.read().decode("utf-8", errors="replace")
|
||
data = json.loads(text)
|
||
node = (data.get("data") or {}).get(f"hk{code}", {})
|
||
return node.get("qfqday") or node.get("day") or []
|
||
|
||
|
||
def main():
|
||
conn = sqlite3.connect(DB_PATH, timeout=30)
|
||
conn.execute("PRAGMA busy_timeout=30000")
|
||
codes = [r[0] for r in conn.execute(
|
||
"SELECT code FROM hk_connect_stocks WHERE is_active=1 ORDER BY code").fetchall()]
|
||
print(f"港股通个股长历史回填：{len(codes)} 只，每只 {COUNT} 根", flush=True)
|
||
|
||
total_new = 0
|
||
ok = empty = fail = 0
|
||
t0 = time.time()
|
||
for idx, code in enumerate(codes):
|
||
try:
|
||
bars = fetch_klines(code)
|
||
if not bars:
|
||
empty += 1
|
||
continue
|
||
new = 0
|
||
for b in bars:
|
||
# 健壮解析：只取前6字段(date,open,close,high,low,volume)，且都须为数字
|
||
# （腾讯港股部分 bar 末尾带 dict 附加字段，会导致 float() 报错）
|
||
try:
|
||
if not isinstance(b, (list, tuple)) or len(b) < 6:
|
||
continue
|
||
date = b[0]
|
||
open_, close, high, low = float(b[1]), float(b[2]), float(b[3]), float(b[4])
|
||
volume = float(b[5]) if b[5] not in (None, "") else 0.0
|
||
except (IndexError, ValueError, TypeError):
|
||
continue
|
||
cur = conn.execute(
|
||
"INSERT OR IGNORE INTO stock_daily (code, date, open, close, high, low, volume) "
|
||
"VALUES (?,?,?,?,?,?,?)",
|
||
(code, date, open_, close, high, low, volume))
|
||
new += cur.rowcount
|
||
total_new += new
|
||
ok += 1
|
||
if (idx + 1) % 50 == 0:
|
||
conn.commit()
|
||
print(f" [{idx+1}/{len(codes)}] ok={ok} empty={empty} fail={fail} 新增{total_new}根 | {time.time()-t0:.0f}s", flush=True)
|
||
except Exception as e:
|
||
fail += 1
|
||
if fail <= 5:
|
||
print(f" FAIL {code}: {str(e)[:60]}", flush=True)
|
||
time.sleep(SLEEP)
|
||
conn.commit()
|
||
|
||
# 验证
|
||
stat = conn.execute(
|
||
"SELECT COUNT(DISTINCT code), MIN(date), MAX(date) FROM stock_daily WHERE length(code)=5").fetchone()
|
||
conn.close()
|
||
print(f"\n完成：ok={ok} empty={empty} fail={fail} 新增{total_new}根 耗时{time.time()-t0:.0f}s", flush=True)
|
||
print(f"stock_daily 港股: {stat[0]} 只, {stat[1]} ~ {stat[2]}", flush=True)
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|