91 lines
3.3 KiB
Python
91 lines
3.3 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
"""backfill_hk_index.py — 恒生指数(hkHSI)历史日K回填（一次性脚本）
|
||
|
||
背景（阶段3 港股接入）：stock_daily 的 hkHSI 仅 2024 起（643 条），温区回填
|
||
需要更长历史（对齐 A 股约 10 年）。本脚本从腾讯日K接口拉取 hkHSI 全量历史
|
||
（约 3000 根 ≈ 12 年），写入 stock_daily。
|
||
|
||
数据源（与 market_data.fetch_tx_klines 同一腾讯接口，hkHSI 已验证返回 day 键）：
|
||
http://ifzq.gtimg.cn/appstock/app/fqkline/get?param=hkHSI,day,,,{count},qfq
|
||
|
||
用法：
|
||
python3 backfill_hk_index.py # 拉 3000 根（约12年）
|
||
python3 backfill_hk_index.py --count 4000
|
||
|
||
写入：INSERT OR IGNORE（(code,date) 唯一键防重复，已有日期跳过）。
|
||
"""
|
||
import json
|
||
import sqlite3
|
||
import sys
|
||
import urllib.request
|
||
from pathlib import Path
|
||
|
||
_SCRIPT_DIR = Path(__file__).resolve().parent
|
||
_MOFIN_ROOT = _SCRIPT_DIR.parent.parent
|
||
DB_PATH = Path(_MOFIN_ROOT) / "data" / "mofin.db"
|
||
|
||
UA = "Mozilla/5.0"
|
||
INDEX_CODE = "hkHSI"
|
||
|
||
|
||
def fetch_hsi(count=3000):
|
||
"""从腾讯接口拉 hkHSI 日K（count 根）"""
|
||
url = (f"http://ifzq.gtimg.cn/appstock/app/fqkline/get?"
|
||
f"param={INDEX_CODE},day,,,{count},qfq")
|
||
req = urllib.request.Request(url, headers={"User-Agent": UA})
|
||
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
||
with opener.open(req, timeout=30) as r:
|
||
text = r.read().decode("utf-8", errors="replace")
|
||
data = json.loads(text)
|
||
node = data.get("data", {}).get(INDEX_CODE, {})
|
||
bars = node.get("qfqday") or node.get("day") or []
|
||
return bars
|
||
|
||
|
||
def main():
|
||
count = 3000
|
||
for a in sys.argv[1:]:
|
||
if a.startswith("--count"):
|
||
count = int(a.split("=")[-1] if "=" in a else sys.argv[sys.argv.index(a) + 1])
|
||
|
||
print(f"拉取 {INDEX_CODE} 最近 {count} 根日K...")
|
||
bars = fetch_hsi(count)
|
||
if not bars:
|
||
print("拉取失败/为空，终止")
|
||
return 1
|
||
print(f"拉到 {len(bars)} 根，时间范围 {bars[0][0]} ~ {bars[-1][0]}")
|
||
|
||
conn = sqlite3.connect(str(DB_PATH), timeout=30)
|
||
conn.execute("PRAGMA busy_timeout=30000")
|
||
# stock_daily 列：code, date, open, close, high, low, volume（与 daily_kline_collector 一致）
|
||
inserted = 0
|
||
skipped = 0
|
||
for b in bars:
|
||
# 格式: [date, open, close, high, low, volume, ...]
|
||
try:
|
||
date, open_, close, high, low = b[0], b[1], b[2], b[3], b[4]
|
||
volume = float(b[5]) if len(b) > 5 and b[5] not in (None, "") else 0
|
||
except (IndexError, ValueError):
|
||
skipped += 1
|
||
continue
|
||
cur = conn.execute(
|
||
"INSERT OR IGNORE INTO stock_daily (code, date, open, close, high, low, volume) "
|
||
"VALUES (?,?,?,?,?,?,?)",
|
||
(INDEX_CODE, date, float(open_), float(close), float(high), float(low), volume)
|
||
)
|
||
inserted += cur.rowcount
|
||
conn.commit()
|
||
# 验证
|
||
total = conn.execute(
|
||
"SELECT COUNT(*), MIN(date), MAX(date) FROM stock_daily WHERE code=?", (INDEX_CODE,)
|
||
).fetchone()
|
||
conn.close()
|
||
print(f"写入 {inserted} 根（跳过 {skipped} 根格式异常 / 已有日期自动 IGNORE）")
|
||
print(f"stock_daily 的 {INDEX_CODE}: 共 {total[0]} 条，{total[1]} ~ {total[2]}")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|