118 lines
4.4 KiB
Python
118 lines
4.4 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
"""backfill_hk_index.py — 恒生指数(hkHSI)历史日K回填（一次性脚本）
|
||
|
||
背景（阶段3 港股接入）：stock_daily 的 hkHSI 仅 2024 起（643 条），温区回填
|
||
需要更长历史（对齐 A 股约 10 年）。本脚本从腾讯日K接口拉取 hkHSI 全量历史
|
||
（约 3000 根 ≈ 12 年），写入 stock_daily。
|
||
|
||
数据源（与 market_data.fetch_tx_klines 同一腾讯接口，hkHSI 已验证返回 day 键）：
|
||
http://ifzq.gtimg.cn/appstock/app/fqkline/get?param=hkHSI,day,,,{count},qfq
|
||
|
||
用法：
|
||
python3 backfill_hk_index.py # 拉 3000 根（约12年）
|
||
python3 backfill_hk_index.py --count 4000
|
||
|
||
写入：INSERT OR IGNORE（(code,date) 唯一键防重复，已有日期跳过）。
|
||
"""
|
||
import json
|
||
import sqlite3
|
||
import sys
|
||
import urllib.request
|
||
from pathlib import Path
|
||
|
||
_SCRIPT_DIR = Path(__file__).resolve().parent
|
||
_MOFIN_ROOT = _SCRIPT_DIR.parent.parent
|
||
DB_PATH = Path(_MOFIN_ROOT) / "data" / "mofin.db"
|
||
|
||
UA = "Mozilla/5.0"
|
||
INDEX_CODE = "hkHSI"
|
||
PAGE_SIZE = 2000 # 腾讯接口单次上限（实测 count=2000 ok, 2500 报 param error）
|
||
|
||
|
||
def _fetch_page(count, end_date=""):
|
||
"""拉一页：param=hkHSI,day,start,end,count,qfq。end_date 空=截至最新"""
|
||
url = (f"https://web.ifzq.gtimg.cn/appstock/app/fqkline/get?"
|
||
f"param={INDEX_CODE},day,,{end_date},{count},qfq")
|
||
req = urllib.request.Request(url, headers={"User-Agent": UA})
|
||
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
||
with opener.open(req, timeout=30) as r:
|
||
text = r.read().decode("utf-8", errors="replace")
|
||
data = json.loads(text)
|
||
node = (data.get("data") or {}).get(INDEX_CODE, {})
|
||
return node.get("qfqday") or node.get("day") or []
|
||
|
||
|
||
def fetch_hsi(count=3000):
|
||
"""分页拉取 hkHSI 日K（单次上限 2000，超过则向前翻页）"""
|
||
from datetime import datetime as _dt, timedelta as _td
|
||
import time as _t
|
||
uniq = {}
|
||
end_date = ""
|
||
while len(uniq) < count:
|
||
want = min(PAGE_SIZE, count - len(uniq) + 5)
|
||
page = _fetch_page(want, end_date)
|
||
if not page:
|
||
break
|
||
new_cnt = 0
|
||
for b in page:
|
||
if b[0] not in uniq:
|
||
uniq[b[0]] = b
|
||
new_cnt += 1
|
||
if new_cnt == 0:
|
||
break # 没有新数据，历史到头
|
||
# 向前翻页：以本页最早日期的前一天为 end
|
||
earliest = page[0][0]
|
||
end_date = (_dt.strptime(earliest, "%Y-%m-%d") - _td(days=1)).strftime("%Y-%m-%d")
|
||
if len(page) < 100: # 不足一页说明历史拉完
|
||
break
|
||
_t.sleep(1) # 翻页间隔，遵守访问频率
|
||
return [uniq[d] for d in sorted(uniq)]
|
||
|
||
|
||
def main():
|
||
count = 3000
|
||
for a in sys.argv[1:]:
|
||
if a.startswith("--count"):
|
||
count = int(a.split("=")[-1] if "=" in a else sys.argv[sys.argv.index(a) + 1])
|
||
|
||
print(f"拉取 {INDEX_CODE} 最近 {count} 根日K...")
|
||
bars = fetch_hsi(count)
|
||
if not bars:
|
||
print("拉取失败/为空，终止")
|
||
return 1
|
||
print(f"拉到 {len(bars)} 根，时间范围 {bars[0][0]} ~ {bars[-1][0]}")
|
||
|
||
conn = sqlite3.connect(str(DB_PATH), timeout=30)
|
||
conn.execute("PRAGMA busy_timeout=30000")
|
||
# stock_daily 列：code, date, open, close, high, low, volume（与 daily_kline_collector 一致）
|
||
inserted = 0
|
||
skipped = 0
|
||
for b in bars:
|
||
# 格式: [date, open, close, high, low, volume, ...]
|
||
try:
|
||
date, open_, close, high, low = b[0], b[1], b[2], b[3], b[4]
|
||
volume = float(b[5]) if len(b) > 5 and b[5] not in (None, "") else 0
|
||
except (IndexError, ValueError):
|
||
skipped += 1
|
||
continue
|
||
cur = conn.execute(
|
||
"INSERT OR IGNORE INTO stock_daily (code, date, open, close, high, low, volume) "
|
||
"VALUES (?,?,?,?,?,?,?)",
|
||
(INDEX_CODE, date, float(open_), float(close), float(high), float(low), volume)
|
||
)
|
||
inserted += cur.rowcount
|
||
conn.commit()
|
||
# 验证
|
||
total = conn.execute(
|
||
"SELECT COUNT(*), MIN(date), MAX(date) FROM stock_daily WHERE code=?", (INDEX_CODE,)
|
||
).fetchone()
|
||
conn.close()
|
||
print(f"写入 {inserted} 根（跳过 {skipped} 根格式异常 / 已有日期自动 IGNORE）")
|
||
print(f"stock_daily 的 {INDEX_CODE}: 共 {total[0]} 条，{total[1]} ~ {total[2]}")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|