#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""backfill_hk_stocks.py — 港股通个股长历史日K回填（周末·阶段4前置）

背景：stock_daily 港股个股只有约2年（2024-07起，522条），回测样本不足。
本脚本对港股通名单（hk_connect_stocks 613只）每只拉 2000 根（腾讯接口单次上限，约8年），
回填入 stock_daily，供港股策略回测 + 实盘技术分析。

数据源：腾讯日K接口 hk 前缀（单次上限2000根，恒指回填已验证）。
  http://ifzq.gtimg.cn/appstock/app/fqkline/get?param=hk{code},day,,,2000,qfq

跑法（heavy_run 受控，约10分钟）：
  flock + nice19 + ionice，见 heavy_run.sh 等效约束
幂等：INSERT OR IGNORE（code,date 唯一键），可重复执行。
"""
import json
import sqlite3
import sys
import time
import urllib.request
from pathlib import Path

DB_PATH = "/home/hmo/MoFin/data/mofin.db"
UA = "Mozilla/5.0"
COUNT = 2000       # 腾讯单次上限
SLEEP = 0.3        # 限速防封（铁律）


def fetch_klines(code):
    """拉单只港股 2000 根日K（腾讯 hk 前缀）"""
    url = (f"http://ifzq.gtimg.cn/appstock/app/fqkline/get?"
           f"param=hk{code},day,,,{COUNT},qfq")
    req = urllib.request.Request(url, headers={"User-Agent": UA})
    opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
    with opener.open(req, timeout=20) as r:
        text = r.read().decode("utf-8", errors="replace")
    data = json.loads(text)
    node = (data.get("data") or {}).get(f"hk{code}", {})
    return node.get("qfqday") or node.get("day") or []


def main():
    conn = sqlite3.connect(DB_PATH, timeout=30)
    conn.execute("PRAGMA busy_timeout=30000")
    codes = [r[0] for r in conn.execute(
        "SELECT code FROM hk_connect_stocks WHERE is_active=1 ORDER BY code").fetchall()]
    print(f"港股通个股长历史回填：{len(codes)} 只，每只 {COUNT} 根", flush=True)

    total_new = 0
    ok = empty = fail = 0
    t0 = time.time()
    for idx, code in enumerate(codes):
        try:
            bars = fetch_klines(code)
            if not bars:
                empty += 1
                continue
            new = 0
            for b in bars:
                # 健壮解析：只取前6字段(date,open,close,high,low,volume)，且都须为数字
                # （腾讯港股部分 bar 末尾带 dict 附加字段，会导致 float() 报错）
                try:
                    if not isinstance(b, (list, tuple)) or len(b) < 6:
                        continue
                    date = b[0]
                    open_, close, high, low = float(b[1]), float(b[2]), float(b[3]), float(b[4])
                    volume = float(b[5]) if b[5] not in (None, "") else 0.0
                except (IndexError, ValueError, TypeError):
                    continue
                cur = conn.execute(
                    "INSERT OR IGNORE INTO stock_daily (code, date, open, close, high, low, volume) "
                    "VALUES (?,?,?,?,?,?,?)",
                    (code, date, open_, close, high, low, volume))
                new += cur.rowcount
            total_new += new
            ok += 1
            if (idx + 1) % 50 == 0:
                conn.commit()
                print(f"  [{idx+1}/{len(codes)}] ok={ok} empty={empty} fail={fail} 新增{total_new}根 | {time.time()-t0:.0f}s", flush=True)
        except Exception as e:
            fail += 1
            if fail <= 5:
                print(f"  FAIL {code}: {str(e)[:60]}", flush=True)
        time.sleep(SLEEP)
    conn.commit()

    # 验证
    stat = conn.execute(
        "SELECT COUNT(DISTINCT code), MIN(date), MAX(date) FROM stock_daily WHERE length(code)=5").fetchone()
    conn.close()
    print(f"\n完成：ok={ok} empty={empty} fail={fail} 新增{total_new}根 耗时{time.time()-t0:.0f}s", flush=True)
    print(f"stock_daily 港股: {stat[0]} 只, {stat[1]} ~ {stat[2]}", flush=True)
    return 0


if __name__ == "__main__":
    sys.exit(main())