diff --git a/deploy/profile-scripts/backfill_hk_index.py b/deploy/profile-scripts/backfill_hk_index.py index 22ff6611..666eb908 100644 --- a/deploy/profile-scripts/backfill_hk_index.py +++ b/deploy/profile-scripts/backfill_hk_index.py @@ -27,20 +27,47 @@ DB_PATH = Path(_MOFIN_ROOT) / "data" / "mofin.db" UA = "Mozilla/5.0" INDEX_CODE = "hkHSI" +PAGE_SIZE = 2000 # 腾讯接口单次上限(实测 count=2000 ok, 2500 报 param error) -def fetch_hsi(count=3000): - """从腾讯接口拉 hkHSI 日K(count 根)""" - url = (f"http://ifzq.gtimg.cn/appstock/app/fqkline/get?" - f"param={INDEX_CODE},day,,,{count},qfq") +def _fetch_page(count, end_date=""): + """拉一页:param=hkHSI,day,start,end,count,qfq。end_date 空=截至最新""" + url = (f"https://web.ifzq.gtimg.cn/appstock/app/fqkline/get?" + f"param={INDEX_CODE},day,,{end_date},{count},qfq") req = urllib.request.Request(url, headers={"User-Agent": UA}) opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) with opener.open(req, timeout=30) as r: text = r.read().decode("utf-8", errors="replace") data = json.loads(text) - node = data.get("data", {}).get(INDEX_CODE, {}) - bars = node.get("qfqday") or node.get("day") or [] - return bars + node = (data.get("data") or {}).get(INDEX_CODE, {}) + return node.get("qfqday") or node.get("day") or [] + + +def fetch_hsi(count=3000): + """分页拉取 hkHSI 日K(单次上限 2000,超过则向前翻页)""" + from datetime import datetime as _dt, timedelta as _td + import time as _t + uniq = {} + end_date = "" + while len(uniq) < count: + want = min(PAGE_SIZE, count - len(uniq) + 5) + page = _fetch_page(want, end_date) + if not page: + break + new_cnt = 0 + for b in page: + if b[0] not in uniq: + uniq[b[0]] = b + new_cnt += 1 + if new_cnt == 0: + break # 没有新数据,历史到头 + # 向前翻页:以本页最早日期的前一天为 end + earliest = page[0][0] + end_date = (_dt.strptime(earliest, "%Y-%m-%d") - _td(days=1)).strftime("%Y-%m-%d") + if len(page) < 100: # 不足一页说明历史拉完 + break + _t.sleep(1) # 翻页间隔,遵守访问频率 + return [uniq[d] for d in sorted(uniq)] def main():