feat: 港股数据补采1——hk_sector_collector行业归属采集(东财公司档案→stock_sectors复用表,source=hk_em)

2026-08-14 21:43:23 +08:00
parent 8c6e41b619
commit ad5208c6cd
1 changed files with 73 additions and 0 deletions
@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""hk_sector_collector.py — 港股通行业归属采集（阶段2数据补采）
+
+数据源：akshare stock_hk_company_profile_em（东财港股公司档案，含"所属行业"）。
+写入：stock_sectors 表（code+sector_name 主键，source='hk_em'，与A股同表复用）。
+调度：一次性补采 + 之后随名单变动增量（周级）。
+限速：0.4s/只（铁律），620只约5分钟。
+"""
+import sqlite3
+import sys
+import time
+import warnings
+warnings.filterwarnings("ignore")
+
+DB_PATH = "/home/hmo/MoFin/data/mofin.db"
+SLEEP = 0.4
+
+
+def fetch_industry(code):
+    """东财港股公司档案 → 所属行业"""
+    import akshare as ak
+    try:
+        df = ak.stock_hk_company_profile_em(symbol=code)
+        if df is None or df.empty:
+            return None
+        row = df.iloc[0]
+        ind = row.get("所属行业")
+        return str(ind).strip() if ind and str(ind) != "nan" else None
+    except Exception:
+        return None
+
+
+def main():
+    conn = sqlite3.connect(DB_PATH, timeout=30)
+    conn.execute("PRAGMA busy_timeout=30000")
+    codes = [r[0] for r in conn.execute(
+        "SELECT code FROM hk_connect_stocks WHERE is_active=1 ORDER BY code").fetchall()]
+    print(f"港股通行业归属采集：{len(codes)} 只", flush=True)
+
+    ok = empty = fail = 0
+    t0 = time.time()
+    for idx, code in enumerate(codes):
+        ind = fetch_industry(code)
+        if ind:
+            conn.execute(
+                "INSERT OR REPLACE INTO stock_sectors (code, sector_name, source, updated_at) "
+                "VALUES (?,?, 'hk_em', datetime('now','localtime'))",
+                (code, ind))
+            ok += 1
+        elif ind is None:
+            empty += 1
+        if (idx + 1) % 100 == 0:
+            conn.commit()
+            print(f"  [{idx+1}/{len(codes)}] ok={ok} empty={empty} | {time.time()-t0:.0f}s", flush=True)
+        time.sleep(SLEEP)
+    conn.commit()
+
+    n = conn.execute("SELECT COUNT(*) FROM stock_sectors WHERE source='hk_em'").fetchone()[0]
+    top = conn.execute(
+        "SELECT sector_name, COUNT(*) c FROM stock_sectors WHERE source='hk_em' "
+        "GROUP BY sector_name ORDER BY c DESC LIMIT 10").fetchall()
+    conn.close()
+    print(f"\n完成：ok={ok} empty={empty} 耗时{time.time()-t0:.0f}s", flush=True)
+    print(f"stock_sectors 港股行业记录: {n} 条", flush=True)
+    print("行业分布 top10:", flush=True)
+    for name, c in top:
+        print(f"  {name}: {c}", flush=True)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())