feat: 同花顺新闻采集固化——ths_news.py(backfill/daily双模式)

deploy/profile-scripts/ths_news.py

@@ -0,0 +1,143 @@
+"""
+deploy/profile-scripts/ths_news.py — 同花顺新闻采集
+API: news.10jqka.com.cn/timeline_web/web/v1/news/list
+offset=publishTime/1000（秒级游标），无限回溯
+用法: python ths_news.py --backfill    # 全量回填
+      python ths_news.py --daily       # 每日增量
+"""
+import sys, os, json, time, sqlite3, requests, random
+from datetime import datetime
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+DB = os.environ.get('MOFIN_DB', os.path.join(os.path.dirname(__file__), '..', 'data', 'mofin.db'))
+
+URL = "https://news.10jqka.com.cn/timeline_web/web/v1/news/list"
+HEADERS = {"User-Agent": "Mozilla/5.0", "Referer": "http://stockpage.10jqka.com.cn/"}
+SLEEP = (0.3, 0.8)
+MAX_PAGES = 500
+
+
+def get_market_id(code):
+    """17=上交所(6开头), 33=深交所(0/3开头)"""
+    return 17 if code.startswith('6') else 33
+
+
+def get_codes():
+    """从 v8.1 5y 策略交易取 58 只股票"""
+    conn = sqlite3.connect(DB)
+    codes = set()
+    for r in conn.execute("SELECT results_json FROM strategy_research WHERE version='v8.1' AND period_tag='5y'").fetchall():
+        for t in json.loads(r[0])['trades']:
+            codes.add(t['code'])
+    conn.close()
+    return sorted(codes)
+
+
+def fetch_stock(code, max_pages=MAX_PAGES):
+    """翻页抓取一只股票的全部历史新闻"""
+    market_id = get_market_id(code)
+    items = []
+    offset = None
+    for page in range(max_pages):
+        params = {"marketId": market_id, "code": code, "size": 20}
+        if offset:
+            params["offset"] = offset
+        try:
+            r = requests.get(URL, params=params, headers=HEADERS, timeout=15)
+            data = r.json()
+            news = data.get('data', {}).get('newsList', [])
+        except:
+            break
+        if not news:
+            break
+        for it in news:
+            ts = it.get('publishTime')
+            if ts:
+                items.append({
+                    'code': code,
+                    'title': it.get('title', ''),
+                    'content': it.get('summary', ''),
+                    'source': it.get('source', 'ths'),
+                    'date': datetime.fromtimestamp(ts/1000).strftime('%Y-%m-%d %H:%M:%S'),
+                    'url': it.get('jumpUrl', ''),
+                })
+            offset = ts / 1000.0
+        time.sleep(random.uniform(*SLEEP))
+    return items
+
+
+def init_table(conn):
+    conn.execute("""
+        CREATE TABLE IF NOT EXISTS stock_news (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            code TEXT NOT NULL, title TEXT, content TEXT,
+            source TEXT, date TEXT, url TEXT,
+            sentiment TEXT, created_at TEXT DEFAULT (datetime('now','localtime')),
+            UNIQUE(code, date, title)
+        )
+    """)
+    conn.commit()
+
+
+def save_batch(conn, rows):
+    cur = conn.cursor()
+    n = 0
+    for r in rows:
+        try:
+            cur.execute(
+                "INSERT OR IGNORE INTO stock_news (code, title, content, source, date, url) VALUES (?,?,?,?,?,?)",
+                (r['code'], r['title'], r['content'], r['source'], r['date'], r['url']))
+            if cur.rowcount: n += 1
+        except: pass
+    conn.commit()
+    return n
+
+
+def run_backfill():
+    """一次性全量回填"""
+    codes = get_codes()
+    print(f"=== 同花顺新闻回填: {len(codes)} 只股票 ===", flush=True)
+    conn = sqlite3.connect(DB)
+    init_table(conn)
+    total_new = 0
+    for idx, code in enumerate(codes, 1):
+        exist = conn.execute("SELECT COUNT(*) FROM stock_news WHERE code=? AND source LIKE '同花顺%'", (code,)).fetchone()[0]
+        if exist > 100:
+            print(f"  [{idx}/{len(codes)}] {code} 已有{exist}条, 跳过", flush=True)
+            continue
+        arts = fetch_stock(code)
+        if arts:
+            n = save_batch(conn, arts)
+            total_new += n
+            dates = sorted(set(a['date'][:10] for a in arts))
+            print(f"  [{idx}/{len(codes)}] {code}: +{n}条 ({dates[0]}~{dates[-1]})", flush=True)
+        else:
+            print(f"  [{idx}/{len(codes)}] {code}: 0条", flush=True)
+    total = conn.execute("SELECT COUNT(*) FROM stock_news").fetchone()[0]
+    print(f"\n完成: 新增{total_new}条, 总计{total}条", flush=True)
+    conn.close()
+
+
+def run_daily():
+    """每日增量（只取最新一页）"""
+    codes = get_codes()
+    print(f"=== 同花顺每日更新: {len(codes)} 只股票 ===", flush=True)
+    conn = sqlite3.connect(DB)
+    init_table(conn)
+    total_new = 0
+    for idx, code in enumerate(codes, 1):
+        arts = fetch_stock(code, max_pages=3)
+        if arts:
+            n = save_batch(conn, arts)
+            total_new += n
+    print(f"完成: +{total_new}条", flush=True)
+    conn.close()
+
+
+if __name__ == '__main__':
+    if '--backfill' in sys.argv:
+        run_backfill()
+    elif '--daily' in sys.argv:
+        run_daily()
+    else:
+        print("用法: --backfill | --daily")