diff --git a/deploy/profile-scripts/ths_news.py b/deploy/profile-scripts/ths_news.py new file mode 100644 index 00000000..dc2af2d8 --- /dev/null +++ b/deploy/profile-scripts/ths_news.py @@ -0,0 +1,143 @@ +""" +deploy/profile-scripts/ths_news.py — 同花顺新闻采集 +API: news.10jqka.com.cn/timeline_web/web/v1/news/list +offset=publishTime/1000(秒级游标),无限回溯 +用法: python ths_news.py --backfill # 全量回填 + python ths_news.py --daily # 每日增量 +""" +import sys, os, json, time, sqlite3, requests, random +from datetime import datetime + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) +DB = os.environ.get('MOFIN_DB', os.path.join(os.path.dirname(__file__), '..', 'data', 'mofin.db')) + +URL = "https://news.10jqka.com.cn/timeline_web/web/v1/news/list" +HEADERS = {"User-Agent": "Mozilla/5.0", "Referer": "http://stockpage.10jqka.com.cn/"} +SLEEP = (0.3, 0.8) +MAX_PAGES = 500 + + +def get_market_id(code): + """17=上交所(6开头), 33=深交所(0/3开头)""" + return 17 if code.startswith('6') else 33 + + +def get_codes(): + """从 v8.1 5y 策略交易取 58 只股票""" + conn = sqlite3.connect(DB) + codes = set() + for r in conn.execute("SELECT results_json FROM strategy_research WHERE version='v8.1' AND period_tag='5y'").fetchall(): + for t in json.loads(r[0])['trades']: + codes.add(t['code']) + conn.close() + return sorted(codes) + + +def fetch_stock(code, max_pages=MAX_PAGES): + """翻页抓取一只股票的全部历史新闻""" + market_id = get_market_id(code) + items = [] + offset = None + for page in range(max_pages): + params = {"marketId": market_id, "code": code, "size": 20} + if offset: + params["offset"] = offset + try: + r = requests.get(URL, params=params, headers=HEADERS, timeout=15) + data = r.json() + news = data.get('data', {}).get('newsList', []) + except: + break + if not news: + break + for it in news: + ts = it.get('publishTime') + if ts: + items.append({ + 'code': code, + 'title': it.get('title', ''), + 'content': it.get('summary', ''), + 'source': it.get('source', 'ths'), + 'date': datetime.fromtimestamp(ts/1000).strftime('%Y-%m-%d %H:%M:%S'), + 'url': it.get('jumpUrl', ''), + }) + offset = ts / 1000.0 + time.sleep(random.uniform(*SLEEP)) + return items + + +def init_table(conn): + conn.execute(""" + CREATE TABLE IF NOT EXISTS stock_news ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + code TEXT NOT NULL, title TEXT, content TEXT, + source TEXT, date TEXT, url TEXT, + sentiment TEXT, created_at TEXT DEFAULT (datetime('now','localtime')), + UNIQUE(code, date, title) + ) + """) + conn.commit() + + +def save_batch(conn, rows): + cur = conn.cursor() + n = 0 + for r in rows: + try: + cur.execute( + "INSERT OR IGNORE INTO stock_news (code, title, content, source, date, url) VALUES (?,?,?,?,?,?)", + (r['code'], r['title'], r['content'], r['source'], r['date'], r['url'])) + if cur.rowcount: n += 1 + except: pass + conn.commit() + return n + + +def run_backfill(): + """一次性全量回填""" + codes = get_codes() + print(f"=== 同花顺新闻回填: {len(codes)} 只股票 ===", flush=True) + conn = sqlite3.connect(DB) + init_table(conn) + total_new = 0 + for idx, code in enumerate(codes, 1): + exist = conn.execute("SELECT COUNT(*) FROM stock_news WHERE code=? AND source LIKE '同花顺%'", (code,)).fetchone()[0] + if exist > 100: + print(f" [{idx}/{len(codes)}] {code} 已有{exist}条, 跳过", flush=True) + continue + arts = fetch_stock(code) + if arts: + n = save_batch(conn, arts) + total_new += n + dates = sorted(set(a['date'][:10] for a in arts)) + print(f" [{idx}/{len(codes)}] {code}: +{n}条 ({dates[0]}~{dates[-1]})", flush=True) + else: + print(f" [{idx}/{len(codes)}] {code}: 0条", flush=True) + total = conn.execute("SELECT COUNT(*) FROM stock_news").fetchone()[0] + print(f"\n完成: 新增{total_new}条, 总计{total}条", flush=True) + conn.close() + + +def run_daily(): + """每日增量(只取最新一页)""" + codes = get_codes() + print(f"=== 同花顺每日更新: {len(codes)} 只股票 ===", flush=True) + conn = sqlite3.connect(DB) + init_table(conn) + total_new = 0 + for idx, code in enumerate(codes, 1): + arts = fetch_stock(code, max_pages=3) + if arts: + n = save_batch(conn, arts) + total_new += n + print(f"完成: +{total_new}条", flush=True) + conn.close() + + +if __name__ == '__main__': + if '--backfill' in sys.argv: + run_backfill() + elif '--daily' in sys.argv: + run_daily() + else: + print("用法: --backfill | --daily")