144 lines
4.8 KiB
Python
144 lines
4.8 KiB
Python
"""
|
|
deploy/profile-scripts/ths_news.py — 同花顺新闻采集
|
|
API: news.10jqka.com.cn/timeline_web/web/v1/news/list
|
|
offset=publishTime/1000（秒级游标），无限回溯
|
|
用法: python ths_news.py --backfill # 全量回填
|
|
python ths_news.py --daily # 每日增量
|
|
"""
|
|
import sys, os, json, time, sqlite3, requests, random
|
|
from datetime import datetime
|
|
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
|
|
DB = os.environ.get('MOFIN_DB', os.path.join(os.path.dirname(__file__), '..', 'data', 'mofin.db'))
|
|
|
|
URL = "https://news.10jqka.com.cn/timeline_web/web/v1/news/list"
|
|
HEADERS = {"User-Agent": "Mozilla/5.0", "Referer": "http://stockpage.10jqka.com.cn/"}
|
|
SLEEP = (0.3, 0.8)
|
|
MAX_PAGES = 500
|
|
|
|
|
|
def get_market_id(code):
|
|
"""17=上交所(6开头), 33=深交所(0/3开头)"""
|
|
return 17 if code.startswith('6') else 33
|
|
|
|
|
|
def get_codes():
|
|
"""从 v8.1 5y 策略交易取 58 只股票"""
|
|
conn = sqlite3.connect(DB)
|
|
codes = set()
|
|
for r in conn.execute("SELECT results_json FROM strategy_research WHERE version='v8.1' AND period_tag='5y'").fetchall():
|
|
for t in json.loads(r[0])['trades']:
|
|
codes.add(t['code'])
|
|
conn.close()
|
|
return sorted(codes)
|
|
|
|
|
|
def fetch_stock(code, max_pages=MAX_PAGES):
|
|
"""翻页抓取一只股票的全部历史新闻"""
|
|
market_id = get_market_id(code)
|
|
items = []
|
|
offset = None
|
|
for page in range(max_pages):
|
|
params = {"marketId": market_id, "code": code, "size": 20}
|
|
if offset:
|
|
params["offset"] = offset
|
|
try:
|
|
r = requests.get(URL, params=params, headers=HEADERS, timeout=15)
|
|
data = r.json()
|
|
news = data.get('data', {}).get('newsList', [])
|
|
except:
|
|
break
|
|
if not news:
|
|
break
|
|
for it in news:
|
|
ts = it.get('publishTime')
|
|
if ts:
|
|
items.append({
|
|
'code': code,
|
|
'title': it.get('title', ''),
|
|
'content': it.get('summary', ''),
|
|
'source': it.get('source', 'ths'),
|
|
'date': datetime.fromtimestamp(ts/1000).strftime('%Y-%m-%d %H:%M:%S'),
|
|
'url': it.get('jumpUrl', ''),
|
|
})
|
|
offset = ts / 1000.0
|
|
time.sleep(random.uniform(*SLEEP))
|
|
return items
|
|
|
|
|
|
def init_table(conn):
|
|
conn.execute("""
|
|
CREATE TABLE IF NOT EXISTS stock_news (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
code TEXT NOT NULL, title TEXT, content TEXT,
|
|
source TEXT, date TEXT, url TEXT,
|
|
sentiment TEXT, created_at TEXT DEFAULT (datetime('now','localtime')),
|
|
UNIQUE(code, date, title)
|
|
)
|
|
""")
|
|
conn.commit()
|
|
|
|
|
|
def save_batch(conn, rows):
|
|
cur = conn.cursor()
|
|
n = 0
|
|
for r in rows:
|
|
try:
|
|
cur.execute(
|
|
"INSERT OR IGNORE INTO stock_news (code, title, content, source, date, url) VALUES (?,?,?,?,?,?)",
|
|
(r['code'], r['title'], r['content'], r['source'], r['date'], r['url']))
|
|
if cur.rowcount: n += 1
|
|
except: pass
|
|
conn.commit()
|
|
return n
|
|
|
|
|
|
def run_backfill():
|
|
"""一次性全量回填"""
|
|
codes = get_codes()
|
|
print(f"=== 同花顺新闻回填: {len(codes)} 只股票 ===", flush=True)
|
|
conn = sqlite3.connect(DB)
|
|
init_table(conn)
|
|
total_new = 0
|
|
for idx, code in enumerate(codes, 1):
|
|
exist = conn.execute("SELECT COUNT(*) FROM stock_news WHERE code=? AND source LIKE '同花顺%'", (code,)).fetchone()[0]
|
|
if exist > 100:
|
|
print(f" [{idx}/{len(codes)}] {code} 已有{exist}条, 跳过", flush=True)
|
|
continue
|
|
arts = fetch_stock(code)
|
|
if arts:
|
|
n = save_batch(conn, arts)
|
|
total_new += n
|
|
dates = sorted(set(a['date'][:10] for a in arts))
|
|
print(f" [{idx}/{len(codes)}] {code}: +{n}条 ({dates[0]}~{dates[-1]})", flush=True)
|
|
else:
|
|
print(f" [{idx}/{len(codes)}] {code}: 0条", flush=True)
|
|
total = conn.execute("SELECT COUNT(*) FROM stock_news").fetchone()[0]
|
|
print(f"\n完成: 新增{total_new}条, 总计{total}条", flush=True)
|
|
conn.close()
|
|
|
|
|
|
def run_daily():
|
|
"""每日增量（只取最新一页）"""
|
|
codes = get_codes()
|
|
print(f"=== 同花顺每日更新: {len(codes)} 只股票 ===", flush=True)
|
|
conn = sqlite3.connect(DB)
|
|
init_table(conn)
|
|
total_new = 0
|
|
for idx, code in enumerate(codes, 1):
|
|
arts = fetch_stock(code, max_pages=3)
|
|
if arts:
|
|
n = save_batch(conn, arts)
|
|
total_new += n
|
|
print(f"完成: +{total_new}条", flush=True)
|
|
conn.close()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
if '--backfill' in sys.argv:
|
|
run_backfill()
|
|
elif '--daily' in sys.argv:
|
|
run_daily()
|
|
else:
|
|
print("用法: --backfill | --daily")
|