feat: 东方财富新闻翻页采集器(500条/股, 近6个月) 替换mofin_news 10条模式
This commit is contained in:
@@ -0,0 +1,125 @@
|
|||||||
|
"""
|
||||||
|
news_collector.py — 东方财富新闻翻页采集（替换 mofin_news.py 的 10 条模式）
|
||||||
|
跑一次回填近 4 个月 (~500 条/股），之后每日增量更新
|
||||||
|
"""
|
||||||
|
import sys, os, json, re, time, sqlite3
|
||||||
|
from datetime import datetime, timedelta
|
||||||
|
import requests
|
||||||
|
|
||||||
|
DB = '/home/hmo/MoFin/data/mofin.db'
|
||||||
|
URL = "https://search-api-web.eastmoney.com/search/jsonp"
|
||||||
|
UA = {"User-Agent": "Mozilla/5.0", "Referer": "https://so.eastmoney.com/news/s"}
|
||||||
|
PAGE_SIZE = 50
|
||||||
|
SLEEP = 0.5 # 请求间隔
|
||||||
|
|
||||||
|
def fetch_stock_news(code, max_pages=10):
|
||||||
|
"""翻页采集一只股票的东方财富新闻"""
|
||||||
|
all_arts = []
|
||||||
|
for page in range(1, max_pages + 1):
|
||||||
|
inner = {
|
||||||
|
"uid": "", "keyword": code, "type": ["cmsArticleWebOld"],
|
||||||
|
"client": "web", "clientType": "web", "clientVersion": "curr",
|
||||||
|
"param": {"cmsArticleWebOld": {
|
||||||
|
"searchScope": "default", "sort": "default",
|
||||||
|
"pageIndex": page, "pageSize": PAGE_SIZE,
|
||||||
|
"preTag": "<em>", "postTag": "</em>",
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
params = {"cb": "jQuery_nc", "param": json.dumps(inner, ensure_ascii=False), "_": "1"}
|
||||||
|
try:
|
||||||
|
r = requests.get(URL, params=params, headers=UA, timeout=20)
|
||||||
|
m = re.search(r'jQuery_nc\((.*)\)', r.text)
|
||||||
|
if not m: break
|
||||||
|
data = json.loads(m.group(1))
|
||||||
|
arts = data.get('result', {}).get('cmsArticleWebOld', [])
|
||||||
|
if not arts: break
|
||||||
|
for a in arts:
|
||||||
|
all_arts.append({
|
||||||
|
'code': code,
|
||||||
|
'title': re.sub(r'<[^>]+>', '', a.get('title', '')),
|
||||||
|
'content': re.sub(r'<[^>]+>', '', a.get('content', '')),
|
||||||
|
'source': a.get('mediaName', ''),
|
||||||
|
'date': a.get('date', '')[:19],
|
||||||
|
'url': f"http://finance.eastmoney.com/a/{a.get('code','')}.html",
|
||||||
|
})
|
||||||
|
time.sleep(SLEEP)
|
||||||
|
except Exception:
|
||||||
|
break
|
||||||
|
return all_arts
|
||||||
|
|
||||||
|
|
||||||
|
def init_table(conn):
|
||||||
|
conn.execute("""
|
||||||
|
CREATE TABLE IF NOT EXISTS stock_news (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
code TEXT NOT NULL,
|
||||||
|
title TEXT,
|
||||||
|
content TEXT,
|
||||||
|
source TEXT,
|
||||||
|
date TEXT,
|
||||||
|
url TEXT,
|
||||||
|
sentiment TEXT,
|
||||||
|
created_at TEXT DEFAULT (datetime('now','localtime')),
|
||||||
|
UNIQUE(code, date, title)
|
||||||
|
)
|
||||||
|
""")
|
||||||
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_sn_code_date ON stock_news(code, date)")
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
conn = sqlite3.connect(DB)
|
||||||
|
init_table(conn)
|
||||||
|
|
||||||
|
# 取策略用到的 58 只股票
|
||||||
|
codes = set()
|
||||||
|
for r in conn.execute("SELECT results_json FROM strategy_research WHERE version='v8.1' AND period_tag='5y'").fetchall():
|
||||||
|
for t in json.loads(r[0])['trades']:
|
||||||
|
codes.add(t['code'])
|
||||||
|
codes = sorted(codes)
|
||||||
|
|
||||||
|
# 回填模式
|
||||||
|
is_backfill = '--backfill' in sys.argv
|
||||||
|
is_daily = '--daily' in sys.argv
|
||||||
|
max_pages = 15 if is_backfill else 3 # 回填翻15页，日常只翻3页
|
||||||
|
|
||||||
|
if is_backfill:
|
||||||
|
print(f"=== 回填 {len(codes)} 只股票 (最多{max_pages}页/PAGE_SIZE={PAGE_SIZE}) ===", flush=True)
|
||||||
|
else:
|
||||||
|
print(f"=== 增量更新 {len(codes)} 只股票 ({PAGE_SIZE}条/页, 最多{max_pages}页) ===", flush=True)
|
||||||
|
|
||||||
|
total_new = 0
|
||||||
|
for idx, code in enumerate(codes, 1):
|
||||||
|
# 回填模式不过滤，日常只取最新
|
||||||
|
if not is_backfill:
|
||||||
|
latest = conn.execute("SELECT date FROM stock_news WHERE code=? ORDER BY date DESC LIMIT 1", (code,)).fetchone()
|
||||||
|
if latest:
|
||||||
|
# 只需取最新日期之后的数据
|
||||||
|
print(f" [{idx}/{len(codes)}] {code} 已有，跳到下一只", flush=True)
|
||||||
|
continue
|
||||||
|
|
||||||
|
arts = fetch_stock_news(code, max_pages)
|
||||||
|
if not arts:
|
||||||
|
print(f" [{idx}/{len(codes)}] {code}: 0条", flush=True)
|
||||||
|
continue
|
||||||
|
|
||||||
|
cursor = conn.cursor()
|
||||||
|
n = 0
|
||||||
|
for a in arts:
|
||||||
|
try:
|
||||||
|
cursor.execute("INSERT OR IGNORE INTO stock_news (code, title, content, source, date, url) VALUES (?,?,?,?,?,?)",
|
||||||
|
(a['code'], a['title'], a['content'], a['source'], a['date'], a['url']))
|
||||||
|
if cursor.rowcount:
|
||||||
|
n += 1
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
conn.commit()
|
||||||
|
total_new += n
|
||||||
|
if arts:
|
||||||
|
dates = sorted(set(a['date'][:10] for a in arts))
|
||||||
|
print(f" [{idx}/{len(codes)}] {code}: +{n}条 {dates[0]}~{dates[-1] if len(dates)>1 else dates[0]}", flush=True)
|
||||||
|
time.sleep(SLEEP)
|
||||||
|
|
||||||
|
total = conn.execute("SELECT COUNT(*) FROM stock_news").fetchone()[0]
|
||||||
|
print(f"\n完成: 新增{total_new}条, 总共{total}条", flush=True)
|
||||||
|
conn.close()
|
||||||
Reference in New Issue
Block a user