From 9fad6ff2857416c045c759f2a86116326141281d Mon Sep 17 00:00:00 2001
From: hmo <staymo7777@gmail.com>
Date: Fri, 31 Jul 2026 02:22:28 +0800
Subject: [PATCH] =?UTF-8?q?feat:=20=E4=B8=9C=E6=96=B9=E8=B4=A2=E5=AF=8C?=
 =?UTF-8?q?=E6=96=B0=E9=97=BB=E7=BF=BB=E9=A1=B5=E9=87=87=E9=9B=86=E5=99=A8?=
 =?UTF-8?q?(500=E6=9D=A1/=E8=82=A1,=20=E8=BF=916=E4=B8=AA=E6=9C=88)=20?=
 =?UTF-8?q?=E6=9B=BF=E6=8D=A2mofin=5Fnews=2010=E6=9D=A1=E6=A8=A1=E5=BC=8F?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 deploy/profile-scripts/news_collector.py | 125 +++++++++++++++++++++++
 1 file changed, 125 insertions(+)
 create mode 100644 deploy/profile-scripts/news_collector.py

diff --git a/deploy/profile-scripts/news_collector.py b/deploy/profile-scripts/news_collector.py
new file mode 100644
index 00000000..ab8027bf
--- /dev/null
+++ b/deploy/profile-scripts/news_collector.py
@@ -0,0 +1,125 @@
+"""
+news_collector.py — 东方财富新闻翻页采集（替换 mofin_news.py 的 10 条模式）
+跑一次回填近 4 个月 (~500 条/股），之后每日增量更新
+"""
+import sys, os, json, re, time, sqlite3
+from datetime import datetime, timedelta
+import requests
+
+DB = '/home/hmo/MoFin/data/mofin.db'
+URL = "https://search-api-web.eastmoney.com/search/jsonp"
+UA = {"User-Agent": "Mozilla/5.0", "Referer": "https://so.eastmoney.com/news/s"}
+PAGE_SIZE = 50
+SLEEP = 0.5  # 请求间隔
+
+def fetch_stock_news(code, max_pages=10):
+    """翻页采集一只股票的东方财富新闻"""
+    all_arts = []
+    for page in range(1, max_pages + 1):
+        inner = {
+            "uid": "", "keyword": code, "type": ["cmsArticleWebOld"],
+            "client": "web", "clientType": "web", "clientVersion": "curr",
+            "param": {"cmsArticleWebOld": {
+                "searchScope": "default", "sort": "default",
+                "pageIndex": page, "pageSize": PAGE_SIZE,
+                "preTag": "<em>", "postTag": "</em>",
+            }},
+        }
+        params = {"cb": "jQuery_nc", "param": json.dumps(inner, ensure_ascii=False), "_": "1"}
+        try:
+            r = requests.get(URL, params=params, headers=UA, timeout=20)
+            m = re.search(r'jQuery_nc\((.*)\)', r.text)
+            if not m: break
+            data = json.loads(m.group(1))
+            arts = data.get('result', {}).get('cmsArticleWebOld', [])
+            if not arts: break
+            for a in arts:
+                all_arts.append({
+                    'code': code,
+                    'title': re.sub(r'<[^>]+>', '', a.get('title', '')),
+                    'content': re.sub(r'<[^>]+>', '', a.get('content', '')),
+                    'source': a.get('mediaName', ''),
+                    'date': a.get('date', '')[:19],
+                    'url': f"http://finance.eastmoney.com/a/{a.get('code','')}.html",
+                })
+            time.sleep(SLEEP)
+        except Exception:
+            break
+    return all_arts
+
+
+def init_table(conn):
+    conn.execute("""
+        CREATE TABLE IF NOT EXISTS stock_news (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            code TEXT NOT NULL,
+            title TEXT,
+            content TEXT,
+            source TEXT,
+            date TEXT,
+            url TEXT,
+            sentiment TEXT,
+            created_at TEXT DEFAULT (datetime('now','localtime')),
+            UNIQUE(code, date, title)
+        )
+    """)
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_sn_code_date ON stock_news(code, date)")
+    conn.commit()
+
+
+if __name__ == '__main__':
+    conn = sqlite3.connect(DB)
+    init_table(conn)
+
+    # 取策略用到的 58 只股票
+    codes = set()
+    for r in conn.execute("SELECT results_json FROM strategy_research WHERE version='v8.1' AND period_tag='5y'").fetchall():
+        for t in json.loads(r[0])['trades']:
+            codes.add(t['code'])
+    codes = sorted(codes)
+
+    # 回填模式
+    is_backfill = '--backfill' in sys.argv
+    is_daily = '--daily' in sys.argv
+    max_pages = 15 if is_backfill else 3  # 回填翻15页，日常只翻3页
+
+    if is_backfill:
+        print(f"=== 回填 {len(codes)} 只股票 (最多{max_pages}页/PAGE_SIZE={PAGE_SIZE}) ===", flush=True)
+    else:
+        print(f"=== 增量更新 {len(codes)} 只股票 ({PAGE_SIZE}条/页, 最多{max_pages}页) ===", flush=True)
+
+    total_new = 0
+    for idx, code in enumerate(codes, 1):
+        # 回填模式不过滤，日常只取最新
+        if not is_backfill:
+            latest = conn.execute("SELECT date FROM stock_news WHERE code=? ORDER BY date DESC LIMIT 1", (code,)).fetchone()
+            if latest:
+                # 只需取最新日期之后的数据
+                print(f"  [{idx}/{len(codes)}] {code} 已有，跳到下一只", flush=True)
+                continue
+
+        arts = fetch_stock_news(code, max_pages)
+        if not arts:
+            print(f"  [{idx}/{len(codes)}] {code}: 0条", flush=True)
+            continue
+
+        cursor = conn.cursor()
+        n = 0
+        for a in arts:
+            try:
+                cursor.execute("INSERT OR IGNORE INTO stock_news (code, title, content, source, date, url) VALUES (?,?,?,?,?,?)",
+                              (a['code'], a['title'], a['content'], a['source'], a['date'], a['url']))
+                if cursor.rowcount:
+                    n += 1
+            except:
+                pass
+        conn.commit()
+        total_new += n
+        if arts:
+            dates = sorted(set(a['date'][:10] for a in arts))
+            print(f"  [{idx}/{len(codes)}] {code}: +{n}条 {dates[0]}~{dates[-1] if len(dates)>1 else dates[0]}", flush=True)
+        time.sleep(SLEEP)
+
+    total = conn.execute("SELECT COUNT(*) FROM stock_news").fetchone()[0]
+    print(f"\n完成: 新增{total_new}条, 总共{total}条", flush=True)
+    conn.close()