"""
news_collector.py — 东方财富新闻翻页采集（替换 mofin_news.py 的 10 条模式）
跑一次回填近 4 个月 (~500 条/股），之后每日增量更新
"""
import sys, os, json, re, time, sqlite3
from datetime import datetime, timedelta
import requests

DB = '/home/hmo/MoFin/data/mofin.db'
URL = "https://search-api-web.eastmoney.com/search/jsonp"
UA = {"User-Agent": "Mozilla/5.0", "Referer": "https://so.eastmoney.com/news/s"}
PAGE_SIZE = 50
SLEEP = 0.5  # 请求间隔

def fetch_stock_news(code, max_pages=10):
    """翻页采集一只股票的东方财富新闻"""
    all_arts = []
    for page in range(1, max_pages + 1):
        inner = {
            "uid": "", "keyword": code, "type": ["cmsArticleWebOld"],
            "client": "web", "clientType": "web", "clientVersion": "curr",
            "param": {"cmsArticleWebOld": {
                "searchScope": "default", "sort": "default",
                "pageIndex": page, "pageSize": PAGE_SIZE,
                "preTag": "<em>", "postTag": "</em>",
            }},
        }
        params = {"cb": "jQuery_nc", "param": json.dumps(inner, ensure_ascii=False), "_": "1"}
        try:
            r = requests.get(URL, params=params, headers=UA, timeout=20)
            m = re.search(r'jQuery_nc\((.*)\)', r.text)
            if not m: break
            data = json.loads(m.group(1))
            arts = data.get('result', {}).get('cmsArticleWebOld', [])
            if not arts: break
            for a in arts:
                all_arts.append({
                    'code': code,
                    'title': re.sub(r'<[^>]+>', '', a.get('title', '')),
                    'content': re.sub(r'<[^>]+>', '', a.get('content', '')),
                    'source': a.get('mediaName', ''),
                    'date': a.get('date', '')[:19],
                    'url': f"http://finance.eastmoney.com/a/{a.get('code','')}.html",
                })
            time.sleep(SLEEP)
        except Exception:
            break
    return all_arts


def init_table(conn):
    conn.execute("""
        CREATE TABLE IF NOT EXISTS stock_news (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            code TEXT NOT NULL,
            title TEXT,
            content TEXT,
            source TEXT,
            date TEXT,
            url TEXT,
            sentiment TEXT,
            created_at TEXT DEFAULT (datetime('now','localtime')),
            UNIQUE(code, date, title)
        )
    """)
    conn.execute("CREATE INDEX IF NOT EXISTS idx_sn_code_date ON stock_news(code, date)")
    conn.commit()


if __name__ == '__main__':
    conn = sqlite3.connect(DB)
    init_table(conn)

    # 取策略用到的 58 只股票
    codes = set()
    for r in conn.execute("SELECT results_json FROM strategy_research WHERE version='v8.1' AND period_tag='5y'").fetchall():
        for t in json.loads(r[0])['trades']:
            codes.add(t['code'])
    codes = sorted(codes)

    # 回填模式
    is_backfill = '--backfill' in sys.argv
    is_daily = '--daily' in sys.argv
    max_pages = 15 if is_backfill else 3  # 回填翻15页，日常只翻3页

    if is_backfill:
        print(f"=== 回填 {len(codes)} 只股票 (最多{max_pages}页/PAGE_SIZE={PAGE_SIZE}) ===", flush=True)
    else:
        print(f"=== 增量更新 {len(codes)} 只股票 ({PAGE_SIZE}条/页, 最多{max_pages}页) ===", flush=True)

    total_new = 0
    for idx, code in enumerate(codes, 1):
        # 回填模式不过滤，日常只取最新
        if not is_backfill:
            latest = conn.execute("SELECT date FROM stock_news WHERE code=? ORDER BY date DESC LIMIT 1", (code,)).fetchone()
            if latest:
                # 只需取最新日期之后的数据
                print(f"  [{idx}/{len(codes)}] {code} 已有，跳到下一只", flush=True)
                continue

        arts = fetch_stock_news(code, max_pages)
        if not arts:
            print(f"  [{idx}/{len(codes)}] {code}: 0条", flush=True)
            continue

        cursor = conn.cursor()
        n = 0
        for a in arts:
            try:
                cursor.execute("INSERT OR IGNORE INTO stock_news (code, title, content, source, date, url) VALUES (?,?,?,?,?,?)",
                              (a['code'], a['title'], a['content'], a['source'], a['date'], a['url']))
                if cursor.rowcount:
                    n += 1
            except:
                pass
        conn.commit()
        total_new += n
        if arts:
            dates = sorted(set(a['date'][:10] for a in arts))
            print(f"  [{idx}/{len(codes)}] {code}: +{n}条 {dates[0]}~{dates[-1] if len(dates)>1 else dates[0]}", flush=True)
        time.sleep(SLEEP)

    total = conn.execute("SELECT COUNT(*) FROM stock_news").fetchone()[0]
    print(f"\n完成: 新增{total_new}条, 总共{total}条", flush=True)
    conn.close()