"""
deploy/profile-scripts/ths_news.py — 同花顺新闻采集
API: news.10jqka.com.cn/timeline_web/web/v1/news/list
offset=publishTime/1000（秒级游标），无限回溯
用法: python ths_news.py --backfill    # 全量回填
      python ths_news.py --daily       # 每日增量
"""
import sys, os, json, time, sqlite3, requests, random
from datetime import datetime

sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
DB = os.environ.get('MOFIN_DB', os.path.join(os.path.dirname(__file__), '..', 'data', 'mofin.db'))

URL = "https://news.10jqka.com.cn/timeline_web/web/v1/news/list"
HEADERS = {"User-Agent": "Mozilla/5.0", "Referer": "http://stockpage.10jqka.com.cn/"}
SLEEP = (0.3, 0.8)
MAX_PAGES = 500


def get_market_id(code):
    """17=上交所(6开头), 33=深交所(0/3开头)"""
    return 17 if code.startswith('6') else 33


def get_codes():
    """从 v8.1 5y 策略交易取 58 只股票"""
    conn = sqlite3.connect(DB)
    codes = set()
    for r in conn.execute("SELECT results_json FROM strategy_research WHERE version='v8.1' AND period_tag='5y'").fetchall():
        for t in json.loads(r[0])['trades']:
            codes.add(t['code'])
    conn.close()
    return sorted(codes)


def fetch_stock(code, max_pages=MAX_PAGES):
    """翻页抓取一只股票的全部历史新闻"""
    market_id = get_market_id(code)
    items = []
    offset = None
    for page in range(max_pages):
        params = {"marketId": market_id, "code": code, "size": 20}
        if offset:
            params["offset"] = offset
        try:
            r = requests.get(URL, params=params, headers=HEADERS, timeout=15)
            data = r.json()
            news = data.get('data', {}).get('newsList', [])
        except:
            break
        if not news:
            break
        for it in news:
            ts = it.get('publishTime')
            if ts:
                items.append({
                    'code': code,
                    'title': it.get('title', ''),
                    'content': it.get('summary', ''),
                    'source': it.get('source', 'ths'),
                    'date': datetime.fromtimestamp(ts/1000).strftime('%Y-%m-%d %H:%M:%S'),
                    'url': it.get('jumpUrl', ''),
                })
            offset = ts / 1000.0
        time.sleep(random.uniform(*SLEEP))
    return items


def init_table(conn):
    conn.execute("""
        CREATE TABLE IF NOT EXISTS stock_news (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            code TEXT NOT NULL, title TEXT, content TEXT,
            source TEXT, date TEXT, url TEXT,
            sentiment TEXT, created_at TEXT DEFAULT (datetime('now','localtime')),
            UNIQUE(code, date, title)
        )
    """)
    conn.commit()


def save_batch(conn, rows):
    cur = conn.cursor()
    n = 0
    for r in rows:
        try:
            cur.execute(
                "INSERT OR IGNORE INTO stock_news (code, title, content, source, date, url) VALUES (?,?,?,?,?,?)",
                (r['code'], r['title'], r['content'], r['source'], r['date'], r['url']))
            if cur.rowcount: n += 1
        except: pass
    conn.commit()
    return n


def run_backfill():
    """一次性全量回填"""
    codes = get_codes()
    print(f"=== 同花顺新闻回填: {len(codes)} 只股票 ===", flush=True)
    conn = sqlite3.connect(DB)
    init_table(conn)
    total_new = 0
    for idx, code in enumerate(codes, 1):
        exist = conn.execute("SELECT COUNT(*) FROM stock_news WHERE code=? AND source LIKE '同花顺%'", (code,)).fetchone()[0]
        if exist > 100:
            print(f"  [{idx}/{len(codes)}] {code} 已有{exist}条, 跳过", flush=True)
            continue
        arts = fetch_stock(code)
        if arts:
            n = save_batch(conn, arts)
            total_new += n
            dates = sorted(set(a['date'][:10] for a in arts))
            print(f"  [{idx}/{len(codes)}] {code}: +{n}条 ({dates[0]}~{dates[-1]})", flush=True)
        else:
            print(f"  [{idx}/{len(codes)}] {code}: 0条", flush=True)
    total = conn.execute("SELECT COUNT(*) FROM stock_news").fetchone()[0]
    print(f"\n完成: 新增{total_new}条, 总计{total}条", flush=True)
    conn.close()


def run_daily():
    """每日增量（只取最新一页）"""
    codes = get_codes()
    print(f"=== 同花顺每日更新: {len(codes)} 只股票 ===", flush=True)
    conn = sqlite3.connect(DB)
    init_table(conn)
    total_new = 0
    for idx, code in enumerate(codes, 1):
        arts = fetch_stock(code, max_pages=3)
        if arts:
            n = save_batch(conn, arts)
            total_new += n
    print(f"完成: +{total_new}条", flush=True)
    conn.close()


if __name__ == '__main__':
    if '--backfill' in sys.argv:
        run_backfill()
    elif '--daily' in sys.argv:
        run_daily()
    else:
        print("用法: --backfill | --daily")