423 lines
15 KiB
Python
423 lines
15 KiB
Python
#!/usr/bin/env python3
|
||
"""collect_evaluation_data.py — 六维评估原始数据采集
|
||
|
||
纯数据收集脚本（no_agent），不做任何评估/判断/RR计算。
|
||
输出：data/evaluation_input.json — 供 21:00 LLM cron 使用。
|
||
|
||
采集内容：
|
||
D1 宏观环境 — 五大指数（上证/深证/恒生/恒科/A50）
|
||
D2 行业表现 — 持仓+自选按行业分组
|
||
D3 技术面（当前） — 今开/今高/今低/昨收/现价/成交量
|
||
D4 基本面 — PE/PB/总市值/52周高/52周低
|
||
D5 消息面 — (此脚本不采集，LLM cron web_search)
|
||
D6 资金面 — 成交额/换手率/量比
|
||
|
||
日期：2026-06-18 v1 — 初始版本
|
||
"""
|
||
|
||
import json
|
||
import os
|
||
import sys
|
||
from datetime import datetime
|
||
from pathlib import Path
|
||
|
||
# ── 消息通道统一路由(broadcast/xmpp by delivery) ──
|
||
try:
|
||
from messenger import install_stdio_hook as _msh
|
||
_msh()
|
||
except Exception:
|
||
pass
|
||
|
||
DATA_DIR = Path(__file__).parent.parent / "data"
|
||
PROFILES_PATH = DATA_DIR / "stock_profiles.json"
|
||
OUTPUT_PATH = DATA_DIR / "evaluation_input.json"
|
||
|
||
|
||
def load_json(path, default=None):
|
||
try:
|
||
with open(path, encoding="utf-8") as f:
|
||
return json.load(f)
|
||
except (FileNotFoundError, json.JSONDecodeError):
|
||
return {} if default is None else default
|
||
|
||
|
||
def save_json(path, data):
|
||
Path(path).parent.mkdir(parents=True, exist_ok=True)
|
||
with open(path, "w", encoding="utf-8") as f:
|
||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||
|
||
|
||
def fetch_tencent_data(symbols):
|
||
"""批量拉行情。读 DB live_prices + stock_daily（2026-08-26 分层铁律：消费层不直连腾讯API）"""
|
||
if not symbols:
|
||
return {}
|
||
import sqlite3
|
||
result = {}
|
||
try:
|
||
conn = sqlite3.connect('/home/hmo/MoFin/data/mofin.db', timeout=5)
|
||
syms = list(symbols)
|
||
ph = ",".join("?" * len(syms))
|
||
# 实时价：一次查
|
||
price_rows = conn.execute(
|
||
f"SELECT code, price, change_pct FROM live_prices WHERE code IN ({ph})", syms
|
||
).fetchall()
|
||
prices = {r[0]: (r[1], r[2]) for r in price_rows}
|
||
# 名称
|
||
name_rows = conn.execute(
|
||
f"SELECT code, name FROM stocks WHERE code IN ({ph})", syms
|
||
).fetchall()
|
||
names = {r[0]: r[1] or "" for r in name_rows}
|
||
# 最近日K（昨收/今开/高低/量）
|
||
sd = {}
|
||
for c in syms:
|
||
row = conn.execute(
|
||
"SELECT close, open, high, low, volume FROM stock_daily "
|
||
"WHERE code=? ORDER BY date DESC LIMIT 1", (c,)
|
||
).fetchone()
|
||
if row:
|
||
sd[c] = row
|
||
conn.close()
|
||
|
||
for code in syms:
|
||
if code not in prices:
|
||
continue
|
||
price = prices[code][0]
|
||
change_pct = prices[code][1] or 0
|
||
if not price or price <= 0:
|
||
continue
|
||
row = sd.get(code)
|
||
row_c = row[0] if row else 0
|
||
row_o = row[1] if row else 0
|
||
row_h = row[2] if row else 0
|
||
row_l = row[3] if row else 0
|
||
row_v = row[4] if row else 0
|
||
result[code] = {
|
||
"name": names.get(code, ""),
|
||
"price": price,
|
||
"prev_close": row_c,
|
||
"open": row_o,
|
||
"change_pct": change_pct,
|
||
"high": row_h,
|
||
"low": row_l,
|
||
"volume": row_v,
|
||
}
|
||
except Exception as e:
|
||
print(f"行情拉取失败: {e}", file=sys.stderr)
|
||
return {}
|
||
return result
|
||
|
||
|
||
def fetch_indices():
|
||
"""拉指数：读 DB stock_daily 最近收盘（2026-08-26 分层铁律：消费层不直连腾讯API）"""
|
||
import sqlite3
|
||
index_codes = {
|
||
"sh000001": "上证指数",
|
||
"sz399001": "深证成指",
|
||
"sz399006": "创业板指",
|
||
"hkHSI": "恒生指数",
|
||
"hkHSTECH": "恒生科技",
|
||
}
|
||
result = {}
|
||
try:
|
||
conn = sqlite3.connect('/home/hmo/MoFin/data/mofin.db', timeout=5)
|
||
for c, n in index_codes.items():
|
||
rows = conn.execute(
|
||
"SELECT date, close, open, high, low FROM stock_daily "
|
||
"WHERE code=? ORDER BY date DESC LIMIT 2", (c,)
|
||
).fetchall()
|
||
if not rows:
|
||
continue # 读不到则跳过（中性）
|
||
latest = rows[0]
|
||
prev = rows[1] if len(rows) > 1 else None
|
||
prev_close = prev[1] if prev else latest[1]
|
||
price = latest[1] or 0
|
||
change_pct = (price - prev_close) / prev_close * 100 if prev_close else 0
|
||
result[n] = {
|
||
"price": safe_float(price),
|
||
"prev_close": safe_float(prev_close if prev else None),
|
||
"change_pct": safe_float(change_pct),
|
||
"high": safe_float(latest[3] or price),
|
||
"low": safe_float(latest[4] or price),
|
||
"timestamp": latest[0],
|
||
}
|
||
conn.close()
|
||
except Exception as e:
|
||
print(f"指数拉取失败: {e}", file=sys.stderr)
|
||
return {}
|
||
return result
|
||
|
||
|
||
def safe_float(v):
|
||
try:
|
||
return float(v) if v else None
|
||
except (ValueError, TypeError):
|
||
return None
|
||
|
||
|
||
def parse_stock_data(code, fields, is_hk=False):
|
||
"""从腾讯 API 字段解析为结构化数据"""
|
||
# fields可能是dict(已解析)或list(原始split)
|
||
if isinstance(fields, dict):
|
||
data = {
|
||
"code": code,
|
||
"name": fields.get("name", code),
|
||
"price": safe_float(fields.get("price", 0)),
|
||
"prev_close": safe_float(fields.get("prev_close", 0)),
|
||
"open": safe_float(fields.get("open", 0)) if not is_hk else None,
|
||
"change_pct": safe_float(fields.get("change_pct", 0)),
|
||
"high": safe_float(fields.get("high", 0)),
|
||
"low": safe_float(fields.get("low", 0)),
|
||
"volume": safe_float(fields.get("volume", 0)),
|
||
}
|
||
else:
|
||
data = {
|
||
"code": code,
|
||
"name": fields[1] if len(fields) > 1 else code,
|
||
"price": safe_float(fields[3]),
|
||
"prev_close": safe_float(fields[4]),
|
||
"open": safe_float(fields[5]) if not is_hk else None,
|
||
"change_pct": safe_float(fields[32]),
|
||
"high": safe_float(fields[33]),
|
||
"low": safe_float(fields[34]),
|
||
"volume": safe_float(fields[6]),
|
||
}
|
||
# A股特有字段 (index 35+)
|
||
if not is_hk:
|
||
if isinstance(fields, dict):
|
||
data["turnover_rate"] = safe_float(fields.get("turnover_rate", 0))
|
||
data["amplitude"] = safe_float(fields.get("amplitude", 0))
|
||
data["pe"] = safe_float(fields.get("pe", 0))
|
||
data["market_cap"] = safe_float(fields.get("market_cap", 0))
|
||
else:
|
||
data["turnover_rate"] = safe_float(fields[38]) if len(fields) > 38 else 0
|
||
data["amplitude"] = safe_float(fields[39]) if len(fields) > 39 else 0
|
||
data["pe"] = safe_float(fields[39]) if len(fields) > 39 else 0
|
||
data["market_cap"] = safe_float(fields[45]) if len(fields) > 45 else 0
|
||
# 港股特有
|
||
if is_hk:
|
||
data["market"] = "HK"
|
||
if isinstance(fields, dict):
|
||
data["high_52w"] = safe_float(fields.get("high_52w", 0))
|
||
data["low_52w"] = safe_float(fields.get("low_52w", 0))
|
||
data["amplitude"] = safe_float(fields.get("amplitude", 0))
|
||
else:
|
||
data["high_52w"] = safe_float(fields[48]) if len(fields) > 48 else 0
|
||
data["low_52w"] = safe_float(fields[49]) if len(fields) > 49 else 0
|
||
data["amplitude"] = safe_float(fields[43]) if len(fields) > 43 else 0
|
||
# 港股特有字段
|
||
if is_hk:
|
||
if isinstance(fields, dict):
|
||
data["pe"] = safe_float(fields.get("pe", 0))
|
||
data["total_market_cap"] = safe_float(fields.get("total_market_cap", 0))
|
||
data["high_52w"] = safe_float(fields.get("high_52w", 0))
|
||
data["low_52w"] = safe_float(fields.get("low_52w", 0))
|
||
else:
|
||
if len(fields) > 70:
|
||
data["pe"] = safe_float(fields[71])
|
||
data["total_market_cap"] = safe_float(fields[69])
|
||
data["high_52w"] = safe_float(fields[48])
|
||
data["low_52w"] = safe_float(fields[49])
|
||
return data
|
||
|
||
|
||
def get_sector_mapping(profiles, decisions):
|
||
"""
|
||
从 stock_profiles.json 和 decisions.json 建立
|
||
{code: {name, sector, business, market, type}} 映射
|
||
"""
|
||
mapping = {}
|
||
# 先读 stock_profiles
|
||
profile_list = profiles.get("profiles", []) if isinstance(profiles, dict) else profiles
|
||
if isinstance(profile_list, list):
|
||
for p in profile_list:
|
||
code = p.get("code", "")
|
||
if code:
|
||
mapping[code] = {
|
||
"name": p.get("name", ""),
|
||
"sector": p.get("sector", ""),
|
||
"business": p.get("business", ""),
|
||
"market": p.get("market", ""),
|
||
"type": p.get("type", ""),
|
||
}
|
||
# 再补全 decisions.json 中的信息
|
||
for d in decisions.get("decisions", []):
|
||
code = d.get("code", "")
|
||
if code and code not in mapping:
|
||
trig = d.get("trigger", {})
|
||
mapping[code] = {
|
||
"name": d.get("name", code),
|
||
"sector": trig.get("sector_name", d.get("sector_name", "")),
|
||
"business": "",
|
||
"market": "港股" if len(code) == 5 else "A股",
|
||
"type": d.get("type", "持仓策略"),
|
||
}
|
||
return mapping
|
||
|
||
|
||
def get_portfolio_info(portfolio):
|
||
"""建立 {code: {cost, shares, position_pct}} 映射"""
|
||
result = {}
|
||
for h in portfolio.get("holdings", []):
|
||
code = h.get("code", "")
|
||
result[code] = {
|
||
"cost": h.get("cost", 0),
|
||
"shares": h.get("shares", 0),
|
||
"position_pct": h.get("position_pct", 0),
|
||
}
|
||
return result
|
||
|
||
|
||
def get_decisions_info(decisions):
|
||
"""提取 decisions.json 中的策略参数"""
|
||
return decisions.get("decisions", [])
|
||
|
||
|
||
def run():
|
||
# 加载数据
|
||
from mo_data import read_decisions, read_portfolio
|
||
decisions = read_decisions()
|
||
portfolio = read_portfolio()
|
||
profiles = load_json(PROFILES_PATH, {"profiles": []})
|
||
|
||
# 获取行业映射
|
||
sector_mapping = get_sector_mapping(profiles, decisions)
|
||
|
||
# 获取持仓信息
|
||
portfolio_info = get_portfolio_info(portfolio)
|
||
|
||
# 收集所有代码
|
||
all_codes = set()
|
||
for d in decisions.get("decisions", []):
|
||
code = d.get("code", "")
|
||
if code:
|
||
all_codes.add(code)
|
||
for h in portfolio.get("holdings", []):
|
||
code = h.get("code", "")
|
||
if code:
|
||
all_codes.add(code)
|
||
|
||
# 区分 A/H 股
|
||
a_codes = [c for c in all_codes if len(c) != 5]
|
||
hk_codes = [c for c in all_codes if len(c) == 5]
|
||
|
||
# 拉行情
|
||
a_prices = fetch_tencent_data(a_codes) if a_codes else {}
|
||
hk_prices = fetch_tencent_data(hk_codes) if hk_codes else {}
|
||
|
||
# 拉指数
|
||
index_data = fetch_indices()
|
||
|
||
# 解析个股数据
|
||
stock_data = {}
|
||
for code in a_codes:
|
||
if code in a_prices:
|
||
stock_data[code] = parse_stock_data(code, a_prices[code], is_hk=False)
|
||
for code in hk_codes:
|
||
if code in hk_prices:
|
||
stock_data[code] = parse_stock_data(code, hk_prices[code], is_hk=True)
|
||
|
||
# 组装输出
|
||
stocks = []
|
||
all_codes_sorted = sorted(all_codes)
|
||
|
||
for code in all_codes_sorted:
|
||
raw = stock_data.get(code, {})
|
||
sector_info = sector_mapping.get(code, {})
|
||
port = portfolio_info.get(code, {})
|
||
strategy = None
|
||
for d in decisions.get("decisions", []):
|
||
if d.get("code") == code:
|
||
trig = d.get("trigger", {})
|
||
strategy = {
|
||
"action": trig.get("action", d.get("action", "")),
|
||
"entry_zone": trig.get("entry_zone", ""),
|
||
"stop_loss": trig.get("stop_loss", d.get("stop_loss", "")),
|
||
"take_profit": trig.get("take_profit", d.get("take_profit", "")),
|
||
"type": d.get("type", "持仓策略"),
|
||
"tech_snapshot": trig.get("tech_snapshot", d.get("tech_snapshot", "")),
|
||
}
|
||
break
|
||
|
||
stock_entry = {
|
||
"code": code,
|
||
"name": raw.get("name", sector_info.get("name", code)),
|
||
"market": "港股" if len(code) == 5 else "A股",
|
||
"type": sector_info.get("type", "持仓策略"),
|
||
"sector": sector_info.get("sector", ""),
|
||
"business": sector_info.get("business", ""),
|
||
# 当天行情
|
||
"price": raw.get("price"),
|
||
"prev_close": raw.get("prev_close"),
|
||
"open": raw.get("open"),
|
||
"high": raw.get("high"),
|
||
"low": raw.get("low"),
|
||
"change_pct": raw.get("change_pct"),
|
||
"volume": raw.get("volume"),
|
||
# 基本面
|
||
"pe": raw.get("pe"),
|
||
"total_market_cap": raw.get("total_market_cap"),
|
||
"high_52w": raw.get("high_52w"),
|
||
"low_52w": raw.get("low_52w"),
|
||
"turnover_rate": raw.get("turnover_rate"),
|
||
"amplitude": raw.get("amplitude"),
|
||
# 持仓
|
||
"cost": port.get("cost", 0),
|
||
"shares": port.get("shares", 0),
|
||
"position_pct": port.get("position_pct", 0),
|
||
# 现策略
|
||
"strategy": strategy,
|
||
}
|
||
# 浮亏%
|
||
cost = port.get("cost", 0)
|
||
price = raw.get("price", 0)
|
||
if cost > 0 and price > 0:
|
||
stock_entry["pnl_pct"] = round((price - cost) / cost * 100, 2)
|
||
else:
|
||
stock_entry["pnl_pct"] = None
|
||
|
||
stocks.append(stock_entry)
|
||
|
||
# 按行业分组统计
|
||
sector_groups = {}
|
||
for s in stocks:
|
||
sector = s.get("sector", "未分类")
|
||
if sector not in sector_groups:
|
||
sector_groups[sector] = []
|
||
sector_groups[sector].append({
|
||
"code": s["code"],
|
||
"name": s["name"],
|
||
"change_pct": s["change_pct"],
|
||
"pnl_pct": s["pnl_pct"],
|
||
"type": s["type"],
|
||
})
|
||
|
||
# 汇总
|
||
total = len(stocks)
|
||
up_count = sum(1 for s in stocks if s["change_pct"] is not None and s["change_pct"] > 0)
|
||
down_count = sum(1 for s in stocks if s["change_pct"] is not None and s["change_pct"] < 0)
|
||
deep_loss = sum(1 for s in stocks if s["pnl_pct"] is not None and s["pnl_pct"] < -20)
|
||
|
||
output = {
|
||
"collected_at": datetime.now().isoformat(),
|
||
"total_stocks": total,
|
||
"summary": {
|
||
"up_count": up_count,
|
||
"down_count": down_count,
|
||
"deep_loss_count": deep_loss,
|
||
"holdings_count": len(portfolio_info),
|
||
"watchlist_count": total - len(portfolio_info),
|
||
},
|
||
"index_data": index_data,
|
||
"sector_groups": sector_groups,
|
||
"stocks": stocks,
|
||
}
|
||
|
||
save_json(OUTPUT_PATH, output)
|
||
print(f"数据收集完成: {total}只股票, {len(index_data)}个指数, {len(sector_groups)}个行业分组")
|
||
print(f" 上涨{up_count} 下跌{down_count} 深套{deep_loss}")
|
||
print(f" 输出: {OUTPUT_PATH}")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
run()
|