feat: 批量重评分片并发(老爸:这么多key不能并发?)
- batch_reassess --shard K/N: index%N==K分片,N进程互不重叠 - llm_client LLM_KEY_OFFSET: key池起始位错开,避免N进程压同一首选key - SKIP_FLUSH=1时worker不发摘要,parallel_batch.sh收尾统一flush - parallel_batch.sh: N路并行+等待+统一digest
This commit is contained in:
@@ -0,0 +1,21 @@
|
||||
#!/bin/bash
|
||||
# parallel_batch.sh — 并发批量12维重评(分片+key池偏移+统一flush)
|
||||
# 用法: bash parallel_batch.sh [N] [type]
|
||||
# N=worker数(默认4) type=holding|watchlist|all(默认all)
|
||||
N=${1:-4}
|
||||
TYPE=${2:-all}
|
||||
cd /home/hmo/MoFin/deploy/profile-scripts
|
||||
echo "[parallel] launching $N workers (type=$TYPE) at $(date '+%F %T')"
|
||||
pids=()
|
||||
for ((k=0; k<N; k++)); do
|
||||
SKIP_FLUSH=1 LLM_KEY_OFFSET=$k python3 batch_reassess.py --type "$TYPE" --shard "$k/$N" \
|
||||
> "/tmp/reassess_parallel_w$k.log" 2>&1 &
|
||||
pids+=($!)
|
||||
echo "[parallel] worker $k pid=${pids[$k]}"
|
||||
done
|
||||
for p in "${pids[@]}"; do wait "$p"; done
|
||||
echo "[parallel] all workers done at $(date '+%F %T')"
|
||||
# 统一推荐摘要(防各worker半成品)
|
||||
python3 -c "from mofin_db import flush_rec_digest; flush_rec_digest()"
|
||||
echo "[parallel] digest flushed"
|
||||
grep -h "^完成:" /tmp/reassess_parallel_w*.log
|
||||
Reference in New Issue
Block a user