feat: 批量重评分片并发(老爸:这么多key不能并发?)

- batch_reassess --shard K/N: index%N==K分片,N进程互不重叠
- llm_client LLM_KEY_OFFSET: key池起始位错开,避免N进程压同一首选key
- SKIP_FLUSH=1时worker不发摘要,parallel_batch.sh收尾统一flush
- parallel_batch.sh: N路并行+等待+统一digest
This commit is contained in:
hmo
2026-07-24 09:02:34 +08:00
parent 9fad0cfa14
commit a559fc84dd
3 changed files with 48 additions and 10 deletions
+8 -3
View File
@@ -107,14 +107,19 @@ _KEY_POOL_TS = 0
def _get_key_pool():
"""key 池缓存 5 分钟。"""
"""key 池缓存 5 分钟。LLM_KEY_OFFSET 环境变量:并发 worker 起始 key 错开,
避免 N 个进程同时压同一个首选 key(2026-07-24 并发分片配套)。"""
global _KEY_POOL, _KEY_POOL_TS
import time as _t
import time as _t, os as _os
if not _KEY_POOL or (_t.time() - _KEY_POOL_TS) > 300:
_KEY_POOL = _load_key_pool()
_KEY_POOL_TS = _t.time()
if _KEY_POOL:
print(f" [LLM] key池: {[k for k, _ in _KEY_POOL]}", flush=True)
_off = int(_os.environ.get("LLM_KEY_OFFSET", "0") or 0)
if _off and len(_KEY_POOL) > 1:
_off = _off % len(_KEY_POOL)
_KEY_POOL = _KEY_POOL[_off:] + _KEY_POOL[:_off]
print(f" [LLM] key池: {[k for k, _ in _KEY_POOL]}" + (f" (offset={_off})" if _off else ""), flush=True)
return _KEY_POOL