feat(bot): screenshot OCR pipeline + ack delay 120s
D fix: screenshots were silently dropped (empty body + OOB url). - register xep_0066, capture msg['oob']['url'] when body empty - also handle body-as-URL messages (some clients put URL in body) - download from upload.yoin.fun, OCR via SenseNova (sensenova-6.7-flash-lite) - inject OCR text as context into LLM call - config at /home/hmo/.config/mofin/ocr_config.json (outside repo) - replaces dead node122 GLM-OCR path (host unreachable) A+B fix: ACK_DELAY 15s -> 120s. 15s fired on every normal LLM cold-start (20-100s), now only signals genuine hangs.
This commit is contained in:
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"key": "sk-aRNj3UwKSLPsDfh15QNTPwbHxahblfaO",
|
||||
"base_url": "https://token.sensenova.cn/v1",
|
||||
"model": "sensenova-6.7-flash-lite"
|
||||
}
|
||||
@@ -48,7 +48,7 @@ MUC_NICK = ""
|
||||
AGENT_MENTION = ""
|
||||
HTTP_PORT = 5808
|
||||
AGENT_NICK = ""
|
||||
ACK_DELAY = 15
|
||||
ACK_DELAY = 120 # 真卡死才提示(普通 LLM 冷启动 20-100s 不应触发)
|
||||
GATEWAY_URL = ""
|
||||
GATEWAY_API_KEY = ""
|
||||
GATEWAY_SESSION_ID = ""
|
||||
@@ -56,6 +56,89 @@ GATEWAY_DEADLINE_SECONDS = 180
|
||||
CALL_HERMES_TIMEOUT = 180
|
||||
FALLBACK_REPLY = "请稍等,我在处理..."
|
||||
|
||||
# ── 图片 OCR 配置(截图消息 → SenseNova vision)──
|
||||
OCR_CONFIG_FILE = "/home/hmo/.config/mofin/ocr_config.json"
|
||||
OCR_TIMEOUT = 90
|
||||
IMAGE_EXTS = ('.jpg', '.jpeg', '.png', '.webp', '.gif')
|
||||
|
||||
|
||||
def _load_ocr_config():
|
||||
"""从 /home/hmo/.config/mofin/ocr_config.json 读取 SenseNova 配置。
|
||||
文件不存在或未配置 → 返回 (None, None, None),图片功能降级为仅提示收到。"""
|
||||
try:
|
||||
with open(OCR_CONFIG_FILE, 'r', encoding='utf-8') as f:
|
||||
cfg = json.load(f)
|
||||
return cfg.get('key'), cfg.get('base_url'), cfg.get('model', 'sensenova-6.7-flash-lite')
|
||||
except Exception:
|
||||
return None, None, None
|
||||
|
||||
|
||||
def _download_image(url, timeout=30):
|
||||
"""下载图片字节。失败返回 None。"""
|
||||
import urllib.request
|
||||
import ssl
|
||||
try:
|
||||
ctx = ssl.create_default_context()
|
||||
ctx.check_hostname = False
|
||||
ctx.verify_mode = ssl.CERT_NONE
|
||||
req = urllib.request.Request(url, headers={'User-Agent': 'curl/8.5.0'})
|
||||
with urllib.request.urlopen(req, timeout=timeout, context=ctx) as resp:
|
||||
data = resp.read()
|
||||
return data if len(data) > 100 else None
|
||||
except Exception as e:
|
||||
log.error(f"图片下载失败 {url[:80]}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def _ocr_image(img_data, prompt=None):
|
||||
"""调 SenseNova vision OCR。返回 (ok, text)。"""
|
||||
key, base, model = _load_ocr_config()
|
||||
if not (key and base):
|
||||
return False, "OCR未配置"
|
||||
import base64
|
||||
import urllib.request
|
||||
b64 = base64.b64encode(img_data).decode()
|
||||
text_prompt = prompt or "请识别这张图片中的所有文字内容,包括数字、股票名称、金额、日期。用中文回复。"
|
||||
payload = json.dumps({
|
||||
"model": model,
|
||||
"messages": [{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{b64}"}},
|
||||
{"type": "text", "text": text_prompt},
|
||||
]
|
||||
}],
|
||||
"max_tokens": 1500,
|
||||
}).encode()
|
||||
req = urllib.request.Request(
|
||||
f"{base.rstrip('/')}/chat/completions", data=payload,
|
||||
headers={"Content-Type": "application/json", "Authorization": f"Bearer {key}"})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=OCR_TIMEOUT) as resp:
|
||||
data = json.loads(resp.read().decode())
|
||||
msg = data.get('choices', [{}])[0].get('message', {})
|
||||
text = msg.get('content', '') or msg.get('reasoning', '')
|
||||
return True, text.strip() if text.strip() else "(OCR无内容)"
|
||||
except Exception as e:
|
||||
return False, f"OCR失败: {str(e)[:120]}"
|
||||
|
||||
|
||||
def _is_image_url(url):
|
||||
u = url.lower().split('?')[0]
|
||||
return ('upload.yoin.fun' in u or '/upload/' in u or u.endswith(IMAGE_EXTS))
|
||||
|
||||
|
||||
def _process_image_message(url):
|
||||
"""下载+OCR 一张截图,返回注入 LLM 的上下文文本。"""
|
||||
img = _download_image(url)
|
||||
if not img:
|
||||
return "[老爸发来一张截图,但下载失败,无法识别]"
|
||||
ok, text = _ocr_image(img)
|
||||
if not ok:
|
||||
return f"[老爸发来一张截图,OCR识别失败: {text}]"
|
||||
return (f"[老爸发来一张截图,OCR识别内容如下]\n{text}\n"
|
||||
f"[截图内容结束] 请基于截图内容回应老爸。")
|
||||
|
||||
# ── 全局队列 ──
|
||||
_outbound_queue = []
|
||||
_outbound_lock = threading.Lock()
|
||||
@@ -160,6 +243,7 @@ class XmppAgent(slixmpp.ClientXMPP):
|
||||
self.register_plugin('xep_0030')
|
||||
self.register_plugin('xep_0045')
|
||||
self.register_plugin('xep_0199')
|
||||
self.register_plugin('xep_0066') # OOB 附件(截图 URL)
|
||||
|
||||
async def on_start(self, event):
|
||||
self.send_presence()
|
||||
@@ -183,8 +267,21 @@ class XmppAgent(slixmpp.ClientXMPP):
|
||||
def on_msg(self, msg):
|
||||
if msg['type'] in ('chat', 'groupchat'):
|
||||
body = str(msg['body']).strip()
|
||||
# body 为空时检查 OOB 附件(XEP-0066:截图/文件以 OOB url 送达)
|
||||
if not body:
|
||||
return
|
||||
try:
|
||||
oob_url = str(msg['oob']['url'] or '').strip()
|
||||
except Exception:
|
||||
oob_url = ''
|
||||
if oob_url and _is_image_url(oob_url):
|
||||
body = f"[IMAGE] {oob_url}"
|
||||
log.info(f"📩 收到图片(OOB): from={msg['from']} url={oob_url[:80]}")
|
||||
else:
|
||||
return
|
||||
# body 本身是上传 URL(部分客户端把 URL 直接放 body)
|
||||
elif body.startswith('http') and _is_image_url(body):
|
||||
log.info(f"📩 收到图片(URL body): from={msg['from']} url={body[:80]}")
|
||||
body = f"[IMAGE] {body}"
|
||||
log.info(f"📩 收到: from={msg['from']} type={msg['type']} body={body[:60]}")
|
||||
if ('[executor]' in body and 'gateway_zhiwei' in body):
|
||||
log.info(f"过滤 executor 消息: {body[:80]}...")
|
||||
@@ -246,6 +343,12 @@ def _inbound_loop(bot):
|
||||
sender, body, msg_type = _inbound_queue.pop(0)
|
||||
log.info(f"🔄 inbound处理: {body[:40]}")
|
||||
ack_mgr.start(body[:40], sender, body)
|
||||
# 截图消息:下载 + OCR → 注入上下文
|
||||
if body.startswith('[IMAGE]'):
|
||||
img_url = body[len('[IMAGE]'):].strip()
|
||||
log.info(f"🖼️ 图片消息处理: {img_url[:80]}")
|
||||
body = _process_image_message(img_url)
|
||||
log.info(f"🖼️ OCR完成: {body[:100]}")
|
||||
reply = call_hermes(body)
|
||||
ack_mgr.ack(body[:40])
|
||||
if reply:
|
||||
|
||||
Reference in New Issue
Block a user