hmo/MoFin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(bot): screenshot OCR pipeline + ack delay 120s

Browse Source 
D fix: screenshots were silently dropped (empty body + OOB url).
- register xep_0066, capture msg['oob']['url'] when body empty
- also handle body-as-URL messages (some clients put URL in body)
- download from upload.yoin.fun, OCR via SenseNova (sensenova-6.7-flash-lite)
- inject OCR text as context into LLM call
- config at /home/hmo/.config/mofin/ocr_config.json (outside repo)
- replaces dead node122 GLM-OCR path (host unreachable)

A+B fix: ACK_DELAY 15s -> 120s. 15s fired on every normal LLM
cold-start (20-100s), now only signals genuine hangs.
This commit is contained in:
hmo
2026-07-19 23:23:46 +08:00
parent e0f47f9349
commit bb1529909b
5 changed files with 170 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deploy/bot/ocr_config.example.json
+5
View File
@@ -0,0 +1,5 @@
{
"key": "sk-aRNj3UwKSLPsDfh15QNTPwbHxahblfaO",
"base_url": "https://token.sensenova.cn/v1",
"model": "sensenova-6.7-flash-lite"
}
deploy/bot/xmpp_agent_core.py
+105 -2
View File
@@ -48,7 +48,7 @@ MUC_NICK = ""
AGENT_MENTION = ""
HTTP_PORT = 5808
AGENT_NICK = ""
ACK_DELAY = 15
ACK_DELAY = 120 # 真卡死才提示（普通 LLM 冷启动 20-100s 不应触发）
GATEWAY_URL = ""
GATEWAY_API_KEY = ""
GATEWAY_SESSION_ID = ""
@@ -56,6 +56,89 @@ GATEWAY_DEADLINE_SECONDS = 180
CALL_HERMES_TIMEOUT = 180
FALLBACK_REPLY = "请稍等，我在处理..."
# ── 图片 OCR 配置（截图消息 → SenseNova vision）──
OCR_CONFIG_FILE = "/home/hmo/.config/mofin/ocr_config.json"
OCR_TIMEOUT = 90
IMAGE_EXTS = ('.jpg', '.jpeg', '.png', '.webp', '.gif')
def _load_ocr_config():
"""从 /home/hmo/.config/mofin/ocr_config.json 读取 SenseNova 配置。
文件不存在或未配置 → 返回 (None, None, None)，图片功能降级为仅提示收到。"""
try:
with open(OCR_CONFIG_FILE, 'r', encoding='utf-8') as f:
cfg = json.load(f)
return cfg.get('key'), cfg.get('base_url'), cfg.get('model', 'sensenova-6.7-flash-lite')
except Exception:
return None, None, None
def _download_image(url, timeout=30):
"""下载图片字节。失败返回 None。"""
import urllib.request
import ssl
try:
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
req = urllib.request.Request(url, headers={'User-Agent': 'curl/8.5.0'})
with urllib.request.urlopen(req, timeout=timeout, context=ctx) as resp:
data = resp.read()
return data if len(data) > 100 else None
except Exception as e:
log.error(f"图片下载失败 {url[:80]}: {e}")
return None
def _ocr_image(img_data, prompt=None):
"""调 SenseNova vision OCR。返回 (ok, text)。"""
key, base, model = _load_ocr_config()
if not (key and base):
return False, "OCR未配置"
import base64
import urllib.request
b64 = base64.b64encode(img_data).decode()
text_prompt = prompt or "请识别这张图片中的所有文字内容，包括数字、股票名称、金额、日期。用中文回复。"
payload = json.dumps({
"model": model,
"messages": [{
"role": "user",
"content": [
{"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{b64}"}},
{"type": "text", "text": text_prompt},
]
}],
"max_tokens": 1500,
}).encode()
req = urllib.request.Request(
f"{base.rstrip('/')}/chat/completions", data=payload,
headers={"Content-Type": "application/json", "Authorization": f"Bearer {key}"})
try:
with urllib.request.urlopen(req, timeout=OCR_TIMEOUT) as resp:
data = json.loads(resp.read().decode())
msg = data.get('choices', [{}])[0].get('message', {})
text = msg.get('content', '') or msg.get('reasoning', '')
return True, text.strip() if text.strip() else "(OCR无内容)"
except Exception as e:
return False, f"OCR失败: {str(e)[:120]}"
def _is_image_url(url):
u = url.lower().split('?')[0]
return ('upload.yoin.fun' in u or '/upload/' in u or u.endswith(IMAGE_EXTS))
def _process_image_message(url):
"""下载+OCR 一张截图，返回注入 LLM 的上下文文本。"""
img = _download_image(url)
if not img:
return "[老爸发来一张截图，但下载失败，无法识别]"
ok, text = _ocr_image(img)
if not ok:
return f"[老爸发来一张截图，OCR识别失败: {text}]"
return (f"[老爸发来一张截图，OCR识别内容如下]\n{text}\n"
f"[截图内容结束] 请基于截图内容回应老爸。")
# ── 全局队列 ──
_outbound_queue = []
_outbound_lock = threading.Lock()
@@ -160,6 +243,7 @@ class XmppAgent(slixmpp.ClientXMPP):
self.register_plugin('xep_0030')
self.register_plugin('xep_0045')
self.register_plugin('xep_0199')
self.register_plugin('xep_0066') # OOB 附件（截图 URL）
async def on_start(self, event):
self.send_presence()
@@ -183,8 +267,21 @@ class XmppAgent(slixmpp.ClientXMPP):
def on_msg(self, msg):
if msg['type'] in ('chat', 'groupchat'):
body = str(msg['body']).strip()
# body 为空时检查 OOB 附件（XEP-0066：截图/文件以 OOB url 送达）
if not body:
return
try:
oob_url = str(msg['oob']['url'] or '').strip()
except Exception:
oob_url = ''
if oob_url and _is_image_url(oob_url):
body = f"[IMAGE] {oob_url}"
log.info(f"📩 收到图片(OOB): from={msg['from']} url={oob_url[:80]}")
else:
return
# body 本身是上传 URL（部分客户端把 URL 直接放 body）
elif body.startswith('http') and _is_image_url(body):
log.info(f"📩 收到图片(URL body): from={msg['from']} url={body[:80]}")
body = f"[IMAGE] {body}"
log.info(f"📩 收到: from={msg['from']} type={msg['type']} body={body[:60]}")
if ('[executor]' in body and 'gateway_zhiwei' in body):
log.info(f"过滤 executor 消息: {body[:80]}...")
@@ -246,6 +343,12 @@ def _inbound_loop(bot):
sender, body, msg_type = _inbound_queue.pop(0)
log.info(f"🔄 inbound处理: {body[:40]}")
ack_mgr.start(body[:40], sender, body)
# 截图消息：下载 + OCR → 注入上下文
if body.startswith('[IMAGE]'):
img_url = body[len('[IMAGE]'):].strip()
log.info(f"🖼️ 图片消息处理: {img_url[:80]}")
body = _process_image_message(img_url)
log.info(f"🖼️ OCR完成: {body[:100]}")
reply = call_hermes(body)
ack_mgr.ack(body[:40])
if reply:
scripts/syntax_check.py
+8
View File
@@ -0,0 +1,8 @@
import ast, sys
src = open('/home/hmo/MoFin/deploy/bot/xmpp_agent_core.py').read()
try:
ast.parse(src)
print('SYNTAX OK, lines:', len(src.splitlines()))
except SyntaxError as e:
print('SYNTAX ERROR:', e)
sys.exit(1)
scripts/test_image_pipeline.py
+23
View File
@@ -0,0 +1,23 @@
import sys
sys.argv = ['test', '--agent', 'zhiwei']
# Prevent bot from actually connecting — just test the image functions
src = open('/home/hmo/MoFin/deploy/bot/xmpp_agent_core.py').read()
# Cut off at main entry to avoid starting the bot
cut = src.find("if __name__ ==")
if cut > 0:
src = src[:cut]
ns = {}
exec(compile(src, 'xmpp_agent_core.py', 'exec'), ns)
url = "https://upload.yoin.fun/upload/d22cef590582300cc5580c722a19280054bf0d6a/brEbU8RxXwipU9VuhZqhhFOIAmQI4epG5dp5so6U/LBbTaPJdRfmzxjBg0uh21A.jpg"
print("== _is_image_url ==", ns['_is_image_url'](url))
print("== download ==")
img = ns['_download_image'](url)
print("bytes:", len(img) if img else None)
if img:
print("== OCR ==")
ok, text = ns['_ocr_image'](img)
print("ok:", ok)
print("text:", text[:600])
print("== full pipeline ==")
print(ns['_process_image_message'](url)[:700])
scripts/test_sn_ocr2.py
+29
View File
@@ -0,0 +1,29 @@
import json, urllib.request, base64
KEY = "sk-aRNj3UwKSLPsDfh15QNTPwbHxahblfaO"
BASE = "https://token.sensenova.cn/v1"
MODEL = "sensenova-6.7-flash-lite"
img_b64 = base64.b64encode(open('/tmp/test_shot.jpg', 'rb').read()).decode()
payload = json.dumps({
"model": MODEL,
"messages": [{
"role": "user",
"content": [
{"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{img_b64}"}},
{"type": "text", "text": "请识别这张图片中的所有文字内容，包括数字、股票名称、金额、日期。用中文回复。"}
]
}],
"max_tokens": 1500,
}).encode()
req = urllib.request.Request(f"{BASE}/chat/completions", data=payload,
headers={"Content-Type": "application/json", "Authorization": f"Bearer {KEY}"})
try:
resp = urllib.request.urlopen(req, timeout=90)
data = json.loads(resp.read().decode())
msg = data.get('choices', [{}])[0].get('message', {})
text = msg.get('content', '') or msg.get('reasoning', '')
print('OCR OK:')
print(text[:1500])
except Exception as e:
print('OCR FAIL:', e)