From bb1529909b5663e6fd552ff1ece83d58b98ca22a Mon Sep 17 00:00:00 2001 From: hmo Date: Sun, 19 Jul 2026 23:23:46 +0800 Subject: [PATCH] feat(bot): screenshot OCR pipeline + ack delay 120s D fix: screenshots were silently dropped (empty body + OOB url). - register xep_0066, capture msg['oob']['url'] when body empty - also handle body-as-URL messages (some clients put URL in body) - download from upload.yoin.fun, OCR via SenseNova (sensenova-6.7-flash-lite) - inject OCR text as context into LLM call - config at /home/hmo/.config/mofin/ocr_config.json (outside repo) - replaces dead node122 GLM-OCR path (host unreachable) A+B fix: ACK_DELAY 15s -> 120s. 15s fired on every normal LLM cold-start (20-100s), now only signals genuine hangs. --- deploy/bot/ocr_config.example.json | 5 ++ deploy/bot/xmpp_agent_core.py | 107 ++++++++++++++++++++++++++++- scripts/syntax_check.py | 8 +++ scripts/test_image_pipeline.py | 23 +++++++ scripts/test_sn_ocr2.py | 29 ++++++++ 5 files changed, 170 insertions(+), 2 deletions(-) create mode 100644 deploy/bot/ocr_config.example.json create mode 100644 scripts/syntax_check.py create mode 100644 scripts/test_image_pipeline.py create mode 100644 scripts/test_sn_ocr2.py diff --git a/deploy/bot/ocr_config.example.json b/deploy/bot/ocr_config.example.json new file mode 100644 index 00000000..0c35d728 --- /dev/null +++ b/deploy/bot/ocr_config.example.json @@ -0,0 +1,5 @@ +{ + "key": "sk-aRNj3UwKSLPsDfh15QNTPwbHxahblfaO", + "base_url": "https://token.sensenova.cn/v1", + "model": "sensenova-6.7-flash-lite" +} diff --git a/deploy/bot/xmpp_agent_core.py b/deploy/bot/xmpp_agent_core.py index e5e6e8a4..f44c3829 100644 --- a/deploy/bot/xmpp_agent_core.py +++ b/deploy/bot/xmpp_agent_core.py @@ -48,7 +48,7 @@ MUC_NICK = "" AGENT_MENTION = "" HTTP_PORT = 5808 AGENT_NICK = "" -ACK_DELAY = 15 +ACK_DELAY = 120 # 真卡死才提示（普通 LLM 冷启动 20-100s 不应触发） GATEWAY_URL = "" GATEWAY_API_KEY = "" GATEWAY_SESSION_ID = "" @@ -56,6 +56,89 @@ GATEWAY_DEADLINE_SECONDS = 180 CALL_HERMES_TIMEOUT = 180 FALLBACK_REPLY = "请稍等，我在处理..." +# ── 图片 OCR 配置（截图消息 → SenseNova vision）── +OCR_CONFIG_FILE = "/home/hmo/.config/mofin/ocr_config.json" +OCR_TIMEOUT = 90 +IMAGE_EXTS = ('.jpg', '.jpeg', '.png', '.webp', '.gif') + + +def _load_ocr_config(): + """从 /home/hmo/.config/mofin/ocr_config.json 读取 SenseNova 配置。 + 文件不存在或未配置 → 返回 (None, None, None)，图片功能降级为仅提示收到。""" + try: + with open(OCR_CONFIG_FILE, 'r', encoding='utf-8') as f: + cfg = json.load(f) + return cfg.get('key'), cfg.get('base_url'), cfg.get('model', 'sensenova-6.7-flash-lite') + except Exception: + return None, None, None + + +def _download_image(url, timeout=30): + """下载图片字节。失败返回 None。""" + import urllib.request + import ssl + try: + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + req = urllib.request.Request(url, headers={'User-Agent': 'curl/8.5.0'}) + with urllib.request.urlopen(req, timeout=timeout, context=ctx) as resp: + data = resp.read() + return data if len(data) > 100 else None + except Exception as e: + log.error(f"图片下载失败 {url[:80]}: {e}") + return None + + +def _ocr_image(img_data, prompt=None): + """调 SenseNova vision OCR。返回 (ok, text)。""" + key, base, model = _load_ocr_config() + if not (key and base): + return False, "OCR未配置" + import base64 + import urllib.request + b64 = base64.b64encode(img_data).decode() + text_prompt = prompt or "请识别这张图片中的所有文字内容，包括数字、股票名称、金额、日期。用中文回复。" + payload = json.dumps({ + "model": model, + "messages": [{ + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{b64}"}}, + {"type": "text", "text": text_prompt}, + ] + }], + "max_tokens": 1500, + }).encode() + req = urllib.request.Request( + f"{base.rstrip('/')}/chat/completions", data=payload, + headers={"Content-Type": "application/json", "Authorization": f"Bearer {key}"}) + try: + with urllib.request.urlopen(req, timeout=OCR_TIMEOUT) as resp: + data = json.loads(resp.read().decode()) + msg = data.get('choices', [{}])[0].get('message', {}) + text = msg.get('content', '') or msg.get('reasoning', '') + return True, text.strip() if text.strip() else "(OCR无内容)" + except Exception as e: + return False, f"OCR失败: {str(e)[:120]}" + + +def _is_image_url(url): + u = url.lower().split('?')[0] + return ('upload.yoin.fun' in u or '/upload/' in u or u.endswith(IMAGE_EXTS)) + + +def _process_image_message(url): + """下载+OCR 一张截图，返回注入 LLM 的上下文文本。""" + img = _download_image(url) + if not img: + return "[老爸发来一张截图，但下载失败，无法识别]" + ok, text = _ocr_image(img) + if not ok: + return f"[老爸发来一张截图，OCR识别失败: {text}]" + return (f"[老爸发来一张截图，OCR识别内容如下]\n{text}\n" + f"[截图内容结束] 请基于截图内容回应老爸。") + # ── 全局队列 ── _outbound_queue = [] _outbound_lock = threading.Lock() @@ -160,6 +243,7 @@ class XmppAgent(slixmpp.ClientXMPP): self.register_plugin('xep_0030') self.register_plugin('xep_0045') self.register_plugin('xep_0199') + self.register_plugin('xep_0066') # OOB 附件（截图 URL） async def on_start(self, event): self.send_presence() @@ -183,8 +267,21 @@ class XmppAgent(slixmpp.ClientXMPP): def on_msg(self, msg): if msg['type'] in ('chat', 'groupchat'): body = str(msg['body']).strip() + # body 为空时检查 OOB 附件（XEP-0066：截图/文件以 OOB url 送达） if not body: - return + try: + oob_url = str(msg['oob']['url'] or '').strip() + except Exception: + oob_url = '' + if oob_url and _is_image_url(oob_url): + body = f"[IMAGE] {oob_url}" + log.info(f"📩 收到图片(OOB): from={msg['from']} url={oob_url[:80]}") + else: + return + # body 本身是上传 URL（部分客户端把 URL 直接放 body） + elif body.startswith('http') and _is_image_url(body): + log.info(f"📩 收到图片(URL body): from={msg['from']} url={body[:80]}") + body = f"[IMAGE] {body}" log.info(f"📩 收到: from={msg['from']} type={msg['type']} body={body[:60]}") if ('[executor]' in body and 'gateway_zhiwei' in body): log.info(f"过滤 executor 消息: {body[:80]}...") @@ -246,6 +343,12 @@ def _inbound_loop(bot): sender, body, msg_type = _inbound_queue.pop(0) log.info(f"🔄 inbound处理: {body[:40]}") ack_mgr.start(body[:40], sender, body) + # 截图消息：下载 + OCR → 注入上下文 + if body.startswith('[IMAGE]'): + img_url = body[len('[IMAGE]'):].strip() + log.info(f"🖼️ 图片消息处理: {img_url[:80]}") + body = _process_image_message(img_url) + log.info(f"🖼️ OCR完成: {body[:100]}") reply = call_hermes(body) ack_mgr.ack(body[:40]) if reply: diff --git a/scripts/syntax_check.py b/scripts/syntax_check.py new file mode 100644 index 00000000..6fc8ff9f --- /dev/null +++ b/scripts/syntax_check.py @@ -0,0 +1,8 @@ +import ast, sys +src = open('/home/hmo/MoFin/deploy/bot/xmpp_agent_core.py').read() +try: + ast.parse(src) + print('SYNTAX OK, lines:', len(src.splitlines())) +except SyntaxError as e: + print('SYNTAX ERROR:', e) + sys.exit(1) \ No newline at end of file diff --git a/scripts/test_image_pipeline.py b/scripts/test_image_pipeline.py new file mode 100644 index 00000000..15bce7c1 --- /dev/null +++ b/scripts/test_image_pipeline.py @@ -0,0 +1,23 @@ +import sys +sys.argv = ['test', '--agent', 'zhiwei'] +# Prevent bot from actually connecting — just test the image functions +src = open('/home/hmo/MoFin/deploy/bot/xmpp_agent_core.py').read() +# Cut off at main entry to avoid starting the bot +cut = src.find("if __name__ ==") +if cut > 0: + src = src[:cut] +ns = {} +exec(compile(src, 'xmpp_agent_core.py', 'exec'), ns) + +url = "https://upload.yoin.fun/upload/d22cef590582300cc5580c722a19280054bf0d6a/brEbU8RxXwipU9VuhZqhhFOIAmQI4epG5dp5so6U/LBbTaPJdRfmzxjBg0uh21A.jpg" +print("== _is_image_url ==", ns['_is_image_url'](url)) +print("== download ==") +img = ns['_download_image'](url) +print("bytes:", len(img) if img else None) +if img: + print("== OCR ==") + ok, text = ns['_ocr_image'](img) + print("ok:", ok) + print("text:", text[:600]) +print("== full pipeline ==") +print(ns['_process_image_message'](url)[:700]) \ No newline at end of file diff --git a/scripts/test_sn_ocr2.py b/scripts/test_sn_ocr2.py new file mode 100644 index 00000000..8c6df72c --- /dev/null +++ b/scripts/test_sn_ocr2.py @@ -0,0 +1,29 @@ +import json, urllib.request, base64 + +KEY = "sk-aRNj3UwKSLPsDfh15QNTPwbHxahblfaO" +BASE = "https://token.sensenova.cn/v1" +MODEL = "sensenova-6.7-flash-lite" + +img_b64 = base64.b64encode(open('/tmp/test_shot.jpg', 'rb').read()).decode() +payload = json.dumps({ + "model": MODEL, + "messages": [{ + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{img_b64}"}}, + {"type": "text", "text": "请识别这张图片中的所有文字内容，包括数字、股票名称、金额、日期。用中文回复。"} + ] + }], + "max_tokens": 1500, +}).encode() +req = urllib.request.Request(f"{BASE}/chat/completions", data=payload, + headers={"Content-Type": "application/json", "Authorization": f"Bearer {KEY}"}) +try: + resp = urllib.request.urlopen(req, timeout=90) + data = json.loads(resp.read().decode()) + msg = data.get('choices', [{}])[0].get('message', {}) + text = msg.get('content', '') or msg.get('reasoning', '') + print('OCR OK:') + print(text[:1500]) +except Exception as e: + print('OCR FAIL:', e) \ No newline at end of file