异常浮窗+统一告警通道+冷却1小时: alert_logger统一异常写入(alerts.json+history.log+XMPP), 前端右上角浮动异常区(自动展开5秒折叠,只显示active), price_monitor接入异常钩子(重评失败/拉取失败/DB同步), 冷却30分->1小时+15分5%突发例外
This commit is contained in:
+104
@@ -0,0 +1,104 @@
|
|||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""alert_logger.py — MoFin 统一异常告警通道(2026-08-12 落地)
|
||||||
|
|
||||||
|
任何脚本/任务出错时调用 record_alert():
|
||||||
|
- 写入 data/alerts.json(active 异常列表,前端浮窗读取 /api/overview 的 alerts)
|
||||||
|
- 追加 data/alerts_history.log(历史异常实时日志,永久保留)
|
||||||
|
- 同时通过 xmpp_logger 发送 XMPP 告警(老莫铁律:任何错误/异常 → xmpp告警 + MoFin监控)
|
||||||
|
|
||||||
|
用法:
|
||||||
|
from alert_logger import record_alert
|
||||||
|
record_alert(level="error", source="price_monitor", title="重评失败",
|
||||||
|
detail="688002 LLM 调用超时", code="688002")
|
||||||
|
|
||||||
|
level: info / warning / error / critical
|
||||||
|
"""
|
||||||
|
import json
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
DATA_DIR = Path("/home/hmo/MoFin/data")
|
||||||
|
ALERTS_FILE = DATA_DIR / "alerts.json"
|
||||||
|
HISTORY_FILE = DATA_DIR / "alerts_history.log"
|
||||||
|
|
||||||
|
# 保留的 active 异常数(浮窗只显示 active,最多保留 20 条)
|
||||||
|
MAX_ACTIVE = 20
|
||||||
|
|
||||||
|
|
||||||
|
def _now_str():
|
||||||
|
from datetime import datetime
|
||||||
|
return datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||||
|
|
||||||
|
|
||||||
|
def record_alert(level="error", source="", title="", detail="", code="", clear_after=0):
|
||||||
|
"""记录一条异常。clear_after>0 表示自动清除的秒数(可空)"""
|
||||||
|
ts = time.time()
|
||||||
|
ts_str = _now_str()
|
||||||
|
entry = {
|
||||||
|
"ts": ts,
|
||||||
|
"ts_str": ts_str,
|
||||||
|
"level": level,
|
||||||
|
"source": source,
|
||||||
|
"title": title,
|
||||||
|
"detail": detail,
|
||||||
|
"code": code,
|
||||||
|
"clear_after": clear_after,
|
||||||
|
}
|
||||||
|
|
||||||
|
# 1. active 列表(alerts.json)
|
||||||
|
try:
|
||||||
|
alerts = json.loads(ALERTS_FILE.read_text(encoding="utf-8")) if ALERTS_FILE.exists() else []
|
||||||
|
if not isinstance(alerts, list):
|
||||||
|
alerts = []
|
||||||
|
# 同 source+code 的旧告警先移除(避免重复堆积),再插到头部
|
||||||
|
alerts = [a for a in alerts if not (a.get("source") == source and a.get("code") == code)]
|
||||||
|
alerts.insert(0, entry)
|
||||||
|
# 清除超时的
|
||||||
|
now = time.time()
|
||||||
|
alerts = [a for a in alerts if not a.get("clear_after") or now - a.get("ts", 0) < a["clear_after"]]
|
||||||
|
alerts = alerts[:MAX_ACTIVE]
|
||||||
|
ALERTS_FILE.write_text(json.dumps(alerts, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# 2. 历史日志(永久追加)
|
||||||
|
try:
|
||||||
|
with HISTORY_FILE.open("a", encoding="utf-8") as f:
|
||||||
|
f.write(f"[{ts_str}] [{level.upper()}] {source} | {title} | {detail} | {code}\n")
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# 3. XMPP 告警(非 info 级别才发,避免轰炸)→ 本地 XMPP 发送服务 :5805
|
||||||
|
if level != "info":
|
||||||
|
try:
|
||||||
|
import urllib.request as _ur
|
||||||
|
body = f"⚠️[{level.upper()}] {source}: {title} {detail} {code}".strip()
|
||||||
|
req = _ur.Request(
|
||||||
|
"http://127.0.0.1:5805/",
|
||||||
|
data=body.encode("utf-8"),
|
||||||
|
headers={"Content-Type": "text/plain"},
|
||||||
|
method="POST",
|
||||||
|
)
|
||||||
|
_ur.urlopen(req, timeout=5).read()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
return entry
|
||||||
|
|
||||||
|
|
||||||
|
def clear_alert(source="", code=""):
|
||||||
|
"""清除指定 source+code 的 active 告警(恢复后调用)"""
|
||||||
|
try:
|
||||||
|
alerts = json.loads(ALERTS_FILE.read_text(encoding="utf-8")) if ALERTS_FILE.exists() else []
|
||||||
|
if not isinstance(alerts, list):
|
||||||
|
return
|
||||||
|
alerts = [a for a in alerts if not (a.get("source") == source and (not code or a.get("code") == code))]
|
||||||
|
ALERTS_FILE.write_text(json.dumps(alerts, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
# 自测
|
||||||
|
record_alert(level="info", source="alert_logger", title="自测", detail="告警通道正常", code="test")
|
||||||
|
print("alerts.json 写入 OK:", ALERTS_FILE.exists())
|
||||||
@@ -9,6 +9,7 @@ import sqlite3
|
|||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
from mo_data import read_decisions
|
from mo_data import read_decisions
|
||||||
|
from alert_logger import record_alert, clear_alert
|
||||||
|
|
||||||
BREACH_PATH = "/home/hmo/.hermes/zone_breach.json"
|
BREACH_PATH = "/home/hmo/.hermes/zone_breach.json"
|
||||||
STATE_PATH = "/home/hmo/.hermes/price_trigger_state.json"
|
STATE_PATH = "/home/hmo/.hermes/price_trigger_state.json"
|
||||||
@@ -105,6 +106,10 @@ def fetch_all_prices(codes):
|
|||||||
text = r.read().decode("gbk")
|
text = r.read().decode("gbk")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"⚠️ 批量拉取失败: {e}", file=sys.stderr)
|
print(f"⚠️ 批量拉取失败: {e}", file=sys.stderr)
|
||||||
|
try:
|
||||||
|
record_alert(level="warning", source="price_monitor", title="批量拉取失败", detail=str(e)[:200])
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
results = {}
|
results = {}
|
||||||
@@ -149,6 +154,10 @@ def refresh_data_prices():
|
|||||||
conn.close()
|
conn.close()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"⚠️ 从DB读代码失败: {e}", file=sys.stderr)
|
print(f"⚠️ 从DB读代码失败: {e}", file=sys.stderr)
|
||||||
|
try:
|
||||||
|
record_alert(level="warning", source="price_monitor", title="读取持仓失败", detail=str(e)[:200])
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
if not all_codes:
|
if not all_codes:
|
||||||
@@ -295,6 +304,10 @@ def refresh_data_prices():
|
|||||||
except Exception: pass
|
except Exception: pass
|
||||||
conn = None
|
conn = None
|
||||||
print(f"⚠️ DB同步异常: {e}", file=sys.stderr)
|
print(f"⚠️ DB同步异常: {e}", file=sys.stderr)
|
||||||
|
try:
|
||||||
|
record_alert(level="error", source="price_monitor", title="DB同步异常", detail=str(e)[:200])
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
break
|
break
|
||||||
else:
|
else:
|
||||||
# for-else: loop exhausted without break
|
# for-else: loop exhausted without break
|
||||||
@@ -537,7 +550,10 @@ def run_once(round_label=""):
|
|||||||
record_event(code, name, "swing_re", price, str(_ma10))
|
record_event(code, name, "swing_re", price, str(_ma10))
|
||||||
_push_action("波段再进", f"📈 {name}({code}) {price} 收回MA10({_ma10:.2f})且突破前一日高点 → 波段再进,可接回")
|
_push_action("波段再进", f"📈 {name}({code}) {price} 收回MA10({_ma10:.2f})且突破前一日高点 → 波段再进,可接回")
|
||||||
except Exception as _e:
|
except Exception as _e:
|
||||||
pass
|
try:
|
||||||
|
record_alert(level="warning", source="price_monitor", title="波段检测异常", detail=str(_e)[:200], code=code)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
# 时间预算检查:如果超时,跳过重评只做状态记录
|
# 时间预算检查:如果超时,跳过重评只做状态记录
|
||||||
_budget_low = (time.time() - start) > TIME_BUDGET
|
_budget_low = (time.time() - start) > TIME_BUDGET
|
||||||
@@ -578,6 +594,10 @@ def run_once(round_label=""):
|
|||||||
outputs.append(f" 📨 止损重评→已推送Dad: {action}")
|
outputs.append(f" 📨 止损重评→已推送Dad: {action}")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
outputs.append(f" ⚠️ 止损重评失败: {e}")
|
outputs.append(f" ⚠️ 止损重评失败: {e}")
|
||||||
|
try:
|
||||||
|
record_alert(level="error", source="price_monitor", title="止损重评失败", detail=str(e)[:200], code=code)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
else:
|
else:
|
||||||
extra = ""
|
extra = ""
|
||||||
if "_price" in key:
|
if "_price" in key:
|
||||||
@@ -626,6 +646,10 @@ def run_once(round_label=""):
|
|||||||
outputs.append(f" 📋 本地日志(不推): {reason}")
|
outputs.append(f" 📋 本地日志(不推): {reason}")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
outputs.append(f" ⚠️ 区间重评失败: {e}")
|
outputs.append(f" ⚠️ 区间重评失败: {e}")
|
||||||
|
try:
|
||||||
|
record_alert(level="error", source="price_monitor", title="区间重评失败", detail=str(e)[:200], code=code)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
state[code][key] = True
|
state[code][key] = True
|
||||||
state_updated = True
|
state_updated = True
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user