feat: XMPP health auto-collection every 5min via cron pipeline
This commit is contained in:
+26
-21
@@ -5,6 +5,7 @@ agents_health_check.py — MoFin Tier1 快速健康检查
|
||||
====================================================
|
||||
每 5 分钟运行一次(crontab)。检查关键服务的端口/HTTP/DB 可用性。
|
||||
全正常时静默(不输出)。异常时写入 TODO 文件和 JSON 报告。
|
||||
同时采集 XMPP 通道健康数据到时间序列日志。
|
||||
|
||||
部署: crontab */5 * * * * cd /home/hmo/MoFin && python3 agents_health_check.py
|
||||
"""
|
||||
@@ -33,6 +34,23 @@ SERVICES = [
|
||||
{"name": "mofin_db", "label": "MoFin 数据库", "host": "127.0.0.1", "port": 0, "type": "db", "check": "/home/hmo/web-dashboard/data/mofin.db"},
|
||||
]
|
||||
|
||||
# ── XMPP 通道健康采集 ──────────────────────────────
|
||||
|
||||
XMPP_HEALTH_LOG = LOGS_DIR / "xmpp_health_log.jsonl"
|
||||
|
||||
|
||||
def _collect_xmpp_health(now):
|
||||
"""采集 XMPP 通道健康数据,追加到时间序列日志。"""
|
||||
try:
|
||||
from xmpp_logger import health as xmpp_health
|
||||
h = xmpp_health()
|
||||
except Exception as e:
|
||||
h = {"status": "collector_error", "error": str(e)[:200]}
|
||||
entry = {"timestamp": now.strftime("%Y-%m-%d %H:%M:%S"), **h}
|
||||
XMPP_HEALTH_LOG.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(XMPP_HEALTH_LOG, "a", encoding="utf-8") as f:
|
||||
f.write(json.dumps(entry, ensure_ascii=False) + "\n")
|
||||
|
||||
|
||||
# ---- Checkers ----
|
||||
|
||||
@@ -83,57 +101,44 @@ def run():
|
||||
ok, detail = False, "unknown type"
|
||||
|
||||
results.append({
|
||||
"name": svc["name"],
|
||||
"label": svc["label"],
|
||||
"type": svc["type"],
|
||||
"port": svc["port"],
|
||||
"health": {"ok": ok},
|
||||
"detail": detail,
|
||||
"name": svc["name"], "label": svc["label"],
|
||||
"type": svc["type"], "port": svc["port"],
|
||||
"health": {"ok": ok}, "detail": detail,
|
||||
})
|
||||
|
||||
if not ok:
|
||||
issues.append(svc)
|
||||
|
||||
# Write report
|
||||
report = {
|
||||
"services": results,
|
||||
"summary": {
|
||||
"ok": sum(1 for r in results if r["health"]["ok"]),
|
||||
"total": len(results),
|
||||
},
|
||||
"summary": {"ok": sum(1 for r in results if r["health"]["ok"]), "total": len(results)},
|
||||
"generated_at": now.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
}
|
||||
with open(REPORT_FILE, "w", encoding="utf-8") as f:
|
||||
json.dump(report, f, ensure_ascii=False, indent=2)
|
||||
|
||||
# ── XMPP 通道健康采集 ──
|
||||
_collect_xmpp_health(now)
|
||||
|
||||
# Handle issues
|
||||
if issues:
|
||||
# Write TODO entries
|
||||
with open(TODO_FILE, "a", encoding="utf-8") as f:
|
||||
for svc in issues:
|
||||
entry = {
|
||||
"service": svc["name"],
|
||||
"label": svc["label"],
|
||||
"service": svc["name"], "label": svc["label"],
|
||||
"reason": next((r["detail"] for r in results if r["name"] == svc["name"]), "unknown"),
|
||||
"timestamp": now.isoformat(),
|
||||
}
|
||||
f.write(json.dumps(entry, ensure_ascii=False) + "\n")
|
||||
|
||||
# Log to file
|
||||
with open(LOG_FILE, "a", encoding="utf-8") as f:
|
||||
f.write(f"[{now.strftime('%Y-%m-%d %H:%M:%S')}] ISSUES: {len(issues)} failed\n")
|
||||
for svc in issues:
|
||||
detail = next((r["detail"] for r in results if r["name"] == svc["name"]), "")
|
||||
f.write(f" - {svc['label']}: {detail}\n")
|
||||
|
||||
# Print to stdout (visible in cron log)
|
||||
print(f"[{now.strftime('%H:%M')}] Health check: {len(issues)}/{len(SERVICES)} services failed")
|
||||
for svc in issues:
|
||||
detail = next((r["detail"] for r in results if r["name"] == svc["name"]), "")
|
||||
print(f" FAIL: {svc['label']} ({svc['name']}) — {detail}")
|
||||
else:
|
||||
# All OK → silent
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user