From 57377e9dd39b84e3a17ee922a0f8f7f6b71c6399 Mon Sep 17 00:00:00 2001 From: hmo Date: Mon, 20 Jul 2026 01:12:38 +0800 Subject: [PATCH] feat(auto_heal): multi-profile key switching + fix two error classes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Error investigation (from restored monitoring) found 2 root causes: 1. 'Blocked: script path resolves outside scripts dir' (5+ cron jobs): Jul 17 symlink refactor replaced real scripts with symlinks; the hermes cron scheduler's security check (Path.resolve + relative_to) rejects symlink escape. ALL no_agent script jobs blocked since Jul 17 23:12. FIX: converted 102 symlinks to hardlinks (same inode, resolve() stays inside scripts_dir, single-source still works). Permanent structural fix. 2. HTTP 429 on default profile (知识研究/梦境循环/wiki-self-growth/ evolution-pulse/大脑任务执行): default gateway used ocg-key1 (weekly 100%). FIX: switched default profile to ocg-key6 + added missing provider block. LLM verified working (3.1s). auto_heal extended to actually cover these automatically next time: - PROFILES registry: zhiwei (8643, system svc) + default (8642, user svc) - current_provider/switch_key/_scan_agent_log parameterized by profile - health() now reports llm_provider_default (agent.log scan) - auto_heal: per-profile 429 detection -> best_key -> switch_key(profile) - _ensure_provider_block: injects missing provider credentials from zhiwei config (single source of truth) into target config --- scripts/analyze_health.py | 45 +++++++ scripts/check_missing_scripts.py | 31 +++++ scripts/cron_status.py | 31 +++++ scripts/find_cron_errors.py | 15 +++ scripts/fix_symlinks.py | 33 ++++++ scripts/test_multi_profile.py | 22 ++++ scripts/validate_fixes.py | 37 ++++++ xmpp_logger.py | 197 +++++++++++++++++++------------ 8 files changed, 338 insertions(+), 73 deletions(-) create mode 100644 scripts/analyze_health.py create mode 100644 scripts/check_missing_scripts.py create mode 100644 scripts/cron_status.py create mode 100644 scripts/find_cron_errors.py create mode 100644 scripts/fix_symlinks.py create mode 100644 scripts/test_multi_profile.py create mode 100644 scripts/validate_fixes.py diff --git a/scripts/analyze_health.py b/scripts/analyze_health.py new file mode 100644 index 00000000..f23c7859 --- /dev/null +++ b/scripts/analyze_health.py @@ -0,0 +1,45 @@ +import json + +d = json.load(open('/tmp/mofin_health.json')) + +print("generated_at:", d.get('generated_at')) +print() + +# 1. Pipelines with error/warn status +print("=== PIPELINES (error/warn) ===") +for p in d.get('pipelines', []): + if p.get('status') in ('error', 'fail', 'warn'): + print(f"[{p['status']}] {p.get('name') or p.get('script')} | profile={p.get('profile')} | type={p.get('type')} | schedule={p.get('schedule')} | last_run={p.get('last_run')}") + +print() +print("=== FEATURE TREE (non-ok nodes) ===") +def walk(n, path=''): + label = n.get('label', '?') + p = f"{path}/{label}" + if n.get('status') not in ('ok', None): + print(f"[{n.get('status')}] {p}") + for pipe in n.get('pipes', []): + if pipe.get('status') != 'ok': + print(f" pipe: [{pipe.get('status')}] {pipe.get('name') or pipe.get('script')} | last_run={pipe.get('last_run')}") + for c in n.get('children', []): + walk(c, p) +walk(d.get('feature_tree', {})) + +print() +print("=== DATA ENTITIES (orphan/write_only) ===") +for e in d.get('entities', []): + if e.get('flow_status') in ('orphan', 'write_only'): + print(f"[{e['flow_status']}] {e['name']} | rows={e.get('rows')} | writers={e.get('writers')} | readers={e.get('readers')} | {e.get('desc','')[:60]}") + +print() +print("=== ARCHITECTURE violations ===") +arch = d.get('architecture', {}) +print("violation_count:", arch.get('violation_count')) +for v in (arch.get('price_api_violations') or [])[:10]: + print(f" {v.get('script')} L{v.get('line')}") + +print() +print("=== JSON files with warn ===") +for j in d.get('json_files', []): + if j.get('warn'): + print(f"[warn] {j['name']} | {j.get('desc','')[:60]} | readers={j.get('readers')}") \ No newline at end of file diff --git a/scripts/check_missing_scripts.py b/scripts/check_missing_scripts.py new file mode 100644 index 00000000..f181a989 --- /dev/null +++ b/scripts/check_missing_scripts.py @@ -0,0 +1,31 @@ +import os + +files = ['meta_growth.py', 'macro_context_collector.py', 'divergence_detector.py', + 'memory_guardian.py', 'fix_gateway_port.py'] +d = '/home/hmo/.hermes/profiles/position-analyst/scripts/' +for f in files: + p = os.path.join(d, f) + if os.path.islink(p): + target = os.readlink(p) + ok = os.path.exists(p) + print(f"{f} -> symlink -> {target} [{'OK' if ok else 'BROKEN'}]") + elif os.path.exists(p): + print(f"{f} -> real file") + else: + print(f"{f} -> MISSING") + +print() +print('=== deploy/profile-scripts candidates ===') +deploy = '/home/hmo/MoFin/deploy/profile-scripts/' +for f in os.listdir(deploy): + if any(k in f for k in ['meta_growth', 'macro_context', 'divergence', 'memory_guardian', 'fix_gateway']): + print(' ', f) + +print() +print('=== search whole MoFin for the 5 scripts ===') +import subprocess +for f in files: + r = subprocess.run(['find', '/home/hmo/MoFin', '/home/hmo/projects', '-name', f, + '-not', '-path', '*/venv/*'], capture_output=True, text=True, timeout=30) + found = [l for l in r.stdout.splitlines() if l.strip()] + print(f"{f}: {found if found else 'NOT FOUND'}") \ No newline at end of file diff --git a/scripts/cron_status.py b/scripts/cron_status.py new file mode 100644 index 00000000..1b5de1d1 --- /dev/null +++ b/scripts/cron_status.py @@ -0,0 +1,31 @@ +import json +from datetime import datetime, timezone, timedelta + +d = json.load(open('/home/hmo/.hermes/profiles/position-analyst/cron/jobs.json')) +jobs = d if isinstance(d, list) else d.get('jobs', []) +now = datetime.now(timezone.utc) +print(f"{'name':32} {'last_run':20} {'status':8} {'next_run':20} {'en'}") +print('-' * 95) +for j in sorted(jobs, key=lambda x: x.get('last_run_at') or ''): + if not j.get('enabled', True): + continue + lr = (j.get('last_run_at') or '?')[:19] + nr = (j.get('next_run_at') or '?')[:19] + st = str(j.get('last_status', '?')) + print(f"{j.get('name','?')[:32]:32} {lr:20} {st:8} {nr:20} {j.get('enabled')}") + +# also check gateway-side cron (default profile) +print() +print('=== default profile cron ===') +try: + d2 = json.load(open('/home/hmo/.hermes/cron/jobs.json')) + jobs2 = d2 if isinstance(d2, list) else d2.get('jobs', []) + for j in sorted(jobs2, key=lambda x: x.get('last_run_at') or ''): + if not j.get('enabled', True): + continue + lr = (j.get('last_run_at') or '?')[:19] + nr = (j.get('next_run_at') or '?')[:19] + st = str(j.get('last_status', '?')) + print(f"{j.get('name','?')[:32]:32} {lr:20} {st:8} {nr:20} {j.get('enabled')}") +except Exception as e: + print('err:', e) \ No newline at end of file diff --git a/scripts/find_cron_errors.py b/scripts/find_cron_errors.py new file mode 100644 index 00000000..eda4f02e --- /dev/null +++ b/scripts/find_cron_errors.py @@ -0,0 +1,15 @@ +import json, os, glob + +# find job ids for the failing jobs +targets = { + 'pa': ('/home/hmo/.hermes/profiles/position-analyst/cron/jobs.json', + ['大脑任务执行', 'Gateway看门狗-知微', '元自成长-每日', '记忆守卫-每日', '宏观新闻采集', '跨市场背离检测']), + 'default': ('/home/hmo/.hermes/cron/jobs.json', + ['大脑任务执行', 'evolution-pulse', 'wiki-self-growth', '知识研究-日常', '梦境循环-知识库归并']), +} +for label, (jf, names) in targets.items(): + d = json.load(open(jf)) + jobs = d if isinstance(d, list) else d.get('jobs', []) + for j in jobs: + if j.get('name') in names: + print(f"{label} | {j['name']} | id={j.get('id')} | status={j.get('last_status')} | error={str(j.get('last_error'))[:200]}") \ No newline at end of file diff --git a/scripts/fix_symlinks.py b/scripts/fix_symlinks.py new file mode 100644 index 00000000..69c10a49 --- /dev/null +++ b/scripts/fix_symlinks.py @@ -0,0 +1,33 @@ +import os, sys + +scripts_dir = '/home/hmo/.hermes/profiles/position-analyst/scripts' +fixed, skipped, errors = 0, 0, [] +for name in os.listdir(scripts_dir): + p = os.path.join(scripts_dir, name) + if not os.path.islink(p): + skipped += 1 + continue + target = os.path.realpath(p) + if not os.path.exists(target): + errors.append(f"{name}: broken symlink -> {target}") + continue + try: + os.unlink(p) + os.link(target, p) # hardlink: same inode, .resolve() stays in scripts_dir + fixed += 1 + except Exception as e: + errors.append(f"{name}: {e}") + +print(f"fixed: {fixed}, skipped (non-symlink): {skipped}, errors: {len(errors)}") +for e in errors: + print(' ', e) + +# verify with the scheduler's own check +from pathlib import Path +scripts_resolved = Path(scripts_dir).resolve() +test = (Path(scripts_dir) / 'meta_growth.py').resolve() +try: + test.relative_to(scripts_resolved) + print('VERIFY OK: meta_growth.py now passes relative_to check') +except ValueError: + print('VERIFY FAIL: still resolves outside') \ No newline at end of file diff --git a/scripts/test_multi_profile.py b/scripts/test_multi_profile.py new file mode 100644 index 00000000..01d40aaf --- /dev/null +++ b/scripts/test_multi_profile.py @@ -0,0 +1,22 @@ +import ast, sys, json +p = '/home/hmo/MoFin/xmpp_logger.py' +try: + ast.parse(open(p).read()) + print('SYNTAX OK') +except SyntaxError as e: + print('SYNTAX ERROR:', e) + sys.exit(1) + +sys.path.insert(0, '/home/hmo/MoFin') +import importlib, xmpp_logger +importlib.reload(xmpp_logger) +x = xmpp_logger +import time + +print('== current_provider(zhiwei):', x.current_provider('zhiwei')) +print('== current_provider(default):', x.current_provider('default')) +print('== scan zhiwei:', json.dumps(x._scan_agent_log(time.time(), 'zhiwei'), ensure_ascii=False)) +print('== scan default:', json.dumps(x._scan_agent_log(time.time(), 'default'), ensure_ascii=False)) +print() +print('== auto_heal ==') +print(json.dumps(x.auto_heal(), ensure_ascii=False, indent=2)) \ No newline at end of file diff --git a/scripts/validate_fixes.py b/scripts/validate_fixes.py new file mode 100644 index 00000000..442bc2be --- /dev/null +++ b/scripts/validate_fixes.py @@ -0,0 +1,37 @@ +import json, urllib.request +from pathlib import Path + +print("=== 验证1: 5个被 Blocked 的脚本现在是否通过调度器路径检查 ===") +scripts_dir = Path('/home/hmo/.hermes/profiles/position-analyst/scripts') +resolved = scripts_dir.resolve() +files = ['meta_growth.py', 'macro_context_collector.py', 'divergence_detector.py', + 'memory_guardian.py', 'fix_gateway_port.py'] +all_ok = True +for f in files: + p = (scripts_dir / f).resolve() + try: + p.relative_to(resolved) + exists = p.exists() + print(f" PASS {f} (exists={exists})") + if not exists: + all_ok = False + except ValueError: + all_ok = False + print(f" FAIL {f} still blocked") +print(' =>', 'ALL PASS' if all_ok else 'STILL FAILING') + +print() +print("=== 验证2: default gateway (8642) LLM 调用（原 key1 429，现 key6）===") +payload = json.dumps({'model': 'deepseek-v4-flash', + 'messages': [{'role': 'user', 'content': 'reply with one word: ok'}], + 'max_tokens': 10}).encode() +req = urllib.request.Request('http://127.0.0.1:8642/v1/chat/completions', data=payload, + headers={'Content-Type': 'application/json', + 'Authorization': 'Bearer hermes123'}) +try: + resp = urllib.request.urlopen(req, timeout=90) + d = json.loads(resp.read().decode()) + content = d.get('choices', [{}])[0].get('message', {}).get('content', '') + print(' LLM OK:', content[:100]) +except Exception as e: + print(' LLM FAIL:', e) \ No newline at end of file diff --git a/xmpp_logger.py b/xmpp_logger.py index f316385e..5d329ca5 100644 --- a/xmpp_logger.py +++ b/xmpp_logger.py @@ -19,9 +19,26 @@ LOG_DIR = Path(__file__).resolve().parent / "gateway" / "logs" LOG_FILE = LOG_DIR / "xmpp_messages.jsonl" MAX_AGE_DAYS = 7 -# Position-analyst profile config path (where model.provider is set) -HERMES_CONFIG = Path("/home/hmo/.hermes/profiles/position-analyst/config.yaml") -GATEWAY_SERVICE = "hermes-gateway-zhiwei.service" +# 监控的 Hermes Gateway profiles +PROFILES = { + "zhiwei": { + "config": Path("/home/hmo/.hermes/profiles/position-analyst/config.yaml"), + "agent_log": Path("/home/hmo/.hermes/profiles/position-analyst/logs/agent.log"), + "service": "hermes-gateway-zhiwei.service", + "user_service": False, + "gateway_port": 8643, + }, + "default": { + "config": Path("/home/hmo/.hermes/config.yaml"), + "agent_log": Path("/home/hmo/.hermes/logs/agent.log"), + "service": "hermes-gateway.service", + "user_service": True, + "gateway_port": 8642, + }, +} +# 向后兼容（旧引用） +HERMES_CONFIG = PROFILES["zhiwei"]["config"] +GATEWAY_SERVICE = PROFILES["zhiwei"]["service"] RESTART_COOLDOWN_FILE = LOG_DIR / "last_restart.txt" RESTART_COOLDOWN_SEC = 180 # 3 min: don't trigger another restart within 3 min of last @@ -38,15 +55,16 @@ KEY_TO_PROVIDER = { } -def current_provider() -> str | None: +def current_provider(profile="zhiwei") -> str | None: """Read current model.provider from Hermes config. Walks the file tracking the model: block to find its nested `provider:` (avoiding other blocks like `agent.alerts[0].provider:`). """ + cfg_path = PROFILES.get(profile, PROFILES["zhiwei"])["config"] try: in_model = False - for line in HERMES_CONFIG.read_text().splitlines(): + for line in cfg_path.read_text().splitlines(): stripped = line.rstrip() # Detect top-level "model:" at column 0 if stripped == "model:" or stripped.startswith("model:") and not line.startswith(" "): @@ -65,50 +83,74 @@ def current_provider() -> str | None: return None -def switch_key(key_id: str) -> dict: +def _ensure_provider_block(target_cfg: Path, provider: str) -> bool: + """若目标 config 缺少 provider 凭据块，从 zhiwei config（凭据单一事实源）复制注入。""" + txt = target_cfg.read_text() + if _re.search(rf"^ {provider}:\s*$", txt, flags=_re.MULTILINE): + return True + src = PROFILES["zhiwei"]["config"].read_text() + m = _re.search(rf"(^ {provider}:\n(?: .+\n)+)", src, flags=_re.MULTILINE) + if not m: + return False + block = m.group(1) + # 插到 providers: 区块第一个 provider 之前 + m2 = _re.search(r"^providers:\s*$", txt, flags=_re.MULTILINE) + if not m2: + return False + txt = txt[:m2.end()] + "\n" + block + txt[m2.end():] + target_cfg.write_text(txt) + return True + + +def _restart_gateway(profile: str) -> dict: + """异步重启指定 profile 的 gateway。""" + p = PROFILES[profile] + try: + if p["user_service"]: + _sp.Popen(["systemctl", "--user", "restart", p["service"]], + stdout=_sp.DEVNULL, stderr=_sp.DEVNULL, start_new_session=True) + else: + _sp.Popen(["sudo", "-n", "systemctl", "restart", p["service"]], + stdout=_sp.DEVNULL, stderr=_sp.DEVNULL, start_new_session=True) + return {"success": True, "detail": "restart triggered (async)"} + except Exception as e: + return {"success": False, "detail": str(e)[:100]} + + +def switch_key(key_id: str, profile: str = "zhiwei") -> dict: """Switch Hermes config to use the given key_id's provider, then restart Gateway.""" provider = KEY_TO_PROVIDER.get(key_id) if not provider: return {"switched": False, "detail": f"unknown key_id {key_id}"} - old = current_provider() + cfg_path = PROFILES[profile]["config"] + old = current_provider(profile) if old == provider: return {"switched": False, "old": old, "new": provider, "detail": "already on this key"} try: - txt = HERMES_CONFIG.read_text() + if not _ensure_provider_block(cfg_path, provider): + return {"switched": False, "detail": f"provider {provider} 凭据块缺失且无法注入"} + txt = cfg_path.read_text() new_txt, n = _re.subn(r"^ provider:\s*\S+\s*$", f" provider: {provider}", txt, count=1, flags=_re.MULTILINE) if n == 0: return {"switched": False, "detail": "no provider: line found"} - HERMES_CONFIG.write_text(new_txt) + cfg_path.write_text(new_txt) except Exception as e: return {"switched": False, "detail": f"config edit failed: {e}"} - # systemctl restart blocks until gateway drain completes (60+s). - # Fire asynchronously via Popen to avoid blocking cron; verify later. - try: - _sp.Popen(["sudo", "-n", "systemctl", "restart", GATEWAY_SERVICE], - stdout=_sp.DEVNULL, stderr=_sp.DEVNULL, start_new_session=True) - RESTART_COOLDOWN_FILE.write_text(str(_time.time())) - ok = True - detail = "restart triggered (async)" - except Exception as e: - ok = False - detail = f"restart trigger failed: {e}" - - # Wait long enough for systemd to drain + restart the gateway (max 90s) - _time.sleep(45) - verify = _verify_llm() + r = _restart_gateway(profile) + RESTART_COOLDOWN_FILE.write_text(str(_time.time())) return { "switched": True, "old": old, "new": provider, "key_id": key_id, - "restart": ok, - "verify": verify, - "detail": detail, + "profile": profile, + "restart": r["success"], + "detail": r["detail"], } @@ -340,7 +382,9 @@ def health(): # LLM 健康：扫 gateway agent.log 最近一次真实调用结果（零成本，不发 LLM 请求） # agent.log 里每次真实调用都有记录：成功 "API call #N: ... latency=Xs" / 失败 "HTTP 429..." - result["llm_provider"] = _scan_agent_log(now_epoch) + result["llm_provider"] = _scan_agent_log(now_epoch, "zhiwei") + # default profile（莫荷 8642）同样监控——它的 429 曾导致整组 cron 失败 + result["llm_provider_default"] = _scan_agent_log(now_epoch, "default") except Exception as e: result["gateways"] = {"error": str(e)[:100]} @@ -365,22 +409,20 @@ def health(): return result -AGENT_LOG = Path("/home/hmo/.hermes/profiles/position-analyst/logs/agent.log") - - -def _scan_agent_log(now_epoch, tail_lines=300): +def _scan_agent_log(now_epoch, profile="zhiwei", tail_lines=300): """扫 gateway agent.log 尾部，取最近一次真实 LLM 调用的结果。 成功行: "2026-07-19 20:04:42,880 INFO ... API call #4: ... latency=16.0s ..." 失败行: "2026-07-19 19:35:59 WARNING ... API call failed ... HTTP 429: Weekly usage limit reached ..." 返回 {"status": "ok"|"error"|"unknown", "latency"/"error", "age_sec"} """ - if not AGENT_LOG.exists(): + log_path = PROFILES.get(profile, PROFILES["zhiwei"])["agent_log"] + if not log_path.exists(): return {"status": "unknown", "error": "agent.log not found"} try: # 只读尾部（大文件不全读） - size = AGENT_LOG.stat().st_size - with open(AGENT_LOG, "rb") as f: + size = log_path.stat().st_size + with open(log_path, "rb") as f: f.seek(max(0, size - 64 * 1024)) chunk = f.read().decode("utf-8", errors="replace") lines = [l for l in chunk.splitlines() if l.strip()][-tail_lines:] @@ -484,53 +526,62 @@ def auto_heal(): h = health() actions = [] - # 0. LLM Provider 异常 → 先查是否有更好的 Key 可切换 - llm_status = h.get("llm_provider", {}).get("status") - llm_error = h.get("llm_provider", {}).get("error", "") + # 0. 检查每个 profile 的 LLM 状态（zhiwei + default），429 → 切 key + bk_cache = None + for profile in ("zhiwei", "default"): + llm = h.get("llm_provider" if profile == "zhiwei" else "llm_provider_default", {}) + llm_status = llm.get("status") + llm_error = llm.get("error", "") or "" - # Detect rate-limit / timeout / error - is_rate_limited = (llm_status == "rate_limited" or - "429" in llm_error or "Weekly usage" in llm_error or - "RateLimit" in llm_error) - is_timeout_or_error = llm_status in ("timeout", "error") + is_rate_limited = (llm_status == "rate_limited" or + "429" in llm_error or "Weekly usage" in llm_error or + "RateLimit" in llm_error) + if not is_rate_limited: + continue - if is_rate_limited or is_timeout_or_error: - bk = best_key() - if bk: + if bk_cache is None: + bk_cache = best_key() + bk = bk_cache + if not bk: + continue + actions.append({ + "action": "check_keys", + "profile": profile, + "best_key": bk["key_id"], + "current_provider": current_provider(profile), + "best_provider": KEY_TO_PROVIDER.get(bk["key_id"]), + "weekly_pct": bk["weekly"]["usage_percent"], + }) + + target_provider = KEY_TO_PROVIDER.get(bk["key_id"]) + current = current_provider(profile) + if target_provider is None: actions.append({ - "action": "check_keys", - "best_key": bk["key_id"], - "current_provider": current_provider(), - "best_provider": KEY_TO_PROVIDER.get(bk["key_id"]), - "rolling_pct": bk["rolling"]["usage_percent"], - "weekly_pct": bk["weekly"]["usage_percent"], - "issues": bk["issues"], + "action": "skip_switch", + "profile": profile, + "reason": f"best key {bk['key_id']} is non-OCG (kimi), stay on deepseek", }) - - # 如果 best key 对应的 provider 跟当前不同 → 切换 - target_provider = KEY_TO_PROVIDER.get(bk["key_id"]) - current = current_provider() - if target_provider is None: - # best key is a non-OCG provider (e.g. key7=kimi) — never switch zhiwei there + elif target_provider != current: + if bk["weekly"]["status"] == "ok": + action = switch_key(bk["key_id"], profile=profile) + action["action_group"] = "switch_key" + action["profile"] = profile + actions.append(action) + else: actions.append({ "action": "skip_switch", - "reason": f"best key {bk['key_id']} is non-OCG (not in KEY_TO_PROVIDER), zhiwei must stay on deepseek", - "best_key": bk["key_id"], + "profile": profile, + "reason": f"best key {bk['key_id']} weekly status {bk['weekly']['status']}, no improvement", }) - elif target_provider != current: - # 只有当 best key 自身至少 weekly ok 才切，否则切过去也白搭 - if bk["weekly"]["status"] == "ok": - action = switch_key(bk["key_id"]) - action["action_group"] = "switch_key" - actions.append(action) - else: - actions.append({ - "action": "skip_switch", - "reason": f"best key {bk['key_id']} weekly status {bk['weekly']['status']}, no improvement", - }) + + # zhiwei LLM timeout/error（非429）→ 重启 Gateway + llm_status = h.get("llm_provider", {}).get("status") + llm_error = h.get("llm_provider", {}).get("error", "") or "" + is_rate_limited_zw = ("429" in llm_error or "Weekly usage" in llm_error or "RateLimit" in llm_error) + is_timeout_or_error = llm_status in ("timeout", "error") # 1. LLM Provider timeout/error（非429）→ 重启 Gateway（不是切 key） - if is_timeout_or_error and not is_rate_limited: + if is_timeout_or_error and not is_rate_limited_zw: gw = h.get("gateways", {}).get("zhiwei", {}) if gw.get("alive"): # Check cooldown to avoid restart loops