MythEclipse/pr-agent-server
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(health-check): read key from on-disk file first, no BWS dependency

Browse Source 
Root cause: health-check binary runs inside a Nix venv that doesn't have
/usr/local/bin/bws on PATH. When the shell wrapper's BWS_ACCESS_TOKEN
export fails (e.g. sudo unavailable, gateway lacks bws group), get_key()
returns empty → false 'MODELS FAILING' alert.

Fix: read the router API key from the on-disk omniroute_key file first
(maintained by sync-key.py on every service start via ExecStartPre —
always current, zero subprocess/BWS dependency). Fall back to BWS CLI
only if the file is missing/stale.

Also: all config values now read from env vars (no hardcoded paths),
alert message clarified to indicate both resolution paths failed.
This commit is contained in:
asepharyana
2026-09-09 21:05:10 +07:00
parent 33928c78f5
commit ae5356a4ea
1 changed files with 77 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/health-check.py
+77 -17
View File
@@ -13,39 +13,84 @@ Design: alert only when EVERY configured model fails (primary AND all
fallbacks). If any model works, the server's own fallback chain will succeed,
so the system is healthy even if the primary is down/slow. This prevents
false alerts from a single slow/failed model.
Key resolution order (no hardcoding):
1. On-disk key file maintained by sync-key.py (source of truth for the
running server — always current after every service start/restart).
2. BWS_ACCESS_TOKEN env var (shell wrapper or gateway-injected).
3. /etc/bws-token file → bws secret get (with sudo fallback for
non-root processes).
"""
import os, sys, json, hashlib, subprocess
from pathlib import Path
BWS_SECRET_ID = "2aef2194-971d-4dae-99dd-b49a0041f97c"
ROUTER_BASE = "https://9router.asepharyana.my.id/v1"
PRIMARY = "openai/claude-opus-5"
FALLBACKS = ["openai/claude-sonnet-5", "openai/claude-haiku-4-5-20251001", "openai/ATLAS", "openai/gemini", "openai/text", "openai/deepseek-v4-flash-free"]
# Configurable paths — no hardcoding; everything reads from env or known
# locations that the Nix build / systemd unit define.
BWS_SECRET_ID = os.environ.get(
"BWS_ROUTER_KEY_SECRET_ID", "2aef2194-971d-4dae-99dd-b49a0041f97c"
)
ROUTER_BASE = os.environ.get(
"ROUTER_BASE_URL", "https://9router.asepharyana.my.id/v1"
)
PRIMARY = os.environ.get("HEALTH_CHECK_PRIMARY_MODEL", "openai/claude-opus-5")
FALLBACKS = os.environ.get(
"HEALTH_CHECK_FALLBACK_MODELS",
"openai/claude-sonnet-5,openai/claude-haiku-4-5-20251001,openai/ATLAS,openai/gemini,openai/text,openai/deepseek-v4-flash-free"
).split(",")
# Caddy 9router route is now response_header_timeout 120s / read 300s.
# LLM combo TTFT often 30-40s+. Give the check room to complete.
HTTP_TIMEOUT = 150
CONSECUTIVE_FAIL_FILE = Path("/tmp/pr-agent-health-fail-count")
HTTP_TIMEOUT = int(os.environ.get("HEALTH_CHECK_HTTP_TIMEOUT", "150"))
CONSECUTIVE_FAIL_FILE = Path(
os.environ.get("HEALTH_CHECK_FAIL_COUNT_FILE", "/tmp/pr-agent-health-fail-count")
)
# ── key resolution ──────────────────────────────────────────────────────────
# On-disk key file — maintained by sync-key.py (runs on every service start
# via systemd ExecStartPre). This is the SAME key the server uses, always
# current, no BWS dependency.
APP_DIR = Path(os.environ.get("PR_AGENT_APP_DIR", "/var/lib/pr-agent-server"))
KEYFILE = APP_DIR / "omniroute_key"
def _read_key_from_disk() -> str:
"""Read the router key from the on-disk file maintained by sync-key.py.
This is the primary source — always current after service start."""
try:
if KEYFILE.is_file():
key = KEYFILE.read_text().strip()
if len(key) >= 10:
return key
except (PermissionError, OSError):
pass
return ""
# ── key from BWS ────────────────────────────────────────────────────────────
def _read_token() -> str:
"""Read BWS token. Direct read fails for non-root (root:bws 640), so fall
back to `sudo -n cat` (cron user `code` is in sudo group, NOPASSWD)."""
"""Read BWS access token. Direct read fails for non-root (root:bws 640),
so fall back to `sudo -n cat` (cron user `code` is in sudo group, NOPASSWD)."""
# Try direct read first (works when gateway has bws group)
for path in (Path("/etc/bws-token"),):
try:
if path.is_file():
return path.read_text().strip()
except PermissionError:
pass
# Fallback: sudo (works when user has NOPASSWD sudo)
try:
r = subprocess.run(["sudo", "-n", "cat", "/etc/bws-token"],
capture_output=True, text=True, timeout=10)
r = subprocess.run(
["sudo", "-n", "cat", "/etc/bws-token"],
capture_output=True, text=True, timeout=10,
)
if r.returncode == 0:
return r.stdout.strip()
except Exception:
pass
return ""
def get_key() -> str:
def _fetch_key_from_bws() -> str:
"""Fetch the router key from BWS via the bws CLI."""
token = os.environ.get("BWS_ACCESS_TOKEN", "")
if not token:
token = _read_token()
@@ -59,9 +104,7 @@ def get_key() -> str:
)
if r.returncode != 0:
return ""
# Value is shell-quoted KEY="value" — take first line only. BWS sometimes
# appends "# one or more secrets have been commented-out..."; only the
# first line is the real key value.
# Value is shell-quoted KEY="value" — take first line only.
line = r.stdout.split("\n")[0]
if "=" not in line:
return ""
@@ -73,7 +116,21 @@ def get_key() -> str:
return ""
def get_key() -> str:
"""Resolve the router API key. Order: on-disk file → BWS CLI.
The on-disk file is always current (sync-key runs on every service start)
and has zero external dependencies — preferred path for the health check."""
# 1. On-disk file (fast, no subprocess, no BWS dependency)
key = _read_key_from_disk()
if key:
return key
# 2. BWS CLI fallback (for edge cases where the file is missing/stale)
key = _fetch_key_from_bws()
return key
# ── health check ────────────────────────────────────────────────────────────
def check_model(model: str, key: str) -> tuple:
"""Returns (ok: bool, detail: str). Uses raw HTTP (no litellm dependency).
@@ -108,7 +165,10 @@ def check_model(model: str, key: str) -> tuple:
def main() -> int:
key = get_key()
if not key:
print("⚠️ pr-agent health: cannot fetch router key from BWS (bws unavailable)")
print(
"⚠️ pr-agent health: cannot resolve router key "
"(on-disk file missing and BWS unavailable)"
)
return 1
results = {}
@@ -154,4 +214,4 @@ def main() -> int:
if __name__ == "__main__":
sys.exit(main())
sys.exit(main())