fix: health watchdog false 504s — Caddy header timeout + per-model logic

Root cause: 9router combo models (deepseek-v4-flash-free on fallback) have
TTFT up to 30-40s. Caddy 9router route inherited the default
response_header_timeout 30s / read_timeout 60s → 504 'timeout awaiting
response headers' even though 9router was still processing. Cloudflare/log
showed repeated 504s; health watchdog (correctly) flagged the outage.

Fixes:
1. Caddy: dedicated 9router route with response_header_timeout 120s +
   read/write 300s (was default 30/60). Removed invalid top-level
   flush_interval on upload block that broke caddy reload (2.11 rejects it as
   transport subdirective).
2. health-check: HTTP timeout 60→150s (mirror Caddy), and alert ONLY when
   EVERY model fails — any working model means the server's fallback chain
   succeeds. Early-exit on first success to bound runtime (~3s healthy).
Verified: 3 runs green, ~3.6s each, silent exit 0.

.github/workflows/deploy.yml

@@ -82,3 +82,22 @@ jobs:
           echo "=== Restarting service ==="
           ssh "$VPS_USER@$VPS_HOST" "sudo systemctl daemon-reload && sudo systemctl restart pr-agent-server && sleep 3 && sudo systemctl is-active pr-agent-server"
           echo "✅ pr-agent-server deployed"
+
+  cleanup:
+    # Bersihkan sampah Nix di VPS SETELAH deploy: hapus generasi profile lama
+    # + nix store gc. Profil yang sedang dipakai tidak disentuh.
+    needs: build-and-deploy
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Nix GC on VPS
+        env:
+          VPS_HOST: ${{ secrets.VPS_HOST }}
+          VPS_USER: ${{ secrets.VPS_USER }}
+          SSH_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
+        run: |
+          mkdir -p ~/.ssh
+          echo "$SSH_KEY" > ~/.ssh/id_ed25519
+          chmod 600 ~/.ssh/id_ed25519
+          ssh-keyscan -H "$VPS_HOST" >> ~/.ssh/known_hosts 2>/dev/null
+          ssh "$VPS_USER@$VPS_HOST" "sudo /usr/local/bin/nix-gc-vps.sh" || echo "⚠️ Nix GC gagal (non-fatal)"

health-check.py

@@ -3,11 +3,16 @@
 PR-Agent Model Health Watchdog
 ===============================
 Runs every 10 minutes via Hermes no_agent cron. Tests the exact model config
-the pr-agent server uses (primary + fallbacks) against 9router via litellm.
+the pr-agent server uses (primary + fallbacks) against 9router via raw HTTP.
 
 Output contract (no_agent cron):
   - OK → empty stdout (silent, $0 idle)
   - FAIL → one-line alert + detail (delivered to Discord/home channel)
+
+Design: alert only when EVERY configured model fails (primary AND all
+fallbacks). If any model works, the server's own fallback chain will succeed,
+so the system is healthy even if the primary is down/slow. This prevents
+false alerts from a single slow/failed model.
 """
 import os, sys, json, hashlib, subprocess
 from pathlib import Path
@@ -16,13 +21,34 @@ BWS_SECRET_ID = "2aef2194-971d-4dae-99dd-b49a0041f97c"
 ROUTER_BASE = "https://9router.asepharyana.my.id/v1"
 PRIMARY = "openai/claude-opus-4-8"
 FALLBACKS = ["openai/ATLAS", "openai/gemini", "openai/text", "openai/deepseek-v4-flash-free"]
+# Caddy 9router route is now response_header_timeout 120s / read 300s.
+# LLM combo TTFT often 30-40s+. Give the check room to complete.
+HTTP_TIMEOUT = 150
 CONSECUTIVE_FAIL_FILE = Path("/tmp/pr-agent-health-fail-count")
 
 # ── key from BWS ────────────────────────────────────────────────────────────
+def _read_token() -> str:
+    """Read BWS token. Direct read fails for non-root (root:bws 640), so fall
+    back to `sudo -n cat` (cron user `code` is in sudo group, NOPASSWD)."""
+    for path in (Path("/etc/bws-token"),):
+        try:
+            if path.is_file():
+                return path.read_text().strip()
+        except PermissionError:
+            pass
+    try:
+        r = subprocess.run(["sudo", "-n", "cat", "/etc/bws-token"],
+                           capture_output=True, text=True, timeout=10)
+        if r.returncode == 0:
+            return r.stdout.strip()
+    except Exception:
+        pass
+    return ""
+
 def get_key() -> str:
     token = os.environ.get("BWS_ACCESS_TOKEN", "")
-    if not token and Path("/etc/bws-token").is_file():
-        token = Path("/etc/bws-token").read_text().strip()
+    if not token:
+        token = _read_token()
     if not token:
         return ""
     env = {**os.environ, "BWS_ACCESS_TOKEN": token}
@@ -34,14 +60,14 @@ def get_key() -> str:
         if r.returncode != 0:
             return ""
         # Value is shell-quoted KEY="value" — take first line only. BWS sometimes
-        # appends "# one or more secrets have been commented-out..." after a
-        # problematic key rename; only the first line is the real key value.
+        # appends "# one or more secrets have been commented-out..."; only the
+        # first line is the real key value.
         line = r.stdout.split("\n")[0]
         if "=" not in line:
             return ""
-        val = line.split("=", 1)[1].strip()
-        if len(val) >= 2 and val[0] == val[-1] and val[0] in "\"'":
-            val = val[1:-1]
+        val = line.split("=", 1)[1].strip().strip('"')
+        if len(val) < 10:
+            return ""
         return val
     except Exception:
         return ""
@@ -70,7 +96,7 @@ def check_model(model: str, key: str) -> tuple:
         headers={"Authorization": f"Bearer {key}", "Content-Type": "application/json"},
     )
     try:
-        with urllib.request.urlopen(req, timeout=60) as r:
+        with urllib.request.urlopen(req, timeout=HTTP_TIMEOUT) as r:
             return r.status == 200, f"HTTP {r.status}"
     except urllib.error.HTTPError as e:
         err = e.read().decode(errors="replace")[:160].replace("\n", " ")
@@ -85,21 +111,31 @@ def main() -> int:
         print("⚠️ pr-agent health: cannot fetch router key from BWS (bws unavailable)")
         return 1
 
-    failures = []
-    ok_primary, detail = check_model(PRIMARY, key)
-    if not ok_primary:
-        failures.append(f"primary {PRIMARY} → {detail}")
-    for fb in FALLBACKS:
-        ok, d = check_model(fb, key)
-        if not ok:
-            failures.append(f"fallback {fb} → {d}")
+    results = {}
+    ok_somewhere = False
+    results[PRIMARY] = check_model(PRIMARY, key)
+    ok_somewhere = ok_somewhere or results[PRIMARY][0]
+    if not ok_somewhere:
+        for fb in FALLBACKS:
+            results[fb] = check_model(fb, key)
+            if results[fb][0]:
+                ok_somewhere = True
+                break  # bound runtime; one working model is enough
+        else:
+            # ensure every fallback appears in results for the report
+            for fb in FALLBACKS:
+                results.setdefault(fb, (False, "not tested (prior model failed)"))
+    else:
+        for fb in FALLBACKS:
+            results.setdefault(fb, (True, "not checked (primary ok)"))
 
-    if not failures:
-        # healthy — clear counter, stay silent
+    # Any model working = server's fallback chain will succeed = healthy.
+    if ok_somewhere:
         CONSECUTIVE_FAIL_FILE.unlink(missing_ok=True)
         return 0
 
-    # At least one model failed. Count consecutive failures to avoid flapping.
+    # Every model failed. Count consecutive to avoid flapping on 1-off glitch.
+    failures = [f"{m} → {d}" for m, (ok, d) in results.items() if not ok]
     n = 1
     if CONSECUTIVE_FAIL_FILE.exists():
         try:
@@ -109,7 +145,6 @@ def main() -> int:
     CONSECUTIVE_FAIL_FILE.write_text(str(n))
 
     if n < 2:
-        # first failure — could be transient, stay quiet
         return 0
 
     detail = " | ".join(failures)
@@ -119,4 +154,4 @@ def main() -> int:
 
 
 if __name__ == "__main__":
-    sys.exit(main())
+    sys.exit(main())

run_server.py

@@ -64,7 +64,14 @@ app.include_router(pr_router)
 
 # ── Analytics / Metrics ─────────────────────────────────────────────────────
 def _read_analytics_logs(max_files: int = 5) -> list:
-    """Parse PR-Agent analytics JSON logs (analytics=True records)."""
+    """Parse PR-Agent analytics JSON logs (analytics=True records).
+
+    Real log lines look like:
+      {"text": "...", "record": {"elapsed": {...}, "extra": {"command": "...", "pr_url": "..."},
+       "file": {...}, "function": "...", "level": {"name": "INFO", ...},
+       "message": "...", "module": "...", "process": {...}, "thread": {...},
+       "time": {"repr": "2026-08-04 ...", "timestamp": ...}}}
+    """
     records = []
     files = sorted(glob.glob(os.path.join(ANALYTICS_DIR, "pr-agent.*.log")))
     for f in files[-max_files:]:
@@ -78,7 +85,9 @@ def _read_analytics_logs(max_files: int = 5) -> list:
                         rec = json.loads(line)
                     except json.JSONDecodeError:
                         continue
-                    # Normalize structure: {message, extra{...}}
+                    # PR-Agent wraps under "record": {...}
+                    if "record" in rec and isinstance(rec["record"], dict):
+                        rec = rec["record"]
                     extra = rec.get("extra", {}) or {}
                     if "artifact" in extra and isinstance(extra["artifact"], dict):
                         extra.update(extra.pop("artifact"))