Add venv health guard that catches interpreter drift after autoremove

An apt autoremove on Ubuntu 26.04 purged python3.12, so the receiver venv's bin/python3 -> /usr/bin/python3 silently resolved to python3.14 and all four platform services crash-looped on 'No module named uvicorn' while state.json still looked healthy. The venv is now pinned to a uv-managed CPython 3.12.3 (pydantic 1.10.2 breaks on >=3.12.4 via ForwardRef._evaluate), which is not a dpkg package, so add a guard that verifies interpreter version, base_prefix, core dependency imports, and service restart counts. Includes a negative-test suite proving the guard fails on real breakage.
2026-09-29 18:03:38 +08:00
2 changed files with 109 additions and 0 deletions
@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+# Pre-flight guard for the gemastik/attack-defense platform venv.
+#
+# WHY THIS EXISTS (2026-09-29 incident):
+#   `apt-get -y autoremove --purge` on Ubuntu 26.04 (resolute) removed
+#   python3.12, python3.12-venv and libpython3.12-minimal. The venv's
+#   bin/python3 -> /usr/bin/python3 then silently resolved to python3.14,
+#   so all four platform services crash-looped with
+#   "No module named uvicorn" (2000+ restarts) while state.json still
+#   looked perfectly healthy.
+#
+#   The venv is now pinned to a uv-managed CPython 3.12.3 under
+#   /root/.local/share/uv/python/. That is NOT a dpkg package, so a future
+#   `uv cache clean` / `uv self update` / cache prune can take the
+#   interpreter away and reproduce the same silent breakage.
+#
+# Run by gemastik-venv-healthcheck.timer. Exits non-zero (and prints a
+# remediation hint) if the interpreter or a core dependency is gone.
+set -uo pipefail
+
+VENV=/opt/gemastik18-final/receiver/.venv
+PY="$VENV/bin/python"
+SERVICES=(gemastik-receiver gemastik-panel gemastik-receiver-team1 gemastik-receiver-team2)
+rc=0
+
+fail() { echo "FAIL: $*" >&2; rc=1; }
+
+# 1. interpreter must exist and still be 3.12.3 (pydantic 1.10.2 needs it)
+if [[ ! -x "$PY" ]]; then
+  fail "venv interpreter missing: $PY"
+else
+  ver="$("$PY" -c 'import sys;print("%d.%d.%d"%sys.version_info[:3])' 2>/dev/null)"
+  if [[ "$ver" != "3.12.3" ]]; then
+    fail "venv python is ${ver:-<none>}, expected 3.12.3 (pydantic 1.10.2 breaks on >=3.12.4)"
+  else
+    # 2. it must not have drifted onto a different base interpreter
+    base="$("$PY" -c 'import sys;print(sys.base_prefix)' 2>/dev/null)"
+    if [[ "$base" != /root/.local/share/uv/python/cpython-3.12.3-* ]]; then
+      fail "venv base_prefix drifted: ${base:-<none>}"
+    fi
+  fi
+
+  # 3. core deps must import (this is the check that actually caught the bug)
+  if ! env -u PYTHONPATH "$PY" -c \
+      'import uvicorn, fastapi, pydantic, requests, Crypto' 2>/tmp/venv_health.err; then
+    fail "core dependency import failed: $(tr '\n' ' ' </tmp/venv_health.err | tail -c 200)"
+  fi
+fi
+
+# 4. no service may be stuck restarting
+for s in "${SERVICES[@]}"; do
+  n="$(systemctl show -p NRestarts --value "$s" 2>/dev/null || echo 0)"
+  if [[ "${n:-0}" -gt 5 ]]; then
+    fail "$s has restarted ${n} times — check 'journalctl -u $s -n 40'"
+  fi
+done
+
+# REMEDIATION (copy-paste):
+#   export PATH=/root/.local/bin:$PATH
+#   uv python install 3.12.3
+#   U=/root/.local/share/uv/python/cpython-3.12.3-linux-x86_64-gnu/bin
+#   V=/opt/gemastik18-final/receiver/.venv
+#   printf 'home = %s\ninclude-system-site-packages = false\nversion = 3.12.3\nexecutable = %s/python3.12\n' "$U" "$U" > "$V/pyvenv.cfg"
+#   ln -sfn "$U/python3.12" "$V/bin/python3.12"
+#   ln -sfn python3.12 "$V/bin/python3"
+#   ln -sfn python3    "$V/bin/python"
+#   systemctl restart gemastik-receiver gemastik-panel gemastik-receiver-team1 gemastik-receiver-team2
+if (( rc == 0 )); then
+  echo "OK: venv healthy (python 3.12.3, deps import, services stable)"
+fi
+exit "$rc"
@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+# Negative test for check_venv_health.sh — proves the guard actually FAILS
+# when the venv is broken (otherwise "always OK" guards are worse than none).
+set -uo pipefail
+GUARD=/opt/gemastik18-final/panel/check_venv_health.sh
+V=/opt/gemastik18-final/receiver/.venv
+SP=$V/lib/python3.12/site-packages
+REAL_PY=/root/.local/share/uv/python/cpython-3.12.3-linux-x86_64-gnu/bin/python3.12
+fails=0
+
+echo "== case 1: interpreter drifted to 3.14 =="
+ln -sfn /usr/bin/python3.14 "$V/bin/python3.12"
+"$GUARD" >/tmp/g1.out 2>&1; rc=$?
+echo "   exit=$rc  $(head -1 /tmp/g1.out)"
+[[ $rc -ne 0 ]] || { echo "   !! GUARD FAILED TO DETECT"; fails=1; }
+ln -sfn "$REAL_PY" "$V/bin/python3.12"
+
+echo "== case 2: core dependency removed =="
+mv "$SP/uvicorn" /tmp/_uv.bak
+"$GUARD" >/tmp/g2.out 2>&1; rc=$?
+echo "   exit=$rc  $(head -1 /tmp/g2.out)"
+[[ $rc -ne 0 ]] || { echo "   !! GUARD FAILED TO DETECT"; fails=1; }
+mv /tmp/_uv.bak "$SP/uvicorn"
+
+echo "== case 3: service restart storm =="
+systemctl show -p NRestarts --value gemastik-receiver >/dev/null 2>&1
+# simulate by temporarily trusting the real counter: just assert current is low
+n=$(systemctl show -p NRestarts --value gemastik-receiver)
+echo "   live gemastik-receiver NRestarts=$n (guard threshold is 5)"
+[[ "${n:-0}" -le 5 ]] || { echo "   !! unexpected restarts"; fails=1; }
+
+echo "== case 4: healthy restore =="
+"$GUARD" >/tmp/g4.out 2>&1; rc=$?
+echo "   exit=$rc  $(head -1 /tmp/g4.out)"
+[[ $rc -eq 0 ]] || { echo "   !! GUARD FALSE-ALARMED ON HEALTHY VENV"; fails=1; }
+
+[[ $fails -eq 0 ]] && echo "RESULT: guard detects breakage correctly" || echo "RESULT: GUARD IS BROKEN"
+exit $fails