diff --git a/panel/check_venv_health.sh b/panel/check_venv_health.sh new file mode 100755 index 0000000..ac13758 --- /dev/null +++ b/panel/check_venv_health.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# Pre-flight guard for the gemastik/attack-defense platform venv. +# +# WHY THIS EXISTS (2026-09-29 incident): +# `apt-get -y autoremove --purge` on Ubuntu 26.04 (resolute) removed +# python3.12, python3.12-venv and libpython3.12-minimal. The venv's +# bin/python3 -> /usr/bin/python3 then silently resolved to python3.14, +# so all four platform services crash-looped with +# "No module named uvicorn" (2000+ restarts) while state.json still +# looked perfectly healthy. +# +# The venv is now pinned to a uv-managed CPython 3.12.3 under +# /root/.local/share/uv/python/. That is NOT a dpkg package, so a future +# `uv cache clean` / `uv self update` / cache prune can take the +# interpreter away and reproduce the same silent breakage. +# +# Run by gemastik-venv-healthcheck.timer. Exits non-zero (and prints a +# remediation hint) if the interpreter or a core dependency is gone. +set -uo pipefail + +VENV=/opt/gemastik18-final/receiver/.venv +PY="$VENV/bin/python" +SERVICES=(gemastik-receiver gemastik-panel gemastik-receiver-team1 gemastik-receiver-team2) +rc=0 + +fail() { echo "FAIL: $*" >&2; rc=1; } + +# 1. interpreter must exist and still be 3.12.3 (pydantic 1.10.2 needs it) +if [[ ! -x "$PY" ]]; then + fail "venv interpreter missing: $PY" +else + ver="$("$PY" -c 'import sys;print("%d.%d.%d"%sys.version_info[:3])' 2>/dev/null)" + if [[ "$ver" != "3.12.3" ]]; then + fail "venv python is ${ver:-}, expected 3.12.3 (pydantic 1.10.2 breaks on >=3.12.4)" + else + # 2. it must not have drifted onto a different base interpreter + base="$("$PY" -c 'import sys;print(sys.base_prefix)' 2>/dev/null)" + if [[ "$base" != /root/.local/share/uv/python/cpython-3.12.3-* ]]; then + fail "venv base_prefix drifted: ${base:-}" + fi + fi + + # 3. core deps must import (this is the check that actually caught the bug) + if ! env -u PYTHONPATH "$PY" -c \ + 'import uvicorn, fastapi, pydantic, requests, Crypto' 2>/tmp/venv_health.err; then + fail "core dependency import failed: $(tr '\n' ' ' /dev/null || echo 0)" + if [[ "${n:-0}" -gt 5 ]]; then + fail "$s has restarted ${n} times — check 'journalctl -u $s -n 40'" + fi +done + +# REMEDIATION (copy-paste): +# export PATH=/root/.local/bin:$PATH +# uv python install 3.12.3 +# U=/root/.local/share/uv/python/cpython-3.12.3-linux-x86_64-gnu/bin +# V=/opt/gemastik18-final/receiver/.venv +# printf 'home = %s\ninclude-system-site-packages = false\nversion = 3.12.3\nexecutable = %s/python3.12\n' "$U" "$U" > "$V/pyvenv.cfg" +# ln -sfn "$U/python3.12" "$V/bin/python3.12" +# ln -sfn python3.12 "$V/bin/python3" +# ln -sfn python3 "$V/bin/python" +# systemctl restart gemastik-receiver gemastik-panel gemastik-receiver-team1 gemastik-receiver-team2 +if (( rc == 0 )); then + echo "OK: venv healthy (python 3.12.3, deps import, services stable)" +fi +exit "$rc" diff --git a/panel/test_venv_guard.sh b/panel/test_venv_guard.sh new file mode 100755 index 0000000..6514c1a --- /dev/null +++ b/panel/test_venv_guard.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Negative test for check_venv_health.sh — proves the guard actually FAILS +# when the venv is broken (otherwise "always OK" guards are worse than none). +set -uo pipefail +GUARD=/opt/gemastik18-final/panel/check_venv_health.sh +V=/opt/gemastik18-final/receiver/.venv +SP=$V/lib/python3.12/site-packages +REAL_PY=/root/.local/share/uv/python/cpython-3.12.3-linux-x86_64-gnu/bin/python3.12 +fails=0 + +echo "== case 1: interpreter drifted to 3.14 ==" +ln -sfn /usr/bin/python3.14 "$V/bin/python3.12" +"$GUARD" >/tmp/g1.out 2>&1; rc=$? +echo " exit=$rc $(head -1 /tmp/g1.out)" +[[ $rc -ne 0 ]] || { echo " !! GUARD FAILED TO DETECT"; fails=1; } +ln -sfn "$REAL_PY" "$V/bin/python3.12" + +echo "== case 2: core dependency removed ==" +mv "$SP/uvicorn" /tmp/_uv.bak +"$GUARD" >/tmp/g2.out 2>&1; rc=$? +echo " exit=$rc $(head -1 /tmp/g2.out)" +[[ $rc -ne 0 ]] || { echo " !! GUARD FAILED TO DETECT"; fails=1; } +mv /tmp/_uv.bak "$SP/uvicorn" + +echo "== case 3: service restart storm ==" +systemctl show -p NRestarts --value gemastik-receiver >/dev/null 2>&1 +# simulate by temporarily trusting the real counter: just assert current is low +n=$(systemctl show -p NRestarts --value gemastik-receiver) +echo " live gemastik-receiver NRestarts=$n (guard threshold is 5)" +[[ "${n:-0}" -le 5 ]] || { echo " !! unexpected restarts"; fails=1; } + +echo "== case 4: healthy restore ==" +"$GUARD" >/tmp/g4.out 2>&1; rc=$? +echo " exit=$rc $(head -1 /tmp/g4.out)" +[[ $rc -eq 0 ]] || { echo " !! GUARD FALSE-ALARMED ON HEALTHY VENV"; fails=1; } + +[[ $fails -eq 0 ]] && echo "RESULT: guard detects breakage correctly" || echo "RESULT: GUARD IS BROKEN" +exit $fails