I keep hitting an issue where nvidia’s drivers get messed up on Linux and it causes my plex media server to crash on startup, requiring me to restart. nvidia-smi would return this error: Failed to initialize NVML: Driver/library version mismatch.
I finally found some time to investigate and came up with a script to cleanly unload/reload nvidia drivers and allow plex to start up without a restart of my machine. Script below:
#!/usr/bin/env bash
# Ensure the script is executed with root privileges
if [ "$EUID" -ne 0 ]; then
echo "Error: This script must be run as root. Please run with sudo." >&2
exit 1
fi
echo "===================================================="
echo "Starting Live NVIDIA Driver Version Mismatch Fix"
echo "===================================================="
# 1. Detect and stop the active display manager
DM=""
if systemctl is-active --quiet gdm3; then
DM="gdm3"
elif systemctl is-active --quiet gdm; then
DM="gdm"
elif systemctl is-active --quiet sddm; then
DM="sddm"
elif systemctl is-active --quiet lightdm; then
DM="lightdm"
fi
if [ -n "$DM" ]; then
echo "[+] Detected active display manager: $DM"
echo "[+] Stopping $DM to free up GPU handles..."
systemctl stop "$DM"
# Provide a short pause for the system to settle the graphical stack
sleep 2
else
echo "[!] No standard active display manager detected (headless server mode). Proceeding..."
fi
# 2. Force-terminate lingering processes accessing NVIDIA devices
echo "[+] Checking for lingering processes accessing NVIDIA device files..."
if lsof /dev/nvidia* >/dev/null 2>&1; then
echo "[!] Found processes clinging to the GPU. Forcing termination via fuser..."
fuser -k -9 /dev/nvidia* 2>/dev/null
sleep 2
else
echo "[+] No user-space processes currently locking the GPU."
fi
# 3. Unload existing kernel modules sequentially from most to least dependent
echo "[+] Unloading active NVIDIA kernel modules..."
modules=("nvidia_drm" "nvidia_modeset" "nvidia_uvm" "nvidia")
for mod in "${modules[@]}"; do
if lsmod | grep -q "^${mod}"; then
echo " -> Removing module: $mod"
if ! rmmod "$mod"; then
echo " [!] Error: Failed to remove $mod. Checking for persistent locks..." >&2
lsof /dev/nvidia* 2>/dev/null
echo " [!] Please manually stop any background tasks listed above and rerun this script." >&2
exit 1
fi
else
echo " -> Module $mod is not currently loaded or already removed."
fi
done
# 4. Reload updated kernel modules sequentially from least to most dependent
echo "[+] Reloading updated NVIDIA kernel modules..."
reload_modules=("nvidia" "nvidia_uvm" "nvidia_modeset" "nvidia_drm")
for mod in "${reload_modules[@]}"; do
echo " -> Loading module: $mod"
if ! modprobe "$mod"; then
echo " [!] Critical Error: Failed to load module $mod." >&2
exit 1
fi
done
# 5. Restart the display manager if one was originally running
if [ -n "$DM" ]; then
echo "[+] Restarting display manager: $DM..."
systemctl start "$DM"
fi
# 6. Verify NVML initialization status
echo "===================================================="
echo "Verification Check"
echo "===================================================="
if command -v nvidia-smi &> /dev/null; then
echo "[+] Running nvidia-smi to confirm fix..."
nvidia-smi
else
echo "[+] Modules reloaded successfully, but nvidia-smi utility is not in the system PATH."
fi
