Skip to content
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
159 changes: 159 additions & 0 deletions deploy/docker/cluster-entrypoint.sh
Original file line number Diff line number Diff line change
Expand Up @@ -328,6 +328,165 @@ if [ "${GPU_ENABLED:-}" = "true" ]; then
cp "$manifest" "$K3S_MANIFESTS/"
done
fi

# -------------------------------------------------------------------
# WSL2 GPU support: CDI mode + libdxcore.so injection + node labeling
# -------------------------------------------------------------------
# WSL2 virtualises GPU access through /dev/dxg instead of native
# /dev/nvidia* device nodes. The legacy nvidia-container-runtime
# injection path fails because:
# 1. NVML can't initialise without libdxcore.so (the bridge between
# Linux NVML and the Windows DirectX GPU Kernel via /dev/dxg)
# 2. NFD can't detect NVIDIA PCI vendor (WSL2 hides PCI topology)
#
# Fix: switch to CDI mode, patch the CDI spec with libdxcore.so, and
# add a k3s manifest that labels the node for the device plugin
# DaemonSet affinity.
if [ -c /dev/dxg ]; then
echo "WSL2 detected (/dev/dxg present) — configuring CDI mode for GPU"

# 1. Build a complete CDI spec from scratch.
# nvidia-ctk cdi generate has two WSL2 bugs:
# a) only creates name=all but the device plugin assigns by UUID
# b) misses libdxcore.so (the NVML-to-DXG bridge library)
# Writing the spec directly avoids fragile sed patching of YAML.
if command -v nvidia-ctk >/dev/null 2>&1 && command -v nvidia-smi >/dev/null 2>&1; then
mkdir -p /var/run/cdi

GPU_UUID=$(nvidia-smi --query-gpu=gpu_uuid --format=csv,noheader 2>/dev/null | tr -d ' ' | head -1)
DXCORE_PATH=$(find /usr/lib -name "libdxcore.so" 2>/dev/null | head -1)
DXCORE_DIR=$(dirname "$DXCORE_PATH" 2>/dev/null || echo "/usr/lib/x86_64-linux-gnu")
DRIVER_DIR=$(ls -d /usr/lib/wsl/drivers/nv*.inf_amd64_* 2>/dev/null | head -1)

if [ -n "$DRIVER_DIR" ] && [ -n "$GPU_UUID" ]; then
cat > /var/run/cdi/nvidia.yaml <<CDIEOF
---
cdiVersion: "0.5.0"
kind: nvidia.com/gpu
devices:
- name: all
containerEdits:
deviceNodes:
- path: /dev/dxg
- name: "${GPU_UUID}"
containerEdits:
deviceNodes:
- path: /dev/dxg
- name: "0"
containerEdits:
deviceNodes:
- path: /dev/dxg
containerEdits:
env:
- NVIDIA_VISIBLE_DEVICES=void
hooks:
- hookName: createContainer
path: /usr/bin/nvidia-cdi-hook
args:
- nvidia-cdi-hook
- create-symlinks
- --link
- ${DRIVER_DIR}/nvidia-smi::/usr/bin/nvidia-smi
env:
- NVIDIA_CTK_DEBUG=false
- hookName: createContainer
path: /usr/bin/nvidia-cdi-hook
args:
- nvidia-cdi-hook
- update-ldcache
- --folder
- ${DRIVER_DIR}
- --folder
- ${DXCORE_DIR}
env:
- NVIDIA_CTK_DEBUG=false
mounts:
- hostPath: ${DXCORE_PATH}
containerPath: ${DXCORE_PATH}
options: [ro, nosuid, nodev, rbind, rprivate]
- hostPath: ${DRIVER_DIR}/libcuda.so.1.1
containerPath: ${DRIVER_DIR}/libcuda.so.1.1
options: [ro, nosuid, nodev, rbind, rprivate]
- hostPath: ${DRIVER_DIR}/libcuda_loader.so
containerPath: ${DRIVER_DIR}/libcuda_loader.so
options: [ro, nosuid, nodev, rbind, rprivate]
- hostPath: ${DRIVER_DIR}/libnvdxgdmal.so.1
containerPath: ${DRIVER_DIR}/libnvdxgdmal.so.1
options: [ro, nosuid, nodev, rbind, rprivate]
- hostPath: ${DRIVER_DIR}/libnvidia-ml.so.1
containerPath: ${DRIVER_DIR}/libnvidia-ml.so.1
options: [ro, nosuid, nodev, rbind, rprivate]
- hostPath: ${DRIVER_DIR}/libnvidia-ml_loader.so
containerPath: ${DRIVER_DIR}/libnvidia-ml_loader.so
options: [ro, nosuid, nodev, rbind, rprivate]
- hostPath: ${DRIVER_DIR}/libnvidia-ptxjitcompiler.so.1
containerPath: ${DRIVER_DIR}/libnvidia-ptxjitcompiler.so.1
options: [ro, nosuid, nodev, rbind, rprivate]
- hostPath: ${DRIVER_DIR}/nvcubins.bin
containerPath: ${DRIVER_DIR}/nvcubins.bin
options: [ro, nosuid, nodev, rbind, rprivate]
- hostPath: ${DRIVER_DIR}/nvidia-smi
containerPath: ${DRIVER_DIR}/nvidia-smi
options: [ro, nosuid, nodev, rbind, rprivate]
CDIEOF
echo "CDI spec written (GPU: $GPU_UUID, driver: $DRIVER_DIR, dxcore: $DXCORE_PATH)"
else
echo "Warning: could not detect GPU UUID or WSL driver store — CDI spec not written"
fi
fi

# 5. Switch nvidia container runtime to CDI mode
NVIDIA_RUNTIME_CONFIG="/etc/nvidia-container-runtime/config.toml"
if [ -f "$NVIDIA_RUNTIME_CONFIG" ]; then
sed -i 's/mode = "auto"/mode = "cdi"/' "$NVIDIA_RUNTIME_CONFIG"
echo "nvidia-container-runtime switched to CDI mode"
fi

# 6. Create a k3s manifest to label the node with NVIDIA PCI vendor
# (NFD can't detect it on WSL2 since PCI topology is virtualised)
cat > "$K3S_MANIFESTS/wsl2-gpu-node-label.yaml" <<'WSLEOF'
apiVersion: batch/v1
kind: Job
metadata:
name: wsl2-gpu-node-label
namespace: kube-system
spec:
template:
spec:
serviceAccountName: default
hostNetwork: true
tolerations:
- operator: Exists
containers:
- name: label
image: rancher/mirrored-library-busybox:1.37.0
command:
- /bin/sh
- -c
- |
# Wait for the API server, then label the node
until wget -qO- --no-check-certificate https://kubernetes.default.svc/api/v1/nodes 2>/dev/null | grep -q '"items"'; do
sleep 2
done
NODE=$(wget -qO- --no-check-certificate \
-H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" \
https://kubernetes.default.svc/api/v1/nodes 2>/dev/null \
| sed -n 's/.*"name":"\([^"]*\)".*/\1/p' | head -1)
if [ -n "$NODE" ]; then
wget -qO- --no-check-certificate \
-H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" \
-H "Content-Type: application/strategic-merge-patch+json" \
--method=PATCH \
--body-data='{"metadata":{"labels":{"feature.node.kubernetes.io/pci-10de.present":"true"}}}' \
"https://kubernetes.default.svc/api/v1/nodes/$NODE" >/dev/null 2>&1 \
&& echo "Labeled node $NODE with pci-10de.present=true" \
|| echo "Warning: failed to label node $NODE"
fi
restartPolicy: OnFailure
backoffLimit: 10
WSLEOF
echo "WSL2 GPU node-label job manifest installed"
fi
fi

# ---------------------------------------------------------------------------
Expand Down
Loading