diff --git a/deploy/docker/cluster-entrypoint.sh b/deploy/docker/cluster-entrypoint.sh index 19fae35d..1e0f8ca8 100644 --- a/deploy/docker/cluster-entrypoint.sh +++ b/deploy/docker/cluster-entrypoint.sh @@ -328,6 +328,165 @@ if [ "${GPU_ENABLED:-}" = "true" ]; then cp "$manifest" "$K3S_MANIFESTS/" done fi + + # ------------------------------------------------------------------- + # WSL2 GPU support: CDI mode + libdxcore.so injection + node labeling + # ------------------------------------------------------------------- + # WSL2 virtualises GPU access through /dev/dxg instead of native + # /dev/nvidia* device nodes. The legacy nvidia-container-runtime + # injection path fails because: + # 1. NVML can't initialise without libdxcore.so (the bridge between + # Linux NVML and the Windows DirectX GPU Kernel via /dev/dxg) + # 2. NFD can't detect NVIDIA PCI vendor (WSL2 hides PCI topology) + # + # Fix: switch to CDI mode, patch the CDI spec with libdxcore.so, and + # add a k3s manifest that labels the node for the device plugin + # DaemonSet affinity. + if [ -c /dev/dxg ]; then + echo "WSL2 detected (/dev/dxg present) — configuring CDI mode for GPU" + + # 1. Build a complete CDI spec from scratch. + # nvidia-ctk cdi generate has two WSL2 bugs: + # a) only creates name=all but the device plugin assigns by UUID + # b) misses libdxcore.so (the NVML-to-DXG bridge library) + # Writing the spec directly avoids fragile sed patching of YAML. + if command -v nvidia-ctk >/dev/null 2>&1 && command -v nvidia-smi >/dev/null 2>&1; then + mkdir -p /var/run/cdi + + GPU_UUID=$(nvidia-smi --query-gpu=gpu_uuid --format=csv,noheader 2>/dev/null | tr -d ' ' | head -1) + DXCORE_PATH=$(find /usr/lib -name "libdxcore.so" 2>/dev/null | head -1) + DXCORE_DIR=$(dirname "$DXCORE_PATH" 2>/dev/null || echo "/usr/lib/x86_64-linux-gnu") + DRIVER_DIR=$(ls -d /usr/lib/wsl/drivers/nv*.inf_amd64_* 2>/dev/null | head -1) + + if [ -n "$DRIVER_DIR" ] && [ -n "$GPU_UUID" ]; then + cat > /var/run/cdi/nvidia.yaml < "$K3S_MANIFESTS/wsl2-gpu-node-label.yaml" <<'WSLEOF' +apiVersion: batch/v1 +kind: Job +metadata: + name: wsl2-gpu-node-label + namespace: kube-system +spec: + template: + spec: + serviceAccountName: default + hostNetwork: true + tolerations: + - operator: Exists + containers: + - name: label + image: rancher/mirrored-library-busybox:1.37.0 + command: + - /bin/sh + - -c + - | + # Wait for the API server, then label the node + until wget -qO- --no-check-certificate https://kubernetes.default.svc/api/v1/nodes 2>/dev/null | grep -q '"items"'; do + sleep 2 + done + NODE=$(wget -qO- --no-check-certificate \ + -H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" \ + https://kubernetes.default.svc/api/v1/nodes 2>/dev/null \ + | sed -n 's/.*"name":"\([^"]*\)".*/\1/p' | head -1) + if [ -n "$NODE" ]; then + wget -qO- --no-check-certificate \ + -H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" \ + -H "Content-Type: application/strategic-merge-patch+json" \ + --method=PATCH \ + --body-data='{"metadata":{"labels":{"feature.node.kubernetes.io/pci-10de.present":"true"}}}' \ + "https://kubernetes.default.svc/api/v1/nodes/$NODE" >/dev/null 2>&1 \ + && echo "Labeled node $NODE with pci-10de.present=true" \ + || echo "Warning: failed to label node $NODE" + fi + restartPolicy: OnFailure + backoffLimit: 10 +WSLEOF + echo "WSL2 GPU node-label job manifest installed" + fi fi # ---------------------------------------------------------------------------