diff --git a/.github/workflows/marketplace-build.yml b/.github/workflows/marketplace-build.yml index 61413b7dd..9ecabdb10 100644 --- a/.github/workflows/marketplace-build.yml +++ b/.github/workflows/marketplace-build.yml @@ -11,6 +11,11 @@ on: description: "Registry base URL override" required: false type: string + build_cuda: + description: "Also build CUDA bundle variants on the self-hosted GPU runner" + required: false + type: boolean + default: true secrets: MINISIGN_SECRET_KEY: required: true @@ -28,6 +33,9 @@ env: RUST_BACKTRACE: 1 RUSTC_WRAPPER: sccache SHERPA_ONNX_VERSION: "1.12.17" + # GPU (CUDA-enabled) sherpa-onnx shared build; vendored into cuda variants so + # the execution-provider-agnostic plugin .so dispatches to the GPU at runtime. + SHERPA_ONNX_GPU_ARCHIVE: "sherpa-onnx-v1.12.17-linux-x64-gpu-shared.tar.bz2" MINISIGN_DEB_URL: "http://launchpadlibrarian.net/780165111/minisign_0.12-1_amd64.deb" REGISTRY_BASE_URL: ${{ inputs.registry_base_url || 'https://streamkit.dev/registry' }} @@ -210,9 +218,142 @@ jobs: path: ~/.cache/sccache key: sccache-marketplace-${{ runner.os }}-${{ hashFiles('plugins/**/Cargo.lock') }} + build-marketplace-cuda: + name: Build Marketplace CUDA Variants + needs: [build-marketplace] + if: ${{ inputs.build_cuda }} + # GPU variants are best-effort: a failure on the (potentially flaky) + # self-hosted runner must never regress CPU registry publishing or per-plugin + # releases. continue-on-error keeps the overall workflow green so downstream + # CPU jobs still run; the failure is still visible on this job. + continue-on-error: true + # Self-hosted NVIDIA (Ada) runner: CUDA toolkit + nvcc must be preinstalled. + # Adjust the labels to match your runner registration if they differ. + runs-on: [self-hosted, linux, x64, gpu] + permissions: + contents: read + steps: + - uses: actions/checkout@v5 + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y cmake pkg-config libclang-dev libfontconfig1-dev wget libopenblas-dev zstd patchelf python3-yaml python3-tomli libfdk-aac-dev \ + build-essential g++ libharfbuzz-dev libfreetype6-dev \ + libegl1-mesa-dev libgl1-mesa-dev libgbm-dev libdrm-dev libunwind-dev \ + libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev \ + gstreamer1.0-plugins-base gstreamer1.0-plugins-good gstreamer1.0-plugins-bad \ + libdbus-1-dev libpulse-dev libxkbcommon-dev nasm + + - name: Install minisign + run: | + deb_path="/tmp/minisign.deb" + wget -O "${deb_path}" "${MINISIGN_DEB_URL}" + if ! sudo dpkg -i "${deb_path}"; then + echo "dpkg failed, retrying with apt-get" + sudo env NEEDRESTART_MODE=a apt-get install -y "${deb_path}" || true + fi + command -v minisign >/dev/null + + - name: Install CUDA-enabled sherpa-onnx + run: | + cd /tmp + wget "https://github.com/k2-fsa/sherpa-onnx/releases/download/v${SHERPA_ONNX_VERSION}/${SHERPA_ONNX_GPU_ARCHIVE}" + tar xf "${SHERPA_ONNX_GPU_ARCHIVE}" + dir="${SHERPA_ONNX_GPU_ARCHIVE%.tar.bz2}" + sudo cp -r "${dir}/lib/." /usr/local/lib/ + sudo cp -r "${dir}/include/." /usr/local/include/ + sudo ldconfig + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@master + with: + toolchain: "1.95.0" + + - name: Restore sccache cache + uses: actions/cache/restore@v4 + with: + path: ~/.cache/sccache + key: sccache-marketplace-cuda-${{ runner.os }}-${{ hashFiles('plugins/**/Cargo.lock') }} + restore-keys: | + sccache-marketplace-cuda-${{ runner.os }}- + + - uses: mozilla-actions/sccache-action@v0.0.9 + + - name: Build CUDA plugins + run: | + bash scripts/marketplace/build_official_plugins_cuda.sh + + - name: Download CPU registry + uses: actions/download-artifact@v4 + with: + name: marketplace-registry + path: dist/registry-cpu + + - name: Write minisign key + env: + MINISIGN_SECRET_KEY: ${{ secrets.MINISIGN_SECRET_KEY }} + run: | + if [ -z "${MINISIGN_SECRET_KEY}" ]; then + echo "MINISIGN_SECRET_KEY is not set" + exit 1 + fi + echo "${MINISIGN_SECRET_KEY}" > /tmp/streamkit.key + chmod 600 /tmp/streamkit.key + + - name: Build CUDA registry variants + run: | + python3 scripts/marketplace/build_registry.py \ + --plugins marketplace/official-plugins.json \ + --existing-registry dist/registry-cpu \ + --accelerator cuda \ + --bundle-url-template "https://github.com/${{ github.repository }}/releases/download/plugin-{plugin_id}-v{version}" \ + --registry-base-url "${REGISTRY_BASE_URL}" \ + --bundles-out dist/bundles-cuda \ + --registry-out dist/registry \ + --signing-key /tmp/streamkit.key \ + --new-plugins-out dist/new-plugins-cuda.json + + - name: Verify CUDA bundle portability + run: | + python3 scripts/marketplace/verify_bundles.py \ + --plugins marketplace/official-plugins.json \ + --bundles dist/bundles-cuda \ + --accelerator cuda + + - name: Upload CUDA bundles + uses: actions/upload-artifact@v4 + with: + name: marketplace-bundles-cuda + path: dist/bundles-cuda/*.tar.zst + if-no-files-found: ignore + + - name: Upload new CUDA plugins manifest + uses: actions/upload-artifact@v4 + with: + name: new-plugins-cuda + path: dist/new-plugins-cuda.json + + - name: Upload merged registry metadata + uses: actions/upload-artifact@v4 + with: + name: marketplace-registry + overwrite: true + path: dist/registry/** + + - name: Save sccache cache + if: always() + uses: actions/cache/save@v4 + with: + path: ~/.cache/sccache + key: sccache-marketplace-cuda-${{ runner.os }}-${{ hashFiles('plugins/**/Cargo.lock') }} + publish-registry: name: Publish Registry (PR) - needs: [build-marketplace] + needs: [build-marketplace, build-marketplace-cuda] + # A successful CPU build is sufficient to publish; the CUDA job is optional + # and, when it runs, supersedes the artifact with a merged registry. + if: ${{ always() && needs.build-marketplace.result == 'success' }} runs-on: ubuntu-22.04 permissions: contents: write diff --git a/.github/workflows/marketplace-release.yml b/.github/workflows/marketplace-release.yml index 8bd11c552..08f138675 100644 --- a/.github/workflows/marketplace-release.yml +++ b/.github/workflows/marketplace-release.yml @@ -42,6 +42,20 @@ jobs: name: new-plugins path: artifacts + - name: Download CUDA bundles + uses: actions/download-artifact@v4 + continue-on-error: true + with: + name: marketplace-bundles-cuda + path: artifacts/bundles-cuda + + - name: Download new CUDA plugins manifest + uses: actions/download-artifact@v4 + continue-on-error: true + with: + name: new-plugins-cuda + path: artifacts/cuda + - name: Load plugin metadata id: plugins run: | @@ -108,3 +122,26 @@ jobs: if result.returncode != 0: print(f'::warning::Skipping {tag}: release may already exist') " + + - name: Attach CUDA bundles to releases + if: hashFiles('artifacts/cuda/new-plugins-cuda.json') != '' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + python3 -c " + import json, os, pathlib, subprocess + + data = json.load(open('artifacts/cuda/new-plugins-cuda.json')) + for plugin in data.get('plugins', []): + pid = plugin['id'] + version = plugin['version'] + tag = f'plugin-{pid}-v{version}' + bundle = f'artifacts/bundles-cuda/{pid}-{version}-cuda-bundle.tar.zst' + if not pathlib.Path(bundle).exists(): + print(f'::warning::Missing CUDA bundle for {tag}: {bundle}') + continue + print(f'Uploading CUDA bundle to {tag}') + result = subprocess.run(['gh', 'release', 'upload', tag, bundle, '--clobber']) + if result.returncode != 0: + print(f'::warning::Failed to upload CUDA bundle for {tag}') + " diff --git a/Cargo.lock b/Cargo.lock index e16c9c170..63e818afd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6541,6 +6541,7 @@ dependencies = [ "image", "jemalloc_pprof", "jsonwebtoken", + "libloading 0.9.0", "liblzma", "mime_guess", "moq-lite", diff --git a/Dockerfile.demo b/Dockerfile.demo index 774b904aa..b32ff34bf 100644 --- a/Dockerfile.demo +++ b/Dockerfile.demo @@ -5,9 +5,6 @@ # Demo image: CPU-only with all core plugins (including Pocket TTS, Supertonic, and Slint) and sample pipelines # syntax=docker/dockerfile:1 -# Version configuration -ARG SHERPA_ONNX_VERSION=1.12.17 - # Stage 1: Build Rust dependencies FROM rust:1.95-slim-bookworm AS rust-deps @@ -122,498 +119,331 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \ mkdir -p /build/bin && cp /build/target/release-lto/skit /build/bin/skit \ ' -# Stage 4: Build Whisper plugin -FROM rust:1.95-slim-bookworm AS whisper-builder - -WORKDIR /build - -# Install build dependencies for whisper.cpp -RUN apt-get update && apt-get install -y \ - pkg-config \ - libssl-dev \ - g++ \ - cmake \ - curl \ - libclang-dev \ - clang \ - git \ - && rm -rf /var/lib/apt/lists/* - -# whisper.cpp/ggml defaults to enabling `GGML_NATIVE` (i.e. `-march=native`) when not cross-compiling. -# That can produce binaries that crash with SIGILL on older CPUs. Setting SOURCE_DATE_EPOCH disables -# `GGML_NATIVE_DEFAULT` upstream, making the build portable by default. -ENV SOURCE_DATE_EPOCH=1 - -# Extra defense-in-depth: ensure the toolchain doesn't auto-enable newer x86 features. -ENV CFLAGS="-O3 -pipe -fPIC -march=x86-64 -mtune=generic" \ - CXXFLAGS="-O3 -pipe -fPIC -march=x86-64 -mtune=generic" - -# Copy only what's needed to build whisper plugin -# Note: Cargo.toml needed for workspace dependency resolution in core/ -COPY Cargo.toml Cargo.lock ./ -COPY crates/core ./crates/core -COPY sdks/plugin-sdk ./sdks/plugin-sdk -COPY plugins/native/whisper ./plugins/native/whisper - -# Build whisper plugin -RUN --mount=type=cache,id=cargo-registry-whisper,target=/usr/local/cargo/registry \ - --mount=type=cache,id=cargo-git-whisper,target=/usr/local/cargo/git \ - --mount=type=cache,id=whisper-target-portable-v2,target=/build/plugins/native/whisper/target-portable \ - cd plugins/native/whisper && \ - cargo build --release --target-dir target-portable && \ - mkdir -p /build/dist/native/whisper && \ - cp target-portable/release/libwhisper.so /build/dist/native/whisper/ && \ - cp plugin.yml /build/dist/native/whisper/ - -# Download Whisper models (demo image uses a single tiny multilingual model). -# - ggml-tiny-q5_1.bin: Tiny multilingual STT (quantized) -# NOTE: This model is not yet uploaded to streamkit/whisper-models, using original source -# - silero_vad.onnx: VAD model for Whisper -RUN mkdir -p /build/models && \ - curl -L -o /build/models/ggml-tiny-q5_1.bin \ - https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny-q5_1.bin && \ - curl -L -o /build/models/silero_vad.onnx \ - https://huggingface.co/streamkit/whisper-models/resolve/main/silero_vad.onnx - -# Stage 5: Build Kokoro TTS plugin -FROM rust:1.95-slim-bookworm AS kokoro-builder - -WORKDIR /build - -# Install build dependencies -RUN apt-get update && apt-get install -y \ - pkg-config \ - libssl-dev \ - curl \ - wget \ - bzip2 \ - libclang-dev \ - clang \ - && rm -rf /var/lib/apt/lists/* - -# Download and install sherpa-onnx shared library -ARG SHERPA_ONNX_VERSION -RUN cd /tmp && \ - wget https://github.com/k2-fsa/sherpa-onnx/releases/download/v${SHERPA_ONNX_VERSION}/sherpa-onnx-v${SHERPA_ONNX_VERSION}-linux-x64-shared.tar.bz2 && \ - tar xf sherpa-onnx-v${SHERPA_ONNX_VERSION}-linux-x64-shared.tar.bz2 && \ - cp sherpa-onnx-v${SHERPA_ONNX_VERSION}-linux-x64-shared/lib/*.so* /usr/local/lib/ && \ - ldconfig - -# Copy only what's needed to build kokoro plugin -# Note: Cargo.toml needed for workspace dependency resolution in core/ -COPY Cargo.toml Cargo.lock ./ -COPY crates/core ./crates/core -COPY sdks/plugin-sdk ./sdks/plugin-sdk -COPY plugins/native/kokoro ./plugins/native/kokoro - -# Build kokoro plugin -RUN --mount=type=cache,id=cargo-registry-kokoro,target=/usr/local/cargo/registry \ - --mount=type=cache,id=cargo-git-kokoro,target=/usr/local/cargo/git \ - --mount=type=cache,target=/build/plugins/native/kokoro/target \ - cd plugins/native/kokoro && \ - RUSTFLAGS="-L /usr/local/lib" cargo build --release --target-dir target && \ - mkdir -p /build/dist/native/kokoro && \ - cp target/release/libkokoro.so /build/dist/native/kokoro/ && \ - cp plugin.yml /build/dist/native/kokoro/ - -# Download Kokoro TTS models -RUN mkdir -p /build/models && \ - cd /build/models && \ - curl -L -o kokoro-multi-lang-v1_1.tar.bz2 \ - https://huggingface.co/streamkit/kokoro-models/resolve/main/kokoro-multi-lang-v1_1.tar.bz2 && \ - tar xf kokoro-multi-lang-v1_1.tar.bz2 && \ - rm kokoro-multi-lang-v1_1.tar.bz2 - -# Stage 6: Build Piper TTS plugin -FROM rust:1.95-slim-bookworm AS piper-builder - -WORKDIR /build - -# Install build dependencies -RUN apt-get update && apt-get install -y \ - pkg-config \ - libssl-dev \ - curl \ - wget \ - bzip2 \ - libclang-dev \ - clang \ +# --------------------------------------------------------------- +# Marketplace plugin bundles +# +# Plugins are pulled as signed prebuilt CPU bundles from the same +# marketplace users install from, so the demo validates the artifacts +# users actually receive. Each sherpa-backed bundle is self-contained +# (vendors its own libsherpa-onnx-c-api.so + libonnxruntime.so with +# RUNPATH=$ORIGIN), so the demo needs no system sherpa-onnx libs. +# +# When bumping a plugin, update its `_PLUGIN_VERSION` and +# `_BUNDLE_SHA256` ARGs together from +# `docs/public/registry/plugins///manifest.json`; model HF +# SHAs come from the same manifest's `models[*].sha256` / +# `models[*].file_checksums` fields. Each sha256 pin is the trust +# boundary, matching the model the registry's signed manifest pins. +# --------------------------------------------------------------- + +# Shared base for all bundle-fetch stages -- one apt-get cached across all. +FROM debian:bookworm-slim AS bundle-deps +RUN apt-get update && apt-get install -y --no-install-recommends \ + curl ca-certificates zstd tar bzip2 \ && rm -rf /var/lib/apt/lists/* -# Copy sherpa-onnx from kokoro-builder (reuse to avoid duplicate downloads) -COPY --from=kokoro-builder /usr/local/lib/*.so* /usr/local/lib/ -RUN ldconfig - -# Copy only what's needed to build piper plugin -# Note: Cargo.toml needed for workspace dependency resolution in core/ -COPY Cargo.toml Cargo.lock ./ -COPY crates/core ./crates/core -COPY sdks/plugin-sdk ./sdks/plugin-sdk -COPY plugins/native/piper ./plugins/native/piper - -# Build piper plugin -RUN --mount=type=cache,id=cargo-registry-piper,target=/usr/local/cargo/registry \ - --mount=type=cache,id=cargo-git-piper,target=/usr/local/cargo/git \ - --mount=type=cache,target=/build/plugins/native/piper/target \ - cd plugins/native/piper && \ - RUSTFLAGS="-L /usr/local/lib" cargo build --release --target-dir target && \ - mkdir -p /build/dist/native/piper && \ - cp target/release/libpiper.so /build/dist/native/piper/ && \ - cp plugin.yml /build/dist/native/piper/ - -# Download Piper TTS models (English + Spanish for translation output) -RUN mkdir -p /build/models && \ - cd /build/models && \ - curl -L -o vits-piper-en_US-libritts_r-medium.tar.bz2 \ - https://huggingface.co/streamkit/piper-models/resolve/main/vits-piper-en_US-libritts_r-medium.tar.bz2 && \ - tar xf vits-piper-en_US-libritts_r-medium.tar.bz2 && \ - rm vits-piper-en_US-libritts_r-medium.tar.bz2 && \ - cd vits-piper-en_US-libritts_r-medium && \ - if [ ! -f "model.onnx" ] && [ -f "en_US-libritts_r-medium.onnx" ]; then \ - ln -sf en_US-libritts_r-medium.onnx model.onnx; \ - fi && \ - cd /build/models && \ - curl -L -o vits-piper-es_MX-claude-high.tar.bz2 \ - https://huggingface.co/streamkit/piper-models/resolve/main/vits-piper-es_MX-claude-high.tar.bz2 && \ - tar xf vits-piper-es_MX-claude-high.tar.bz2 && \ - rm vits-piper-es_MX-claude-high.tar.bz2 && \ - cd vits-piper-es_MX-claude-high && \ - if [ ! -f "model.onnx" ] && [ -f "es_MX-claude-high.onnx" ]; then \ - ln -sf es_MX-claude-high.onnx model.onnx; \ +# --------------------------------------------------------------- +# Stage: whisper bundle + tiny multilingual model + Silero VAD. +# The demo deliberately ships the tiny *multilingual* ggml model +# (ggml-tiny-q5_1.bin from ggerganov/whisper.cpp), which is not one of +# the registry's manifest models (those are tiny.en/base.en/base) -- the +# sample pipelines below are rewritten to reference it. +# --------------------------------------------------------------- +FROM bundle-deps AS whisper-bundle +ARG WHISPER_PLUGIN_VERSION=0.3.0 +ARG WHISPER_BUNDLE_SHA256=79212cfa57376ebfcb955fd12522ecf36164071cdc94cf3bf964122837e8f4d2 +ARG WHISPER_GGML_SHA256=818710568da3ca15689e31a743197b520007872ff9576237bda97bd1b469c3d7 +ARG SILERO_VAD_SHA256=1a153a22f4509e292a94e67d6f9b85e8deb25b4988682b7e174c65279d8788e3 + +WORKDIR /bundle +RUN set -eux; \ + url="https://github.com/streamer45/streamkit/releases/download/plugin-whisper-v${WHISPER_PLUGIN_VERSION}/whisper-${WHISPER_PLUGIN_VERSION}-bundle.tar.zst"; \ + curl --proto '=https' --tlsv1.2 -fsSL -o whisper-bundle.tar.zst "$url"; \ + echo "${WHISPER_BUNDLE_SHA256} whisper-bundle.tar.zst" | sha256sum -c -; \ + mkdir -p extracted; \ + tar --zstd -xf whisper-bundle.tar.zst -C extracted; \ + test -f extracted/libwhisper.so + +WORKDIR /models +RUN set -eux; \ + curl --proto '=https' --tlsv1.2 -fsSL -o ggml-tiny-q5_1.bin \ + "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny-q5_1.bin"; \ + echo "${WHISPER_GGML_SHA256} ggml-tiny-q5_1.bin" | sha256sum -c -; \ + curl --proto '=https' --tlsv1.2 -fsSL -o silero_vad.onnx \ + "https://huggingface.co/streamkit/whisper-models/resolve/main/silero_vad.onnx"; \ + echo "${SILERO_VAD_SHA256} silero_vad.onnx" | sha256sum -c - + +# --------------------------------------------------------------- +# Stage: kokoro bundle + Kokoro multi-lang v1.1 models. +# --------------------------------------------------------------- +FROM bundle-deps AS kokoro-bundle +ARG KOKORO_PLUGIN_VERSION=0.3.0 +ARG KOKORO_BUNDLE_SHA256=0458c130b47a860428431427c26d9c2c80f21a371c7c565138fc351b2d0f652c +ARG KOKORO_MODELS_SHA256=a3f4c73d043860e3fd2e5b06f36795eb81de0fc8e8de6df703245edddd87dbad + +WORKDIR /bundle +RUN set -eux; \ + url="https://github.com/streamer45/streamkit/releases/download/plugin-kokoro-v${KOKORO_PLUGIN_VERSION}/kokoro-${KOKORO_PLUGIN_VERSION}-bundle.tar.zst"; \ + curl --proto '=https' --tlsv1.2 -fsSL -o kokoro-bundle.tar.zst "$url"; \ + echo "${KOKORO_BUNDLE_SHA256} kokoro-bundle.tar.zst" | sha256sum -c -; \ + mkdir -p extracted; \ + tar --zstd -xf kokoro-bundle.tar.zst -C extracted; \ + test -f extracted/libkokoro.so + +WORKDIR /models +RUN set -eux; \ + curl --proto '=https' --tlsv1.2 -fsSL -o kokoro-multi-lang-v1_1.tar.bz2 \ + "https://huggingface.co/streamkit/kokoro-models/resolve/main/kokoro-multi-lang-v1_1.tar.bz2"; \ + echo "${KOKORO_MODELS_SHA256} kokoro-multi-lang-v1_1.tar.bz2" | sha256sum -c -; \ + tar xf kokoro-multi-lang-v1_1.tar.bz2; \ + rm kokoro-multi-lang-v1_1.tar.bz2; \ + test -d kokoro-multi-lang-v1_1 + +# --------------------------------------------------------------- +# Stage: piper bundle + en_US / es_MX voices. +# --------------------------------------------------------------- +FROM bundle-deps AS piper-bundle +ARG PIPER_PLUGIN_VERSION=0.3.0 +ARG PIPER_BUNDLE_SHA256=28186b6fcff6af6048fb34576a96afe3b49ce5e640d9335a1999f2cd935c2a64 +ARG PIPER_MODEL_EN_SHA256=78c137daa7eddaf57190cf05c020efd6e593015f62c82ee999ef570fc2dff496 +ARG PIPER_MODEL_ES_SHA256=ec33fb689c248fe64810aab564cba97babf0f506672cfd404928d46e751a4721 + +WORKDIR /bundle +RUN set -eux; \ + url="https://github.com/streamer45/streamkit/releases/download/plugin-piper-v${PIPER_PLUGIN_VERSION}/piper-${PIPER_PLUGIN_VERSION}-bundle.tar.zst"; \ + curl --proto '=https' --tlsv1.2 -fsSL -o piper-bundle.tar.zst "$url"; \ + echo "${PIPER_BUNDLE_SHA256} piper-bundle.tar.zst" | sha256sum -c -; \ + mkdir -p extracted; \ + tar --zstd -xf piper-bundle.tar.zst -C extracted; \ + test -f extracted/libpiper.so + +WORKDIR /models +RUN set -eux; \ + curl --proto '=https' --tlsv1.2 -fsSL -o vits-piper-en_US-libritts_r-medium.tar.bz2 \ + "https://huggingface.co/streamkit/piper-models/resolve/main/vits-piper-en_US-libritts_r-medium.tar.bz2"; \ + echo "${PIPER_MODEL_EN_SHA256} vits-piper-en_US-libritts_r-medium.tar.bz2" | sha256sum -c -; \ + tar xf vits-piper-en_US-libritts_r-medium.tar.bz2; \ + rm vits-piper-en_US-libritts_r-medium.tar.bz2; \ + if [ ! -f vits-piper-en_US-libritts_r-medium/model.onnx ] && [ -f vits-piper-en_US-libritts_r-medium/en_US-libritts_r-medium.onnx ]; then \ + ln -sf en_US-libritts_r-medium.onnx vits-piper-en_US-libritts_r-medium/model.onnx; \ + fi; \ + curl --proto '=https' --tlsv1.2 -fsSL -o vits-piper-es_MX-claude-high.tar.bz2 \ + "https://huggingface.co/streamkit/piper-models/resolve/main/vits-piper-es_MX-claude-high.tar.bz2"; \ + echo "${PIPER_MODEL_ES_SHA256} vits-piper-es_MX-claude-high.tar.bz2" | sha256sum -c -; \ + tar xf vits-piper-es_MX-claude-high.tar.bz2; \ + rm vits-piper-es_MX-claude-high.tar.bz2; \ + if [ ! -f vits-piper-es_MX-claude-high/model.onnx ] && [ -f vits-piper-es_MX-claude-high/es_MX-claude-high.onnx ]; then \ + ln -sf es_MX-claude-high.onnx vits-piper-es_MX-claude-high/model.onnx; \ fi -# Stage 7: Build SenseVoice STT plugin -FROM rust:1.95-slim-bookworm AS sensevoice-builder - -WORKDIR /build - -# Install build dependencies -RUN apt-get update && apt-get install -y \ - pkg-config \ - libssl-dev \ - curl \ - wget \ - bzip2 \ - libclang-dev \ - clang \ - && rm -rf /var/lib/apt/lists/* - -# Copy sherpa-onnx from kokoro-builder (reuse to avoid duplicate downloads) -COPY --from=kokoro-builder /usr/local/lib/*.so* /usr/local/lib/ -RUN ldconfig - -# Copy only what's needed to build sensevoice plugin -# Note: Cargo.toml needed for workspace dependency resolution in core/ -COPY Cargo.toml Cargo.lock ./ -COPY crates/core ./crates/core -COPY sdks/plugin-sdk ./sdks/plugin-sdk -COPY plugins/native/sensevoice ./plugins/native/sensevoice - -# Build sensevoice plugin -RUN --mount=type=cache,id=cargo-registry-sensevoice,target=/usr/local/cargo/registry \ - --mount=type=cache,id=cargo-git-sensevoice,target=/usr/local/cargo/git \ - --mount=type=cache,target=/build/plugins/native/sensevoice/target \ - cd plugins/native/sensevoice && \ - RUSTFLAGS="-L /usr/local/lib" cargo build --release --target-dir target && \ - mkdir -p /build/dist/native/sensevoice && \ - cp target/release/libsensevoice.so /build/dist/native/sensevoice/ && \ - cp plugin.yml /build/dist/native/sensevoice/ - -# Download SenseVoice models -RUN mkdir -p /build/models && \ - cd /build/models && \ - curl -L -o sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2025-09-09.tar.bz2 \ - https://huggingface.co/streamkit/sensevoice-models/resolve/main/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2025-09-09.tar.bz2 && \ - tar xf sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2025-09-09.tar.bz2 && \ - rm sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2025-09-09.tar.bz2 - -# Stage 8: Build VAD plugin -FROM rust:1.95-slim-bookworm AS vad-builder - -WORKDIR /build - -# Install build dependencies -RUN apt-get update && apt-get install -y \ - pkg-config \ - libssl-dev \ - curl \ - wget \ - bzip2 \ - libclang-dev \ - clang \ - && rm -rf /var/lib/apt/lists/* - -# Copy sherpa-onnx from kokoro-builder (reuse to avoid duplicate downloads) -COPY --from=kokoro-builder /usr/local/lib/*.so* /usr/local/lib/ -RUN ldconfig - -# Copy only what's needed to build vad plugin -# Note: Cargo.toml needed for workspace dependency resolution in core/ -COPY Cargo.toml Cargo.lock ./ -COPY crates/core ./crates/core -COPY sdks/plugin-sdk ./sdks/plugin-sdk -COPY plugins/native/vad ./plugins/native/vad - -# Build vad plugin -RUN --mount=type=cache,id=cargo-registry-vad,target=/usr/local/cargo/registry \ - --mount=type=cache,id=cargo-git-vad,target=/usr/local/cargo/git \ - --mount=type=cache,target=/build/plugins/native/vad/target \ - cd plugins/native/vad && \ - RUSTFLAGS="-L /usr/local/lib" cargo build --release --target-dir target && \ - mkdir -p /build/dist/native/vad && \ - cp target/release/libvad.so /build/dist/native/vad/ && \ - cp plugin.yml /build/dist/native/vad/ - -# Download ten-vad model -RUN mkdir -p /build/models && \ - curl -L -o /build/models/ten-vad.onnx \ - https://huggingface.co/streamkit/vad-models/resolve/main/ten-vad.onnx - -# Stage 9: Build Matcha TTS plugin -FROM rust:1.95-slim-bookworm AS matcha-builder - -WORKDIR /build - -# Install build dependencies -RUN apt-get update && apt-get install -y \ - pkg-config \ - libssl-dev \ - curl \ - wget \ - bzip2 \ - libclang-dev \ - clang \ - && rm -rf /var/lib/apt/lists/* - -# Copy sherpa-onnx from kokoro-builder (reuse to avoid duplicate downloads) -COPY --from=kokoro-builder /usr/local/lib/*.so* /usr/local/lib/ -RUN ldconfig - -# Copy only what's needed to build matcha plugin -# Note: Cargo.toml needed for workspace dependency resolution in core/ -COPY Cargo.toml Cargo.lock ./ -COPY crates/core ./crates/core -COPY sdks/plugin-sdk ./sdks/plugin-sdk -COPY plugins/native/matcha ./plugins/native/matcha - -# Build matcha plugin -RUN --mount=type=cache,id=cargo-registry-matcha,target=/usr/local/cargo/registry \ - --mount=type=cache,id=cargo-git-matcha,target=/usr/local/cargo/git \ - --mount=type=cache,target=/build/plugins/native/matcha/target \ - cd plugins/native/matcha && \ - RUSTFLAGS="-L /usr/local/lib" cargo build --release --target-dir target && \ - mkdir -p /build/dist/native/matcha && \ - cp target/release/libmatcha.so /build/dist/native/matcha/ && \ - cp plugin.yml /build/dist/native/matcha/ - -# Download Matcha TTS models -RUN mkdir -p /build/models && \ - cd /build/models && \ - curl -L -o matcha-icefall-en_US-ljspeech.tar.bz2 \ - https://huggingface.co/streamkit/matcha-models/resolve/main/matcha-icefall-en_US-ljspeech.tar.bz2 && \ - tar xf matcha-icefall-en_US-ljspeech.tar.bz2 && \ - rm matcha-icefall-en_US-ljspeech.tar.bz2 && \ - cd matcha-icefall-en_US-ljspeech && \ - curl -L -o vocos-22khz-univ.onnx \ - https://huggingface.co/streamkit/matcha-models/resolve/main/matcha-icefall-en_US-ljspeech/vocos-22khz-univ.onnx - -# Stage 10: Build Helsinki Translation plugin (CPU-only) -FROM rust:1.95-slim-bookworm AS helsinki-builder - -WORKDIR /build - -# Install dependencies (Rust + Python for model conversion) -RUN apt-get update && apt-get install -y \ - curl \ - git \ - pkg-config \ - libssl-dev \ - libclang-dev \ - clang \ - python3 \ - python3-pip \ - && rm -rf /var/lib/apt/lists/* - -# Copy only what's needed to build helsinki plugin -COPY Cargo.toml Cargo.lock ./ -COPY crates/core ./crates/core -COPY sdks/plugin-sdk ./sdks/plugin-sdk -COPY plugins/native/helsinki ./plugins/native/helsinki - -# Build helsinki plugin (CPU-only, no cuda feature) -RUN --mount=type=cache,id=helsinki-cargo-registry,target=/usr/local/cargo/registry \ - --mount=type=cache,id=helsinki-cargo-git,target=/usr/local/cargo/git \ - --mount=type=cache,id=helsinki-target,target=/build/plugins/native/helsinki/target \ - cd plugins/native/helsinki && \ - cargo build --release --target-dir target && \ - mkdir -p /build/dist/native/helsinki && \ - cp target/release/libhelsinki.so /build/dist/native/helsinki/ && \ - cp plugin.yml /build/dist/native/helsinki/ - -# Download and convert OPUS-MT models (EN<->ES) -RUN PIP_BREAK_SYSTEM_PACKAGES=1 pip3 install --no-cache-dir \ - transformers \ - sentencepiece \ - safetensors \ - torch \ - tokenizers && \ - python3 plugins/native/helsinki/download-models.py - -# Stage 11: Build Pocket TTS plugin -FROM rust:1.95-slim-bookworm AS pocket-tts-builder - -WORKDIR /build - -# Install build dependencies -RUN apt-get update && apt-get install -y \ - pkg-config \ - libssl-dev \ - g++ \ - cmake \ - curl \ - bzip2 \ - libclang-dev \ - clang \ - git \ - && rm -rf /var/lib/apt/lists/* - -# Copy only what's needed to build pocket-tts plugin -COPY Cargo.toml Cargo.lock ./ -COPY crates/core ./crates/core -COPY sdks/plugin-sdk ./sdks/plugin-sdk -COPY plugins/native/pocket-tts ./plugins/native/pocket-tts - -# Build pocket-tts plugin -RUN --mount=type=cache,id=cargo-registry-pocket-tts,target=/usr/local/cargo/registry \ - --mount=type=cache,id=cargo-git-pocket-tts,target=/usr/local/cargo/git \ - --mount=type=cache,id=pocket-tts-target,target=/build/plugins/native/pocket-tts/target \ - cd plugins/native/pocket-tts && \ - cargo build --release --target-dir target && \ - mkdir -p /build/dist/native/pocket-tts && \ - cp target/release/libpocket_tts.so /build/dist/native/pocket-tts/ && \ - cp plugin.yml /build/dist/native/pocket-tts/ - -# Download Pocket TTS models -RUN mkdir -p /build/models && \ - cd /build/models && \ - curl -L -o pocket-tts-b6369a24.tar.bz2 \ - https://huggingface.co/streamkit/pocket-tts-models/resolve/main/pocket-tts-b6369a24.tar.bz2 && \ - tar xf pocket-tts-b6369a24.tar.bz2 && \ - rm pocket-tts-b6369a24.tar.bz2 - -# Stage 12: Build Supertonic TTS plugin -FROM rust:1.95-slim-bookworm AS supertonic-builder - -WORKDIR /build - -# Install build dependencies -RUN apt-get update && apt-get install -y \ - pkg-config \ - libssl-dev \ - g++ \ - cmake \ - curl \ - bzip2 \ - libclang-dev \ - clang \ - git \ - && rm -rf /var/lib/apt/lists/* - -# Copy only what's needed to build supertonic plugin -COPY Cargo.toml Cargo.lock ./ -COPY crates/core ./crates/core -COPY sdks/plugin-sdk ./sdks/plugin-sdk -COPY plugins/native/supertonic ./plugins/native/supertonic - -# Build supertonic plugin -RUN --mount=type=cache,id=cargo-registry-supertonic,target=/usr/local/cargo/registry \ - --mount=type=cache,id=cargo-git-supertonic,target=/usr/local/cargo/git \ - --mount=type=cache,id=supertonic-target,target=/build/plugins/native/supertonic/target \ - cd plugins/native/supertonic && \ - cargo build --release --target-dir target && \ - mkdir -p /build/dist/native/supertonic && \ - cp target/release/libsupertonic.so /build/dist/native/supertonic/ && \ - cp plugin.yml /build/dist/native/supertonic/ - -# Download Supertonic models -RUN mkdir -p /build/models && \ - cd /build/models && \ - curl -L -o supertonic-v2-onnx.tar.bz2 \ - https://huggingface.co/streamkit/supertonic-models/resolve/main/supertonic-v2-onnx.tar.bz2 && \ - tar xf supertonic-v2-onnx.tar.bz2 && \ - rm supertonic-v2-onnx.tar.bz2 - -# Stage 13: Build AAC Encoder plugin -FROM rust:1.95-slim-bookworm AS aac-encoder-builder - -WORKDIR /build - -# Install build dependencies (libfdk-aac-dev needed by shiguredo_fdk_aac) -# libfdk-aac-dev is in Debian's non-free component. -RUN sed -i 's/^Components: main$/Components: main non-free/' /etc/apt/sources.list.d/debian.sources && \ - apt-get update && apt-get install -y \ - pkg-config \ - libssl-dev \ - libclang-dev \ - clang \ - libfdk-aac-dev \ - && rm -rf /var/lib/apt/lists/* - -# Copy only what's needed to build aac-encoder plugin -COPY Cargo.toml Cargo.lock ./ -COPY crates/core ./crates/core -COPY sdks/plugin-sdk ./sdks/plugin-sdk -COPY plugins/native/aac-encoder ./plugins/native/aac-encoder - -# Build aac-encoder plugin -RUN --mount=type=cache,id=cargo-registry-aac-encoder,target=/usr/local/cargo/registry \ - --mount=type=cache,id=cargo-git-aac-encoder,target=/usr/local/cargo/git \ - --mount=type=cache,id=aac-encoder-target,target=/build/plugins/native/aac-encoder/target \ - cd plugins/native/aac-encoder && \ - cargo build --release --target-dir target && \ - mkdir -p /build/dist/native/aac-encoder && \ - cp target/release/libaac_encoder.so /build/dist/native/aac-encoder/ && \ - cp plugin.yml /build/dist/native/aac-encoder/ - -# Stage 14: Build Slint UI plugin -FROM rust:1.95-slim-bookworm AS slint-builder - -WORKDIR /build - -# Install build dependencies (libfontconfig1-dev needed by Slint's software renderer) -RUN apt-get update && apt-get install -y \ - pkg-config \ - libssl-dev \ - g++ \ - cmake \ - curl \ - libclang-dev \ - clang \ - git \ - libfontconfig1-dev \ - && rm -rf /var/lib/apt/lists/* - -# Copy only what's needed to build slint plugin -COPY Cargo.toml Cargo.lock ./ -COPY crates/core ./crates/core -COPY sdks/plugin-sdk ./sdks/plugin-sdk -COPY plugins/native/slint ./plugins/native/slint - -# Build slint plugin -RUN --mount=type=cache,id=cargo-registry-slint,target=/usr/local/cargo/registry \ - --mount=type=cache,id=cargo-git-slint,target=/usr/local/cargo/git \ - --mount=type=cache,id=slint-target,target=/build/plugins/native/slint/target \ - cd plugins/native/slint && \ - cargo build --release --target-dir target && \ - mkdir -p /build/dist/native/slint && \ - cp target/release/libslint.so /build/dist/native/slint/ && \ - cp plugin.yml /build/dist/native/slint/ +# --------------------------------------------------------------- +# Stage: sensevoice bundle + SenseVoice int8 model + Silero VAD. +# --------------------------------------------------------------- +FROM bundle-deps AS sensevoice-bundle +ARG SENSEVOICE_PLUGIN_VERSION=0.3.0 +ARG SENSEVOICE_BUNDLE_SHA256=eeb8b7f8505e7f7d212a9180b8e6449e76a72112395579ace7a87e263b4a26ce +ARG SENSEVOICE_MODEL_SHA256=7305f7905bfcf77fa0b39388a313f3da35c68d971661a65475b56fb2162c8e63 + +WORKDIR /bundle +RUN set -eux; \ + url="https://github.com/streamer45/streamkit/releases/download/plugin-sensevoice-v${SENSEVOICE_PLUGIN_VERSION}/sensevoice-${SENSEVOICE_PLUGIN_VERSION}-bundle.tar.zst"; \ + curl --proto '=https' --tlsv1.2 -fsSL -o sensevoice-bundle.tar.zst "$url"; \ + echo "${SENSEVOICE_BUNDLE_SHA256} sensevoice-bundle.tar.zst" | sha256sum -c -; \ + mkdir -p extracted; \ + tar --zstd -xf sensevoice-bundle.tar.zst -C extracted; \ + test -f extracted/libsensevoice.so + +WORKDIR /models +RUN set -eux; \ + f="sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2025-09-09"; \ + curl --proto '=https' --tlsv1.2 -fsSL -o "${f}.tar.bz2" \ + "https://huggingface.co/streamkit/sensevoice-models/resolve/main/${f}.tar.bz2"; \ + echo "${SENSEVOICE_MODEL_SHA256} ${f}.tar.bz2" | sha256sum -c -; \ + tar xf "${f}.tar.bz2"; \ + rm "${f}.tar.bz2"; \ + test -d "${f}" + +# --------------------------------------------------------------- +# Stage: vad bundle + ten-vad model. +# --------------------------------------------------------------- +FROM bundle-deps AS vad-bundle +ARG VAD_PLUGIN_VERSION=0.4.0 +ARG VAD_BUNDLE_SHA256=5122e4a24598ad761b4dc385e7ebc2343cb7249f7714546a0d0a56d69ab9d957 +ARG TEN_VAD_SHA256=718cb7eef47e3cf5ddbe7e967a7503f46b8b469c0706872f494dfa921b486206 + +WORKDIR /bundle +RUN set -eux; \ + url="https://github.com/streamer45/streamkit/releases/download/plugin-vad-v${VAD_PLUGIN_VERSION}/vad-${VAD_PLUGIN_VERSION}-bundle.tar.zst"; \ + curl --proto '=https' --tlsv1.2 -fsSL -o vad-bundle.tar.zst "$url"; \ + echo "${VAD_BUNDLE_SHA256} vad-bundle.tar.zst" | sha256sum -c -; \ + mkdir -p extracted; \ + tar --zstd -xf vad-bundle.tar.zst -C extracted; \ + test -f extracted/libvad.so + +WORKDIR /models +RUN set -eux; \ + curl --proto '=https' --tlsv1.2 -fsSL -o ten-vad.onnx \ + "https://huggingface.co/streamkit/vad-models/resolve/main/ten-vad.onnx"; \ + echo "${TEN_VAD_SHA256} ten-vad.onnx" | sha256sum -c - + +# --------------------------------------------------------------- +# Stage: matcha bundle + Matcha LJSpeech models. +# --------------------------------------------------------------- +FROM bundle-deps AS matcha-bundle +ARG MATCHA_PLUGIN_VERSION=0.3.0 +ARG MATCHA_BUNDLE_SHA256=46fa83617d0e53d619d9f52ae4b70c9f3a45f31ea9da327ac2bdd012396257bb +ARG MATCHA_MODEL_SHA256=f7862f5d93b956561ee7aca86bf33504f47726c1d5a559066f3cef8fab6c3e23 +ARG MATCHA_VOCOS_SHA256=0574a135aa1db2de6e181050db2ec528496cacd4a4701fc5d7faf9f9804c0081 + +WORKDIR /bundle +RUN set -eux; \ + url="https://github.com/streamer45/streamkit/releases/download/plugin-matcha-v${MATCHA_PLUGIN_VERSION}/matcha-${MATCHA_PLUGIN_VERSION}-bundle.tar.zst"; \ + curl --proto '=https' --tlsv1.2 -fsSL -o matcha-bundle.tar.zst "$url"; \ + echo "${MATCHA_BUNDLE_SHA256} matcha-bundle.tar.zst" | sha256sum -c -; \ + mkdir -p extracted; \ + tar --zstd -xf matcha-bundle.tar.zst -C extracted; \ + test -f extracted/libmatcha.so + +WORKDIR /models +RUN set -eux; \ + curl --proto '=https' --tlsv1.2 -fsSL -o matcha-icefall-en_US-ljspeech.tar.bz2 \ + "https://huggingface.co/streamkit/matcha-models/resolve/main/matcha-icefall-en_US-ljspeech.tar.bz2"; \ + echo "${MATCHA_MODEL_SHA256} matcha-icefall-en_US-ljspeech.tar.bz2" | sha256sum -c -; \ + tar xf matcha-icefall-en_US-ljspeech.tar.bz2; \ + rm matcha-icefall-en_US-ljspeech.tar.bz2; \ + curl --proto '=https' --tlsv1.2 -fsSL -o matcha-icefall-en_US-ljspeech/vocos-22khz-univ.onnx \ + "https://huggingface.co/streamkit/matcha-models/resolve/main/matcha-icefall-en_US-ljspeech/vocos-22khz-univ.onnx"; \ + echo "${MATCHA_VOCOS_SHA256} matcha-icefall-en_US-ljspeech/vocos-22khz-univ.onnx" | sha256sum -c - + +# --------------------------------------------------------------- +# Stage: helsinki bundle + OPUS-MT EN<->ES models. +# The from-source build used a Python/torch conversion step; we now pull +# the pre-converted CTranslate2 model tarballs from streamkit/helsinki-models. +# --------------------------------------------------------------- +FROM bundle-deps AS helsinki-bundle +ARG HELSINKI_PLUGIN_VERSION=0.3.0 +ARG HELSINKI_BUNDLE_SHA256=b8cdfef21d66cfc75596bdb8080cfb20115f6be883c2c1990b2e6edc50190045 +ARG HELSINKI_EN_ES_SHA256=6624ec0babce458c0771f493460e62f7e7dc6d4d832e56dbde1621444d4b37cf +ARG HELSINKI_ES_EN_SHA256=01a0ddd203b3343d02c013539d4d8f0dfcd747585c9b26f0c3f90f7a11d7cde9 + +WORKDIR /bundle +RUN set -eux; \ + url="https://github.com/streamer45/streamkit/releases/download/plugin-helsinki-v${HELSINKI_PLUGIN_VERSION}/helsinki-${HELSINKI_PLUGIN_VERSION}-bundle.tar.zst"; \ + curl --proto '=https' --tlsv1.2 -fsSL -o helsinki-bundle.tar.zst "$url"; \ + echo "${HELSINKI_BUNDLE_SHA256} helsinki-bundle.tar.zst" | sha256sum -c -; \ + mkdir -p extracted; \ + tar --zstd -xf helsinki-bundle.tar.zst -C extracted; \ + test -f extracted/libhelsinki.so + +WORKDIR /models +RUN set -eux; \ + hf_base="https://huggingface.co/streamkit/helsinki-models/resolve/main"; \ + for spec in \ + "opus-mt-en-es ${HELSINKI_EN_ES_SHA256}" \ + "opus-mt-es-en ${HELSINKI_ES_EN_SHA256}" \ + ; do \ + name=$(echo "$spec" | awk '{print $1}'); \ + sha=$(echo "$spec" | awk '{print $2}'); \ + curl --proto '=https' --tlsv1.2 -fsSL -o "${name}.tar.bz2" "${hf_base}/${name}.tar.bz2"; \ + echo "${sha} ${name}.tar.bz2" | sha256sum -c -; \ + tar xf "${name}.tar.bz2"; \ + rm "${name}.tar.bz2"; \ + test -d "${name}"; \ + done + +# --------------------------------------------------------------- +# Stage: pocket-tts bundle + Kyutai Pocket TTS models. +# --------------------------------------------------------------- +FROM bundle-deps AS pocket-tts-bundle +ARG POCKET_TTS_PLUGIN_VERSION=0.3.0 +ARG POCKET_TTS_BUNDLE_SHA256=9f86803c8fce08f84da8484f2e7c38037ed20215df10d0188a423631bc7f76b0 +ARG POCKET_TTS_MODELS_SHA256=7661d610217e8d2b0ae1d8739d384756e50c734fb136047679ca651385ed3035 + +WORKDIR /bundle +RUN set -eux; \ + url="https://github.com/streamer45/streamkit/releases/download/plugin-pocket-tts-v${POCKET_TTS_PLUGIN_VERSION}/pocket-tts-${POCKET_TTS_PLUGIN_VERSION}-bundle.tar.zst"; \ + curl --proto '=https' --tlsv1.2 -fsSL -o pocket-tts-bundle.tar.zst "$url"; \ + echo "${POCKET_TTS_BUNDLE_SHA256} pocket-tts-bundle.tar.zst" | sha256sum -c -; \ + mkdir -p extracted; \ + tar --zstd -xf pocket-tts-bundle.tar.zst -C extracted; \ + test -f extracted/libpocket_tts.so + +WORKDIR /models +RUN set -eux; \ + curl --proto '=https' --tlsv1.2 -fsSL -o pocket-tts-models.tar.bz2 \ + "https://huggingface.co/streamkit/pocket-tts-models/resolve/main/pocket-tts-b6369a24.tar.bz2"; \ + echo "${POCKET_TTS_MODELS_SHA256} pocket-tts-models.tar.bz2" | sha256sum -c -; \ + tar xf pocket-tts-models.tar.bz2; \ + rm pocket-tts-models.tar.bz2; \ + test -d pocket-tts + +# --------------------------------------------------------------- +# Stage: supertonic bundle + Supertonic v2 ONNX models. +# --------------------------------------------------------------- +FROM bundle-deps AS supertonic-bundle +ARG SUPERTONIC_PLUGIN_VERSION=0.3.0 +ARG SUPERTONIC_BUNDLE_SHA256=cc815d2f807021cdd424936fd8c0d84321773c29eeb6b8c79394b92becf8ae02 +ARG SUPERTONIC_MODELS_SHA256=3c3ba6326cd6c8ee48d4c7322322d1f1f4ebf188bf4a7d80fc218babca186f41 + +WORKDIR /bundle +RUN set -eux; \ + url="https://github.com/streamer45/streamkit/releases/download/plugin-supertonic-v${SUPERTONIC_PLUGIN_VERSION}/supertonic-${SUPERTONIC_PLUGIN_VERSION}-bundle.tar.zst"; \ + curl --proto '=https' --tlsv1.2 -fsSL -o supertonic-bundle.tar.zst "$url"; \ + echo "${SUPERTONIC_BUNDLE_SHA256} supertonic-bundle.tar.zst" | sha256sum -c -; \ + mkdir -p extracted; \ + tar --zstd -xf supertonic-bundle.tar.zst -C extracted; \ + test -f extracted/libsupertonic.so + +WORKDIR /models +RUN set -eux; \ + curl --proto '=https' --tlsv1.2 -fsSL -o supertonic-v2-onnx.tar.bz2 \ + "https://huggingface.co/streamkit/supertonic-models/resolve/main/supertonic-v2-onnx.tar.bz2"; \ + echo "${SUPERTONIC_MODELS_SHA256} supertonic-v2-onnx.tar.bz2" | sha256sum -c -; \ + tar xf supertonic-v2-onnx.tar.bz2; \ + rm supertonic-v2-onnx.tar.bz2; \ + test -d supertonic-v2-onnx + +# --------------------------------------------------------------- +# Stage: aac-encoder bundle. No models. +# --------------------------------------------------------------- +FROM bundle-deps AS aac-encoder-bundle +ARG AAC_ENCODER_PLUGIN_VERSION=0.2.0 +ARG AAC_ENCODER_BUNDLE_SHA256=6ccf39fdcc4a9b9984980dad0fa73cbcd9d092323b359538785aa0edd6421e7c + +WORKDIR /bundle +RUN set -eux; \ + url="https://github.com/streamer45/streamkit/releases/download/plugin-aac-encoder-v${AAC_ENCODER_PLUGIN_VERSION}/aac-encoder-${AAC_ENCODER_PLUGIN_VERSION}-bundle.tar.zst"; \ + curl --proto '=https' --tlsv1.2 -fsSL -o aac-encoder-bundle.tar.zst "$url"; \ + echo "${AAC_ENCODER_BUNDLE_SHA256} aac-encoder-bundle.tar.zst" | sha256sum -c -; \ + mkdir -p extracted; \ + tar --zstd -xf aac-encoder-bundle.tar.zst -C extracted; \ + test -f extracted/libaac_encoder.so + +# --------------------------------------------------------------- +# Stage: slint bundle. No models. +# --------------------------------------------------------------- +FROM bundle-deps AS slint-bundle +ARG SLINT_PLUGIN_VERSION=0.4.0 +ARG SLINT_BUNDLE_SHA256=2b33767cdd35b9eeba3bec4e1801aa020a59ef9a61c54b28f7c5d71fa087034d + +WORKDIR /bundle +COPY plugins/native/slint/plugin.yml plugin.yml.src +RUN set -eux; \ + url="https://github.com/streamer45/streamkit/releases/download/plugin-slint-v${SLINT_PLUGIN_VERSION}/slint-${SLINT_PLUGIN_VERSION}-bundle.tar.zst"; \ + curl --proto '=https' --tlsv1.2 -fsSL -o slint-bundle.tar.zst "$url"; \ + echo "${SLINT_BUNDLE_SHA256} slint-bundle.tar.zst" | sha256sum -c -; \ + mkdir -p extracted; \ + tar --zstd -xf slint-bundle.tar.zst -C extracted; \ + test -f extracted/libslint.so; \ + sed "s/^version:.*/version: ${SLINT_PLUGIN_VERSION}/" plugin.yml.src > extracted/plugin.yml # Runtime stage FROM debian:bookworm-slim # Install runtime dependencies (include gdb for debugging demo image crashes) # libfdk-aac2 is in Debian's non-free component. +# sherpa-onnx / ONNX Runtime are NOT installed here: each sherpa-backed +# bundle vendors its own libs (RUNPATH=$ORIGIN). RUN sed -i 's/^Components: main$/Components: main non-free/' /etc/apt/sources.list.d/debian.sources && \ apt-get update && apt-get install -y \ ca-certificates \ @@ -634,58 +464,49 @@ RUN useradd -m -u 1000 -s /bin/bash app # Copy binary from rust builder COPY --from=rust-builder /build/bin/skit /usr/local/bin/skit -# Copy sherpa-onnx shared libraries from kokoro-builder -COPY --from=kokoro-builder /usr/local/lib/*.so* /usr/local/lib/ -RUN ldconfig - -# Copy whisper plugin and models -# Plugins are shipped as directory bundles (plugins/native// with a -# plugin.yml + the .so) — the layout the loader expects; bare .so files in -# plugins/native/ are ignored. -# IMPORTANT: Use --chown=app:app on every COPY to avoid a bulk `chown -R` later -# which would duplicate every file in a new layer (~12GB waste). -COPY --chown=app:app --from=whisper-builder /build/dist /opt/streamkit/plugins -COPY --chown=app:app --from=whisper-builder /build/models /opt/streamkit/models - -# Copy kokoro plugin and models (merge into /opt/streamkit/plugins and /opt/streamkit/models) -COPY --chown=app:app --from=kokoro-builder /build/dist/native/kokoro /opt/streamkit/plugins/native/kokoro -COPY --chown=app:app --from=kokoro-builder /build/models/kokoro-multi-lang-v1_1 /opt/streamkit/models/kokoro-multi-lang-v1_1 - -# Copy piper plugin and models (merge into /opt/streamkit/plugins and /opt/streamkit/models) -COPY --chown=app:app --from=piper-builder /build/dist/native/piper /opt/streamkit/plugins/native/piper -COPY --chown=app:app --from=piper-builder /build/models/vits-piper-en_US-libritts_r-medium /opt/streamkit/models/vits-piper-en_US-libritts_r-medium -COPY --chown=app:app --from=piper-builder /build/models/vits-piper-es_MX-claude-high /opt/streamkit/models/vits-piper-es_MX-claude-high - -# Copy sensevoice plugin and models (merge into /opt/streamkit/plugins and /opt/streamkit/models) -COPY --chown=app:app --from=sensevoice-builder /build/dist/native/sensevoice /opt/streamkit/plugins/native/sensevoice -COPY --chown=app:app --from=sensevoice-builder /build/models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2025-09-09 /opt/streamkit/models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2025-09-09 - -# Copy helsinki plugin and models -COPY --chown=app:app --from=helsinki-builder /build/dist/native/helsinki /opt/streamkit/plugins/native/helsinki -COPY --chown=app:app --from=helsinki-builder /build/models/opus-mt-en-es /opt/streamkit/models/opus-mt-en-es -COPY --chown=app:app --from=helsinki-builder /build/models/opus-mt-es-en /opt/streamkit/models/opus-mt-es-en - -# Copy vad plugin and model (merge into /opt/streamkit/plugins and /opt/streamkit/models) -COPY --chown=app:app --from=vad-builder /build/dist/native/vad /opt/streamkit/plugins/native/vad -COPY --chown=app:app --from=vad-builder /build/models/ten-vad.onnx /opt/streamkit/models/ten-vad.onnx - -# Copy matcha plugin and models (merge into /opt/streamkit/plugins and /opt/streamkit/models) -COPY --chown=app:app --from=matcha-builder /build/dist/native/matcha /opt/streamkit/plugins/native/matcha -COPY --chown=app:app --from=matcha-builder /build/models/matcha-icefall-en_US-ljspeech /opt/streamkit/models/matcha-icefall-en_US-ljspeech - -# Copy pocket-tts plugin and models -COPY --chown=app:app --from=pocket-tts-builder /build/dist/native/pocket-tts /opt/streamkit/plugins/native/pocket-tts -COPY --chown=app:app --from=pocket-tts-builder /build/models/pocket-tts /opt/streamkit/models/pocket-tts - -# Copy supertonic plugin and models -COPY --chown=app:app --from=supertonic-builder /build/dist/native/supertonic /opt/streamkit/plugins/native/supertonic -COPY --chown=app:app --from=supertonic-builder /build/models/supertonic-v2-onnx /opt/streamkit/models/supertonic-v2-onnx - -# Copy slint plugin (no models needed) -COPY --chown=app:app --from=slint-builder /build/dist/native/slint /opt/streamkit/plugins/native/slint - -# Copy aac-encoder plugin (no models needed) -COPY --chown=app:app --from=aac-encoder-builder /build/dist/native/aac-encoder /opt/streamkit/plugins/native/aac-encoder +# Plugins are shipped as directory bundles (plugins/native// with the +# .so + embedded manifest.json + any vendored libs) -- the layout the +# loader expects. Each marketplace bundle is copied verbatim from its +# fetch stage so $ORIGIN-relative vendored libs stay co-located. +# IMPORTANT: Use --chown=app:app on every COPY to avoid a bulk `chown -R` +# later which would duplicate every file in a new layer. +COPY --chown=app:app --from=whisper-bundle /bundle/extracted /opt/streamkit/plugins/native/whisper +COPY --chown=app:app --from=whisper-bundle /models/ggml-tiny-q5_1.bin /opt/streamkit/models/ggml-tiny-q5_1.bin +COPY --chown=app:app --from=whisper-bundle /models/silero_vad.onnx /opt/streamkit/models/silero_vad.onnx + +COPY --chown=app:app --from=kokoro-bundle /bundle/extracted /opt/streamkit/plugins/native/kokoro +COPY --chown=app:app --from=kokoro-bundle /models/kokoro-multi-lang-v1_1 /opt/streamkit/models/kokoro-multi-lang-v1_1 + +COPY --chown=app:app --from=piper-bundle /bundle/extracted /opt/streamkit/plugins/native/piper +COPY --chown=app:app --from=piper-bundle /models/vits-piper-en_US-libritts_r-medium /opt/streamkit/models/vits-piper-en_US-libritts_r-medium +COPY --chown=app:app --from=piper-bundle /models/vits-piper-es_MX-claude-high /opt/streamkit/models/vits-piper-es_MX-claude-high + +COPY --chown=app:app --from=sensevoice-bundle /bundle/extracted /opt/streamkit/plugins/native/sensevoice +COPY --chown=app:app --from=sensevoice-bundle /models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2025-09-09 /opt/streamkit/models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2025-09-09 + +COPY --chown=app:app --from=helsinki-bundle /bundle/extracted /opt/streamkit/plugins/native/helsinki +COPY --chown=app:app --from=helsinki-bundle /models/opus-mt-en-es /opt/streamkit/models/opus-mt-en-es +COPY --chown=app:app --from=helsinki-bundle /models/opus-mt-es-en /opt/streamkit/models/opus-mt-es-en + +COPY --chown=app:app --from=vad-bundle /bundle/extracted /opt/streamkit/plugins/native/vad +COPY --chown=app:app --from=vad-bundle /models/ten-vad.onnx /opt/streamkit/models/ten-vad.onnx + +COPY --chown=app:app --from=matcha-bundle /bundle/extracted /opt/streamkit/plugins/native/matcha +COPY --chown=app:app --from=matcha-bundle /models/matcha-icefall-en_US-ljspeech /opt/streamkit/models/matcha-icefall-en_US-ljspeech + +COPY --chown=app:app --from=pocket-tts-bundle /bundle/extracted /opt/streamkit/plugins/native/pocket-tts +COPY --chown=app:app --from=pocket-tts-bundle /models/pocket-tts /opt/streamkit/models/pocket-tts + +COPY --chown=app:app --from=supertonic-bundle /bundle/extracted /opt/streamkit/plugins/native/supertonic +COPY --chown=app:app --from=supertonic-bundle /models/supertonic-v2-onnx /opt/streamkit/models/supertonic-v2-onnx + +# The pinned slint bundle predates self-describing bundles, so the slint-bundle +# stage injects a plugin.yml (version-matched to the pinned bundle) to keep the +# "Slint Files" asset type + /api/v1/assets/plugin/slint endpoints. Drop the +# injection once SLINT_PLUGIN_VERSION points at a self-describing bundle. +COPY --chown=app:app --from=slint-bundle /bundle/extracted /opt/streamkit/plugins/native/slint + +COPY --chown=app:app --from=aac-encoder-bundle /bundle/extracted /opt/streamkit/plugins/native/aac-encoder # Copy sample pipelines + small bundled audio samples (Opus/Ogg only) COPY --chown=app:app samples/pipelines /opt/streamkit/samples/pipelines diff --git a/agent_docs/adding-plugins.md b/agent_docs/adding-plugins.md index 0a0bee6ae..7b6a1655b 100644 --- a/agent_docs/adding-plugins.md +++ b/agent_docs/adding-plugins.md @@ -31,3 +31,28 @@ the following: Hugging Face repo so they remain accessible indefinitely (license permitting). - **Human review required** before bundling any new third-party shared libraries (licensing, security, size, and distro compatibility). + +## GPU (CUDA) bundle variants + +A plugin can ship an optional `cuda` bundle variant alongside its canonical CPU +bundle. Clients auto-detect CUDA at install time (or honour an explicit +`accelerator`) and fall back to the CPU bundle when no GPU is present. To make a +plugin CUDA-capable: + +- Declare `accelerators: [cpu, cuda]` in `plugins/native//plugin.yml` (the + default is CPU-only). This is propagated into `marketplace/official-plugins.json`. +- For compile-time GPU plugins (e.g. whisper, helsinki), add a `cuda` Cargo + feature that enables the backend's CUDA path. `build_official_plugins_cuda.sh` + builds with `--features cuda` automatically when the feature exists. +- sherpa-onnx plugins (kokoro, sensevoice, vad, matcha) are execution-provider + agnostic: the same `.so` is repackaged against the CUDA sherpa runtime, so no + feature flag is needed — just the `accelerators` declaration. +- The CUDA registry pass runs on the self-hosted Ada GPU runner in + `.github/workflows/marketplace-build.yml` (`build-marketplace-cuda` job). It + vendors the GPU ONNX Runtime provider libs (`libonnxruntime_providers_cuda.so`, + `libonnxruntime_providers_shared.so`) and layers a `cuda` variant onto the + already-published CPU manifest (append-only; a published variant is immutable). +- CUDA bundles are named `--cuda-bundle.tar.zst` and uploaded to the + same per-plugin release as the CPU bundle. `verify_bundles.py --accelerator + cuda` permits CUDA NEEDED/RUNPATH deps (libcudart/libcublas/libcudnn) that the + strict CPU gate rejects. diff --git a/apps/skit/Cargo.toml b/apps/skit/Cargo.toml index 0a7c38ea9..9edf31d12 100644 --- a/apps/skit/Cargo.toml +++ b/apps/skit/Cargo.toml @@ -38,6 +38,9 @@ schemars = "1.2.0" base64 = "0.22" semver = "1.0" +# For probing CUDA driver availability when selecting plugin bundle variants +libloading = "0.9" + # For serializing the default config into TOML format toml = "1.1" diff --git a/apps/skit/src/config.rs b/apps/skit/src/config.rs index 7aed38349..7c9e180af 100644 --- a/apps/skit/src/config.rs +++ b/apps/skit/src/config.rs @@ -587,6 +587,12 @@ pub struct PluginMarketplaceConfig { /// Native plugins run in-process and are unsafe without full trust. #[serde(default)] pub allow_native_marketplace: bool, + /// Default accelerator for bundle variant selection (e.g. `"cpu"` or + /// `"cuda"`). When unset, the installer auto-detects CUDA availability and + /// otherwise falls back to the CPU bundle. A per-install request can + /// override this. + #[serde(default)] + pub default_accelerator: Option, #[serde(flatten, default)] pub security: PluginMarketplaceSecurityConfig, } diff --git a/apps/skit/src/marketplace.rs b/apps/skit/src/marketplace.rs index dc50af9a5..0e7d0ab82 100644 --- a/apps/skit/src/marketplace.rs +++ b/apps/skit/src/marketplace.rs @@ -227,8 +227,15 @@ pub struct PluginManifest { pub entrypoint: String, /// Marketplace bundle info. Required for marketplace-distributed plugins; /// absent for local-only plugins that ship alongside their `.so`. + /// + /// This is the canonical **CPU** bundle. Accelerator-specific builds (e.g. + /// CUDA) live in [`PluginManifest::variants`]. #[serde(default)] pub bundle: Option, + /// Accelerator-specific bundle variants. Empty means CPU-only (use + /// [`PluginManifest::bundle`]). Older manifests omit this field entirely. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub variants: Vec, pub compatibility: Option, #[serde(default)] pub models: Vec, @@ -241,6 +248,40 @@ pub struct PluginManifest { pub assets: Vec, } +impl PluginManifest { + /// Selects the bundle to install for the requested `accelerator`. + /// + /// `Some("cuda")` (or any non-`"cpu"` tag) matches a [`variants`] entry by + /// accelerator; if no variant matches — including `None`, `"cpu"`, or an + /// unknown tag — this falls back to the canonical CPU [`bundle`]. + /// + /// [`variants`]: PluginManifest::variants + /// [`bundle`]: PluginManifest::bundle + #[must_use] + pub fn resolve_bundle(&self, accelerator: Option<&str>) -> Option> { + if let Some(acc) = accelerator { + if !acc.eq_ignore_ascii_case("cpu") { + if let Some(variant) = + self.variants.iter().find(|v| v.accelerator.eq_ignore_ascii_case(acc)) + { + return Some(ResolvedBundle { + accelerator: &variant.accelerator, + url: &variant.url, + sha256: &variant.sha256, + size_bytes: variant.size_bytes, + }); + } + } + } + self.bundle.as_ref().map(|bundle| ResolvedBundle { + accelerator: "cpu", + url: &bundle.url, + sha256: &bundle.sha256, + size_bytes: bundle.size_bytes, + }) + } +} + /// An asset type declared by a plugin in its manifest. /// /// Each spec causes the server to register generic CRUD endpoints and @@ -299,6 +340,29 @@ pub struct PluginBundle { pub size_bytes: Option, } +/// An accelerator-specific bundle variant (e.g. a CUDA build). +/// +/// The canonical CPU bundle stays in [`PluginManifest::bundle`]; `variants` +/// carries additional builds tagged by [`PluginBundleVariant::accelerator`] +/// (currently `"cpu"` or `"cuda"`). Clients select a variant at install time. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PluginBundleVariant { + pub accelerator: String, + pub url: String, + pub sha256: String, + pub size_bytes: Option, +} + +/// A bundle chosen by [`PluginManifest::resolve_bundle`], borrowing from the +/// canonical bundle or one of its variants. +#[derive(Debug, Clone, Copy)] +pub struct ResolvedBundle<'a> { + pub accelerator: &'a str, + pub url: &'a str, + pub sha256: &'a str, + pub size_bytes: Option, +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PluginCompatibility { pub streamkit: Option, @@ -600,6 +664,84 @@ mod tests { }; use crate::marketplace_security::origin_key; + fn manifest_with_variants( + bundle: Option, + variants: Vec, + ) -> PluginManifest { + PluginManifest { + schema_version: 1, + id: "demo".to_string(), + name: None, + version: "1.0.0".to_string(), + node_kind: "demo".to_string(), + kind: PluginKind::Native, + description: None, + license: None, + license_url: None, + homepage: None, + repository: None, + entrypoint: "libdemo.so".to_string(), + bundle, + variants, + compatibility: None, + models: Vec::new(), + assets: Vec::new(), + } + } + + fn cpu_bundle() -> PluginBundle { + PluginBundle { + url: "https://example.com/demo-1.0.0-bundle.tar.zst".to_string(), + sha256: "cpu".to_string(), + size_bytes: Some(1), + } + } + + fn cuda_variant() -> PluginBundleVariant { + PluginBundleVariant { + accelerator: "cuda".to_string(), + url: "https://example.com/demo-1.0.0-cuda-bundle.tar.zst".to_string(), + sha256: "cuda".to_string(), + size_bytes: Some(2), + } + } + + #[test] + fn resolve_bundle_defaults_to_cpu() { + let manifest = manifest_with_variants(Some(cpu_bundle()), vec![cuda_variant()]); + let resolved = manifest.resolve_bundle(None).unwrap(); + assert_eq!(resolved.accelerator, "cpu"); + assert_eq!(resolved.sha256, "cpu"); + } + + #[test] + fn resolve_bundle_selects_matching_variant() { + let manifest = manifest_with_variants(Some(cpu_bundle()), vec![cuda_variant()]); + let resolved = manifest.resolve_bundle(Some("cuda")).unwrap(); + assert_eq!(resolved.accelerator, "cuda"); + assert_eq!(resolved.sha256, "cuda"); + } + + #[test] + fn resolve_bundle_falls_back_when_variant_missing() { + let manifest = manifest_with_variants(Some(cpu_bundle()), Vec::new()); + let resolved = manifest.resolve_bundle(Some("cuda")).unwrap(); + assert_eq!(resolved.accelerator, "cpu"); + } + + #[test] + fn resolve_bundle_cpu_request_ignores_variants() { + let manifest = manifest_with_variants(Some(cpu_bundle()), vec![cuda_variant()]); + let resolved = manifest.resolve_bundle(Some("cpu")).unwrap(); + assert_eq!(resolved.accelerator, "cpu"); + } + + #[test] + fn resolve_bundle_none_when_no_bundle() { + let manifest = manifest_with_variants(None, Vec::new()); + assert!(manifest.resolve_bundle(Some("cuda")).is_none()); + } + fn permissive_policy() -> MarketplaceUrlPolicy { let config = PluginConfig { marketplace: PluginMarketplaceConfig { diff --git a/apps/skit/src/marketplace_installer.rs b/apps/skit/src/marketplace_installer.rs index c6809f70f..117e10d58 100644 --- a/apps/skit/src/marketplace_installer.rs +++ b/apps/skit/src/marketplace_installer.rs @@ -6,7 +6,7 @@ use std::{ collections::{HashMap, HashSet, VecDeque}, fmt::Write, path::{Component, Path, PathBuf}, - sync::Arc, + sync::{Arc, OnceLock}, time::{Duration, SystemTime, UNIX_EPOCH}, }; @@ -58,6 +58,11 @@ pub struct InstallPluginRequest { pub install_models: bool, #[serde(default)] pub model_ids: Option>, + /// Accelerator variant to install (e.g. `"cpu"` or `"cuda"`). When `None`, + /// the installer uses the configured default or auto-detects CUDA, falling + /// back to the canonical CPU bundle. + #[serde(default)] + pub accelerator: Option, } #[derive(Debug, Clone, Serialize)] @@ -181,6 +186,12 @@ impl InstallJobQueue { allow_model_urls: config.marketplace.security.allow_model_urls, marketplace_policy, registries: config.registries.clone(), + default_accelerator: config + .marketplace + .default_accelerator + .as_ref() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()), }, )?; Ok(Self { @@ -511,6 +522,7 @@ struct PluginInstaller { allow_model_urls: bool, marketplace_policy: MarketplaceUrlPolicy, registries: Vec, + default_accelerator: Option, } struct PluginInstallerSettings { @@ -521,6 +533,7 @@ struct PluginInstallerSettings { allow_model_urls: bool, marketplace_policy: MarketplaceUrlPolicy, registries: Vec, + default_accelerator: Option, } struct DownloadModelRequest<'a> { @@ -562,6 +575,7 @@ impl PluginInstaller { allow_model_urls: settings.allow_model_urls, marketplace_policy: settings.marketplace_policy, registries: settings.registries, + default_accelerator: settings.default_accelerator, }) } @@ -782,70 +796,130 @@ impl PluginInstaller { }, }; - let bundle_dir = self.plugin_dir.join("bundles").join(&manifest.id).join(&manifest.version); - if bundle_dir.exists() { - if !request.install_models { - let err = anyhow!("Bundle version '{}' is already installed", manifest.version); + // Resolve the requested accelerator once. `resolve_accelerator` probes + // the CUDA runtime, so threading the result through `download_bundle` + // avoids re-probing and keeps the recorded and downloaded variants in + // lockstep. + let requested_accelerator = self.resolve_accelerator(request.accelerator.as_deref()); + let selected_accelerator = + if let Some(bundle) = manifest.resolve_bundle(requested_accelerator.as_deref()) { + bundle.accelerator.to_ascii_lowercase() + } else { + let err = anyhow!("Plugin manifest missing required `bundle` section"); tracker.fail_step(STEP_DOWNLOAD_BUNDLE, err.to_string()).await; return Err(err.into()); - } + }; + if let Err(err) = + plugin_paths::validate_path_component("accelerator", &selected_accelerator) + { + tracker.fail_step(STEP_DOWNLOAD_BUNDLE, err.to_string()).await; + return Err(err.into()); + } + + // Install dirs are keyed by id + version + accelerator so CPU and CUDA + // builds of the same version coexist and switching variants never + // silently no-ops. + let bundle_dir = self + .plugin_dir + .join("bundles") + .join(&manifest.id) + .join(&manifest.version) + .join(&selected_accelerator); + + let bundle_dir = if bundle_dir.exists() { if let Err(err) = plugin_paths::ensure_existing_dir_under(&base_real, &bundle_dir, "bundle").await { tracker.fail_step(STEP_DOWNLOAD_BUNDLE, err.to_string()).await; return Err(err.into()); } + + let already_active = + self.read_active_record(&manifest.id).await.is_some_and(|record| { + record.version == manifest.version + && record.accelerator.eq_ignore_ascii_case(&selected_accelerator) + }); + if already_active && !request.install_models { + let err = anyhow!( + "Plugin '{}' v{} ({} variant) is already installed", + manifest.id, + manifest.version, + selected_accelerator, + ); + tracker.fail_step(STEP_DOWNLOAD_BUNDLE, err.to_string()).await; + return Err(err.into()); + } + tracker.succeed_step(STEP_DOWNLOAD_BUNDLE).await; - for step in [STEP_EXTRACT_BUNDLE, STEP_ACTIVATE, STEP_LOAD_PLUGIN] { - Self::mark_step_succeeded(tracker, step).await; + Self::mark_step_succeeded(tracker, STEP_EXTRACT_BUNDLE).await; + + if already_active { + for step in [STEP_ACTIVATE, STEP_LOAD_PLUGIN] { + Self::mark_step_succeeded(tracker, step).await; + } + return Ok(()); } - return Ok(()); - } - let bundle_path = match self - .download_bundle(manifest, tracker, cancel, registry_origin, &base_real) - .await - { - Ok(path) => path, - Err(InstallError::Cancelled) => return Err(InstallError::Cancelled), - Err(InstallError::Other(err)) => { - tracker.fail_step(STEP_DOWNLOAD_BUNDLE, err.to_string()).await; - return Err(InstallError::Other(err)); - }, - }; - tracker.succeed_step(STEP_DOWNLOAD_BUNDLE).await; + bundle_dir + } else { + let bundle_path = match self + .download_bundle( + manifest, + tracker, + cancel, + registry_origin, + &base_real, + requested_accelerator.as_deref(), + ) + .await + { + Ok(path) => path, + Err(InstallError::Cancelled) => return Err(InstallError::Cancelled), + Err(InstallError::Other(err)) => { + tracker.fail_step(STEP_DOWNLOAD_BUNDLE, err.to_string()).await; + return Err(InstallError::Other(err)); + }, + }; + tracker.succeed_step(STEP_DOWNLOAD_BUNDLE).await; - Self::ensure_not_cancelled(cancel)?; + Self::ensure_not_cancelled(cancel)?; - tracker.start_step(STEP_EXTRACT_BUNDLE).await; - let bundle_dir = match self.extract_bundle(manifest, &bundle_path, &base_real, cancel).await - { - Ok(dir) => dir, - Err(InstallError::Cancelled) => return Err(InstallError::Cancelled), - Err(InstallError::Other(err)) => { - tracker.fail_step(STEP_EXTRACT_BUNDLE, err.to_string()).await; - return Err(InstallError::Other(err)); - }, - }; - tracker.succeed_step(STEP_EXTRACT_BUNDLE).await; + tracker.start_step(STEP_EXTRACT_BUNDLE).await; + let bundle_dir = match self + .extract_bundle(manifest, &selected_accelerator, &bundle_path, &base_real, cancel) + .await + { + Ok(dir) => dir, + Err(InstallError::Cancelled) => return Err(InstallError::Cancelled), + Err(InstallError::Other(err)) => { + tracker.fail_step(STEP_EXTRACT_BUNDLE, err.to_string()).await; + return Err(InstallError::Other(err)); + }, + }; + tracker.succeed_step(STEP_EXTRACT_BUNDLE).await; + + // The local loader rediscovers asset types from a YAML plugin.yml + // beside the entrypoint, not from the marketplace JSON manifest, so + // serialize one here for restart survival. + if let Err(err) = write_manifest_yml(manifest, &bundle_dir).await { + tracing::warn!( + plugin_id = %manifest.id, + error = %err, + "Failed to write plugin.yml into bundle directory; \ + asset types may not survive restart" + ); + } - // Write a plugin.yml into the bundle directory so that - // `read_local_plugin_manifest` can rediscover asset types on server - // restart. The marketplace manifest is JSON; the local loader expects - // YAML, so we serialize the manifest here. - if let Err(err) = write_manifest_yml(manifest, &bundle_dir).await { - tracing::warn!( - plugin_id = %manifest.id, - error = %err, - "Failed to write plugin.yml into bundle directory; \ - asset types may not survive restart" - ); - } + bundle_dir + }; Self::ensure_not_cancelled(cancel)?; tracker.start_step(STEP_ACTIVATE).await; - let entrypoint_path = match self.activate_bundle(manifest, &bundle_dir, &base_real).await { + let entrypoint_path = match self + .activate_bundle(manifest, &selected_accelerator, &bundle_dir, &base_real) + .await + { Ok(path) => path, Err(InstallError::Cancelled) => return Err(InstallError::Cancelled), Err(InstallError::Other(err)) => { @@ -858,7 +932,10 @@ impl PluginInstaller { Self::ensure_not_cancelled(cancel)?; tracker.start_step(STEP_LOAD_PLUGIN).await; - match self.load_plugin(manifest, &entrypoint_path, namespaced_kind).await { + match self + .load_plugin(manifest, &entrypoint_path, namespaced_kind, &selected_accelerator) + .await + { Ok(_) => { tracker.succeed_step(STEP_LOAD_PLUGIN).await; }, @@ -938,6 +1015,30 @@ impl PluginInstaller { Err(anyhow!("Registry '{registry}' is not configured")) } + /// Determines the accelerator to install for, honoring (in order) an + /// explicit per-request value, the configured default, and finally runtime + /// CUDA auto-detection. `None` means "use the canonical CPU bundle". + fn resolve_accelerator(&self, requested: Option<&str>) -> Option { + if let Some(value) = requested.map(str::trim).filter(|value| !value.is_empty()) { + return Some(value.to_string()); + } + if let Some(value) = + self.default_accelerator.as_deref().map(str::trim).filter(|value| !value.is_empty()) + { + return Some(value.to_string()); + } + if cuda_runtime_available() { + return Some("cuda".to_string()); + } + None + } + + async fn read_active_record(&self, plugin_id: &str) -> Option { + let record_path = plugin_record_path(&self.plugin_dir, plugin_id).ok()?; + let bytes = tokio::fs::read(&record_path).await.ok()?; + serde_json::from_slice(&bytes).ok() + } + async fn download_bundle( &self, manifest: &crate::marketplace::PluginManifest, @@ -945,13 +1046,22 @@ impl PluginInstaller { cancel: &CancellationToken, registry_origin: &OriginKey, base_real: &Path, + accelerator: Option<&str>, ) -> Result { - let bundle = manifest.bundle.as_ref().ok_or_else(|| { + let bundle = manifest.resolve_bundle(accelerator).ok_or_else(|| { InstallError::Other(anyhow!("Plugin manifest missing required `bundle` section")) })?; + if let Some(requested) = accelerator { + tracing::info!( + plugin_id = %manifest.id, + requested_accelerator = requested, + selected_accelerator = bundle.accelerator, + "Resolved plugin bundle variant" + ); + } let bundle_url = self .marketplace_policy - .validate_url("bundle url", &bundle.url, Some(registry_origin)) + .validate_url("bundle url", bundle.url, Some(registry_origin)) .await?; let cache_dir = self.plugin_dir.join("cache").join(&manifest.id).join(&manifest.version); plugin_paths::ensure_dir_under(base_real, &cache_dir, "cache").await?; @@ -991,7 +1101,7 @@ impl PluginInstaller { // For 206 responses, content_length is the remaining bytes. response.content_length().map(|cl| cl + resume_from.unwrap_or(0)) } else { - response.content_length() + response.content_length().or(bundle.size_bytes) }; let mut stream = response.bytes_stream(); @@ -1055,8 +1165,8 @@ impl PluginInstaller { })?; let actual_hash = to_hex(&hasher.finalize()); - if !actual_hash.eq_ignore_ascii_case(&bundle.sha256) { - let expected = bundle.sha256.as_str(); + if !actual_hash.eq_ignore_ascii_case(bundle.sha256) { + let expected = bundle.sha256; let actual = actual_hash.as_str(); hash_mismatch = true; return Err( @@ -1092,6 +1202,7 @@ impl PluginInstaller { async fn extract_bundle( &self, manifest: &crate::marketplace::PluginManifest, + accelerator: &str, bundle_path: &Path, base_real: &Path, cancel: &CancellationToken, @@ -1100,17 +1211,20 @@ impl PluginInstaller { return Err(InstallError::Cancelled); } - let bundles_root = self.plugin_dir.join("bundles").join(&manifest.id); - plugin_paths::ensure_dir_under(base_real, &bundles_root, "bundles").await?; + let version_dir = + self.plugin_dir.join("bundles").join(&manifest.id).join(&manifest.version); + plugin_paths::ensure_dir_under(base_real, &version_dir, "bundles").await?; - let bundle_dir = bundles_root.join(&manifest.version); + let bundle_dir = version_dir.join(accelerator); if bundle_dir.exists() { let version = manifest.version.as_str(); - return Err(anyhow!("Bundle version '{version}' is already installed").into()); + return Err( + anyhow!("Bundle version '{version}' ({accelerator}) is already installed").into() + ); } let temp_id = Uuid::new_v4(); - let temp_dir = bundles_root.join(format!(".tmp-{temp_id}")); + let temp_dir = version_dir.join(format!(".tmp-{temp_id}")); tokio::fs::create_dir_all(&temp_dir).await.with_context(|| { format!("Failed to create temp dir {temp_dir}", temp_dir = temp_dir.display()) })?; @@ -1180,6 +1294,7 @@ impl PluginInstaller { async fn activate_bundle( &self, manifest: &crate::marketplace::PluginManifest, + accelerator: &str, bundle_dir: &Path, base_real: &Path, ) -> Result { @@ -1194,6 +1309,7 @@ impl PluginInstaller { kind: manifest.kind.clone(), entrypoint: entrypoint_path.to_string_lossy().into_owned(), installed_at_ms: now_ms(), + accelerator: accelerator.to_string(), }; let record_path = plugin_record_path(&self.plugin_dir, &manifest.id)?; let payload = serde_json::to_vec_pretty(&record) @@ -1213,6 +1329,7 @@ impl PluginInstaller { manifest: &crate::marketplace::PluginManifest, entrypoint_path: &Path, expected_kind: &str, + accelerator: &str, ) -> Result { let plugin_type = match manifest.kind { PluginKind::Wasm => PluginType::Wasm, @@ -1245,12 +1362,23 @@ impl PluginInstaller { self.plugin_asset_registry.unregister_plugin(&manifest.id).await; } - let summary = tokio::task::spawn_blocking(move || { - let mut mgr = manager.blocking_lock(); - mgr.load_from_path(plugin_type, entrypoint_path) + let accelerator = accelerator.to_ascii_lowercase(); + let mut summary = tokio::task::spawn_blocking({ + let expected_kind = expected_kind_owned.clone(); + let accelerator = accelerator.clone(); + move || { + let mut mgr = manager.blocking_lock(); + let summary = mgr.load_from_path(plugin_type, entrypoint_path)?; + if summary.kind == expected_kind { + mgr.set_plugin_accelerator(&summary.kind, Some(accelerator)); + } + drop(mgr); + anyhow::Ok(summary) + } }) .await .context("Plugin load task failed")??; + summary.accelerator = Some(accelerator); if summary.kind != expected_kind { let manager = Arc::clone(&self.plugin_manager); @@ -2034,6 +2162,40 @@ fn now_ms() -> u128 { SystemTime::now().duration_since(UNIX_EPOCH).map_or(0, |duration| duration.as_millis()) } +/// System libraries that must all be loadable before a host is considered able +/// to run a CUDA plugin bundle. `libcuda.so.1` is the driver stub, but the +/// bundles also link the CUDA *runtime* (`libcudart`), which is absent on +/// driver-only hosts; probing the driver alone would auto-select a CUDA bundle +/// that then fails to load. +const CUDA_RUNTIME_LIBS: &[&str] = &["libcuda.so.1", "libcudart.so"]; + +/// Returns `true` if every named library is loadable via `dlopen`. +fn libs_loadable(names: &[&str]) -> bool { + names.iter().all(|name| { + // SAFETY: We only load well-known system libraries and never call into + // them; `libloading` unloads them on drop. No symbols are dereferenced. + // allow(unsafe_code): dlopen is the only portable way to probe for the + // CUDA stack at runtime; the load is side-effect free. + #[allow(unsafe_code)] + let loaded = unsafe { libloading::Library::new(*name) }.is_ok(); + loaded + }) +} + +/// Returns `true` if the full CUDA runtime stack is loadable, used to +/// auto-select CUDA plugin bundle variants. The result is cached for the +/// process lifetime. +fn cuda_runtime_available() -> bool { + static AVAILABLE: OnceLock = OnceLock::new(); + *AVAILABLE.get_or_init(|| { + let available = libs_loadable(CUDA_RUNTIME_LIBS); + if available { + tracing::debug!("CUDA runtime detected (driver + libcudart loadable)"); + } + available + }) +} + #[cfg(test)] mod tests { use super::*; @@ -2051,6 +2213,13 @@ mod tests { use tokio::sync::oneshot; use tokio::task::JoinHandle; + #[test] + fn libs_loadable_requires_all_named_libraries() { + assert!(libs_loadable(&[])); + assert!(!libs_loadable(&["libdefinitely-not-real-streamkit.so.999"])); + assert!(!libs_loadable(&["libc.so.6", "libdefinitely-not-real.so.999"])); + } + fn make_job(status: JobStatus) -> InstallJob { InstallJob { info: JobInfo { @@ -2067,6 +2236,7 @@ mod tests { version: None, install_models: false, model_ids: None, + accelerator: None, }, permissions: Permissions::default(), } @@ -2175,6 +2345,7 @@ mod tests { sha256: "deadbeef".to_string(), size_bytes: None, }), + variants: Vec::new(), compatibility: None, models, assets: Vec::new(), @@ -2221,6 +2392,7 @@ mod tests { marketplace_url_allowlist: vec!["http://127.0.0.1:*".to_string()], ..crate::config::PluginMarketplaceSecurityConfig::default() }, + default_accelerator: None, }, trusted_pubkeys: Vec::new(), registries: Vec::new(), @@ -2324,6 +2496,7 @@ mod tests { marketplace_url_allowlist: vec!["http://127.0.0.1:*".to_string()], ..crate::config::PluginMarketplaceSecurityConfig::default() }, + default_accelerator: None, }, trusted_pubkeys: Vec::new(), registries: Vec::new(), @@ -2427,6 +2600,7 @@ mod tests { marketplace_url_allowlist: vec!["http://127.0.0.1:*".to_string()], ..crate::config::PluginMarketplaceSecurityConfig::default() }, + default_accelerator: None, }, trusted_pubkeys: Vec::new(), registries: Vec::new(), @@ -2493,6 +2667,7 @@ mod tests { allow_model_urls: false, ..crate::config::PluginMarketplaceSecurityConfig::default() }, + default_accelerator: None, }, trusted_pubkeys: Vec::new(), registries: Vec::new(), @@ -3050,6 +3225,7 @@ mod tests { marketplace_enabled: true, allow_native_marketplace: true, security, + default_accelerator: None, }, trusted_pubkeys, registries, @@ -3111,6 +3287,7 @@ mod tests { version: None, install_models: false, model_ids: None, + accelerator: None, }; let job_id = queue.enqueue(request, Permissions::default()).await; @@ -3339,6 +3516,7 @@ mod tests { sha256: bundle_sha, size_bytes: None, }), + variants: Vec::new(), compatibility: None, models: Vec::new(), assets: Vec::new(), @@ -3394,6 +3572,7 @@ mod tests { version: None, install_models: false, model_ids: None, + accelerator: None, }; // Loading a fabricated native library fails at dlopen, so install() @@ -3416,7 +3595,7 @@ mod tests { assert!(step_succeeded(STEP_EXTRACT_BUNDLE), "bundle extraction should succeed"); assert!(step_succeeded(STEP_ACTIVATE), "activation should succeed"); - let bundle_dir = plugin_dir.join("bundles").join("demo").join("1.0.0"); + let bundle_dir = plugin_dir.join("bundles").join("demo").join("1.0.0").join("cpu"); assert!(bundle_dir.join("libdemo.so").exists(), "entrypoint should be extracted"); let _ = shutdown_tx.send(()); @@ -3424,6 +3603,152 @@ mod tests { Ok(()) } + #[tokio::test] + async fn install_keys_bundle_dir_by_accelerator_for_side_by_side_variants() -> Result<()> { + let temp = tempfile::tempdir()?; + let plugin_dir = temp.path().join("plugins"); + tokio::fs::create_dir_all(&plugin_dir).await?; + + let cpu_bytes = tar_with_entry("libdemo.so", b"cpu build")?; + let cuda_bytes = tar_with_entry("libdemo.so", b"cuda build")?; + let sha = |bytes: &[u8]| { + let mut hasher = Sha256::new(); + hasher.update(bytes); + to_hex(&hasher.finalize()) + }; + + let listener = match TcpListener::bind("127.0.0.1:0").await { + Ok(listener) => listener, + Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => { + tracing::warn!(error = %err, "Skipping side-by-side variant test"); + return Ok(()); + }, + Err(err) => return Err(err.into()), + }; + let addr = listener.local_addr()?; + + let manifest = crate::marketplace::PluginManifest { + schema_version: 1, + id: "demo".to_string(), + name: None, + version: "1.0.0".to_string(), + node_kind: "demo".to_string(), + kind: PluginKind::Native, + description: None, + license: None, + license_url: None, + homepage: None, + repository: None, + entrypoint: "libdemo.so".to_string(), + bundle: Some(crate::marketplace::PluginBundle { + url: format!("http://{addr}/cpu.tar"), + sha256: sha(&cpu_bytes), + size_bytes: None, + }), + variants: vec![crate::marketplace::PluginBundleVariant { + accelerator: "cuda".to_string(), + url: format!("http://{addr}/cuda.tar"), + sha256: sha(&cuda_bytes), + size_bytes: None, + }], + compatibility: None, + models: Vec::new(), + assets: Vec::new(), + }; + let manifest_bytes = serde_json::to_vec(&manifest)?; + let (pubkey, signature) = minisign_keypair_and_sign(&manifest_bytes)?; + + let index = RegistryIndex { + schema_version: 1, + plugins: vec![crate::marketplace::RegistryPlugin { + id: "demo".to_string(), + name: None, + description: None, + latest: Some("1.0.0".to_string()), + versions: vec![crate::marketplace::RegistryPluginVersion { + version: "1.0.0".to_string(), + manifest_url: format!("http://{addr}/manifest.json"), + signature_url: Some(format!("http://{addr}/manifest.json.minisig")), + published_at: None, + }], + }], + }; + let registry_url = format!("http://{addr}/index.json"); + + let (shutdown_tx, server_handle) = serve_static_routes( + listener, + vec![ + ("/index.json".to_string(), Bytes::from(serde_json::to_vec(&index)?)), + ("/manifest.json".to_string(), Bytes::from(manifest_bytes)), + ("/manifest.json.minisig".to_string(), Bytes::from(signature)), + ("/cpu.tar".to_string(), Bytes::from(cpu_bytes)), + ("/cuda.tar".to_string(), Bytes::from(cuda_bytes)), + ], + ); + + let security = crate::config::PluginMarketplaceSecurityConfig { + allow_model_urls: true, + marketplace_scheme_policy: crate::config::MarketplaceSchemePolicy::AllowHttp, + marketplace_host_policy: crate::config::MarketplaceHostPolicy::AllowPrivate, + marketplace_url_allowlist: vec!["http://127.0.0.1:*".to_string()], + ..crate::config::PluginMarketplaceSecurityConfig::default() + }; + let queue = + build_queue_with(&plugin_dir, vec![registry_url.clone()], vec![pubkey], security)?; + let permissions = + Permissions { allowed_plugins: vec!["*".to_string()], ..Permissions::default() }; + + let install = |accelerator: Option<&str>, job: &str| { + let queue = queue.clone(); + let registry_url = registry_url.clone(); + let permissions = permissions.clone(); + let accelerator = accelerator.map(str::to_string); + let job = job.to_string(); + async move { + insert_job(&queue, &job, JobStatus::Running).await; + let tracker = JobTracker { job_id: job.clone(), queue: queue.clone() }; + let request = InstallPluginRequest { + registry: registry_url, + plugin_id: "demo".to_string(), + version: None, + install_models: false, + model_ids: None, + accelerator, + }; + // Fabricated .so fails at dlopen, but the record is written during + // the preceding ACTIVATE step, which is what we assert on. + let _ = queue + .installer + .install(request, permissions, tracker, CancellationToken::new()) + .await; + } + }; + + install(None, "cpu-job").await; + let cpu_dir = plugin_dir.join("bundles").join("demo").join("1.0.0").join("cpu"); + assert!(cpu_dir.join("libdemo.so").exists(), "cpu variant should extract under cpu/"); + + let record_path = plugin_record_path(&plugin_dir, "demo")?; + let record: ActivePluginRecord = + serde_json::from_slice(&tokio::fs::read(&record_path).await?)?; + assert_eq!(record.accelerator, "cpu", "active record should track cpu"); + + // Switching to the cuda variant must download it (no silent no-op) and + // leave the cpu variant installed side-by-side. + install(Some("cuda"), "cuda-job").await; + let cuda_dir = plugin_dir.join("bundles").join("demo").join("1.0.0").join("cuda"); + assert!(cuda_dir.join("libdemo.so").exists(), "cuda variant should extract under cuda/"); + assert!(cpu_dir.join("libdemo.so").exists(), "cpu variant must remain side-by-side"); + + let record: ActivePluginRecord = + serde_json::from_slice(&tokio::fs::read(&record_path).await?)?; + assert_eq!(record.accelerator, "cuda", "active record should switch to cuda"); + + let _ = shutdown_tx.send(()); + server_handle.await.context("file server task panicked")??; + Ok(()) + } + #[tokio::test] async fn install_rejects_unconfigured_registry_and_empty_plugin_id() -> Result<()> { let temp = tempfile::tempdir()?; @@ -3438,6 +3763,7 @@ mod tests { version: None, install_models: false, model_ids: None, + accelerator: None, }; let result = queue .installer @@ -3459,6 +3785,7 @@ mod tests { version: None, install_models: false, model_ids: None, + accelerator: None, }; let result = queue .installer diff --git a/apps/skit/src/plugin_records.rs b/apps/skit/src/plugin_records.rs index 47e387bbe..a11d4d2d4 100644 --- a/apps/skit/src/plugin_records.rs +++ b/apps/skit/src/plugin_records.rs @@ -17,6 +17,10 @@ pub struct ActivePluginRecord { pub kind: PluginKind, pub entrypoint: String, pub installed_at_ms: u128, + /// Accelerator variant of the activated bundle (e.g. `cpu`, `cuda`). + /// Empty on records written before bundles were keyed by accelerator. + #[serde(default)] + pub accelerator: String, } pub fn active_dir(plugin_dir: &Path) -> PathBuf { diff --git a/apps/skit/src/plugins.rs b/apps/skit/src/plugins.rs index cd779364f..ef63c435d 100644 --- a/apps/skit/src/plugins.rs +++ b/apps/skit/src/plugins.rs @@ -60,6 +60,7 @@ pub struct PluginSummary { pub loaded_at_ms: u128, pub plugin_type: PluginType, pub version: Option, + pub accelerator: Option, } impl PluginSummary { @@ -88,6 +89,7 @@ impl PluginSummary { loaded_at_ms, plugin_type: entry.plugin_type, version: entry.version.clone(), + accelerator: entry.accelerator.clone(), } } } @@ -106,6 +108,7 @@ struct ManagedPlugin { original_kind: String, plugin_type: PluginType, version: Option, + accelerator: Option, } impl ManagedPlugin { @@ -123,6 +126,7 @@ impl ManagedPlugin { original_kind, plugin_type: PluginType::Wasm, version: None, + accelerator: None, } } @@ -140,6 +144,7 @@ impl ManagedPlugin { original_kind, plugin_type: PluginType::Native, version: None, + accelerator: None, } } } @@ -473,10 +478,14 @@ impl UnifiedPluginManager { } let version = record.version; + let accelerator = + (!record.accelerator.is_empty()).then(|| record.accelerator.to_ascii_lowercase()); if let Some(managed) = self.plugins.get_mut(&summary.kind) { managed.version = Some(version.clone()); + managed.accelerator.clone_from(&accelerator); } summary.version = Some(version); + summary.accelerator = accelerator; Some(summary) } @@ -900,6 +909,14 @@ impl UnifiedPluginManager { self.plugins.contains_key(kind) } + /// Records the accelerator variant for a loaded plugin so it shows up in + /// plugin listings without a server restart. + pub fn set_plugin_accelerator(&mut self, kind: &str, accelerator: Option) { + if let Some(managed) = self.plugins.get_mut(kind) { + managed.accelerator = accelerator; + } + } + fn update_loaded_gauge(&self) { let wasm_count = self.plugins.values().filter(|p| p.plugin_type == PluginType::Wasm).count() as u64; @@ -1901,7 +1918,7 @@ mod tests { fn load_active_plugin_record_happy_path_registers_with_version() { // Full active-record path: entrypoint exists under plugin_base_dir // and the record's node_kind matches what the .so reports. Plugin - // is registered with the version from the record. + // is registered with the version and accelerator from the record. let Some(so) = panicking_plugin_so_or_skip() else { return }; let tmp = TempDir::new().unwrap(); let mut mgr = make_manager(&tmp); @@ -1917,6 +1934,7 @@ mod tests { "kind": "native", "entrypoint": entrypoint.display().to_string(), "installed_at_ms": 0u64, + "accelerator": "CUDA", }); write_active_record(tmp.path(), "panicking.json", &record.to_string()); @@ -1925,7 +1943,10 @@ mod tests { let s = &summaries[0]; assert_eq!(s.kind, "plugin::native::panicking"); assert_eq!(s.version.as_deref(), Some("0.1.0")); + assert_eq!(s.accelerator.as_deref(), Some("cuda")); assert!(mgr.is_plugin_loaded("plugin::native::panicking")); + let listed = mgr.list_plugins(); + assert_eq!(listed[0].accelerator.as_deref(), Some("cuda")); } #[test] diff --git a/apps/skit/src/server/plugins.rs b/apps/skit/src/server/plugins.rs index 84621eb73..16f793d23 100644 --- a/apps/skit/src/server/plugins.rs +++ b/apps/skit/src/server/plugins.rs @@ -627,6 +627,22 @@ pub(super) async fn find_active_record_for_kind( None } +async fn remove_dir_if_empty( + base_real: &std::path::Path, + dir: &std::path::Path, + label: &str, +) -> Result<(), anyhow::Error> { + if !tokio::fs::try_exists(dir).await.unwrap_or(false) { + return Ok(()); + } + let dir_real = plugin_paths::ensure_existing_dir_under(base_real, dir, label).await?; + let mut entries = tokio::fs::read_dir(&dir_real).await?; + if entries.next_entry().await?.is_none() { + let _ = tokio::fs::remove_dir(&dir_real).await; + } + Ok(()) +} + pub(super) async fn remove_active_record_and_bundle( plugin_dir: &std::path::Path, record_path: &std::path::Path, @@ -641,7 +657,19 @@ pub(super) async fn remove_active_record_and_bundle( plugin_paths::validate_path_component("plugin version", &record.version)?; let bundles_root = plugin_dir.join("bundles").join(&record.plugin_id); - let bundle_dir = bundles_root.join(&record.version); + let version_dir = bundles_root.join(&record.version); + + // Records written before bundles were keyed by accelerator have an empty + // `accelerator`; those live directly under the version dir (legacy layout), + // so removing the version dir is correct. Newer records key by accelerator, + // so only that variant is removed, leaving sibling variants installed. + let bundle_dir = if record.accelerator.is_empty() { + version_dir.clone() + } else { + plugin_paths::validate_path_component("accelerator", &record.accelerator)?; + version_dir.join(&record.accelerator) + }; + if tokio::fs::try_exists(&bundle_dir).await.unwrap_or(false) { let bundle_dir_real = plugin_paths::ensure_existing_dir_under(&base_real, &bundle_dir, "bundle").await?; @@ -653,14 +681,8 @@ pub(super) async fn remove_active_record_and_bundle( })?; } - if tokio::fs::try_exists(&bundles_root).await.unwrap_or(false) { - let bundles_root_real = - plugin_paths::ensure_existing_dir_under(&base_real, &bundles_root, "bundles").await?; - let mut entries = tokio::fs::read_dir(&bundles_root_real).await?; - if entries.next_entry().await?.is_none() { - let _ = tokio::fs::remove_dir(&bundles_root_real).await; - } - } + remove_dir_if_empty(&base_real, &version_dir, "version").await?; + remove_dir_if_empty(&base_real, &bundles_root, "bundles").await?; let cache_root = plugin_dir.join("cache").join(&record.plugin_id); if tokio::fs::try_exists(&cache_root).await.unwrap_or(false) { diff --git a/apps/skit/tests/plugin_integration_test.rs b/apps/skit/tests/plugin_integration_test.rs index 2cd8bc78f..b70662d84 100644 --- a/apps/skit/tests/plugin_integration_test.rs +++ b/apps/skit/tests/plugin_integration_test.rs @@ -376,6 +376,7 @@ async fn test_load_active_plugin_on_startup() { kind: PluginKind::Native, entrypoint: entrypoint_path.to_string_lossy().into_owned(), installed_at_ms: 0, + accelerator: "cpu".to_string(), }; let record_path = plugin_record_path(&plugins_dir, "gain").unwrap(); fs::write(&record_path, serde_json::to_vec_pretty(&record).unwrap()).await.unwrap(); @@ -406,7 +407,11 @@ async fn test_uninstall_marketplace_plugin_removes_bundle() { let plugins_dir = temp_dir.path().join("plugins"); fs::create_dir_all(&plugins_dir).await.unwrap(); - let bundle_dir = plugins_dir.join("bundles").join("gain").join("1.0.0"); + // Mirror the installer's accelerator-keyed layout + // (bundles////) so uninstall exercises the + // variant-dir removal path, not just the entrypoint file removal. + let version_dir = plugins_dir.join("bundles").join("gain").join("1.0.0"); + let bundle_dir = version_dir.join("cpu"); fs::create_dir_all(&bundle_dir).await.unwrap(); let entrypoint_path = bundle_dir .join(plugin_path.file_name().expect("plugin file name").to_string_lossy().to_string()); @@ -420,6 +425,7 @@ async fn test_uninstall_marketplace_plugin_removes_bundle() { kind: PluginKind::Native, entrypoint: entrypoint_path.to_string_lossy().into_owned(), installed_at_ms: 0, + accelerator: "cpu".to_string(), }; let record_path = plugin_record_path(&plugins_dir, "gain").unwrap(); fs::write(&record_path, serde_json::to_vec_pretty(&record).unwrap()).await.unwrap(); @@ -447,7 +453,11 @@ async fn test_uninstall_marketplace_plugin_removes_bundle() { assert!(!tokio::fs::try_exists(&record_path).await.unwrap(), "Active record should be removed"); assert!( !tokio::fs::try_exists(&bundle_dir).await.unwrap(), - "Bundle directory should be removed" + "Accelerator variant directory should be removed" + ); + assert!( + !tokio::fs::try_exists(&version_dir).await.unwrap(), + "Empty version directory should be pruned" ); server.shutdown().await; diff --git a/marketplace/official-plugins.json b/marketplace/official-plugins.json index 9cb41007f..88140bb43 100644 --- a/marketplace/official-plugins.json +++ b/marketplace/official-plugins.json @@ -20,6 +20,10 @@ "kind": "native", "entrypoint": "libhelsinki.so", "artifact": "target/plugins/release/libhelsinki.so", + "accelerators": [ + "cpu", + "cuda" + ], "description": "Neural machine translation using OPUS-MT", "license": "MPL-2.0", "models": [ @@ -63,6 +67,10 @@ "kind": "native", "entrypoint": "libkokoro.so", "artifact": "target/plugins/release/libkokoro.so", + "accelerators": [ + "cpu", + "cuda" + ], "description": "Text-to-speech using Sherpa-ONNX Kokoro models", "license": "MPL-2.0", "models": [ @@ -91,6 +99,10 @@ "kind": "native", "entrypoint": "libmatcha.so", "artifact": "target/plugins/release/libmatcha.so", + "accelerators": [ + "cpu", + "cuda" + ], "description": "Text-to-speech using Matcha models", "license": "MPL-2.0", "models": [ @@ -266,6 +278,10 @@ "kind": "native", "entrypoint": "libsensevoice.so", "artifact": "target/plugins/release/libsensevoice.so", + "accelerators": [ + "cpu", + "cuda" + ], "description": "Streaming speech-to-text using SenseVoice", "license": "MPL-2.0", "models": [ @@ -315,7 +331,7 @@ { "id": "slint", "name": "Slint", - "version": "0.4.0", + "version": "0.5.0", "node_kind": "slint", "kind": "native", "entrypoint": "libslint.so", @@ -373,6 +389,10 @@ "kind": "native", "entrypoint": "libvad.so", "artifact": "target/plugins/release/libvad.so", + "accelerators": [ + "cpu", + "cuda" + ], "description": "Voice activity detection", "license": "MPL-2.0", "models": [ @@ -401,6 +421,10 @@ "kind": "native", "entrypoint": "libwhisper.so", "artifact": "target/plugins/release/libwhisper.so", + "accelerators": [ + "cpu", + "cuda" + ], "description": "Streaming speech-to-text using whisper.cpp", "license": "MPL-2.0", "models": [ diff --git a/plugins/native/helsinki/plugin.yml b/plugins/native/helsinki/plugin.yml index 92fc92c6b..1ea88f044 100644 --- a/plugins/native/helsinki/plugin.yml +++ b/plugins/native/helsinki/plugin.yml @@ -5,6 +5,9 @@ node_kind: helsinki kind: native entrypoint: libhelsinki.so artifact: target/plugins/release/libhelsinki.so +accelerators: +- cpu +- cuda description: Neural machine translation using OPUS-MT license: MPL-2.0 models: diff --git a/plugins/native/kokoro/plugin.yml b/plugins/native/kokoro/plugin.yml index 778c3fe17..45cc31a69 100644 --- a/plugins/native/kokoro/plugin.yml +++ b/plugins/native/kokoro/plugin.yml @@ -5,6 +5,9 @@ node_kind: kokoro kind: native entrypoint: libkokoro.so artifact: target/plugins/release/libkokoro.so +accelerators: +- cpu +- cuda description: Text-to-speech using Sherpa-ONNX Kokoro models license: MPL-2.0 models: diff --git a/plugins/native/matcha/plugin.yml b/plugins/native/matcha/plugin.yml index 765d2f8f8..80bea8054 100644 --- a/plugins/native/matcha/plugin.yml +++ b/plugins/native/matcha/plugin.yml @@ -5,6 +5,9 @@ node_kind: matcha kind: native entrypoint: libmatcha.so artifact: target/plugins/release/libmatcha.so +accelerators: +- cpu +- cuda description: Text-to-speech using Matcha models license: MPL-2.0 models: diff --git a/plugins/native/sensevoice/plugin.yml b/plugins/native/sensevoice/plugin.yml index 0de62151a..79fff4d75 100644 --- a/plugins/native/sensevoice/plugin.yml +++ b/plugins/native/sensevoice/plugin.yml @@ -5,6 +5,9 @@ node_kind: sensevoice kind: native entrypoint: libsensevoice.so artifact: target/plugins/release/libsensevoice.so +accelerators: +- cpu +- cuda description: Streaming speech-to-text using SenseVoice license: MPL-2.0 models: diff --git a/plugins/native/slint/Cargo.lock b/plugins/native/slint/Cargo.lock index 2dda4d3c7..fdf4d2242 100644 --- a/plugins/native/slint/Cargo.lock +++ b/plugins/native/slint/Cargo.lock @@ -3830,7 +3830,7 @@ dependencies = [ [[package]] name = "slint-plugin-native" -version = "0.4.0" +version = "0.5.0" dependencies = [ "pollster", "serde", diff --git a/plugins/native/slint/Cargo.toml b/plugins/native/slint/Cargo.toml index 1497e300b..bf7da5fe8 100644 --- a/plugins/native/slint/Cargo.toml +++ b/plugins/native/slint/Cargo.toml @@ -4,7 +4,7 @@ [package] name = "slint-plugin-native" -version = "0.4.0" +version = "0.5.0" edition = "2021" license = "MPL-2.0" diff --git a/plugins/native/slint/plugin.yml b/plugins/native/slint/plugin.yml index d94332aeb..6ee6bbcf9 100644 --- a/plugins/native/slint/plugin.yml +++ b/plugins/native/slint/plugin.yml @@ -1,6 +1,6 @@ id: slint name: Slint -version: 0.4.0 +version: 0.5.0 node_kind: slint kind: native entrypoint: libslint.so diff --git a/plugins/native/vad/plugin.yml b/plugins/native/vad/plugin.yml index b95383c06..48a569f28 100644 --- a/plugins/native/vad/plugin.yml +++ b/plugins/native/vad/plugin.yml @@ -5,6 +5,9 @@ node_kind: vad kind: native entrypoint: libvad.so artifact: target/plugins/release/libvad.so +accelerators: +- cpu +- cuda description: Voice activity detection license: MPL-2.0 models: diff --git a/plugins/native/whisper/Cargo.toml b/plugins/native/whisper/Cargo.toml index 630118211..e7ee973a9 100644 --- a/plugins/native/whisper/Cargo.toml +++ b/plugins/native/whisper/Cargo.toml @@ -22,6 +22,12 @@ ort = "2.0.0-rc.10" ndarray = "0.16" once_cell = "1.20" +[features] +default = [] +# Enables CUDA-accelerated inference via whisper.cpp's CUDA backend. Requires a +# CUDA toolkit at build time; the marketplace cuda bundle is built with this on. +cuda = ["whisper-rs/cuda"] + [profile.release] opt-level = 3 lto = true diff --git a/plugins/native/whisper/plugin.yml b/plugins/native/whisper/plugin.yml index 7d3a72c59..a26dc8948 100644 --- a/plugins/native/whisper/plugin.yml +++ b/plugins/native/whisper/plugin.yml @@ -5,6 +5,9 @@ node_kind: whisper kind: native entrypoint: libwhisper.so artifact: target/plugins/release/libwhisper.so +accelerators: +- cpu +- cuda description: Streaming speech-to-text using whisper.cpp license: MPL-2.0 models: diff --git a/scripts/marketplace/build_official_plugins.sh b/scripts/marketplace/build_official_plugins.sh index 6e3872449..74eb2d3ef 100755 --- a/scripts/marketplace/build_official_plugins.sh +++ b/scripts/marketplace/build_official_plugins.sh @@ -42,6 +42,15 @@ while IFS= read -r plugin; do echo "Building native plugin: ${plugin}" ( cd "plugins/native/${plugin}" + # whisper.cpp/ggml enables GGML_NATIVE (-march=native) by default, which can + # emit AVX/AVX512 that SIGILLs on older CPUs than the build runner. The + # published bundle must run on any x86-64 host, so pin a portable baseline + # (SOURCE_DATE_EPOCH disables ggml's native-default upstream). + if [ "${plugin}" = "whisper" ]; then + export SOURCE_DATE_EPOCH=1 + export CFLAGS="-O3 -pipe -fPIC -march=x86-64 -mtune=generic" + export CXXFLAGS="-O3 -pipe -fPIC -march=x86-64 -mtune=generic" + fi CARGO_TARGET_DIR="${CARGO_TARGET_DIR:-$(cd ../../.. && pwd)/target/plugins}" cargo build --release ) done <<< "${plugins}" diff --git a/scripts/marketplace/build_official_plugins_cuda.sh b/scripts/marketplace/build_official_plugins_cuda.sh new file mode 100755 index 000000000..93192eb52 --- /dev/null +++ b/scripts/marketplace/build_official_plugins_cuda.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: © 2025 StreamKit Contributors +# SPDX-License-Identifier: MPL-2.0 + +# Builds the CUDA-capable official plugins for the marketplace `cuda` variant. +# +# Plugins are selected from marketplace/official-plugins.json by their +# `accelerators` list containing "cuda". Plugins that declare a `cuda` Cargo +# feature (e.g. whisper, helsinki) are built with `--features cuda`; sherpa-onnx +# plugins are execution-provider agnostic, so their CPU `.so` is rebuilt as-is +# and later packaged against the GPU sherpa runtime by build_registry.py. + +set -euo pipefail + +python3 scripts/marketplace/generate_official_plugins.py + +cuda_plugins=$(python3 - <<'PY' +import json +import pathlib + +metadata = json.loads(pathlib.Path("marketplace/official-plugins.json").read_text()) +for plugin in metadata.get("plugins", []): + if "cuda" in plugin.get("accelerators", []): + print(f"{plugin['id']}\t{plugin['artifact']}") +PY +) + +if [ -z "${cuda_plugins}" ]; then + echo "No CUDA-capable plugins declared in marketplace/official-plugins.json" >&2 + exit 0 +fi + +target_dir="${CARGO_TARGET_DIR:-$(pwd)/target/plugins}" + +while IFS=$'\t' read -r plugin artifact; do + if [ -z "${plugin}" ]; then + continue + fi + plugin_dir="plugins/native/${plugin}" + if [ ! -d "${plugin_dir}" ]; then + echo "Missing plugin directory: ${plugin_dir}" >&2 + exit 1 + fi + + features=() + has_cuda_feature=0 + if grep -qE '^[[:space:]]*cuda[[:space:]]*=' "${plugin_dir}/Cargo.toml"; then + features=(--features cuda) + has_cuda_feature=1 + fi + + echo "Building CUDA plugin: ${plugin} ${features[*]:-}" + ( + cd "${plugin_dir}" + # whisper.cpp/ggml enables GGML_NATIVE (-march=native) by default, which can + # emit AVX/AVX512 that SIGILLs on older CPUs than the build runner. Even in + # the CUDA build the CPU ggml kernels are still compiled, so pin the same + # portable baseline as build_official_plugins.sh (SOURCE_DATE_EPOCH disables + # ggml's native-default upstream). These flags only affect host C/C++ code, + # not the nvcc-compiled GPU kernels. + if [ "${plugin}" = "whisper" ]; then + export SOURCE_DATE_EPOCH=1 + export CFLAGS="-O3 -pipe -fPIC -march=x86-64 -mtune=generic" + export CXXFLAGS="-O3 -pipe -fPIC -march=x86-64 -mtune=generic" + fi + CARGO_TARGET_DIR="${target_dir}" cargo build --release ${features[@]+"${features[@]}"} + ) + + # Guard against silently shipping an unaccelerated CPU build as a `cuda` + # variant. A cuda-declared plugin must either compile its own `cuda` feature + # or link the (execution-provider-agnostic) sherpa-onnx runtime that + # build_registry.py later packages against the GPU sherpa libs. If neither + # holds, the produced .so is a plain CPU build masquerading as cuda. + artifact_path="${target_dir}/release/$(basename "${artifact}")" + if [ "${has_cuda_feature}" -eq 0 ]; then + if [ ! -f "${artifact_path}" ]; then + echo "ERROR: expected artifact not found after build: ${artifact_path}" >&2 + exit 1 + fi + if ! readelf -d "${artifact_path}" 2>/dev/null | grep -q 'libsherpa-onnx-c-api.so'; then + echo "ERROR: plugin '${plugin}' declares the 'cuda' accelerator but has" \ + "neither a 'cuda' Cargo feature nor sherpa-onnx linkage; refusing to" \ + "package a plain CPU build as a cuda variant." >&2 + exit 1 + fi + fi +done <<< "${cuda_plugins}" diff --git a/scripts/marketplace/build_registry.py b/scripts/marketplace/build_registry.py index 99413d3fd..27dc780b8 100644 --- a/scripts/marketplace/build_registry.py +++ b/scripts/marketplace/build_registry.py @@ -13,6 +13,14 @@ import subprocess import sys +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) + +from manifest_builder import ( # noqa: E402 + SHERPA_CORE_LIBS, + SHERPA_CUDA_LIBS, + build_manifest, +) + def sha256_file(path: pathlib.Path) -> str: hasher = hashlib.sha256() @@ -60,9 +68,11 @@ def require_tool(name: str) -> None: raise RuntimeError(f"Missing required tool: {name}") -def ensure_sherpa_runtime(work_dir: pathlib.Path) -> None: +def ensure_sherpa_runtime(work_dir: pathlib.Path, accelerator: str = "cpu") -> None: lib_dir = pathlib.Path(os.environ.get("SHERPA_ONNX_LIB_DIR", "/usr/local/lib")) - sherpa_libs = ["libsherpa-onnx-c-api.so", "libonnxruntime.so"] + sherpa_libs = list(SHERPA_CORE_LIBS) + if accelerator == "cuda": + sherpa_libs.extend(SHERPA_CUDA_LIBS) for lib in sherpa_libs: src = lib_dir / lib if not src.exists(): @@ -81,15 +91,22 @@ def build_bundle( bundles_out: pathlib.Path, work_root: pathlib.Path, embedded_manifest: dict | None = None, + accelerator: str = "cpu", ) -> dict: plugin_id = plugin["id"] - artifact = pathlib.Path(plugin["artifact"]) + # CUDA passes may ship a separately compiled artifact (e.g. whisper/helsinki + # built with their `cuda` feature). Fall back to the canonical artifact. + artifact = pathlib.Path( + plugin.get(f"artifact_{accelerator}", plugin["artifact"]) + if accelerator != "cpu" + else plugin["artifact"] + ) entrypoint = pathlib.Path(plugin["entrypoint"]) if not artifact.exists(): raise FileNotFoundError(f"Missing artifact: {artifact}") - work_dir = work_root / f"{plugin_id}-{version}" + work_dir = work_root / f"{plugin_id}-{version}-{accelerator}" if work_dir.exists(): shutil.rmtree(work_dir) ensure_dir(work_dir) @@ -99,7 +116,7 @@ def build_bundle( needed, _ = readelf_dynamic(entrypoint_path) if "libsherpa-onnx-c-api.so" in needed: - ensure_sherpa_runtime(work_dir) + ensure_sherpa_runtime(work_dir, accelerator) set_runpath_origin(entrypoint_path) for extra in plugin.get("extra_files", []): @@ -113,8 +130,12 @@ def build_bundle( if embedded_manifest is not None: write_json(work_dir / "manifest.json", embedded_manifest) + write_plugin_yml(entrypoint_path.parent, embedded_manifest) - bundle_name = f"{plugin_id}-{version}-bundle.tar.zst" + if accelerator == "cpu": + bundle_name = f"{plugin_id}-{version}-bundle.tar.zst" + else: + bundle_name = f"{plugin_id}-{version}-{accelerator}-bundle.tar.zst" bundle_path = bundles_out / bundle_name ensure_dir(bundles_out) @@ -144,8 +165,27 @@ def write_json(path: pathlib.Path, payload: dict) -> None: path.write_text(json.dumps(payload, indent=2, sort_keys=False)) -def strip_none(payload: dict) -> dict: - return {key: value for key, value in payload.items() if value is not None} +def write_plugin_yml(yml_dir: pathlib.Path, manifest: dict) -> None: + """Embed a `plugin.yml` next to the entrypoint so the bundle is + self-describing. + + The server's `read_local_plugin_manifest` discovers a plugin's asset types + (and other metadata) from a `plugin.yml`/`plugin.yaml` beside the `.so`, not + from `manifest.json`. Consumers that extract a bundle directly (e.g. the demo + image) therefore need this file to recover declarations like Slint's asset + type. The marketplace installer writes the same file from the downloaded + manifest; embedding it keeps raw-extraction consumers in parity. + """ + try: + import yaml + except ImportError as exc: + raise RuntimeError( + "PyYAML is required. Install python3-yaml or pip install pyyaml." + ) from exc + ensure_dir(yml_dir) + (yml_dir / "plugin.yml").write_text( + yaml.safe_dump(manifest, sort_keys=False, allow_unicode=True) + ) def dump_manifest_bytes(manifest: dict) -> bytes: @@ -153,32 +193,6 @@ def dump_manifest_bytes(manifest: dict) -> bytes: return (json.dumps(manifest, indent=2, sort_keys=False) + "\n").encode("utf-8") -def build_manifest( - plugin: dict, - plugin_version: str, - bundle_block: dict | None, -) -> dict: - """Build manifest dict from plugin metadata and bundle info.""" - manifest = { - "schema_version": 1, - "id": plugin["id"], - "name": plugin.get("name"), - "version": plugin_version, - "node_kind": plugin["node_kind"], - "kind": plugin["kind"], - "description": plugin.get("description"), - "license": plugin.get("license"), - "license_url": plugin.get("license_url"), - "homepage": plugin.get("homepage"), - "repository": plugin.get("repo"), - "entrypoint": plugin["entrypoint"], - "bundle": bundle_block, - "compatibility": plugin.get("compatibility"), - "models": plugin.get("models", []), - } - return strip_none(manifest) - - def verify_existing_signature( manifest_path: pathlib.Path, signature_path: pathlib.Path, @@ -359,7 +373,17 @@ def main() -> int: "--new-plugins-out", help="Path to write JSON file listing newly built plugins (id + version)", ) + parser.add_argument( + "--accelerator", + default="cpu", + help=( + "Accelerator to build. 'cpu' (default) builds the canonical bundle; " + "any other value (e.g. 'cuda') builds a variant and merges it into " + "the matching existing manifest (requires --existing-registry)." + ), + ) args = parser.parse_args() + accelerator = args.accelerator.strip().lower() plugins_path = pathlib.Path(args.plugins) bundles_out = pathlib.Path(args.bundles_out) @@ -409,6 +433,20 @@ def main() -> int: if work_root.exists(): shutil.rmtree(work_root) + # Variant passes (e.g. cuda) layer onto an already-published registry, so + # copy the full existing tree forward; untouched plugins must stay present + # and only the variant-updated manifests are overwritten below. + if accelerator != "cpu": + if not args.existing_registry: + print( + "ERROR: --accelerator other than 'cpu' requires --existing-registry", + file=sys.stderr, + ) + return 1 + src_plugins = existing_registry_path / "plugins" + if src_plugins.exists(): + shutil.copytree(src_plugins, registry_out / "plugins", dirs_exist_ok=True) + for plugin in plugins: plugin_id = plugin["id"] plugin_version = plugin.get("version") @@ -418,17 +456,125 @@ def main() -> int: key = (plugin_id, plugin_version) + # Variant pass: add an accelerator-specific bundle to an existing + # manifest rather than (re)building the canonical CPU bundle. + if accelerator != "cpu": + if accelerator not in plugin.get("accelerators", ["cpu"]): + continue + if key not in existing_registry: + print( + f"ERROR: {plugin_id}@{plugin_version} has no published CPU " + f"manifest to attach a '{accelerator}' variant to. Build and " + f"publish the CPU bundle first.", + file=sys.stderr, + ) + return 1 + existing = existing_registry[key] + existing_manifest = existing["manifest"] + + # Append-only: an already-published variant is immutable, so reuse + # the manifest carried forward above instead of rebuilding it. + # Match case-insensitively to mirror the installer's + # eq_ignore_ascii_case bundle resolution. + if any( + (v.get("accelerator") or "").lower() == accelerator + for v in existing_manifest.get("variants", []) + ): + print( + f"Reusing existing {accelerator} variant for " + f"{plugin_id} v{plugin_version}" + ) + continue + + embedded_manifest = build_manifest(plugin, plugin_version, bundle_block=None) + bundle_info = build_bundle( + plugin, plugin_version, bundles_out, work_root, + accelerator=accelerator, + embedded_manifest=embedded_manifest, + ) + bundle_base = bundle_url_template.format( + plugin_id=plugin_id, version=plugin_version, + ) + variant_block = { + "accelerator": accelerator, + "url": f"{bundle_base}/{bundle_info['bundle_name']}", + "sha256": bundle_info["sha256"], + "size_bytes": bundle_info["size_bytes"], + } + merged_variants = [ + v + for v in existing_manifest.get("variants", []) + if (v.get("accelerator") or "").lower() != accelerator + ] + merged_variants.append(variant_block) + would_be_manifest = build_manifest( + plugin, + plugin_version, + existing_manifest.get("bundle"), + variants=merged_variants, + ) + + # Only the variants list may differ; everything else is immutable. + existing_core = {k: v for k, v in existing_manifest.items() if k != "variants"} + would_be_core = {k: v for k, v in would_be_manifest.items() if k != "variants"} + if existing_core != would_be_core: + print( + f"ERROR: {plugin_id}@{plugin_version} '{accelerator}' variant " + f"pass would change non-variant manifest fields; bump version.", + file=sys.stderr, + ) + existing_json = json.dumps(existing_core, indent=2, sort_keys=False) + would_be_json = json.dumps(would_be_core, indent=2, sort_keys=False) + diff = difflib.unified_diff( + existing_json.splitlines(keepends=True), + would_be_json.splitlines(keepends=True), + fromfile="existing", + tofile="would-be", + ) + print("".join(diff), file=sys.stderr) + return 1 + + if not verify_existing_signature( + existing["manifest_path"], + existing["signature_path"], + public_key_path, + ): + print( + f"ERROR: {plugin_id}@{plugin_version} — existing manifest " + f"signature verification failed before attaching the " + f"'{accelerator}' variant. The manifest may have been " + f"modified after signing; revert the changes or bump the " + f"version to trigger re-signing.", + file=sys.stderr, + ) + return 1 + + manifest_dir = registry_out / "plugins" / plugin_id / plugin_version + manifest_path = manifest_dir / "manifest.json" + write_json(manifest_path, would_be_manifest) + sign_manifest(manifest_path, signing_key) + new_plugins.append( + {"id": plugin_id, "version": plugin_version, "accelerator": accelerator} + ) + print( + f"Added {accelerator} variant for {plugin_id} v{plugin_version} " + f"-> {bundle_info['bundle_name']} ({bundle_info['sha256']})" + ) + continue + # Check if this version already exists in the registry if key in existing_registry: # Verify immutability: check if republishing with same version would change manifest existing = existing_registry[key] existing_manifest = existing["manifest"] - # Build would-be manifest using current plugin fields but existing bundle + # Build would-be manifest using current plugin fields but existing + # bundle and variants (CPU passes never touch published variants). would_be_manifest = build_manifest( plugin, plugin_version, existing_manifest["bundle"], + variants=existing_manifest.get("variants"), ) # Compare parsed JSON objects (robust to formatting differences like trailing newlines) diff --git a/scripts/marketplace/check_registry_versions.py b/scripts/marketplace/check_registry_versions.py index ed65c9218..db7078537 100644 --- a/scripts/marketplace/check_registry_versions.py +++ b/scripts/marketplace/check_registry_versions.py @@ -21,6 +21,10 @@ import pathlib import sys +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) + +from manifest_builder import build_manifest # noqa: E402 + def load_yaml(path: pathlib.Path) -> dict: try: @@ -32,30 +36,16 @@ def load_yaml(path: pathlib.Path) -> dict: return yaml.safe_load(path.read_text()) -def strip_none(payload: dict) -> dict: - return {key: value for key, value in payload.items() if value is not None} - - -def build_manifest_from_plugin(plugin: dict, bundle_block: dict | None) -> dict: - """Mirror the manifest shape produced by build_registry.py.""" - manifest = { - "schema_version": 1, - "id": plugin["id"], - "name": plugin.get("name"), - "version": plugin.get("version"), - "node_kind": plugin["node_kind"], - "kind": plugin["kind"], - "description": plugin.get("description"), - "license": plugin.get("license"), - "license_url": plugin.get("license_url"), - "homepage": plugin.get("homepage"), - "repository": plugin.get("repo"), - "entrypoint": plugin["entrypoint"], - "bundle": bundle_block, - "compatibility": plugin.get("compatibility"), - "models": plugin.get("models", []), - } - return strip_none(manifest) +def build_manifest_from_plugin( + plugin: dict, bundle_block: dict | None, variants: list[dict] | None = None +) -> dict: + """Rebuild the would-be manifest from plugin.yml for the append-only check. + + `variants` are carried over verbatim from the committed manifest; they are + produced by the build pipeline (not plugin.yml) and must survive the + append-only comparison untouched. + """ + return build_manifest(plugin, plugin.get("version"), bundle_block, variants) def main() -> int: @@ -92,7 +82,9 @@ def main() -> int: continue committed = json.loads(committed_manifest_path.read_text()) - would_be = build_manifest_from_plugin(plugin, committed.get("bundle")) + would_be = build_manifest_from_plugin( + plugin, committed.get("bundle"), committed.get("variants") + ) if committed != would_be: errors += 1 diff --git a/scripts/marketplace/generate_official_plugins.py b/scripts/marketplace/generate_official_plugins.py index b097b8126..2715df802 100644 --- a/scripts/marketplace/generate_official_plugins.py +++ b/scripts/marketplace/generate_official_plugins.py @@ -63,6 +63,7 @@ def validate_plugin(plugin: dict, plugin_dir: pathlib.Path) -> dict: "kind", "entrypoint", "artifact", + "accelerators", "description", "license", "license_url", diff --git a/scripts/marketplace/manifest_builder.py b/scripts/marketplace/manifest_builder.py new file mode 100644 index 000000000..b08ae4a21 --- /dev/null +++ b/scripts/marketplace/manifest_builder.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: © 2025 StreamKit Contributors +# SPDX-License-Identifier: MPL-2.0 + +"""Shared manifest/registry helpers. + +The registry is append-only: a committed manifest must round-trip byte-identically +when rebuilt from plugin.yml. `build_registry.py` (the builder) and +`check_registry_versions.py` (the pre-merge guard) therefore have to agree on the +exact manifest shape, and `verify_bundles.py` has to agree with the builder on +which GPU libraries a cuda bundle carries. Keeping those definitions here makes a +single source of truth instead of byte-fragile copies. +""" + +# Core sherpa runtime libraries vendored into every sherpa-backed bundle. +SHERPA_CORE_LIBS = ["libsherpa-onnx-c-api.so", "libonnxruntime.so"] + +# Additional ONNX Runtime execution-provider libraries required when the +# vendored libonnxruntime.so is the CUDA-enabled build. The same plugin `.so` +# loads these at runtime to dispatch to the GPU. +SHERPA_CUDA_LIBS = [ + "libonnxruntime_providers_cuda.so", + "libonnxruntime_providers_shared.so", +] + + +def strip_none(payload: dict) -> dict: + return {key: value for key, value in payload.items() if value is not None} + + +def build_manifest( + plugin: dict, + version: str, + bundle_block: dict | None, + variants: list[dict] | None = None, +) -> dict: + """Build the registry manifest dict from plugin metadata and bundle info. + + `variants` carries accelerator-specific bundles (e.g. a CUDA build) and is + inserted right after `bundle` for readable diffs; it is omitted entirely when + empty so CPU-only manifests stay byte-identical to those produced before + variant support existed. + """ + manifest = { + "schema_version": 1, + "id": plugin["id"], + "name": plugin.get("name"), + "version": version, + "node_kind": plugin["node_kind"], + "kind": plugin["kind"], + "description": plugin.get("description"), + "license": plugin.get("license"), + "license_url": plugin.get("license_url"), + "homepage": plugin.get("homepage"), + "repository": plugin.get("repo"), + "entrypoint": plugin["entrypoint"], + "bundle": bundle_block, + "compatibility": plugin.get("compatibility"), + "models": plugin.get("models", []), + "assets": plugin.get("assets") or None, + } + manifest = strip_none(manifest) + if variants: + if bundle_block is None: + raise ValueError( + "variants require a bundle block: variants are anchored after " + "the 'bundle' key and would be silently dropped without one" + ) + ordered: dict = {} + for key, value in manifest.items(): + ordered[key] = value + if key == "bundle": + ordered["variants"] = variants + manifest = ordered + return manifest diff --git a/scripts/marketplace/test_append_only.py b/scripts/marketplace/test_append_only.py index 10cfde90a..ca55ce689 100755 --- a/scripts/marketplace/test_append_only.py +++ b/scripts/marketplace/test_append_only.py @@ -15,7 +15,7 @@ import tempfile -def setup_test_registry(registry_path: pathlib.Path) -> None: +def setup_test_registry(registry_path: pathlib.Path, with_variant: bool = False) -> None: """Create a minimal existing registry with one plugin version.""" plugin_id = "test-plugin" version = "0.1.0" @@ -42,6 +42,15 @@ def setup_test_registry(registry_path: pathlib.Path) -> None: }, "models": [], } + if with_variant: + manifest["variants"] = [ + { + "accelerator": "cuda", + "url": "https://example.com/test-plugin-0.1.0-cuda-bundle.tar.zst", + "sha256": "deadbeef" * 8, + "size_bytes": 2048, + } + ] manifest_path = manifest_dir / "manifest.json" manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=False) + "\n") @@ -128,8 +137,8 @@ def test_identical_reuse(tmp_dir: pathlib.Path) -> bool: str(plugins_json), "--existing-registry", str(existing_registry), - "--bundle-base-url", - "https://example.com/bundles", + "--bundle-url-template", + "https://example.com/bundles/{plugin_id}/{version}", "--registry-base-url", "https://example.com/registry", "--bundles-out", @@ -199,8 +208,8 @@ def test_changed_metadata_fails(tmp_dir: pathlib.Path) -> bool: str(plugins_json), "--existing-registry", str(existing_registry), - "--bundle-base-url", - "https://example.com/bundles", + "--bundle-url-template", + "https://example.com/bundles/{plugin_id}/{version}", "--registry-base-url", "https://example.com/registry", "--bundles-out", @@ -230,6 +239,67 @@ def test_changed_metadata_fails(tmp_dir: pathlib.Path) -> bool: return True +def test_cpu_pass_preserves_variants(tmp_dir: pathlib.Path) -> bool: + """A CPU pass over a manifest carrying a cuda variant must preserve it.""" + print("\n=== Test 3: CPU pass preserves published cuda variant ===") + + existing_registry = tmp_dir / "existing_registry" + setup_test_registry(existing_registry, with_variant=True) + + plugins_json = tmp_dir / "plugins.json" + create_test_plugin_metadata(plugins_json) + + output_registry = tmp_dir / "output_registry" + bundles_out = tmp_dir / "bundles" + dummy_key = tmp_dir / "dummy.key" + dummy_key.write_text("dummy signing key\n") + public_key = existing_registry / "streamkit.pub" + + result = subprocess.run( + [ + "python3", + "scripts/marketplace/build_registry.py", + "--plugins", + str(plugins_json), + "--existing-registry", + str(existing_registry), + "--bundle-url-template", + "https://example.com/bundles/{plugin_id}/{version}", + "--registry-base-url", + "https://example.com/registry", + "--bundles-out", + str(bundles_out), + "--registry-out", + str(output_registry), + "--signing-key", + str(dummy_key), + "--public-key", + str(public_key), + ], + capture_output=True, + text=True, + ) + + if result.returncode != 0: + print(f"FAIL: Expected success but got exit code {result.returncode}") + print(f"STDERR: {result.stderr}") + return False + + manifest_path = output_registry / "plugins" / "test-plugin" / "0.1.0" / "manifest.json" + if not manifest_path.exists(): + print(f"FAIL: Manifest not found at {manifest_path}") + return False + + manifest = json.loads(manifest_path.read_text()) + variants = manifest.get("variants") + if not variants or variants[0].get("accelerator") != "cuda": + print(f"FAIL: cuda variant not preserved; got variants={variants}") + return False + + print("PASS: CPU pass preserved the published cuda variant") + return True + + def main() -> int: """Run all tests.""" print("Running append-only registry tests...") @@ -245,8 +315,12 @@ def main() -> int: test2_dir.mkdir() test2_passed = test_changed_metadata_fails(test2_dir) + test3_dir = tmp_dir / "test3" + test3_dir.mkdir() + test3_passed = test_cpu_pass_preserves_variants(test3_dir) + print("\n" + "=" * 60) - if test1_passed and test2_passed: + if test1_passed and test2_passed and test3_passed: print("✓ All tests passed!") return 0 else: @@ -255,6 +329,8 @@ def main() -> int: print(" - Test 1 (identical reuse) FAILED") if not test2_passed: print(" - Test 2 (immutability check) FAILED") + if not test3_passed: + print(" - Test 3 (variant preservation) FAILED") return 1 diff --git a/scripts/marketplace/test_build_registry.py b/scripts/marketplace/test_build_registry.py index f7542e406..9fbc800bb 100644 --- a/scripts/marketplace/test_build_registry.py +++ b/scripts/marketplace/test_build_registry.py @@ -63,6 +63,151 @@ def test_check_registry_omits_repository_when_repo_absent(self): assert "repository" not in manifest +SAMPLE_BUNDLE = { + "url": "https://example.com/test-plugin-1.0.0-bundle.tar.zst", + "sha256": "ab" * 32, + "size_bytes": 1024, +} + +CUDA_VARIANT = { + "accelerator": "cuda", + "url": "https://example.com/test-plugin-1.0.0-cuda-bundle.tar.zst", + "sha256": "cd" * 32, + "size_bytes": 2048, +} + + +class TestBuildManifestVariants: + """Variant handling in build_registry / check_registry manifest builders.""" + + def test_no_variants_omits_field(self): + manifest = build_registry.build_manifest(SAMPLE_PLUGIN, "1.0.0", SAMPLE_BUNDLE) + assert "variants" not in manifest + + def test_empty_variants_omits_field(self): + manifest = build_registry.build_manifest( + SAMPLE_PLUGIN, "1.0.0", SAMPLE_BUNDLE, variants=[] + ) + assert "variants" not in manifest + + def test_variants_inserted_after_bundle(self): + manifest = build_registry.build_manifest( + SAMPLE_PLUGIN, "1.0.0", SAMPLE_BUNDLE, variants=[CUDA_VARIANT] + ) + keys = list(manifest.keys()) + assert manifest["variants"] == [CUDA_VARIANT] + assert keys.index("variants") == keys.index("bundle") + 1 + + def test_check_registry_carries_variants(self): + manifest = check_registry_versions.build_manifest_from_plugin( + SAMPLE_PLUGIN, SAMPLE_BUNDLE, variants=[CUDA_VARIANT] + ) + assert manifest["variants"] == [CUDA_VARIANT] + + def test_build_and_check_manifests_match_with_variants(self): + built = build_registry.build_manifest( + SAMPLE_PLUGIN, "1.0.0", SAMPLE_BUNDLE, variants=[CUDA_VARIANT] + ) + checked = check_registry_versions.build_manifest_from_plugin( + {**SAMPLE_PLUGIN, "version": "1.0.0"}, SAMPLE_BUNDLE, variants=[CUDA_VARIANT] + ) + assert built == checked + + +SAMPLE_ASSET = { + "type_id": "slint", + "label": "Slint Files", + "extensions": ["slint"], + "max_size_bytes": 1048576, + "content_type": "text", + "icon_hint": "code", + "node_param": "slint_file", + "system_dir": "samples/slint/system", +} + + +class TestBuildManifestAssets: + """Asset-type declarations must survive into manifest.json (and the embedded + plugin.yml) so the server can rediscover them after install/extraction.""" + + def test_assets_included_when_present(self): + plugin = {**SAMPLE_PLUGIN, "assets": [SAMPLE_ASSET]} + manifest = build_registry.build_manifest(plugin, "1.0.0", bundle_block=None) + assert manifest["assets"] == [SAMPLE_ASSET] + + def test_assets_omitted_when_absent(self): + manifest = build_registry.build_manifest(SAMPLE_PLUGIN, "1.0.0", bundle_block=None) + assert "assets" not in manifest + + def test_assets_omitted_when_empty(self): + plugin = {**SAMPLE_PLUGIN, "assets": []} + manifest = build_registry.build_manifest(plugin, "1.0.0", bundle_block=None) + assert "assets" not in manifest + + def test_build_and_check_manifests_match_with_assets(self): + plugin = {**SAMPLE_PLUGIN, "assets": [SAMPLE_ASSET]} + built = build_registry.build_manifest(plugin, "1.0.0", SAMPLE_BUNDLE) + checked = check_registry_versions.build_manifest_from_plugin( + {**plugin, "version": "1.0.0"}, SAMPLE_BUNDLE + ) + assert built == checked + + +class TestWritePluginYml: + """build_bundle embeds a plugin.yml so raw-extraction consumers (the demo) + recover asset types the server only reads from plugin.yml.""" + + def test_writes_parseable_yaml_with_assets(self, tmp_path): + import yaml + + plugin = {**SAMPLE_PLUGIN, "assets": [SAMPLE_ASSET]} + manifest = build_registry.build_manifest(plugin, "1.0.0", bundle_block=None) + build_registry.write_plugin_yml(tmp_path, manifest) + + yml_path = tmp_path / "plugin.yml" + assert yml_path.exists() + parsed = yaml.safe_load(yml_path.read_text()) + assert parsed["assets"] == [SAMPLE_ASSET] + assert parsed["id"] == "test-plugin" + + +class TestEnsureSherpaRuntime: + """ensure_sherpa_runtime copies extra GPU libs for cuda variants.""" + + def _make_libs(self, lib_dir: pathlib.Path, names) -> None: + lib_dir.mkdir(parents=True, exist_ok=True) + for name in names: + (lib_dir / name).write_bytes(b"\x7fELF") + + def test_cpu_copies_core_libs_only(self, tmp_path, monkeypatch): + lib_dir = tmp_path / "libs" + self._make_libs( + lib_dir, + build_registry.SHERPA_CORE_LIBS + build_registry.SHERPA_CUDA_LIBS, + ) + monkeypatch.setenv("SHERPA_ONNX_LIB_DIR", str(lib_dir)) + work = tmp_path / "work" + work.mkdir() + build_registry.ensure_sherpa_runtime(work, "cpu") + for name in build_registry.SHERPA_CORE_LIBS: + assert (work / name).exists() + for name in build_registry.SHERPA_CUDA_LIBS: + assert not (work / name).exists() + + def test_cuda_copies_gpu_provider_libs(self, tmp_path, monkeypatch): + lib_dir = tmp_path / "libs" + self._make_libs( + lib_dir, + build_registry.SHERPA_CORE_LIBS + build_registry.SHERPA_CUDA_LIBS, + ) + monkeypatch.setenv("SHERPA_ONNX_LIB_DIR", str(lib_dir)) + work = tmp_path / "work" + work.mkdir() + build_registry.ensure_sherpa_runtime(work, "cuda") + for name in build_registry.SHERPA_CORE_LIBS + build_registry.SHERPA_CUDA_LIBS: + assert (work / name).exists() + + class TestVerifyExistingSignature: """Tests for verify_existing_signature used during manifest reuse.""" diff --git a/scripts/marketplace/verify_bundles.py b/scripts/marketplace/verify_bundles.py index bb263d581..c71e3fba3 100644 --- a/scripts/marketplace/verify_bundles.py +++ b/scripts/marketplace/verify_bundles.py @@ -9,6 +9,10 @@ import sys import tempfile +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) + +from manifest_builder import SHERPA_CUDA_LIBS # noqa: E402 + def readelf_dynamic(path: pathlib.Path) -> tuple[list[str], list[str]]: result = subprocess.run( @@ -37,11 +41,19 @@ def extract_bundle(bundle_path: pathlib.Path, dest: pathlib.Path) -> None: ) +# NEEDED/RUNPATH substrings tolerated only for cuda-tagged bundles. CPU bundles +# stay strict and must not reference any of these. +CUDA_DEP_ALLOWLIST = ("libcudart", "libcublas", "libcudnn", "libcuda", "libnvrtc") + + def find_bundle( - bundles_dir: pathlib.Path, plugin_id: str, version: str + bundles_dir: pathlib.Path, plugin_id: str, version: str, accelerator: str = "cpu" ) -> pathlib.Path | None: """Find bundle for specific plugin version. Returns None if not found.""" - bundle_name = f"{plugin_id}-{version}-bundle.tar.zst" + if accelerator == "cpu": + bundle_name = f"{plugin_id}-{version}-bundle.tar.zst" + else: + bundle_name = f"{plugin_id}-{version}-{accelerator}-bundle.tar.zst" bundle_path = bundles_dir / bundle_name return bundle_path if bundle_path.exists() else None @@ -50,7 +62,13 @@ def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--plugins", required=True, help="Path to plugin metadata JSON") parser.add_argument("--bundles", required=True, help="Directory with bundle archives") + parser.add_argument( + "--accelerator", + default="cpu", + help="Accelerator variant to verify ('cpu' default, or e.g. 'cuda').", + ) args = parser.parse_args() + accelerator = args.accelerator.strip().lower() plugins_path = pathlib.Path(args.plugins) bundles_dir = pathlib.Path(args.bundles) @@ -77,11 +95,15 @@ def main() -> int: errors.append(f"{plugin_id}: missing version field in metadata") continue + # For variant passes only verify plugins that declare the accelerator. + if accelerator != "cpu" and accelerator not in plugin.get("accelerators", ["cpu"]): + continue + entrypoint = plugin["entrypoint"] - bundle_path = find_bundle(bundles_dir, plugin_id, plugin_version) + bundle_path = find_bundle(bundles_dir, plugin_id, plugin_version, accelerator) if bundle_path is None: # Bundle not found - assume it was published earlier (append-only mode) - print(f"Skipping {plugin_id} v{plugin_version} (bundle not in {bundles_dir}, likely already published)") + print(f"Skipping {plugin_id} v{plugin_version} ({accelerator} bundle not in {bundles_dir}, likely already published)") continue with tempfile.TemporaryDirectory() as tmp_dir: @@ -100,6 +122,17 @@ def main() -> int: f"{plugin_id}: entrypoint has RPATH/RUNPATH referencing /usr/local/lib" ) + # CPU bundles must stay free of CUDA dependencies; cuda bundles may + # legitimately link them. + if accelerator == "cpu": + cuda_needed = [ + lib for lib in needed if any(tag in lib for tag in CUDA_DEP_ALLOWLIST) + ] + if cuda_needed: + errors.append( + f"{plugin_id}: CPU bundle links CUDA libraries {cuda_needed}" + ) + if "libsherpa-onnx-c-api.so" in needed: sherpa_lib = tmp_path / "libsherpa-onnx-c-api.so" onnx_lib = tmp_path / "libonnxruntime.so" @@ -111,6 +144,12 @@ def main() -> int: errors.append( f"{plugin_id}: missing libonnxruntime.so in bundle" ) + if accelerator == "cuda": + for cuda_lib in SHERPA_CUDA_LIBS: + if not (tmp_path / cuda_lib).exists(): + errors.append( + f"{plugin_id}: cuda bundle missing GPU provider lib {cuda_lib}" + ) if errors: print("Portability verification failed:") diff --git a/ui/src/types/marketplace.ts b/ui/src/types/marketplace.ts index ce234fdf5..ce06b64ba 100644 --- a/ui/src/types/marketplace.ts +++ b/ui/src/types/marketplace.ts @@ -35,6 +35,13 @@ export type PluginBundle = { size_bytes?: number | null; }; +export type PluginBundleVariant = { + accelerator: string; + url: string; + sha256: string; + size_bytes?: number | null; +}; + export type PluginCompatibility = { streamkit?: string | null; os: string[]; @@ -79,6 +86,7 @@ export type PluginManifest = { repository?: string | null; entrypoint: string; bundle: PluginBundle | null; + variants?: PluginBundleVariant[]; compatibility?: PluginCompatibility | null; models: ModelSpec[]; }; @@ -104,6 +112,7 @@ export type InstallPluginRequest = { version?: string | null; install_models?: boolean; model_ids?: string[] | null; + accelerator?: string | null; }; export type InstallPluginResponse = { diff --git a/ui/src/types/types.ts b/ui/src/types/types.ts index 980df7a30..58129b6dc 100644 --- a/ui/src/types/types.ts +++ b/ui/src/types/types.ts @@ -78,4 +78,6 @@ export interface PluginSummary { plugin_type: PluginType; /** Plugin version from the marketplace record, if available */ version?: string | null; + /** Accelerator variant of the installed bundle (e.g. "cpu", "cuda"), if known */ + accelerator?: string | null; } diff --git a/ui/src/views/plugins/InstalledPluginsTab.tsx b/ui/src/views/plugins/InstalledPluginsTab.tsx index fb08730b1..45ef1cd46 100644 --- a/ui/src/views/plugins/InstalledPluginsTab.tsx +++ b/ui/src/views/plugins/InstalledPluginsTab.tsx @@ -130,6 +130,7 @@ const InstalledPluginsTab: React.FC = () => { {plugin.version && Version: {plugin.version}} + {plugin.accelerator && Accelerator: {plugin.accelerator}} Original kind: {plugin.original_kind} File: {plugin.file_name} Loaded: {loadedAt} diff --git a/ui/src/views/plugins/MarketplacePanels.tsx b/ui/src/views/plugins/MarketplacePanels.tsx index 8366e2beb..0891362ef 100644 --- a/ui/src/views/plugins/MarketplacePanels.tsx +++ b/ui/src/views/plugins/MarketplacePanels.tsx @@ -184,6 +184,8 @@ const MarketplaceInstallWarnings: React.FC<{ type MarketplaceDetailsPanelProps = { details: MarketplacePluginDetails | null; selectedVersion: string | null; + selectedAccelerator: string; + onAcceleratorChange: (value: string) => void; loading: boolean; licenseAccepted: boolean; requiresLicenseAcceptance: boolean; @@ -206,6 +208,8 @@ type MarketplaceDetailsPanelProps = { export const MarketplaceDetailsPanel: React.FC = ({ details, selectedVersion, + selectedAccelerator, + onAcceleratorChange, loading, licenseAccepted, requiresLicenseAcceptance, @@ -260,6 +264,8 @@ export const MarketplaceDetailsPanel: React.FC = ( details={details} selectedVersion={selectedVersion} onVersionChange={onVersionChange} + selectedAccelerator={selectedAccelerator} + onAcceleratorChange={onAcceleratorChange} /> void; + selectedAccelerator: string; + onAcceleratorChange: (value: string) => void; +}; + +// The canonical bundle is always the CPU build; `variants` carries additional +// accelerator-specific builds (e.g. CUDA). +export const manifestAccelerators = (manifest: MarketplacePluginDetails['manifest']): string[] => { + const variants = manifest.variants ?? []; + const accelerators = ['cpu', ...variants.map((variant) => variant.accelerator.toLowerCase())]; + return [...new Set(accelerators)]; }; const MarketplaceDetailsFields: React.FC = ({ details, selectedVersion, onVersionChange, + selectedAccelerator, + onAcceleratorChange, }) => { const signatureLabel = details.signature.verified ? `\u2713 Verified (${details.signature.key_id ?? 'trusted key'})` @@ -354,6 +372,11 @@ const MarketplaceDetailsFields: React.FC = ({ ))} + Node kind {details.manifest.node_kind} Entry point @@ -391,6 +414,33 @@ const MarketplaceDetailsFields: React.FC = ({ ); }; +const MarketplaceAcceleratorRow: React.FC<{ + manifest: MarketplacePluginDetails['manifest']; + selectedAccelerator: string; + onAcceleratorChange: (value: string) => void; +}> = ({ manifest, selectedAccelerator, onAcceleratorChange }) => { + const accelerators = manifestAccelerators(manifest); + if (accelerators.length < 2) return null; + return ( + <> + Accelerator + + + + + ); +}; + const MarketplaceCompatibilityRows: React.FC<{ compatibility: MarketplacePluginDetails['manifest']['compatibility']; }> = ({ compatibility }) => { diff --git a/ui/src/views/plugins/MarketplaceTab.tsx b/ui/src/views/plugins/MarketplaceTab.tsx index d39f87226..d5065468d 100644 --- a/ui/src/views/plugins/MarketplaceTab.tsx +++ b/ui/src/views/plugins/MarketplaceTab.tsx @@ -214,6 +214,7 @@ const startInstall = async ({ details, installModels, selectedModelIds, + selectedAccelerator, resetJob, setInstalling, setJobId, @@ -222,6 +223,7 @@ const startInstall = async ({ details: MarketplacePluginDetails | null; installModels: boolean; selectedModelIds: string[]; + selectedAccelerator: string; resetJob: () => void; setInstalling: React.Dispatch>; setJobId: React.Dispatch>; @@ -241,6 +243,7 @@ const startInstall = async ({ version: details.version.version, install_models: installModels, model_ids: modelIds, + accelerator: selectedAccelerator || undefined, }); setJobId(response.job_id); resetJob(); @@ -255,6 +258,7 @@ const startInstall = async ({ const useMarketplaceHandlers = ({ details, selectedModelIds, + selectedAccelerator, resetJob, setInstalling, setJobId, @@ -265,6 +269,7 @@ const useMarketplaceHandlers = ({ }: { details: MarketplacePluginDetails | null; selectedModelIds: string[]; + selectedAccelerator: string; resetJob: () => void; setInstalling: React.Dispatch>; setJobId: React.Dispatch>; @@ -280,12 +285,13 @@ const useMarketplaceHandlers = ({ details, installModels: hasSelectedModels, selectedModelIds, + selectedAccelerator, resetJob, setInstalling, setJobId, toast, }); - }, [details, selectedModelIds, resetJob, setInstalling, setJobId, toast]); + }, [details, selectedModelIds, selectedAccelerator, resetJob, setInstalling, setJobId, toast]); const handleClearJob = useCallback(() => { setJobId(null); @@ -344,6 +350,7 @@ const MarketplaceTab: React.FC = ({ active }) => { const { searchInput, setSearchInput, debouncedSearch } = useDebouncedSearch('', 300); const [selectedPluginId, setSelectedPluginId] = useState(null); const [licenseAccepted, setLicenseAccepted] = useState(false); + const [selectedAccelerator, setSelectedAccelerator] = useState(''); const [selectedModelIds, setSelectedModelIds] = useState([]); const [jobId, setJobId] = useState(null); const [installing, setInstalling] = useState(false); @@ -368,7 +375,10 @@ const MarketplaceTab: React.FC = ({ active }) => { }, [index, selectedPluginId]); useEffect(() => { - startTransition(() => setLicenseAccepted(false)); + startTransition(() => { + setLicenseAccepted(false); + setSelectedAccelerator(''); + }); }, [selectedPluginId, selectedVersion]); useEffect(() => { @@ -412,6 +422,7 @@ const MarketplaceTab: React.FC = ({ active }) => { } = useMarketplaceHandlers({ details, selectedModelIds, + selectedAccelerator, resetJob, setInstalling, setJobId, @@ -512,6 +523,8 @@ const MarketplaceTab: React.FC = ({ active }) => { details={details} selectedVersion={selectedVersion} onVersionChange={setSelectedVersion} + selectedAccelerator={selectedAccelerator} + onAcceleratorChange={setSelectedAccelerator} loading={detailsLoading} licenseAccepted={licenseAccepted} onLicenseAccepted={setLicenseAccepted}