diff --git a/.github/backend-matrix.yml b/.github/backend-matrix.yml index e7bada4b4e91..adbb1aefac83 100644 --- a/.github/backend-matrix.yml +++ b/.github/backend-matrix.yml @@ -11,6 +11,8 @@ # CUDA / ROCm / SYCL / Vulkan variants). # - macOS -> the `includeDarwin:` matrix (Apple Silicon / arm64; Metal where # the engine supports it, otherwise a native arm64 CPU build). +# - Windows -> the `includeWindows:` matrix (x86_64 / amd64; native builds +# under MSYS2, no WSL/Docker — see backend_build_windows.yml). # # New backends must target EVERY OS they can build for, not just Linux. A backend # listed only under `include:` is silently unavailable on macOS even when its code @@ -24,6 +26,12 @@ # `metal:` capability + `metal-` image entries, a `run.sh` Darwin/DYLD # branch for C/C++ backends, and the inferBackendPathDarwin case in # scripts/lib/backend-filter.mjs so the path filter actually builds it). +# +# Windows builds are bespoke for now: every entry builds via a per-backend make +# target + MSYS2 build script (see scripts/build/llama-cpp-windows.sh), the +# index.yaml `windows:` capability + `windows-` image entries, a +# run-windows launcher next to run.sh, and the inferBackendPathWindows case in +# scripts/lib/backend-filter.mjs. # Linux matrix (consumed by backend-jobs). include: @@ -6540,3 +6548,11 @@ includeDarwin: - backend: "ds4" tag-suffix: "-metal-darwin-arm64-ds4" lang: "go" + +# Windows matrix (consumed by backend-jobs-windows). +# Native windows/amd64 builds under MSYS2 — no WSL, no Docker. Each entry builds +# via a bespoke make target + scripts/build/-windows.sh. +includeWindows: + - backend: "llama-cpp" + tag-suffix: "-windows-amd64-llama-cpp" + lang: "go" diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 13e67c6fe4d0..a1331fe3bdff 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -34,9 +34,11 @@ jobs: outputs: matrix-multiarch: ${{ steps.set-matrix.outputs['matrix-multiarch'] }} matrix-darwin: ${{ steps.set-matrix.outputs['matrix-darwin'] }} + matrix-windows: ${{ steps.set-matrix.outputs['matrix-windows'] }} merge-matrix-multiarch: ${{ steps.set-matrix.outputs['merge-matrix-multiarch'] }} has-backends-multiarch: ${{ steps.set-matrix.outputs['has-backends-multiarch'] }} has-backends-darwin: ${{ steps.set-matrix.outputs['has-backends-darwin'] }} + has-backends-windows: ${{ steps.set-matrix.outputs['has-backends-windows'] }} has-merges-multiarch: ${{ steps.set-matrix.outputs['has-merges-multiarch'] }} # Single-arch backends are sharded across SINGLEARCH_SHARDS matrix jobs to # stay under GitHub's 256-jobs-per-matrix limit (see changed-backends.js). @@ -368,3 +370,23 @@ jobs: strategy: fail-fast: false matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix-darwin) }} + + backend-jobs-windows: + needs: generate-matrix + if: needs.generate-matrix.outputs.has-backends-windows == 'true' + uses: ./.github/workflows/backend_build_windows.yml + with: + backend: ${{ matrix.backend }} + build-type: ${{ matrix.build-type }} + go-version: "1.25.x" + tag-suffix: ${{ matrix.tag-suffix }} + lang: ${{ matrix.lang || 'go' }} + runs-on: "windows-latest" + secrets: + dockerUsername: ${{ secrets.DOCKERHUB_USERNAME }} + dockerPassword: ${{ secrets.DOCKERHUB_PASSWORD }} + quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }} + quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix-windows) }} diff --git a/.github/workflows/backend_build_windows.yml b/.github/workflows/backend_build_windows.yml new file mode 100644 index 000000000000..736b58c31255 --- /dev/null +++ b/.github/workflows/backend_build_windows.yml @@ -0,0 +1,256 @@ +--- +name: 'build windows backend container images (reusable)' + +on: + workflow_call: + inputs: + backend: + description: 'Backend to build' + required: true + type: string + build-type: + description: 'Build type' + default: '' + type: string + lang: + description: 'Programming language (e.g. go)' + default: 'python' + type: string + go-version: + description: 'Go version to use' + default: '1.25.x' + type: string + tag-suffix: + description: 'Tag suffix for the built image' + required: true + type: string + runs-on: + description: 'Runner to use' + default: 'windows-latest' + type: string + secrets: + dockerUsername: + required: false + dockerPassword: + required: false + quayUsername: + required: true + quayPassword: + required: true + +jobs: + windows-backend-build: + runs-on: ${{ inputs.runs-on }} + strategy: + matrix: + go-version: ['${{ inputs.go-version }}'] + env: + # Every CMake variant below (gRPC + the three llama.cpp variants) compiles + # the same source trees with overlapping flags; ccache dedupes them. + # CCACHE_DIR is set in a run step (below), not here: a job-level env value + # is used verbatim, and MSYS2's $HOME must be expanded at runtime so it + # matches the ~/.cache/ccache path handed to actions/cache. + CMAKE_ARGS: "-DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" + steps: + - name: Clone + uses: actions/checkout@v7 + with: + submodules: true + + - name: Setup Go ${{ matrix.go-version }} + uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go-version }} + # Caches ~/go/pkg/mod and %LOCALAPPDATA%\go-build keyed on go.sum. + cache: true + + - name: Display Go version + run: go version + + # ---- MSYS2 toolchain ---- + # Native windows builds need a mingw gcc toolchain; windows-latest ships + # none by default (build-test-windows works only because LocalAI itself + # builds with CGO_ENABLED=0). UCRT64 matches the /ucrt64/bin DLL bundling + # in scripts/build/llama-cpp-windows.sh. `perl` is required by gRPC's + # third_party/openssl build (openssl's Configure is a perl script). + # `install` is a folded scalar (>-): every line folds into the pacman + # command, so no `#` comments may live inside it - they would become + # literal package names ("target not found: Vulkan:"). Vulkan packages: + # headers + loader (ucrt64) for FindVulkan, and the mingw64 shaderc + # build for glslc (no ucrt64 shaderc exists; glslc is a standalone tool + # so the prefix split is irrelevant). The loader's vulkan-1.dll is + # bundled next to the image's DLLs; ggml-vulkan loads it dynamically, so + # it is never a hard import. spirv-headers provides the cmake config + # ggml-vulkan requires (find_package(SPIRV-Headers CONFIG REQUIRED)). + - name: Set up MSYS2 + uses: msys2/setup-msys2@v2 + with: + msystem: UCRT64 + update: false + install: >- + git + make + cmake + mingw-w64-ucrt-x86_64-cmake + ninja + pkg-config + perl + patch + unzip + curl + mingw-w64-ucrt-x86_64-gcc + mingw-w64-ucrt-x86_64-gcc-libs + mingw-w64-ucrt-x86_64-binutils + mingw-w64-ucrt-x86_64-ccache + mingw-w64-ucrt-x86_64-vulkan-headers + mingw-w64-ucrt-x86_64-vulkan-loader + mingw-w64-ucrt-x86_64-spirv-headers + mingw-w64-x86_64-shaderc + + # Run steps that need bash/msys2 tools (make, ccache, grep) declare + # `shell: msys2 {0}` individually instead of a job-level defaults block: + # the msys2 shell only exists once the Set up MSYS2 step above has run. + - name: Set CCACHE_DIR + shell: msys2 {0} + run: echo "CCACHE_DIR=$HOME/.cache/ccache" >> "$GITHUB_ENV" + + - name: Display toolchain versions + shell: msys2 {0} + run: | + gcc --version | head -1 + cmake --version | head -1 + make --version | head -1 + ccache --version | head -1 + + # ---- ccache for llama.cpp CMake builds ---- + # Same shape as the Darwin workflow: key on the pinned LLAMA_VERSION so a + # pin bump invalidates cleanly; restore-keys fall back to the latest entry + # for the same pin so unchanged TUs stay warm. + - name: Compute llama.cpp version + if: inputs.backend == 'llama-cpp' + id: llama-version + shell: msys2 {0} + run: | + version=$(grep '^LLAMA_VERSION' backend/cpp/llama-cpp/Makefile | head -1 | cut -d= -f2 | cut -d'?' -f1 | tr -d ' ') + echo "version=${version}" >> "$GITHUB_OUTPUT" + + - name: Restore ccache + if: inputs.backend == 'llama-cpp' + id: ccache-cache + uses: actions/cache/restore@v6 + with: + path: ~/.cache/ccache + key: ccache-llama-windows-amd64-${{ steps.llama-version.outputs.version }}-${{ github.run_id }} + restore-keys: | + ccache-llama-windows-amd64-${{ steps.llama-version.outputs.version }}- + + # Only llama-cpp has a windows build path today - the matrix's + # includeWindows section lists exactly this backend. Fail loudly rather + # than upload an empty tar if a future entry dispatches here without a + # build step of its own. Keep in sync with WINDOWS_BESPOKE_BUILDERS in + # scripts/lib/backend-filter.mjs. + - name: Check backend is supported + if: inputs.backend != 'llama-cpp' + run: | + echo "::error::no windows build path for backend '${{ inputs.backend }}'" + exit 1 + + # The msys2 shell below resets PATH, so the Go toolchain setup-go put on + # the runner PATH is invisible to it (and setup-go only exports GOROOT + # for Go < 1.9). Resolve the install dir with the default shell, where + # `go` is reachable, and hand it to the script to prepend. + - name: Resolve Go toolchain path + id: go-toolchain + if: inputs.backend == 'llama-cpp' + shell: bash + run: echo "root=$(go env GOROOT)" >> "$GITHUB_OUTPUT" + + - name: Build ${{ inputs.backend }} (llama-cpp) + if: inputs.backend == 'llama-cpp' + shell: msys2 {0} + env: + GO_TOOLCHAIN_ROOT: ${{ steps.go-toolchain.outputs.root }} + run: | + make backends/llama-cpp-windows + + - name: ccache stats + if: inputs.backend == 'llama-cpp' + shell: msys2 {0} + run: ccache -s + + - name: Save ccache + if: inputs.backend == 'llama-cpp' && github.event_name != 'pull_request' + uses: actions/cache/save@v6 + with: + path: ~/.cache/ccache + key: ccache-llama-windows-amd64-${{ steps.llama-version.outputs.version }}-${{ github.run_id }} + + - name: Upload ${{ inputs.backend }}.tar + uses: actions/upload-artifact@v7 + with: + name: ${{ inputs.backend }}-tar + path: backend-images/${{ inputs.backend }}.tar + + windows-backend-publish: + needs: windows-backend-build + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Download ${{ inputs.backend }}.tar + uses: actions/download-artifact@v8 + with: + name: ${{ inputs.backend }}-tar + path: . + + - name: Install crane + run: | + curl -L https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz | tar -xz + sudo mv crane /usr/local/bin/ + + - name: Log in to DockerHub + run: | + echo "${{ secrets.dockerPassword }}" | crane auth login docker.io -u "${{ secrets.dockerUsername }}" --password-stdin + + - name: Log in to quay.io + run: | + echo "${{ secrets.quayPassword }}" | crane auth login quay.io -u "${{ secrets.quayUsername }}" --password-stdin + + - name: Docker meta + id: meta + uses: docker/metadata-action@v6 + with: + images: | + localai/localai-backends + tags: | + type=ref,event=branch + type=semver,pattern={{raw}} + type=sha + flavor: | + latest=auto + suffix=${{ inputs.tag-suffix }},onlatest=true + + - name: Docker meta + id: quaymeta + uses: docker/metadata-action@v6 + with: + images: | + quay.io/go-skynet/local-ai-backends + tags: | + type=ref,event=branch + type=semver,pattern={{raw}} + type=sha + flavor: | + latest=auto + suffix=${{ inputs.tag-suffix }},onlatest=true + + - name: Push Docker image (DockerHub) + run: | + for tag in $(echo "${{ steps.meta.outputs.tags }}" | tr ',' '\n'); do + crane push ${{ inputs.backend }}.tar $tag + done + + - name: Push Docker image (Quay) + run: | + for tag in $(echo "${{ steps.quaymeta.outputs.tags }}" | tr ',' '\n'); do + crane push ${{ inputs.backend }}.tar $tag + done diff --git a/.github/workflows/backend_pr.yml b/.github/workflows/backend_pr.yml index c13c444c455b..f88306c35b9c 100644 --- a/.github/workflows/backend_pr.yml +++ b/.github/workflows/backend_pr.yml @@ -13,9 +13,11 @@ jobs: outputs: matrix-multiarch: ${{ steps.set-matrix.outputs['matrix-multiarch'] }} matrix-darwin: ${{ steps.set-matrix.outputs['matrix-darwin'] }} + matrix-windows: ${{ steps.set-matrix.outputs['matrix-windows'] }} merge-matrix-multiarch: ${{ steps.set-matrix.outputs['merge-matrix-multiarch'] }} has-backends-multiarch: ${{ steps.set-matrix.outputs['has-backends-multiarch'] }} has-backends-darwin: ${{ steps.set-matrix.outputs['has-backends-darwin'] }} + has-backends-windows: ${{ steps.set-matrix.outputs['has-backends-windows'] }} has-merges-multiarch: ${{ steps.set-matrix.outputs['has-merges-multiarch'] }} # Single-arch backends are sharded across SINGLEARCH_SHARDS matrix jobs to # stay under GitHub's 256-jobs-per-matrix limit (see changed-backends.js). @@ -292,3 +294,21 @@ jobs: strategy: fail-fast: true matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix-darwin) }} + + backend-jobs-windows: + needs: generate-matrix + uses: ./.github/workflows/backend_build_windows.yml + if: needs.generate-matrix.outputs.has-backends-windows == 'true' + with: + backend: ${{ matrix.backend }} + build-type: ${{ matrix.build-type }} + go-version: "1.25.x" + tag-suffix: ${{ matrix.tag-suffix }} + lang: ${{ matrix.lang || 'go' }} + runs-on: "windows-latest" + secrets: + quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }} + quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }} + strategy: + fail-fast: true + matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix-windows) }} diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index 08fb2d0842a3..d875b31483c2 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -53,6 +53,35 @@ jobs: - name: Run GoReleaser run: | make ${{ github.event_name == 'pull_request' && 'dev-dist-single' || 'dev-dist' }} + # Windows ships no GNU toolchain on the default runner: make is not + # installed there and Git for Windows does not ship it (it bundles sh, uname, + # unzip, grep, awk, sed only), so it is installed via Chocolatey below. The + # Makefile finds Git for Windows' sh itself and runs recipes through it (see + # the SHELL setup at the top of the Makefile), so the explicit usr/bin PATH + # step below is belt-and-suspenders for environments where make must locate + # sh before the makefile is read. The server builds with CGO_ENABLED=0, so no + # mingw/gcc is needed; the Makefile downloads protoc and installs the Go + # protobuf plugins itself (see the protoc/protogen-go targets). + build-test-windows: + runs-on: windows-latest + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + fetch-depth: 0 + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: 1.25 + - name: Add Git usr/bin to PATH + run: | + echo "$env:ProgramFiles\Git\usr\bin" | Out-File -Append -Encoding utf8 $env:GITHUB_PATH + - name: Install GNU Make + run: choco install make -y + - name: Build LocalAI (CGO disabled, mirrors release config) + env: + CGO_ENABLED: '0' + run: make build launcher-build-darwin: runs-on: macos-latest steps: diff --git a/.gitignore b/.gitignore index a4c84a0e8dfe..b1fe53accc10 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ prepare-sources /backend-images /result.yaml protoc +protoc.exe *.log @@ -25,6 +26,7 @@ go-bert # LocalAI build binary LocalAI /local-ai +/local-ai.exe /local-ai-launcher # Root-level build artifacts when running `go build ./...` against # Go backend packages whose main lives under backend/go/. @@ -129,3 +131,6 @@ formal-verification/out/ # root, which is what a contributor testing a build does. Nothing under here is # source: it is the instance's own models, outputs, traces and identity. /data/ + +# Model gallery index cache fetched by `local-ai` at runtime (cache/gallery/). +/cache/ diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 88d6e9ecd9e4..67bc8cf33b0e 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -20,7 +20,7 @@ builds: goos: - linux - darwin - #- windows + - windows goarch: - amd64 - arm64 diff --git a/Makefile b/Makefile index 1f25e246c423..fddf123694ee 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,28 @@ # Disable parallel execution for backend builds -.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/bonsai backends/outetts backends/piper backends/stablediffusion-ggml backends/trellis2cpp backends/trellis2cpp-darwin backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/nemo-speech-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/valkey-store backends/cloud-proxy backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/moss-tts-cpp backends/magpie-tts-cpp backends/vllm-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin backends/audio-cpp backends/audio-cpp-darwin +.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/bonsai backends/outetts backends/piper backends/stablediffusion-ggml backends/trellis2cpp backends/trellis2cpp-darwin backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/nemo-speech-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/valkey-store backends/cloud-proxy backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/moss-tts-cpp backends/magpie-tts-cpp backends/vllm-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin backends/audio-cpp backends/audio-cpp-darwin backends/llama-cpp-windows + +# Native GNU make for Windows (e.g. ezwinports) only behaves like the Unix +# make when it can find sh.exe: otherwise recipe lines run under cmd.exe and +# $(shell ...) degrades to bare CreateProcess, which breaks every POSIX +# recipe and the uname/tput expansion below. Seed the exported PATH with the +# sh directories from the standard Git-for-Windows / MSYS2 installs (missing +# entries are harmless in a Windows PATH) and force SHELL to sh, mirroring +# what CI does. Detection uses the cmd environment OS variable (Windows_NT); +# OS is re-derived from uname further down. Nothing matches on +# Linux/macOS/WSL, so the block is inert there. +ifeq ($(findstring Windows,$(OS)),Windows) + export PATH := C:/Program Files/Git/usr/bin;$(if $(LOCALAPPDATA),$(LOCALAPPDATA)/Programs/Git/usr/bin,);C:/msys64/usr/bin;$(PATH) + export SHELL := sh +endif GOCMD=go GOTEST=$(GOCMD) test GOVET=$(GOCMD) vet -BINARY_NAME=local-ai +# Windows builds get the .exe suffix so the artifact is runnable from cmd / +# PowerShell (an extensionless PE needs a POSIX shell to launch it). OS is +# assigned below via uname; this is a recursive = so the suffix is expanded +# at use time, after OS exists. Empty on Linux/macOS, so nothing else changes. +BINARY_NAME=local-ai$(if $(findstring NT,$(OS)),.exe) LAUNCHER_BINARY_NAME=local-ai-launcher UBUNTU_VERSION?=2404 @@ -532,10 +550,18 @@ help: ## Show this help. .PHONY: protogen protogen: protogen-go +# The win64 protoc zip ships bin/protoc.exe while the unix zips ship +# bin/protoc. protogen-go invokes a bare ./protoc, and the protoc target must +# stay up-to-date once the binary is in place, so on Windows we extract the +# .exe and rename it to ./protoc (MSYS sh runs extensionless PE binaries). +PROTOC_MEMBER := bin/protoc$(if $(findstring NT,$(OS)),.exe) + protoc: @OS_NAME=$$(uname -s | tr '[:upper:]' '[:lower:]'); \ ARCH_NAME=$$(uname -m); \ - if [ "$$OS_NAME" = "darwin" ]; then \ + if echo "$$OS_NAME" | grep -qE 'mingw|msys|cygwin'; then \ + FILE=protoc-31.1-win64.zip; \ + elif [ "$$OS_NAME" = "darwin" ]; then \ if [ "$$ARCH_NAME" = "arm64" ]; then \ FILE=protoc-31.1-osx-aarch_64.zip; \ elif [ "$$ARCH_NAME" = "x86_64" ]; then \ @@ -562,18 +588,20 @@ protoc: fi; \ URL=https://github.com/protocolbuffers/protobuf/releases/download/v31.1/$$FILE; \ curl -L $$URL -o protoc.zip && \ - unzip -j -d $(CURDIR) protoc.zip bin/protoc && rm protoc.zip + unzip -o -j -d $(CURDIR) protoc.zip $(PROTOC_MEMBER) && \ + rm -f protoc.zip && \ + [ ! -f ./protoc.exe ] || mv -f ./protoc.exe ./protoc .PHONY: protogen-go protogen-go: protoc install-go-tools mkdir -p pkg/grpc/proto # install-go-tools writes protoc-gen-go and protoc-gen-go-grpc into - # $(shell go env GOPATH)/bin, which isn't on every dev's PATH. protoc - # resolves its code-gen plugins via PATH, so without this prefix the - # generate step fails with "protoc-gen-go: program not found". Prepend - # GOPATH/bin so the freshly-installed plugins win without requiring a - # shell-profile change. - PATH="$$(go env GOPATH)/bin:$$PATH" ./protoc --experimental_allow_proto3_optional -Ibackend/ --go_out=pkg/grpc/proto/ --go_opt=paths=source_relative --go-grpc_out=pkg/grpc/proto/ --go-grpc_opt=paths=source_relative \ + # $(shell go env GOPATH)/bin, which isn't on every dev's PATH. Point + # protoc at the plugins explicitly (--plugin) so discovery doesn't depend + # on PATH separator conventions (POSIX ':' vs Windows ';'). + ./protoc --experimental_allow_proto3_optional -Ibackend/ --go_out=pkg/grpc/proto/ --go_opt=paths=source_relative --go-grpc_out=pkg/grpc/proto/ --go-grpc_opt=paths=source_relative \ + --plugin=protoc-gen-go="$$(go env GOPATH)/bin/protoc-gen-go$(if $(findstring NT,$(OS)),.exe,)" \ + --plugin=protoc-gen-go-grpc="$$(go env GOPATH)/bin/protoc-gen-go-grpc$(if $(findstring NT,$(OS)),.exe,)" \ backend/backend.proto core/config/inference_defaults.json: ## Fetch inference defaults from unsloth (only if missing) @@ -1223,6 +1251,21 @@ backends/audio-cpp-darwin: build bash ./scripts/build/audio-cpp-darwin.sh ./local-ai backends install "ocifile://$(abspath ./backend-images/audio-cpp.tar)" +# Windows-specific backends (keep as explicit targets since they have special build logic). +# Built on windows-latest under MSYS2 — see .github/workflows/backend_build_windows.yml. +# Unlike the darwin targets this does not depend on `build`: the windows runner +# builds local-ai.exe with setup-go (go build ./cmd/local-ai), and the full +# `make build` would additionally need node for the React UI, which this +# packaging path does not use. The script still builds local-ai itself when it +# is missing so the target works outside CI too. +# `ocifile://` hands the path straight to tarball.ImageFromPath, which on +# Windows needs a drive-letter path — $(abspath) yields the MSYS /d/... form, +# so it is converted via cygpath -m. +backends/llama-cpp-windows: + bash ./scripts/build/llama-cpp-windows.sh + ./$(BINARY_NAME) backends install "ocifile://$(shell cygpath -m $(abspath ./backend-images/llama-cpp.tar))" + + build-darwin-python-backend: build bash ./scripts/build/python-darwin.sh diff --git a/backend/cpp/llama-cpp/run.ps1 b/backend/cpp/llama-cpp/run.ps1 new file mode 100644 index 000000000000..22efbd13978f --- /dev/null +++ b/backend/cpp/llama-cpp/run.ps1 @@ -0,0 +1,31 @@ +# run.ps1 - backend launcher for the native windows/amd64 llama-cpp backend +# images. The mirror of ../run.sh for a host with no POSIX shell: pick the best +# binary for the host, set up the library search path, then run it, forwarding +# arguments and exit code. pkg/model/process.go starts it via +# `powershell.exe -NoProfile -ExecutionPolicy Bypass -File run.ps1`. + +$ErrorActionPreference = "Stop" + +$curDir = Split-Path -Parent $MyInvocation.MyCommand.Path + +$binary = "llama-cpp-fallback.exe" +if (Test-Path (Join-Path $curDir "llama-cpp-cpu-all.exe")) { + $binary = "llama-cpp-cpu-all.exe" +} + +if ($env:LLAMACPP_GRPC_SERVERS) { + if (Test-Path (Join-Path $curDir "llama-cpp-grpc.exe")) { + $binary = "llama-cpp-grpc.exe" + } +} + +# ggml's shared backends (libggml-cpu-*.dll) live next to the executable so +# ggml's own registry finds them; a lib\ dir is still honoured when a variant +# ships one, mirroring the run.sh LD_LIBRARY_PATH handling. +$lib = Join-Path $curDir "lib" +if (Test-Path $lib) { + $env:PATH = "$lib;$env:PATH" +} + +& (Join-Path $curDir $binary) @args +exit $LASTEXITCODE diff --git a/backend/index.yaml b/backend/index.yaml index 3420d85496b7..14f80ef77400 100644 --- a/backend/index.yaml +++ b/backend/index.yaml @@ -25,6 +25,7 @@ intel: "intel-sycl-f16-llama-cpp" amd: "rocm-llama-cpp" metal: "metal-llama-cpp" + windows: "windows-llama-cpp" vulkan: "vulkan-llama-cpp" nvidia-l4t: "nvidia-l4t-arm64-llama-cpp" nvidia-cuda-13: "cuda13-llama-cpp" @@ -2049,6 +2050,7 @@ intel: "intel-sycl-f16-llama-cpp-development" amd: "rocm-llama-cpp-development" metal: "metal-llama-cpp-development" + windows: "windows-llama-cpp-development" vulkan: "vulkan-llama-cpp-development" nvidia-l4t: "nvidia-l4t-arm64-llama-cpp-development" nvidia-cuda-13: "cuda13-llama-cpp-development" @@ -2696,6 +2698,16 @@ uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-llama-cpp" mirrors: - localai/localai-backends:master-metal-darwin-arm64-llama-cpp +- !!merge <<: *llamacpp + name: "windows-llama-cpp" + uri: "quay.io/go-skynet/local-ai-backends:latest-windows-amd64-llama-cpp" + mirrors: + - localai/localai-backends:latest-windows-amd64-llama-cpp +- !!merge <<: *llamacpp + name: "windows-llama-cpp-development" + uri: "quay.io/go-skynet/local-ai-backends:master-windows-amd64-llama-cpp" + mirrors: + - localai/localai-backends:master-windows-amd64-llama-cpp - !!merge <<: *llamacpp name: "cuda12-llama-cpp-development" uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-llama-cpp" diff --git a/core/gallery/backends_test.go b/core/gallery/backends_test.go index 1b7e059bee9b..27ad6779cc80 100644 --- a/core/gallery/backends_test.go +++ b/core/gallery/backends_test.go @@ -376,7 +376,7 @@ var _ = Describe("Gallery Backends", func() { }, URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-llama-cpp", } - Expect(cudaBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.Nvidia, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue()) + Expect(cudaBackend.IsCompatibleWith(system.NewCapabilityState(system.Nvidia))).To(BeTrue()) }) It("should be compatible with cuda13 backend on nvidia GPU", func() { @@ -386,7 +386,7 @@ var _ = Describe("Gallery Backends", func() { }, URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-13-llama-cpp", } - Expect(cuda13Backend.IsCompatibleWith(&system.SystemState{GPUVendor: system.Nvidia, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue()) + Expect(cuda13Backend.IsCompatibleWith(system.NewCapabilityState(system.Nvidia))).To(BeTrue()) }) }) }) @@ -417,7 +417,7 @@ var _ = Describe("Gallery Backends", func() { }, URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-rocm-hipblas-llama-cpp", } - Expect(rocmBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.AMD, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue()) + Expect(rocmBackend.IsCompatibleWith(system.NewCapabilityState(system.AMD))).To(BeTrue()) }) It("should be compatible with hipblas backend on AMD GPU", func() { @@ -427,7 +427,7 @@ var _ = Describe("Gallery Backends", func() { }, URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-hip-llama-cpp", } - Expect(hipBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.AMD, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue()) + Expect(hipBackend.IsCompatibleWith(system.NewCapabilityState(system.AMD))).To(BeTrue()) }) }) }) @@ -458,7 +458,7 @@ var _ = Describe("Gallery Backends", func() { }, URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-sycl-f16-llama-cpp", } - Expect(intelBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.Intel, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue()) + Expect(intelBackend.IsCompatibleWith(system.NewCapabilityState(system.Intel))).To(BeTrue()) }) It("should be compatible with intel-sycl-f32 backend on Intel GPU", func() { @@ -468,7 +468,7 @@ var _ = Describe("Gallery Backends", func() { }, URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-sycl-f32-llama-cpp", } - Expect(intelF32Backend.IsCompatibleWith(&system.SystemState{GPUVendor: system.Intel, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue()) + Expect(intelF32Backend.IsCompatibleWith(system.NewCapabilityState(system.Intel))).To(BeTrue()) }) It("should be compatible with intel-transformers backend on Intel GPU", func() { @@ -478,7 +478,7 @@ var _ = Describe("Gallery Backends", func() { }, URI: "quay.io/go-skynet/local-ai-backends:latest-intel-transformers", } - Expect(intelTransformersBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.Intel, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue()) + Expect(intelTransformersBackend.IsCompatibleWith(system.NewCapabilityState(system.Intel))).To(BeTrue()) }) }) }) @@ -551,28 +551,28 @@ var _ = Describe("Gallery Backends", func() { } else { // Test with NVIDIA system state - nvidiaSystemState := &system.SystemState{GPUVendor: "nvidia", VRAM: 1000000000000} + nvidiaSystemState := system.NewCapabilityState("nvidia") bestBackend := metaBackend.FindBestBackendFromMeta(nvidiaSystemState, backends) Expect(bestBackend).To(Equal(nvidiaBackend)) // Test with AMD system state - amdSystemState := &system.SystemState{GPUVendor: "amd", VRAM: 1000000000000} + amdSystemState := system.NewCapabilityState("amd") bestBackend = metaBackend.FindBestBackendFromMeta(amdSystemState, backends) Expect(bestBackend).To(Equal(amdBackend)) // Test with default system state (not enough VRAM) - defaultSystemState := &system.SystemState{GPUVendor: "amd"} + defaultSystemState := system.NewCapabilityState("default") bestBackend = metaBackend.FindBestBackendFromMeta(defaultSystemState, backends) Expect(bestBackend).To(Equal(defaultBackend)) // Test with default system state - defaultSystemState = &system.SystemState{GPUVendor: "default"} + defaultSystemState = system.NewCapabilityState("default") bestBackend = metaBackend.FindBestBackendFromMeta(defaultSystemState, backends) Expect(bestBackend).To(Equal(defaultBackend)) backends = GalleryElements[*GalleryBackend]{nvidiaBackend, amdBackend, metalBackend} // Test with unsupported GPU vendor - unsupportedSystemState := &system.SystemState{GPUVendor: "unsupported"} + unsupportedSystemState := system.NewCapabilityState("unsupported") bestBackend = metaBackend.FindBestBackendFromMeta(unsupportedSystemState, backends) Expect(bestBackend).To(BeNil()) } @@ -621,11 +621,7 @@ var _ = Describe("Gallery Backends", func() { Expect(err).NotTo(HaveOccurred()) // Test with NVIDIA system state - nvidiaSystemState := &system.SystemState{ - GPUVendor: "nvidia", - VRAM: 1000000000000, - Backend: system.Backend{BackendsPath: tempDir}, - } + nvidiaSystemState := system.NewCapabilityState("nvidia", system.WithBackendPath(tempDir)) err = InstallBackendFromGallery(context.TODO(), []config.Gallery{gallery}, nvidiaSystemState, ml, "meta-backend", nil, true, false) Expect(err).NotTo(HaveOccurred()) @@ -701,11 +697,7 @@ var _ = Describe("Gallery Backends", func() { Expect(err).NotTo(HaveOccurred()) // Test with NVIDIA system state - nvidiaSystemState := &system.SystemState{ - GPUVendor: "nvidia", - VRAM: 1000000000000, - Backend: system.Backend{BackendsPath: tempDir}, - } + nvidiaSystemState := system.NewCapabilityState("nvidia", system.WithBackendPath(tempDir)) err = InstallBackendFromGallery(context.TODO(), []config.Gallery{gallery}, nvidiaSystemState, ml, "meta-backend", nil, true, false) Expect(err).NotTo(HaveOccurred()) @@ -785,11 +777,7 @@ var _ = Describe("Gallery Backends", func() { Expect(err).NotTo(HaveOccurred()) // Test with NVIDIA system state - nvidiaSystemState := &system.SystemState{ - GPUVendor: "nvidia", - VRAM: 1000000000000, - Backend: system.Backend{BackendsPath: tempDir}, - } + nvidiaSystemState := system.NewCapabilityState("nvidia", system.WithBackendPath(tempDir)) err = InstallBackendFromGallery(context.TODO(), []config.Gallery{gallery}, nvidiaSystemState, ml, "meta-backend", nil, true, false) Expect(err).NotTo(HaveOccurred()) diff --git a/core/gallery/gallery_test.go b/core/gallery/gallery_test.go index cb2070c1c6eb..1455f7e87360 100644 --- a/core/gallery/gallery_test.go +++ b/core/gallery/gallery_test.go @@ -510,7 +510,7 @@ var _ = Describe("Gallery", func() { }, }, } - result := FindGalleryElement(modelsWithPath, "bert/embeddings") + result := FindGalleryElement(modelsWithPath, "bert"+string(os.PathSeparator)+"embeddings") Expect(result).NotTo(BeNil()) Expect(result.GetName()).To(Equal("bert__embeddings")) }) diff --git a/core/gallery/model_artifacts_test.go b/core/gallery/model_artifacts_test.go index cd9d4cdd9866..bc31ddb5b095 100644 --- a/core/gallery/model_artifacts_test.go +++ b/core/gallery/model_artifacts_test.go @@ -67,7 +67,7 @@ unknown_extension: Expect(err).NotTo(HaveOccurred()) Expect(fake.seen).To(HaveLen(1)) Expect(installed.Model).To(Equal("owner/repo")) - Expect(installed.ModelFileName()).To(Equal(fake.result.RelativePath)) + Expect(installed.ModelFileName()).To(Equal(filepath.FromSlash(fake.result.RelativePath))) data, err := os.ReadFile(filepath.Join(modelsPath, "managed.yaml")) Expect(err).NotTo(HaveOccurred()) diff --git a/core/gallery/resolve_variant_test.go b/core/gallery/resolve_variant_test.go index c5bd8648111e..bf4cd58a736a 100644 --- a/core/gallery/resolve_variant_test.go +++ b/core/gallery/resolve_variant_test.go @@ -1056,6 +1056,9 @@ var _ = Describe("HostResolveEnv engine preference wiring", func() { if runtime.GOOS == "darwin" { Skip("darwin reports metal or darwin-x86 before the VRAM floor is consulted") } + if runtime.GOOS == "windows" { + Skip("windows reports its own capability before the VRAM floor is consulted") + } Expect(os.Unsetenv(capabilityEnv)).To(Succeed()) // A capability run file on the machine would override detection. Expect(os.Setenv(capabilityRunFileEnv, filepath.Join(GinkgoT().TempDir(), "absent"))).To(Succeed()) diff --git a/core/http/endpoints/localai/video_internal_test.go b/core/http/endpoints/localai/video_internal_test.go index 383ae5fe4115..848f945c9958 100644 --- a/core/http/endpoints/localai/video_internal_test.go +++ b/core/http/endpoints/localai/video_internal_test.go @@ -5,6 +5,7 @@ import ( "encoding/base64" "os" "path/filepath" + "runtime" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -23,7 +24,11 @@ var _ = Describe("video media staging", func() { Expect(os.ReadFile(path)).To(Equal(content)) info, err := os.Stat(path) Expect(err).NotTo(HaveOccurred()) - Expect(info.Mode().Perm()).To(Equal(os.FileMode(0o600))) + // POSIX permission bits are not enforceable on Windows, where the file + // is reported with the default 0666 mode regardless of chmod. + if runtime.GOOS != "windows" { + Expect(info.Mode().Perm()).To(Equal(os.FileMode(0o600))) + } }) It("accepts browser data URIs with codec parameters", func() { diff --git a/core/services/testutil/testdb.go b/core/services/testutil/testdb.go index 80e511201b7d..ebecc3c8bc79 100644 --- a/core/services/testutil/testdb.go +++ b/core/services/testutil/testdb.go @@ -22,6 +22,9 @@ func SetupTestDB() *gorm.DB { if runtime.GOOS == "darwin" { Skip("testcontainers requires Docker, not available on macOS CI") } + if runtime.GOOS == "windows" { + Skip("testcontainers requires Docker, not available on Windows CI") + } ctx := context.Background() pgC, err := tcpostgres.Run(ctx, "postgres:16", tcpostgres.WithDatabase("testdb"), diff --git a/docs/content/features/GPU-acceleration.md b/docs/content/features/GPU-acceleration.md index 94a97b8f56ad..fed97df3511c 100644 --- a/docs/content/features/GPU-acceleration.md +++ b/docs/content/features/GPU-acceleration.md @@ -17,6 +17,10 @@ For advanced use cases or to override auto-detection, you can use the `LOCALAI_F - `nvidia`: Forces backends compiled with CUDA support for NVIDIA GPUs. - `amd`: Forces backends compiled with ROCm support for AMD GPUs. - `intel`: Forces backends compiled with SYCL/oneAPI support for Intel GPUs. +- `vulkan`: Forces backends compiled with Vulkan support (e.g. the Windows llama-cpp backend, which auto-detects a Vulkan device and falls back to CPU — see [Windows]({{% relref "getting-started/windows" %}})). +- `metal`: Forces backends compiled with Metal support (macOS/Apple Silicon). +- `darwin-x86`: Forces the Intel-macOS build of a backend. +- `windows`: Forces the native Windows build of a backend (only `llama-cpp` ships a Windows build today). ## Model configuration @@ -379,6 +383,13 @@ On an integrated Intel GPU, the amount of free graphics memory can only be read If using nvidia, follow the steps in the [CUDA](#cudanvidia-acceleration) section to configure your docker runtime to allow access to the GPU. +### Native Windows + +The Windows binary ships the llama-cpp backend with Vulkan baked in: it +auto-detects a Vulkan device at runtime and offloads `gpu_layers` to it, +falling back to CPU when no Vulkan driver is present. No configuration or +driver passthrough is required — see [Windows]({{% relref "getting-started/windows" %}}). + ### Container images To use Vulkan, use the images with the `vulkan` tag, for example `{{< version >}}-gpu-vulkan`. diff --git a/docs/content/getting-started/build.md b/docs/content/getting-started/build.md index 77884a4126b8..7d9185af64d5 100644 --- a/docs/content/getting-started/build.md +++ b/docs/content/getting-started/build.md @@ -51,6 +51,17 @@ go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@1958fcbe2ca8bd93af633f1 ``` +{{% /tab %}} +{{% tab title="Windows" %}} + +The `make` recipes are POSIX shell. Git for Windows bundles the POSIX tools they need (`sh`, `uname`, `unzip`, `grep`, `awk`, `sed`) — but **not** `make` itself — so install a GNU make for Windows as well: + +```powershell +winget install ezwinports.make +``` + +The Makefile locates Git for Windows' `sh` on its own and runs every recipe through it, downloads `protoc` and installs the Go protobuf plugins itself — so no PATH edits or extra toolchain are needed. `make build` also compiles the React UI, so install [Node.js](https://nodejs.org) (with npm) as well. + {{% /tab %}} {{% tab title="From source" %}} @@ -72,6 +83,8 @@ make build This should produce the binary `local-ai` +On Windows, `make build` produces `local-ai.exe` instead. + #### Container image Requirements: @@ -143,6 +156,27 @@ make clean make build ``` +### Example: Build on Windows + +Building the server binary on Windows needs no C toolchain: the release config compiles with `CGO_ENABLED=0`. The `make` recipes are POSIX shell, so you need Git for Windows (which bundles `sh` and the POSIX tools) plus a GNU make for Windows — Git for Windows does not ship `make`. The Makefile finds `sh` on its own, so no PATH edits are required. + +```powershell +# Install Go (https://go.dev/dl), Git for Windows (https://git-scm.com/downloads), +# Node.js for the React UI build (https://nodejs.org), then a GNU make for +# Windows, e.g.: +winget install ezwinports.make + +# Build with CGO disabled like the release config. +$env:CGO_ENABLED = '0' + +git clone https://github.com/mudler/LocalAI.git +cd LocalAI + +make build +``` + +This produces the binary `local-ai.exe`. The `llama-cpp` backend is also distributed natively for Windows — see [Building the Windows backend]({{% relref "getting-started/windows" %}}#building-from-source) for the MSYS2 UCRT64 requirements and the `make backends/llama-cpp-windows` target. Other backends are still built as Linux container images and are not distributed for Windows. + ## Build backends LocalAI have several backends available for installation in the backend gallery. The backends can be also built by source. As backends might vary from language and dependencies that they require, the documentation will provide generic guidance for few of the backends, which can be applied with some slight modifications also to the others. diff --git a/docs/content/getting-started/install.md b/docs/content/getting-started/install.md index d9d76161c2f2..734c076e5f93 100644 --- a/docs/content/getting-started/install.md +++ b/docs/content/getting-started/install.md @@ -18,9 +18,10 @@ Choose the installation method that best suits your needs: 1. **[Containers]({{% relref "getting-started/containers" %}})** ⭐ **Recommended** - Works on all platforms, supports Docker and Podman 2. **[macOS]({{% relref "getting-started/macos" %}})** - Download and install the DMG application -3. **[Linux]({{% relref "getting-started/linux" %}})** - Install on Linux using binaries -4. **[Kubernetes]({{% relref "getting-started/kubernetes" %}})** - Deploy LocalAI on Kubernetes clusters -5. **[Build from Source]({{% relref "getting-started/build" %}})** - Build LocalAI from source code +3. **[Windows]({{% relref "getting-started/windows" %}})** - Run natively on Windows, no WSL or Docker required +4. **[Linux]({{% relref "getting-started/linux" %}})** - Install on Linux using binaries +5. **[Kubernetes]({{% relref "getting-started/kubernetes" %}})** - Deploy LocalAI on Kubernetes clusters +6. **[Build from Source]({{% relref "getting-started/build" %}})** - Build LocalAI from source code ## Quick Start @@ -38,6 +39,7 @@ This will start LocalAI. The API will be available at `http://localhost:8080`. For other platforms: - **macOS**: Download the [DMG]({{% relref "getting-started/macos" %}}) +- **Windows**: See the [Windows installation guide]({{% relref "getting-started/windows" %}}) for native, WSL-free installation. - **Linux**: See the [Linux installation guide]({{% relref "getting-started/linux" %}}) for binary installation. For detailed instructions, see the [Containers installation guide]({{% relref "getting-started/containers" %}}). diff --git a/docs/content/getting-started/models.md b/docs/content/getting-started/models.md index e78ac00a995b..ecd9b64822d0 100644 --- a/docs/content/getting-started/models.md +++ b/docs/content/getting-started/models.md @@ -286,7 +286,7 @@ curl http://localhost:8080/v1/completions -H "Content-Type: application/json" -d For other Docker images, please refer to the table in [Getting Started](https://localai.io/basics/getting_started/#container-images). {{% /notice %}} -Note: If you are on Windows, ensure the project is on the Linux filesystem to avoid slow model loading. For more information, see the [Microsoft Docs](https://learn.microsoft.com/en-us/windows/wsl/filesystems). +Note: When running LocalAI under Docker/WSL2 on Windows, ensure the project is on the Linux filesystem to avoid slow model loading. For more information, see the [Microsoft Docs](https://learn.microsoft.com/en-us/windows/wsl/filesystems). {{% /tab %}} {{% tab title="Kubernetes" %}} diff --git a/docs/content/getting-started/troubleshooting.md b/docs/content/getting-started/troubleshooting.md index 3e925125be4e..1330d829e1b4 100644 --- a/docs/content/getting-started/troubleshooting.md +++ b/docs/content/getting-started/troubleshooting.md @@ -51,7 +51,7 @@ docker run --rm --gpus all nvidia/cuda:12.8.0-base-ubuntu24.04 nvidia-smi ```bash chmod +x local-ai-* -./local-ai-Linux-x86_64 run +./local-ai--linux-amd64 run ``` If you see "cannot execute binary file: Exec format error", you downloaded the wrong architecture. Verify with: diff --git a/docs/content/getting-started/windows.md b/docs/content/getting-started/windows.md new file mode 100644 index 000000000000..b812bc8ee50f --- /dev/null +++ b/docs/content/getting-started/windows.md @@ -0,0 +1,170 @@ +--- +title: Windows +weight: 8 +description: > + Install and run LocalAI on Windows without WSL or Docker. Native + windows/amd64 backend support. +--- + +## Native Windows backends (no WSL / Docker required) + +LocalAI can run entirely natively on Windows using pre-built backend images +packaged as OCI tarballs. The llama-cpp backend is the first backend with +native Windows support; other backends will follow. + +## Prerequisites + +- Windows 10/11, **amd64** +- PowerShell 7+ (recommended) or cmd.exe +- No WSL, no Docker Desktop, no MSYS2 needed at runtime + +The `local-ai` launcher downloads the Windows backend image on first run and +executes it as a native process. + +## Install + +```powershell +# Download the Windows release asset `local-ai--windows-amd64.exe` +# from the releases page: https://github.com/mudler/LocalAI/releases + +# Or via winget (when available): +# winget install mudler.localai +``` + +## First run + +```powershell +# Download a model (for example, Llama 3.2 1B Instruct GGUF) +.\local-ai.exe model install huggingface://bartowski/Llama-3.2-1B-Instruct-GGUF:Q4_K_M + +# Start LocalAI +.\local-ai.exe run +``` + +LocalAI selects the best llama-cpp binary shipped in the Windows backend image +automatically. No configuration is required. + +## Backend selection on Windows + +Windows backend images ship three llama-cpp executables: + +- `llama-cpp-cpu-all.exe` — CPU-only build with all ggml CPU backends (preferred + when present). +- `llama-cpp-grpc.exe` — gRPC-RPC build, selected automatically when the + `LLAMACPP_GRPC_SERVERS` environment variable is set. +- `llama-cpp-fallback.exe` — static fallback used when neither of the above + applies. + +A small `run.ps1` launcher (Windows has no shell to run the `run.sh` stub +through, so `pkg/model/process.go` starts `run.ps1` via `powershell.exe`) +picks the right binary at startup. + +### GPU (Vulkan) support + +The Windows backend image ships `ggml-vulkan.dll` alongside the CPU backends. +The single `llama-cpp-cpu-all.exe` build auto-detects a Vulkan device at +startup and offloads `gpu_layers` to it when a model requests them, falling +back to CPU inference on hosts with no Vulkan driver. No configuration is +required — the Vulkan loader (`vulkan-1.dll`) is bundled with the image and +loaded dynamically, so the backend never hard-depends on it. + +## Environment variables + +| Variable | Effect | +|----------|--------| +| `LLAMACPP_GRPC_SERVERS` | Forces use of `llama-cpp-grpc.exe` when set. | + +## Troubleshooting + +### Missing Visual C++ runtime + +If the backend fails to start with a `DLL not found` error, install the +[Visual C++ Redistributable](https://learn.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist). + +### Antivirus / SmartScreen + +Windows Defender SmartScreen may warn about the unsigned `local-ai.exe` and +backend binaries. Click **More info** → **Run anyway** to proceed. Future +releases will be signed. + +### Firewall + +LocalAI listens on `http://localhost:8080` by default. Allow it through the +firewall on first run when prompted. + +## Building from source + +See `scripts/build/llama-cpp-windows.sh` and the Makefile target +`backends/llama-cpp-windows` if you want to rebuild the Windows backend image +locally. The target is re-runnable: an interrupted build can simply be run +again (the script re-arms its source clones, and the long gRPC build tree is +reused between runs). +The build requires an MSYS2 UCRT64 environment with mingw-w64 GCC, +CMake 4.x, Ninja, the Vulkan headers/loader and glslc (see below), and Go +(to build the `local-ai.exe` that assembles the OCI image tar; it must be +reachable via `GOROOT`, `GO_TOOLCHAIN_ROOT`, or `PATH`). + +### Setting up MSYS2 UCRT64 + +1. Download and install MSYS2 from https://www.msys2.org/ +2. Open the **MSYS2 UCRT64** terminal (not plain MSYS2) +3. Install the build toolchain: + +```bash +pacman -S --needed --noconfirm base-devel git cmake ninja mingw-w64-ucrt-x86_64-gcc mingw-w64-ucrt-x86_64-cmake mingw-w64-ucrt-x86_64-vulkan-headers mingw-w64-ucrt-x86_64-vulkan-loader mingw-w64-ucrt-x86_64-spirv-headers mingw-w64-x86_64-shaderc +``` + +The Vulkan packages are required: `GGML_VULKAN=ON` makes CMake's `FindVulkan` +treat `glslc` as a REQUIRED component, and ggml-vulkan runs +`find_package(SPIRV-Headers CONFIG REQUIRED)`. MSYS2 ships glslc only in the +mingw64 variant of `shaderc` (there is no ucrt64 build) — it is a standalone +tool, so the build script adds `/mingw64/bin` to `PATH` when no glslc is +already available (e.g. from a Vulkan SDK installed via the registry). The +script also installs `mingw-w64-ucrt-x86_64-spirv-headers` itself if its cmake +config is missing, so the docs' package list is only a head start. + +4. Run the build. From a plain Windows shell (PowerShell, `cmd`, or Git bash) + the script detects that it is not under MSYS2 and re-executes itself under + an MSYS2 UCRT64 login shell automatically — no need to open the UCRT64 + terminal yourself: + +```powershell +cd C:\source_path\LocalAI +make backends/llama-cpp-windows +``` + + Inside the MSYS2 UCRT64 terminal the script runs directly: + +```bash +cd /source_path/LocalAI +bash scripts/build/llama-cpp-windows.sh +``` + +The script packages the result as an OCI tarball (`backend-images/llama-cpp.tar`) +and — when run through the Makefile — installs it with +`./local-ai.exe backends install "ocifile://..."`. + +### GitHub Actions build + +CI builds the Windows backend in the reusable +[`.github/workflows/backend_build_windows.yml`](https://github.com/mudler/LocalAI/blob/master/.github/workflows/backend_build_windows.yml) +workflow, dispatched from `.github/workflows/backend.yml` for the `includeWindows` +matrix entry. Only `llama-cpp` has a Windows build path today — a future entry +dispatched without one fails loudly rather than upload an empty tarball. + +The pipeline: + +1. **Build job** (`windows-latest`): sets up MSYS2 UCRT64 with the same + package set as the local instructions above, caches ccache keyed on the + pinned `LLAMA_VERSION`, then runs `make backends/llama-cpp-windows` in an + MSYS2 shell (the Go toolchain from `actions/setup-go` is handed in via + `GO_TOOLCHAIN_ROOT`, since the MSYS2 shell resets `PATH`). The produced + `backend-images/llama-cpp.tar` is uploaded as a workflow artifact. +2. **Publish job** (`ubuntu-latest`, skipped on PRs): downloads the tarball, + logs into DockerHub and quay.io with `crane`, tags it with the standard + branch/semver/sha metadata (the `-windows-amd64-llama-cpp` tag suffix from + the matrix), + and pushes the OCI image to both registries. + +The gallery entry (with its `verification:` block) points at the published +image, so `local-ai backends install` fetches the exact artifact CI produced. diff --git a/docs/content/reference/binaries.md b/docs/content/reference/binaries.md index 178f311277c2..18b2677b62c1 100644 --- a/docs/content/reference/binaries.md +++ b/docs/content/reference/binaries.md @@ -5,7 +5,7 @@ title = "LocalAI binaries" weight = 26 +++ -LocalAI binaries are available for both Linux and MacOS platforms and can be executed directly from your command line. These binaries are continuously updated and hosted on [our GitHub Releases page](https://github.com/mudler/LocalAI/releases). This method also supports Windows users via the Windows Subsystem for Linux (WSL). +LocalAI binaries are available for Linux, macOS, and Windows platforms and can be executed directly from your command line. These binaries are continuously updated and hosted on [our GitHub Releases page](https://github.com/mudler/LocalAI/releases). ### macOS Download @@ -17,19 +17,22 @@ You can download the DMG and install the application: > Note: the DMGs are not signed by Apple as quarantined. See https://github.com/mudler/LocalAI/issues/6268 for a workaround, fix is tracked here: https://github.com/mudler/LocalAI/issues/6244 -Otherwise, use the following one-liner command in your terminal to download and run LocalAI on Linux or MacOS: +Otherwise, use the following one-liner command in your terminal to download and run LocalAI on Linux or MacOS (set `VERSION` to the current tag, e.g. `v4.8.2`): ```bash -curl -Lo local-ai "https://github.com/mudler/LocalAI/releases/download/{{< version >}}/local-ai-$(uname -s)-$(uname -m)" && chmod +x local-ai && ./local-ai +VERSION=v4.8.2; ARCH=$(uname -m | sed 's/x86_64/amd64/; s/aarch64/arm64/'); \ +curl -Lo local-ai "https://github.com/mudler/LocalAI/releases/download/$VERSION/local-ai-$VERSION-$(uname -s | tr '[:upper:]' '[:lower:]')-$ARCH" \ + && chmod +x local-ai && ./local-ai ``` Otherwise, here are the links to the binaries: | OS | Link | | --- | --- | -| Linux (amd64) | [Download](https://github.com/mudler/LocalAI/releases/download/{{< version >}}/local-ai-Linux-x86_64) | -| Linux (arm64) | [Download](https://github.com/mudler/LocalAI/releases/download/{{< version >}}/local-ai-Linux-arm64) | -| MacOS (arm64) | [Download](https://github.com/mudler/LocalAI/releases/download/{{< version >}}/local-ai-Darwin-arm64) | +| Linux (amd64) | [Download](https://github.com/mudler/LocalAI/releases/download/{{< version >}}/local-ai-{{< version >}}-linux-amd64) | +| Linux (arm64) | [Download](https://github.com/mudler/LocalAI/releases/download/{{< version >}}/local-ai-{{< version >}}-linux-arm64) | +| MacOS (arm64) | [Download](https://github.com/mudler/LocalAI/releases/download/{{< version >}}/local-ai-{{< version >}}-darwin-arm64) | +| Windows (amd64) | [Download](https://github.com/mudler/LocalAI/releases/download/{{< version >}}/local-ai-{{< version >}}-windows-amd64.exe) | {{% notice icon="⚡" context="warning" %}} @@ -38,4 +41,5 @@ Binaries do have limited support compared to container images: - Python-based backends are not shipped with binaries (e.g. `diffusers` or `transformers`) - MacOS binaries and Linux-arm64 do not ship TTS nor `stablediffusion-cpp` backends - Linux binaries do not ship `stablediffusion-cpp` backend +- The Windows binary ships only the `llama-cpp` backend (native, no WSL required); see the [Windows guide]({{% relref "getting-started/windows" %}}) for details {{% /notice %}} diff --git a/pkg/downloader/auth_progress_test.go b/pkg/downloader/auth_progress_test.go index debf45f9e79d..622e652720e5 100644 --- a/pkg/downloader/auth_progress_test.go +++ b/pkg/downloader/auth_progress_test.go @@ -7,6 +7,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "runtime" "sync" . "github.com/onsi/ginkgo/v2" @@ -141,7 +142,11 @@ var _ = Describe("authenticated HTTP downloads", func() { Expect(errors.Is(err, context.Canceled)).To(BeTrue()) info, statErr := os.Stat(target + ".partial") Expect(statErr).NotTo(HaveOccurred()) - Expect(info.Mode().Perm()).To(Equal(os.FileMode(0o600))) + // POSIX permission bits are not enforceable on Windows, where the file + // is reported with the default 0666 mode regardless of chmod. + if runtime.GOOS != "windows" { + Expect(info.Mode().Perm()).To(Equal(os.FileMode(0o600))) + } }) It("keeps the legacy total empty when the response length is unknown", func() { diff --git a/pkg/downloader/partial_resume_test.go b/pkg/downloader/partial_resume_test.go index 9cf953dee2ef..42e213011477 100644 --- a/pkg/downloader/partial_resume_test.go +++ b/pkg/downloader/partial_resume_test.go @@ -8,6 +8,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "runtime" "strconv" "strings" "sync" @@ -157,6 +158,12 @@ var _ = Describe("DownloadFile with a leftover .partial", func() { }) It("reports an informative error when the partial cannot be stat'd", func() { + if runtime.GOOS == "windows" { + // A self-referencing symlink is the portable trick that makes + // os.Stat fail with ELOOP, but creating symlinks on Windows needs + // admin rights or Developer Mode, so the setup itself errors first. + Skip("symlink creation requires privileges on Windows") + } server := rangeServer(true, nil, nil) defer server.Close() diff --git a/pkg/downloader/uri.go b/pkg/downloader/uri.go index 0239781304fb..425244b75eeb 100644 --- a/pkg/downloader/uri.go +++ b/pkg/downloader/uri.go @@ -756,6 +756,10 @@ func (uri URI) DownloadFileWithContext(ctx context.Context, filePath, sha string if startPos > 0 && resp.StatusCode != http.StatusPartialContent { _ = resp.Body.Close() + // The origin ignored the resume range, so the partial is garbage; + // a retry must start clean. Drop the write handle first — Windows + // cannot delete a file that is still open (no FILE_SHARE_DELETE). + _ = outFile.Close() _ = removePartialFile(tmpFilePath) // The partial has just been discarded, so a further attempt starts // clean and no longer needs the server to honour the range. @@ -805,6 +809,13 @@ func (uri URI) DownloadFileWithContext(ctx context.Context, filePath, sha string // after filesystem permissions while the disk was perfectly healthy. tracked := &readErrorRecorder{r: source} _, err = xio.Copy(ctx, io.MultiWriter(outFile, progress), tracked) + // Windows cannot rename or remove a file that still has an open handle: + // Go opens files without FILE_SHARE_DELETE, so MoveFileEx / DeleteFile fail + // with a sharing violation while outFile is live. The copy is the last use + // of the handle, so close it before any of the removal/rename paths below — + // the error paths (non-206 resume, user cancel, SHA mismatch) remove the + // partial too, and POSIX unlink works on open files while Windows does not. + _ = outFile.Close() if err != nil { // Detect cancellation via the context (a cause-cancelled read surfaces // the cause, not context.Canceled). Keep the .partial for resume, diff --git a/pkg/downloader/uri_test.go b/pkg/downloader/uri_test.go index 9cb667b57864..d4e9c28a90c9 100644 --- a/pkg/downloader/uri_test.go +++ b/pkg/downloader/uri_test.go @@ -281,12 +281,15 @@ var _ = Describe("Download Test", func() { mockServer := getMockServer(true) defer mockServer.Close() uri := URI(mockServer.URL) - // Create a partial file + // Create a partial file. The handle must be released before the + // download runs: DownloadFile renames the partial over the target, + // and Windows cannot rename a file with an open handle. tmpFilePath := filePath + ".partial" file, err := os.OpenFile(tmpFilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) Expect(err).ToNot(HaveOccurred()) _, err = file.Write(mockData[0:10000]) Expect(err).ToNot(HaveOccurred()) + Expect(file.Close()).To(Succeed()) err = uri.DownloadFile(filePath, mockDataSha, 1, 1, func(s1, s2, s3 string, f float64) {}) Expect(err).ToNot(HaveOccurred()) }) @@ -295,12 +298,15 @@ var _ = Describe("Download Test", func() { mockServer := getMockServer(false) defer mockServer.Close() uri := URI(mockServer.URL) - // Create a partial file + // Create a partial file. The handle must be released before the + // download runs: DownloadFile removes the partial and Windows cannot + // delete a file with an open handle. tmpFilePath := filePath + ".partial" file, err := os.OpenFile(tmpFilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) Expect(err).ToNot(HaveOccurred()) _, err = file.Write(mockData[0:10000]) Expect(err).ToNot(HaveOccurred()) + Expect(file.Close()).To(Succeed()) err = uri.DownloadFile(filePath, mockDataSha, 1, 1, func(s1, s2, s3 string, f float64) {}) Expect(err).ToNot(HaveOccurred()) }) diff --git a/pkg/model/loader_test.go b/pkg/model/loader_test.go index 1a882943127f..0182ad23d1ef 100644 --- a/pkg/model/loader_test.go +++ b/pkg/model/loader_test.go @@ -65,7 +65,7 @@ var _ = Describe("ModelLoader", func() { BeforeEach(func() { // Setup the model loader with a test directory - modelPath = "/tmp/test_model_path" + modelPath = filepath.Join(os.TempDir(), "test_model_path") os.Mkdir(modelPath, 0755) systemState, err := system.GetSystemState( diff --git a/pkg/model/process.go b/pkg/model/process.go index f6533e57827a..e065f9b60d6c 100644 --- a/pkg/model/process.go +++ b/pkg/model/process.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path/filepath" + "runtime" "strconv" "strings" "time" @@ -252,6 +253,24 @@ func (ml *ModelLoader) startProcess(grpcProcess, id string, serverAddress string return nil, err } + // A Windows host has no shell to execute the run.sh stub the gallery + // contract names, so the image also ships run.ps1 and os.StartProcess runs + // it through the bundled Windows PowerShell. run.sh stays in the image so + // discovery, validation and upgrades stay uniform with the other platforms. + processName := filepath.Base(grpcProcess) + processArgs := args + if runtime.GOOS == "windows" { + if _, err := os.Stat(filepath.Join(workDir, "run.ps1")); err == nil { + grpcProcess = filepath.Join(workDir, "run.ps1") + processName = "powershell.exe" + processArgs = append([]string{ + "-NoProfile", + "-ExecutionPolicy", "Bypass", + "-File", grpcProcess, + }, args...) + } + } + env := os.Environ() // Vulkan backends are self-contained: they bundle their own loader and // Mesa driver .so files in lib/ plus the matching ICD manifests in @@ -263,8 +282,8 @@ func (ml *ModelLoader) startProcess(grpcProcess, id string, serverAddress string grpcControlProcess := process.New( process.WithTemporaryStateDir(), - process.WithName(filepath.Base(grpcProcess)), - process.WithArgs(append(args, []string{"--addr", serverAddress}...)...), + process.WithName(processName), + process.WithArgs(append(processArgs, []string{"--addr", serverAddress}...)...), process.WithEnvironment(env...), process.WithWorkDir(workDir), ) diff --git a/pkg/system/capabilities.go b/pkg/system/capabilities.go index ffe9a4c96594..16ffcca7ddba 100644 --- a/pkg/system/capabilities.go +++ b/pkg/system/capabilities.go @@ -25,6 +25,7 @@ const ( darwinX86 = "darwin-x86" metal = "metal" vulkan = "vulkan" + windows = "windows" nvidiaCuda13 = "nvidia-cuda-13" nvidiaCuda12 = "nvidia-cuda-12" @@ -36,15 +37,16 @@ const ( defaultRunFile = "/run/localai/capability" // Backend detection tokens (private) - backendTokenDarwin = "darwin" - backendTokenMLX = "mlx" - backendTokenMetal = "metal" - backendTokenL4T = "l4t" - backendTokenCUDA = "cuda" - backendTokenROCM = "rocm" - backendTokenHIP = "hip" - backendTokenSYCL = "sycl" - backendTokenCPU = "cpu" + backendTokenDarwin = "darwin" + backendTokenMLX = "mlx" + backendTokenMetal = "metal" + backendTokenL4T = "l4t" + backendTokenCUDA = "cuda" + backendTokenROCM = "rocm" + backendTokenHIP = "hip" + backendTokenSYCL = "sycl" + backendTokenCPU = "cpu" + backendTokenWindows = "windows" // Engine names (private). Unlike the tokens above these are whole backend // identities as a gallery entry's `backend:` field spells them, not build @@ -117,6 +119,9 @@ var backendBuildTagPreferenceRules = []backendPreferenceRule{ {metal, []string{backendTokenMetal, backendTokenCPU}}, {darwinX86, []string{darwinX86, backendTokenCPU}}, {vulkan, []string{vulkan, backendTokenCPU}}, + // Windows ships native windows/amd64 backend builds; prefer them over the + // Linux CPU builds, which cannot run on a Windows host. + {windows, []string{backendTokenWindows, backendTokenCPU}}, } // defaultBackendBuildTagTokens is what a host with no matching rule prefers. @@ -184,6 +189,11 @@ var engineNamePreferenceRules = []backendPreferenceRule{ // IsBackendCompatible admits any darwin-tokened engine on this capability // and will not drop it. {darwinX86, []string{engineLlamaCpp, engineVLLM, engineSGLang}}, + // A Windows host runs exactly the native llama.cpp build; vLLM and SGLang + // do not publish windows/amd64 images, so ranking them above llama.cpp + // (or even equal to it) could auto-select an engine a Windows host cannot + // run whenever a variant set is offered. + {windows, []string{engineLlamaCpp}}, } // defaultEnginePreferenceTokens is empty on purpose. It is now only reached by @@ -375,6 +385,15 @@ func (s *SystemState) getSystemCapabilities() string { return s.systemCapabilities } + // Windows backends are published as native windows/amd64 OCI images, so a + // Windows host gets its own capability to select them deterministically + // instead of falling through to "default" and racing the Linux CPU builds. + if runtime.GOOS == "windows" { + xlog.Info("Using windows capability", "env", capabilityEnv) + s.systemCapabilities = windows + return s.systemCapabilities + } + // If arm64 on linux and a nvidia gpu is detected, we will return nvidia-l4t if runtime.GOOS == "linux" && runtime.GOARCH == "arm64" { if s.GPUVendor == Nvidia { @@ -496,6 +515,11 @@ func (s *SystemState) IsBackendCompatible(name, uri string) bool { return capability == metal || capability == darwinX86 } + // Check for Windows-specific backends (native windows/amd64 builds) + if strings.Contains(combined, backendTokenWindows) { + return capability == windows + } + // Check for NVIDIA L4T-specific backends (arm64 Linux with NVIDIA GPU) // This must be checked before the general NVIDIA check as L4T backends // may also contain "cuda" or "nvidia" in their names diff --git a/pkg/system/capabilities_test.go b/pkg/system/capabilities_test.go index 0844e52f44a5..523c29dd5f82 100644 --- a/pkg/system/capabilities_test.go +++ b/pkg/system/capabilities_test.go @@ -19,8 +19,8 @@ var _ = Describe("getSystemCapabilities", func() { ) BeforeEach(func() { - if runtime.GOOS == "darwin" { - Skip("darwin short-circuits before reaching CUDA logic") + if runtime.GOOS == "darwin" || runtime.GOOS == "windows" { + Skip("platform short-circuits before reaching CUDA logic") } origEnv = os.Getenv(capabilityEnv) @@ -168,6 +168,7 @@ var _ = Describe("BackendPreferenceTokens", func() { Entry("metal", metal, []string{"metal", "cpu"}), Entry("darwin-x86", darwinX86, []string{"darwin-x86", "cpu"}), Entry("vulkan", vulkan, []string{"vulkan", "cpu"}), + Entry("windows", windows, []string{"windows", "cpu"}), Entry("unknown capability", "some-future-accelerator", []string{"cpu"}), ) @@ -175,7 +176,7 @@ var _ = Describe("BackendPreferenceTokens", func() { // The generic form of the lock above: whatever anyone adds to the build // tag table later, an engine name in it is a merged vocabulary. engineNames := []string{"vllm", "sglang", "llama-cpp", "mlx"} - for _, capability := range []string{"nvidia", AMD, Intel, metal, darwinX86, vulkan, "default"} { + for _, capability := range []string{"nvidia", AMD, Intel, metal, darwinX86, vulkan, windows, "default"} { Expect(tokensFor(capability)).ToNot(ContainElements(engineNames), "capability %q leaked an engine name into the build tag table", capability) } @@ -235,8 +236,8 @@ var _ = Describe("EnginePreferenceTokens", func() { It("names only engines, never a build tag", func() { // The mirror of the lock on BackendPreferenceTokens. A build tag here // matches no gallery `backend:` value and silently disables ranking. - buildTags := []string{"cuda", "rocm", "hip", "sycl", "vulkan", "metal", "cpu", "darwin-x86"} - for _, capability := range []string{"nvidia", AMD, Intel, metal, vulkan, defaultCapability, darwinX86} { + buildTags := []string{"cuda", "rocm", "hip", "sycl", "vulkan", "metal", "cpu", "darwin-x86", "windows"} + for _, capability := range []string{"nvidia", AMD, Intel, metal, vulkan, defaultCapability, darwinX86, windows} { Expect(tokensFor(capability)).ToNot(ContainElements(buildTags), "capability %q leaked a build tag into the engine table", capability) } @@ -253,6 +254,10 @@ var _ = Describe("EnginePreferenceTokens", func() { Expect(tokensFor(darwinX86)).To(Equal([]string{"llama-cpp", "vllm", "sglang"})) }) + It("prefers llama.cpp on windows, the only engine with native windows builds", func() { + Expect(tokensFor(windows)).To(Equal([]string{"llama-cpp"})) + }) + It("ranks the GPU serving engines below llama.cpp rather than leaving them tied", func() { // Enumerating them is what stops download size deciding between vLLM and // SGLang once llama.cpp is out of the running. diff --git a/scripts/build/llama-cpp-windows.sh b/scripts/build/llama-cpp-windows.sh new file mode 100644 index 000000000000..4cad1f2952ea --- /dev/null +++ b/scripts/build/llama-cpp-windows.sh @@ -0,0 +1,793 @@ +#!/bin/bash + +# Builds the native windows/amd64 llama-cpp backend image on a windows-latest +# GitHub runner under MSYS2 (UCRT64). Driven from the top-level Makefile via: +# +# make backends/llama-cpp-windows +# +# Unlike Linux (prebuilt base-grpc images) and Darwin (Homebrew gRPC), a +# Windows runner has neither, so gRPC is compiled from source into +# backend/cpp/grpc/installed_packages -- the exact layout the llama-cpp +# Makefile's BUILD_GRPC_FOR_BACKEND_LLAMA path expects, so the flags it would +# pass to cmake (absl_DIR / Protobuf_DIR / utf8_range_DIR / gRPC_DIR) resolve +# to real directories. +# +# This script drives cmake directly instead of the llama-cpp Makefile's +# llama-cpp-cpu-all / llama-cpp-grpc / llama-cpp-fallback targets: those copy +# binaries around as bare `grpc-server` paths, which MSYS2's cp cannot resolve +# to grpc-server.exe. The cmake flags below mirror exactly what those targets +# would pass (see backend/cpp/llama-cpp/Makefile). +# +# The build host also has no docker daemon, so the result is packaged as an OCI +# tar (--platform windows/amd64) that LocalAI's `backends install` extracts and +# runs as a native process -- see pkg/model/process.go (run.ps1) and +# backend/cpp/llama-cpp/run.ps1. + +set -ex + +# --------------------------------------------------------------------------- +# Auto-dispatch into MSYS2 (UCRT64). +# +# `make backends/llama-cpp-windows` runs this script through the recipe shell. +# GNU make for Windows falls back to cmd.exe unless it finds sh; the Makefile's +# OS-detection block (top of file) then points SHELL at Git for Windows' sh, so +# a local run normally starts under a MINGW64 bash. That bash cannot resolve +# /ucrt64/bin (the path maps to C:\Program Files\Git\ucrt64, which does not +# exist) and nothing below can build without it. Re-exec under the real MSYS2 +# bash instead. A non-login invocation deliberately keeps the inherited working +# directory (the repo root) and the Windows PATH (Go, Git); `bash -l` would +# reset PATH and cd to $HOME. MSYSTEM is not a usable discriminator here (Git +# bash reports MINGW64 too) - the existence of /ucrt64/bin under MSYS2's own +# runtime is the discriminator. +if [ ! -d /ucrt64/bin ]; then + if [ -x /c/msys64/usr/bin/bash.exe ]; then + echo "==> not running under MSYS2; re-exec'ing under C:/msys64 ..." >&2 + SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) + REPO_ROOT=$(cd "$SCRIPT_DIR/../.." && pwd) + # Run the script by name, not `exec bash "$0"`: a bare `bash` here would + # resolve to Git for Windows' bash via the inherited PATH (re-dispatch loop); + # the shebang's /bin/bash is an absolute MSYS2 path and lands on the real + # MSYS2 bash, where the /ucrt64/bin check below takes the direct branch. + # + # Mirror msys2_shell.cmd -ucrt64 -use-full-path: a LOGIN bash with + # MSYSTEM=UCRT64 + MSYS2_PATH_TYPE=inherit. This is the one configuration in + # which MSYS2's own git works when launched from a foreign shell (a plain + # non-login `bash.exe -c` inherits the Git-for-Windows mount table and its + # https remote helper dies with "fatal: remote helper 'https' aborted + # session"). `bash -l` resets PATH to MSYS2 defaults and drops `go` (it is + # on the Windows PATH but GOROOT is unset), so re-add the bin dir that the + # invoking shell resolved before the profile runs. + GO_DIR=$(command -v go 2>/dev/null) + [ -n "$GO_DIR" ] && GO_DIR=$(dirname "$GO_DIR") + # The login profile rebuilds the environment from a whitelist and drops + # Windows vars the MSYS2 runtime does not know (USERPROFILE etc.), so + # carry the ones go/ccache need over from the invoking shell. + exec env MSYSTEM=UCRT64 MSYS2_PATH_TYPE=inherit /c/msys64/usr/bin/bash.exe -l -c \ + "export PATH='$GO_DIR':\$PATH; export USERPROFILE='$USERPROFILE'; cd '$REPO_ROOT' && exec ./scripts/build/llama-cpp-windows.sh" + else + echo "ERROR: this script must run under MSYS2 UCRT64 (C:\\msys64)." >&2 + echo " Install MSYS2 (winget install MSYS2.MSYS2) and retry 'make backends/llama-cpp-windows'." >&2 + exit 1 + fi +fi + +ROOT=$(pwd) +IMAGE_NAME="${IMAGE_NAME:-localai/llama-cpp-windows}" +PLATFORMARCH="${PLATFORMARCH:-windows/amd64}" + +# nproc exists on MSYS2; fall back like the llama-cpp Makefile does otherwise. +# Honor an explicit JOBS override (e.g. `JOBS=4 make backends/llama-cpp-windows`): +# on memory-limited Windows hosts -j$(nproc) can spike past available RAM and +# several parallel g++ processes get killed mid-compile with no error text +# (silent "Error 1" on unrelated .obj targets). +JOBS="${JOBS:-$(nproc 2>/dev/null || echo 1)}" +export JOBS + +# Resolve cmake to the native UCRT64 build (/ucrt64/bin/cmake), not MSYS2's +# /usr/bin/cmake. The MSYS/Cygwin cmake reports UNIX=TRUE, so gRPC's +# `if(UNIX)` block in CMakeLists.txt appends ${CMAKE_DL_LIBS} m rt to every +# target's link line and every gRPC executable dies with "cannot find -ldl / +# -lrt" (no such libraries on Windows). The native cmake also sets MINGW +# (gRPC then defines -D_WIN32_WINNT=0x600 itself) and keeps plain exe names +# (the MSYS cmake emits protoc-24.3.0.exe). Everything else the script needs +# (make, git, perl, curl, unzip) lives in /usr/bin, so only cmake's resolution +# changes. +export PATH="/ucrt64/bin:$PATH" + +# The MSYS2 runtime does not export PROCESSOR_ARCHITECTURE, so cmake's Windows +# host detection yields an empty CMAKE_SYSTEM_PROCESSOR and ggml treats the +# machine as UNKNOWN (GGML_CPU_ALL_VARIANTS then refuses to configure). This +# backend is x86_64-only. +export PROCESSOR_ARCHITECTURE=AMD64 + +# go.exe resolves GOPATH (default %USERPROFILE%\go) and its build cache +# (%LocalAppData%\go-build) through Windows env vars the login profile drops; +# ccache needs a user-profile dir too. Fall back to the MSYS2 home for direct +# launches that forgot to set it. +export USERPROFILE="${USERPROFILE:-$(cygpath -w "$HOME")}" +export GOCACHE="${GOCACHE:-$(cygpath -w "$HOME")/go-build}" +mkdir -p "$GOCACHE" + +# ggml-vulkan compiles its shaders at build time with glslc, and CMake's +# FindVulkan declares it a REQUIRED component. MSYS2 only ships glslc in the +# mingw64 variant of shaderc (there is no ucrt64 build); it is a standalone +# tool, so its prefix can sit next to the UCRT64 one. When a Vulkan SDK +# (VULKAN_SDK, found by FindVulkan through the registry on Windows) already +# provides glslc, leave PATH alone. +# +# The mingw64 dir must be APPENDED, never prepended: the UCRT64-built tools +# (protoc, gcc, ...) resolve their runtime DLLs through PATH, and a mingw64 +# libstdc++/libgcc_s_seh placed ahead of /ucrt64/bin makes them load the wrong +# ABI runtime and die silently (protoc exits 127 mid-build). +if ! command -v glslc >/dev/null 2>&1 && [ -x /mingw64/bin/glslc.exe ]; then + export PATH="$PATH:/mingw64/bin" +fi + +# ggml-vulkan runs find_package(SPIRV-Headers CONFIG REQUIRED) at configure +# time; MSYS2 ships the config as mingw-w64-ucrt-x86_64-spirv-headers. +if [ ! -f /ucrt64/share/cmake/SPIRV-Headers/SPIRV-HeadersConfig.cmake ]; then + echo "==> SPIRV-Headers cmake config missing, installing mingw-w64-ucrt-x86_64-spirv-headers" + pacman -S --noconfirm mingw-w64-ucrt-x86_64-spirv-headers +fi + +# actions/setup-go adds go to the runner PATH but only exports GOROOT for Go +# < 1.9; the msys2 shell resets PATH, so `go` is not reachable here. The +# workflow resolves the install dir (GO_TOOLCHAIN_ROOT) with the default shell +# and hands it over; GOROOT is the fallback for standalone runs. Re-add the +# bin dir (translated to an msys path) unless a go binary already resolves. +GO_BIN_DIR="${GO_TOOLCHAIN_ROOT:-$GOROOT}" +if ! command -v go >/dev/null 2>&1 && [ -n "$GO_BIN_DIR" ]; then + GO_BIN_DIR="$GO_BIN_DIR/bin" + case "$GO_BIN_DIR" in + [A-Za-z]:*) GO_BIN_DIR=$(cygpath -u "$GO_BIN_DIR") ;; + esac + export PATH="$GO_BIN_DIR:$PATH" +fi + +# --------------------------------------------------------------------------- +# gRPC from source (backend/cpp/grpc/installed_packages) +# --------------------------------------------------------------------------- +GRPC_DIR="$ROOT/backend/cpp/grpc" +mkdir -p "$GRPC_DIR/grpc_repo/grpc" "$GRPC_DIR/grpc_build" +cd "$GRPC_DIR/grpc_repo/grpc" +git init -q +# Re-runnable: a previous (possibly failed) run may already have wired origin. +git remote remove origin 2>/dev/null || true +git remote add origin https://github.com/grpc/grpc.git +git fetch origin refs/tags/v1.59.0:refs/tags/v1.59.0 --depth 1 +git checkout -q v1.59.0 +git submodule update --init --recursive --depth 1 --single-branch + +# abseil's cctz win32 local-time-zone path (USE_WIN32_LOCAL_TIME_ZONE) uses +# WindowsCreateStringReference / WindowsDeleteString / WindowsGetStringRawBuffer +# from , which the MSVC SDK pulls in transitively but mingw-w64's +# roapi.h/windows.globalization.h do not. Patch the one TU to include it +# explicitly; without this the whole gRPC build dies at time_zone_lookup.cc. +# (\R matches either LF or CRLF since git may have normalized line endings.) +perl -0pi -e 's/(#include \R#include \R)(#endif)/$1#include \n$2/' \ + third_party/abseil-cpp/absl/time/internal/cctz/src/time_zone_lookup.cc + +# c-ares (bundled with gRPC 1.59.0) has a broken CMake build under mingw-w64 +# UCRT64 + CMake 4.x. Its CMakeLists uses CHECK_INCLUDE_FILES to detect +# winsock2.h / ws2tcpip.h / windows.h on WIN32, but the probes run BEFORE +# CMAKE_REQUIRED_DEFINITIONS is set (so without WIN32_LEAN_AND_MEAN), causing +# windows.h to pull in winsock.h which conflicts with winsock2.h. When the +# checks fail, CMAKE_EXTRA_INCLUDE_FILES stays empty, so the downstream +# CARES_TYPE_EXISTS / CHECK_SYMBOL_EXISTS probes for SOCKET, struct +# sockaddr_in6, struct addrinfo, recv, send etc. all fail too. That leaves +# the generated ares_config.h without HAVE_RECV / HAVE_SEND / +# HAVE_STRUCT_SOCKADDR_IN6 / HAVE_STRUCT_ADDRINFO / RECV_TYPE_* / SEND_TYPE_*, +# which trips setup_once.h's compile-time Error guards and makes ares_ipv6.h +# redefine structs already in ws2tcpip.h. +# +# Pre-seeding the cache vars via -D didn't work: CMake 4.x's CHECK_INCLUDE_FILES +# still runs and overwrites them. Instead, patch the ares_config.h.cmake +# template to append Windows-specific forced defines at the end. This way, +# regardless of what CMake detection produces, the generated ares_config.h +# always has the correct Windows values. +cat >> third_party/cares/cares/src/lib/ares_config.h.cmake <<'CARES_WIN32_FIX' + +/* ---- LocalAI mingw-w64 build fix ---- + * The CMake detection in this old c-ares (pinned by gRPC 1.59.0) doesn't + * reliably detect Windows socket types and functions under mingw-w64's + * UCRT64 headers + CMake 4.x. Force the known-correct values on _WIN32 + * so the build doesn't trip setup_once.h's Error guards or ares_ipv6.h's + * struct redefinitions. Values mirror config-win32.h (the hand-crafted + * config c-ares uses for non-CMake Windows builds). */ +#ifdef _WIN32 +#ifndef HAVE_WINDOWS_H +#define HAVE_WINDOWS_H 1 +#endif +#ifndef HAVE_WINSOCK_H +#define HAVE_WINSOCK_H 1 +#endif +#ifndef HAVE_WINSOCK2_H +#define HAVE_WINSOCK2_H 1 +#endif +#ifndef HAVE_WS2TCPIP_H +#define HAVE_WS2TCPIP_H 1 +#endif +#ifndef HAVE_ASSERT_H +#define HAVE_ASSERT_H 1 +#endif +#ifndef HAVE_ERRNO_H +#define HAVE_ERRNO_H 1 +#endif +#ifndef HAVE_GETOPT_H +#define HAVE_GETOPT_H 1 +#endif +#ifndef HAVE_LIMITS_H +#define HAVE_LIMITS_H 1 +#endif +#ifndef HAVE_PROCESS_H +#define HAVE_PROCESS_H 1 +#endif +#ifndef HAVE_SIGNAL_H +#define HAVE_SIGNAL_H 1 +#endif +#ifndef HAVE_TIME_H +#define HAVE_TIME_H 1 +#endif +#ifndef HAVE_UNISTD_H +#define HAVE_UNISTD_H 1 +#endif +#ifndef HAVE_STDBOOL_H +#define HAVE_STDBOOL_H 1 +#endif +#ifndef HAVE_STDINT_H +#define HAVE_STDINT_H 1 +#endif +#ifndef HAVE_STDLIB_H +#define HAVE_STDLIB_H 1 +#endif +#ifndef HAVE_STRING_H +#define HAVE_STRING_H 1 +#endif +#ifndef HAVE_SOCKLEN_T +#define HAVE_SOCKLEN_T 1 +#endif +#ifndef HAVE_TYPE_SOCKET +#define HAVE_TYPE_SOCKET 1 +#endif +#ifndef HAVE_BOOL_T +#define HAVE_BOOL_T 1 +#endif +#ifndef HAVE_SSIZE_T +#define HAVE_SSIZE_T 1 +#endif +#ifndef HAVE_LONGLONG +#define HAVE_LONGLONG 1 +#endif +#ifndef HAVE_SIG_ATOMIC_T +#define HAVE_SIG_ATOMIC_T 1 +#endif +#ifndef HAVE_STRUCT_ADDRINFO +#define HAVE_STRUCT_ADDRINFO 1 +#endif +#ifndef HAVE_STRUCT_IN6_ADDR +#define HAVE_STRUCT_IN6_ADDR 1 +#endif +#ifndef HAVE_STRUCT_SOCKADDR_IN6 +#define HAVE_STRUCT_SOCKADDR_IN6 1 +#endif +#ifndef HAVE_STRUCT_SOCKADDR_STORAGE +#define HAVE_STRUCT_SOCKADDR_STORAGE 1 +#endif +#ifndef HAVE_STRUCT_TIMEVAL +#define HAVE_STRUCT_TIMEVAL 1 +#endif +#ifndef HAVE_AF_INET6 +#define HAVE_AF_INET6 1 +#endif +#ifndef HAVE_PF_INET6 +#define HAVE_PF_INET6 1 +#endif +#ifndef HAVE_FIONBIO +#define HAVE_FIONBIO 1 +#endif +#ifndef HAVE_CLOSESOCKET +#define HAVE_CLOSESOCKET 1 +#endif +#ifndef HAVE_CONNECT +#define HAVE_CONNECT 1 +#endif +#ifndef HAVE_FREEADDRINFO +#define HAVE_FREEADDRINFO 1 +#endif +#ifndef HAVE_GETADDRINFO +#define HAVE_GETADDRINFO 1 +#endif +#ifndef HAVE_GETHOSTBYADDR +#define HAVE_GETHOSTBYADDR 1 +#endif +#ifndef HAVE_GETHOSTBYNAME +#define HAVE_GETHOSTBYNAME 1 +#endif +#ifndef HAVE_GETHOSTNAME +#define HAVE_GETHOSTNAME 1 +#endif +#ifndef HAVE_GETNAMEINFO +#define HAVE_GETNAMEINFO 1 +#endif +#ifndef HAVE_GETTIMEOFDAY +#define HAVE_GETTIMEOFDAY 1 +#endif +#ifndef HAVE_INET_NTOP +#define HAVE_INET_NTOP 1 +#endif +#ifndef HAVE_INET_PTON +#define HAVE_INET_PTON 1 +#endif +#ifndef HAVE_IOCTLSOCKET +#define HAVE_IOCTLSOCKET 1 +#endif +/* NOTE: HAVE_IOCTL_FIONBIO is deliberately NOT defined. c-ares's + * setsocknonblock() prefers HAVE_IOCTL_FIONBIO (POSIX ioctl(FIONBIO)) over + * HAVE_IOCTLSOCKET_FIONBIO (Windows ioctlsocket(FIONBIO)), and the POSIX + * branch calls ioctl() which does not exist on Windows. config-win32.h — + * the hand-crafted config c-ares uses for non-CMake Windows builds — also + * omits it, defining only HAVE_IOCTLSOCKET_FIONBIO. */ +#ifndef HAVE_IOCTLSOCKET_FIONBIO +#define HAVE_IOCTLSOCKET_FIONBIO 1 +#endif +#ifndef HAVE_RECV +#define HAVE_RECV 1 +#endif +#ifndef HAVE_RECVFROM +#define HAVE_RECVFROM 1 +#endif +#ifndef HAVE_SEND +#define HAVE_SEND 1 +#endif +#ifndef HAVE_SETSOCKOPT +#define HAVE_SETSOCKOPT 1 +#endif +#ifndef HAVE_SOCKET +#define HAVE_SOCKET 1 +#endif +#ifndef HAVE_STRDUP +#define HAVE_STRDUP 1 +#endif +#ifndef HAVE_STRICMP +#define HAVE_STRICMP 1 +#endif +#ifndef HAVE_STRNICMP +#define HAVE_STRNICMP 1 +#endif +#ifndef HAVE_GETENV +#define HAVE_GETENV 1 +#endif +/* HAVE_IOCTL_FIONBIO is intentionally absent and HAVE_IOCTLSOCKET_FIONBIO + * is already defined above; see the NOTE above the first + * HAVE_IOCTLSOCKET_FIONBIO define. */ +#ifndef HAVE_GETADDRINFO_THREADSAFE +#define HAVE_GETADDRINFO_THREADSAFE 1 +#endif +#ifndef HAVE_SOCKADDR_IN6_SIN6_SCOPE_ID +#define HAVE_SOCKADDR_IN6_SIN6_SCOPE_ID 1 +#endif +#ifndef RECV_TYPE_ARG1 +#define RECV_TYPE_ARG1 SOCKET +#endif +#ifndef RECV_TYPE_ARG2 +#define RECV_TYPE_ARG2 char * +#endif +#ifndef RECV_TYPE_ARG3 +#define RECV_TYPE_ARG3 int +#endif +#ifndef RECV_TYPE_ARG4 +#define RECV_TYPE_ARG4 int +#endif +#ifndef RECV_TYPE_RETV +#define RECV_TYPE_RETV int +#endif +#ifndef SEND_QUAL_ARG2 +#define SEND_QUAL_ARG2 +#endif +#ifndef SEND_TYPE_ARG1 +#define SEND_TYPE_ARG1 SOCKET +#endif +#ifndef SEND_TYPE_ARG2 +#define SEND_TYPE_ARG2 char * +#endif +#ifndef SEND_TYPE_ARG3 +#define SEND_TYPE_ARG3 int +#endif +#ifndef SEND_TYPE_ARG4 +#define SEND_TYPE_ARG4 int +#endif +#ifndef SEND_TYPE_RETV +#define SEND_TYPE_RETV int +#endif +#ifndef RECVFROM_TYPE_ARG1 +#define RECVFROM_TYPE_ARG1 SOCKET +#endif +#ifndef RECVFROM_TYPE_ARG2 +#define RECVFROM_TYPE_ARG2 char * +#endif +#ifndef RECVFROM_TYPE_ARG3 +#define RECVFROM_TYPE_ARG3 int +#endif +#ifndef RECVFROM_TYPE_ARG4 +#define RECVFROM_TYPE_ARG4 int +#endif +#ifndef RECVFROM_TYPE_ARG5 +#define RECVFROM_TYPE_ARG5 "struct sockaddr *" +#endif +#ifndef RECVFROM_TYPE_ARG6 +#define RECVFROM_TYPE_ARG6 "int *" +#endif +#ifndef RECVFROM_TYPE_RETV +#define RECVFROM_TYPE_RETV int +#endif +#ifndef RETSIGTYPE +#define RETSIGTYPE void +#endif +#ifndef CARES_TYPEOF_ARES_SOCKLEN_T +#define CARES_TYPEOF_ARES_SOCKLEN_T socklen_t +#endif +#ifndef CARES_TYPEOF_ARES_SSIZE_T +#define CARES_TYPEOF_ARES_SSIZE_T "long long" +#endif +#endif /* _WIN32 */ +CARES_WIN32_FIX + +# The c-ares CMake detection above fails every HAVE_* probe (SOCKLEN_T, +# TYPE_SOCKET, STRUCT_ADDRINFO, ...), so its CMakeLists never adds the Winsock +# library to the link line. Without it the static libcares.a references +# WSAStartup / select / ntohl / gethostname / ... which then fail to link with +# "undefined reference to __imp_*". +# +# The static lib target (c-ares_static) links CARES_DEPENDENT_LIBS +# (ws2_32/advapi32/iphlpapi) PUBLICly, so gRPC's own consumers are fine. But +# the diagnostic tools (ahost/adig/acountry) link ${PROJECT_NAME} = "c-ares" +# by name; with CARES_SHARED=OFF there is no "c-ares" target, so CMake treats +# it as a plain library name and resolves it to libcares.a WITHOUT the PUBLIC +# Winsock deps -- the tools then fail to link with "undefined reference to +# __imp_*". The backend image doesn't need these standalone diagnostic +# utilities (gRPC links libcares.a, not the tools), so disable them entirely +# via -DCARES_BUILD_TOOLS=OFF in the cmake configure below. + +# boringssl's ssl_file.cc and ssl_x509.cc use X509_NAME as a C++ type, but +# Windows' wincrypt.h (pulled in transitively by in some +# mingw-w64 header chains) #defines X509_NAME as L"Name", a string literal. +# The macro turns `const X509_NAME *const *a` into `const L"Name" *const *a` +# and the whole file fails to compile. A per-file #undef doesn't work because +# wincrypt.h is re-included transitively through boringssl's own headers. +# Instead, define WIN32_LEAN_AND_MEAN globally via CMAKE_C/CXX_FLAGS so +# windows.h never pulls in wincrypt.h at all, eliminating the collision +# everywhere. This is the standard Windows idiom for avoiding wincrypt.h +# macro pollution (X509_NAME, X509_CERT, etc.) in code that uses OpenSSL/ +# boringssl types. +# +# _WIN32_WINNT=0x600 must ride along on the same flags: abseil's +# win32_waiter.cc guards its Win32Waiter definitions on +# _WIN32_WINNT >= _WIN32_WINNT_VISTA, and win32_waiter.h is included before +# any header sets a default _WIN32_WINNT, so without an explicit define the +# gate stays off. per_thread_sem.cc pulls win32_waiter.h in late (after +# thread_identity.h has already defined the default), so it compiles with the +# gate ON and the synchronization archive ends up referencing Win32Waiter +# while win32_waiter.cc.o is empty -- protoc dies with "undefined reference +# to Win32Waiter::*" that no --start-group can fix, because the symbols are +# not in the archive at all. (gRPC's own CMakeLists only adds +# -D_WIN32_WINNT=0x600 under `if (MINGW)`, which the native cmake sets but +# the MSYS2/Cygwin cmake that CI resolves via PATH does not.) +# +# The flags are spelled out on each cmake configure below instead of through +# CMAKE_ARGS: ${CMAKE_ARGS} is word-split on spaces by bash, so a +# space-containing -D value cannot travel through it. +export CMAKE_ARGS="${CMAKE_ARGS:-}" + +# boringssl's bssl CLI tool uses Winsock functions (socket, connect, select, +# WSAStartup, ...) directly in transport_common.cc, but its CMakeLists only +# links bssl with ssl crypto, omitting ws2_32. A global +# -DCMAKE_EXE_LINKER_FLAGS=-lws2_32 cannot fix this: CMake places those flags +# before the object files on the link line, so ld cannot use the library to +# satisfy the references that only appear later, and bssl still fails with +# "undefined reference to `__imp_select'". Patch the CMakeLists instead. +# gRPC adds the whole boringssl-with-bazel dir via add_subdirectory, so the +# bssl target lives in the ROOT CMakeLists.txt (its binary dir shows as +# third_party/boringssl-with-bazel/CMakeFiles/bssl.dir/), not src/CMakeLists.txt. +# Both files declare the same `target_link_libraries(bssl ssl crypto)` line, so +# rewrite it in place to add ws2_32. Editing the exact declaration (instead of +# appending a guarded block) cannot be skipped by target visibility or block +# evaluation order: when the line is read, ssl and crypto are already linked +# and the bssl target already exists. +for bssl_cmake in \ + third_party/boringssl-with-bazel/CMakeLists.txt \ + third_party/boringssl-with-bazel/src/CMakeLists.txt +do + if [ -f "$bssl_cmake" ]; then + sed -i 's/^target_link_libraries(bssl ssl crypto)/target_link_libraries(bssl ssl crypto ws2_32)/' "$bssl_cmake" + grep -q 'target_link_libraries(bssl ssl crypto ws2_32)' "$bssl_cmake" || \ + { echo "ERROR: failed to patch bssl link in $bssl_cmake" >&2; exit 1; } + fi +done + +# zlib bundled with gRPC 1.59.0 ships win32/zlib1.rc which the mingw-w64 +# binutils 2.46+ windres rejects ("zlib1.rc:7: syntax error" — the VERSIONINFO +# resource format it emits is no longer accepted). The .rc is only used to +# embed Windows version metadata into the DLL; it is unnecessary for a static +# zlib linked into protobuf/gRPC. Overwriting the file with a minimal valid +# resource is format-independent (no need to know the pinned submodule's +# CMakeLists spelling, and the file is unconditionally named win32/zlib1.rc on +# WIN32) and produces no side effects for a static library. +cat > third_party/zlib/win32/zlib1.rc <<'ZLIB_RC_FIX' +1 VERSIONINFO +FILEVERSION 1,3,0,0 +PRODUCTVERSION 1,3,0,0 +BEGIN +END +ZLIB_RC_FIX + +# The grpc Makefile's MSYS/MINGW branches already add OPENSSL_NO_ASM=ON. We +# still pass an explicit generator + compiler: a bare cmake on windows-latest +# would pick the Visual Studio generator, producing an MSVC gRPC that the +# mingw-built grpc-server cannot link against. +cd "$GRPC_DIR/grpc_build" +# gRPC 1.59.0's bundled c-ares still declares cmake_minimum_required < 3.5, +# which cmake >= 4 removed support for. MSYS2 ships cmake 4.x, so configure +# with the minimum policy version c-ares' CMakeLists can still read. +# Static-library resolution on MinGW is order-sensitive: GNU ld scans each +# archive once and a definition that lives before the reference in the link +# line is never revisited. That is why protoc dies with "__imp_Sym*" +# (absl's symbolize needs dbghelp) unless dbghelp is reachable from a cyclic +# group: CMake places CMAKE_EXE_LINKER_FLAGS before the object files, so a +# bare -ldbghelp would be scanned before any reference to it exists and never +# revisited. The --start-group here is what makes it work - it extends to the +# end of the link line (ld closes it implicitly) and makes ld rescan all +# archives cyclically until no new member is pulled, so dbghelp resolves +# regardless of archive order. (The historical "Win32Waiter::*" failures are +# NOT a link-order problem; see the WIN32_LEAN_AND_MEAN/_WIN32_WINNT comment +# above - those symbols were missing from the archive entirely.) The same +# applies to shared links. +cmake -G "Unix Makefiles" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER=gcc \ + -DCMAKE_CXX_COMPILER=g++ \ + -DCMAKE_C_FLAGS="-DWIN32_LEAN_AND_MEAN -D_WIN32_WINNT=0x600" \ + -DCMAKE_CXX_FLAGS="-DWIN32_LEAN_AND_MEAN -D_WIN32_WINNT=0x600" \ + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DCMAKE_EXE_LINKER_FLAGS="-Wl,--start-group -ldbghelp" \ + -DCMAKE_SHARED_LINKER_FLAGS="-Wl,--start-group -ldbghelp" \ + ${CMAKE_ARGS:-} \ + -DgRPC_INSTALL=ON \ + -DEXECUTABLE_OUTPUT_PATH=../installed_packages/grpc/bin \ + -DLIBRARY_OUTPUT_PATH=../installed_packages/grpc/lib \ + -DgRPC_BUILD_TESTS=OFF \ + -DCARES_BUILD_TOOLS=OFF \ + -DgRPC_BUILD_CSHARP_EXT=OFF \ + -DgRPC_BUILD_GRPC_CPP_PLUGIN=ON \ + -DgRPC_BUILD_GRPC_CSHARP_PLUGIN=OFF \ + -DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF \ + -DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF \ + -DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF \ + -DgRPC_BUILD_GRPC_PYTHON_PLUGIN=ON \ + -DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF \ + -Dprotobuf_WITH_ZLIB=ON \ + -DRE2_BUILD_TESTING=OFF \ + -DCMAKE_INSTALL_PREFIX=../installed_packages \ + -DOPENSSL_NO_ASM=ON \ + ../grpc_repo/grpc +# Diagnostics: print the bssl link line that CMake generated. bssl uses Winsock +# directly and dies with __imp_select/__imp_* undefined references ~30 minutes +# into the build if ws2_32 is not on its link line, so surface it now while the +# CI log still has the configure context. Verify ws2_32 actually landed. +echo "=== cmake flavor (must be the UCRT64 native build; MSYS2's /usr/bin/cmake +splits the Windows exe names and, unlike the native build, does not set MINGW) ===" +which cmake +cmake --version | head -1 +echo "=== bssl link (expect ws2_32) ===" +if [ -f third_party/boringssl-with-bazel/CMakeFiles/bssl.dir/linkLibs.rsp ]; then + cat third_party/boringssl-with-bazel/CMakeFiles/bssl.dir/linkLibs.rsp +elif [ -f third_party/boringssl-with-bazel/CMakeFiles/bssl.dir/build.make ]; then + grep 'Linking CXX executable bssl' -A1 third_party/boringssl-with-bazel/CMakeFiles/bssl.dir/build.make || true +else + echo "WARNING: bssl build files not generated" +fi +grep -rq 'ws2_32' third_party/boringssl-with-bazel/CMakeFiles/bssl.dir/ && \ + echo "OK: bssl link includes ws2_32" || echo "FAIL: ws2_32 missing from bssl link" +# Diagnostics: protoc's link rule must carry --start-group + dbghelp, otherwise +# the absl static archives (symbolize) fail ~40 minutes in with undefined +# references that CMake cannot reorder away. +echo "=== protoc link (expect --start-group and -ldbghelp) ===" +if [ -f third_party/protobuf/CMakeFiles/protoc.dir/build.make ]; then + grep 'Linking CXX executable' -A1 third_party/protobuf/CMakeFiles/protoc.dir/build.make || true +else + echo "WARNING: protoc build files not generated" +fi +grep -rq -- '--start-group' third_party/protobuf/CMakeFiles/protoc.dir/ && \ + echo "OK: protoc link has --start-group" || echo "FAIL: --start-group missing from protoc link" +cmake --build . -j "$JOBS" +# Diagnostics: the _WIN32_WINNT=0x600 flag above must have compiled +# win32_waiter.cc with its gate open; confirm the archive actually defines +# Win32Waiter (a successful protoc link already implies it, but surface the +# count for the log). +echo "=== absl synchronization archive (expect Win32Waiter T symbols) ===" +# LIBRARY_OUTPUT_PATH=../installed_packages/grpc/lib is relative to the top +# build dir, so the archive lands next to the build tree, not under +# third_party/abseil-cpp. +nm ../installed_packages/grpc/lib/libabsl_synchronization.a 2>/dev/null | \ + grep -c 'Win32Waiter' || echo "WARNING: no Win32Waiter symbols in libabsl_synchronization.a" +cmake --build . --target install + +INSTALLED_PACKAGES="$GRPC_DIR/installed_packages" +ADDED_CMAKE_ARGS="-Dabsl_DIR=${INSTALLED_PACKAGES}/lib/cmake/absl \ + -DProtobuf_DIR=${INSTALLED_PACKAGES}/lib/cmake/protobuf \ + -Dutf8_range_DIR=${INSTALLED_PACKAGES}/lib/cmake/utf8_range \ + -DgRPC_DIR=${INSTALLED_PACKAGES}/lib/cmake/grpc \ + -DCMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES=${INSTALLED_PACKAGES}/include" +# find_program(protoc / grpc_cpp_plugin) in the grpc-server CMakeLists searches +# PATH, so the freshly installed protoc + grpc_cpp_plugin must be reachable. +export PATH="${INSTALLED_PACKAGES}/bin:${PATH}" + +# --------------------------------------------------------------------------- +# llama.cpp source at the pinned version, with the LocalAI grpc-server patch +# --------------------------------------------------------------------------- +cd "$ROOT/backend/cpp/llama-cpp" +LLAMA_VERSION=$(grep '^LLAMA_VERSION' Makefile | head -1 | cut -d= -f2 | cut -d'?' -f1 | tr -d ' ') +mkdir -p llama.cpp +cd llama.cpp +git init -q +# Re-runnable like the gRPC clone above; -B resets a stale `build` branch. +git remote remove origin 2>/dev/null || true +git remote add origin https://github.com/ggerganov/llama.cpp +# Shallow-fetch the pinned commit (allowAnySHA1InWant is on for public repos), +# then the pinned submodules shallowly. Equivalent content to `make llama.cpp` +# without downloading the full llama.cpp history on every build. +git fetch origin "$LLAMA_VERSION" --depth 1 +git checkout -q -B build "$LLAMA_VERSION" +# prepare.sh patches the source tree in place; re-running against the same +# commit would otherwise find the patch already applied and fail. Restore the +# pristine tree (tracked edits, untracked .rej/.orig and the generated +# tools/grpc-server staging dir). +git reset -q --hard +git clean -qfd +git submodule update --init --recursive --depth 1 --single-branch +cd "$ROOT/backend/cpp/llama-cpp" +bash prepare.sh + +# --------------------------------------------------------------------------- +# The three cmake variants (flags mirror backend/cpp/llama-cpp/Makefile) +# --------------------------------------------------------------------------- +build_variant() { + local name="$1" + local targets="$2" + shift 2 + # ggml turns ccache on when it finds it, but on MSYS2 ccache wants + # Windows-style USERPROFILE/LOCALAPPDATA the login profile drops - disable + # it (the variant builds are from scratch anyway). + rm -rf "llama.cpp/build-${name}" + cmake -G "Unix Makefiles" \ + -S llama.cpp -B "llama.cpp/build-${name}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER=gcc \ + -DCMAKE_CXX_COMPILER=g++ \ + -DCMAKE_C_FLAGS="-DWIN32_LEAN_AND_MEAN -D_WIN32_WINNT=0xA00" \ + -DCMAKE_CXX_FLAGS="-DWIN32_LEAN_AND_MEAN -D_WIN32_WINNT=0xA00" \ + -DCMAKE_EXE_LINKER_FLAGS="-Wl,--start-group -ldbghelp" \ + ${CMAKE_ARGS:-} \ + -DGGML_NATIVE=OFF \ + -DLLAMA_OPENSSL=OFF \ + -DLLAMA_CURL=OFF \ + -DBUILD_SHARED_LIBS=OFF \ + -DGGML_CCACHE=OFF \ + "$@" \ + $ADDED_CMAKE_ARGS + cmake --build "llama.cpp/build-${name}" --config Release -j "$JOBS" --target $targets +} + +# CPU_ALL_VARIANTS: one grpc-server plus the dlopen-able libggml-cpu-*.dll set. +# ggml/llama go shared so the dynamic CPU backends work; gRPC stays static. +# GGML_VULKAN=ON adds the dlopen-able libggml-vulkan.dll to the same image, so +# the single grpc-server auto-detects a Vulkan device at runtime (ggml-vulkan +# loads vulkan-1.dll dynamically - it is never in the import table) and falls +# back to CPU when the host has no Vulkan driver. No separate variant or launcher +# change is needed: one image works everywhere and GPU offload just works when +# a model requests gpu_layers. +build_variant "cpu-all" "grpc-server ggml" \ + -DBUILD_SHARED_LIBS=ON \ + -DGGML_BACKEND_DL=ON \ + -DGGML_CPU_ALL_VARIANTS=ON \ + -DGGML_VULKAN=ON + +# gRPC-RPC server + the ggml-rpc-server companion binary. +build_variant "rpc" "grpc-server ggml-rpc-server" \ + -DGGML_RPC=ON \ + -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX512=off \ + -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off + +# Static fallback, mirrors llama-cpp-fallback. +build_variant "fallback" "grpc-server" \ + -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX512=off \ + -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off + +# --------------------------------------------------------------------------- +# Stage the image contents +# --------------------------------------------------------------------------- +STAGE="$ROOT/build/windows" +rm -rf "$STAGE" +mkdir -p "$STAGE" "$ROOT/backend-images" + +cp llama.cpp/build-cpu-all/bin/grpc-server.exe "$STAGE/llama-cpp-cpu-all.exe" +# ggml's shared backends are loadable DLLs; they go next to the executable so +# the registry finds them without any PATH setup (mirrors run.sh LD_LIBRARY_PATH). +find llama.cpp/build-cpu-all/bin -maxdepth 1 -name '*.dll' -exec cp -v {} "$STAGE/" \; + +cp llama.cpp/build-rpc/bin/grpc-server.exe "$STAGE/llama-cpp-grpc.exe" +cp llama.cpp/build-rpc/bin/ggml-rpc-server.exe "$STAGE/llama-cpp-rpc-server.exe" + +cp llama.cpp/build-fallback/bin/grpc-server.exe "$STAGE/llama-cpp-fallback.exe" + +cd "$ROOT" +# The launcher is the PowerShell script pkg/model/process.go starts on Windows +# (run.ps1, mirroring run.sh); run.sh below is the stub that keeps +# discovery/validation/upgrade uniform with the other platforms. +cp backend/cpp/llama-cpp/run.ps1 "$STAGE/run.ps1" +cp backend/cpp/llama-cpp/run.sh "$STAGE/run.sh" + +# Bundle the runtime DLLs (mingw gcc runtime, openssl when gRPC links it +# dynamically) so the image runs on a host with no MSYS2. ggml's shared +# backends are already in $STAGE; scanning them too (not just the .exe files) +# catches the runtime deps of the loadable modules (ggml-vulkan.dll pulls in +# libgcc/libstdc++/libwinpthread) and lets the vulkan loader be bundled below. +for bin in "$STAGE"/*.exe "$STAGE"/*.dll; do + objdump -p "$bin" | awk '/DLL Name:/ {print $3}' | while read -r dll; do + if [ -f "/ucrt64/bin/$dll" ] && [ ! -e "$STAGE/$dll" ]; then + cp -v "/ucrt64/bin/$dll" "$STAGE/" + fi + done +done + +# ggml-vulkan loads vulkan-1.dll dynamically (volk-style LoadLibrary), so it is +# not in any import table and the scan above cannot see it. Bundle the loader +# from the MSYS2 vulkan-loader package anyway: on a host with no Vulkan driver +# the loader still initializes, ggml-vulkan registers zero devices, and CPU +# inference keeps working - but on a host whose driver does not drop its own +# copy into system32 the GPU would otherwise be invisible. +if [ -f "/ucrt64/bin/vulkan-1.dll" ] && [ ! -e "$STAGE/vulkan-1.dll" ]; then + cp -v "/ucrt64/bin/vulkan-1.dll" "$STAGE/" +fi + +echo "Bundled DLLs:" +ls -la "$STAGE" + +# --------------------------------------------------------------------------- +# Package as an OCI image tar +# --------------------------------------------------------------------------- +# local-ai.exe is only the tool that assembles the OCI tar - the image carries +# $STAGE, not this binary. cmd/local-ai still needs the prereqs of the upstream +# `make build`: pkg/grpc/proto/*.pb.go (gitignored, absent in a fresh checkout; +# regenerate with the protoc built above + the pinned Go plugins, mirroring +# `make protogen-go`) and the embedded react-ui dist (stubbed like lint.yml; +# the real bundle would need node, and this binary never serves it). +GO_PLUGIN_DIR="$(go env GOPATH)/bin" +GO_PLUGIN_DIR_MSYS="$(cygpath -u "$GO_PLUGIN_DIR")" +if [ ! -x "$GO_PLUGIN_DIR_MSYS/protoc-gen-go" ]; then + go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.2 +fi +if [ ! -x "$GO_PLUGIN_DIR_MSYS/protoc-gen-go-grpc" ]; then + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@1958fcbe2ca8bd93af633f11e97d44e567e945af +fi +mkdir -p pkg/grpc/proto +"$GRPC_DIR/installed_packages/bin/protoc.exe" --experimental_allow_proto3_optional \ + -Ibackend/ \ + --go_out=pkg/grpc/proto/ --go_opt=paths=source_relative \ + --go-grpc_out=pkg/grpc/proto/ --go-grpc_opt=paths=source_relative \ + --plugin=protoc-gen-go="$GO_PLUGIN_DIR/protoc-gen-go.exe" \ + --plugin=protoc-gen-go-grpc="$GO_PLUGIN_DIR/protoc-gen-go-grpc.exe" \ + backend/backend.proto +mkdir -p core/http/react-ui/dist +[ -f core/http/react-ui/dist/index.html ] || touch core/http/react-ui/dist/index.html + +if [ ! -f local-ai.exe ] && [ ! -f local-ai ]; then + CGO_ENABLED=0 go build -o local-ai.exe ./cmd/local-ai +fi +mkdir -p backend-images +./local-ai util create-oci-image \ + build/windows/. \ + --output ./backend-images/llama-cpp.tar \ + --image-name "$IMAGE_NAME" \ + --platform "$PLATFORMARCH" + +rm -rf build/windows \ No newline at end of file diff --git a/scripts/changed-backends.js b/scripts/changed-backends.js index 08a024fc63e8..caa1fdc6410a 100644 --- a/scripts/changed-backends.js +++ b/scripts/changed-backends.js @@ -16,11 +16,12 @@ import { const matrixYml = yaml.load(fs.readFileSync(BACKEND_MATRIX_FILE, "utf8")); const includes = matrixYml.include; const includesDarwin = matrixYml.includeDarwin; +const includesWindows = matrixYml.includeWindows || []; const eventPath = process.env.GITHUB_EVENT_PATH; const event = JSON.parse(fs.readFileSync(eventPath, "utf8")); -const allBackendPaths = getAllBackendPaths(includes, includesDarwin); +const allBackendPaths = getAllBackendPaths(includes, includesDarwin, includesWindows); const token = process.env.GITHUB_TOKEN; const octokit = new Octokit({ auth: token }); @@ -107,6 +108,7 @@ async function getPreviousMatrix(event) { return { include: previous.include || [], includeDarwin: previous.includeDarwin || [], + includeWindows: previous.includeWindows || [], }; } catch (err) { console.log( @@ -267,9 +269,11 @@ function emitFullMatrix() { fs.appendFileSync(process.env.GITHUB_OUTPUT, `run-all=true\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends-multiarch=${multiarch.length > 0 ? 'true' : 'false'}\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends-darwin=true\n`); + fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends-windows=true\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-merges-multiarch=${hasMergesMultiarch}\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix-multiarch=${JSON.stringify({ include: multiarch })}\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix-darwin=${JSON.stringify({ include: includesDarwin })}\n`); + fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix-windows=${JSON.stringify({ include: includesWindows })}\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `merge-matrix-multiarch=${JSON.stringify(mergeMatrixMultiarch)}\n`); emitSinglearchShards(singlearch); for (const backend of allBackendPaths.keys()) { @@ -280,9 +284,10 @@ function emitFullMatrix() { function emitFilteredMatrix(changedFiles, previousMatrix, protoRevisions) { console.log("Changed files:", changedFiles); - const { filtered, filteredDarwin, changedBackends } = filterMatrix({ + const { filtered, filteredDarwin, filteredWindows, changedBackends } = filterMatrix({ includes, includesDarwin, + includesWindows, changedFiles, previousMatrix, protoRevisions, @@ -290,13 +295,16 @@ function emitFilteredMatrix(changedFiles, previousMatrix, protoRevisions) { console.log("Filtered files:", filtered); console.log("Filtered files Darwin:", filteredDarwin); + console.log("Filtered files Windows:", filteredWindows); const { multiarch, singlearch } = splitByArch(filtered); const hasBackendsMultiarch = multiarch.length > 0 ? 'true' : 'false'; const hasBackendsDarwin = filteredDarwin.length > 0 ? 'true' : 'false'; + const hasBackendsWindows = filteredWindows.length > 0 ? 'true' : 'false'; console.log("Has single-arch backends?:", singlearch.length > 0 ? 'true' : 'false'); console.log("Has multi-arch backends?:", hasBackendsMultiarch); console.log("Has Darwin backends?:", hasBackendsDarwin); + console.log("Has Windows backends?:", hasBackendsWindows); const mergeMatrixMultiarch = computeMergeMatrix(multiarch); const hasMergesMultiarch = mergeMatrixMultiarch.include.length > 0 ? 'true' : 'false'; @@ -304,9 +312,11 @@ function emitFilteredMatrix(changedFiles, previousMatrix, protoRevisions) { fs.appendFileSync(process.env.GITHUB_OUTPUT, `run-all=false\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends-multiarch=${hasBackendsMultiarch}\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends-darwin=${hasBackendsDarwin}\n`); + fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends-windows=${hasBackendsWindows}\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-merges-multiarch=${hasMergesMultiarch}\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix-multiarch=${JSON.stringify({ include: multiarch })}\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix-darwin=${JSON.stringify({ include: filteredDarwin })}\n`); + fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix-windows=${JSON.stringify({ include: filteredWindows })}\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `merge-matrix-multiarch=${JSON.stringify(mergeMatrixMultiarch)}\n`); emitSinglearchShards(singlearch); diff --git a/scripts/lib/backend-filter.mjs b/scripts/lib/backend-filter.mjs index 9312cbf6d0c0..02239df396fa 100644 --- a/scripts/lib/backend-filter.mjs +++ b/scripts/lib/backend-filter.mjs @@ -129,8 +129,24 @@ export function inferBackendPathDarwin(item) { return `backend/${item.lang}/${item.backend}/`; } +export function inferBackendPathWindows(item) { + // llama-cpp on Windows builds from the C++ sources (via the MSYS2 build + // script, scripts/build/llama-cpp-windows.sh), not a backend/go/llama-cpp + // tree (which doesn't exist). The Windows job is matrix-driven with lang=go + // for runner/toolchain selection, but the source path is C++ — exactly the + // Darwin convention. + if (item.backend === "llama-cpp") { + return `backend/cpp/llama-cpp/`; + } + if (!item.lang) { + return `backend/python/${item.backend}/`; + } + + return `backend/${item.lang}/${item.backend}/`; +} + // Build a deduplicated map of backend name -> path prefix from all matrix entries -export function getAllBackendPaths(includes, includesDarwin) { +export function getAllBackendPaths(includes, includesDarwin, includesWindows = []) { const paths = new Map(); for (const item of includes) { const p = inferBackendPath(item); @@ -144,6 +160,12 @@ export function getAllBackendPaths(includes, includesDarwin) { paths.set(item.backend, p); } } + for (const item of includesWindows) { + const p = inferBackendPathWindows(item); + if (p && !paths.has(item.backend)) { + paths.set(item.backend, p); + } + } return paths; } @@ -173,6 +195,14 @@ const DARWIN_BESPOKE_BUILDERS = new Set([ const isDarwinGenericGo = item => !!item.lang && !DARWIN_BESPOKE_BUILDERS.has(item.backend); +// backend_build_windows.yml builds every Windows entry through +// `make backends/llama-cpp-windows` -> scripts/build/llama-cpp-windows.sh; there +// is no generic Go builder for Windows yet. Keep this set in sync with that +// workflow. +const WINDOWS_BESPOKE_BUILDERS = new Set(["llama-cpp"]); +const isWindowsGenericGo = item => + !!item.lang && !WINDOWS_BESPOKE_BUILDERS.has(item.backend); + const isLinuxPython = item => item.dockerfile.endsWith("python"); // Dockerfile.golang is the only Linux dockerfile that compiles Go (it runs @@ -360,6 +390,7 @@ export const SHARED_BUILD_INPUTS = [ matches: file => file === BACKEND_PROTO_FILE, linux: always, darwin: always, + windows: always, }, { // COPY'd into every Python image (Dockerfile.python) and into every Darwin @@ -367,6 +398,7 @@ export const SHARED_BUILD_INPUTS = [ matches: file => file.startsWith("backend/python/common/"), linux: isLinuxPython, darwin: isDarwinPython, + windows: never, }, { // Compiled into every Go backend binary (see GO_BACKEND_PKG_PREFIXES). @@ -377,11 +409,18 @@ export const SHARED_BUILD_INPUTS = [ // lang=go for runner selection only — their sources are C++ and link no // Go, so isDarwinGenericGo is the correct predicate here, exactly as it // is for scripts/build/golang-darwin.sh. + // + // Windows: llama-cpp's launcher is the run.ps1 script (no Go, no compile) + // and local-ai.exe is rebuilt from the current checkout in the workflow + // itself, so a pkg/ change never invalidates a published Windows backend + // image. A future generic Go Windows builder would switch this to + // isWindowsGenericGo. matches: file => GO_BACKEND_PKG_PREFIXES.some(prefix => file.startsWith(prefix)) && !file.endsWith("_test.go"), linux: isLinuxGo, darwin: isDarwinGenericGo, + windows: never, }, { // The reusable build workflows own build-args, packaging and push for @@ -390,11 +429,19 @@ export const SHARED_BUILD_INPUTS = [ matches: file => file === ".github/workflows/backend_build.yml", linux: always, darwin: never, + windows: never, }, { matches: file => file === ".github/workflows/backend_build_darwin.yml", linux: never, darwin: always, + windows: never, + }, + { + matches: file => file === ".github/workflows/backend_build_windows.yml", + linux: never, + darwin: never, + windows: always, }, { // Decides which GPU libraries end up inside an image. Every Linux image @@ -406,31 +453,43 @@ export const SHARED_BUILD_INPUTS = [ matches: file => file === "scripts/build/package-gpu-libs.sh", linux: always, darwin: never, + windows: never, }, { matches: file => file === "scripts/build/python-darwin.sh", linux: never, darwin: isDarwinPython, + windows: never, }, { matches: file => file === "scripts/build/golang-darwin.sh", linux: never, darwin: isDarwinGenericGo, + windows: never, }, { matches: file => file === "scripts/build/llama-cpp-darwin.sh", linux: never, darwin: item => item.backend === "llama-cpp", + windows: never, }, { matches: file => file === "scripts/build/ds4-darwin.sh", linux: never, darwin: item => item.backend === "ds4", + windows: never, }, { matches: file => file === "scripts/build/privacy-filter-darwin.sh", linux: never, darwin: item => item.backend === "privacy-filter", + windows: never, + }, + { + matches: file => file === "scripts/build/llama-cpp-windows.sh", + linux: never, + darwin: never, + windows: item => item.backend === "llama-cpp", }, { // Catch-all, deliberately last and deliberately broad: anything else under @@ -443,6 +502,7 @@ export const SHARED_BUILD_INPUTS = [ file.startsWith("scripts/build/") && !file.endsWith("_test.sh"), linux: always, darwin: always, + windows: always, }, ]; @@ -478,7 +538,7 @@ export const BACKEND_MATRIX_FILE = ".github/backend-matrix.yml"; // Identity of a matrix entry across revisions. tag-suffix names the image; // per-arch legs of the same image are distinguished by platform-tag. Verified -// unique across all 432 Linux and 57 Darwin entries. +// unique across all 432 Linux, 57 Darwin and 1 Windows entries. export function matrixEntryKey(item) { return JSON.stringify([item["tag-suffix"] || "", item["platform-tag"] || ""]); } @@ -515,12 +575,14 @@ function matchedSharedRules(changedFiles) { return rules; } -// Filter both matrices against a changed-file list. Returns the surviving -// entries plus the set of backend names considered changed, which drives the -// per-backend boolean outputs consumed by test-extra.yml. +// Filter the Linux, Darwin and Windows matrices against a changed-file list. +// Returns the surviving entries plus the set of backend names considered +// changed, which drives the per-backend boolean outputs consumed by +// test-extra.yml. export function filterMatrix({ includes, includesDarwin, + includesWindows, changedFiles, previousMatrix, protoRevisions, @@ -553,6 +615,10 @@ export function filterMatrix({ matrixFileChanged && previousMatrix ? changedEntryKeys(includesDarwin, previousMatrix.includeDarwin || []) : null; + const changedWindowsKeys = + matrixFileChanged && previousMatrix + ? changedEntryKeys(includesWindows || [], previousMatrix.includeWindows || []) + : null; const filtered = includes.filter(item => { const backendPath = inferBackendPath(item); @@ -572,12 +638,21 @@ export function filterMatrix({ return sharedRules.some(rule => rule.darwin(item)); }); + const filteredWindows = (includesWindows || []).filter(item => { + const backendPath = inferBackendPathWindows(item); + if (changedFiles.some(file => file.startsWith(backendPath))) return true; + if (matrixDiffUnavailable) return true; + if (changedWindowsKeys && changedWindowsKeys.has(matrixEntryKey(item))) return true; + return sharedRules.some(rule => rule.windows(item)); + }); + const changedBackends = new Set(); for (const item of filtered) changedBackends.add(item.backend); for (const item of filteredDarwin) changedBackends.add(item.backend); - for (const [backend, pathPrefix] of getAllBackendPaths(includes, includesDarwin)) { + for (const item of filteredWindows) changedBackends.add(item.backend); + for (const [backend, pathPrefix] of getAllBackendPaths(includes, includesDarwin, includesWindows)) { if (backendChanged(backend, pathPrefix, changedFiles)) changedBackends.add(backend); } - return { filtered, filteredDarwin, changedBackends }; + return { filtered, filteredDarwin, filteredWindows, changedBackends }; } diff --git a/scripts/lib/backend-filter_test.mjs b/scripts/lib/backend-filter_test.mjs index f419afd3a37a..1c6da028ff8d 100644 --- a/scripts/lib/backend-filter_test.mjs +++ b/scripts/lib/backend-filter_test.mjs @@ -12,6 +12,7 @@ import { filterMatrix, inferBackendPath, inferBackendPathDarwin, + inferBackendPathWindows, } from "./backend-filter.mjs"; test("trellis2cpp maps to its Go backend source directory", () => { @@ -81,8 +82,12 @@ const includesDarwin = [ { backend: "ds4", lang: "go", "tag-suffix": "-metal-darwin-arm64-ds4", "build-type": "metal" }, ]; +const includesWindows = [ + { backend: "llama-cpp", lang: "go", "tag-suffix": "-windows-amd64-llama-cpp" }, +]; + const run = (changedFiles, previousMatrix) => - filterMatrix({ includes, includesDarwin, changedFiles, previousMatrix }); + filterMatrix({ includes, includesDarwin, includesWindows, changedFiles, previousMatrix }); const names = entries => entries.map(e => e.backend).sort(); @@ -196,12 +201,13 @@ test("a bespoke Darwin build script rebuilds only its own backend", () => { }); test("an unclassified scripts/build/ file conservatively rebuilds everything", () => { - const { filtered, filteredDarwin } = run([ + const { filtered, filteredDarwin, filteredWindows } = run([ "scripts/build/package-something-new.sh", ]); assert.equal(filtered.length, includes.length); assert.equal(filteredDarwin.length, includesDarwin.length); + assert.equal(filteredWindows.length, includesWindows.length); }); test("tests for the packaging scripts do not rebuild anything", () => { @@ -257,6 +263,87 @@ test("audio-cpp resolves to its C++ sources on Darwin, not backend/go", () => { ); }); +// --------------------------------------------------------------------------- +// Windows: native windows/amd64 backend images built under MSYS2 +// --------------------------------------------------------------------------- + +test("llama-cpp resolves to its C++ sources on Windows, not backend/go", () => { + // lang=go on a Windows entry only selects the runner and toolchain (the + // MSYS2 build compiles C++); routing it by that field would point at + // backend/go/llama-cpp/, which does not exist. + assert.equal( + inferBackendPathWindows({ + backend: "llama-cpp", + lang: "go", + "tag-suffix": "-windows-amd64-llama-cpp", + }), + "backend/cpp/llama-cpp/", + ); +}); + +test("llama-cpp source changes trigger the Windows build", () => { + const { filtered, filteredDarwin, filteredWindows, changedBackends } = run([ + "backend/cpp/llama-cpp/grpc-server.cpp", + ]); + + assert.deepEqual(names(filtered), ["llama-cpp", "turboquant"]); + assert.deepEqual(names(filteredDarwin), ["llama-cpp"]); + assert.deepEqual(names(filteredWindows), ["llama-cpp"]); + assert.ok(changedBackends.has("llama-cpp")); +}); + +test("backend.proto rebuilds the Windows entry too", () => { + const { filteredWindows } = run(["backend/backend.proto"]); + + assert.equal(filteredWindows.length, includesWindows.length); +}); + +test("the reusable Windows build workflow rebuilds Windows only", () => { + const { filtered, filteredDarwin, filteredWindows } = run([ + ".github/workflows/backend_build_windows.yml", + ]); + + assert.deepEqual(filtered, []); + assert.deepEqual(filteredDarwin, []); + assert.equal(filteredWindows.length, includesWindows.length); +}); + +test("the Windows llama-cpp build script rebuilds only its backend", () => { + const { filtered, filteredDarwin, filteredWindows } = run([ + "scripts/build/llama-cpp-windows.sh", + ]); + + assert.deepEqual(filtered, []); + assert.deepEqual(filteredDarwin, []); + assert.deepEqual(names(filteredWindows), ["llama-cpp"]); +}); + +test("a pkg/ subtree change never rebuilds the Windows entry", () => { + // The launcher is the run.ps1 script (no Go, no compile) and local-ai.exe + // is rebuilt in the workflow itself, so the shipped image does not depend on + // LocalAI pkg/ code. + const { filteredWindows } = run(["pkg/grpc/server.go"]); + + assert.deepEqual(filteredWindows, []); +}); + +test("editing the Windows matrix entry rebuilds it", () => { + const previousMatrix = previousOf( + e => e, + e => e, + e => ({ ...e, "tag-suffix": "-windows-amd64-llama-cpp-old" }) + ); + + const { filtered, filteredDarwin, filteredWindows } = run( + [".github/backend-matrix.yml"], + previousMatrix + ); + + assert.deepEqual(filtered, []); + assert.deepEqual(filteredDarwin, []); + assert.deepEqual(names(filteredWindows), ["llama-cpp"]); +}); + // --------------------------------------------------------------------------- // pkg/ subtrees linked into every Go backend binary // --------------------------------------------------------------------------- @@ -333,9 +420,10 @@ test("tests for the linked pkg subtrees do not rebuild anything", () => { // A matrix file is only ever compared against a previous revision of itself, // so build the "before" side by mutating a copy of the fixtures. -const previousOf = (linuxPatch = e => e, darwinPatch = e => e) => ({ +const previousOf = (linuxPatch = e => e, darwinPatch = e => e, windowsPatch = e => e) => ({ include: includes.map(e => linuxPatch({ ...e })), includeDarwin: includesDarwin.map(e => darwinPatch({ ...e })), + includeWindows: includesWindows.map(e => windowsPatch({ ...e })), }); test("editing an existing entry's base-image rebuilds exactly that entry", () => { @@ -372,15 +460,17 @@ test("a brand-new matrix entry for an existing backend rebuilds only itself", () const previousMatrix = { include: includes.filter(e => e.backend !== "kokoros"), includeDarwin: includesDarwin, + includeWindows: includesWindows, }; - const { filtered, filteredDarwin } = run( + const { filtered, filteredDarwin, filteredWindows } = run( [".github/backend-matrix.yml"], previousMatrix ); assert.deepEqual(names(filtered), ["kokoros"]); assert.deepEqual(filteredDarwin, []); + assert.deepEqual(filteredWindows, []); }); test("editing a Darwin entry rebuilds only that Darwin entry", () => { @@ -421,11 +511,17 @@ test("removing an entry rebuilds nothing", () => { "base-image": "nvidia/cuda:11.8.0-devel-ubuntu22.04", }], includeDarwin: includesDarwin, + includeWindows: includesWindows, }; - const { filtered } = run([".github/backend-matrix.yml"], previousMatrix); + const { filtered, filteredDarwin, filteredWindows } = run( + [".github/backend-matrix.yml"], + previousMatrix + ); assert.deepEqual(filtered, []); + assert.deepEqual(filteredDarwin, []); + assert.deepEqual(filteredWindows, []); }); test("a previous matrix is ignored when backend-matrix.yml did not change", () => { @@ -443,10 +539,11 @@ test("a previous matrix is ignored when backend-matrix.yml did not change", () = test("an unavailable previous matrix conservatively rebuilds everything", () => { // Same posture as changed-backends.js's run-all fallbacks: if we cannot // resolve what the entries used to be, we must not claim nothing changed. - const { filtered, filteredDarwin } = run([".github/backend-matrix.yml"], null); + const { filtered, filteredDarwin, filteredWindows } = run([".github/backend-matrix.yml"], null); assert.equal(filtered.length, includes.length); assert.equal(filteredDarwin.length, includesDarwin.length); + assert.equal(filteredWindows.length, includesWindows.length); }); // --- backend.proto: additive changes must not rebuild the world -------------