From f27270028e3855ec1a2bbcd70b23e7975235eba0 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> Date: Mon, 10 Aug 2026 12:13:41 +0000 Subject: [PATCH 1/2] feat(mlx): add Apple Silicon video backend Add a Darwin-only MLX-Video backend for LTX-2 and converted Wan checkpoints, expose it through the existing video API, and wire packaging, discovery, tests, docs, and an example. Assisted-by: Codex:gpt-5 --- .github/backend-matrix.yml | 3 + Makefile | 8 +- backend/index.yaml | 29 +++ backend/python/mlx-video/.gitignore | 5 + backend/python/mlx-video/Makefile | 16 ++ backend/python/mlx-video/backend.py | 169 ++++++++++++++++++ backend/python/mlx-video/install.sh | 13 ++ backend/python/mlx-video/mlx_video_utils.py | 102 +++++++++++ backend/python/mlx-video/requirements-mps.txt | 1 + backend/python/mlx-video/requirements.txt | 4 + backend/python/mlx-video/run.sh | 11 ++ backend/python/mlx-video/test.py | 86 +++++++++ backend/python/mlx-video/test.sh | 11 ++ core/config/backend_capabilities.go | 7 + docs/content/features/backends.md | 2 +- docs/content/features/video-generation.md | 27 ++- docs/content/reference/compatibility-table.md | 1 + examples/mlx-video/README.md | 26 +++ examples/mlx-video/ltx2-mlx.yaml | 8 + 19 files changed, 526 insertions(+), 3 deletions(-) create mode 100644 backend/python/mlx-video/.gitignore create mode 100644 backend/python/mlx-video/Makefile create mode 100755 backend/python/mlx-video/backend.py create mode 100755 backend/python/mlx-video/install.sh create mode 100644 backend/python/mlx-video/mlx_video_utils.py create mode 100644 backend/python/mlx-video/requirements-mps.txt create mode 100644 backend/python/mlx-video/requirements.txt create mode 100755 backend/python/mlx-video/run.sh create mode 100644 backend/python/mlx-video/test.py create mode 100755 backend/python/mlx-video/test.sh create mode 100644 examples/mlx-video/README.md create mode 100644 examples/mlx-video/ltx2-mlx.yaml diff --git a/.github/backend-matrix.yml b/.github/backend-matrix.yml index e7bada4b4e91..72b595ccf168 100644 --- a/.github/backend-matrix.yml +++ b/.github/backend-matrix.yml @@ -6311,6 +6311,9 @@ includeDarwin: - backend: "mlx" tag-suffix: "-metal-darwin-arm64-mlx" build-type: "mps" + - backend: "mlx-video" + tag-suffix: "-metal-darwin-arm64-mlx-video" + build-type: "mps" - backend: "chatterbox" tag-suffix: "-metal-darwin-arm64-chatterbox" build-type: "mps" diff --git a/Makefile b/Makefile index 1f25e246c423..744e6c1d29e9 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Disable parallel execution for backend builds -.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/bonsai backends/outetts backends/piper backends/stablediffusion-ggml backends/trellis2cpp backends/trellis2cpp-darwin backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/nemo-speech-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/valkey-store backends/cloud-proxy backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/moss-tts-cpp backends/magpie-tts-cpp backends/vllm-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin backends/audio-cpp backends/audio-cpp-darwin +.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/bonsai backends/outetts backends/piper backends/stablediffusion-ggml backends/trellis2cpp backends/trellis2cpp-darwin backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/nemo-speech-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/valkey-store backends/cloud-proxy backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/mlx-video backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/moss-tts-cpp backends/magpie-tts-cpp backends/vllm-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin backends/audio-cpp backends/audio-cpp-darwin GOCMD=go GOTEST=$(GOCMD) test @@ -599,6 +599,7 @@ prepare-test-extra: protogen-python $(MAKE) -C backend/python/vllm $(MAKE) -C backend/python/vllm-omni $(MAKE) -C backend/python/longcat-video + $(MAKE) -C backend/python/mlx-video $(MAKE) -C backend/python/sglang $(MAKE) -C backend/python/vibevoice $(MAKE) -C backend/python/liquid-audio @@ -631,6 +632,7 @@ test-extra: prepare-test-extra $(MAKE) -C backend/python/vllm test $(MAKE) -C backend/python/vllm-omni test $(MAKE) -C backend/python/longcat-video test + $(MAKE) -C backend/python/mlx-video test $(MAKE) -C backend/python/vibevoice test $(MAKE) -C backend/python/liquid-audio test $(MAKE) -C backend/python/moonshine test @@ -1233,6 +1235,10 @@ backends/mlx: BACKEND=mlx $(MAKE) build-darwin-python-backend ./local-ai backends install "ocifile://$(abspath ./backend-images/mlx.tar)" +backends/mlx-video: + BACKEND=mlx-video $(MAKE) build-darwin-python-backend + ./local-ai backends install "ocifile://$(abspath ./backend-images/mlx-video.tar)" + backends/diffuser-darwin: BACKEND=diffusers $(MAKE) build-darwin-python-backend ./local-ai backends install "ocifile://$(abspath ./backend-images/diffusers.tar)" diff --git a/backend/index.yaml b/backend/index.yaml index 3420d85496b7..c4d1ddf56276 100644 --- a/backend/index.yaml +++ b/backend/index.yaml @@ -1060,6 +1060,21 @@ nvidia-l4t: "nvidia-l4t-mlx" nvidia-l4t-cuda-12: "nvidia-l4t-mlx" nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-mlx" +- &mlx-video + name: "mlx-video" + icon: https://avatars.githubusercontent.com/u/102832242?s=200&v=4 + urls: + - https://github.com/Blaizzy/mlx-video + license: MIT + description: | + Generate videos with LTX-2 and converted Wan2.1/Wan2.2 checkpoints using + MLX on Apple Silicon. + tags: + - text-to-video + - image-to-video + - MLX + capabilities: + metal: "metal-mlx-video" - &mlx-vlm name: "mlx-vlm" icon: https://avatars.githubusercontent.com/u/102832242?s=200&v=4 @@ -2164,6 +2179,20 @@ uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-mlx" mirrors: - localai/localai-backends:latest-metal-darwin-arm64-mlx +- !!merge <<: *mlx-video + name: "metal-mlx-video" + uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-mlx-video" + mirrors: + - localai/localai-backends:latest-metal-darwin-arm64-mlx-video +- !!merge <<: *mlx-video + name: "metal-mlx-video-development" + uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-mlx-video" + mirrors: + - localai/localai-backends:master-metal-darwin-arm64-mlx-video +- !!merge <<: *mlx-video + name: "mlx-video-development" + capabilities: + metal: "metal-mlx-video-development" - !!merge <<: *mlx name: "metal-mlx-development" uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-mlx" diff --git a/backend/python/mlx-video/.gitignore b/backend/python/mlx-video/.gitignore new file mode 100644 index 000000000000..8cf675e0a0aa --- /dev/null +++ b/backend/python/mlx-video/.gitignore @@ -0,0 +1,5 @@ +__pycache__/ +backend_pb2.py +backend_pb2_grpc.py +common/ +venv/ diff --git a/backend/python/mlx-video/Makefile b/backend/python/mlx-video/Makefile new file mode 100644 index 000000000000..de59578b32c4 --- /dev/null +++ b/backend/python/mlx-video/Makefile @@ -0,0 +1,16 @@ +.PHONY: all run test protogen-clean clean + +all: + bash install.sh + +run: all + bash run.sh + +test: + bash test.sh + +protogen-clean: + $(RM) backend_pb2.py backend_pb2_grpc.py + +clean: protogen-clean + rm -rf __pycache__ venv diff --git a/backend/python/mlx-video/backend.py b/backend/python/mlx-video/backend.py new file mode 100755 index 000000000000..3e86f1172f9e --- /dev/null +++ b/backend/python/mlx-video/backend.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT + +import argparse +import os +import platform +import signal +import subprocess +import sys +import traceback +from concurrent import futures + +import grpc + +import backend_pb2 +import backend_pb2_grpc +from mlx_video_utils import build_generation_command, classify_model, parse_options + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "common")) +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "common")) +from grpc_auth import get_auth_interceptors # noqa: E402 + + +MAX_WORKERS = int(os.environ.get("PYTHON_GRPC_MAX_WORKERS", "1")) + + +class BackendServicer(backend_pb2_grpc.BackendServicer): + def __init__(self): + self.family = None + self.model = None + self.options = {} + + def Health(self, request, context): + return backend_pb2.Reply(message=b"OK") + + def LoadModel(self, request, context): + if platform.system() != "Darwin" or platform.machine() != "arm64": + return self._fail( + context, + grpc.StatusCode.FAILED_PRECONDITION, + "mlx-video requires macOS on Apple Silicon", + ) + + model = ( + request.ModelFile + if request.ModelFile and os.path.isdir(request.ModelFile) + else request.Model + ) + family = classify_model(model) + if family is None: + return self._fail( + context, + grpc.StatusCode.INVALID_ARGUMENT, + "mlx-video supports LTX-2 and converted Wan2.1/Wan2.2 MLX checkpoints", + ) + if family == "wan" and not os.path.isdir(model): + return self._fail( + context, + grpc.StatusCode.INVALID_ARGUMENT, + "Wan models must be converted to MLX and configured as a local model directory", + ) + + self.family = family + self.model = model + self.options = parse_options(request.Options) + return backend_pb2.Result(message="mlx-video model configured", success=True) + + def Free(self, request, context): + self.family = None + self.model = None + self.options = {} + return backend_pb2.Result(message="Model released", success=True) + + def GenerateVideo(self, request, context): + if not self.family or not self.model: + return self._fail( + context, grpc.StatusCode.FAILED_PRECONDITION, "model is not loaded" + ) + if not request.prompt.strip(): + return self._fail(context, grpc.StatusCode.INVALID_ARGUMENT, "prompt is required") + if not request.dst: + return self._fail( + context, + grpc.StatusCode.INVALID_ARGUMENT, + "output destination is required", + ) + + process = None + try: + command = build_generation_command( + family=self.family, + model=self.model, + prompt=request.prompt, + destination=request.dst, + negative_prompt=request.negative_prompt, + start_image=request.start_image, + end_image=request.end_image, + audio=request.audio, + width=request.width, + height=request.height, + num_frames=request.num_frames, + fps=request.fps, + seed=request.seed, + steps=request.step, + cfg_scale=request.cfg_scale, + params={**self.options, **dict(request.params)}, + ) + os.makedirs( + os.path.dirname(request.dst) or ".", mode=0o750, exist_ok=True + ) + process = subprocess.Popen(command) + if hasattr(context, "add_callback"): + context.add_callback( + lambda: process.poll() is None and process.terminate() + ) + return_code = process.wait() + if return_code != 0: + raise RuntimeError( + f"upstream generator exited with status {return_code}" + ) + if not os.path.isfile(request.dst) or os.path.getsize(request.dst) == 0: + raise RuntimeError("upstream generator did not create the requested video") + return backend_pb2.Result(message="Video generated successfully", success=True) + except ValueError as err: + return self._fail(context, grpc.StatusCode.INVALID_ARGUMENT, str(err)) + except Exception as err: + print(f"mlx-video generation failed: {err}", file=sys.stderr) + traceback.print_exc() + return self._fail( + context, + grpc.StatusCode.INTERNAL, + f"mlx-video generation failed: {err}", + ) + + @staticmethod + def _fail(context, code, message): + context.set_code(code) + context.set_details(message) + return backend_pb2.Result(message=message, success=False) + + +def serve(address): + server = grpc.server( + futures.ThreadPoolExecutor(max_workers=MAX_WORKERS), + options=[ + ("grpc.max_message_length", 64 * 1024 * 1024), + ("grpc.max_send_message_length", 64 * 1024 * 1024), + ("grpc.max_receive_message_length", 64 * 1024 * 1024), + ], + interceptors=get_auth_interceptors(), + ) + backend_pb2_grpc.add_BackendServicer_to_server(BackendServicer(), server) + server.add_insecure_port(address) + server.start() + print(f"MLX Video backend listening on {address}", file=sys.stderr) + + def stop_server(signum, frame): + del signum, frame + server.stop(0) + + signal.signal(signal.SIGINT, stop_server) + signal.signal(signal.SIGTERM, stop_server) + server.wait_for_termination() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Run the MLX Video gRPC backend") + parser.add_argument("--addr", default="localhost:50051") + serve(parser.parse_args().addr) diff --git a/backend/python/mlx-video/install.sh b/backend/python/mlx-video/install.sh new file mode 100755 index 000000000000..0ec541e04882 --- /dev/null +++ b/backend/python/mlx-video/install.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MIT +set -euo pipefail + +USE_PIP=true +PYTHON_VERSION="" +backend_dir=$(dirname "$0") +if [ -d "${backend_dir}/common" ]; then + source "${backend_dir}/common/libbackend.sh" +else + source "${backend_dir}/../common/libbackend.sh" +fi +installRequirements diff --git a/backend/python/mlx-video/mlx_video_utils.py b/backend/python/mlx-video/mlx_video_utils.py new file mode 100644 index 000000000000..5146f6fc46a2 --- /dev/null +++ b/backend/python/mlx-video/mlx_video_utils.py @@ -0,0 +1,102 @@ +# SPDX-License-Identifier: MIT + +import os +import sys + + +def parse_options(options): + parsed = {} + for option in options: + key, separator, value = option.partition(":") + parsed[key] = value if separator else "true" + return parsed + + +def classify_model(model): + normalized = model.rstrip("/").lower() + name = os.path.basename(normalized) + if "ltx-2" in normalized or "ltx2" in normalized: + return "ltx2" + if "wan2" in name or "wan_2" in name: + return "wan" + return None + + +def _append(command, flag, value): + if value is not None and value != "": + command.extend([flag, str(value)]) + + +def build_generation_command( + *, + family, + model, + prompt, + destination, + negative_prompt="", + start_image="", + end_image="", + audio="", + width=0, + height=0, + num_frames=0, + fps=0, + seed=0, + steps=0, + cfg_scale=0, + params=None, +): + params = params or {} + if audio: + raise ValueError("audio conditioning is not supported by mlx-video") + + if family == "ltx2": + command = [sys.executable, "-m", "mlx_video.models.ltx_2.generate"] + _append(command, "--model-repo", model) + _append(command, "--prompt", prompt) + _append(command, "--output-path", destination) + _append(command, "--negative-prompt", negative_prompt) + _append(command, "--image", start_image) + _append(command, "--end-image", end_image) + _append(command, "--width", width or None) + _append(command, "--height", height or None) + _append(command, "--num-frames", num_frames or None) + _append(command, "--fps", fps or None) + _append(command, "--seed", seed if seed > 0 else None) + _append(command, "--steps", steps or None) + _append(command, "--cfg-scale", cfg_scale or None) + for key, flag in { + "pipeline": "--pipeline", + "tiling": "--tiling", + "image_strength": "--image-strength", + "end_image_strength": "--end-image-strength", + "text_encoder_repo": "--text-encoder-repo", + }.items(): + _append(command, flag, params.get(key)) + return command + + if family == "wan": + if end_image: + raise ValueError("Wan models do not support end_image conditioning") + command = [sys.executable, "-m", "mlx_video.models.wan_2.generate"] + _append(command, "--model-dir", model) + _append(command, "--prompt", prompt) + _append(command, "--output-path", destination) + _append(command, "--negative-prompt", negative_prompt) + _append(command, "--image", start_image) + _append(command, "--width", width or None) + _append(command, "--height", height or None) + _append(command, "--num-frames", num_frames or None) + _append(command, "--seed", seed if seed > 0 else None) + _append(command, "--steps", steps or None) + _append(command, "--guide-scale", cfg_scale or None) + for key, flag in { + "scheduler": "--scheduler", + "shift": "--shift", + "tiling": "--tiling", + "trim_first_frames": "--trim-first-frames", + }.items(): + _append(command, flag, params.get(key)) + return command + + raise ValueError(f"unsupported mlx-video model family: {family}") diff --git a/backend/python/mlx-video/requirements-mps.txt b/backend/python/mlx-video/requirements-mps.txt new file mode 100644 index 000000000000..45f2fa4c50e5 --- /dev/null +++ b/backend/python/mlx-video/requirements-mps.txt @@ -0,0 +1 @@ +git+https://github.com/Blaizzy/mlx-video.git@87db56a51758fefb748a359b90a5283bb8ba4837 diff --git a/backend/python/mlx-video/requirements.txt b/backend/python/mlx-video/requirements.txt new file mode 100644 index 000000000000..fe67cdb50ebe --- /dev/null +++ b/backend/python/mlx-video/requirements.txt @@ -0,0 +1,4 @@ +grpcio==1.71.0 +protobuf +certifi +setuptools diff --git a/backend/python/mlx-video/run.sh b/backend/python/mlx-video/run.sh new file mode 100755 index 000000000000..54066119924f --- /dev/null +++ b/backend/python/mlx-video/run.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MIT +set -euo pipefail + +backend_dir=$(dirname "$0") +if [ -d "${backend_dir}/common" ]; then + source "${backend_dir}/common/libbackend.sh" +else + source "${backend_dir}/../common/libbackend.sh" +fi +startBackend "$@" diff --git a/backend/python/mlx-video/test.py b/backend/python/mlx-video/test.py new file mode 100644 index 000000000000..b3f2cf97acfb --- /dev/null +++ b/backend/python/mlx-video/test.py @@ -0,0 +1,86 @@ +# SPDX-License-Identifier: MIT + +import os +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from mlx_video_utils import ( # noqa: E402 + build_generation_command, + classify_model, + parse_options, +) + + +class MLXVideoUtilsTest(unittest.TestCase): + def test_parses_model_options_and_preserves_colons(self): + self.assertEqual( + parse_options(["pipeline:dev", "tiling:aggressive", "flag"]), + {"pipeline": "dev", "tiling": "aggressive", "flag": "true"}, + ) + + def test_classifies_supported_model_families(self): + self.assertEqual(classify_model("Lightricks/LTX-2"), "ltx2") + self.assertEqual(classify_model("/models/Wan2.2-TI2V-5B-mlx"), "wan") + self.assertIsNone(classify_model("some-org/unrelated-model")) + + def test_maps_ltx_request_without_invoking_a_shell(self): + command = build_generation_command( + family="ltx2", + model="prince-canuma/LTX-2.3-distilled-mlx", + prompt="waves; touch /tmp/nope", + destination="/tmp/out.mp4", + negative_prompt="blurry", + start_image="/tmp/start.png", + end_image="/tmp/end.png", + width=768, + height=512, + num_frames=49, + fps=24, + seed=7, + steps=12, + cfg_scale=3.5, + params={"pipeline": "dev", "tiling": "aggressive"}, + ) + + self.assertEqual(command[:3], [sys.executable, "-m", "mlx_video.models.ltx_2.generate"]) + self.assertIn("waves; touch /tmp/nope", command) + self.assertEqual(command[command.index("--model-repo") + 1], "prince-canuma/LTX-2.3-distilled-mlx") + self.assertEqual(command[command.index("--pipeline") + 1], "dev") + self.assertEqual(command[command.index("--cfg-scale") + 1], "3.5") + self.assertIn("--end-image", command) + + def test_maps_wan_request_and_rejects_unsupported_audio(self): + command = build_generation_command( + family="wan", + model="/models/wan22-mlx", + prompt="a fox", + destination="/tmp/out.mp4", + width=1280, + height=704, + num_frames=81, + fps=16, + seed=9, + steps=20, + cfg_scale=4.0, + params={"scheduler": "unipc", "shift": "5.0"}, + ) + + self.assertEqual(command[:3], [sys.executable, "-m", "mlx_video.models.wan_2.generate"]) + self.assertEqual(command[command.index("--model-dir") + 1], "/models/wan22-mlx") + self.assertEqual(command[command.index("--guide-scale") + 1], "4.0") + self.assertNotIn("--fps", command) + + with self.assertRaisesRegex(ValueError, "audio conditioning"): + build_generation_command( + family="wan", + model="/models/wan22-mlx", + prompt="a fox", + destination="/tmp/out.mp4", + audio="/tmp/input.wav", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/backend/python/mlx-video/test.sh b/backend/python/mlx-video/test.sh new file mode 100755 index 000000000000..32f5fd0fb9aa --- /dev/null +++ b/backend/python/mlx-video/test.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MIT +set -euo pipefail + +backend_dir=$(dirname "$0") +if [ -d "${backend_dir}/common" ]; then + source "${backend_dir}/common/libbackend.sh" +else + source "${backend_dir}/../common/libbackend.sh" +fi +runUnittests diff --git a/core/config/backend_capabilities.go b/core/config/backend_capabilities.go index 50ace9b3634f..e2d47bc2ed5f 100644 --- a/core/config/backend_capabilities.go +++ b/core/config/backend_capabilities.go @@ -382,6 +382,13 @@ var BackendCapabilities = map[string]BackendCapability{ AcceptsAudios: true, Description: "LongCat-Video — text, image, and audio-conditioned avatar video generation on NVIDIA CUDA", }, + "mlx-video": { + GRPCMethods: []GRPCMethod{MethodGenerateVideo}, + PossibleUsecases: []string{UsecaseVideo}, + DefaultUsecases: []string{UsecaseVideo}, + AcceptsImages: true, + Description: "MLX-Video — LTX-2 and Wan video generation on Apple Silicon", + }, "stablediffusion": { GRPCMethods: []GRPCMethod{MethodGenerateImage}, PossibleUsecases: []string{UsecaseImage}, diff --git a/docs/content/features/backends.md b/docs/content/features/backends.md index 4aebf97ac1bc..6338ee49085d 100644 --- a/docs/content/features/backends.md +++ b/docs/content/features/backends.md @@ -172,7 +172,7 @@ LocalAI supports various types of backends: - **Text-to-Speech Backends**: For speech synthesis (e.g., piper, Kokoro, VibeVoice, Qwen3-TTS, [NeMo-Speech.cpp]({{%relref "features/nemo-speech-cpp" %}}), [audio.cpp]({{%relref "features/audio-cpp" %}})) - **Sound Generation Backends**: For music and audio generation (e.g., ACE-Step, [audio.cpp]({{%relref "features/audio-cpp" %}})) - **Sound Classification Backends**: For sound-event classification / audio tagging - identifying everyday sounds like baby cry, glass breaking, alarms (e.g., ced.cpp) -- **Image & Video Generation Backends**: For diffusion and audio-conditioned avatar models (e.g., stable-diffusion.cpp, diffusers, vLLM-Omni, [LongCat-Video]({{%relref "features/video-generation" %}}), [vllm.cpp / MiniMax-H3]({{%relref "features/video-generation" %}})) +- **Image & Video Generation Backends**: For diffusion and audio-conditioned avatar models (e.g., stable-diffusion.cpp, diffusers, vLLM-Omni, [MLX-Video on Apple Silicon]({{%relref "features/video-generation" %}}), [LongCat-Video]({{%relref "features/video-generation" %}}), [vllm.cpp / MiniMax-H3]({{%relref "features/video-generation" %}})) - **3D Generation Backends**: For image-to-3D mesh generation ([trellis2.cpp]({{%relref "features/3d-generation" %}}) — Microsoft TRELLIS.2, producing GLB assets with PBR textures) - **Vision & Detection Backends**: For object detection, segmentation, depth, and face/voice recognition (e.g., rf-detr.cpp, locate-anything.cpp, sam3.cpp, insightface) - **Audio Processing Backends**: For voice activity detection and audio enhancement (e.g., Silero VAD, LocalVQE, [audio.cpp]({{%relref "features/audio-cpp" %}})) diff --git a/docs/content/features/video-generation.md b/docs/content/features/video-generation.md index db2f39a70ac9..d514d62df81b 100644 --- a/docs/content/features/video-generation.md +++ b/docs/content/features/video-generation.md @@ -6,7 +6,32 @@ url = "/features/video-generation/" aliases = ["/features/longcat-video/"] +++ -LocalAI can generate videos from text prompts and optional image or audio conditioning via the `/video` endpoint. Supported backends include `diffusers`, `stablediffusion`, `vllm-omni`, `vllm-cpp` (MiniMax-H3, which generates video **and** audio together), and the dedicated `longcat-video` backend. +LocalAI can generate videos from text prompts and optional image or audio conditioning via the `/video` endpoint. Supported backends include `diffusers`, `stablediffusion`, `vllm-omni`, `vllm-cpp` (MiniMax-H3, which generates video **and** audio together), `mlx-video` on Apple Silicon, and the dedicated `longcat-video` backend. + +## MLX-Video on Apple Silicon + +Install the native Metal backend, then configure an LTX-2 repository: + +```bash +local-ai backends install mlx-video +``` + +```yaml +name: ltx2-mlx +backend: mlx-video +parameters: + model: prince-canuma/LTX-2.3-distilled-mlx +usecases: + - video +options: + - pipeline:distilled +``` + +The backend accepts text-to-video and start-image conditioning through `/video`. +LTX-2 also accepts an `end_image`. Converted Wan2.1/Wan2.2 checkpoints must be +configured as a local model directory and do not support `end_image`. Useful +backend-specific `params` include `pipeline` and `tiling` for LTX-2, and +`scheduler`, `shift`, and `tiling` for Wan. ## API diff --git a/docs/content/reference/compatibility-table.md b/docs/content/reference/compatibility-table.md index 28e8c2a1decb..98bbd6ea0489 100644 --- a/docs/content/reference/compatibility-table.md +++ b/docs/content/reference/compatibility-table.md @@ -94,6 +94,7 @@ All backends listed here can be installed on demand from the [Backend Gallery]({ |---------|-------------|-------------| | [stable-diffusion.cpp](https://github.com/leejet/stable-diffusion.cpp) | Stable Diffusion, Flux, PhotoMaker, Ideogram in C/C++ | CPU, CUDA 12/13, Intel SYCL, Vulkan, Metal, Jetson L4T | | [diffusers](https://github.com/huggingface/diffusers) | HuggingFace diffusion models (image and video generation) | CPU, CUDA 12/13, ROCm, Intel SYCL, Metal, Jetson L4T | +| [mlx-video](https://github.com/Blaizzy/mlx-video) | LTX-2 and converted Wan2.1/Wan2.2 video generation | Metal (Apple Silicon) | | [vLLM Omni](https://github.com/vllm-project/vllm-omni) | Multimodal generation including text-to-image and text-to-video | CUDA 12/13, ROCm, Jetson L4T | ## Vision, Detection & Recognition diff --git a/examples/mlx-video/README.md b/examples/mlx-video/README.md new file mode 100644 index 000000000000..9890258b262d --- /dev/null +++ b/examples/mlx-video/README.md @@ -0,0 +1,26 @@ +# MLX-Video on Apple Silicon + +Install the backend and start LocalAI with the example model configuration: + +```bash +local-ai backends install mlx-video +local-ai run --models-path ./examples/mlx-video +``` + +Generate a short video: + +```bash +curl http://localhost:8080/video \ + -H 'Content-Type: application/json' \ + -d '{ + "model": "ltx2-mlx", + "prompt": "Ocean waves at golden hour, locked camera", + "width": 768, + "height": 512, + "num_frames": 49, + "fps": 24 + }' +``` + +MLX-Video and its models require substantial unified memory. Reduce the canvas +and frame count first if generation exhausts memory. diff --git a/examples/mlx-video/ltx2-mlx.yaml b/examples/mlx-video/ltx2-mlx.yaml new file mode 100644 index 000000000000..6d1cbc22a529 --- /dev/null +++ b/examples/mlx-video/ltx2-mlx.yaml @@ -0,0 +1,8 @@ +name: ltx2-mlx +backend: mlx-video +parameters: + model: prince-canuma/LTX-2.3-distilled-mlx +usecases: + - video +options: + - pipeline:distilled From 72625523d2661ce86d652c97d8bc398482c29867 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> Date: Mon, 10 Aug 2026 14:06:35 +0000 Subject: [PATCH 2/2] fix(mlx-video): use supported Python on Darwin The pinned mlx-video package requires Python 3.11 or newer, while an empty PYTHON_VERSION selected the backend helper default of 3.10. Pin the available 3.11.13 portable runtime for the Darwin package build. Assisted-by: Codex:gpt-5 --- backend/python/mlx-video/install.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/python/mlx-video/install.sh b/backend/python/mlx-video/install.sh index 0ec541e04882..747f3f7ed848 100755 --- a/backend/python/mlx-video/install.sh +++ b/backend/python/mlx-video/install.sh @@ -3,7 +3,8 @@ set -euo pipefail USE_PIP=true -PYTHON_VERSION="" +PYTHON_VERSION="3.11" +PYTHON_PATCH="13" backend_dir=$(dirname "$0") if [ -d "${backend_dir}/common" ]; then source "${backend_dir}/common/libbackend.sh"