feat(mlx): add Apple Silicon video backend

.github/backend-matrix.yml

@@ -6311,6 +6311,9 @@ includeDarwin:
   - backend: "mlx"
     tag-suffix: "-metal-darwin-arm64-mlx"
     build-type: "mps"
+  - backend: "mlx-video"
+    tag-suffix: "-metal-darwin-arm64-mlx-video"
+    build-type: "mps"
   - backend: "chatterbox"
     tag-suffix: "-metal-darwin-arm64-chatterbox"
     build-type: "mps"

Makefile

@@ -1,5 +1,5 @@
 # Disable parallel execution for backend builds
-.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/bonsai backends/outetts backends/piper backends/stablediffusion-ggml backends/trellis2cpp backends/trellis2cpp-darwin backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/nemo-speech-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/valkey-store backends/cloud-proxy backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/moss-tts-cpp backends/magpie-tts-cpp backends/vllm-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin backends/audio-cpp backends/audio-cpp-darwin
+.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/bonsai backends/outetts backends/piper backends/stablediffusion-ggml backends/trellis2cpp backends/trellis2cpp-darwin backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/nemo-speech-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/valkey-store backends/cloud-proxy backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/mlx-video backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/moss-tts-cpp backends/magpie-tts-cpp backends/vllm-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin backends/audio-cpp backends/audio-cpp-darwin
 
 GOCMD=go
 GOTEST=$(GOCMD) test
@@ -599,6 +599,7 @@ prepare-test-extra: protogen-python
 	$(MAKE) -C backend/python/vllm
 	$(MAKE) -C backend/python/vllm-omni
 	$(MAKE) -C backend/python/longcat-video
+	$(MAKE) -C backend/python/mlx-video
 	$(MAKE) -C backend/python/sglang
 	$(MAKE) -C backend/python/vibevoice
 	$(MAKE) -C backend/python/liquid-audio
@@ -631,6 +632,7 @@ test-extra: prepare-test-extra
 	$(MAKE) -C backend/python/vllm test
 	$(MAKE) -C backend/python/vllm-omni test
 	$(MAKE) -C backend/python/longcat-video test
+	$(MAKE) -C backend/python/mlx-video test
 	$(MAKE) -C backend/python/vibevoice test
 	$(MAKE) -C backend/python/liquid-audio test
 	$(MAKE) -C backend/python/moonshine test
@@ -1233,6 +1235,10 @@ backends/mlx:
 	BACKEND=mlx $(MAKE) build-darwin-python-backend
 	./local-ai backends install "ocifile://$(abspath ./backend-images/mlx.tar)"
 
+backends/mlx-video:
+	BACKEND=mlx-video $(MAKE) build-darwin-python-backend
+	./local-ai backends install "ocifile://$(abspath ./backend-images/mlx-video.tar)"
+
 backends/diffuser-darwin:
 	BACKEND=diffusers $(MAKE) build-darwin-python-backend
 	./local-ai backends install "ocifile://$(abspath ./backend-images/diffusers.tar)"

backend/index.yaml

@@ -563,7 +563,7 @@
   capabilities:
     nvidia: "cuda12-rfdetr"
     intel: "intel-rfdetr"
     #amd: "rocm-rfdetr"
     nvidia-l4t: "nvidia-l4t-arm64-rfdetr"
     metal: "metal-rfdetr"
     default: "cpu-rfdetr"
@@ -1060,6 +1060,21 @@
     nvidia-l4t: "nvidia-l4t-mlx"
     nvidia-l4t-cuda-12: "nvidia-l4t-mlx"
     nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-mlx"
+- &mlx-video
+  name: "mlx-video"
+  icon: https://avatars.githubusercontent.com/u/102832242?s=200&v=4
+  urls:
+    - https://github.com/Blaizzy/mlx-video
+  license: MIT
+  description: |
+      Generate videos with LTX-2 and converted Wan2.1/Wan2.2 checkpoints using
+      MLX on Apple Silicon.
+  tags:
+    - text-to-video
+    - image-to-video
+    - MLX
+  capabilities:
+    metal: "metal-mlx-video"
 - &mlx-vlm
   name: "mlx-vlm"
   icon: https://avatars.githubusercontent.com/u/102832242?s=200&v=4
@@ -2164,6 +2179,20 @@
   uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-mlx"
   mirrors:
     - localai/localai-backends:latest-metal-darwin-arm64-mlx
+- !!merge <<: *mlx-video
+  name: "metal-mlx-video"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-mlx-video"
+  mirrors:
+    - localai/localai-backends:latest-metal-darwin-arm64-mlx-video
+- !!merge <<: *mlx-video
+  name: "metal-mlx-video-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-mlx-video"
+  mirrors:
+    - localai/localai-backends:master-metal-darwin-arm64-mlx-video
+- !!merge <<: *mlx-video
+  name: "mlx-video-development"
+  capabilities:
+    metal: "metal-mlx-video-development"
 - !!merge <<: *mlx
   name: "metal-mlx-development"
   uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-mlx"
@@ -4315,7 +4344,7 @@
   capabilities:
     nvidia: "cuda12-rfdetr-development"
     intel: "intel-rfdetr-development"
     #amd: "rocm-rfdetr-development"
     nvidia-l4t: "nvidia-l4t-arm64-rfdetr-development"
     metal: "metal-rfdetr-development"
     default: "cpu-rfdetr-development"

backend/python/mlx-video/.gitignore

@@ -0,0 +1,5 @@
+__pycache__/
+backend_pb2.py
+backend_pb2_grpc.py
+common/
+venv/

backend/python/mlx-video/Makefile

@@ -0,0 +1,16 @@
+.PHONY: all run test protogen-clean clean
+
+all:
+	bash install.sh
+
+run: all
+	bash run.sh
+
+test:
+	bash test.sh
+
+protogen-clean:
+	$(RM) backend_pb2.py backend_pb2_grpc.py
+
+clean: protogen-clean
+	rm -rf __pycache__ venv

backend/python/mlx-video/backend.py

@@ -0,0 +1,169 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: MIT
+
+import argparse
+import os
+import platform
+import signal
+import subprocess
+import sys
+import traceback
+from concurrent import futures
+
+import grpc
+
+import backend_pb2
+import backend_pb2_grpc
+from mlx_video_utils import build_generation_command, classify_model, parse_options
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "common"))
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "common"))
+from grpc_auth import get_auth_interceptors  # noqa: E402
+
+
+MAX_WORKERS = int(os.environ.get("PYTHON_GRPC_MAX_WORKERS", "1"))
+
+
+class BackendServicer(backend_pb2_grpc.BackendServicer):
+    def __init__(self):
+        self.family = None
+        self.model = None
+        self.options = {}
+
+    def Health(self, request, context):
+        return backend_pb2.Reply(message=b"OK")
+
+    def LoadModel(self, request, context):
+        if platform.system() != "Darwin" or platform.machine() != "arm64":
+            return self._fail(
+                context,
+                grpc.StatusCode.FAILED_PRECONDITION,
+                "mlx-video requires macOS on Apple Silicon",
+            )
+
+        model = (
+            request.ModelFile
+            if request.ModelFile and os.path.isdir(request.ModelFile)
+            else request.Model
+        )
+        family = classify_model(model)
+        if family is None:
+            return self._fail(
+                context,
+                grpc.StatusCode.INVALID_ARGUMENT,
+                "mlx-video supports LTX-2 and converted Wan2.1/Wan2.2 MLX checkpoints",
+            )
+        if family == "wan" and not os.path.isdir(model):
+            return self._fail(
+                context,
+                grpc.StatusCode.INVALID_ARGUMENT,
+                "Wan models must be converted to MLX and configured as a local model directory",
+            )
+
+        self.family = family
+        self.model = model
+        self.options = parse_options(request.Options)
+        return backend_pb2.Result(message="mlx-video model configured", success=True)
+
+    def Free(self, request, context):
+        self.family = None
+        self.model = None
+        self.options = {}
+        return backend_pb2.Result(message="Model released", success=True)
+
+    def GenerateVideo(self, request, context):
+        if not self.family or not self.model:
+            return self._fail(
+                context, grpc.StatusCode.FAILED_PRECONDITION, "model is not loaded"
+            )
+        if not request.prompt.strip():
+            return self._fail(context, grpc.StatusCode.INVALID_ARGUMENT, "prompt is required")
+        if not request.dst:
+            return self._fail(
+                context,
+                grpc.StatusCode.INVALID_ARGUMENT,
+                "output destination is required",
+            )
+
+        process = None
+        try:
+            command = build_generation_command(
+                family=self.family,
+                model=self.model,
+                prompt=request.prompt,
+                destination=request.dst,
+                negative_prompt=request.negative_prompt,
+                start_image=request.start_image,
+                end_image=request.end_image,
+                audio=request.audio,
+                width=request.width,
+                height=request.height,
+                num_frames=request.num_frames,
+                fps=request.fps,
+                seed=request.seed,
+                steps=request.step,
+                cfg_scale=request.cfg_scale,
+                params={**self.options, **dict(request.params)},
+            )
+            os.makedirs(
+                os.path.dirname(request.dst) or ".", mode=0o750, exist_ok=True
+            )
+            process = subprocess.Popen(command)
+            if hasattr(context, "add_callback"):
+                context.add_callback(
+                    lambda: process.poll() is None and process.terminate()
+                )
+            return_code = process.wait()
+            if return_code != 0:
+                raise RuntimeError(
+                    f"upstream generator exited with status {return_code}"
+                )
+            if not os.path.isfile(request.dst) or os.path.getsize(request.dst) == 0:
+                raise RuntimeError("upstream generator did not create the requested video")
+            return backend_pb2.Result(message="Video generated successfully", success=True)
+        except ValueError as err:
+            return self._fail(context, grpc.StatusCode.INVALID_ARGUMENT, str(err))
+        except Exception as err:
+            print(f"mlx-video generation failed: {err}", file=sys.stderr)
+            traceback.print_exc()
+            return self._fail(
+                context,
+                grpc.StatusCode.INTERNAL,
+                f"mlx-video generation failed: {err}",
+            )
+
+    @staticmethod
+    def _fail(context, code, message):
+        context.set_code(code)
+        context.set_details(message)
+        return backend_pb2.Result(message=message, success=False)
+
+
+def serve(address):
+    server = grpc.server(
+        futures.ThreadPoolExecutor(max_workers=MAX_WORKERS),
+        options=[
+            ("grpc.max_message_length", 64 * 1024 * 1024),
+            ("grpc.max_send_message_length", 64 * 1024 * 1024),
+            ("grpc.max_receive_message_length", 64 * 1024 * 1024),
+        ],
+        interceptors=get_auth_interceptors(),
+    )
+    backend_pb2_grpc.add_BackendServicer_to_server(BackendServicer(), server)
+    server.add_insecure_port(address)
+    server.start()
+    print(f"MLX Video backend listening on {address}", file=sys.stderr)
+
+    def stop_server(signum, frame):
+        del signum, frame
+        server.stop(0)
+
+    signal.signal(signal.SIGINT, stop_server)
+    signal.signal(signal.SIGTERM, stop_server)
+    server.wait_for_termination()
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Run the MLX Video gRPC backend")
+    parser.add_argument("--addr", default="localhost:50051")
+    serve(parser.parse_args().addr)

backend/python/mlx-video/install.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: MIT
+set -euo pipefail
+
+USE_PIP=true
+PYTHON_VERSION="3.11"
+PYTHON_PATCH="13"
+backend_dir=$(dirname "$0")
+if [ -d "${backend_dir}/common" ]; then
+    source "${backend_dir}/common/libbackend.sh"
+else
+    source "${backend_dir}/../common/libbackend.sh"
+fi
+installRequirements

backend/python/mlx-video/mlx_video_utils.py

@@ -0,0 +1,102 @@
+# SPDX-License-Identifier: MIT
+
+import os
+import sys
+
+
+def parse_options(options):
+    parsed = {}
+    for option in options:
+        key, separator, value = option.partition(":")
+        parsed[key] = value if separator else "true"
+    return parsed
+
+
+def classify_model(model):
+    normalized = model.rstrip("/").lower()
+    name = os.path.basename(normalized)
+    if "ltx-2" in normalized or "ltx2" in normalized:
+        return "ltx2"
+    if "wan2" in name or "wan_2" in name:
+        return "wan"
+    return None
+
+
+def _append(command, flag, value):
+    if value is not None and value != "":
+        command.extend([flag, str(value)])
+
+
+def build_generation_command(
+    *,
+    family,
+    model,
+    prompt,
+    destination,
+    negative_prompt="",
+    start_image="",
+    end_image="",
+    audio="",
+    width=0,
+    height=0,
+    num_frames=0,
+    fps=0,
+    seed=0,
+    steps=0,
+    cfg_scale=0,
+    params=None,
+):
+    params = params or {}
+    if audio:
+        raise ValueError("audio conditioning is not supported by mlx-video")
+
+    if family == "ltx2":
+        command = [sys.executable, "-m", "mlx_video.models.ltx_2.generate"]
+        _append(command, "--model-repo", model)
+        _append(command, "--prompt", prompt)
+        _append(command, "--output-path", destination)
+        _append(command, "--negative-prompt", negative_prompt)
+        _append(command, "--image", start_image)
+        _append(command, "--end-image", end_image)
+        _append(command, "--width", width or None)
+        _append(command, "--height", height or None)
+        _append(command, "--num-frames", num_frames or None)
+        _append(command, "--fps", fps or None)
+        _append(command, "--seed", seed if seed > 0 else None)
+        _append(command, "--steps", steps or None)
+        _append(command, "--cfg-scale", cfg_scale or None)
+        for key, flag in {
+            "pipeline": "--pipeline",
+            "tiling": "--tiling",
+            "image_strength": "--image-strength",
+            "end_image_strength": "--end-image-strength",
+            "text_encoder_repo": "--text-encoder-repo",
+        }.items():
+            _append(command, flag, params.get(key))
+        return command
+
+    if family == "wan":
+        if end_image:
+            raise ValueError("Wan models do not support end_image conditioning")
+        command = [sys.executable, "-m", "mlx_video.models.wan_2.generate"]
+        _append(command, "--model-dir", model)
+        _append(command, "--prompt", prompt)
+        _append(command, "--output-path", destination)
+        _append(command, "--negative-prompt", negative_prompt)
+        _append(command, "--image", start_image)
+        _append(command, "--width", width or None)
+        _append(command, "--height", height or None)
+        _append(command, "--num-frames", num_frames or None)
+        _append(command, "--seed", seed if seed > 0 else None)
+        _append(command, "--steps", steps or None)
+        _append(command, "--guide-scale", cfg_scale or None)
+        for key, flag in {
+            "scheduler": "--scheduler",
+            "shift": "--shift",
+            "tiling": "--tiling",
+            "trim_first_frames": "--trim-first-frames",
+        }.items():
+            _append(command, flag, params.get(key))
+        return command
+
+    raise ValueError(f"unsupported mlx-video model family: {family}")

backend/python/mlx-video/requirements-mps.txt

@@ -0,0 +1 @@
+git+https://github.com/Blaizzy/mlx-video.git@87db56a51758fefb748a359b90a5283bb8ba4837