Add llama-bench-style benchmark: llama-tornado --bench (pp/tg/pg matrix, avg±stddev t/s, md/csv/json)

README.md

@@ -493,6 +493,41 @@ Any OpenAI SDK works by pointing `base_url` at the server. Generation is seriali
 context (requests queue); the reusable core is `server/InferenceService`. Smoke test:
 `scripts/server-smoke-test.sh http://localhost:8080`.
 
+## 📊 Benchmarking (`--bench`, llama-bench style)
+
+A [llama-bench](https://github.com/ggml-org/llama.cpp/tree/master/tools/llama-bench)-equivalent
+benchmark for the GPU forward path: a cartesian matrix of tests over one or more models —
+prompt processing (`pp N`), token generation (`tg N`) and combined (`pg`) — each repeated
+`-r` times after an untimed warmup, reported as **average tokens/s ± stddev**. Timings cover
+the forward pass only (no tokenization, no sampling), matching llama-bench methodology.
+
+```bash
+# defaults: -p 512 -n 128 -r 5, markdown output
+llama-tornado --gpu --cuda --model model1.gguf --bench
+
+# multiple models, custom matrix, CSV
+llama-tornado --gpu --cuda --model model1.gguf --bench \
+    --bench-args="-m model2.gguf -p 256,512 -n 64,128 -pg 512,128 -d 0,4096 -r 5 -o csv"
+```
+
+Benchmark options (`--bench-args="..."` — use the `=` form): `-m` extra models
+(comma/repeatable), `-p` prompt sizes, `-n` generation lengths, `-pg pp,tg` combined tests,
+`-b` prompt-processing batch size — when >1, `pp` tokens run through the batched-prefill
+tensor-core (MMA) path, `b` tokens per forward (compute-bound, the llama-bench `-b`);
+generation stays single-token decode. Supported models: Llama / Qwen3 / Mistral (FP16).
+`-d` context depths (untimed KV prefill of `d` positions before each timed test, e.g.
+`tg128@d4096`), `-r` repetitions, `-o md|csv|json|jsonl|sql`, `-oe <fmt>` second format to
+stderr, `--delay <s>` pause between tests, `--no-warmup`.
+
+Example output (RTX 4090, CUDA backend):
+
+| model | quant | size | params | backend | test | t/s |
+| ----- | ----- | ---: | -----: | ------- | ---- | --: |
+| beehive-llama-3.2-1b-instruct-fp16 | FP16 | 2.31 GiB | 1.24 B | TornadoVM CUDA | pp512 | 86.62 ± 2.18 |
+| beehive-llama-3.2-1b-instruct-fp16 | FP16 | 2.31 GiB | 1.24 B | TornadoVM CUDA | tg128 | 101.72 ± 0.43 |
+| Qwen3-1.7B-f16 | FP16 | 3.79 GiB | 2.03 B | TornadoVM CUDA | pp512 | 57.29 ± 0.18 |
+| Qwen3-1.7B-f16 | FP16 | 3.79 GiB | 2.03 B | TornadoVM CUDA | tg128 | 57.71 ± 0.10 |
+
 ## Debug & Profiling Options
 View TornadoVM's internal behavior:
 ```bash

llama-tornado

@@ -196,13 +196,18 @@ class LlamaRunner:
                 ]
             )
 
+        if getattr(args, "server", False):
+            main_class = "org.beehive.gpullama3.server.OpenAIServer"
+        elif getattr(args, "bench", False):
+            main_class = "org.beehive.gpullama3.bench.LlamaBench"
+        else:
+            main_class = "org.beehive.gpullama3.LlamaApp"
+
         module_config.extend(
             [
                 "-cp",
                 self._find_llama_jar(),
-                "org.beehive.gpullama3.server.OpenAIServer"
-                if getattr(args, "server", False)
-                else "org.beehive.gpullama3.LlamaApp",
+                main_class,
             ]
         )
         cmd.extend(module_config)
@@ -235,6 +240,12 @@ class LlamaRunner:
             if args.use_gpu:
                 srv.append("--gpu")
             return cmd + srv
+        if getattr(args, "bench", False):
+            # llama-bench mode: model(s) + benchmark options only. If --bench-args already
+            # supplies -m, use those verbatim (don't also prepend --model → duplicate rows).
+            extra = args.bench_args.split() if args.bench_args else []
+            bench_args = extra if ("-m" in extra or "--model" in extra) else ["-m", args.model_path] + extra
+            return cmd + bench_args
         llama_args = [
             "-m",
             args.model_path,
@@ -426,6 +437,24 @@ def create_parser() -> argparse.ArgumentParser:
         help="Run in instruction mode (default)",
     )
 
+    # OpenAI-compatible server
+    server_group = parser.add_argument_group("OpenAI-compatible server")
+    server_group.add_argument("--server", action="store_true", help="Run the OpenAI-compatible HTTP server instead of inference")
+    server_group.add_argument("--port", type=int, default=8080, help="Server port (default 8080)")
+
+    # Benchmark (llama-bench style)
+    bench_group = parser.add_argument_group("Benchmark (llama-bench style)")
+    bench_group.add_argument(
+        "--bench",
+        action="store_true",
+        help="Run the llama-bench-style benchmark (bench.LlamaBench) instead of inference",
+    )
+    bench_group.add_argument(
+        "--bench-args",
+        default="",
+        help='Extra benchmark options (use --bench-args="..."), e.g. "-p 512 -n 128 -d 0,4096 -r 5 -o md" (see LlamaBench)',
+    )
+
     # Hardware configuration
     hw_group = parser.add_argument_group("Hardware Configuration")
     hw_group.add_argument(
@@ -475,11 +504,6 @@ def create_parser() -> argparse.ArgumentParser:
         help="Directory for profiler output",
     )
 
-    # OpenAI-compatible server
-    server_group = parser.add_argument_group("OpenAI-compatible server")
-    server_group.add_argument("--server", action="store_true", help="Run the OpenAI-compatible HTTP server instead of inference")
-    server_group.add_argument("--port", type=int, default=8080, help="Server port (default 8080)")
-
     # TornadoVM Execution Verbose options
     verbose_group = parser.add_argument_group("TornadoVM Execution Verbose")
     verbose_group.add_argument(

llamaTornado

@@ -17,7 +17,8 @@ record Config(
     boolean printBytecodes, boolean threads, boolean printKernel,
     boolean fullDump, boolean verboseInit,
     boolean showCommand, boolean executeAfterShow,
-    String openclFlags, int maxWaitEvents, boolean verbose
+    String openclFlags, int maxWaitEvents, boolean verbose,
+    boolean bench, String benchArgs
 ) {}
 
 Config parseArgs(String[] args) {
@@ -51,6 +52,8 @@ Config parseArgs(String[] args) {
     String openclFlags = "-cl-denorms-are-zero -cl-no-signed-zeros -cl-finite-math-only";
     int maxWaitEvents = 32000;
     boolean verbose = false;
+    boolean bench = false;
+    String benchArgs = "";
 
     for (int i = 0; i < args.length; i++) {
         switch (args[i]) {
@@ -86,6 +89,8 @@ Config parseArgs(String[] args) {
             case "--opencl-flags" -> openclFlags = args[++i];
             case "--max-wait-events" -> maxWaitEvents = Integer.parseInt(args[++i]);
             case "--verbose", "-v" -> verbose = true;
+            case "--bench" -> bench = true;
+            case "--bench-args" -> benchArgs = args[++i];
             default -> {
                 System.err.println("Unknown option: " + args[i]);
                 System.exit(1);
@@ -111,7 +116,8 @@ Config parseArgs(String[] args) {
     return new Config(modelPath, prompt, systemPrompt, temperature, topP, seed, maxTokens,
             stream, echo, interactive, instruct, useGpu, backend, gpuMemory, heapMin, heapMax, directMemory,
             debug, profiler, profilerDumpDir, printBytecodes, threads, printKernel, fullDump,
-            verboseInit, showCommand, executeAfterShow, openclFlags, maxWaitEvents, verbose);
+            verboseInit, showCommand, executeAfterShow, openclFlags, maxWaitEvents, verbose,
+            bench, benchArgs);
 }
 
 String parseAndScale(String memoryValue, int multiplier) {
@@ -170,6 +176,10 @@ void printUsage() {
               --execute-after-show    Execute after showing command
               --verbose, -v           Verbose output
 
+            Benchmark (llama-bench style):
+              --bench                 Run the llama-bench-style benchmark instead of inference
+              --bench-args <str>      Extra benchmark options, e.g. "-p 512 -n 128 -r 5 -o md"
+
               -help                   Show this help
               -version                Show version
             """.formatted(name));
@@ -281,7 +291,25 @@ List<String> buildCommand(Config cfg, String javaHome, String tornadoSdk, String
         }
     }
 
-    cmd.addAll(List.of("-cp", findLlamaJar(llamaRoot), "org.beehive.gpullama3.LlamaApp"));
+    String mainClass = cfg.bench()
+        ? "org.beehive.gpullama3.bench.LlamaBench"
+        : "org.beehive.gpullama3.LlamaApp";
+    cmd.addAll(List.of("-cp", findLlamaJar(llamaRoot), mainClass));
+
+    if (cfg.bench()) {
+        // llama-bench mode: model(s) + benchmark options only. If --bench-args already
+        // supplies -m, use those verbatim (don't also prepend --model → duplicate rows).
+        var extra = cfg.benchArgs() == null || cfg.benchArgs().isBlank()
+            ? List.<String>of()
+            : List.of(cfg.benchArgs().trim().split("\\s+"));
+        if (extra.contains("-m") || extra.contains("--model")) {
+            cmd.addAll(extra);
+        } else {
+            cmd.addAll(List.of("-m", cfg.modelPath()));
+            cmd.addAll(extra);
+        }
+        return cmd;
+    }
 
     // LLaMA arguments
     cmd.addAll(List.of(