OpenAI-compatible server (llama-tornado --server): /v1/chat/completions + /v1/completions, streaming, zero-dep

README.md

@@ -465,6 +465,32 @@ Advanced Options:
 
 ```
 
+## 🔌 OpenAI-compatible server (`--server`)
+
+Serve the model behind the HTTP API OpenAI clients already speak — no external dependencies
+(JDK `HttpServer`), streaming (SSE) and non-streaming.
+
+```bash
+llama-tornado --gpu --cuda --model model.gguf --server --port 8080
+# or directly:
+java ... org.beehive.gpullama3.server.OpenAIServer --model model.gguf --port 8080 --gpu
+```
+
+Endpoints: `POST /v1/chat/completions`, `POST /v1/completions`, `GET /v1/models`, `GET /health`.
+
+```bash
+curl http://localhost:8080/v1/chat/completions -H 'Content-Type: application/json' \
+  -d '{"messages":[{"role":"user","content":"Capital of France?"}],"max_tokens":16}'
+
+# streaming
+curl -N http://localhost:8080/v1/chat/completions -H 'Content-Type: application/json' \
+  -d '{"messages":[{"role":"user","content":"Tell me a joke"}],"stream":true}'
+```
+
+Any OpenAI SDK works by pointing `base_url` at the server. Generation is serialized on one GPU
+context (requests queue); the reusable core is `server/InferenceService`. Smoke test:
+`scripts/server-smoke-test.sh http://localhost:8080`.
+
 ## Debug & Profiling Options
 View TornadoVM's internal behavior:
 ```bash

llama-tornado

@@ -200,7 +200,9 @@ class LlamaRunner:
             [
                 "-cp",
                 self._find_llama_jar(),
-                "org.beehive.gpullama3.LlamaApp",
+                "org.beehive.gpullama3.server.OpenAIServer"
+                if getattr(args, "server", False)
+                else "org.beehive.gpullama3.LlamaApp",
             ]
         )
         cmd.extend(module_config)
@@ -227,6 +229,12 @@ class LlamaRunner:
 
     def _add_llama_args(self, cmd: List[str], args: argparse.Namespace) -> List[str]:
         """Add LLaMA-specific arguments to the command."""
+        if getattr(args, "server", False):
+            # OpenAI server mode: the server parses only --model / --port / --gpu.
+            srv = ["--model", args.model_path, "--port", str(getattr(args, "port", 8080))]
+            if args.use_gpu:
+                srv.append("--gpu")
+            return cmd + srv
         llama_args = [
             "-m",
             args.model_path,
@@ -467,6 +475,11 @@ def create_parser() -> argparse.ArgumentParser:
         help="Directory for profiler output",
     )
 
+    # OpenAI-compatible server
+    server_group = parser.add_argument_group("OpenAI-compatible server")
+    server_group.add_argument("--server", action="store_true", help="Run the OpenAI-compatible HTTP server instead of inference")
+    server_group.add_argument("--port", type=int, default=8080, help="Server port (default 8080)")
+
     # TornadoVM Execution Verbose options
     verbose_group = parser.add_argument_group("TornadoVM Execution Verbose")
     verbose_group.add_argument(

scripts/server-smoke-test.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+# Smoke test for the GPULlama3 OpenAI-compatible server.
+# Usage: scripts/server-smoke-test.sh [host]   (default http://localhost:8080)
+# Exits non-zero on the first failed check.
+set -u
+H="${1:-http://localhost:8080}"
+pass=0; fail=0
+check() { # name, condition(0/1)
+  if [ "$2" -eq 0 ]; then echo "  ok   $1"; pass=$((pass+1)); else echo "  FAIL $1"; fail=$((fail+1)); fi
+}
+j() { python3 -c "import json,sys; d=json.load(sys.stdin); print($1)"; }
+
+echo "== health =="
+curl -sf "$H/health" | grep -q '"status":"ok"'; check "GET /health" $?
+
+echo "== models =="
+curl -sf "$H/v1/models" | grep -q '"object":"list"'; check "GET /v1/models" $?
+
+echo "== chat completion (non-stream) =="
+R=$(curl -sf "$H/v1/chat/completions" -H 'Content-Type: application/json' \
+  -d '{"messages":[{"role":"user","content":"Capital of France? One word."}],"max_tokens":10,"temperature":0}')
+echo "$R" | j "d['choices'][0]['message']['content']" | grep -qi paris; check "chat returns Paris" $?
+echo "$R" | grep -q '"total_tokens"'; check "usage present" $?
+
+echo "== text completion =="
+curl -sf "$H/v1/completions" -H 'Content-Type: application/json' \
+  -d '{"prompt":"The capital of France is","max_tokens":8,"temperature":0}' \
+  | j "d['choices'][0]['text']" | grep -qi paris; check "completion returns Paris" $?
+
+echo "== streaming (SSE) =="
+S=$(curl -sfN "$H/v1/chat/completions" -H 'Content-Type: application/json' \
+  -d '{"messages":[{"role":"user","content":"Say hi."}],"max_tokens":6,"temperature":0,"stream":true}')
+echo "$S" | grep -q 'chat.completion.chunk'; check "SSE emits chunks" $?
+echo "$S" | grep -q 'data: \[DONE\]'; check "SSE terminates with [DONE]" $?
+
+echo "== errors =="
+[ "$(curl -s -o /dev/null -w '%{http_code}' "$H/v1/chat/completions" -d '{bad')" = 400 ]; check "400 on bad JSON" $?
+[ "$(curl -s -o /dev/null -w '%{http_code}' "$H/v1/chat/completions" -d '{}')" = 400 ]; check "400 on missing messages" $?
+[ "$(curl -s -o /dev/null -w '%{http_code}' "$H/v1/chat/completions")" = 405 ]; check "405 on GET" $?
+
+echo "== concurrency (5 parallel) =="
+rm -f /tmp/smoke_*.done
+for i in 1 2 3 4 5; do
+  ( curl -sf "$H/v1/chat/completions" -d '{"messages":[{"role":"user","content":"Hi"}],"max_tokens":4,"temperature":0}' \
+      -o "/tmp/smoke_$i.json" && touch "/tmp/smoke_$i.done" ) &
+done; wait
+[ "$(ls /tmp/smoke_*.done 2>/dev/null | wc -l)" -eq 5 ]; check "5 concurrent requests all succeed" $?
+
+echo
+echo "passed=$pass failed=$fail"
+[ "$fail" -eq 0 ]

src/main/java/org/beehive/gpullama3/server/InferenceService.java

@@ -0,0 +1,108 @@
+package org.beehive.gpullama3.server;
+
+import org.beehive.gpullama3.inference.sampler.Sampler;
+import org.beehive.gpullama3.inference.state.State;
+import org.beehive.gpullama3.model.Model;
+import org.beehive.gpullama3.model.format.ChatFormat;
+import org.beehive.gpullama3.tornadovm.TornadoVMMasterPlan;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.function.IntConsumer;
+
+/**
+ * Reusable, thread-safe inference wrapper over a single loaded {@link Model}.
+ *
+ * <p>Holds one {@link State} and one {@link TornadoVMMasterPlan} (the expensive, compile-once GPU
+ * task graph) and serializes generation on them — the GPU is a single context, so requests run
+ * one at a time. This is the reuse seam shared by the OpenAI server and any other front-end:
+ * build the prompt, stream decoded tokens through a callback, return the full text. The single
+ * proven single-token decode path is used verbatim (KV cache is overwritten from position 0 each
+ * request, so state is safely reused without reallocation).</p>
+ */
+public final class InferenceService {
+
+    private final Model model;
+    private final State state;
+    private final TornadoVMMasterPlan plan;
+    private final boolean gpu;
+    private final int initialToken;   // model start/BOS token that createNewState() seeds
+    private final Object lock = new Object();
+
+    public InferenceService(Model model, boolean gpu) {
+        this.model = model;
+        this.gpu = gpu;
+        this.state = model.createNewState();
+        // createNewState() seeds latestToken with the model's start token (BOS / header) — the
+        // first forward pass consumes it. Capture it so each request can reset to a clean start.
+        this.initialToken = state.latestToken;
+        this.plan = gpu ? TornadoVMMasterPlan.initializeTornadoVMPlan(state, model) : null;
+    }
+
+    public Model model() {
+        return model;
+    }
+
+    /** A single generation request. {@code messages} are role/content turns (chat); a lone
+     *  user turn covers the completions endpoint. */
+    public record Request(List<ChatFormat.Message> messages, int maxTokens, float temperature, float topP, long seed) {}
+
+    /** Result of a generation: the text and the token counts (for {@code usage}). */
+    public record Result(String text, int promptTokens, int completionTokens, boolean stopped) {}
+
+    /**
+     * Generate a completion. If {@code onToken} is non-null each decoded token's text is streamed
+     * to it as it is produced; the full text is always returned. Thread-safe (serialized).
+     */
+    public Result generate(Request req, java.util.function.Consumer<String> onToken) {
+        synchronized (lock) {
+            ChatFormat cf = model.chatFormat();
+            List<Integer> promptTokens = new ArrayList<>();
+            if (model.shouldAddBeginOfText()) {
+                promptTokens.add(cf.getBeginOfText());
+            }
+            for (ChatFormat.Message m : req.messages()) {
+                promptTokens.addAll(cf.encodeMessage(m));
+            }
+            promptTokens.addAll(cf.encodeHeader(new ChatFormat.Message(ChatFormat.Role.ASSISTANT, "")));
+            if (model.shouldIncludeReasoning()) {
+                promptTokens.addAll(model.tokenizer().encode("<think>\n", model.tokenizer().getSpecialTokens().keySet()));
+            }
+
+            int maxTokens = req.maxTokens() > 0 ? promptTokens.size() + req.maxTokens() : 0;
+            Sampler sampler = Sampler.selectSampler(model.configuration().vocabularySize(),
+                    req.temperature(), req.topP(), req.seed());
+            Set<Integer> stopTokens = cf.getStopTokens();
+
+            StringBuilder full = new StringBuilder();
+            IntConsumer tokenConsumer = token -> {
+                if (model.tokenizer().shouldDisplayToken(token)) {
+                    String piece = model.tokenizer().decode(List.of(token));
+                    full.append(piece);
+                    if (onToken != null) {
+                        onToken.accept(piece);
+                    }
+                }
+            };
+
+            // Reset to the fresh-state convention: model start token + KV overwritten from pos 0.
+            state.latestToken = initialToken;
+
+            List<Integer> responseTokens = gpu
+                    ? model.generateTokensGPU(state, 0, promptTokens, stopTokens, maxTokens, sampler, false, tokenConsumer, plan)
+                    : model.generateTokens(state, 0, promptTokens, stopTokens, maxTokens, sampler, false, tokenConsumer);
+
+            boolean stopped = !responseTokens.isEmpty() && stopTokens.contains(responseTokens.getLast());
+            int completion = responseTokens.size() - (stopped ? 1 : 0);
+            return new Result(full.toString(), promptTokens.size(), Math.max(0, completion), stopped);
+        }
+    }
+
+    /** Free the GPU plan (call on server shutdown). */
+    public void close() {
+        if (plan != null) {
+            plan.freeTornadoExecutionPlan();
+        }
+    }
+}