Show HN: Jinfer – AI inference engine for the JVM. AI in a jar

Some call AI on the JVM quixotic.

We call it Quixotic AI.

Finally, AI in a jar.

Every component of the AI stack, a la carte.

jbang header + imports

///usr/bin/env jbang "$0" "$@" ; exit $?

//JAVA 25+

//RUNTIME_OPTIONS --add-modules jdk.incubator.vector --enable-native-access=ALL-UNNAMED

//DEPS com.qxotic:jinfer-bom:0.2.0@pom

//DEPS com.qxotic:jinfer-spring-ai com.qxotic:jinfer-models-all

//DEPS com.qxotic:jam-native com.qxotic:jam-vector

//DEPS org.slf4j:slf4j-nop:2.0.17

import com.qxotic.jinfer.spring.ai.*;

void main() {

try (var model = JinferChatModel.builder()

.model("LiquidAI/LFM2.5-350M-GGUF:Q8_0")

.build()) {

System.out.println(model.call("What is the capital of France?"));

}

}$ jbang Chat.java # "The capital of France is Paris."

jbang header + imports

///usr/bin/env jbang "$0" "$@" ; exit $?

//JAVA 25+

//RUNTIME_OPTIONS --add-modules jdk.incubator.vector --enable-native-access=ALL-UNNAMED

//DEPS com.qxotic:jinfer-bom:0.2.0@pom

//DEPS com.qxotic:jinfer-spring-ai

//DEPS com.qxotic:jinfer-kokoro

//DEPS com.qxotic:jam-native com.qxotic:jam-vector

//DEPS org.slf4j:slf4j-nop:2.0.17

import com.qxotic.jinfer.spring.ai.JinferSpeechModel;

import java.nio.file.*;

void main() throws Exception {

try (var tts = JinferSpeechModel.builder()

.model("simonfxr/kokoro.cpp-GGUF:Q8_0")

.companion("voice", "simonfxr/kokoro.cpp-GGUF/voices/kokoro-voice-af_heart.gguf")

.build()) {

var line = "Turns out, matrix multiplications can talk. "

+ "The JVM just crunched a few billion numbers to say this.";

Files.write(Path.of("kokoro.wav"), tts.call(line));

}

}$ jbang TextToSpeech.java # kokoro.wav · seven seconds, in a warm voice · Kokoro-82M

jbang header + imports

///usr/bin/env jbang "$0" "$@" ; exit $?

//JAVA 25+

//RUNTIME_OPTIONS --add-modules jdk.incubator.vector --enable-native-access=ALL-UNNAMED

//DEPS com.qxotic:jinfer-bom:0.2.0@pom

//DEPS com.qxotic:jinfer-spring-ai com.qxotic:jinfer-models-all

//DEPS org.springframework.ai:spring-ai-client-chat:2.0.1

//DEPS com.qxotic:jam-native com.qxotic:jam-vector

//DEPS org.slf4j:slf4j-nop:2.0.17

import com.qxotic.jinfer.spring.ai.*;

import org.springframework.ai.chat.client.ChatClient;

import org.springframework.core.io.UrlResource;

import org.springframework.util.MimeType;

void main() throws Exception {

try (var gemma = JinferChatModel.builder()

.model("unsloth/gemma-4-E2B-it-GGUF:Q4_K_M")

.companion("media", "unsloth/gemma-4-E2B-it-GGUF/mmproj-BF16.gguf")

.build()) {

// jfk.wav: JFK's own voice, inaugural address, January 20, 1961

var speech = new UrlResource("https://qxotic.ai/snippets/jfk.wav");

var transcript = ChatClient.create(gemma).prompt()

.user(u -> u.text("Transcribe this recording.")

.media(MimeType.valueOf("audio/wav"), speech))

.call().content();

System.out.println(transcript);

}

}$ jbang Audio.java # "ask not what your country can do for you ..." · every word heard

jbang header + imports

///usr/bin/env jbang "$0" "$@" ; exit $?

//JAVA 25+

//RUNTIME_OPTIONS --add-modules jdk.incubator.vector --enable-native-access=ALL-UNNAMED

//DEPS com.qxotic:jinfer-bom:0.2.0@pom

//DEPS com.qxotic:jinfer-spring-ai com.qxotic:jinfer-models-all

//DEPS org.springframework.ai:spring-ai-client-chat:2.0.1

//DEPS com.qxotic:jam-native com.qxotic:jam-vector

//DEPS org.slf4j:slf4j-nop:2.0.17

import com.qxotic.jinfer.spring.ai.*;

import org.springframework.ai.chat.client.ChatClient;

import org.springframework.core.io.UrlResource;

import org.springframework.util.MimeTypeUtils;

void main() throws Exception {

try (var vision = JinferChatModel.builder()

.model("LiquidAI/LFM2.5-VL-3B-GGUF:Q8_0")

.companion("media", "LiquidAI/LFM2.5-VL-3B-GGUF/mmproj-LFM2.5-VL-3B-Q8_0.gguf")

.build()) {

// The windmills of Consuegra, La Mancha: Don Quixote's "giants"

var windmills = new UrlResource("https://qxotic.ai/snippets/consuegra.jpg");

var answer = ChatClient.create(vision).prompt()

.user(u -> u.text("Don Quixote saw giants here. What do you see?")

.media(MimeTypeUtils.IMAGE_JPEG, windmills))

.call().content();

System.out.println(answer);

}

}$ jbang Vision.java # "several traditional windmills standing on a hill under a clear blue sky"

jbang header + imports

///usr/bin/env jbang "$0" "$@" ; exit $?

//JAVA 25+

//RUNTIME_OPTIONS --add-modules jdk.incubator.vector --enable-native-access=ALL-UNNAMED

//DEPS com.qxotic:jinfer-bom:0.2.0@pom

//DEPS com.qxotic:jinfer-spring-ai com.qxotic:jinfer-models-all

//DEPS com.qxotic:jam-native com.qxotic:jam-vector

//DEPS org.slf4j:slf4j-nop:2.0.17

import com.qxotic.jinfer.spring.ai.JinferEmbeddingModel;

void main() {

try (var emb = JinferEmbeddingModel.builder()

.model("LiquidAI/LFM2.5-Embedding-350M-GGUF:Q8_0")

.build()) {

var a = emb.embed("AI on the JVM");

var b = emb.embed("the JVM thinks now");

var c = emb.embed("the python shed its skin");

System.out.printf("similar %.3f%n", cosineSimilarity(a, b));

System.out.printf("unrelated %.3f%n", cosineSimilarity(a, c));

}

}

cosineSimilarity helper

float cosineSimilarity(float[] a, float[] b) {

float dot = 0, na = 0, nb = 0;

for (int i = 0; i < a.length; i++) {

dot += a[i] * b[i]; na += a[i] * a[i]; nb += b[i] * b[i];

}

return (float) (dot / Math.sqrt(na * nb));

}$ jbang Embed.java # similar 0.609 · unrelated 0.081

Benchmarking is hard ... take these numbers as indicative, not definitive, always measure yourself.

jinfer-bench vs. llama-bench on CPU, ↑ higher is better.

Local AI, end-to-end on the JVM: no sidecars, no services, no IPC.

No Python runtime, no ONNX, no glue code. Every component, from tokenizers to the inference engine, built for the JVM, not bolted onto it.

Ships as a single self-contained binary: millisecond startup, small footprint, no JVM at runtime.

One Tensor API, seven backends: Java, C, CUDA, HIP, Metal, OpenCL, and Mojo.