feat: tool calling and in-JVM embeddings (0.3.0)

.github/workflows/model-integration.yml

@@ -94,6 +94,7 @@ jobs:
           :backend-java:downloadMiniCpm51BQ4KMModel
           :backend-java:downloadQwen25Math15BQ4KMModel
           :backend-java:downloadEuroLlm17BQ4KMModel
+          :backend-java:downloadQwen3Embedding06BQ80Model
           --no-daemon
           --stacktrace
 
@@ -110,6 +111,15 @@ jobs:
           --tests com.integrallis.models.backend.purejava.MiniCpm5ModelFixtureIntegrationTest
           --tests com.integrallis.models.backend.purejava.Qwen25MathModelFixtureIntegrationTest
           --tests com.integrallis.models.backend.purejava.EuroLlmModelFixtureIntegrationTest
+          --tests com.integrallis.models.backend.purejava.Qwen3EmbeddingModelFixtureIntegrationTest
+          --no-daemon
+          --stacktrace
+
+      - name: Run framework embedding adapter integration tests
+        working-directory: models
+        run: >
+          ./gradlew
+          :models-rag-bench:embeddingAdaptersIntegrationTest
           --no-daemon
           --stacktrace
 

CHANGELOG.md

@@ -4,6 +4,48 @@ All notable changes to models are documented here.
 
 ## [Unreleased]
 
+## [0.3.0] - 2026-08-05
+
+### Added
+
+- Added tool calling. `ToolSpec` and `ToolCall` describe declarations and
+  invocations without introducing a JSON dependency, because argument text is
+  carried verbatim rather than parsed. `ToolSyntax` records how each model family
+  expresses calls, taken from its published chat template, and drives both
+  rendering and recovery so no per-family parser is required. `ChatTemplate`
+  gained `render(messages, tools)`, `toolSyntax()`, `supportsTools()`, and
+  `canParseToolCalls()`.
+- Added tool-call recovery through `ToolCallScanner`, which strips markdown code
+  fences, accepts either the `arguments` or `parameters` spelling, and degrades
+  to plain text rather than failing a turn on malformed output.
+- Surfaced tool calls natively in both framework adapters: Spring AI through
+  `AssistantMessage.getToolCalls()`, and LangChain4j through
+  `AiMessage.toolExecutionRequests()` with `FinishReason.TOOL_EXECUTION`.
+- Added embedding support. `EmbeddingBackend` and `Pooling` define the contract,
+  `GgufEmbeddingBackend` implements it over the pure-Java forward pass, and both
+  the Llama-family and Gemma 4 decoders can now return the final normalized
+  hidden state instead of vocabulary logits. Producing an embedding skips the
+  vocabulary projection, so it costs less per token than generating one.
+- Added `ModelsSpringAiEmbeddingModel` and `ModelsEmbeddingModel`, letting a
+  Spring AI or LangChain4j application keep embeddings inside the JVM.
+- Added `Tokenizer.tokenId(String)` for resolving a token id from its exact
+  vocabulary text, needed because families disagree on the ids behind identical
+  tool-call delimiters.
+
+### Changed
+
+- **Breaking:** moved `EmbeddingBackend` from `com.integrallis.models.embedding`
+  to `com.integrallis.models.api`. It is a contract, and leaving it in
+  `models-embedding` would have forced every backend implementing it to depend on
+  `vectors-db`. No published artifact implemented it.
+- `ChatMessage` now carries `toolCalls`. Blank text remains invalid except on an
+  assistant turn consisting solely of a tool call. The two-argument constructor
+  and the factory methods are unchanged.
+- Replaced the sampler's full-vocabulary sort with a bounded-heap top-k
+  selection, measured at 19.251 ms to 0.848 ms per sampled token at a
+  151,936-token vocabulary. Tie-breaking still prefers the lower token id, so
+  seeded output is unchanged.
+
 ## [0.2.6] - 2026-08-04
 
 ### Added

README.md

@@ -136,16 +136,16 @@ directly:
 
 ```kotlin
 dependencies {
-    implementation("com.integrallis:models:0.2.6")
-    implementation("com.integrallis:backend-java:0.2.6") // or backend-native
+    implementation("com.integrallis:models:0.3.0")
+    implementation("com.integrallis:backend-java:0.3.0") // or backend-native
 }
 ```
 
 Use Apple's on-device system model on a supported Apple Silicon Mac:
 
 ```kotlin
 dependencies {
-    implementation("com.integrallis:backend-apple:0.2.6")
+    implementation("com.integrallis:backend-apple:0.3.0")
 }
 ```
 

backend-java/build.gradle.kts

@@ -43,6 +43,10 @@ val modelFixtures =
             "downloadQwen317BQ80Model",
             "qwen3_1_7b_q8_0",
         ),
+        modelFixture(
+            "downloadQwen3Embedding06BQ80Model",
+            "qwen3_embedding_0_6b_q8_0",
+        ),
         modelFixture(
             "downloadQwen38BQ4KMModel",
             "qwen3_8b_q4_k_m",
@@ -210,6 +214,25 @@ tasks.register<Test>("qwen306BQ40IntegrationTest") {
     maxHeapSize = "4g"
 }
 
+tasks.register<Test>("qwen3EmbeddingIntegrationTest") {
+    description = "Run the pinned Qwen3-Embedding 0.6B pure-Java embedding integration tests"
+    group = "verification"
+    testClassesDirs = sourceSets["test"].output.classesDirs
+    classpath = sourceSets["test"].runtimeClasspath
+    useJUnitPlatform {
+        includeTags("integration")
+    }
+    filter {
+        includeTestsMatching(
+            "com.integrallis.models.backend.purejava.Qwen3EmbeddingModelFixtureIntegrationTest",
+        )
+    }
+    dependsOn(tasks.named("downloadQwen3Embedding06BQ80Model"))
+    outputs.upToDateWhen { false }
+    maxParallelForks = 1
+    maxHeapSize = "4g"
+}
+
 tasks.register<Test>("qwen25Math15BIntegrationTest") {
     description = "Run the pinned Qwen2.5-Math 1.5B model integration tests"
     group = "verification"

backend-java/src/main/java/com/integrallis/models/backend/purejava/Gemma4DecoderAdapter.java

@@ -67,6 +67,21 @@ public float[] prefill(int[] tokens, int startPosition) {
     return decoder.prefill(tokens, startPosition);
   }
 
+  @Override
+  public float[] prefillHiddenState(int[] tokens, int startPosition) {
+    return decoder.prefillHiddenState(tokens, startPosition);
+  }
+
+  @Override
+  public float[] hiddenState(int token, int position) {
+    return decoder.hiddenState(token, position);
+  }
+
+  @Override
+  public boolean supportsHiddenState() {
+    return true;
+  }
+
   @Override
   public Session openSession() {
     return new Gemma4Session(decoder.openSession());

backend-java/src/main/java/com/integrallis/models/backend/purejava/GgufEmbeddingBackend.java

@@ -0,0 +1,182 @@
+/*
+ * Copyright 2025-2026 Integrallis Software, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.integrallis.models.backend.purejava;
+
+import com.integrallis.models.api.EmbeddingBackend;
+import com.integrallis.models.api.Pooling;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Produces sentence embeddings from a GGUF model using the pure-Java forward pass.
+ *
+ * <p>An embedding is the generative pass with its head removed: the transformer stack runs
+ * unchanged, and the vocabulary projection — the widest matmul in the pass — is skipped in favour
+ * of the activation it would have consumed. Embedding a token therefore costs strictly less than
+ * generating one on the same weights.
+ *
+ * <p>Pooling and normalization travel with the instance because they are properties of the
+ * embedding model. Getting them wrong does not fail; it quietly degrades retrieval, which is a far
+ * worse failure mode than an exception.
+ *
+ * <p>Not thread-safe: each call drives one sequence through shared backend state and resets it
+ * between texts. Use one instance per thread, or guard it.
+ */
+public final class GgufEmbeddingBackend implements EmbeddingBackend {
+
+  private final PureJavaBackend backend;
+  private final Pooling pooling;
+  private final boolean normalize;
+  private final int dimension;
+  private boolean closed;
+
+  private GgufEmbeddingBackend(Builder builder) {
+    this.backend = builder.backend;
+    this.pooling = builder.pooling;
+    this.normalize = builder.normalize;
+    this.dimension = backend.metadata().embeddingDim();
+    if (!backend.supportsHiddenState()) {
+      throw new IllegalArgumentException(
+          "model architecture "
+              + backend.metadata().modelFamily()
+              + " does not expose hidden states for embedding");
+    }
+  }
+
+  /** Starts configuring an embedding backend over an already-loaded model. */
+  public static Builder builder(PureJavaBackend backend) {
+    return new Builder(backend);
+  }
+
+  @Override
+  public int dimension() {
+    return dimension;
+  }
+
+  @Override
+  public float[] embed(String text) {
+    Objects.requireNonNull(text, "text");
+    checkOpen();
+    int[] tokens = tokenize(text);
+    // Every text is an independent sequence; without this the previous one stays in the KV cache
+    // and results depend on call order.
+    backend.reset();
+    float[] pooled = pooling == Pooling.MEAN ? meanPooled(tokens) : lastTokenPooled(tokens);
+    if (normalize) {
+      l2Normalize(pooled);
+    }
+    return pooled;
+  }
+
+  @Override
+  public float[][] embedAll(List<String> texts) {
+    Objects.requireNonNull(texts, "texts");
+    float[][] rows = new float[texts.size()][];
+    for (int index = 0; index < texts.size(); index++) {
+      rows[index] = embed(Objects.requireNonNull(texts.get(index), "texts must not contain null"));
+    }
+    return rows;
+  }
+
+  @Override
+  public void close() {
+    if (!closed) {
+      closed = true;
+      backend.close();
+    }
+  }
+
+  /**
+   * Encodes text, substituting the beginning-of-sequence token when it yields nothing.
+   *
+   * <p>A blank row in a corpus should embed to something stable rather than abort an ingest run.
+   */
+  private int[] tokenize(String text) {
+    int[] tokens = backend.tokenizer().encode(text);
+    return tokens.length == 0 ? new int[] {backend.tokenizer().bosToken()} : tokens;
+  }
+
+  /** Takes the final position's state: the only one that has attended to the whole input. */
+  private float[] lastTokenPooled(int[] tokens) {
+    return backend.prefillHiddenState(tokens, 0).clone();
+  }
+
+  /** Averages every position's state, which costs one hidden state per token. */
+  private float[] meanPooled(int[] tokens) {
+    double[] sum = new double[dimension];
+    for (int index = 0; index < tokens.length; index++) {
+      float[] hidden = backend.hiddenState(tokens[index], index);
+      for (int component = 0; component < dimension; component++) {
+        sum[component] += hidden[component];
+      }
+    }
+    float[] pooled = new float[dimension];
+    for (int component = 0; component < dimension; component++) {
+      pooled[component] = (float) (sum[component] / tokens.length);
+    }
+    return pooled;
+  }
+
+  /** Scales to unit length so downstream cosine similarity reduces to a dot product. */
+  private static void l2Normalize(float[] vector) {
+    double sumOfSquares = 0;
+    for (float value : vector) {
+      sumOfSquares += (double) value * value;
+    }
+    double magnitude = Math.sqrt(sumOfSquares);
+    if (magnitude == 0.0) {
+      // A zero vector has no direction; scaling would divide by zero.
+      return;
+    }
+    for (int index = 0; index < vector.length; index++) {
+      vector[index] = (float) (vector[index] / magnitude);
+    }
+  }
+
+  private void checkOpen() {
+    if (closed) {
+      throw new IllegalStateException("embedding backend is closed");
+    }
+  }
+
+  /** Configures pooling and normalization, which must match how the model was trained. */
+  public static final class Builder {
+
+    private final PureJavaBackend backend;
+    private Pooling pooling = Pooling.LAST_TOKEN;
+    private boolean normalize = true;
+
+    private Builder(PureJavaBackend backend) {
+      this.backend = Objects.requireNonNull(backend, "backend");
+    }
+
+    /** Defaults to {@link Pooling#LAST_TOKEN}, correct for causal decoder-only embedders. */
+    public Builder pooling(Pooling pooling) {
+      this.pooling = Objects.requireNonNull(pooling, "pooling");
+      return this;
+    }
+
+    /** Defaults to true; unit vectors are what retrieval and semantic caching expect. */
+    public Builder normalize(boolean normalize) {
+      this.normalize = normalize;
+      return this;
+    }
+
+    public GgufEmbeddingBackend build() {
+      return new GgufEmbeddingBackend(this);
+    }
+  }
+}

backend-java/src/main/java/com/integrallis/models/backend/purejava/LlamaDecoder.java

@@ -62,6 +62,21 @@ public float[] prefill(int[] tokens, int startPosition) {
     return forwardPass.prefill(tokens, startPosition);
   }
 
+  @Override
+  public float[] prefillHiddenState(int[] tokens, int startPosition) {
+    return forwardPass.prefillHiddenState(tokens, startPosition);
+  }
+
+  @Override
+  public float[] hiddenState(int token, int position) {
+    return forwardPass.hiddenState(token, position);
+  }
+
+  @Override
+  public boolean supportsHiddenState() {
+    return true;
+  }
+
   @Override
   public Session openSession() {
     return new LlamaSession(forwardPass.openSession());

backend-java/src/main/java/com/integrallis/models/backend/purejava/PureJavaBackend.java

@@ -341,6 +341,36 @@ public float[] prefill(int[] tokens, int startPosition) {
     return decoder.prefill(tokens, startPosition);
   }
 
+  /**
+   * Runs the stack over a sequence and returns the final position's hidden state.
+   *
+   * <p>Skips the vocabulary projection, which is the widest matmul in the pass — so producing an
+   * embedding costs less per token than generating one.
+   *
+   * <p>The array is backend-owned scratch, valid until the next call. Copy it to keep it.
+   */
+  public float[] prefillHiddenState(int[] tokens, int startPosition) {
+    checkOpen();
+    return decoder.prefillHiddenState(tokens, startPosition);
+  }
+
+  /**
+   * Runs one step and returns its hidden state instead of logits.
+   *
+   * <p>Needed for mean pooling, which reduces over every position rather than only the last.
+   *
+   * <p>The array is backend-owned scratch, valid until the next call.
+   */
+  public float[] hiddenState(int token, int position) {
+    checkOpen();
+    return decoder.hiddenState(token, position);
+  }
+
+  /** Whether this model's architecture exposes hidden states for embedding. */
+  public boolean supportsHiddenState() {
+    return decoder.supportsHiddenState();
+  }
+
   @Override
   public InferenceSession openSession() {
     checkOpen();