Expose the inference pipeline

CHANGELOG.md

@@ -4,6 +4,16 @@ All notable changes to models are documented here.
 
 ## [Unreleased]
 
+## [0.2.6] - 2026-08-04
+
+### Added
+
+- Added an owning `InferencePipeline` for coordinated access to structured
+  tokenization, model metadata, active context capacity and position, prefill,
+  forward-pass logits, reset, checkpoint, rewind, and high-level generation.
+- Exposed the runtime-allocated context capacity separately from the maximum
+  context length declared by model metadata.
+
 ## [0.2.5] - 2026-08-03
 
 ### Changed

README.md

@@ -136,16 +136,16 @@ directly:
 
 ```kotlin
 dependencies {
-    implementation("com.integrallis:models:0.2.5")
-    implementation("com.integrallis:backend-java:0.2.5") // or backend-native
+    implementation("com.integrallis:models:0.2.6")
+    implementation("com.integrallis:backend-java:0.2.6") // or backend-native
 }
 ```
 
 Use Apple's on-device system model on a supported Apple Silicon Mac:
 
 ```kotlin
 dependencies {
-    implementation("com.integrallis:backend-apple:0.2.5")
+    implementation("com.integrallis:backend-apple:0.2.6")
 }
 ```
 
@@ -165,7 +165,7 @@ var options = SamplingOptions.builder()
     .build();
 
 try (var runtime = ModelJars.openRuntime(MODEL)) {
-    var prompt = runtime.chatTemplate().render(List.of(
+    ModelPrompt prompt = runtime.chatTemplate().render(List.of(
         ChatMessage.system("Classify the user's intent in one phrase."),
         ChatMessage.user("I want to cancel my order")));
     String result = runtime.model().generate(prompt, options);
@@ -179,6 +179,9 @@ Applications that manage
 their own GGUF files can use the lower-level `PureJavaBackend.load(Path)` and
 `RustFfmBackend.load(Path)` APIs described in the
 [Using Models guide](https://integrallis.github.io/models/docs/models/current/using-models.html).
+Wrap either backend in `InferencePipeline` for ownership-safe access to the
+tokenizer, model metadata, active context window, structured prefill,
+forward-pass logits, reset, checkpoint, and rewind.
 
 Streaming uses the same loaded model:
 

backend-java/src/main/java/com/integrallis/models/backend/purejava/PureJavaBackend.java

@@ -63,14 +63,18 @@ public final class PureJavaBackend implements SpeculativeInferenceBackend, Batch
   private final GgufTokenizer tokenizer;
   private final PureJavaDecoder decoder;
   private final ModelMetadata modelMetadata;
+  private final int contextCapacity;
   private final PureJavaExecutionPlan executionPlan;
   private final BackendDiagnostics diagnostics;
   private final GgufBatchedMatrixKernel batchedMatrixKernel;
   private PureJavaDecoder.Session[] sessionBatch = new PureJavaDecoder.Session[0];
   private boolean closed;
 
   private record LoadedDecoder(
-      PureJavaDecoder decoder, ModelMetadata metadata, PureJavaExecutionPlan executionPlan) {}
+      PureJavaDecoder decoder,
+      ModelMetadata metadata,
+      int contextCapacity,
+      PureJavaExecutionPlan executionPlan) {}
 
   private static final class PureJavaInferenceSession implements InferenceSession {
     private final PureJavaBackend owner;
@@ -104,13 +108,15 @@ private PureJavaBackend(
       GgufTokenizer tokenizer,
       PureJavaDecoder decoder,
       ModelMetadata modelMetadata,
+      int contextCapacity,
       PureJavaExecutionPlan executionPlan,
       BackendDiagnostics diagnostics,
       GgufBatchedMatrixKernel batchedMatrixKernel) {
     this.arena = arena;
     this.tokenizer = tokenizer;
     this.decoder = decoder;
     this.modelMetadata = modelMetadata;
+    this.contextCapacity = contextCapacity;
     this.executionPlan = executionPlan;
     this.diagnostics = diagnostics;
     this.batchedMatrixKernel = batchedMatrixKernel;
@@ -205,6 +211,7 @@ private static PureJavaBackend load(
           tokenizer,
           loaded.decoder(),
           loaded.metadata(),
+          loaded.contextCapacity(),
           loaded.executionPlan(),
           diagnostics,
           batchedMatrixKernel);
@@ -232,12 +239,9 @@ private static LoadedDecoder loadLlama(
             ModelTopology.from(modelFamily, config, weights),
             planConfiguration,
             batchedMatrixKernel);
+    int contextCapacity = runtimeContextLength(config.contextLength());
     KvCache cache =
-        new KvCache(
-            config.numLayers(),
-            runtimeContextLength(config.contextLength()),
-            config.keyDim(),
-            config.valueDim());
+        new KvCache(config.numLayers(), contextCapacity, config.keyDim(), config.valueDim());
     PureJavaDecoder decoder =
         new LlamaDecoder(
             new LlamaForwardPass(config, weights, cache, executionPlan, batchedMatrixKernel));
@@ -251,7 +255,7 @@ private static LoadedDecoder loadLlama(
             config.numLayers(),
             config.numHeads(),
             config.numKvHeads());
-    return new LoadedDecoder(decoder, metadata, executionPlan);
+    return new LoadedDecoder(decoder, metadata, contextCapacity, executionPlan);
   }
 
   private static LoadedDecoder loadGemma4(
@@ -277,9 +281,10 @@ private static LoadedDecoder loadGemma4(
             config.numLayers(),
             config.numHeads(),
             config.numKvHeads(0));
-    Gemma4Decoder decoder =
-        Gemma4Decoder.load(file, runtimeContextLength(config.contextLength()), batchedMatrixKernel);
-    return new LoadedDecoder(new Gemma4DecoderAdapter(decoder), metadata, executionPlan);
+    int contextCapacity = runtimeContextLength(config.contextLength());
+    Gemma4Decoder decoder = Gemma4Decoder.load(file, contextCapacity, batchedMatrixKernel);
+    return new LoadedDecoder(
+        new Gemma4DecoderAdapter(decoder), metadata, contextCapacity, executionPlan);
   }
 
   @Override
@@ -292,6 +297,11 @@ public ModelMetadata metadata() {
     return modelMetadata;
   }
 
+  @Override
+  public int contextCapacity() {
+    return contextCapacity;
+  }
+
   /** Returns the immutable execution plan selected while loading this model. */
   public PureJavaExecutionPlan executionPlan() {
     return executionPlan;

backend-java/src/test/java/com/integrallis/models/backend/purejava/PureJavaBackendTest.java

@@ -271,6 +271,7 @@ void capsRuntimeContextLengthWithoutChangingModelMetadata(@TempDir Path dir)
 
       try (PureJavaBackend backend = PureJavaBackend.load(modelPath)) {
         assertThat(backend.metadata().contextLength()).isEqualTo(CONTEXT);
+        assertThat(backend.contextCapacity()).isEqualTo(4);
         for (int position = 0; position <= 3; position++) {
           assertThat(backend.forward(5, position)).hasSize(VOCAB_SIZE);
         }

backend-java/src/test/java/com/integrallis/models/backend/purejava/Qwen3ModelFixtureIntegrationTest.java

@@ -22,8 +22,12 @@
 import com.integrallis.models.backend.purejava.fixture.ModelFixtureRegistry;
 import com.integrallis.models.backend.purejava.fixture.ModelFixtureRequirement;
 import com.integrallis.models.runtime.GenerationLoop;
+import com.integrallis.models.runtime.InferencePipeline;
 import com.integrallis.models.runtime.SpeculativeGenerationOptions;
+import com.integrallis.models.runtime.chat.ChatMessage;
+import com.integrallis.models.runtime.chat.ChatTemplate;
 import java.nio.file.Files;
+import java.util.List;
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 
@@ -65,6 +69,43 @@ void matchesLlamaCppGreedyTokensForQwen306BQ40() {
         new int[] {34208, 916, 279, 15678});
   }
 
+  @Test
+  void structuredChatMlPipelineMatchesLlamaCppForQwen306BQ40() {
+    ModelFixtureDescriptor descriptor =
+        ModelFixtureRegistry.fromClasspath().resolve(QWEN3_0_6B_Q4_0).orElseThrow();
+    String previous = System.getProperty(PureJavaBackend.MAX_CONTEXT_LENGTH_PROPERTY);
+    System.setProperty(
+        PureJavaBackend.MAX_CONTEXT_LENGTH_PROPERTY, Integer.toString(INTEGRATION_CONTEXT_LENGTH));
+
+    try (InferencePipeline pipeline =
+        new InferencePipeline(PureJavaBackend.load(descriptor.localPath().orElseThrow()))) {
+      var prompt =
+          ChatTemplate.CHATML_NO_THINK.render(
+              List.of(
+                  ChatMessage.system("Follow the user's output format exactly."),
+                  ChatMessage.user("Reply with exactly: JAVA")));
+      SamplingOptions sampling = SamplingOptions.builder().temperature(0.0f).maxTokens(16).build();
+
+      int[] promptTokens = pipeline.tokenize(prompt);
+      assertThat(promptTokens).hasSize(30);
+      assertThat(pipeline.contextWindow().capacity()).isEqualTo(INTEGRATION_CONTEXT_LENGTH);
+      assertThat(pipeline.contextWindow().position()).hasValue(0);
+
+      float[] logits = pipeline.prefill(prompt, 0);
+      int firstToken = argmax(logits);
+      assertThat(pipeline.tokenizer().decode(firstToken)).isEqualTo("JAVA");
+      assertThat(pipeline.contextWindow().position()).hasValue(promptTokens.length);
+
+      pipeline.forward(firstToken, promptTokens.length);
+      assertThat(pipeline.contextWindow().position()).hasValue(promptTokens.length + 1);
+      pipeline.resetContext();
+      assertThat(pipeline.contextWindow().position()).hasValue(0);
+      assertThat(pipeline.generate(prompt, sampling)).isEqualTo("JAVA");
+    } finally {
+      restoreSystemProperty(PureJavaBackend.MAX_CONTEXT_LENGTH_PROPERTY, previous);
+    }
+  }
+
   @Test
   void ngramSpeculationMatchesSequentialQwen306BQ40Generation() {
     ModelFixtureDescriptor descriptor =

backend-native/src/main/java/com/integrallis/models/backend/nativekernel/RustFfmBackend.java

@@ -123,6 +123,11 @@ public ModelMetadata metadata() {
     return delegate.metadata();
   }
 
+  @Override
+  public int contextCapacity() {
+    return delegate.contextCapacity();
+  }
+
   /** Returns the Java transformer execution plan surrounding the native kernels. */
   public PureJavaExecutionPlan executionPlan() {
     return delegate.executionPlan();

backend-native/src/main/rust/model-kernels/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "jmodels-kernels"
-version = "0.2.5"
+version = "0.2.6"
 edition = "2024"
 license = "Apache-2.0"
 publish = false

docs/content/antora.yml

@@ -1,7 +1,7 @@
 name: models
 title: Models
 version: 'current'
-display_version: '0.2.5'
+display_version: '0.2.6'
 prerelease: false
 start_page: ROOT:index.adoc
 nav:
@@ -10,7 +10,7 @@ asciidoc:
   attributes:
     source-language: java
     source-highlighter: highlight.js
-    models-version: '0.2.5'
+    models-version: '0.2.6'
     modeljars-version: '0.1.2'
     vectors-version: '0.1.4'
     url-models-github: https://github.com/integrallis/models

docs/content/modules/ROOT/nav.adoc

@@ -6,6 +6,7 @@
 
 .Using Models
 * xref:using-models.adoc[Using Models]
+* xref:inference-pipeline.adoc[Inference Pipeline]
 * xref:model-support.adoc[Model Support]
 * xref:execution-planning.adoc[Execution Planning]
 * xref:session-batching.adoc[Concurrent Session Batching]

docs/content/modules/ROOT/pages/index.adoc

@@ -69,8 +69,9 @@ try (var runtime = ModelJars.openRuntime(MODEL)) {
 }
 ----
 
-Continue with xref:using-models.adoc[Using Models] for streaming, diagnostics,
-and unmanaged GGUF files. Framework users can go directly to
+Continue with xref:using-models.adoc[Using Models] for streaming and diagnostics,
+or xref:inference-pipeline.adoc[Inference Pipeline] for tokenizer, context,
+prefill, logits, and rewind access. Framework users can go directly to
 xref:langchain4j.adoc[LangChain4j], xref:spring-ai.adoc[Spring AI], or
 xref:spring-boot.adoc[Spring Boot].