From c9bd3dc1dd5859ea34b6238bbe8025a93a449335 Mon Sep 17 00:00:00 2001 From: Osamaali313 Date: Fri, 17 Jul 2026 22:38:45 +0300 Subject: [PATCH] Fix token_to_sequence returning Some(0) for out-of-bounds index `Encoding::token_to_sequence` guarded with `if token > self.len()`, so the index `token == len` (one past the last valid token) fell through. For an unprocessed encoding (empty `sequence_ranges`) it then returned `Some(0)` instead of `None`. Valid token indices are `0..len`, and the sibling accessors treat `index >= len` as out of bounds: `token_to_chars` uses `self.offsets.get(token)` and `token_to_word` uses `self.words.get(token)`, both of which yield `None` at `token == len`. Use `>=` so `token_to_sequence` agrees with them. Processed encodings are unaffected (their populated `sequence_ranges` never contain `len`), so existing tests still pass. Adds a regression test for the unprocessed-encoding boundary. --- tokenizers/src/tokenizer/encoding.rs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tokenizers/src/tokenizer/encoding.rs b/tokenizers/src/tokenizer/encoding.rs index f48f200a5b..d3afaf4eb0 100644 --- a/tokenizers/src/tokenizer/encoding.rs +++ b/tokenizers/src/tokenizer/encoding.rs @@ -210,7 +210,7 @@ impl Encoding { /// Returns the index of the sequence containing the given token pub fn token_to_sequence(&self, token: usize) -> Option { - if token > self.len() { + if token >= self.len() { None } else if self.sequence_ranges.is_empty() { Some(0) @@ -566,6 +566,20 @@ mod tests { use super::*; use std::iter::FromIterator; + #[test] + fn token_to_sequence_out_of_bounds() { + // An unprocessed encoding has no `sequence_ranges`, so every in-range token + // maps to sequence 0. Out-of-range indices (>= len) must return `None`, like + // the sibling accessors `token_to_chars`/`token_to_word` do via `.get(token)`. + let encoding = Encoding { + ids: vec![1, 2, 3], + ..Default::default() + }; + assert_eq!(encoding.token_to_sequence(2), Some(0)); // last valid index + assert_eq!(encoding.token_to_sequence(3), None); // == len, out of bounds + assert_eq!(encoding.token_to_sequence(4), None); // > len, out of bounds + } + #[test] fn merge_encodings() { let mut a = Encoding {