From 0c19e30a4bff1b8b64c35ba07d774dadadafccc6 Mon Sep 17 00:00:00 2001
From: Sohum Trivedi <trivsohum@gmail.com>
Date: Thu, 16 Jul 2026 23:20:11 -0700
Subject: [PATCH] Fix Metaspace decoder dropping internal spaces in the first
 token

Metaspace::decode_chain mapped every replacement char in the first
token to None when prepend_scheme != Never, instead of stripping only
the single leading char that pre-tokenization prepended. Any first
token containing more than one metaspace char (e.g. a BPE merge
spanning a space like "_in_the", or any multi-word token when
split=false) silently lost its internal word-boundary spaces:
decoding ["_in_the", "_house"] produced "inthe house" instead of
"in the house".

Only drop the replacement char at position 0 of the first token; all
other occurrences decode to a space, matching the SentencePiece
convention of stripping exactly the one prepended prefix space.
---
 tokenizers/src/pre_tokenizers/metaspace.rs | 31 ++++++++++++++++++++--
 1 file changed, 29 insertions(+), 2 deletions(-)

diff --git a/tokenizers/src/pre_tokenizers/metaspace.rs b/tokenizers/src/pre_tokenizers/metaspace.rs
index d821f11841..ca422fee31 100644
--- a/tokenizers/src/pre_tokenizers/metaspace.rs
+++ b/tokenizers/src/pre_tokenizers/metaspace.rs
@@ -155,9 +155,13 @@ impl Decoder for Metaspace {
             .map(|(i, token)| {
                 token
                     .chars()
-                    .flat_map(|c| {
+                    .enumerate()
+                    .filter_map(|(j, c)| {
                         if c == self.replacement {
-                            if i == 0 && self.prepend_scheme != PrependScheme::Never {
+                            // Only strip the single replacement char that was
+                            // prepended during pre-tokenization; any other
+                            // occurrence is a real word boundary.
+                            if i == 0 && j == 0 && self.prepend_scheme != PrependScheme::Never {
                                 None
                             } else {
                                 Some(' ')
@@ -367,4 +371,27 @@ mod tests {
             .unwrap();
         assert_eq!(res, vec![" Hey", " friend!"]);
     }
+
+    #[test]
+    fn decode_first_token_with_internal_replacement() {
+        // Only the single prepended replacement char should be stripped from
+        // the first token; internal ones are real word boundaries.
+        let decoder = Metaspace::new('▁', PrependScheme::Always, true);
+        let res = decoder
+            .decode_chain(vec!["▁in▁the".into(), "▁house".into()])
+            .unwrap();
+        assert_eq!(res, vec!["in the", " house"]);
+
+        // Same thing without splitting (single multi-word token).
+        let decoder = Metaspace::new('▁', PrependScheme::Always, false);
+        let res = decoder.decode_chain(vec!["▁Hey▁friend".into()]).unwrap();
+        assert_eq!(res, vec!["Hey friend"]);
+
+        // With `Never`, nothing was prepended so nothing should be stripped.
+        let decoder = Metaspace::new('▁', PrependScheme::Never, true);
+        let res = decoder
+            .decode_chain(vec!["▁in▁the".into(), "▁house".into()])
+            .unwrap();
+        assert_eq!(res, vec![" in the", " house"]);
+    }
 }