Preserve backslashes in generated Python stub docstrings

bindings/python/py_src/tokenizers/normalizers.pyi

@@ -80,7 +80,7 @@ class ByteLevel(Normalizer):
 
         >>> from tokenizers.normalizers import ByteLevel
         >>> normalizer = ByteLevel()
-        >>> normalizer.normalize_str("hello\nworld")
+        >>> normalizer.normalize_str("hello\\nworld")
         'helloĊworld'
     """
     def __new__(cls, /) -> ByteLevel: ...
@@ -115,7 +115,7 @@ class NFC(Normalizer):
 
         >>> from tokenizers.normalizers import NFC
         >>> normalizer = NFC()
-        >>> normalizer.normalize_str("e\u0301")  # 'e' + combining accent
+        >>> normalizer.normalize_str("e\\u0301")  # 'e' + combining accent
         'é'
     """
     def __new__(cls, /) -> NFC: ...
@@ -137,7 +137,7 @@ class NFD(Normalizer):
         >>> from tokenizers.normalizers import NFD
         >>> normalizer = NFD()
         >>> normalizer.normalize_str("Héllo")
-        'He\u0301llo'
+        'He\\u0301llo'
     """
     def __new__(cls, /) -> NFD: ...
 
@@ -155,7 +155,7 @@ class NFKC(Normalizer):
 
         >>> from tokenizers.normalizers import NFKC
         >>> normalizer = NFKC()
-        >>> normalizer.normalize_str("fine caf\u00e9")
+        >>> normalizer.normalize_str("fine caf\\u00e9")
         'fine café'
     """
     def __new__(cls, /) -> NFKC: ...
@@ -192,7 +192,7 @@ class Nmt(Normalizer):
 
         >>> from tokenizers.normalizers import Nmt
         >>> normalizer = Nmt()
-        >>> normalizer.normalize_str("Hello\x00World")
+        >>> normalizer.normalize_str("Hello\\x00World")
         'Hello World'
     """
     def __new__(cls, /) -> Nmt: ...
@@ -310,7 +310,7 @@ class Replace(Normalizer):
         >>> Replace(".", " ").normalize_str("hello.world")
         'hello world'
         >>> # Replace using a regex
-        >>> Replace(Regex(r"\s+"), " ").normalize_str("hello   world")
+        >>> Replace(Regex(r"\\s+"), " ").normalize_str("hello   world")
         'hello world'
     """
     def __new__(cls, /, pattern: str | Regex, content: str) -> Replace: ...

bindings/python/py_src/tokenizers/pre_tokenizers.pyi

@@ -317,7 +317,7 @@ class Split(PreTokenizer):
         >>> pre_tokenizer.pre_tokenize_str("one,two,three")
         [('one', (0, 3)), ('two', (4, 7)), ('three', (8, 13))]
         >>> # Split using a regex, keeping the delimiter isolated
-        >>> Split(Regex(r"\s+"), behavior="isolated").pre_tokenize_str("hello   world")
+        >>> Split(Regex(r"\\s+"), behavior="isolated").pre_tokenize_str("hello   world")
         [('hello', (0, 5)), ('   ', (5, 8)), ('world', (8, 13))]
     """
     def __getnewargs__(self, /) -> tuple: ...
@@ -355,7 +355,7 @@ class UnicodeScripts(PreTokenizer):
 @final
 class Whitespace(PreTokenizer):
     """
-    This pre-tokenizer splits on word boundaries according to the ``\w+|[^\w\s]+``
+    This pre-tokenizer splits on word boundaries according to the ``\\w+|[^\\w\\s]+``
     regex pattern. It splits on word characters or characters that aren't words or
     whitespaces (punctuation such as hyphens, apostrophes, commas, etc.).
 

bindings/python/tools/stub-gen/src/main.rs

@@ -139,7 +139,7 @@ fn generate_stubs(cdylib: &Path, out_dir: &Path) -> Result<(), Box<dyn std::erro
         println!("Found cdylib at {}", cdylib.display());
 
         let main_module_name = "tokenizers";
-        let python_module = pyo3_introspection::introspect_cdylib(&cdylib, main_module_name)
+        let mut python_module = pyo3_introspection::introspect_cdylib(&cdylib, main_module_name)
             .unwrap_or_else(|_| panic!("Failed introspection of {}", main_module_name));
 
         // Sanity check: if docstrings are missing the patched pyo3 in
@@ -150,6 +150,7 @@ fn generate_stubs(cdylib: &Path, out_dir: &Path) -> Result<(), Box<dyn std::erro
         // one well-known class still carries its docstring before writing
         // out otherwise-empty stubs.
         assert_introspection_has_docstrings(&python_module);
+        escape_docstrings_for_python(&mut python_module);
 
         let type_stubs = pyo3_introspection::module_stub_files(&python_module);
 
@@ -176,6 +177,44 @@ fn absolutize_local_imports(contents: &str, root_module: &str) -> String {
         .replace("from .", &format!("from {root_module}."))
 }
 
+fn escape_docstring(docstring: &mut Option<String>) {
+    if let Some(docstring) = docstring {
+        *docstring = docstring.replace('\\', "\\\\");
+    }
+}
+
+fn escape_class_docstrings(class: &mut Class) {
+    escape_docstring(&mut class.docstring);
+    for method in &mut class.methods {
+        escape_docstring(&mut method.docstring);
+    }
+    for attribute in &mut class.attributes {
+        escape_docstring(&mut attribute.docstring);
+    }
+    for inner_class in &mut class.inner_classes {
+        escape_class_docstrings(inner_class);
+    }
+}
+
+/// Preserve runtime docstrings when pyo3-introspection emits ordinary Python
+/// triple-quoted strings. Without this, Python interprets backslash sequences
+/// and warns on regex escapes such as `\s` and `\w`.
+fn escape_docstrings_for_python(module: &mut Module) {
+    escape_docstring(&mut module.docstring);
+    for function in &mut module.functions {
+        escape_docstring(&mut function.docstring);
+    }
+    for attribute in &mut module.attributes {
+        escape_docstring(&mut attribute.docstring);
+    }
+    for class in &mut module.classes {
+        escape_class_docstrings(class);
+    }
+    for submodule in &mut module.modules {
+        escape_docstrings_for_python(submodule);
+    }
+}
+
 /// Walk the introspected module tree and count classes, functions, and
 /// attributes that carry a docstring. Returns `(with_docstring, total)`.
 fn count_docstrings(module: &Module) -> (usize, usize) {
@@ -242,6 +281,23 @@ fn assert_introspection_has_docstrings(module: &Module) {
     );
 }
 
+#[cfg(test)]
+mod tests {
+    use super::escape_docstring;
+
+    #[test]
+    fn escapes_backslashes_in_docstrings() {
+        let mut docstring = Some(r#"Regex(r"\s+") and "e\u0301""#.to_owned());
+
+        escape_docstring(&mut docstring);
+
+        assert_eq!(
+            docstring.as_deref(),
+            Some(r#"Regex(r"\\s+") and "e\\u0301""#)
+        );
+    }
+}
+
 fn build_extension(manifest_dir: &Path) -> Result<(), Box<dyn std::error::Error>> {
     println!("Building and installing extension (release)...");
     match Command::new("maturin").current_dir(manifest_dir).args(["develop", "--release"]).status() {