diff --git a/bindings/python/py_src/tokenizers/normalizers.pyi b/bindings/python/py_src/tokenizers/normalizers.pyi index 78a9f08f3f..556df1d32a 100644 --- a/bindings/python/py_src/tokenizers/normalizers.pyi +++ b/bindings/python/py_src/tokenizers/normalizers.pyi @@ -80,7 +80,7 @@ class ByteLevel(Normalizer): >>> from tokenizers.normalizers import ByteLevel >>> normalizer = ByteLevel() - >>> normalizer.normalize_str("hello\nworld") + >>> normalizer.normalize_str("hello\\nworld") 'helloĊworld' """ def __new__(cls, /) -> ByteLevel: ... @@ -115,7 +115,7 @@ class NFC(Normalizer): >>> from tokenizers.normalizers import NFC >>> normalizer = NFC() - >>> normalizer.normalize_str("e\u0301") # 'e' + combining accent + >>> normalizer.normalize_str("e\\u0301") # 'e' + combining accent 'é' """ def __new__(cls, /) -> NFC: ... @@ -137,7 +137,7 @@ class NFD(Normalizer): >>> from tokenizers.normalizers import NFD >>> normalizer = NFD() >>> normalizer.normalize_str("Héllo") - 'He\u0301llo' + 'He\\u0301llo' """ def __new__(cls, /) -> NFD: ... @@ -155,7 +155,7 @@ class NFKC(Normalizer): >>> from tokenizers.normalizers import NFKC >>> normalizer = NFKC() - >>> normalizer.normalize_str("fine caf\u00e9") + >>> normalizer.normalize_str("fine caf\\u00e9") 'fine café' """ def __new__(cls, /) -> NFKC: ... @@ -192,7 +192,7 @@ class Nmt(Normalizer): >>> from tokenizers.normalizers import Nmt >>> normalizer = Nmt() - >>> normalizer.normalize_str("Hello\x00World") + >>> normalizer.normalize_str("Hello\\x00World") 'Hello World' """ def __new__(cls, /) -> Nmt: ... @@ -310,7 +310,7 @@ class Replace(Normalizer): >>> Replace(".", " ").normalize_str("hello.world") 'hello world' >>> # Replace using a regex - >>> Replace(Regex(r"\s+"), " ").normalize_str("hello world") + >>> Replace(Regex(r"\\s+"), " ").normalize_str("hello world") 'hello world' """ def __new__(cls, /, pattern: str | Regex, content: str) -> Replace: ... diff --git a/bindings/python/py_src/tokenizers/pre_tokenizers.pyi b/bindings/python/py_src/tokenizers/pre_tokenizers.pyi index 41c2abc21e..6367c4981c 100644 --- a/bindings/python/py_src/tokenizers/pre_tokenizers.pyi +++ b/bindings/python/py_src/tokenizers/pre_tokenizers.pyi @@ -317,7 +317,7 @@ class Split(PreTokenizer): >>> pre_tokenizer.pre_tokenize_str("one,two,three") [('one', (0, 3)), ('two', (4, 7)), ('three', (8, 13))] >>> # Split using a regex, keeping the delimiter isolated - >>> Split(Regex(r"\s+"), behavior="isolated").pre_tokenize_str("hello world") + >>> Split(Regex(r"\\s+"), behavior="isolated").pre_tokenize_str("hello world") [('hello', (0, 5)), (' ', (5, 8)), ('world', (8, 13))] """ def __getnewargs__(self, /) -> tuple: ... @@ -355,7 +355,7 @@ class UnicodeScripts(PreTokenizer): @final class Whitespace(PreTokenizer): """ - This pre-tokenizer splits on word boundaries according to the ``\w+|[^\w\s]+`` + This pre-tokenizer splits on word boundaries according to the ``\\w+|[^\\w\\s]+`` regex pattern. It splits on word characters or characters that aren't words or whitespaces (punctuation such as hyphens, apostrophes, commas, etc.). diff --git a/bindings/python/tools/stub-gen/src/main.rs b/bindings/python/tools/stub-gen/src/main.rs index 8acc175b6a..31e012cbbd 100644 --- a/bindings/python/tools/stub-gen/src/main.rs +++ b/bindings/python/tools/stub-gen/src/main.rs @@ -139,7 +139,7 @@ fn generate_stubs(cdylib: &Path, out_dir: &Path) -> Result<(), Box Result<(), Box String { .replace("from .", &format!("from {root_module}.")) } +fn escape_docstring(docstring: &mut Option) { + if let Some(docstring) = docstring { + *docstring = docstring.replace('\\', "\\\\"); + } +} + +fn escape_class_docstrings(class: &mut Class) { + escape_docstring(&mut class.docstring); + for method in &mut class.methods { + escape_docstring(&mut method.docstring); + } + for attribute in &mut class.attributes { + escape_docstring(&mut attribute.docstring); + } + for inner_class in &mut class.inner_classes { + escape_class_docstrings(inner_class); + } +} + +/// Preserve runtime docstrings when pyo3-introspection emits ordinary Python +/// triple-quoted strings. Without this, Python interprets backslash sequences +/// and warns on regex escapes such as `\s` and `\w`. +fn escape_docstrings_for_python(module: &mut Module) { + escape_docstring(&mut module.docstring); + for function in &mut module.functions { + escape_docstring(&mut function.docstring); + } + for attribute in &mut module.attributes { + escape_docstring(&mut attribute.docstring); + } + for class in &mut module.classes { + escape_class_docstrings(class); + } + for submodule in &mut module.modules { + escape_docstrings_for_python(submodule); + } +} + /// Walk the introspected module tree and count classes, functions, and /// attributes that carry a docstring. Returns `(with_docstring, total)`. fn count_docstrings(module: &Module) -> (usize, usize) { @@ -242,6 +281,23 @@ fn assert_introspection_has_docstrings(module: &Module) { ); } +#[cfg(test)] +mod tests { + use super::escape_docstring; + + #[test] + fn escapes_backslashes_in_docstrings() { + let mut docstring = Some(r#"Regex(r"\s+") and "e\u0301""#.to_owned()); + + escape_docstring(&mut docstring); + + assert_eq!( + docstring.as_deref(), + Some(r#"Regex(r"\\s+") and "e\\u0301""#) + ); + } +} + fn build_extension(manifest_dir: &Path) -> Result<(), Box> { println!("Building and installing extension (release)..."); match Command::new("maturin").current_dir(manifest_dir).args(["develop", "--release"]).status() {