diff --git a/Cargo.toml b/Cargo.toml index 512db00..2aa177f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ exclude = ["JSONTestSuite"] async-tokio = ["dep:tokio"] [dependencies] +logos = { git = "https://github.com/Tpt/logos.git", branch = "is-prefix" } tokio = { version = "1.29", optional = true, features = ["io-util"] } [dev-dependencies] diff --git a/src/read.rs b/src/read.rs index d47ede8..25b3ed0 100644 --- a/src/read.rs +++ b/src/read.rs @@ -1,4 +1,5 @@ use crate::JsonEvent; +use logos::{Lexer, Logos}; use std::borrow::Cow; use std::cmp::{max, min}; use std::error::Error; @@ -14,7 +15,6 @@ const MAX_BUFFER_SIZE: usize = 4096 * 4096; /// Parses a JSON file from a [`Read`] implementation. /// -/// /// ``` /// use json_event_parser::{JsonEvent, ReaderJsonParser}; /// use std::borrow::Cow; @@ -266,23 +266,17 @@ impl TokioAsyncReaderJsonParser { /// use std::borrow::Cow; /// /// let mut reader = SliceJsonParser::new(b"{\"foo\": 1}"); +/// assert!(matches!(reader.next(), Some(Ok(JsonEvent::StartObject)))); /// assert!(matches!( -/// reader.parse_next(), -/// Some(Ok(JsonEvent::StartObject)) -/// )); -/// assert!(matches!( -/// reader.parse_next(), +/// reader.next(), /// Some(Ok(JsonEvent::ObjectKey(Cow::Borrowed("foo")))) /// )); /// assert!(matches!( -/// reader.parse_next(), +/// reader.next(), /// Some(Ok(JsonEvent::Number(Cow::Borrowed("1")))) /// )); -/// assert!(matches!( -/// reader.parse_next(), -/// Some(Ok(JsonEvent::EndObject)) -/// )); -/// assert!(matches!(reader.parse_next(), None)); +/// assert!(matches!(reader.next(), Some(Ok(JsonEvent::EndObject)))); +/// assert!(matches!(reader.next(), None)); /// # std::io::Result::Ok(()) /// ``` pub struct SliceJsonParser<'a> { @@ -298,9 +292,12 @@ impl<'a> SliceJsonParser<'a> { parser: LowLevelJsonParser::new(), } } +} - #[inline] - pub fn parse_next(&mut self) -> Option, JsonSyntaxError>> { +impl<'a> Iterator for SliceJsonParser<'a> { + type Item = Result, JsonSyntaxError>; + + fn next(&mut self) -> Option, JsonSyntaxError>> { let LowLevelJsonParserResult { event, consumed_bytes, @@ -310,14 +307,6 @@ impl<'a> SliceJsonParser<'a> { } } -impl<'a> Iterator for SliceJsonParser<'a> { - type Item = Result, JsonSyntaxError>; - - fn next(&mut self) -> Option, JsonSyntaxError>> { - self.parse_next() - } -} - /// A low-level JSON parser acting on a provided buffer. /// /// Does not allocate except a stack to check if array and object opening and closing are properly nested. @@ -507,7 +496,7 @@ impl LowLevelJsonParser { return (None, Some(e)); } if let JsonToken::String(key) = token { - (Some(JsonEvent::ObjectKey(key)), None) + (Some(JsonEvent::ObjectKey(key.into())), None) } else { (None, Some("Object keys must be strings".into())) } @@ -598,8 +587,8 @@ impl LowLevelJsonParser { ), JsonToken::Comma => (None, Some("Unexpected comma, no values to separate".into())), JsonToken::Colon => (None, Some("Unexpected colon, no key to follow".into())), - JsonToken::String(string) => (Some(JsonEvent::String(string)), None), - JsonToken::Number(number) => (Some(JsonEvent::Number(number)), None), + JsonToken::String(string) => (Some(JsonEvent::String(string.into())), None), + JsonToken::Number(number) => (Some(JsonEvent::Number(number.into())), None), JsonToken::True => (Some(JsonEvent::Boolean(true)), None), JsonToken::False => (Some(JsonEvent::Boolean(false)), None), JsonToken::Null => (Some(JsonEvent::Null), None), @@ -653,12 +642,48 @@ enum JsonToken<'a> { Comma, // , Colon, // : String(Cow<'a, str>), // "..." - Number(Cow<'a, str>), // 1.2e3 + Number(&'a str), // 1.2e3 True, // true False, // false Null, // null } +#[derive(Logos, Eq, PartialEq, Clone, Debug)] +#[logos(skip r"[ \t\r\n]+")] +#[logos(utf8 = false)] +enum LogosJsonToken<'a> { + #[token("[")] + OpeningSquareBracket, // [ + #[token("]")] + ClosingSquareBracket, // ] + #[token("{")] + OpeningCurlyBracket, // { + #[token("}")] + ClosingCurlyBracket, // } + #[token(",")] + Comma, // , + #[token(":")] + Colon, // : + #[regex(r#""[^"\\\x00-\x1F]*""#, |lex| { + let slice = lex.slice(); + str::from_utf8(&slice[1..slice.len() - 1]).expect("we already validated UTF-8") + })] + StringWithoutEscapes(&'a str), // "..." + #[regex(r#""[^"\\\x00-\x1F]*(\\(["\\bnfrt/]|u[a-fA-F0-9]{4})[^"\\\x00-\x1F]*)+""#, |lex| { + let slice = lex.slice(); + str::from_utf8(&slice[1..slice.len() - 1]).expect("we already validated UTF-8") + })] + StringWithEscapes(&'a str), // "..." + #[regex(r"-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?", |lex| str::from_utf8(lex.slice()).unwrap())] + Number(&'a str), // 1.2e3 + #[token("true")] + True, // true + #[token("false")] + False, // false + #[token("null")] + Null, // null +} + struct JsonLexer { file_offset: u64, file_line: u64, @@ -668,7 +693,6 @@ struct JsonLexer { } impl JsonLexer { - #[inline] fn read_next_token<'a>( &mut self, mut input_buffer: &'a [u8], @@ -686,202 +710,121 @@ impl JsonLexer { } } - // We skip whitespaces - let mut i = 0; - while let Some(c) = input_buffer.get(i) { - match *c { - b' ' | b'\t' => { - i += 1; - } - b'\n' => { - i += 1; - self.file_line += 1; - self.file_start_of_last_line = self.file_offset + u64::try_from(i).unwrap(); - } - b'\r' => { - i += 1; - if let Some(c) = input_buffer.get(i) { - if *c == b'\n' { - i += 1; // \r\n - } - } else if !is_ending { - // We need an extra byte to check if followed by \n - i -= 1; - self.file_offset += u64::try_from(i).unwrap(); - return None; - } - self.file_line += 1; - self.file_start_of_last_line = self.file_offset + u64::try_from(i).unwrap(); - } - _ => { - break; - } - } - } - self.file_offset += u64::try_from(i).unwrap(); - input_buffer = &input_buffer[i..]; - self.file_start_of_last_token = self.file_offset; - - if is_ending && input_buffer.is_empty() { + let mut lexer = if is_ending { + Lexer::new(input_buffer) + } else { + Lexer::new_partial(input_buffer) + }; + let Some(token) = lexer.next() else { + // A lexer can consume skipped whitespace without emitting a token. + // Preserve that progress while leaving an incomplete partial token + // untouched (its span ends at zero). + self.file_offset += u64::try_from(lexer.span().end).unwrap(); return None; - } - - // we get the first character - match *input_buffer.first()? { - b'{' => { - self.file_offset += 1; - Some(Ok(JsonToken::OpeningCurlyBracket)) - } - b'}' => { - self.file_offset += 1; - Some(Ok(JsonToken::ClosingCurlyBracket)) - } - b'[' => { - self.file_offset += 1; - Some(Ok(JsonToken::OpeningSquareBracket)) - } - b']' => { - self.file_offset += 1; - Some(Ok(JsonToken::ClosingSquareBracket)) - } - b',' => { - self.file_offset += 1; - Some(Ok(JsonToken::Comma)) - } - b':' => { - self.file_offset += 1; - Some(Ok(JsonToken::Colon)) - } - b'"' => self.read_string(input_buffer), - b't' => self.read_constant(input_buffer, is_ending, "true", JsonToken::True), - b'f' => self.read_constant(input_buffer, is_ending, "false", JsonToken::False), - b'n' => self.read_constant(input_buffer, is_ending, "null", JsonToken::Null), - b'-' | b'0'..=b'9' => self.read_number(input_buffer, is_ending), - c => { - self.file_offset += 1; - Some(Err(self.syntax_error( - self.file_offset - 1..self.file_offset, - if c < 128 { - format!("Unexpected char: '{}'", char::from(c)) - } else { - format!("Unexpected byte: \\x{c:X}") - }, - ))) - } - } + }; + let token = token + .map_err(|()| { + let span = lexer.span(); + self.syntax_error( + self.file_offset + u64::try_from(span.start).unwrap() + ..self.file_offset + u64::try_from(span.end).unwrap(), + format!( + "Unexpected input bytes: '{}'", + String::from_utf8_lossy(&input_buffer[span]) + ), + ) + }) + .and_then(|token| { + Ok(match token { + LogosJsonToken::OpeningSquareBracket => JsonToken::OpeningSquareBracket, + LogosJsonToken::ClosingSquareBracket => JsonToken::ClosingSquareBracket, + LogosJsonToken::OpeningCurlyBracket => JsonToken::OpeningCurlyBracket, + LogosJsonToken::ClosingCurlyBracket => JsonToken::ClosingCurlyBracket, + LogosJsonToken::Comma => JsonToken::Comma, + LogosJsonToken::Colon => JsonToken::Colon, + LogosJsonToken::StringWithoutEscapes(v) => JsonToken::String(v.into()), + LogosJsonToken::StringWithEscapes(v) => { + JsonToken::String(self.unescape_string(v.as_bytes())?.into()) + } + LogosJsonToken::Number(v) => JsonToken::Number(v), + LogosJsonToken::True => JsonToken::True, + LogosJsonToken::False => JsonToken::False, + LogosJsonToken::Null => JsonToken::Null, + }) + }); + let span = lexer.span(); + self.file_offset += u64::try_from(span.end).unwrap(); + self.file_start_of_last_token = + self.file_offset - u64::try_from(span.end - span.start).unwrap(); + // TODO file_line and file_start_of_last_line + Some(token) } - #[inline] - fn read_string<'a>( - &mut self, - input_buffer: &'a [u8], - ) -> Option, JsonSyntaxError>> { + fn unescape_string(&self, input_buffer: &[u8]) -> Result { + const TODO: u64 = 0; let mut error = None; - let mut string: Option<(String, usize)> = None; - let mut next_byte_offset = 1; - loop { - match *input_buffer.get(next_byte_offset)? { - b'"' => { - // end of string - let result = Some(if let Some(error) = error { - Err(error) - } else if let Some((mut string, read_until)) = string { - if read_until < next_byte_offset { - let (str, e) = self.decode_utf8( - &input_buffer[read_until..next_byte_offset], - self.file_offset + u64::try_from(read_until).unwrap(), - ); - error = error.or(e); - string.push_str(&str); - } - if let Some(error) = error { - Err(error) - } else { - Ok(JsonToken::String(Cow::Owned(string))) - } - } else { - let (string, error) = self - .decode_utf8(&input_buffer[1..next_byte_offset], self.file_offset + 1); - if let Some(error) = error { - Err(error) - } else { - Ok(JsonToken::String(string)) - } - }); - self.file_offset += u64::try_from(next_byte_offset).unwrap() + 1; - return result; - } + let mut output = Vec::with_capacity(input_buffer.len()); + let mut next_byte_offset = 0; + while let Some(next_byte) = input_buffer.get(next_byte_offset) { + match *next_byte { b'\\' => { // Escape sequences - if string.is_none() { - string = Some((String::new(), 1)) - } - let (string, read_until) = string.as_mut().unwrap(); - if *read_until < next_byte_offset { - let (str, e) = self.decode_utf8( - &input_buffer[*read_until..next_byte_offset], - self.file_offset + u64::try_from(*read_until).unwrap(), - ); - error = error.or(e); - string.push_str(&str); - } next_byte_offset += 1; - match *input_buffer.get(next_byte_offset)? { + match input_buffer[next_byte_offset] { b'"' => { - string.push('"'); + output.push(b'"'); next_byte_offset += 1; } b'\\' => { - string.push('\\'); + output.push(b'\\'); next_byte_offset += 1; } b'/' => { - string.push('/'); + output.push(b'/'); next_byte_offset += 1; } b'b' => { - string.push('\u{8}'); + output.push(b'\x08'); next_byte_offset += 1; } b'f' => { - string.push('\u{C}'); + output.push(b'\x0C'); next_byte_offset += 1; } b'n' => { - string.push('\n'); + output.push(b'\n'); next_byte_offset += 1; } b'r' => { - string.push('\r'); + output.push(b'\r'); next_byte_offset += 1; } b't' => { - string.push('\t'); + output.push(b'\t'); next_byte_offset += 1; } b'u' => { next_byte_offset += 1; - let val = input_buffer.get(next_byte_offset..next_byte_offset + 4)?; + let val = &input_buffer[next_byte_offset..next_byte_offset + 4]; next_byte_offset += 4; let code_point = match read_hexa_char(val) { Ok(cp) => cp, Err(e) => { error = error.or_else(|| { - let pos = self.file_offset - + u64::try_from(next_byte_offset).unwrap(); + let pos = TODO + u64::try_from(next_byte_offset).unwrap(); Some(self.syntax_error(pos - 4..pos, e)) }); char::REPLACEMENT_CHARACTER.into() } }; if let Some(c) = char::from_u32(code_point) { - string.push(c); + let mut buf = [0; 4]; + output.extend_from_slice(c.encode_utf8(&mut buf).as_bytes()); } else { let high_surrogate = code_point; if !(0xD800..=0xDBFF).contains(&high_surrogate) { error = error.or_else(|| { - let pos = self.file_offset - + u64::try_from(next_byte_offset).unwrap(); + let pos = TODO + u64::try_from(next_byte_offset).unwrap(); Some(self.syntax_error( pos - 6..pos, format!( @@ -891,12 +834,13 @@ impl JsonLexer { )) }); } - let val = - input_buffer.get(next_byte_offset..next_byte_offset + 6)?; + let val = input_buffer + .get(next_byte_offset..next_byte_offset + 6) + .unwrap_or(b"\0\0"); next_byte_offset += 6; if !val.starts_with(b"\\u") { error = error.or_else(|| { - let pos = self.file_offset + u64::try_from(next_byte_offset).unwrap(); + let pos =TODO + u64::try_from(next_byte_offset).unwrap(); Some(self.syntax_error( pos - 6..pos, format!( @@ -910,8 +854,8 @@ impl JsonLexer { Ok(cp) => cp, Err(e) => { error = error.or_else(|| { - let pos = self.file_offset - + u64::try_from(next_byte_offset).unwrap(); + let pos = + TODO + u64::try_from(next_byte_offset).unwrap(); Some(self.syntax_error(pos - 6..pos, e)) }); char::REPLACEMENT_CHARACTER.into() @@ -919,8 +863,7 @@ impl JsonLexer { }; if !(0xDC00..=0xDFFF).contains(&low_surrogate) { error = error.or_else(|| { - let pos = self.file_offset - + u64::try_from(next_byte_offset).unwrap(); + let pos = TODO + u64::try_from(next_byte_offset).unwrap(); Some(self.syntax_error( pos - 6..pos, format!( @@ -934,12 +877,17 @@ impl JsonLexer { + ((high_surrogate & 0x03FF) << 10) + (low_surrogate & 0x03FF); if let Some(c) = char::from_u32(code_point) { - string.push(c) + let mut buf = [0; 4]; + output.extend_from_slice(c.encode_utf8(&mut buf).as_bytes()); } else { - string.push(char::REPLACEMENT_CHARACTER); + let mut buf = [0; 4]; + output.extend_from_slice( + char::REPLACEMENT_CHARACTER + .encode_utf8(&mut buf) + .as_bytes(), + ); error = error.or_else(|| { - let pos = self.file_offset - + u64::try_from(next_byte_offset).unwrap(); + let pos = TODO + u64::try_from(next_byte_offset).unwrap(); Some(self.syntax_error( pos - 12..pos, format!( @@ -961,14 +909,16 @@ impl JsonLexer { format!("'\\{}' is not a valid escape sequence", char::from(c)), )) }); - string.push(char::REPLACEMENT_CHARACTER); + let mut buf = [0; 4]; + output.extend_from_slice( + char::REPLACEMENT_CHARACTER.encode_utf8(&mut buf).as_bytes(), + ); } } - *read_until = next_byte_offset; } c @ (0..=0x1F) => { error = error.or_else(|| { - let pos = self.file_offset + u64::try_from(next_byte_offset).unwrap(); + let pos = TODO + u64::try_from(next_byte_offset).unwrap(); Some(self.syntax_error( pos..pos + 1, format!("'{}' is not allowed in JSON strings", char::from(c)), @@ -976,153 +926,15 @@ impl JsonLexer { }); next_byte_offset += 1; } - _ => { - next_byte_offset += 1; - } - } - } - } - - #[inline] - fn read_constant( - &mut self, - input_buffer: &[u8], - is_ending: bool, - expected: &str, - value: JsonToken<'static>, - ) -> Option, JsonSyntaxError>> { - if input_buffer.get(..expected.len())? == expected.as_bytes() { - self.file_offset += u64::try_from(expected.len()).unwrap(); - return Some(Ok(value)); - } - let ascii_chars = input_buffer - .iter() - .take_while(|c| c.is_ascii_alphabetic()) - .count(); - if ascii_chars == input_buffer.len() && !is_ending { - return None; // We might read a bigger token - } - let read = max(1, ascii_chars); // We want to consume at least a byte - let start_offset = self.file_offset; - self.file_offset += u64::try_from(read).unwrap(); - Some(Err(self.syntax_error( - start_offset..self.file_offset, - format!("{} expected", expected), - ))) - } - - #[inline] - fn read_number<'a>( - &mut self, - input_buffer: &'a [u8], - is_ending: bool, - ) -> Option, JsonSyntaxError>> { - let mut next_byte_offset = 0; - if *input_buffer.get(next_byte_offset)? == b'-' { - next_byte_offset += 1; - } - // integer starting with first bytes - match *input_buffer.get(next_byte_offset)? { - b'0' => { - next_byte_offset += 1; - } - b'1'..=b'9' => { - next_byte_offset += 1; - next_byte_offset += read_digits(&input_buffer[next_byte_offset..], is_ending)?; - } - c => { - next_byte_offset += 1; - self.file_offset += u64::try_from(next_byte_offset).unwrap(); - return Some(Err(self.syntax_error( - self.file_offset - 1..self.file_offset, - format!("A number is not allowed to start with '{}'", char::from(c)), - ))); - } - } - - // Dot - if input_buffer.get(next_byte_offset).map_or_else( - || if is_ending { Some(None) } else { None }, - |c| Some(Some(*c)), - )? == Some(b'.') - { - next_byte_offset += 1; - let c = *input_buffer.get(next_byte_offset)?; - next_byte_offset += 1; - if !c.is_ascii_digit() { - self.file_offset += u64::try_from(next_byte_offset).unwrap(); - return Some(Err(self.syntax_error( - self.file_offset - 1..self.file_offset, - format!( - "A number fractional part must start with a digit and not '{}'", - char::from(c) - ), - ))); - } - next_byte_offset += read_digits(&input_buffer[next_byte_offset..], is_ending)?; - } - - // Exp - let c = input_buffer.get(next_byte_offset).map_or_else( - || if is_ending { Some(None) } else { None }, - |c| Some(Some(*c)), - )?; - if c == Some(b'e') || c == Some(b'E') { - next_byte_offset += 1; - match *input_buffer.get(next_byte_offset)? { - b'-' | b'+' => { - next_byte_offset += 1; - let c = *input_buffer.get(next_byte_offset)?; - next_byte_offset += 1; - if !c.is_ascii_digit() { - self.file_offset += u64::try_from(next_byte_offset).unwrap(); - return Some(Err(self.syntax_error( - self.file_offset - 1..self.file_offset, - format!( - "A number exponential part must contain at least a digit, '{}' found", - char::from(c) - ), - ))); - } - } - b'0'..=b'9' => { - next_byte_offset += 1; - } c => { + output.push(c); next_byte_offset += 1; - self.file_offset += u64::try_from(next_byte_offset).unwrap(); - return Some(Err(self.syntax_error( - self.file_offset - 1..self.file_offset, - format!( - "A number exponential part must start with +, - or a digit, '{}' found", - char::from(c) - ), - ))); } } - next_byte_offset += read_digits(&input_buffer[next_byte_offset..], is_ending)?; } - self.file_offset += u64::try_from(next_byte_offset).unwrap(); - Some(Ok(JsonToken::Number(Cow::Borrowed( - str::from_utf8(&input_buffer[..next_byte_offset]).unwrap(), - )))) - } - - #[inline] - fn decode_utf8<'a>( - &self, - input_buffer: &'a [u8], - start_position: u64, - ) -> (Cow<'a, str>, Option) { - match str::from_utf8(input_buffer) { - Ok(str) => (Cow::Borrowed(str), None), - Err(e) => ( - String::from_utf8_lossy(input_buffer), - Some({ - let pos = start_position + u64::try_from(e.valid_up_to()).unwrap(); - self.syntax_error(pos..pos + 1, format!("Invalid UTF-8: {e}")) - }), - ), + match error { + Some(error) => Err(error), + None => Ok(String::from_utf8(output).expect("Already UTF8")), } } diff --git a/tests/errors.rs b/tests/errors.rs index 2117bb4..e33260b 100644 --- a/tests/errors.rs +++ b/tests/errors.rs @@ -56,7 +56,7 @@ fn test_error_messages() { for (json, error) in entries { assert_eq!( SliceJsonParser::new(json) - .parse_next() + .next() .unwrap() .unwrap_err() .to_string(),