diff --git a/tests/test_lexer.c b/tests/test_lexer.c index 81425d4..6467716 100644 --- a/tests/test_lexer.c +++ b/tests/test_lexer.c @@ -190,6 +190,8 @@ test_keywords(void) { "sha256", WG_TOKEN_SHA256 }, { "hmac_sha256",WG_TOKEN_HMAC_SHA256 }, { "uuid4", WG_TOKEN_UUID4 }, + { "crc32_ethernet", WG_TOKEN_CRC32_ETH }, + { "crc32_castagnoli", WG_TOKEN_CRC32_CAST }, { "strlen", WG_TOKEN_STRLEN }, { "str_prefix", WG_TOKEN_STR_PREFIX }, { "to_string", WG_TOKEN_TO_STRING }, @@ -283,6 +285,31 @@ test_integer_and_string(void) e, sizeof(e) / sizeof(e[0])); } +static int +test_string_escapes(void) +{ + /* \" and \\ are escaped bytes inside a string and must not end it. + * The token spans the whole literal, raw bytes preserved (no decode). */ + expect_t e1[] = { + { WG_TOKEN_STRING, "\"a\\\"b\"" }, /* source: "a\"b" */ + { WG_TOKEN_EOF, NULL }, + }; + expect_t e2[] = { + { WG_TOKEN_STRING, "\"a\\\\\"" }, /* source: "a\\" */ + { WG_TOKEN_EOF, NULL }, + }; + /* A lone trailing backslash before the closing quote escapes that + * quote, so the literal is unterminated -> ERROR. */ + expect_t e3[] = { + { WG_TOKEN_ERROR, NULL }, /* source: "a\" */ + }; + int rc = 0; + rc |= expect_stream("string_escape_quote", "\"a\\\"b\"", e1, 2); + rc |= expect_stream("string_escape_backslash", "\"a\\\\\"", e2, 2); + rc |= expect_stream("string_escape_unterminated", "\"a\\\"", e3, 1); + return rc; +} + static int test_comments(void) { @@ -496,6 +523,7 @@ main(void) test_unknown_directive_rollback, test_underscore_disambiguation, test_integer_and_string, + test_string_escapes, test_comments, test_block_comment_multiline, test_crlf, diff --git a/wiig/lexer/lexer.c b/wiig/lexer/lexer.c index 7cdf9e6..0c2cce6 100644 --- a/wiig/lexer/lexer.c +++ b/wiig/lexer/lexer.c @@ -159,8 +159,19 @@ scan_block_comment(wg_lexer_t *lex) static wg_lexer_token_t scan_string(wg_lexer_t *lex) { - /* Opening quote already consumed. */ + /* Opening quote already consumed. A backslash that escapes a quote + * or another backslash (\" or \\) does not terminate the literal; + * wirelog gained this rule with its string-escape fix. wiig keeps + * the bytes raw (it is a formatter, not an evaluator) but must not + * stop early on an escaped quote, else the trailing bytes lex as + * stray tokens and an unterminated string. */ while (!at_end(lex) && peek(lex) != '"') { + if (peek(lex) == '\\' + && (peek_at(lex, 1) == '"' || peek_at(lex, 1) == '\\')) { + advance(lex); /* backslash */ + advance(lex); /* the escaped '"' or '\\' */ + continue; + } if (peek(lex) == '\n') { /* Unterminated on this line: emit ERROR for what we have. */ return make_token(lex, WG_TOKEN_ERROR); @@ -233,6 +244,9 @@ static const wg_keyword_t WG_KEYWORDS[] = { /* uuid */ { "uuid4", 5, WG_TOKEN_UUID4 }, { "uuid5", 5, WG_TOKEN_UUID5 }, + /* crc-32 */ + { "crc32_ethernet", 14, WG_TOKEN_CRC32_ETH }, + { "crc32_castagnoli", 16, WG_TOKEN_CRC32_CAST }, /* string fns */ { "strlen", 6, WG_TOKEN_STRLEN }, { "cat", 3, WG_TOKEN_CAT }, @@ -552,6 +566,8 @@ wg_lexer_token_kind_str(wg_lexer_token_kind_t kind) case WG_TOKEN_HMAC_SHA256: return "HMAC_SHA256"; case WG_TOKEN_UUID4: return "UUID4"; case WG_TOKEN_UUID5: return "UUID5"; + case WG_TOKEN_CRC32_ETH: return "CRC32_ETHERNET"; + case WG_TOKEN_CRC32_CAST: return "CRC32_CASTAGNOLI"; case WG_TOKEN_STRLEN: return "STRLEN"; case WG_TOKEN_CAT: return "CAT"; case WG_TOKEN_SUBSTR: return "SUBSTR"; diff --git a/wiig/lexer/token_kind.h b/wiig/lexer/token_kind.h index c8281da..c6937e7 100644 --- a/wiig/lexer/token_kind.h +++ b/wiig/lexer/token_kind.h @@ -115,6 +115,10 @@ typedef enum { WG_TOKEN_UUID4, /* uuid4 */ WG_TOKEN_UUID5, /* uuid5 */ + /* CRC-32 checksum keywords -------------------------------------------- */ + WG_TOKEN_CRC32_ETH, /* crc32_ethernet */ + WG_TOKEN_CRC32_CAST, /* crc32_castagnoli */ + /* String function keywords -------------------------------------------- */ WG_TOKEN_STRLEN, /* strlen */ WG_TOKEN_CAT, /* cat */