lexer: track wirelog grammar (crc32 keywords, string escapes)

tests/test_lexer.c

@@ -190,6 +190,8 @@ test_keywords(void)
         { "sha256",     WG_TOKEN_SHA256 },
         { "hmac_sha256",WG_TOKEN_HMAC_SHA256 },
         { "uuid4",      WG_TOKEN_UUID4 },
+        { "crc32_ethernet",   WG_TOKEN_CRC32_ETH },
+        { "crc32_castagnoli", WG_TOKEN_CRC32_CAST },
         { "strlen",     WG_TOKEN_STRLEN },
         { "str_prefix", WG_TOKEN_STR_PREFIX },
         { "to_string",  WG_TOKEN_TO_STRING },
@@ -283,6 +285,31 @@ test_integer_and_string(void)
                          e, sizeof(e) / sizeof(e[0]));
 }
 
+static int
+test_string_escapes(void)
+{
+    /* \" and \\ are escaped bytes inside a string and must not end it.
+     * The token spans the whole literal, raw bytes preserved (no decode). */
+    expect_t e1[] = {
+        { WG_TOKEN_STRING, "\"a\\\"b\"" },   /* source: "a\"b"  */
+        { WG_TOKEN_EOF,    NULL },
+    };
+    expect_t e2[] = {
+        { WG_TOKEN_STRING, "\"a\\\\\"" },    /* source: "a\\"   */
+        { WG_TOKEN_EOF,    NULL },
+    };
+    /* A lone trailing backslash before the closing quote escapes that
+     * quote, so the literal is unterminated -> ERROR. */
+    expect_t e3[] = {
+        { WG_TOKEN_ERROR, NULL },            /* source: "a\"    */
+    };
+    int rc = 0;
+    rc |= expect_stream("string_escape_quote", "\"a\\\"b\"", e1, 2);
+    rc |= expect_stream("string_escape_backslash", "\"a\\\\\"", e2, 2);
+    rc |= expect_stream("string_escape_unterminated", "\"a\\\"", e3, 1);
+    return rc;
+}
+
 static int
 test_comments(void)
 {
@@ -496,6 +523,7 @@ main(void)
         test_unknown_directive_rollback,
         test_underscore_disambiguation,
         test_integer_and_string,
+        test_string_escapes,
         test_comments,
         test_block_comment_multiline,
         test_crlf,

wiig/lexer/lexer.c

@@ -159,8 +159,19 @@ scan_block_comment(wg_lexer_t *lex)
 static wg_lexer_token_t
 scan_string(wg_lexer_t *lex)
 {
-    /* Opening quote already consumed. */
+    /* Opening quote already consumed. A backslash that escapes a quote
+     * or another backslash (\" or \\) does not terminate the literal;
+     * wirelog gained this rule with its string-escape fix. wiig keeps
+     * the bytes raw (it is a formatter, not an evaluator) but must not
+     * stop early on an escaped quote, else the trailing bytes lex as
+     * stray tokens and an unterminated string. */
     while (!at_end(lex) && peek(lex) != '"') {
+        if (peek(lex) == '\\'
+            && (peek_at(lex, 1) == '"' || peek_at(lex, 1) == '\\')) {
+            advance(lex); /* backslash */
+            advance(lex); /* the escaped '"' or '\\' */
+            continue;
+        }
         if (peek(lex) == '\n') {
             /* Unterminated on this line: emit ERROR for what we have. */
             return make_token(lex, WG_TOKEN_ERROR);
@@ -233,6 +244,9 @@ static const wg_keyword_t WG_KEYWORDS[] = {
     /* uuid */
     { "uuid4",        5,  WG_TOKEN_UUID4 },
     { "uuid5",        5,  WG_TOKEN_UUID5 },
+    /* crc-32 */
+    { "crc32_ethernet",   14, WG_TOKEN_CRC32_ETH },
+    { "crc32_castagnoli", 16, WG_TOKEN_CRC32_CAST },
     /* string fns */
     { "strlen",       6,  WG_TOKEN_STRLEN },
     { "cat",          3,  WG_TOKEN_CAT },
@@ -552,6 +566,8 @@ wg_lexer_token_kind_str(wg_lexer_token_kind_t kind)
     case WG_TOKEN_HMAC_SHA256:         return "HMAC_SHA256";
     case WG_TOKEN_UUID4:               return "UUID4";
     case WG_TOKEN_UUID5:               return "UUID5";
+    case WG_TOKEN_CRC32_ETH:           return "CRC32_ETHERNET";
+    case WG_TOKEN_CRC32_CAST:          return "CRC32_CASTAGNOLI";
     case WG_TOKEN_STRLEN:              return "STRLEN";
     case WG_TOKEN_CAT:                 return "CAT";
     case WG_TOKEN_SUBSTR:              return "SUBSTR";

wiig/lexer/token_kind.h

@@ -115,6 +115,10 @@ typedef enum {
     WG_TOKEN_UUID4,                 /* uuid4 */
     WG_TOKEN_UUID5,                 /* uuid5 */
 
+    /* CRC-32 checksum keywords -------------------------------------------- */
+    WG_TOKEN_CRC32_ETH,             /* crc32_ethernet */
+    WG_TOKEN_CRC32_CAST,            /* crc32_castagnoli */
+
     /* String function keywords -------------------------------------------- */
     WG_TOKEN_STRLEN,                /* strlen */
     WG_TOKEN_CAT,                   /* cat */