Check tokType in match compiler (#4219)

* Check tokType in match compiler

* Set keyword when tokenlist is missing

Author: pfultz2

lib/token.cpp

@@ -86,6 +86,37 @@ static const std::unordered_set<std::string> controlFlowKeywords = {
     "return"
 };
 
+// Another list of keywords
+static const std::unordered_set<std::string> baseKeywords = {
+    "asm",
+    "auto",
+    "break",
+    "case",
+    "const",
+    "continue",
+    "default",
+    "do",
+    "else",
+    "enum",
+    "extern",
+    "for",
+    "goto",
+    "if",
+    "inline",
+    "register",
+    "restrict",
+    "return",
+    "sizeof",
+    "static",
+    "struct",
+    "switch",
+    "typedef",
+    "union",
+    "volatile",
+    "while",
+    "void"
+};
+
 void Token::update_property_info()
 {
     setFlag(fIsControlFlowKeyword, controlFlowKeywords.find(mStr) != controlFlowKeywords.end());
@@ -102,6 +133,8 @@ void Token::update_property_info()
                 tokType(eVariable);
             else if (mTokensFrontBack && mTokensFrontBack->list && mTokensFrontBack->list->isKeyword(mStr))
                 tokType(eKeyword);
+            else if (baseKeywords.count(mStr) > 0)
+                tokType(eKeyword);
             else if (mTokType != eVariable && mTokType != eFunction && mTokType != eType && mTokType != eKeyword)
                 tokType(eName);
         } else if (std::isdigit((unsigned char)mStr[0]) || (mStr.length() > 1 && mStr[0] == '-' && std::isdigit((unsigned char)mStr[1])))

lib/tokenlist.cpp

@@ -52,6 +52,7 @@ TokenList::TokenList(const Settings* settings) :
     mIsCpp(false)
 {
     mTokensFrontBack.list = this;
+    mKeywords.insert("asm");
     mKeywords.insert("auto");
     mKeywords.insert("break");
     mKeywords.insert("case");

tools/matchcompiler.py

@@ -24,6 +24,77 @@
 import argparse
 import errno
 
+tokTypes = {
+    '+': ['eArithmeticalOp'],
+    '-': ['eArithmeticalOp'],
+    '*': ['eArithmeticalOp'],
+    '/': ['eArithmeticalOp'],
+    '%': ['eArithmeticalOp'],
+    '>>': ['eArithmeticalOp'],
+    '<<': ['eArithmeticalOp'],
+    '=': ['eAssignmentOp'],
+    '+=': ['eAssignmentOp'],
+    '-=': ['eAssignmentOp'],
+    '*=': ['eAssignmentOp'],
+    '/=': ['eAssignmentOp'],
+    '%=': ['eAssignmentOp'],
+    '&=': ['eAssignmentOp'],
+    '|=': ['eAssignmentOp'],
+    '^=': ['eAssignmentOp'],
+    '&': ['eBitOp'],
+    '^': ['eBitOp'],
+    '~': ['eBitOp'],
+    'true': ['eBoolean'],
+    'false': ['eBoolean'],
+    '{': ['eBracket'],
+    '}': ['eBracket'],
+    '<': ['eBracket', 'eComparisonOp'],
+    '>': ['eBracket', 'eComparisonOp'],
+    '==': ['eComparisonOp'],
+    '!=': ['eComparisonOp'],
+    '<=': ['eComparisonOp'],
+    '>=': ['eComparisonOp'],
+    '<=>': ['eComparisonOp'],
+    '...': ['eEllipsis'],
+    ',': ['eExtendedOp'],
+    '?': ['eExtendedOp'],
+    ':': ['eExtendedOp'],
+    '(': ['eExtendedOp'],
+    ')': ['eExtendedOp'],
+    '[': ['eExtendedOp', 'eLambda'],
+    ']': ['eExtendedOp', 'eLambda'],
+    '++': ['eIncDecOp'],
+    '--': ['eIncDecOp'],
+    'asm': ['eKeyword'],
+    'auto': ['eKeyword', 'eType'],
+    'break': ['eKeyword'],
+    'case': ['eKeyword'],
+    'const': ['eKeyword'],
+    'continue': ['eKeyword'],
+    'default': ['eKeyword'],
+    'do': ['eKeyword'],
+    'else': ['eKeyword'],
+    'enum': ['eKeyword'],
+    'extern': ['eKeyword'],
+    'for': ['eKeyword'],
+    'goto': ['eKeyword'],
+    'if': ['eKeyword'],
+    'inline': ['eKeyword'],
+    'register': ['eKeyword'],
+    'restrict': ['eKeyword'],
+    'return': ['eKeyword'],
+    'sizeof': ['eKeyword'],
+    'static': ['eKeyword'],
+    'struct': ['eKeyword'],
+    'switch': ['eKeyword'],
+    'typedef': ['eKeyword'],
+    'union': ['eKeyword'],
+    'volatile': ['eKeyword'],
+    'while': ['eKeyword'],
+    'void': ['eKeyword', 'eType'],
+    '&&': ['eLogicalOp'],
+    '!': ['eLogicalOp']
+}
 
 class MatchCompiler:
 
@@ -117,7 +188,9 @@ def _compileCmd(tok):
             return '(tok->isName() && tok->varId() == varid)'
         elif (len(tok) > 2) and (tok[0] == "%"):
             print("unhandled:" + tok)
-
+        elif tok in tokTypes:
+            cond = ' || '.join(['tok->tokType() == Token::{}'.format(tokType) for tokType in tokTypes[tok]])
+            return '(({cond}) && tok->str() == MatchCompiler::makeConstString("{tok}"))'.format(cond=cond, tok=tok)
         return (
             '(tok->str() == MatchCompiler::makeConstString("' + tok + '"))'
         )