Pascal-compiler/LexicalAnalyzer.py at master · wdvictor/Pascal-compiler · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38


keywords = ['program', 'label', 'type', 'array', 'of', 'var', 'procedure'
                    ,'function', 'begin', 'end', 'if', 'then', 'else', 'while', 'do'
                    , 'or', 'and', 'div', 'not']


symbols = ['.', ';', ',', '(', ')', ':', '=',
           '<', '>', '+', '-', '*', '[', ':=', '..']


def get_tokens(code):
    label = ''
    tokens = []
    for str in code:
        if str in symbols:
            if label in keywords:
                tokens.append('<' + label + '>')
                tokens.append('<' + str + '>')
                label = ''
            else:
                if label is not '': tokens.append('#' + label + '#')
                tokens.append('<' + str + '>')
                label = ''
        elif str is ' ' or str is'\t' or str is '\n':
            if label in keywords:
                if label is not '' : tokens.append('<' + label + '>')
                label = ''
            else:
                if label is not '':
                    tokens.append('#' + label + '#')
                label = ''
        else:
            label += str
    return tokens