29 lines
1.1 KiB
Python
29 lines
1.1 KiB
Python
from compiler.tokenizer import tokenize, Token, L
|
|
|
|
def test_tokenizer_basic() -> None:
|
|
assert tokenize('aaa 123 bbb') == [
|
|
Token(location=L, type='identifier', text='aaa'),
|
|
Token(location=L, type='int_literal', text='123'),
|
|
Token(location=L, type='identifier', text='bbb'),
|
|
]
|
|
|
|
def test_tokenizer_newline() -> None:
|
|
assert tokenize('if 3\nwhile') == [
|
|
Token(location=L, type='identifier', text='if'),
|
|
Token(location=L, type='int_literal', text='3'),
|
|
Token(location=L, type='identifier', text='while'),
|
|
]
|
|
|
|
def test_tokenizer_commments() -> None:
|
|
assert tokenize('aaa 123 bbb ; ) ( >= \n ) # aksdjalksjdkajskdjasd\n != // Another comment $') == [
|
|
Token(location=L, type='identifier', text='aaa'),
|
|
Token(location=L, type='int_literal', text='123'),
|
|
Token(location=L, type='identifier', text='bbb'),
|
|
Token(location=L, type='punctuation', text=';'),
|
|
Token(location=L, type='punctuation', text=')'),
|
|
Token(location=L, type='punctuation', text='('),
|
|
Token(location=L, type='operator', text='>='),
|
|
Token(location=L, type='punctuation', text=')'),
|
|
Token(location=L, type='operator', text='!='),
|
|
]
|