from compiler.tokenizer import tokenize, Token, L

def test_tokenizer_basic() -> None:
  assert tokenize('aaa 123 bbb') == [
    Token(location=L, type='identifier', text='aaa'),
    Token(location=L, type='int_literal', text='123'),
    Token(location=L, type='identifier', text='bbb'),
  ]

def test_tokenizer_newline() -> None:
  assert tokenize('if  3\nwhile') == [
    Token(location=L, type='identifier', text='if'),
    Token(location=L, type='int_literal', text='3'),
    Token(location=L, type='identifier', text='while'),
  ]

def test_tokenizer_commments() -> None:
  assert tokenize('aaa 123 bbb ; ) ( >= \n ) # aksdjalksjdkajskdjasd\n != // Another comment $') == [
    Token(location=L, type='identifier', text='aaa'),
    Token(location=L, type='int_literal', text='123'),
    Token(location=L, type='identifier', text='bbb'),
    Token(location=L, type='punctuation', text=';'),
    Token(location=L, type='punctuation', text=')'),
    Token(location=L, type='punctuation', text='('),
    Token(location=L, type='operator', text='>='),
    Token(location=L, type='punctuation', text=')'),
    Token(location=L, type='operator', text='!='),
  ]