---

2026-06-24 17:24:04 +02:00
commit 00c38a12d9
41 changed files with 7289 additions and 0 deletions
@@ -0,0 +1,28 @@
+from compiler.tokenizer import tokenize, Token, L
+
+def test_tokenizer_basic() -> None:
+  assert tokenize('aaa 123 bbb') == [
+    Token(location=L, type='identifier', text='aaa'),
+    Token(location=L, type='int_literal', text='123'),
+    Token(location=L, type='identifier', text='bbb'),
+  ]
+
+def test_tokenizer_newline() -> None:
+  assert tokenize('if  3\nwhile') == [
+    Token(location=L, type='identifier', text='if'),
+    Token(location=L, type='int_literal', text='3'),
+    Token(location=L, type='identifier', text='while'),
+  ]
+
+def test_tokenizer_commments() -> None:
+  assert tokenize('aaa 123 bbb ; ) ( >= \n ) # aksdjalksjdkajskdjasd\n != // Another comment $') == [
+    Token(location=L, type='identifier', text='aaa'),
+    Token(location=L, type='int_literal', text='123'),
+    Token(location=L, type='identifier', text='bbb'),
+    Token(location=L, type='punctuation', text=';'),
+    Token(location=L, type='punctuation', text=')'),
+    Token(location=L, type='punctuation', text='('),
+    Token(location=L, type='operator', text='>='),
+    Token(location=L, type='punctuation', text=')'),
+    Token(location=L, type='operator', text='!='),
+  ]