Parse multiple tokens.

- Update tests

diff --git a/main.momo b/main.momo
index dad80ffce9ff2e9e8164c4a0c838f6e73f9a1dbf..516690c397f5e695e66e7000a92aa3fc0688d3d2 100644 (file)
--- a/main.momo
+++ b/main.momo
@@ -1,2 +1,4 @@
 
-   foo : string = "Value of foo";
+fun main() -> int {
+    return 0;
+}

diff --git a/main.py b/main.py
index 0fee0a9507a37f0f790d028a3725606da068c408..38b6a201c4fea1bf61cdc8397e133953adcb40dd 100644 (file)
--- a/main.py
+++ b/main.py
@@ -39,6 +39,12 @@ class TokenType(IntEnum):
     STRING = auto()
     COUNT = auto()
 
+token_type_as_str_map: { TokenType : str } = {
+    TokenType.IDENT : "Ident",
+    TokenType.STRING : "String",
+    TokenType.COUNT : "Count",
+}
+
 class Token:
     def __init__(self, typ: TokenType, value: str, loc: Loc):
         self.typ = typ
@@ -46,8 +52,7 @@ class Token:
         self.loc = loc
 
     def __str__(self):
-        return f"Token ({self.typ}, '{self.value}', {self.loc})"
-
+        return f"Token ({token_type_as_str_map[self.typ]}, '{self.value}', {self.loc})"
 
 class Parser:
     def __init__(self, filename: str):
@@ -108,14 +113,14 @@ class Parser:
 
         c = self.consume_char()
 
-        while c.isalpha() or c == '_' or c.isdigit() or self.eof():
+        while (c.isalpha() or c == '_' or c.isdigit()) and not self.eof():
             ident += c
             c = self.consume_char()
 
         return (ident, ident_loc)
 
     def left_trim(self):
-        while self.current_char().isspace():
+        while not self.eof() and self.current_char().isspace():
             if self.current_char() == '\n':
                 self.line += 1
                 self.bol = self.cur + 1
@@ -128,6 +133,9 @@ class Parser:
     def next_token(self) -> Token | None:
         self.left_trim()
 
+        if self.eof():
+            return None
+
         c = self.current_char()
 
         t: Token | None = None
@@ -154,10 +162,15 @@ def main():
 
     parser = Parser(filename)
 
+    tokens: [Token] = []
     token = parser.next_token()
+    tokens.append(token)
+    while token != None:
+        token = parser.next_token()
+        tokens.append(token)
 
-    print(token)
-
+    for t in tokens:
+        pprint.pp(str(t))
 
 if __name__ == '__main__':
     main()

diff --git a/tests/02-string.momo.test b/tests/02-string.momo.test
index 1bfd26967b88e2111285bd5f101546bfb6584901..42e0ee90ddc1b5a7038b798bca745a940041ced8 100644 (file)
--- a/tests/02-string.momo.test
+++ b/tests/02-string.momo.test
@@ -1 +1,2 @@
-Token (2, 'This is a long ass string', ./tests/02-string.momo:1:0)
+"Token (String, 'This is a long ass string', ./tests/02-string.momo:1:0)"
+'None'

diff --git a/tests/03-whitespaced-string.momo.test b/tests/03-whitespaced-string.momo.test
index 0085efeca088e4c1e5287bfcc3380d25629f847b..5523e7f80e3a6f1ae22222d28621d3053a027fab 100644 (file)
--- a/tests/03-whitespaced-string.momo.test
+++ b/tests/03-whitespaced-string.momo.test
@@ -1 +1,3 @@
-Token (2, 'This is a long ass string', ./tests/03-whitespaced-string.momo:3:0)
+("Token (String, 'This is a long ass string', "
+ './tests/03-whitespaced-string.momo:3:0)')
+'None'

diff --git a/tests/04-identifier.momo b/tests/04-identifier.momo
index dad80ffce9ff2e9e8164c4a0c838f6e73f9a1dbf..17dc7cc7bec0e1294080e6cdb0a1306018e4d5f5 100644 (file)
--- a/tests/04-identifier.momo
+++ b/tests/04-identifier.momo
@@ -1,2 +1 @@
-
-   foo : string = "Value of foo";
+   foo

diff --git a/tests/04-identifier.momo.test b/tests/04-identifier.momo.test
index 8eebbcc55e3c3c72ee1c81274064d8866e541df3..17c5818f3797aeb94c3bdf1b9e16a0be5f097b0c 100644 (file)
--- a/tests/04-identifier.momo.test
+++ b/tests/04-identifier.momo.test
@@ -1 +1,2 @@
-Token (1, 'foo', ./tests/04-identifier.momo:2:3)
+"Token (Ident, 'foo', ./tests/04-identifier.momo:1:3)"
+'None'