From 5ffd498220e3c8ea221e95917cc28ec6cb83b43e Mon Sep 17 00:00:00 2001
From: ahmedsamyh <ahmedsamyh10@gmail.com>
Date: Sat, 16 Nov 2024 22:10:06 +0500
Subject: [PATCH] Parse multiple tokens.

- Update tests
---
 main.momo                             |  4 +++-
 main.py                               | 25 +++++++++++++++++++------
 tests/02-string.momo.test             |  3 ++-
 tests/03-whitespaced-string.momo.test |  4 +++-
 tests/04-identifier.momo              |  3 +--
 tests/04-identifier.momo.test         |  3 ++-
 6 files changed, 30 insertions(+), 12 deletions(-)

diff --git a/main.momo b/main.momo
index dad80ff..516690c 100644
--- a/main.momo
+++ b/main.momo
@@ -1,2 +1,4 @@
 
-   foo : string = "Value of foo";
+fun main() -> int {
+    return 0;
+}
diff --git a/main.py b/main.py
index 0fee0a9..38b6a20 100644
--- a/main.py
+++ b/main.py
@@ -39,6 +39,12 @@ class TokenType(IntEnum):
     STRING = auto()
     COUNT = auto()
 
+token_type_as_str_map: { TokenType : str } = {
+    TokenType.IDENT : "Ident",
+    TokenType.STRING : "String",
+    TokenType.COUNT : "Count",
+}
+
 class Token:
     def __init__(self, typ: TokenType, value: str, loc: Loc):
         self.typ = typ
@@ -46,8 +52,7 @@ class Token:
         self.loc = loc
 
     def __str__(self):
-        return f"Token ({self.typ}, '{self.value}', {self.loc})"
-
+        return f"Token ({token_type_as_str_map[self.typ]}, '{self.value}', {self.loc})"
 
 class Parser:
     def __init__(self, filename: str):
@@ -108,14 +113,14 @@ class Parser:
 
         c = self.consume_char()
 
-        while c.isalpha() or c == '_' or c.isdigit() or self.eof():
+        while (c.isalpha() or c == '_' or c.isdigit()) and not self.eof():
             ident += c
             c = self.consume_char()
 
         return (ident, ident_loc)
 
     def left_trim(self):
-        while self.current_char().isspace():
+        while not self.eof() and self.current_char().isspace():
             if self.current_char() == '\n':
                 self.line += 1
                 self.bol = self.cur + 1
@@ -128,6 +133,9 @@ class Parser:
     def next_token(self) -> Token | None:
         self.left_trim()
 
+        if self.eof():
+            return None
+
         c = self.current_char()
 
         t: Token | None = None
@@ -154,10 +162,15 @@ def main():
 
     parser = Parser(filename)
 
+    tokens: [Token] = []
     token = parser.next_token()
+    tokens.append(token)
+    while token != None:
+        token = parser.next_token()
+        tokens.append(token)
 
-    print(token)
-
+    for t in tokens:
+        pprint.pp(str(t))
 
 if __name__ == '__main__':
     main()
diff --git a/tests/02-string.momo.test b/tests/02-string.momo.test
index 1bfd269..42e0ee9 100644
--- a/tests/02-string.momo.test
+++ b/tests/02-string.momo.test
@@ -1 +1,2 @@
-Token (2, 'This is a long ass string', ./tests/02-string.momo:1:0)
+"Token (String, 'This is a long ass string', ./tests/02-string.momo:1:0)"
+'None'
diff --git a/tests/03-whitespaced-string.momo.test b/tests/03-whitespaced-string.momo.test
index 0085efe..5523e7f 100644
--- a/tests/03-whitespaced-string.momo.test
+++ b/tests/03-whitespaced-string.momo.test
@@ -1 +1,3 @@
-Token (2, 'This is a long ass string', ./tests/03-whitespaced-string.momo:3:0)
+("Token (String, 'This is a long ass string', "
+ './tests/03-whitespaced-string.momo:3:0)')
+'None'
diff --git a/tests/04-identifier.momo b/tests/04-identifier.momo
index dad80ff..17dc7cc 100644
--- a/tests/04-identifier.momo
+++ b/tests/04-identifier.momo
@@ -1,2 +1 @@
-
-   foo : string = "Value of foo";
+   foo
diff --git a/tests/04-identifier.momo.test b/tests/04-identifier.momo.test
index 8eebbcc..17c5818 100644
--- a/tests/04-identifier.momo.test
+++ b/tests/04-identifier.momo.test
@@ -1 +1,2 @@
-Token (1, 'foo', ./tests/04-identifier.momo:2:3)
+"Token (Ident, 'foo', ./tests/04-identifier.momo:1:3)"
+'None'
-- 
2.39.5