Revamp ast parsing and structure

src/lexer.rs

@@ -56,23 +56,62 @@ impl<'a> Lexer<'a> {
 
     /// Read the next token
     ///
+    #[expect(
+        clippy::too_many_lines,
+        reason = "token dispatch is intentionally kept together while the lexer is small"
+    )]
     pub fn next_token(&self) -> token::Token {
         // Skip whitespace
-        while self.peek().is_whitespace() {
+        while self.peek_char().is_whitespace() {
             self.read_char();
         }
-        while self.peek() == '\n' {
+        while self.peek_char() == '\n' {
             self.read_char();
         }
 
-        let ch = self.peek();
+        let ch = self.peek_char();
 
         match ch {
             // Example: single-character tokens
             ';' => {
                 self.read_char();
                 token::Token::Semicolon
             }
+            ',' => {
+                self.read_char();
+                token::Token::Comma
+            }
+            ':' => {
+                self.read_char();
+                token::Token::Colon
+            }
+            '=' => {
+                self.read_char();
+                token::Token::Equal
+            }
+            '<' => {
+                self.read_char();
+                match self.peek_char() {
+                    '=' => {
+                        self.read_char();
+                        token::Token::LessEqual
+                    }
+                    '>' => {
+                        self.read_char();
+                        token::Token::NotEqual
+                    }
+                    _ => token::Token::Less,
+                }
+            }
+            '>' => {
+                self.read_char();
+                if self.peek_char() == '=' {
+                    self.read_char();
+                    token::Token::GreaterEqual
+                } else {
+                    token::Token::Greater
+                }
+            }
             '+' => {
                 self.read_char();
                 token::Token::Plus
@@ -166,7 +205,7 @@ impl<'a> Lexer<'a> {
     /// Reads an identifier (e.g., variable/function name)
     fn read_identifier(&self) -> String {
         let mut ident = String::new();
-        while self.peek().is_alphanumeric() || self.peek() == '_' {
+        while self.peek_char().is_alphanumeric() || self.peek_char() == '_' {
             ident.push(self.read_char());
         }
         ident
@@ -175,16 +214,24 @@ impl<'a> Lexer<'a> {
     /// Reads a number (integer only for now)
     fn read_number(&self) -> i64 {
         let mut num = String::new();
-        while self.peek().is_numeric() {
+        while self.peek_char().is_numeric() {
             num.push(self.read_char());
         }
         num.parse().unwrap()
     }
 
+    /// Take a look at the next token without incrementing the location.
+    pub fn peek(&self) -> token::Token {
+        let location = self.location.get();
+        let token = self.next_token();
+        self.location.set(location);
+        token
+    }
+
     /// Take a look at the next character without incrimenting the location
     ///
     /// If used while the location is out of bounds, the function returns 0x00
-    fn peek(&self) -> char {
+    fn peek_char(&self) -> char {
         if self.location.get() < self.size.get() {
             self.input.chars().nth(self.location.get()).unwrap()
         } else {
@@ -200,15 +247,42 @@ mod test {
 
     #[test]
     #[timeout(100)]
-    fn test_peek() {
+    fn test_peek_char() {
         let lex: Lexer<'_> = Lexer::new("Alphabet");
 
-        assert_eq!(lex.peek(), 'A');
-        assert_eq!(lex.peek(), 'A');
-        assert_eq!(lex.peek(), 'A');
+        assert_eq!(lex.peek_char(), 'A');
+        assert_eq!(lex.peek_char(), 'A');
+        assert_eq!(lex.peek_char(), 'A');
         assert_eq!(lex.read_char(), 'A');
 
-        assert_eq!(lex.peek(), 'l');
+        assert_eq!(lex.peek_char(), 'l');
+    }
+
+    #[test]
+    #[timeout(100)]
+    fn test_peek_token() {
+        let lex: Lexer<'_> = Lexer::new("print \"HELLO\";");
+
+        assert_eq!(lex.peek(), token::Token::Print);
+        assert_eq!(lex.peek(), token::Token::Print);
+        assert_eq!(lex.next_token(), token::Token::Print);
+        assert_eq!(lex.peek(), token::Token::String("\"HELLO\"".to_string()));
+        assert_eq!(
+            lex.next_token(),
+            token::Token::String("\"HELLO\"".to_string())
+        );
+        assert_eq!(lex.next_token(), token::Token::Semicolon);
+    }
+
+    #[test]
+    #[timeout(100)]
+    fn test_peek_token_skips_whitespace_without_consuming() {
+        let lex: Lexer<'_> = Lexer::new("   <= 10");
+
+        assert_eq!(lex.peek(), token::Token::LessEqual);
+        assert_eq!(lex.peek(), token::Token::LessEqual);
+        assert_eq!(lex.next_token(), token::Token::LessEqual);
+        assert_eq!(lex.next_token(), token::Token::Integer(10));
     }
 
     #[test]
@@ -267,6 +341,34 @@ mod test {
         assert_eq!(tokens[2], token::Token::Identifier("GAMMA".to_string()));
     }
 
+    #[test]
+    #[timeout(100)]
+    fn test_parse_separators() {
+        let lex: Lexer<'_> = Lexer::new("A,B:C;");
+        let tokens: Vec<token::Token> = lex.parse_tokens();
+
+        assert_eq!(tokens[0], token::Token::Identifier("A".to_string()));
+        assert_eq!(tokens[1], token::Token::Comma);
+        assert_eq!(tokens[2], token::Token::Identifier("B".to_string()));
+        assert_eq!(tokens[3], token::Token::Colon);
+        assert_eq!(tokens[4], token::Token::Identifier("C".to_string()));
+        assert_eq!(tokens[5], token::Token::Semicolon);
+    }
+
+    #[test]
+    #[timeout(100)]
+    fn test_parse_comparison_operators() {
+        let lex: Lexer<'_> = Lexer::new("= < > <= >= <>");
+        let tokens: Vec<token::Token> = lex.parse_tokens();
+
+        assert_eq!(tokens[0], token::Token::Equal);
+        assert_eq!(tokens[1], token::Token::Less);
+        assert_eq!(tokens[2], token::Token::Greater);
+        assert_eq!(tokens[3], token::Token::LessEqual);
+        assert_eq!(tokens[4], token::Token::GreaterEqual);
+        assert_eq!(tokens[5], token::Token::NotEqual);
+    }
+
     #[test]
     #[timeout(100)]
     fn test_string_token() {
@@ -371,5 +473,6 @@ mod test {
             tokens[1],
             token::Token::String("\"HELLO WORLD\"".to_string())
         );
+        assert_eq!(tokens[2], token::Token::Semicolon);
     }
 }

src/lexer/token.rs

@@ -77,4 +77,16 @@ impl Token {
             _ => panic!("Not Identifier!"),
         }
     }
+
+    #[expect(
+        clippy::cast_possible_truncation,
+        reason = "numeric literals are currently represented as f64 in the AST"
+    )]
+    pub fn try_number(&self) -> Option<i64> {
+        match self {
+            Token::Float(num) => Some(*num as i64),
+            Token::Integer(num) => Some(*num),
+            _ => None,
+        }
+    }
 }