From 220b2fb42036a21da1a981f6dc7c5a641efccb1c Mon Sep 17 00:00:00 2001 From: Noah Graf Date: Sat, 25 Apr 2026 03:45:33 +0000 Subject: [PATCH 1/6] Renamed ast file --- src/parser/{ast.rs => outdated_ast.rs} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/parser/{ast.rs => outdated_ast.rs} (100%) diff --git a/src/parser/ast.rs b/src/parser/outdated_ast.rs similarity index 100% rename from src/parser/ast.rs rename to src/parser/outdated_ast.rs From 7b2171ad7cbe9f74676142462297d0058057abde Mon Sep 17 00:00:00 2001 From: Noah Graf Date: Sat, 25 Apr 2026 04:11:58 +0000 Subject: [PATCH 2/6] Added new ast structure --- src/parser.rs | 6 +- src/parser/ast.rs | 124 +++++++++++++++++++++++ src/parser/expression.rs | 201 +++++++++++++++++++++++++++++++++++++ src/parser/outdated_ast.rs | 2 +- 4 files changed, 330 insertions(+), 3 deletions(-) create mode 100644 src/parser/ast.rs create mode 100644 src/parser/expression.rs diff --git a/src/parser.rs b/src/parser.rs index a426ea8..e40d219 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -7,9 +7,11 @@ use crate::{ Lexer, token::{self, Token}, }, - parser::ast::{ASTBinaryOperator, ASTBinaryOperatorKind, ASTExpression, ASTStatement}, + parser::outdated_ast::{ASTBinaryOperator, ASTBinaryOperatorKind, ASTExpression, ASTStatement}, }; -pub mod ast; +mod ast; +mod expression; +mod outdated_ast; #[allow(dead_code)] pub struct Parser { diff --git a/src/parser/ast.rs b/src/parser/ast.rs new file mode 100644 index 0000000..2158f74 --- /dev/null +++ b/src/parser/ast.rs @@ -0,0 +1,124 @@ +//! Defines the AST types for the language and the grammar for how it is composed. +//! +//! This grammar only handles the keywords currently produced by the lexer: +//! `print`, `let`, `if`, `then`, `else`, `for`, `to`, `next`, `goto`, `gosub`, +//! `return`, `end`, and `rem`. +//! +//! Modified from the source here: +//! +//! +//! BNF grammar for the language: +//! +//! ```text +//! ::= * +//! ::= ? +//! ::= +//! ::= | ":" +//! +//! ::= +//! | +//! | +//! | +//! | +//! | +//! | +//! | +//! +//! ::= "let" "=" +//! | "=" +//! +//! ::= "if" "then" ? +//! | "goto" +//! ::= | +//! ::= "else" +//! ::= | +//! +//! ::= "for" "=" "to" +//! | "next" ? +//! +//! ::= "print" ? +//! ::= | +//! ::= | | +//! ::= "," | ";" +//! +//! ::= "gosub" +//! | "return" +//! +//! ::= "end" +//! ::= "rem" ? +//! ::= | | +//! +//! ::= +//! ::= +//! | +//! ::= "=" | "<" | ">" | "<=" | ">=" | "<>" +//! ::= +//! | "+" +//! | "-" +//! +//! ::= +//! | "*" +//! | "/" +//! ::= +//! | +//! | +//! | "(" ")" +//! +//! ::= lexer identifier token +//! ::= lexer integer token +//! ::= lexer float token +//! ::= lexer string token +//! ::= lexer char token +//! ``` +#![allow(dead_code)] + +use crate::parser::expression::Expression; + +pub struct Program { + pub lines: Vec, +} + +pub struct Line { + pub number: Option, + pub statements: Vec, +} + +pub enum Statement { + Let { + name: String, + value: Expression, + }, + Print { + items: Vec, + }, + If { + condition: Expression, + then_branch: Box, + else_branch: Option>, + }, + For { + name: String, + start: Expression, + end: Expression, + }, + Next { + name: Option, + }, + Goto(Target), + Gosub(Target), + Return, + End, + Rem(Option), + Expression(Expression), +} + +pub enum Target { + Line(i64), + Expression(Expression), +} + +pub enum PrintItem { + Expression(Expression), + String(String), + Char(char), +} diff --git a/src/parser/expression.rs b/src/parser/expression.rs new file mode 100644 index 0000000..bab5a48 --- /dev/null +++ b/src/parser/expression.rs @@ -0,0 +1,201 @@ +//! Expression AST nodes. + +#![allow(dead_code)] + +use crate::lexer::token::Token; + +/// An expression node in the AST. +#[derive(Debug, PartialEq)] +pub enum Expression { + /// A literal value. + Literal(LiteralExpression), + /// A named variable or symbol. + Identifier(IdentifierExpression), + /// A binary operation. + Binary(BinaryExpression), + /// A parenthesized expression. + Grouping(GroupingExpression), +} + +/// A literal expression value. +#[derive(Debug, PartialEq)] +pub enum LiteralExpression { + /// An integer literal. + Integer(i64), + /// A floating point literal. + Float(f64), + /// A string literal. + String(String), + /// A character literal. + Char(char), +} + +/// An identifier expression. +#[derive(Debug, PartialEq)] +pub struct IdentifierExpression { + name: String, +} + +/// A binary expression. +#[derive(Debug, PartialEq)] +pub struct BinaryExpression { + left: Box, + operator: BinaryOperator, + right: Box, +} + +/// A parenthesized expression. +#[derive(Debug, PartialEq)] +pub struct GroupingExpression { + expression: Box, +} + +/// A binary expression operator. +#[derive(Debug, PartialEq)] +pub enum BinaryOperator { + /// Addition. + Plus, + /// Subtraction. + Minus, + /// Multiplication. + Multiply, + /// Division. + Divide, + /// Equality comparison. + Equal, + /// Less-than comparison. + Less, + /// Greater-than comparison. + Greater, + /// Less-than-or-equal comparison. + LessEqual, + /// Greater-than-or-equal comparison. + GreaterEqual, + /// Not-equal comparison. + NotEqual, +} + +impl Expression { + /// Creates an integer literal expression. + pub fn integer(value: i64) -> Self { + Self::Literal(LiteralExpression::Integer(value)) + } + + /// Creates a floating point literal expression. + pub fn float(value: f64) -> Self { + Self::Literal(LiteralExpression::Float(value)) + } + + /// Creates a string literal expression. + pub fn string(value: String) -> Self { + Self::Literal(LiteralExpression::String(value)) + } + + /// Creates a character literal expression. + pub fn char(value: char) -> Self { + Self::Literal(LiteralExpression::Char(value)) + } + + /// Creates an identifier expression. + pub fn identifier(name: String) -> Self { + Self::Identifier(IdentifierExpression { name }) + } + + /// Creates a binary expression. + pub fn binary(left: Self, operator: BinaryOperator, right: Self) -> Self { + Self::Binary(BinaryExpression { + left: Box::new(left), + operator, + right: Box::new(right), + }) + } + + /// Creates a parenthesized expression. + pub fn grouping(expression: Self) -> Self { + Self::Grouping(GroupingExpression { + expression: Box::new(expression), + }) + } +} + +impl BinaryOperator { + /// Returns the binding precedence for this operator. + pub fn precedence(&self) -> u8 { + match self { + Self::Equal + | Self::Less + | Self::Greater + | Self::LessEqual + | Self::GreaterEqual + | Self::NotEqual => 1, + Self::Plus | Self::Minus => 2, + Self::Multiply | Self::Divide => 3, + } + } + + /// Converts a lexer token into a binary operator, when possible. + pub fn from_token(token: &Token) -> Option { + match token { + Token::Plus => Some(Self::Plus), + Token::Minus => Some(Self::Minus), + Token::Asterisk | Token::Star => Some(Self::Multiply), + Token::Slash => Some(Self::Divide), + Token::Equal => Some(Self::Equal), + Token::Less => Some(Self::Less), + Token::Greater => Some(Self::Greater), + Token::LessEqual => Some(Self::LessEqual), + Token::GreaterEqual => Some(Self::GreaterEqual), + Token::NotEqual => Some(Self::NotEqual), + _ => None, + } + } +} + +impl From for Expression { + fn from(value: i64) -> Self { + Self::integer(value) + } +} + +impl From for Expression { + fn from(value: f64) -> Self { + Self::float(value) + } +} + +#[cfg(test)] +mod tests { + use super::{BinaryOperator, Expression}; + use crate::lexer::token::Token; + + #[test] + fn binary_operator_from_token() { + assert_eq!( + BinaryOperator::from_token(&Token::Plus), + Some(BinaryOperator::Plus) + ); + assert_eq!( + BinaryOperator::from_token(&Token::LessEqual), + Some(BinaryOperator::LessEqual) + ); + assert_eq!(BinaryOperator::from_token(&Token::Print), None); + } + + #[test] + fn binary_expression_node() { + let expression = Expression::binary( + Expression::integer(1), + BinaryOperator::Plus, + Expression::identifier("x".to_string()), + ); + + assert_eq!( + expression, + Expression::Binary(super::BinaryExpression { + left: Box::new(Expression::integer(1)), + operator: BinaryOperator::Plus, + right: Box::new(Expression::identifier("x".to_string())), + }) + ); + } +} diff --git a/src/parser/outdated_ast.rs b/src/parser/outdated_ast.rs index fca82d0..7e7ad68 100644 --- a/src/parser/outdated_ast.rs +++ b/src/parser/outdated_ast.rs @@ -235,7 +235,7 @@ impl ASTPrinter { mod test { use crate::{ lexer::{Lexer, token}, - parser::{Parser, ast::*}, + parser::{Parser, outdated_ast::*}, }; use ntest::timeout; From 679801e359933861d84d34a14465d1c1a1444fca Mon Sep 17 00:00:00 2001 From: Noah Graf Date: Sat, 25 Apr 2026 04:16:54 +0000 Subject: [PATCH 3/6] Patched the lexer with more tokens --- src/lexer.rs | 68 +++++++++++++++++++++++++++++++++++++++++++ tests/input/print.bsc | 2 +- 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/src/lexer.rs b/src/lexer.rs index abf6180..66b6890 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -56,6 +56,10 @@ impl<'a> Lexer<'a> { /// Read the next token /// + #[expect( + clippy::too_many_lines, + reason = "token dispatch is intentionally kept together while the lexer is small" + )] pub fn next_token(&self) -> token::Token { // Skip whitespace while self.peek().is_whitespace() { @@ -73,6 +77,41 @@ impl<'a> Lexer<'a> { self.read_char(); token::Token::Semicolon } + ',' => { + self.read_char(); + token::Token::Comma + } + ':' => { + self.read_char(); + token::Token::Colon + } + '=' => { + self.read_char(); + token::Token::Equal + } + '<' => { + self.read_char(); + match self.peek() { + '=' => { + self.read_char(); + token::Token::LessEqual + } + '>' => { + self.read_char(); + token::Token::NotEqual + } + _ => token::Token::Less, + } + } + '>' => { + self.read_char(); + if self.peek() == '=' { + self.read_char(); + token::Token::GreaterEqual + } else { + token::Token::Greater + } + } '+' => { self.read_char(); token::Token::Plus @@ -267,6 +306,34 @@ mod test { assert_eq!(tokens[2], token::Token::Identifier("GAMMA".to_string())); } + #[test] + #[timeout(100)] + fn test_parse_separators() { + let lex: Lexer<'_> = Lexer::new("A,B:C;"); + let tokens: Vec = lex.parse_tokens(); + + assert_eq!(tokens[0], token::Token::Identifier("A".to_string())); + assert_eq!(tokens[1], token::Token::Comma); + assert_eq!(tokens[2], token::Token::Identifier("B".to_string())); + assert_eq!(tokens[3], token::Token::Colon); + assert_eq!(tokens[4], token::Token::Identifier("C".to_string())); + assert_eq!(tokens[5], token::Token::Semicolon); + } + + #[test] + #[timeout(100)] + fn test_parse_comparison_operators() { + let lex: Lexer<'_> = Lexer::new("= < > <= >= <>"); + let tokens: Vec = lex.parse_tokens(); + + assert_eq!(tokens[0], token::Token::Equal); + assert_eq!(tokens[1], token::Token::Less); + assert_eq!(tokens[2], token::Token::Greater); + assert_eq!(tokens[3], token::Token::LessEqual); + assert_eq!(tokens[4], token::Token::GreaterEqual); + assert_eq!(tokens[5], token::Token::NotEqual); + } + #[test] #[timeout(100)] fn test_string_token() { @@ -371,5 +438,6 @@ mod test { tokens[1], token::Token::String("\"HELLO WORLD\"".to_string()) ); + assert_eq!(tokens[2], token::Token::Semicolon); } } diff --git a/tests/input/print.bsc b/tests/input/print.bsc index c9c2b9e..31d4d79 100644 --- a/tests/input/print.bsc +++ b/tests/input/print.bsc @@ -1 +1 @@ -print "HELLO WORLD" \ No newline at end of file +print "HELLO WORLD"; From 399acc73b637ea82ebab4f7fa4fd42953a2469f9 Mon Sep 17 00:00:00 2001 From: Noah Graf Date: Sat, 25 Apr 2026 04:36:55 +0000 Subject: [PATCH 4/6] Revamping parser --- src/lexer.rs | 63 +++++-- src/lexer/token.rs | 12 ++ src/parser.rs | 111 ++--------- src/parser/outdated_ast.rs | 365 ------------------------------------- 4 files changed, 79 insertions(+), 472 deletions(-) delete mode 100644 src/parser/outdated_ast.rs diff --git a/src/lexer.rs b/src/lexer.rs index 66b6890..a1eaa4b 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -62,14 +62,14 @@ impl<'a> Lexer<'a> { )] pub fn next_token(&self) -> token::Token { // Skip whitespace - while self.peek().is_whitespace() { + while self.peek_char().is_whitespace() { self.read_char(); } - while self.peek() == '\n' { + while self.peek_char() == '\n' { self.read_char(); } - let ch = self.peek(); + let ch = self.peek_char(); match ch { // Example: single-character tokens @@ -91,7 +91,7 @@ impl<'a> Lexer<'a> { } '<' => { self.read_char(); - match self.peek() { + match self.peek_char() { '=' => { self.read_char(); token::Token::LessEqual @@ -105,7 +105,7 @@ impl<'a> Lexer<'a> { } '>' => { self.read_char(); - if self.peek() == '=' { + if self.peek_char() == '=' { self.read_char(); token::Token::GreaterEqual } else { @@ -205,7 +205,7 @@ impl<'a> Lexer<'a> { /// Reads an identifier (e.g., variable/function name) fn read_identifier(&self) -> String { let mut ident = String::new(); - while self.peek().is_alphanumeric() || self.peek() == '_' { + while self.peek_char().is_alphanumeric() || self.peek_char() == '_' { ident.push(self.read_char()); } ident @@ -214,16 +214,24 @@ impl<'a> Lexer<'a> { /// Reads a number (integer only for now) fn read_number(&self) -> i64 { let mut num = String::new(); - while self.peek().is_numeric() { + while self.peek_char().is_numeric() { num.push(self.read_char()); } num.parse().unwrap() } + /// Take a look at the next token without incrementing the location. + pub fn peek(&self) -> token::Token { + let location = self.location.get(); + let token = self.next_token(); + self.location.set(location); + token + } + /// Take a look at the next character without incrimenting the location /// /// If used while the location is out of bounds, the function returns 0x00 - fn peek(&self) -> char { + fn peek_char(&self) -> char { if self.location.get() < self.size.get() { self.input.chars().nth(self.location.get()).unwrap() } else { @@ -234,20 +242,47 @@ impl<'a> Lexer<'a> { #[cfg(test)] mod test { - use crate::lexer::{Lexer, token}; + use crate::lexer::{token, Lexer}; use ntest::timeout; #[test] #[timeout(100)] - fn test_peek() { + fn test_peek_char() { let lex: Lexer<'_> = Lexer::new("Alphabet"); - assert_eq!(lex.peek(), 'A'); - assert_eq!(lex.peek(), 'A'); - assert_eq!(lex.peek(), 'A'); + assert_eq!(lex.peek_char(), 'A'); + assert_eq!(lex.peek_char(), 'A'); + assert_eq!(lex.peek_char(), 'A'); assert_eq!(lex.read_char(), 'A'); - assert_eq!(lex.peek(), 'l'); + assert_eq!(lex.peek_char(), 'l'); + } + + #[test] + #[timeout(100)] + fn test_peek_token() { + let lex: Lexer<'_> = Lexer::new("print \"HELLO\";"); + + assert_eq!(lex.peek(), token::Token::Print); + assert_eq!(lex.peek(), token::Token::Print); + assert_eq!(lex.next_token(), token::Token::Print); + assert_eq!(lex.peek(), token::Token::String("\"HELLO\"".to_string())); + assert_eq!( + lex.next_token(), + token::Token::String("\"HELLO\"".to_string()) + ); + assert_eq!(lex.next_token(), token::Token::Semicolon); + } + + #[test] + #[timeout(100)] + fn test_peek_token_skips_whitespace_without_consuming() { + let lex: Lexer<'_> = Lexer::new(" <= 10"); + + assert_eq!(lex.peek(), token::Token::LessEqual); + assert_eq!(lex.peek(), token::Token::LessEqual); + assert_eq!(lex.next_token(), token::Token::LessEqual); + assert_eq!(lex.next_token(), token::Token::Integer(10)); } #[test] diff --git a/src/lexer/token.rs b/src/lexer/token.rs index d548c87..3903b47 100644 --- a/src/lexer/token.rs +++ b/src/lexer/token.rs @@ -77,4 +77,16 @@ impl Token { _ => panic!("Not Identifier!"), } } + + #[expect( + clippy::cast_precision_loss, + reason = "numeric literals are currently represented as f64 in the AST" + )] + pub fn try_number(&self) -> Option { + match self { + Token::Float(num) => Some(*num as i64), + Token::Integer(num) => Some(*num), + _ => None, + } + } } diff --git a/src/parser.rs b/src/parser.rs index e40d219..f1a85a3 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -3,120 +3,45 @@ //! This is one of the harder modules to create as I am not familiar with it at all //! Do not be afraid to modify any of the strucutre I have laid out here use crate::{ - lexer::{ - Lexer, - token::{self, Token}, - }, - parser::outdated_ast::{ASTBinaryOperator, ASTBinaryOperatorKind, ASTExpression, ASTStatement}, + lexer::{Lexer, token::Token}, + parser::ast::{Line, Program, Statement}, }; mod ast; mod expression; -mod outdated_ast; #[allow(dead_code)] pub struct Parser { - tokens: Vec, + lexer: Lexer<'static>, position: usize, } #[allow(dead_code)] impl Parser { - pub fn new() -> Self { - Parser { - tokens: Vec::new(), - position: 0, - } - } - - pub fn from_tokens(tokens: Vec) -> Self { - Self { - tokens, - position: 0, - } - } - - pub fn from_input(input: &str) -> Self { + pub fn from_input(input: &'static str) -> Self { let lexer = Lexer::new(input); - let mut tokens: Vec = Vec::new(); - while let Some(token) = Some(lexer.next_token()) { - tokens.push(token); - } - Self { - tokens, - position: 0, - } + Self { lexer, position: 0 } } - pub fn next_statement(&mut self) -> Option { - self.parse_statement() - } - - fn parse_statement(&mut self) -> Option { - //let token = self.current()?; - let expr = self.parse_expression()?; - Some(ASTStatement::expression(expr)) - } - - fn parse_expression(&mut self) -> Option { - self.parse_binary_expression(0) - } - - fn parse_binary_expression(&mut self, precedence: u8) -> Option { - let mut left = self.parse_primary_expression()?; - while let Some(operator) = self.parse_binary_operator() { - self.position += 1; - let operator_precedence = operator.precedence(); - if operator_precedence < precedence { - break; - } - let right: ASTExpression = self.parse_binary_expression(operator_precedence)?; - left = ASTExpression::binary(operator, left, right); + pub fn parse_program(&mut self) -> Program { + let mut lines = Vec::new(); + while self.lexer.next_token() != Token::Eof { + lines.push(self.parse_line()); } - Some(left) + Program { lines } } - fn parse_binary_operator(&mut self) -> Option { - let token: Token = self.current()?; - let kind: Option = match token { - token::Token::Plus => Some(ASTBinaryOperatorKind::Plus), - token::Token::Minus => Some(ASTBinaryOperatorKind::Minus), - token::Token::Asterisk => Some(ASTBinaryOperatorKind::Mult), - token::Token::Slash => Some(ASTBinaryOperatorKind::Divide), - _ => None, - }; - kind.map(|kind: ASTBinaryOperatorKind| ASTBinaryOperator::new(kind, token.clone())) - } + pub fn parse_line(&mut self) -> Line { + let line_num = self.lexer.peek().try_number(); - #[expect( - clippy::cast_precision_loss, - reason = "numeric literals are currently represented as f64 in the AST" - )] - fn parse_primary_expression(&mut self) -> Option { - let token: Token = self.current()?; - // match basic expression types - match token { - token::Token::Integer(int) => { - self.position += 1; - Some(ASTExpression::number(int as f64)) - } - token::Token::LParen => { - self.position += 1; - let expr: ASTExpression = self.parse_expression()?; - let token: Token = self.current()?; - assert_eq!(token, token::Token::RParen, "Expected Right Parentheses"); - self.position += 1; - Some(ASTExpression::paren(expr)) - } - _ => None, + let statements = self.parse_statement_list(); + Line { + number: line_num, + statements, } } - pub fn peek(&self, offset: usize) -> Option { - self.tokens.get(self.position + offset).cloned() - } - - fn current(&self) -> Option { - self.peek(0) + pub fn parse_statement_list(&mut self) -> Vec { + todo!() } } diff --git a/src/parser/outdated_ast.rs b/src/parser/outdated_ast.rs deleted file mode 100644 index 7e7ad68..0000000 --- a/src/parser/outdated_ast.rs +++ /dev/null @@ -1,365 +0,0 @@ -use crate::lexer::token::Token; - -#[derive(Debug, PartialEq)] -#[allow(clippy::upper_case_acronyms)] -pub struct AST { - pub statements: Vec, -} - -#[derive(Debug, PartialEq)] -pub enum ASTStatementKind { - Expression(ASTExpression), -} - -#[derive(Debug, PartialEq)] -pub struct ASTStatement { - kind: ASTStatementKind, -} - -#[derive(Debug, PartialEq)] -pub struct ASTExpression { - kind: ASTExpressionKind, -} - -#[derive(Debug, PartialEq)] -pub enum ASTExpressionKind { - Number(ASTNumberExpression), - Binary(ASTBinaryExpression), - Paren(ASTParenExpression), -} - -#[derive(Debug, PartialEq)] -pub struct ASTBinaryExpression { - left: Box, - operator: ASTBinaryOperator, - right: Box, -} - -#[derive(Debug, PartialEq)] -pub struct ASTBinaryOperator { - kind: ASTBinaryOperatorKind, - token: Token, -} - -#[derive(Debug, PartialEq)] -pub enum ASTBinaryOperatorKind { - Plus, - Minus, - Mult, - Divide, -} - -#[derive(Debug, PartialEq)] -pub struct ASTParenExpression { - expression: Box, -} - -impl ASTBinaryOperator { - pub fn new(kind: ASTBinaryOperatorKind, token: Token) -> Self { - ASTBinaryOperator { kind, token } - } - - pub fn precedence(&self) -> u8 { - match self.kind { - ASTBinaryOperatorKind::Plus | ASTBinaryOperatorKind::Minus => 1, - ASTBinaryOperatorKind::Divide | ASTBinaryOperatorKind::Mult => 2, - } - } -} - -#[derive(Debug, PartialEq)] -pub struct ASTNumberExpression { - number: f64, -} - -impl From for ASTNumberExpression { - fn from(value: f64) -> Self { - ASTNumberExpression { number: value } - } -} - -impl ASTStatement { - pub fn new(kind: ASTStatementKind) -> Self { - ASTStatement { kind } - } - - pub fn expression(expr: ASTExpression) -> Self { - ASTStatement::new(ASTStatementKind::Expression(expr)) - } -} - -impl ASTExpression { - pub fn new(kind: ASTExpressionKind) -> Self { - ASTExpression { kind } - } - - pub fn number(number: f64) -> Self { - ASTExpression::new(ASTExpressionKind::Number(ASTNumberExpression { number })) - } - - pub fn binary(operator: ASTBinaryOperator, left: ASTExpression, right: ASTExpression) -> Self { - ASTExpression::new(ASTExpressionKind::Binary(ASTBinaryExpression { - left: Box::new(left), - operator, - right: Box::new(right), - })) - } - - pub fn paren(paren_expression: ASTExpression) -> Self { - ASTExpression::new(ASTExpressionKind::Paren(ASTParenExpression { - expression: Box::new(paren_expression), - })) - } -} - -impl From for ASTExpression { - fn from(value: f64) -> Self { - Self { - kind: ASTExpressionKind::Number(ASTNumberExpression { number: value }), - } - } -} - -#[allow(unused)] -#[allow(dead_code)] -impl AST { - pub fn new() -> Self { - Self { - statements: Vec::new(), - } - } - - pub fn add_statement(&mut self, statement: ASTStatement) { - self.statements.push(statement); - } - - pub fn visit(&mut self, visitor: &mut dyn ASTVisitor) { - for statement in &self.statements { - visitor.visit_statement(statement); - } - } - - pub fn visualize(&mut self) { - let mut printer = ASTPrinter { indent: 0 }; - self.visit(&mut printer); - } -} - -pub trait ASTVisitor { - fn do_visit_statement(&mut self, statement: &ASTStatement) { - match &statement.kind { - ASTStatementKind::Expression(expr) => { - self.visit_expression(expr); - } - } - } - - fn visit_statement(&mut self, statement: &ASTStatement) { - self.do_visit_statement(statement); - } - fn do_visit_expression(&mut self, expression: &ASTExpression) { - match &expression.kind { - ASTExpressionKind::Number(num) => { - self.visit_number(num); - } - ASTExpressionKind::Binary(expr) => { - self.visit_binary_expression(expr); - } - ASTExpressionKind::Paren(expr) => { - self.visit_paren_expression(expr); - } - } - } - fn visit_expression(&mut self, expression: &ASTExpression) { - self.do_visit_expression(expression); - } - fn visit_number(&mut self, number: &ASTNumberExpression); - - fn visit_binary_expression(&mut self, binary_expression: &ASTBinaryExpression) { - self.visit_expression(&binary_expression.left); - self.visit_expression(&binary_expression.right); - } - fn visit_paren_expression(&mut self, paren_expr: &ASTParenExpression) { - self.visit_expression(&paren_expr.expression); - } -} - -pub struct ASTPrinter { - indent: usize, -} - -const INDENT_LEVEL: usize = 2; -impl ASTVisitor for ASTPrinter { - fn visit_number(&mut self, number: &ASTNumberExpression) { - self.print_with_indent(&format!("Number: {}", number.number)); - } - - fn visit_statement(&mut self, statement: &ASTStatement) { - self.print_with_indent("Statement"); - self.indent += INDENT_LEVEL; - ASTVisitor::do_visit_statement(self, statement); - self.indent -= INDENT_LEVEL; - } - - fn visit_expression(&mut self, expression: &ASTExpression) { - self.print_with_indent("Expression"); - self.indent += INDENT_LEVEL; - ASTVisitor::do_visit_expression(self, expression); - self.indent -= INDENT_LEVEL; - } - - fn visit_binary_expression(&mut self, binary_expression: &ASTBinaryExpression) { - self.print_with_indent("Binary Expression:"); - self.indent += INDENT_LEVEL; - self.print_with_indent(&format!("Operator {:?}", binary_expression.operator.kind)); - self.visit_expression(&binary_expression.left); - self.visit_expression(&binary_expression.right); - self.indent -= INDENT_LEVEL; - } - - fn visit_paren_expression(&mut self, paren_expr: &ASTParenExpression) { - self.print_with_indent("Parenthesized Expression: "); - self.indent += INDENT_LEVEL; - self.visit_expression(&paren_expr.expression); - self.indent -= INDENT_LEVEL; - } -} - -impl ASTPrinter { - fn print_with_indent(&mut self, text: &str) { - println!("{}{}", " ".repeat(self.indent), text); - } -} - -#[allow(unused)] -mod test { - use crate::{ - lexer::{Lexer, token}, - parser::{Parser, outdated_ast::*}, - }; - use ntest::timeout; - - #[test] - #[timeout(100)] - fn test_basic_output() { - let mut ast: AST = AST::new(); - let mut lexer: Lexer = Lexer::new("22"); - let mut tokens: Vec = lexer.parse_tokens(); - let mut parser: Parser = Parser::from_tokens(tokens); - while let Some(stmt) = parser.next_statement() { - ast.add_statement(stmt); - } - ast.visualize(); - - assert_eq!( - ast, - AST { - statements: vec![ASTStatement { - kind: ASTStatementKind::Expression(ASTExpression { - kind: ASTExpressionKind::Number(ASTNumberExpression { number: 22.0 }) - }) - }] - } - ); - } - - #[test] - fn test_basic_addition() { - // I will sacrifice a goat to get this test to work - /* - ,,~~--___---, - / .~, - / _,~ ) - (_-(~) ~, ),,,( /' - Z6 .~`' || \ | - /_,/ || || - ~~~~~~~~~~~~~~~W`~~~~~~W`~~~~~~~~~ - */ - - let mut ast: AST = AST::new(); - let mut lexer: Lexer = Lexer::new("22 + 22"); - let mut tokens: Vec = lexer.parse_tokens(); - let mut parser: Parser = Parser::from_tokens(tokens); - while let Some(stmt) = parser.next_statement() { - ast.add_statement(stmt); - } - ast.visualize(); - - assert_eq!( - ast, - AST { - statements: vec![ASTStatement { - kind: ASTStatementKind::Expression(ASTExpression { - kind: ASTExpressionKind::Binary(ASTBinaryExpression { - left: Box::new(22.0.into()), - operator: ASTBinaryOperator { - kind: ASTBinaryOperatorKind::Plus, - token: Token::Plus - }, - right: Box::new(22.0.into()) - }) - }) - }] - } - ); - } - - #[test] - #[timeout(100)] - fn test_paren_expression() { - let mut ast: AST = AST::new(); - let mut lexer: Lexer = Lexer::new("(22 + 23) * 24"); - let mut tokens: Vec = lexer.parse_tokens(); - let mut parser: Parser = Parser::from_tokens(tokens); - while let Some(stmt) = parser.next_statement() { - ast.add_statement(stmt); - } - ast.visualize(); - - assert_eq!( - ast, - AST { - statements: vec![ASTStatement { - kind: ASTStatementKind::Expression(ASTExpression { - kind: ASTExpressionKind::Binary(ASTBinaryExpression { - left: Box::new(ASTExpression { - kind: ASTExpressionKind::Paren(ASTParenExpression { - expression: Box::new(ASTExpression { - kind: ASTExpressionKind::Binary(ASTBinaryExpression { - left: Box::new(22.0.into()), - operator: ASTBinaryOperator { - kind: ASTBinaryOperatorKind::Plus, - token: token::Token::Plus - }, - right: Box::new(23.0.into()) - }) - }) - }) - }), - operator: ASTBinaryOperator { - kind: ASTBinaryOperatorKind::Mult, - token: token::Token::Asterisk - }, - right: Box::new(24.0.into()) - }) - }) - }] - } - ); - } - - #[test] - #[timeout(100)] - fn test_long_expression() { - let mut ast: AST = AST::new(); - let mut lexer: Lexer = Lexer::new("7 - 30 * 8 / 2"); - let mut tokens: Vec = lexer.parse_tokens(); - let mut parser: Parser = Parser::from_tokens(tokens); - while let Some(stmt) = parser.next_statement() { - ast.add_statement(stmt); - } - ast.visualize(); - - assert_eq!(1, 1); - } -} From 7124c0434fdbe17a4729b667149b0347f64724eb Mon Sep 17 00:00:00 2001 From: Noah Graf Date: Sat, 25 Apr 2026 04:38:51 +0000 Subject: [PATCH 5/6] Tweaks to make clippy happy --- src/lexer.rs | 2 +- src/lexer/token.rs | 2 +- src/parser.rs | 123 ++++++++++++++++++++++++++++++++++++++++++++- src/parser/ast.rs | 5 ++ 4 files changed, 128 insertions(+), 4 deletions(-) diff --git a/src/lexer.rs b/src/lexer.rs index a1eaa4b..82148c9 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -242,7 +242,7 @@ impl<'a> Lexer<'a> { #[cfg(test)] mod test { - use crate::lexer::{token, Lexer}; + use crate::lexer::{Lexer, token}; use ntest::timeout; #[test] diff --git a/src/lexer/token.rs b/src/lexer/token.rs index 3903b47..ff33c56 100644 --- a/src/lexer/token.rs +++ b/src/lexer/token.rs @@ -79,7 +79,7 @@ impl Token { } #[expect( - clippy::cast_precision_loss, + clippy::cast_possible_truncation, reason = "numeric literals are currently represented as f64 in the AST" )] pub fn try_number(&self) -> Option { diff --git a/src/parser.rs b/src/parser.rs index f1a85a3..43295f1 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -4,7 +4,10 @@ //! Do not be afraid to modify any of the strucutre I have laid out here use crate::{ lexer::{Lexer, token::Token}, - parser::ast::{Line, Program, Statement}, + parser::{ + ast::{Line, PrintItem, Program, Statement}, + expression::Expression, + }, }; mod ast; mod expression; @@ -42,6 +45,122 @@ impl Parser { } pub fn parse_statement_list(&mut self) -> Vec { - todo!() + let mut statements = Vec::new(); + + while !matches!(self.lexer.peek(), Token::Eof | Token::Semicolon) { + statements.push(self.parse_statement()); + + if self.lexer.peek() == Token::Colon { + self.lexer.next_token(); + } else { + break; + } + } + + if self.lexer.peek() == Token::Semicolon { + self.lexer.next_token(); + } + + statements + } + + fn parse_statement(&mut self) -> Statement { + match self.lexer.peek() { + Token::End => { + self.lexer.next_token(); + Statement::End + } + Token::Return => { + self.lexer.next_token(); + Statement::Return + } + Token::Rem => { + self.lexer.next_token(); + Statement::Rem(None) + } + Token::Print => self.parse_print_statement(), + _ => Statement::Expression(self.parse_expression()), + } + } + + fn parse_print_statement(&mut self) -> Statement { + self.lexer.next_token(); + let mut items = Vec::new(); + + while !matches!( + self.lexer.peek(), + Token::Colon | Token::Eof | Token::Semicolon + ) { + let item = match self.lexer.next_token() { + Token::String(value) => PrintItem::String(value), + Token::Char(value) => PrintItem::Char(value), + Token::Integer(value) => PrintItem::Expression(Expression::integer(value)), + Token::Float(value) => PrintItem::Expression(Expression::float(value)), + Token::Identifier(value) => PrintItem::Expression(Expression::identifier(value)), + _ => break, + }; + items.push(item); + + if matches!(self.lexer.peek(), Token::Comma | Token::Semicolon) { + self.lexer.next_token(); + } + } + + Statement::Print { items } + } + + fn parse_expression(&mut self) -> Expression { + match self.lexer.next_token() { + Token::Integer(value) => Expression::integer(value), + Token::Float(value) => Expression::float(value), + Token::String(value) => Expression::string(value), + Token::Char(value) => Expression::char(value), + Token::Identifier(value) => Expression::identifier(value), + token => panic!("Expected expression, got {token:?}"), + } + } +} + +#[cfg(test)] +mod tests { + use crate::parser::{ + Parser, + ast::{PrintItem, Statement}, + expression::Expression, + }; + + #[test] + fn parse_single_statement_list() { + let mut parser = Parser::from_input("end;"); + + assert_eq!(parser.parse_statement_list(), vec![Statement::End]); + } + + #[test] + fn parse_colon_separated_statement_list() { + let mut parser = Parser::from_input("print \"HI\": return: end;"); + + assert_eq!( + parser.parse_statement_list(), + vec![ + Statement::Print { + items: vec![PrintItem::String("\"HI\"".to_string())] + }, + Statement::Return, + Statement::End, + ] + ); + } + + #[test] + fn parse_expression_statement_list() { + let mut parser = Parser::from_input("x;"); + + assert_eq!( + parser.parse_statement_list(), + vec![Statement::Expression(Expression::identifier( + "x".to_string() + ))] + ); } } diff --git a/src/parser/ast.rs b/src/parser/ast.rs index 2158f74..a6c7b72 100644 --- a/src/parser/ast.rs +++ b/src/parser/ast.rs @@ -74,15 +74,18 @@ use crate::parser::expression::Expression; +#[derive(Debug, PartialEq)] pub struct Program { pub lines: Vec, } +#[derive(Debug, PartialEq)] pub struct Line { pub number: Option, pub statements: Vec, } +#[derive(Debug, PartialEq)] pub enum Statement { Let { name: String, @@ -112,11 +115,13 @@ pub enum Statement { Expression(Expression), } +#[derive(Debug, PartialEq)] pub enum Target { Line(i64), Expression(Expression), } +#[derive(Debug, PartialEq)] pub enum PrintItem { Expression(Expression), String(String), From 20f8cdc3e0f64429420e03d65da02c3d23448d7f Mon Sep 17 00:00:00 2001 From: Noah Graf Date: Sat, 25 Apr 2026 04:44:04 +0000 Subject: [PATCH 6/6] Added a ton of tests --- src/parser.rs | 261 +++++++++++++++++++++++++- tests/input/control_markers.bsc | 1 + tests/input/empty_print.bsc | 1 + tests/input/expression_statements.bsc | 3 + tests/input/print_items.bsc | 1 + tests/input/statement_list.bsc | 1 + 6 files changed, 258 insertions(+), 10 deletions(-) create mode 100644 tests/input/control_markers.bsc create mode 100644 tests/input/empty_print.bsc create mode 100644 tests/input/expression_statements.bsc create mode 100644 tests/input/print_items.bsc create mode 100644 tests/input/statement_list.bsc diff --git a/src/parser.rs b/src/parser.rs index 43295f1..9850eee 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -27,7 +27,7 @@ impl Parser { pub fn parse_program(&mut self) -> Program { let mut lines = Vec::new(); - while self.lexer.next_token() != Token::Eof { + while self.lexer.peek() != Token::Eof { lines.push(self.parse_line()); } @@ -125,23 +125,33 @@ impl Parser { mod tests { use crate::parser::{ Parser, - ast::{PrintItem, Statement}, + ast::{Line, PrintItem, Program, Statement}, expression::Expression, }; + fn parse_statement_list(input: &'static str) -> Vec { + Parser::from_input(input).parse_statement_list() + } + #[test] fn parse_single_statement_list() { - let mut parser = Parser::from_input("end;"); + assert_eq!(parse_statement_list("end;"), vec![Statement::End]); + } - assert_eq!(parser.parse_statement_list(), vec![Statement::End]); + #[test] + fn parse_empty_statement_list_at_semicolon() { + assert_eq!(parse_statement_list(";"), Vec::new()); } #[test] - fn parse_colon_separated_statement_list() { - let mut parser = Parser::from_input("print \"HI\": return: end;"); + fn parse_empty_statement_list_at_eof() { + assert_eq!(parse_statement_list(""), Vec::new()); + } + #[test] + fn parse_colon_separated_statement_list() { assert_eq!( - parser.parse_statement_list(), + parse_statement_list("print \"HI\": return: end;"), vec![ Statement::Print { items: vec![PrintItem::String("\"HI\"".to_string())] @@ -154,13 +164,244 @@ mod tests { #[test] fn parse_expression_statement_list() { - let mut parser = Parser::from_input("x;"); - assert_eq!( - parser.parse_statement_list(), + parse_statement_list("x;"), vec![Statement::Expression(Expression::identifier( "x".to_string() ))] ); } + + #[test] + fn parse_integer_expression_statement_list() { + assert_eq!( + parse_statement_list("123;"), + vec![Statement::Expression(Expression::integer(123))] + ); + } + + #[test] + fn parse_string_expression_statement_list() { + assert_eq!( + parse_statement_list("\"ABC\";"), + vec![Statement::Expression(Expression::string( + "\"ABC\"".to_string() + ))] + ); + } + + #[test] + fn parse_char_expression_statement_list() { + assert_eq!( + parse_statement_list("'a';"), + vec![Statement::Expression(Expression::char('a'))] + ); + } + + #[test] + fn parse_return_statement_list() { + assert_eq!(parse_statement_list("return;"), vec![Statement::Return]); + } + + #[test] + fn parse_rem_statement_list() { + assert_eq!(parse_statement_list("rem;"), vec![Statement::Rem(None)]); + } + + #[test] + fn parse_empty_print_statement_list() { + assert_eq!( + parse_statement_list("print;"), + vec![Statement::Print { items: Vec::new() }] + ); + } + + #[test] + fn parse_print_string_statement_list() { + assert_eq!( + parse_statement_list("print \"HELLO\";"), + vec![Statement::Print { + items: vec![PrintItem::String("\"HELLO\"".to_string())] + }] + ); + } + + #[test] + fn parse_print_char_statement_list() { + assert_eq!( + parse_statement_list("print 'x';"), + vec![Statement::Print { + items: vec![PrintItem::Char('x')] + }] + ); + } + + #[test] + fn parse_print_integer_statement_list() { + assert_eq!( + parse_statement_list("print 42;"), + vec![Statement::Print { + items: vec![PrintItem::Expression(Expression::integer(42))] + }] + ); + } + + #[test] + fn parse_print_identifier_statement_list() { + assert_eq!( + parse_statement_list("print answer;"), + vec![Statement::Print { + items: vec![PrintItem::Expression(Expression::identifier( + "answer".to_string() + ))] + }] + ); + } + + #[test] + fn parse_print_comma_separated_items() { + assert_eq!( + parse_statement_list("print \"A\", 1, x;"), + vec![Statement::Print { + items: vec![ + PrintItem::String("\"A\"".to_string()), + PrintItem::Expression(Expression::integer(1)), + PrintItem::Expression(Expression::identifier("x".to_string())), + ] + }] + ); + } + + #[test] + fn parse_print_stops_before_colon() { + assert_eq!( + parse_statement_list("print \"A\": end;"), + vec![ + Statement::Print { + items: vec![PrintItem::String("\"A\"".to_string())] + }, + Statement::End, + ] + ); + } + + #[test] + fn parse_program_from_print_fixture() { + let mut parser = Parser::from_input(include_str!("../tests/input/print.bsc")); + + assert_eq!( + parser.parse_program(), + Program { + lines: vec![Line { + number: None, + statements: vec![Statement::Print { + items: vec![PrintItem::String("\"HELLO WORLD\"".to_string())] + }] + }] + } + ); + } + + #[test] + fn parse_program_from_statement_list_fixture() { + let mut parser = Parser::from_input(include_str!("../tests/input/statement_list.bsc")); + + assert_eq!( + parser.parse_program(), + Program { + lines: vec![Line { + number: None, + statements: vec![ + Statement::Print { + items: vec![PrintItem::String("\"A\"".to_string())] + }, + Statement::Return, + Statement::End, + ] + }] + } + ); + } + + #[test] + fn parse_program_from_print_items_fixture() { + let mut parser = Parser::from_input(include_str!("../tests/input/print_items.bsc")); + + assert_eq!( + parser.parse_program(), + Program { + lines: vec![Line { + number: None, + statements: vec![Statement::Print { + items: vec![ + PrintItem::String("\"NAME\"".to_string()), + PrintItem::Expression(Expression::identifier("username".to_string())), + PrintItem::Expression(Expression::integer(7)), + PrintItem::Char('!'), + ] + }] + }] + } + ); + } + + #[test] + fn parse_program_from_expression_statements_fixture() { + let mut parser = + Parser::from_input(include_str!("../tests/input/expression_statements.bsc")); + + assert_eq!( + parser.parse_program(), + Program { + lines: vec![ + Line { + number: None, + statements: vec![Statement::Expression(Expression::identifier( + "answer".to_string() + ))] + }, + Line { + number: None, + statements: vec![Statement::Expression(Expression::string( + "\"forty two\"".to_string() + ))] + }, + Line { + number: None, + statements: vec![Statement::Expression(Expression::char('z'))] + }, + ] + } + ); + } + + #[test] + fn parse_program_from_control_markers_fixture() { + let mut parser = Parser::from_input(include_str!("../tests/input/control_markers.bsc")); + + assert_eq!( + parser.parse_program(), + Program { + lines: vec![Line { + number: None, + statements: vec![Statement::Rem(None), Statement::Return, Statement::End] + }] + } + ); + } + + #[test] + fn parse_program_from_empty_print_fixture() { + let mut parser = Parser::from_input(include_str!("../tests/input/empty_print.bsc")); + + assert_eq!( + parser.parse_program(), + Program { + lines: vec![Line { + number: None, + statements: vec![Statement::Print { items: Vec::new() }] + }] + } + ); + } } diff --git a/tests/input/control_markers.bsc b/tests/input/control_markers.bsc new file mode 100644 index 0000000..e0c6f6d --- /dev/null +++ b/tests/input/control_markers.bsc @@ -0,0 +1 @@ +rem: return: end; diff --git a/tests/input/empty_print.bsc b/tests/input/empty_print.bsc new file mode 100644 index 0000000..657c704 --- /dev/null +++ b/tests/input/empty_print.bsc @@ -0,0 +1 @@ +print; diff --git a/tests/input/expression_statements.bsc b/tests/input/expression_statements.bsc new file mode 100644 index 0000000..3011bc7 --- /dev/null +++ b/tests/input/expression_statements.bsc @@ -0,0 +1,3 @@ +answer; +"forty two"; +'z'; diff --git a/tests/input/print_items.bsc b/tests/input/print_items.bsc new file mode 100644 index 0000000..a1bd443 --- /dev/null +++ b/tests/input/print_items.bsc @@ -0,0 +1 @@ +print "NAME", username, 7, '!'; diff --git a/tests/input/statement_list.bsc b/tests/input/statement_list.bsc new file mode 100644 index 0000000..4d62912 --- /dev/null +++ b/tests/input/statement_list.bsc @@ -0,0 +1 @@ +print "A": return: end;