diff --git a/Cargo.lock b/Cargo.lock index 174b96d2..58d7ef63 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,15 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + [[package]] name = "anstream" version = "1.0.0" @@ -202,6 +211,12 @@ version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "foldhash" version = "0.1.5" @@ -269,6 +284,38 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "logos" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb2c55a318a87600ea870ff8c2012148b44bf18b74fad48d0f835c38c7d07c5f" +dependencies = [ + "logos-derive", +] + +[[package]] +name = "logos-codegen" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58b3ffaa284e1350d017a57d04ada118c4583cf260c8fb01e0fe28a2e9cf8970" +dependencies = [ + "fnv", + "proc-macro2", + "quote", + "regex-automata", + "regex-syntax", + "syn", +] + +[[package]] +name = "logos-derive" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52d3a9855747c17eaf4383823f135220716ab49bea5fbea7dd42cc9a92f8aa31" +dependencies = [ + "logos-codegen", +] + [[package]] name = "memchr" version = "2.8.0" @@ -372,6 +419,23 @@ dependencies = [ "simdeez", ] +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + [[package]] name = "rustc-demangle" version = "0.1.27" @@ -468,6 +532,7 @@ version = "0.1.0" dependencies = [ "color-eyre", "common", + "logos", ] [[package]] diff --git a/crates/lexer/Cargo.toml b/crates/lexer/Cargo.toml index 91639727..06f4acbf 100644 --- a/crates/lexer/Cargo.toml +++ b/crates/lexer/Cargo.toml @@ -6,3 +6,4 @@ edition = "2024" [dependencies] color-eyre.workspace=true common = {path="../common"} +logos = "0.16.1" diff --git a/crates/lexer/src/lib.rs b/crates/lexer/src/lib.rs index d949a4b7..9e8f0157 100644 --- a/crates/lexer/src/lib.rs +++ b/crates/lexer/src/lib.rs @@ -1,14 +1,17 @@ pub mod error; +pub mod tokens; + use std::{collections::VecDeque, ops::Index}; use crate::{ error::LexerError, tokens::{Token, TokenKind}, }; -use common::Span; -pub mod tokens; +use common::ast::Span; +use logos::Logos; + #[derive(Debug)] -///A stream of tokens to be used when parsing the content +/// A stream of tokens to be used when parsing the content pub struct TokenStream { pub stream: VecDeque, pub new_lines: Vec, @@ -23,403 +26,82 @@ impl Iterator for TokenStream { impl Index for TokenStream { type Output = Token; - fn index(&self, index: usize) -> &Self::Output { &self.stream[index] } } + pub struct Lexer; impl Lexer { pub fn tokenize(source: &str) -> Result { - let mut out = VecDeque::new(); - let mut lines = Vec::new(); - let chars = source.chars().collect::>(); - let mut idx = 0; - while idx < chars.len() { - let Some(tk) = chars.get(idx) else { - break; + let mut stream = VecDeque::new(); + let mut new_lines = Vec::new(); + + let raw: Vec<(Result, std::ops::Range)> = + TokenKind::lexer(source).spanned().collect(); + + for i in 0..raw.len() { + let (Ok(result), range) = &raw[i] else { + let range = &raw[i].1; + let c = source[range.clone()].chars().next().unwrap_or('?'); + return Err(LexerError::UnrecognizedChar { + char: c, + index: range.start, + }); }; - let tk = match tk { - '&' => { - if let Some('&') = chars.get(idx + 1) { - idx += 1; - Token::and(idx) - } else { - Token::bitand(idx) - } - } - '|' => { - if let Some('|') = chars.get(idx + 1) { - idx += 1; - Token::or(idx) - } else { - Token::bitor(idx) - } - } - '^' => Token::xor(idx), - '.' => Token::dot(idx), - '(' => Token::lparen(idx), - ')' => Token::rparen(idx), - '{' => Token::lbrace(idx), - '}' => Token::rbrace(idx), - ';' => Token::semicolon(idx), - '<' => { - if let Some('<') = chars.get(idx + 1) { - idx += 1; - Token::shl(idx) - } else if let Some('=') = chars.get(idx + 1) { - idx += 1; - Token::lte(idx) - } else { - Token::lt(idx) - } - } - '>' => { - if let Some('>') = chars.get(idx + 1) { - idx += 1; - Token::shr(idx) - } else if let Some('=') = chars.get(idx + 1) { - idx += 1; - Token::gte(idx) - } else { - Token::gt(idx) - } - } - '=' => { - if let Some('=') = chars.get(idx + 1) { - idx += 1; - Token::eqeq(idx) - } else { - Token::eq(idx) - } - } - - ':' => Token::colon(idx), - ',' => Token::comma(idx), - '+' => { - if let Some('=') = chars.get(idx + 1) { - idx += 1; - Token::pluseq(idx) - } else { - Token::plus(idx) - } - } - '-' => { - if let Some('=') = chars.get(idx + 1) { - idx += 1; - Token::subeq(idx) - } else if let Some('>') = chars.get(idx + 1) { - let out = Token::arrow(idx); - idx += 1; - out - } else { - Token::sub(idx) - } - } - '*' => { - if let Some('=') = chars.get(idx + 1) { - idx += 1; - Token::stareq(idx) - } else { - Token::star(idx) - } - } - '/' => { - if let Some('*') = chars.get(idx + 1) { - while idx < chars.len() { - if let Some('\n') = chars.get(idx) { - lines.push(idx); - } - if let Some('*') = chars.get(idx) - && let Some('/') = chars.get(idx + 1) - { - break; - } - idx += 1; - } - continue; - } else if let Some('=') = chars.get(idx + 1) { - idx += 1; - Token::slasheq(idx) - } else if let Some('/') = chars.get(idx + 1) { - while idx < chars.len() { - if chars.get(idx) == Some(&'\n') { - lines.push(idx); - } - idx += 1; - } - continue; - } else { - Token::slash(idx) - } + match result { + TokenKind::Newline => { + new_lines.push(range.start); } - - '"' => { - let mut buffer = String::new(); - idx += 1; - let start = idx; - while idx < chars.len() - && let Some(c) = chars.get(idx) - && *c != '"' + kind => { + // Reject `..` (double-dot after number) + if matches!(kind, TokenKind::Int(_) | TokenKind::Float(_)) + && let Some((Ok(TokenKind::Dot), dot1)) = raw.get(i + 1) + && let Some((Ok(TokenKind::Dot), _)) = raw.get(i + 2) { - if *c == '\\' { - idx += 1; - match c { - 'n' => buffer.push('\n'), - 't' => buffer.push('\t'), - 'r' => buffer.push('\r'), - c @ '0'..='9' => buffer.push(*c), - '\\' => buffer.push('\\'), - '"' => buffer.push('"'), - _ => buffer.push('\\'), - } - } else { - buffer.push(*c); - } - - idx += 1; - } - Token::string(buffer, start, idx) - } - c if c.is_whitespace() => { - if *c == '\n' { - lines.push(idx); - } - idx += 1; - continue; - } - c if c.is_ascii_digit() => { - let mut buffer = String::new(); - let start = idx; - let mut float_value = false; - let mut last_is_underscore = false; - let mut last_is_dot = false; - let mut should_err = false; - let mut hex_value = false; - let mut binary_value = false; - let mut radix = 0; - let mut octal_value = false; - - if let Some('0') = chars.get(idx) { - match chars.get(idx + 1) { - Some('x') => { - radix += 16; - idx += 2; - hex_value = true; - while let Some(c) = chars.get(idx) - && (c.is_ascii_hexdigit()) - { - buffer.push(*c); - idx += 1; - } - } - Some('b') => { - binary_value = true; - radix += 2; - idx += 2; - while let Some(c) = chars.get(idx) - && (*c == '0' || *c == '1') - { - buffer.push(*c); - idx += 1; - } - } - Some('o') => { - octal_value = true; - radix += 8; - idx += 2; - while let Some(c) = chars.get(idx) - && (c.is_digit(radix)) - { - buffer.push(*c); - idx += 1; - } - } - _ => (), - } - } - - if !hex_value && !binary_value && !octal_value { - while let Some(c) = chars.get(idx) { - if c.is_ascii_digit() { - last_is_dot = false; - last_is_underscore = false; - buffer.push(*c); - idx += 1; - continue; - } - - if *c == '_' { - if last_is_underscore || last_is_dot { - should_err = true; - } - last_is_dot = false; - last_is_underscore = true; - buffer.push(*c); - idx += 1; - continue; - } - - if *c == '.' { - let next = chars.get(idx + 1); - if !float_value - && matches!(next, Some(next) if next.is_ascii_digit()) - { - // Only keep '.' inside a number when it actually starts - // the fractional part of a float literal. - float_value = true; - if last_is_underscore { - should_err = true; - } - last_is_dot = true; - last_is_underscore = false; - buffer.push(*c); - idx += 1; - continue; - } - - if matches!(next, Some('.') | Some('_')) { - should_err = true; - buffer.push(*c); - idx += 1; - } - break; - } - - break; - } - } - if should_err || buffer.ends_with('_') || buffer.ends_with('.') { return Err(LexerError::MalformedNumber { - number: buffer, - init: start, - end: idx, + number: source[range.start..dot1.end].to_string(), + init: range.start, + end: dot1.end, }); } - - idx -= 1; - let buffer = buffer.replace('_', ""); - if hex_value || binary_value || octal_value { - let decimal_value = i32::from_str_radix(&buffer, radix); - - match decimal_value { - Ok(value) => Token::int(value, start, idx), - Err(_) => { - return Err(LexerError::MalformedNumber { - number: buffer, - init: start, - end: idx, - }); - } + // Reject `._` or `.` + // logos won't produce Int with trailing underscore, but catches `1._0` + // pattern: Int Dot Identifier-starting-with-underscore + if matches!(kind, TokenKind::Int(_)) { + if let Some((Ok(TokenKind::Dot), dot_range)) = raw.get(i + 1) + && let Some((Ok(TokenKind::Identifier(id)), _)) = raw.get(i + 2) + && id.starts_with('_') + { + return Err(LexerError::MalformedNumber { + number: source[range.start..dot_range.end].to_string(), + init: range.start, + end: dot_range.end, + }); } - } else if float_value { - match buffer.parse::() { - Ok(value) => Token::float(value, start, idx), - Err(_) => { - return Err(LexerError::MalformedNumber { - number: buffer, - init: start, - end: idx, - }); - } + // Reject `1_.0`: logos lexes `1_` as error, but let's also check + // if the raw source slice for this int ends with '_' + if source[range.clone()].ends_with('_') { + return Err(LexerError::MalformedNumber { + number: source[range.clone()].to_string(), + init: range.start, + end: range.end, + }); } - } else { - match buffer.parse::() { - Ok(value) => Token::int(value, start, idx), - Err(_) => { - return Err(LexerError::MalformedNumber { - number: buffer, - init: start, - end: idx, - }); - } - } - } - } - c if c.is_alphabetic() || *c == '_' => { - let mut buffer = String::new(); - let start = idx; - while let Some(c) = chars.get(idx) - && (chars[idx].is_alphanumeric() || *c == '_') - { - buffer.push(*c); - idx += 1; } - - idx -= 1; - let span = Span { start, end: idx }; - match buffer.as_str() { - "while" => Token { - kind: TokenKind::While, - span, + stream.push_back(Token { + kind: kind.clone(), + span: Span { + start: range.start, + end: range.end, }, - - "let" => Token { - kind: TokenKind::Let, - span, - }, - "mut" => Token { - kind: TokenKind::Mut, - span, - }, - "object" => Token { - kind: TokenKind::Object, - span, - }, - "component" => Token { - kind: TokenKind::Component, - span, - }, - "func" => Token { - kind: TokenKind::Func, - span, - }, - "pub" => Token { - kind: TokenKind::Pub, - span, - }, - "prop" => Token { - kind: TokenKind::Prop, - span, - }, - "if" => Token { - kind: TokenKind::If, - span, - }, - "else" => Token { - kind: TokenKind::Else, - span, - }, - "true" => Token { - kind: TokenKind::True, - span, - }, - "false" => Token { - kind: TokenKind::False, - span, - }, - "alias" => Token { - kind: TokenKind::Alias, - span, - }, - _ => Token::identifier(&buffer, start, idx), - } - } - c => { - return Err(LexerError::UnrecognizedChar { - char: *c, - index: idx, }); } - }; - out.push_back(tk); - idx += 1; + } } - Ok(TokenStream { - stream: out, - new_lines: lines, - }) + + Ok(TokenStream { stream, new_lines }) } } diff --git a/crates/lexer/src/tokens.rs b/crates/lexer/src/tokens.rs index a9ed8842..c08615e0 100644 --- a/crates/lexer/src/tokens.rs +++ b/crates/lexer/src/tokens.rs @@ -1,4 +1,5 @@ -use common::Span; +use common::ast::Span; +use logos::Logos; #[derive(Debug, Clone, PartialEq)] pub struct Token { @@ -6,478 +7,212 @@ pub struct Token { pub span: Span, } -impl std::fmt::Display for TokenKind { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let result = match self { - Self::While => "while".to_string(), - Self::CommonComent => "//".to_string(), - Self::And => "&&".to_string(), - Self::Or => "||".to_string(), - - Self::If => "if".to_string(), - Self::Else => "else".to_string(), - Self::Dot => ".".to_string(), - Self::LParen => "(".to_string(), - Self::RParen => ")".to_string(), - Self::LBrace => "{".to_string(), - Self::RBrace => "}".to_string(), - Self::SemiColon => ";".to_string(), - Self::Lt => "<".to_string(), - Self::LtEq => "<=".to_string(), - Self::Gt => ">".to_string(), - Self::GtEq => ">=".to_string(), - Self::Eq => "=".to_string(), - Self::EqEq => "==".to_string(), - Self::Plus => "+".to_string(), - Self::PlusEq => "+=".to_string(), - Self::Sub => "-".to_string(), - Self::SubEq => "-=".to_string(), - Self::Star => "*".to_string(), - Self::StarEq => "*=".to_string(), - Self::Slash => "/".to_string(), - Self::SlashEq => "/=".to_string(), - Self::Arrow => "->".to_string(), - Self::Comma => ",".to_string(), - Self::Colon => ":".to_string(), - Self::Component => "component".to_string(), - Self::Func => "func".to_string(), - Self::Pub => "pub".to_string(), - Self::Prop => "prop".to_string(), - Self::Alias => "alias".to_string(), - Self::Object => "object".to_string(), - Self::Let => "let".to_string(), - Self::Mut => "mut".to_string(), - Self::True => "true".to_string(), - Self::False => "false".to_string(), - Self::ShiftRight => "<<".to_string(), - Self::ShiftLeft => ">>".to_string(), - Self::Xor => "^".to_string(), - Self::BitAnd => "&".to_string(), - Self::BitOr => "|".to_string(), - Self::BitNot => "~".to_string(), - Self::Float(value) => value.to_string(), - Self::Int(value) => value.to_string(), - Self::String(value) => value.to_string(), - Self::Identifier(value) => value.to_string(), - }; - write!(f, "{}", result) - } -} - -#[derive(Debug, PartialEq, Clone)] +#[derive(Logos, Debug, PartialEq, Clone)] +#[logos(skip r"[ \t\r\f]+")] pub enum TokenKind { - While, + // Comments (skip) + #[regex(r"//[^\n]*", logos::skip, allow_greedy = true)] + #[regex(r"/\*[^*]*\*+(?:[^/*][^*]*\*+)*/", logos::skip)] CommonComent, - LParen, + + // Keywords + #[token("while")] + While, + #[token("if")] If, + #[token("else")] Else, - RParen, - LBrace, - RBrace, - SemiColon, + #[token("component")] + Component, + #[token("func")] + Func, + #[token("pub")] + Pub, + #[token("prop")] + Prop, + #[token("alias")] + Alias, + #[token("object")] + Object, + #[token("let")] + Let, + #[token("mut")] + Mut, + #[token("true")] + True, + #[token("false")] + False, - ShiftRight, + // Multi-char operators (must come before single-char) + #[token("&&")] + And, + #[token("||")] + Or, + #[token("<<")] ShiftLeft, - Xor, + #[token(">>")] + ShiftRight, + #[token("<=")] + LtEq, + #[token(">=")] + GtEq, + #[token("==")] + EqEq, + #[token("+=")] + PlusEq, + #[token("-=")] + SubEq, + #[token("*=")] + StarEq, + #[token("/=")] + SlashEq, + #[token("->")] + Arrow, + + // Single-char operators + #[token("&")] BitAnd, + #[token("|")] BitOr, + #[token("^")] + Xor, + #[token("~")] BitNot, - + #[token(".")] + Dot, + #[token("(")] + LParen, + #[token(")")] + RParen, + #[token("{")] + LBrace, + #[token("}")] + RBrace, + #[token(";")] + SemiColon, + #[token("<")] Lt, - LtEq, - + #[token(">")] Gt, - GtEq, - + #[token("=")] Eq, - EqEq, - + #[token("+")] Plus, - PlusEq, - + #[token("-")] Sub, - SubEq, - + #[token("*")] Star, - StarEq, - + #[token("/")] Slash, - SlashEq, - - And, - Or, - - Arrow, - + #[token(",")] Comma, - + #[token(":")] Colon, - Dot, - Float(f32), + // Literals + #[regex(r"0x[0-9a-fA-F]+", |lex| i32::from_str_radix(&lex.slice()[2..], 16).ok())] + #[regex(r"0b[01]+", |lex| i32::from_str_radix(&lex.slice()[2..], 2).ok())] + #[regex(r"0o[0-7]+", |lex| i32::from_str_radix(&lex.slice()[2..], 8).ok())] + #[regex(r"[0-9][0-9_]*", |lex| lex.slice().replace('_', "").parse::().ok())] Int(i32), - String(String), - Identifier(String), - Component, - Func, - Pub, - Prop, - Alias, + // No underscore immediately before or after the decimal point + #[regex(r"[0-9]([0-9]|_[0-9])*\.[0-9]([0-9]|_[0-9])*", |lex| lex.slice().replace('_', "").parse::().ok())] + Float(f32), - Object, + #[regex(r#""([^"\\]|\\.)*""#, |lex| { + let s = lex.slice(); + // strip surrounding quotes and process escapes + let inner = &s[1..s.len()-1]; + let mut out = String::with_capacity(inner.len()); + let mut chars = inner.chars(); + while let Some(c) = chars.next() { + if c == '\\' { + match chars.next() { + Some('n') => out.push('\n'), + Some('t') => out.push('\t'), + Some('r') => out.push('\r'), + Some('\\') => out.push('\\'), + Some('"') => out.push('"'), + Some(c) => { out.push('\\'); out.push(c); } + None => out.push('\\'), + } + } else { + out.push(c); + } + } + Some(out) + })] + String(String), - Let, - Mut, + #[regex(r"[a-zA-Z_][a-zA-Z0-9_]*", |lex| lex.slice().to_string())] + Identifier(String), - True, - False, + // Newline (tracked for line info) + #[token("\n")] + Newline, } -impl std::fmt::Display for Token { +impl std::fmt::Display for TokenKind { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "'{}'", self.kind) + let s = match self { + Self::While => "while", + Self::CommonComent => "//", + Self::And => "&&", + Self::Or => "||", + Self::If => "if", + Self::Else => "else", + Self::Dot => ".", + Self::LParen => "(", + Self::RParen => ")", + Self::LBrace => "{", + Self::RBrace => "}", + Self::SemiColon => ";", + Self::Lt => "<", + Self::LtEq => "<=", + Self::Gt => ">", + Self::GtEq => ">=", + Self::Eq => "=", + Self::EqEq => "==", + Self::Plus => "+", + Self::PlusEq => "+=", + Self::Sub => "-", + Self::SubEq => "-=", + Self::Star => "*", + Self::StarEq => "*=", + Self::Slash => "/", + Self::SlashEq => "/=", + Self::Arrow => "->", + Self::Comma => ",", + Self::Colon => ":", + Self::Component => "component", + Self::Func => "func", + Self::Pub => "pub", + Self::Prop => "prop", + Self::Alias => "alias", + Self::Object => "object", + Self::Let => "let", + Self::Mut => "mut", + Self::True => "true", + Self::False => "false", + Self::ShiftRight => ">>", + Self::ShiftLeft => "<<", + Self::Xor => "^", + Self::BitAnd => "&", + Self::BitOr => "|", + Self::BitNot => "~", + Self::Newline => "\n", + Self::Float(v) => return write!(f, "{v}"), + Self::Int(v) => return write!(f, "{v}"), + Self::String(v) => return write!(f, "{v}"), + Self::Identifier(v) => return write!(f, "{v}"), + }; + write!(f, "{s}") } } -impl Token { - pub fn comcomment(pos: usize) -> Self { - Self { - kind: TokenKind::CommonComent, - span: Span { - end: pos, - start: pos, - }, - } - } - // constructor helpers for keywords; snake_case to satisfy lint - pub fn if_token(pos: usize) -> Self { - Self { - kind: TokenKind::If, - span: Span { - end: pos, - start: pos, - }, - } - } - pub fn else_token(pos: usize) -> Self { - Self { - kind: TokenKind::Else, - span: Span { - end: pos, - start: pos, - }, - } - } - pub fn shr(pos: usize) -> Self { - Self { - kind: TokenKind::ShiftRight, - span: Span { - end: pos, - start: pos, - }, - } - } - pub fn shl(pos: usize) -> Self { - Self { - kind: TokenKind::ShiftLeft, - span: Span { - end: pos, - start: pos, - }, - } - } - - pub fn xor(pos: usize) -> Self { - Self { - kind: TokenKind::Xor, - span: Span { - end: pos, - start: pos, - }, - } - } - pub fn bitnot(pos: usize) -> Self { - Self { - kind: TokenKind::BitNot, - span: Span { - end: pos, - start: pos, - }, - } - } - - pub fn bitor(pos: usize) -> Self { - Self { - kind: TokenKind::BitOr, - span: Span { - end: pos, - start: pos, - }, - } - } - pub fn bitand(pos: usize) -> Self { - Self { - kind: TokenKind::BitAnd, - span: Span { - end: pos, - start: pos, - }, - } - } - pub fn or(pos: usize) -> Self { - Self { - kind: TokenKind::Or, - span: Span { - end: pos, - start: pos, - }, - } - } - pub fn and(pos: usize) -> Self { - Self { - kind: TokenKind::And, - span: Span { - end: pos, - start: pos, - }, - } - } - pub fn dot(pos: usize) -> Self { - Self { - kind: TokenKind::Dot, - span: Span { - end: pos, - start: pos, - }, - } - } - pub fn lparen(pos: usize) -> Self { - Self { - kind: TokenKind::LParen, - span: Span { - end: pos, - start: pos, - }, - } - } - pub fn rparen(pos: usize) -> Self { - Self { - kind: TokenKind::RParen, - span: Span { - end: pos, - start: pos, - }, - } - } - pub fn lbrace(pos: usize) -> Self { - Self { - kind: TokenKind::LBrace, - span: Span { - end: pos, - start: pos, - }, - } - } - pub fn rbrace(pos: usize) -> Self { - Self { - kind: TokenKind::RBrace, - span: Span { - end: pos, - start: pos, - }, - } - } - pub fn semicolon(pos: usize) -> Self { - Self { - kind: TokenKind::SemiColon, - span: Span { - end: pos, - start: pos, - }, - } - } - pub fn lt(pos: usize) -> Self { - Self { - kind: TokenKind::Lt, - span: Span { - end: pos, - start: pos, - }, - } - } - pub fn gt(pos: usize) -> Self { - Self { - kind: TokenKind::Gt, - span: Span { - end: pos, - start: pos, - }, - } - } - pub fn eq(pos: usize) -> Self { - Self { - kind: TokenKind::Eq, - span: Span { - end: pos, - start: pos, - }, - } - } - pub fn colon(pos: usize) -> Self { - Self { - kind: TokenKind::Colon, - span: Span { - end: pos, - start: pos, - }, - } - } - pub fn comma(pos: usize) -> Self { - Self { - kind: TokenKind::Comma, - span: Span { - end: pos, - start: pos, - }, - } - } - pub fn plus(pos: usize) -> Self { - Self { - kind: TokenKind::Plus, - span: Span { - end: pos, - start: pos, - }, - } - } - pub fn sub(pos: usize) -> Self { - Self { - kind: TokenKind::Sub, - span: Span { - end: pos, - start: pos, - }, - } - } - pub fn star(pos: usize) -> Self { - Self { - kind: TokenKind::Star, - span: Span { - end: pos, - start: pos, - }, - } - } - pub fn slash(pos: usize) -> Self { - Self { - kind: TokenKind::Slash, - span: Span { - end: pos, - start: pos, - }, - } - } - pub fn lte(pos: usize) -> Self { - Self { - kind: TokenKind::LtEq, - span: Span { - end: pos, - start: pos, - }, - } - } - pub fn gte(pos: usize) -> Self { - Self { - kind: TokenKind::GtEq, - span: Span { - end: pos, - start: pos, - }, - } - } - pub fn pluseq(pos: usize) -> Self { - Self { - kind: TokenKind::PlusEq, - span: Span { - end: pos, - start: pos, - }, - } - } - pub fn subeq(pos: usize) -> Self { - Self { - kind: TokenKind::SubEq, - span: Span { - end: pos, - start: pos, - }, - } - } - pub fn stareq(pos: usize) -> Self { - Self { - kind: TokenKind::StarEq, - span: Span { - end: pos, - start: pos, - }, - } - } - pub fn slasheq(pos: usize) -> Self { - Self { - kind: TokenKind::SlashEq, - span: Span { - end: pos, - start: pos, - }, - } - } - pub fn eqeq(pos: usize) -> Self { - Self { - kind: TokenKind::EqEq, - span: Span { - end: pos, - start: pos, - }, - } - } - pub fn float(value: f32, start: usize, end: usize) -> Self { - Self { - kind: TokenKind::Float(value), - span: Span { end, start }, - } - } - pub fn int(value: i32, start: usize, end: usize) -> Self { - Self { - kind: TokenKind::Int(value), - span: Span { end, start }, - } - } - pub fn identifier(buffer: &str, start: usize, end: usize) -> Self { - Self { - kind: TokenKind::Identifier(buffer.to_string()), - span: Span { end, start }, - } - } - pub fn component(start: usize, end: usize) -> Self { - Self { - kind: TokenKind::Component, - span: Span { end, start }, - } - } - pub fn arrow(start: usize) -> Self { - Self { - kind: TokenKind::Arrow, - span: Span { - start, - end: start + 1, - }, - } - } - pub fn string(str: String, start: usize, end: usize) -> Self { - Self { - kind: TokenKind::String(str), - span: Span { end, start }, - } +impl std::fmt::Display for Token { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "'{}'", self.kind) } } diff --git a/src/compilation_context/mod.rs b/src/compilation_context/mod.rs index 0cad5b6d..39efbe5e 100644 --- a/src/compilation_context/mod.rs +++ b/src/compilation_context/mod.rs @@ -16,7 +16,7 @@ use slynx_monomorphizer::Monomorphizer; use slynx_parser::{ASTDeclaration, Parser}; use slynx_typechecker::TypeChecker; -pub use crate::compilation_context::errors::*; +use crate::compilation_context::errors::{SlynxError, helpers::suggestions_from_lexer}; #[derive(Debug)] pub struct CompilationOutput {