From f67e91d8eb8d4e72b057e6f7f809a2f1362f5a78 Mon Sep 17 00:00:00 2001 From: prql-bot <107324867+prql-bot@users.noreply.github.com> Date: Fri, 4 Sep 2026 07:02:16 +0000 Subject: [PATCH] test: assert on the lexer's recovery entry point instead of printing it --- prqlc/prqlc-parser/src/lexer/mod.rs | 4 +- prqlc/prqlc-parser/src/lexer/test.rs | 106 ++++++++++----------------- 2 files changed, 43 insertions(+), 67 deletions(-) diff --git a/prqlc/prqlc-parser/src/lexer/mod.rs b/prqlc/prqlc-parser/src/lexer/mod.rs index 8b8c4e53328e..497d236318db 100644 --- a/prqlc/prqlc-parser/src/lexer/mod.rs +++ b/prqlc/prqlc-parser/src/lexer/mod.rs @@ -61,7 +61,9 @@ fn convert_lexer_error(source: &str, error: &Simple<'_, char>, source_id: u16) - .with_source(ErrorSource::Lexer(error_source)) } -/// Lex PRQL into LR, returning both the LR and any errors encountered +/// Lex PRQL into LR, returning the tokens if the whole source lexed and the +/// errors if it didn't. The lexer doesn't resume after an error, so tokens and +/// errors never come back together. pub fn lex_source_recovery(source: &str, source_id: u16) -> (Option>, Vec) { let result = lexer().parse(source).into_result(); diff --git a/prqlc/prqlc-parser/src/lexer/test.rs b/prqlc/prqlc-parser/src/lexer/test.rs index 491a692a46e8..c698d69915df 100644 --- a/prqlc/prqlc-parser/src/lexer/test.rs +++ b/prqlc/prqlc-parser/src/lexer/test.rs @@ -2,9 +2,8 @@ use chumsky::Parser; use insta::assert_debug_snapshot; use insta::assert_snapshot; -use crate::lexer::lex_source; use crate::lexer::lr::{Literal, TokenKind, Tokens}; -use crate::lexer::{lexer, literal, quoted_string}; +use crate::lexer::{lex_source, lex_source_recovery, lexer, literal, quoted_string}; #[test] fn line_wrap() { @@ -263,8 +262,6 @@ fn range() { #[test] fn test_lex_source() { - use insta::assert_debug_snapshot; - assert_debug_snapshot!(lex_source("5 + 3"), @" Ok( Tokens( @@ -284,10 +281,8 @@ fn test_lex_source() { #[test] fn test_annotation_tokens() { - use insta::assert_debug_snapshot; - // Test basic annotation token - let result = super::lex_source("@{binding_strength=1}"); + let result = lex_source("@{binding_strength=1}"); assert_debug_snapshot!(result, @r#" Ok( Tokens( @@ -305,7 +300,7 @@ fn test_annotation_tokens() { "#); // Test multi-line annotation - let result = super::lex_source( + let result = lex_source( r#" @{binding_strength=1} let add = a b -> a + b @@ -342,17 +337,15 @@ fn test_annotation_tokens() { #[test] fn test_issue_triple_quoted_with_double_quote() { - use insta::assert_debug_snapshot; - - // The specific test case from ISSUE.md that was failing + // A triple-quoted string whose body contains both quote characters: the + // lone `"` is content, and only the closing triple ends the literal. let input = r#"""" '' Canada " """"#; - let result = super::lex_source(input); - eprintln!("Result: {:#?}", result); + let result = lex_source(input); assert_debug_snapshot!(result, @r#" Ok( Tokens( @@ -367,22 +360,11 @@ Canada #[test] fn test_single_curly_quote() { - use insta::assert_debug_snapshot; - - // Test what error we get for a single curly quote character + // A curly quote isn't a string delimiter, so it's rejected outright — and + // the span covers the one character, not its three UTF-8 bytes. let input = "’"; // U+2019 RIGHT SINGLE QUOTATION MARK - eprintln!("\n=== Single Curly Quote Test ==="); - eprintln!("Input: {:?}", input); - eprintln!("Input bytes: {:?}", input.as_bytes()); - eprintln!( - "Char 0: {:?} (U+{:04X})", - input.chars().next().unwrap(), - input.chars().next().unwrap() as u32 - ); - let result = lex_source(input); - eprintln!("Result: {:#?}", result); assert_debug_snapshot!(result, @r#" Err( @@ -405,36 +387,15 @@ fn test_single_curly_quote() { #[test] fn test_mississippi_curly_quotes() { - use insta::assert_debug_snapshot; - // Test error reporting for curly quotes (U+2019) // This is the Mississippi test case from integration tests // NOTE: The quotes in this string are U+2019 RIGHT SINGLE QUOTATION MARK (curly quotes), // not U+0027 APOSTROPHE. Make sure your editor preserves them! let input = "Mississippi has four S’s and four I’s."; - eprintln!("\n=== Mississippi Curly Quotes Test ==="); - eprintln!("Input: {:?}", input); - eprintln!("Input bytes: {:?}", input.as_bytes()); - eprintln!( - "Char 22: {:?} (U+{:04X})", - input.chars().nth(22).unwrap(), - input.chars().nth(22).unwrap() as u32 - ); - eprintln!( - "Char 35: {:?} (U+{:04X})", - input.chars().nth(35).unwrap(), - input.chars().nth(35).unwrap() as u32 - ); - - let result1 = lex_source(input); - eprintln!("{:#?}", result1); - - let (tokens, errors) = super::lex_source_recovery(input, 1); - eprintln!("Tokens: {:#?}", tokens); - eprintln!("Errors: {:#?}", errors); + let result = lex_source(input); - assert_debug_snapshot!(result1, @r#" + assert_debug_snapshot!(result, @r#" Err( [ Error { @@ -455,26 +416,11 @@ fn test_mississippi_curly_quotes() { #[test] fn test_interpolation_empty() { - use insta::assert_debug_snapshot; - - // Test the f"{}" case that's showing a changed error position + // An f-string opened but never closed: the error is reported at the end of + // input rather than at the interpolation's `{`. let input = r#"from x | select f"{}"#; - eprintln!("\n=== Interpolation Empty Test ==="); - eprintln!("Input: {:?}", input); - eprintln!("Input bytes: {:?}", input.as_bytes()); - eprintln!( - "Input length: {} bytes, {} chars", - input.len(), - input.chars().count() - ); - let result = lex_source(input); - eprintln!("lex_source result: {:#?}", result); - - let (tokens, errors) = super::lex_source_recovery(input, 1); - eprintln!("lex_source_recovery tokens: {:#?}", tokens); - eprintln!("lex_source_recovery errors: {:#?}", errors); assert_debug_snapshot!(result, @r#" Err( @@ -577,3 +523,31 @@ fn test_reserved_literals_lex_as_literals() { ); } } + +/// `lex_source_recovery` is the entry point `prqlc`'s parser uses, and the only +/// one that threads a `source_id` — hence the `1:` on the span below, where +/// `lex_source` hard-codes `0:`. Despite the name it doesn't resume after an +/// error, so the tokens come back as `None` rather than as the prefix it had +/// already lexed. Previously this function was called from two tests only to +/// print its output, so nothing asserted on it. +#[test] +fn recovery_returns_no_tokens_on_error() { + let (tokens, errors) = lex_source_recovery("Mississippi has four S’s.", 1); + + assert_debug_snapshot!(tokens, @"None"); + assert_debug_snapshot!(errors, @r#" + [ + Error { + kind: Error, + span: Some( + 1:22-23, + ), + reason: Unexpected { + found: "'’'", + }, + hints: [], + code: None, + }, + ] + "#); +}