From 6327fd68b12620dd96406aed97017e203a29ae32 Mon Sep 17 00:00:00 2001 From: Leynos Date: Fri, 1 Aug 2025 00:05:12 +0100 Subject: [PATCH 1/6] Document tokenizer functions --- src/wrap.rs | 1 + src/wrap/tokenize.rs | 94 +++++++++++++++++++++++++++++++++++++++----- 2 files changed, 86 insertions(+), 9 deletions(-) diff --git a/src/wrap.rs b/src/wrap.rs index b26688ee..770a1790 100644 --- a/src/wrap.rs +++ b/src/wrap.rs @@ -13,6 +13,7 @@ mod tokenize; /// Re-export this so callers of [`crate::textproc`] can implement custom /// transformations without depending on internal modules. pub use tokenize::Token; +pub use tokenize::tokenize_markdown; static FENCE_RE: std::sync::LazyLock = std::sync::LazyLock::new(|| Regex::new(r"^\s*(```|~~~).*").unwrap()); diff --git a/src/wrap/tokenize.rs b/src/wrap/tokenize.rs index a01f052e..47b98d9d 100644 --- a/src/wrap/tokenize.rs +++ b/src/wrap/tokenize.rs @@ -3,6 +3,18 @@ //! This module contains utilities for breaking lines into tokens so that //! inline code spans and Markdown links are preserved during wrapping. +/// Advance `i` while the predicate evaluates to `true`. +/// +/// Returns the index of the first character for which `cond` fails. This small +/// helper keeps the scanning loops concise. +/// +/// # Examples +/// +/// ```rust,ignore +/// let chars: Vec = "abc123".chars().collect(); +/// let end = scan_while(&chars, 0, char::is_alphabetic); +/// assert_eq!(end, 3); +/// ``` fn scan_while(chars: &[char], mut i: usize, cond: F) -> usize where F: Fn(char) -> bool, @@ -13,6 +25,14 @@ where i } +/// Collect a range of characters into a [`String`]. +/// +/// # Examples +/// +/// ```rust,ignore +/// let chars: Vec = ['a', 'b', 'c']; +/// assert_eq!(collect_range(&chars, 0, 2), "ab"); +/// ``` fn collect_range(chars: &[char], start: usize, end: usize) -> String { chars[start..end].iter().collect() } @@ -35,6 +55,15 @@ pub enum Token<'a> { /// Handles nested parentheses within URLs by tracking the depth of opening and /// closing delimiters. Returns the parsed slice and the index after the closing /// parenthesis if one is found. +/// +/// # Examples +/// +/// ```rust,ignore +/// let chars: Vec = "![alt](a(b)c)".chars().collect(); +/// let (tok, idx) = parse_link_or_image(&chars, 0); +/// assert_eq!(tok, "![alt](a(b)c)"); +/// assert_eq!(idx, chars.len()); +/// ``` fn parse_link_or_image(chars: &[char], mut i: usize) -> (String, usize) { let start = i; if chars[i] == '!' { @@ -61,6 +90,17 @@ fn parse_link_or_image(chars: &[char], mut i: usize) -> (String, usize) { (collect_range(chars, start, start + 1), start + 1) } +/// Determine whether a character is considered trailing punctuation. +/// +/// The wrapper treats such punctuation as part of the preceding link when +/// wrapping lines. +/// +/// # Examples +/// +/// ```rust,ignore +/// assert!(is_trailing_punctuation('.')); +/// assert!(!is_trailing_punctuation('a')); +/// ``` fn is_trailing_punctuation(c: char) -> bool { matches!( c, @@ -68,6 +108,21 @@ fn is_trailing_punctuation(c: char) -> bool { ) } +/// Break a single line of text into inline token strings. +/// +/// Code spans, links, images and surrounding whitespace are preserved as +/// separate tokens. This simplifies later wrapping logic which operates on +/// slices of the original text. +/// +/// # Examples +/// +/// ```rust,ignore +/// let tokens = segment_inline("see [link](url) and `code`"); +/// assert_eq!( +/// tokens, +/// vec!["see", " ", "[link](url)", " ", "and", " ", "`code`"] +/// ); +/// ``` pub(super) fn segment_inline(text: &str) -> Vec { let mut tokens = Vec::new(); let chars: Vec = text.chars().collect(); @@ -119,6 +174,17 @@ pub(super) fn segment_inline(text: &str) -> Vec { tokens } +/// Emit [`Token`]s for inline segments within a single line. +/// +/// The function scans for backtick sequences and yields `Token::Code` for +/// matched spans. Text outside code spans is emitted as `Token::Text` via the +/// provided callback. +/// +/// # Examples +/// +/// ```rust,ignore +/// tokenize_inline("run `cmd`", &mut |t| println!("{:?}", t)); +/// ``` fn tokenize_inline<'a, F>(text: &'a str, emit: &mut F) where F: FnMut(Token<'a>), @@ -145,7 +211,25 @@ where } } -/// Tokenize a block of Markdown into [`Token`]s. +/// Tokenize a Markdown snippet using backtick-delimited code spans. +/// +/// The function scans the input line by line. Lines matching [`FENCE_RE`] +/// produce [`Token::Fence`] tokens and toggle fenced mode. Lines inside a +/// fence are yielded verbatim. Outside fenced regions the scanner searches for +/// backtick sequences. Text before a backtick becomes [`Token::Text`]. When a +/// closing backtick follows, the enclosed portion forms a [`Token::Code`] +/// span. If no closing backtick is found the delimiter and remaining text are +/// returned as [`Token::Text`]. Whitespace is preserved exactly as it appears. +/// +/// ```rust,no_run +/// use mdtablefix::wrap::{Token, tokenize_markdown}; +/// +/// let tokens = tokenize_markdown("Example with `code`"); +/// assert_eq!( +/// tokens, +/// vec![Token::Text("Example with "), Token::Code("code")] +/// ); +/// ``` #[must_use] pub fn tokenize_markdown(source: &str) -> Vec> { if source.is_empty() { @@ -183,14 +267,6 @@ pub fn tokenize_markdown(source: &str) -> Vec> { tokens } -/// Split the input string into [`Token`]s by analysing whitespace and backtick -/// delimiters. -/// -/// The tokenizer groups consecutive whitespace into a single [`Token::Text`] and -/// recognises backtick sequences as inline code spans. When a run of backticks -/// is encountered the parser searches forward for an identical delimiter, -/// allowing nested backticks when the span uses a longer fence. Unmatched -/// delimiter sequences are treated as literal text. #[cfg(test)] mod tests { use super::*; From 74be69c7d70ee83c0114fec12bf53d44a9c37857 Mon Sep 17 00:00:00 2001 From: Leynos Date: Sun, 3 Aug 2025 18:03:52 +0100 Subject: [PATCH 2/6] Inline docs for tokenizer re-export --- src/html.rs | 12 +++++++----- src/wrap.rs | 21 +++++++++++++++++---- src/wrap/tokenize.rs | 24 +++++++++++++++++++----- 3 files changed, 43 insertions(+), 14 deletions(-) diff --git a/src/html.rs b/src/html.rs index c3372881..b148e314 100644 --- a/src/html.rs +++ b/src/html.rs @@ -84,7 +84,9 @@ fn is_element(handle: &Handle, tag: &str) -> bool { } /// Returns `true` if `handle` represents a `` or `` element. -fn is_table_cell(handle: &Handle) -> bool { is_element(handle, "td") || is_element(handle, "th") } +fn is_table_cell(handle: &Handle) -> bool { + is_element(handle, "td") || is_element(handle, "th") +} /// Walks the DOM tree collecting `` nodes under `handle`. fn collect_tables(handle: &Handle, tables: &mut Vec) { @@ -112,10 +114,10 @@ fn is_bold_tag(tag: &str) -> bool { /// Returns `true` if `handle` contains a `` or `` descendant. fn contains_strong(handle: &Handle) -> bool { - if let NodeData::Element { name, .. } = &handle.data - && is_bold_tag(name.local.as_ref()) - { - return true; + if let NodeData::Element { name, .. } = &handle.data { + if is_bold_tag(name.local.as_ref()) { + return true; + } } let children = handle.children.borrow(); children.iter().any(contains_strong) diff --git a/src/wrap.rs b/src/wrap.rs index 770a1790..6703be68 100644 --- a/src/wrap.rs +++ b/src/wrap.rs @@ -13,6 +13,11 @@ mod tokenize; /// Re-export this so callers of [`crate::textproc`] can implement custom /// transformations without depending on internal modules. pub use tokenize::Token; +/// Tokenize a Markdown snippet using backtick-delimited code spans. +/// +/// Re-exporting this helper lets downstream crates parse Markdown without +/// depending on the private [`tokenize`] module. +#[doc(inline)] pub use tokenize::tokenize_markdown; static FENCE_RE: std::sync::LazyLock = @@ -51,11 +56,17 @@ struct PrefixHandler { } impl PrefixHandler { - fn build_bullet_prefix(cap: &Captures) -> String { cap[1].to_string() } + fn build_bullet_prefix(cap: &Captures) -> String { + cap[1].to_string() + } - fn build_footnote_prefix(cap: &Captures) -> String { format!("{}{}", &cap[1], &cap[2]) } + fn build_footnote_prefix(cap: &Captures) -> String { + format!("{}{}", &cap[1], &cap[2]) + } - fn build_blockquote_prefix(cap: &Captures) -> String { cap[1].to_string() } + fn build_blockquote_prefix(cap: &Captures) -> String { + cap[1].to_string() + } } static HANDLERS: &[PrefixHandler] = &[ @@ -188,7 +199,9 @@ fn wrap_preserving_code(text: &str, width: usize) -> Vec { } #[doc(hidden)] -pub fn is_fence(line: &str) -> bool { FENCE_RE.is_match(line) } +pub fn is_fence(line: &str) -> bool { + FENCE_RE.is_match(line) +} pub(crate) fn is_markdownlint_directive(line: &str) -> bool { MARKDOWNLINT_DIRECTIVE_RE.is_match(line) diff --git a/src/wrap/tokenize.rs b/src/wrap/tokenize.rs index 47b98d9d..a71215b7 100644 --- a/src/wrap/tokenize.rs +++ b/src/wrap/tokenize.rs @@ -15,9 +15,9 @@ /// let end = scan_while(&chars, 0, char::is_alphabetic); /// assert_eq!(end, 3); /// ``` -fn scan_while(chars: &[char], mut i: usize, cond: F) -> usize +fn scan_while(chars: &[char], mut i: usize, mut cond: F) -> usize where - F: Fn(char) -> bool, + F: FnMut(char) -> bool, { while i < chars.len() && cond(chars[i]) { i += 1; @@ -99,12 +99,13 @@ fn parse_link_or_image(chars: &[char], mut i: usize) -> (String, usize) { /// /// ```rust,ignore /// assert!(is_trailing_punctuation('.')); +/// assert!(is_trailing_punctuation('(')); /// assert!(!is_trailing_punctuation('a')); /// ``` fn is_trailing_punctuation(c: char) -> bool { matches!( c, - '.' | ',' | ';' | ':' | '!' | '?' | ')' | ']' | '"' | '\'' + '.' | ',' | ';' | ':' | '!' | '?' | '(' | ')' | ']' | '"' | '\'' ) } @@ -122,6 +123,13 @@ fn is_trailing_punctuation(c: char) -> bool { /// tokens, /// vec!["see", " ", "[link](url)", " ", "and", " ", "`code`"] /// ); +/// +/// // Example with consecutive and unusual whitespace +/// let tokens = segment_inline("foo bar\tbaz `qux`"); +/// assert_eq!( +/// tokens, +/// vec!["foo", " ", "bar", "\t", "baz", " ", "`qux`"] +/// ); /// ``` pub(super) fn segment_inline(text: &str) -> Vec { let mut tokens = Vec::new(); @@ -183,8 +191,14 @@ pub(super) fn segment_inline(text: &str) -> Vec { /// # Examples /// /// ```rust,ignore +/// // Prints: +/// // Token::Text("run ") +/// // Token::Code("cmd") /// tokenize_inline("run `cmd`", &mut |t| println!("{:?}", t)); /// ``` +/// +/// The callback receives each token as a [`Token<'a>`], such as +/// `Token::Text(&str)` or `Token::Code(&str)`. fn tokenize_inline<'a, F>(text: &'a str, emit: &mut F) where F: FnMut(Token<'a>), @@ -221,8 +235,8 @@ where /// span. If no closing backtick is found the delimiter and remaining text are /// returned as [`Token::Text`]. Whitespace is preserved exactly as it appears. /// -/// ```rust,no_run -/// use mdtablefix::wrap::{Token, tokenize_markdown}; +/// ```rust +/// use crate::wrap::{Token, tokenize_markdown}; /// /// let tokens = tokenize_markdown("Example with `code`"); /// assert_eq!( From 3b5dedbeac26b618f5c3c5d925d9e0ca1e91bc8f Mon Sep 17 00:00:00 2001 From: Leynos Date: Sun, 3 Aug 2025 19:20:38 +0100 Subject: [PATCH 3/6] Remov e duplicate comments --- src/wrap.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/wrap.rs b/src/wrap.rs index 6703be68..6868cf36 100644 --- a/src/wrap.rs +++ b/src/wrap.rs @@ -13,10 +13,6 @@ mod tokenize; /// Re-export this so callers of [`crate::textproc`] can implement custom /// transformations without depending on internal modules. pub use tokenize::Token; -/// Tokenize a Markdown snippet using backtick-delimited code spans. -/// -/// Re-exporting this helper lets downstream crates parse Markdown without -/// depending on the private [`tokenize`] module. #[doc(inline)] pub use tokenize::tokenize_markdown; From d952dfbaa76f337e245a18bf78b07bc9fc561f6d Mon Sep 17 00:00:00 2001 From: Payton McIntosh Date: Sun, 3 Aug 2025 19:52:43 +0100 Subject: [PATCH 4/6] Apply formatting --- src/html.rs | 4 +--- src/wrap.rs | 16 ++++------------ 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/src/html.rs b/src/html.rs index b148e314..2742d7b3 100644 --- a/src/html.rs +++ b/src/html.rs @@ -84,9 +84,7 @@ fn is_element(handle: &Handle, tag: &str) -> bool { } /// Returns `true` if `handle` represents a `
` or `` element. -fn is_table_cell(handle: &Handle) -> bool { - is_element(handle, "td") || is_element(handle, "th") -} +fn is_table_cell(handle: &Handle) -> bool { is_element(handle, "td") || is_element(handle, "th") } /// Walks the DOM tree collecting `` nodes under `handle`. fn collect_tables(handle: &Handle, tables: &mut Vec) { diff --git a/src/wrap.rs b/src/wrap.rs index 5f67fcdc..67775790 100644 --- a/src/wrap.rs +++ b/src/wrap.rs @@ -61,17 +61,11 @@ struct PrefixHandler { } impl PrefixHandler { - fn build_bullet_prefix(cap: &Captures) -> String { - cap[1].to_string() - } + fn build_bullet_prefix(cap: &Captures) -> String { cap[1].to_string() } - fn build_footnote_prefix(cap: &Captures) -> String { - format!("{}{}", &cap[1], &cap[2]) - } + fn build_footnote_prefix(cap: &Captures) -> String { format!("{}{}", &cap[1], &cap[2]) } - fn build_blockquote_prefix(cap: &Captures) -> String { - cap[1].to_string() - } + fn build_blockquote_prefix(cap: &Captures) -> String { cap[1].to_string() } } static HANDLERS: &[PrefixHandler] = &[ @@ -204,9 +198,7 @@ fn wrap_preserving_code(text: &str, width: usize) -> Vec { } #[doc(hidden)] -pub fn is_fence(line: &str) -> bool { - FENCE_RE.is_match(line) -} +pub fn is_fence(line: &str) -> bool { FENCE_RE.is_match(line) } pub(crate) fn is_markdownlint_directive(line: &str) -> bool { MARKDOWNLINT_DIRECTIVE_RE.is_match(line) From 6d26daa3c1d773ec37c1fbaca6c3ee5fbdacc1b9 Mon Sep 17 00:00:00 2001 From: Leynos Date: Sun, 3 Aug 2025 19:57:54 +0100 Subject: [PATCH 5/6] Add missing import --- src/wrap.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wrap.rs b/src/wrap.rs index 67775790..e1c13d86 100644 --- a/src/wrap.rs +++ b/src/wrap.rs @@ -8,7 +8,7 @@ //! The [`Token`] enum and [`tokenize_markdown`] function are public so callers //! can perform custom token-based processing. -use regex::Regex; +use regex::{Captures, Regex}; mod tokenize; /// Token emitted by [`tokenize::segment_inline`] and used by higher-level wrappers. From 9b677faf728749bccf7a598514bc771ad4c3a556 Mon Sep 17 00:00:00 2001 From: Payton McIntosh Date: Sun, 3 Aug 2025 20:06:37 +0100 Subject: [PATCH 6/6] Cleans up unused prefix handling code Removes an unused PrefixHandler abstraction and its associated regex captures import from the markdown wrapping logic. Simplifies the code while preserving all functionality. Also improves code clarity in the strong tag detection by using Rust's let-and pattern matching syntax. --- src/html.rs | 8 ++++---- src/wrap.rs | 38 +------------------------------------- 2 files changed, 5 insertions(+), 41 deletions(-) diff --git a/src/html.rs b/src/html.rs index 2742d7b3..c3372881 100644 --- a/src/html.rs +++ b/src/html.rs @@ -112,10 +112,10 @@ fn is_bold_tag(tag: &str) -> bool { /// Returns `true` if `handle` contains a `` or `` descendant. fn contains_strong(handle: &Handle) -> bool { - if let NodeData::Element { name, .. } = &handle.data { - if is_bold_tag(name.local.as_ref()) { - return true; - } + if let NodeData::Element { name, .. } = &handle.data + && is_bold_tag(name.local.as_ref()) + { + return true; } let children = handle.children.borrow(); children.iter().any(contains_strong) diff --git a/src/wrap.rs b/src/wrap.rs index e1c13d86..9aa0b0ea 100644 --- a/src/wrap.rs +++ b/src/wrap.rs @@ -8,7 +8,7 @@ //! The [`Token`] enum and [`tokenize_markdown`] function are public so callers //! can perform custom token-based processing. -use regex::{Captures, Regex}; +use regex::Regex; mod tokenize; /// Token emitted by [`tokenize::segment_inline`] and used by higher-level wrappers. @@ -53,42 +53,6 @@ static MARKDOWNLINT_DIRECTIVE_RE: std::sync::LazyLock = std::sync::LazyLo .expect("valid markdownlint regex") }); -struct PrefixHandler { - re: &'static std::sync::LazyLock, - is_bq: bool, - build_prefix: fn(&Captures) -> String, - rest_group: usize, -} - -impl PrefixHandler { - fn build_bullet_prefix(cap: &Captures) -> String { cap[1].to_string() } - - fn build_footnote_prefix(cap: &Captures) -> String { format!("{}{}", &cap[1], &cap[2]) } - - fn build_blockquote_prefix(cap: &Captures) -> String { cap[1].to_string() } -} - -static HANDLERS: &[PrefixHandler] = &[ - PrefixHandler { - re: &BULLET_RE, - is_bq: false, - build_prefix: PrefixHandler::build_bullet_prefix, - rest_group: 2, - }, - PrefixHandler { - re: &FOOTNOTE_RE, - is_bq: false, - build_prefix: PrefixHandler::build_footnote_prefix, - rest_group: 3, - }, - PrefixHandler { - re: &BLOCKQUOTE_RE, - is_bq: true, - build_prefix: PrefixHandler::build_blockquote_prefix, - rest_group: 2, - }, -]; - fn wrap_preserving_code(text: &str, width: usize) -> Vec { use unicode_width::UnicodeWidthStr;