diff --git a/benchmark/streaming_performance_benchmark.dart b/benchmark/streaming_performance_benchmark.dart new file mode 100644 index 0000000..936b938 --- /dev/null +++ b/benchmark/streaming_performance_benchmark.dart @@ -0,0 +1,246 @@ +// ignore_for_file: avoid_print + +import 'package:flutter_md/src/parser.dart'; + +/// Benchmark comparing MarkdownDecoder vs StreamingMarkdownDecoder +/// for streaming scenarios. +void main() { + print('='.padRight(80, '=')); + print('Streaming Markdown Decoder Performance Benchmark'); + print('='.padRight(80, '=')); + print(''); + + // Test with different content sizes + final smallContent = _generateMarkdownContent(lines: 50); + final mediumContent = _generateMarkdownContent(lines: 200); + final largeContent = _generateMarkdownContent(lines: 500); + + print('Test Content Sizes:'); + print(' Small: ${smallContent.length} chars, ~50 lines'); + print(' Medium: ${mediumContent.length} chars, ~200 lines'); + print(' Large: ${largeContent.length} chars, ~500 lines'); + print(''); + + // Run benchmarks + _runBenchmark('Small Content (50 lines)', smallContent, chunkSize: 5); + print(''); + + _runBenchmark('Medium Content (200 lines)', mediumContent, chunkSize: 10); + print(''); + + _runBenchmark('Large Content (500 lines)', largeContent, chunkSize: 15); + print(''); + + print('='.padRight(80, '=')); + print('Summary:'); + print(' StreamingMarkdownDecoder is optimized for incremental parsing'); + print(' and should show significant performance gains as content grows.'); + print('='.padRight(80, '=')); +} + +/// Runs a benchmark comparing both decoders. +void _runBenchmark(String name, String content, {int chunkSize = 10}) { + print('$name (chunk size: $chunkSize chars)'); + print('-'.padRight(80, '-')); + + // Simulate streaming chunks + final chunks = _splitIntoChunks(content, chunkSize); + print('Total chunks: ${chunks.length}'); + print(''); + + // Benchmark 1: MarkdownDecoder (full re-parse each time) + final batch = _benchmarkBatchDecoder(chunks); + + // Benchmark 2: StreamingMarkdownDecoder (incremental) + final streaming = _benchmarkStreamingDecoder(chunks); + + // Results + print('Results:'); + print(' MarkdownDecoder:'); + print(' Total time: ${batch.totalTime.inMilliseconds}ms'); + print(' Parse count: ${batch.parseCount}'); + print( + ' Avg per parse: ' + '${batch.avgTimePerParse.inMicroseconds}µs'); + print(' Final blocks: ${batch.finalBlockCount}'); + print(''); + print(' StreamingMarkdownDecoder:'); + print(' Total time: ${streaming.totalTime.inMilliseconds}ms'); + print(' Parse count: ${streaming.parseCount}'); + print( + ' Avg per parse: ' + '${streaming.avgTimePerParse.inMicroseconds}µs'); + print(' Final blocks: ${streaming.finalBlockCount}'); + print(' Lines parsed: ${streaming.totalLinesParsed}'); + print( + ' Avg lines/parse: ' + '${streaming.avgLinesPerParse.toStringAsFixed(1)}'); + print(''); + print(' Performance Gain:'); + final speedup = batch.totalTime.inMicroseconds / + streaming.totalTime.inMicroseconds; + print(' Speedup: ${speedup.toStringAsFixed(2)}x faster'); + final timeSaved = batch.totalTime - streaming.totalTime; + print(' Time saved: ${timeSaved.inMilliseconds}ms'); + print(''); +} + +/// Benchmark result data. +class BenchmarkResult { + const BenchmarkResult({ + required this.totalTime, + required this.parseCount, + required this.finalBlockCount, + this.totalLinesParsed = 0, + }); + + final Duration totalTime; + final int parseCount; + final int finalBlockCount; + final int totalLinesParsed; + + Duration get avgTimePerParse => Duration( + microseconds: totalTime.inMicroseconds ~/ parseCount, + ); + + double get avgLinesPerParse => + totalLinesParsed > 0 ? totalLinesParsed / parseCount : 0; +} + +/// Benchmark MarkdownDecoder (re-parses entire content each time). +BenchmarkResult _benchmarkBatchDecoder(List chunks) { + final stopwatch = Stopwatch()..start(); + var accumulated = ''; + var parseCount = 0; + var finalBlockCount = 0; + + for (final chunk in chunks) { + accumulated += chunk; + final result = markdownDecoder.convert(accumulated); + parseCount++; + finalBlockCount = result.blocks.length; + } + + stopwatch.stop(); + + return BenchmarkResult( + totalTime: stopwatch.elapsed, + parseCount: parseCount, + finalBlockCount: finalBlockCount, + ); +} + +/// Benchmark StreamingMarkdownDecoder (incremental parsing). +BenchmarkResult _benchmarkStreamingDecoder(List chunks) { + final decoder = StreamingMarkdownDecoder(); + final stopwatch = Stopwatch()..start(); + var parseCount = 0; + var finalBlockCount = 0; + var totalLinesParsed = 0; + + for (final chunk in chunks) { + final beforeOpenIndex = decoder.firstOpenIndex; + + decoder.append(chunk); + + final afterLines = decoder.lineCount; + final linesInThisParse = afterLines - beforeOpenIndex; + totalLinesParsed += linesInThisParse; + + parseCount++; + finalBlockCount = decoder.blocks.length; + } + + stopwatch.stop(); + + return BenchmarkResult( + totalTime: stopwatch.elapsed, + parseCount: parseCount, + finalBlockCount: finalBlockCount, + totalLinesParsed: totalLinesParsed, + ); +} + +/// Splits content into chunks of specified size. +List _splitIntoChunks(String content, int chunkSize) { + final chunks = []; + for (var i = 0; i < content.length; i += chunkSize) { + final end = (i + chunkSize).clamp(0, content.length); + chunks.add(content.substring(i, end)); + } + return chunks; +} + +/// Generates markdown content for testing. +String _generateMarkdownContent({required int lines}) { + final buffer = StringBuffer(); + + buffer.writeln('# Benchmark Test Document'); + buffer.writeln(''); + buffer.writeln('This document is generated for performance testing.'); + buffer.writeln(''); + + var lineCount = 4; // Already written 4 lines above + + // Add headings + for (var i = 1; lineCount < lines && i <= 6; i++) { + buffer.writeln('${'#' * i} Heading Level $i'); + lineCount++; + } + + buffer.writeln(''); + lineCount++; + + // Add paragraphs + while (lineCount < lines - 100) { + buffer.writeln( + 'This is a paragraph with **bold**, *italic*, and `code` formatting. ' + 'Lorem ipsum dolor sit amet, consectetur adipiscing elit.', + ); + buffer.writeln(''); + lineCount += 2; + } + + // Add code block + if (lineCount < lines - 50) { + buffer.writeln('```dart'); + lineCount++; + while (lineCount < lines - 40) { + buffer.writeln('void function$lineCount() {'); + buffer.writeln(' print("Line $lineCount");'); + buffer.writeln('}'); + lineCount += 3; + } + buffer.writeln('```'); + buffer.writeln(''); + lineCount += 2; + } + + // Add list + if (lineCount < lines - 20) { + buffer.writeln('## List Items'); + lineCount++; + var itemNum = 1; + while (lineCount < lines - 10) { + buffer.writeln('- Item $itemNum'); + lineCount++; + itemNum++; + } + buffer.writeln(''); + lineCount++; + } + + // Add table + if (lineCount < lines - 5) { + buffer.writeln('| Column A | Column B | Column C |'); + buffer.writeln('|----------|----------|----------|'); + lineCount += 2; + while (lineCount < lines) { + buffer.writeln( + '| Value A$lineCount | Value B$lineCount | Value C$lineCount |'); + lineCount++; + } + } + + return buffer.toString(); +} diff --git a/lib/src/parser.dart b/lib/src/parser.dart index 00bb8a3..1849756 100644 --- a/lib/src/parser.dart +++ b/lib/src/parser.dart @@ -9,252 +9,416 @@ import 'nodes.dart'; /// into a list of [MD$Block] objects. const Converter markdownDecoder = MarkdownDecoder(); -/// {@template markdown_decoder} -/// A [Converter] that decodes Markdown formatted strings -/// into list of [MD$Block] objects. -/// This class is designed to parse Markdown syntax -/// and convert it into a structured format -/// {@endtemplate} -class MarkdownDecoder extends Converter { - /// Creates a new instance of [MarkdownDecoder]. - /// {@macro markdown_decoder} - const MarkdownDecoder(); +// ============================================================================ +// Line State for Streaming +// ============================================================================ + +/// State of a parsed line in streaming context. +enum LineState { + /// Line is not yet confirmed closed, may receive more content + /// or belongs to an unclosed block. + open, + + /// Line belongs to a closed block and will not change. + closed, +} - /// A regular expression pattern to match empty lines. - static final RegExp _emptyPattern = RegExp(r'^(?:[ \t]*)$'); +/// A line with its parsing state for streaming context. +class ParsedLine { + /// Creates a new parsed line. + ParsedLine(this.text, {this.state = LineState.open}); - /// Leading (and trailing) `#` define atx-style headers. - /// - /// Starts with 1-6 unescaped `#` characters which must not be followed by a - /// non-space character. Line may end with any number of `#` characters,. - static final RegExp _headerPattern = RegExp(r'^(#{1,6})'); + /// The text content of the line. + String text; - /// A regular expression pattern to match ordered lists. - /// Matches lines that start with a number followed by a period - /// or parenthesis, or with a bullet point (`*`, `+`, or `-`). - static final RegExp _listPattern = RegExp( - r'^(?[ \t]{0,8})(?(\d{1,9})[\.)]|[*+-])(?[ \t]+(.*))?$'); + /// The parsing state of this line. + LineState state; @override - Markdown convert(String input) { - final lines = LineSplitter.split(input).toList(growable: false); - if (lines.isEmpty) return const Markdown.empty(); - final blocks = Queue(); // Queue to accumulate blocks - final length = lines.length; + String toString() => 'ParsedLine($text, $state)'; +} + +// ============================================================================ +// Shared Parsing Logic +// ============================================================================ + +/// Regular expression pattern to match empty lines. +final RegExp _emptyPattern = RegExp(r'^(?:[ \t]*)$'); + +/// Leading `#` define atx-style headers (1-6 levels). +final RegExp _headerPattern = RegExp(r'^(#{1,6})'); - final paragraph = StringBuffer(); // To accumulate lines for paragraphs +/// Pattern to match list items (ordered and unordered). +final RegExp _listPattern = RegExp( + r'^(?[ \t]{0,8})(?(\d{1,9})[\.)]|[*+-])(?[ \t]+(.*))?$'); + +/// Callback type for when a block is closed (lines [start] to [end] exclusive). +typedef OnBlockClosed = void Function(int start, int end); + +/// Callback type for when a code block is found but not closed. +/// Returns true to continue parsing, false to break. +typedef OnCodeBlockOpen = bool Function(int startLine, String language); + +/// Result of parsing markdown lines. +class _ParseResult { + const _ParseResult({ + required this.blocks, + required this.nextIndex, + this.hasOpenCodeBlock = false, + }); + + final List blocks; + final int nextIndex; + final bool hasOpenCodeBlock; +} - void maybeCommitParagraph() { - if (paragraph.isEmpty) return; - final text = paragraph.toString(); - paragraph.clear(); - blocks.addLast(MD$Paragraph( +/// Core parsing function shared by both [MarkdownDecoder] and +/// [StreamingMarkdownDecoder]. +/// +/// Parameters: +/// - [length]: Total number of lines +/// - [lineAt]: Function to get line text at index +/// - [startIndex]: Index to start parsing from +/// - [existingBlocks]: Optional existing blocks to append to +/// - [onBlockClosed]: Optional callback when a block is closed +/// - [onCodeBlockOpen]: Optional callback when code block is open (not closed) +_ParseResult _parseMarkdownLines({ + required int length, + required String Function(int index) lineAt, + int startIndex = 0, + List? existingBlocks, + OnBlockClosed? onBlockClosed, + OnCodeBlockOpen? onCodeBlockOpen, +}) { + final blocks = existingBlocks ?? []; + final paragraph = StringBuffer(); + var hasOpenCodeBlock = false; + + void maybeCommitParagraph() { + if (paragraph.isEmpty) return; + final text = paragraph.toString(); + paragraph.clear(); + blocks.add(MD$Paragraph( + text: text, + spans: _parseInlineSpans(text), + )); + } + + void pushBlock(MD$Block block) { + maybeCommitParagraph(); + blocks.add(block); + } + + var i = startIndex; + for (; i < length; i++) { + final line = lineAt(i); + + // --- Empty lines / Spacer --- + if (line.isEmpty || _emptyPattern.hasMatch(line)) { + var j = i + 1; + for (; j < length && _emptyPattern.hasMatch(lineAt(j)); j++) continue; + final count = j - i; + + if (j < length) { + onBlockClosed?.call(i, j); + } + + pushBlock(MD$Spacer(count: count)); + if (i + count == length) break; + i = j - 1; + continue; + } + + // --- Horizontal rule --- + if (line.startsWith('---')) { + if (i + 1 < length) { + onBlockClosed?.call(i, i + 1); + } + pushBlock(const MD$Divider()); + continue; + } + + // --- Heading --- + if (line.startsWith('#')) { + final level = + _headerPattern.firstMatch(line)?.group(0)?.length.clamp(1, 6) ?? 1; + final text = line.substring(level).trim(); + + if (i + 1 < length) { + onBlockClosed?.call(i, i + 1); + } + + pushBlock(MD$Heading( + level: level, text: text, spans: _parseInlineSpans(text), )); + continue; } - void pushBlock(MD$Block block) { - maybeCommitParagraph(); - blocks.addLast(block); + // --- Quote --- + if (line.startsWith('>')) { + final buffer = StringBuffer()..write(line.substring(1).trim()); + var j = i + 1; + for (; j < length && lineAt(j).startsWith('>'); j++) { + buffer + ..writeln() + ..write(lineAt(j).substring(1).trim()); + } + final text = buffer.toString(); + final count = j - i; + + if (j < length) { + onBlockClosed?.call(i, j); + } + + pushBlock(MD$Quote( + indent: 1, + text: text, + spans: _parseInlineSpans(text), + )); + + if (i + count == length) break; + i = j - 1; + continue; } - for (var i = 0; i < length; i++) { - // Trim trailing whitespace for consistent parsing - final line = lines[i]; - - // Here you would implement the logic to parse the line - // and create the appropriate MD$Block instances. - // This is a placeholder for demonstration purposes. - if (line.isEmpty || _emptyPattern.hasMatch(line)) { - /// Parse empty lines and combine them into a spacing block. - var j = i + 1; - for (; j < length && _emptyPattern.hasMatch(lines[j]); j++) continue; - final count = j - i; - pushBlock(MD$Spacer(count: count)); - if (i + count == length) break; // Last line is empty - i = j - 1; // Skip the empty lines - continue; - } else if (line.startsWith('---')) { - // Parse horizontal rules - pushBlock(const MD$Divider()); - continue; - } else if (line.startsWith('#')) { - // Parse headings - final level = - _headerPattern.firstMatch(line)?.group(0)?.length.clamp(1, 6) ?? 1; - final text = line.substring(level).trim(); - pushBlock(MD$Heading( - level: level, text: text, spans: _parseInlineSpans(text))); - continue; - } else if (line.startsWith('>')) { - // Parse quotes - final buffer = StringBuffer()..write(line.substring(1).trim()); - var j = i + 1; - for (; j < length && lines[j].startsWith('>'); j++) { - buffer - ..writeln() - ..write(lines[j].substring(1).trim()); + // --- Code block --- + if (line.startsWith('```')) { + final language = line.length > 3 ? line.substring(3).trim() : ''; + var j = i + 1; + for (; j < length && !lineAt(j).startsWith('```'); j++) continue; + + final foundClosing = j < length; + + if (foundClosing) { + onBlockClosed?.call(i, j + 1); + final codeLines = []; + for (var k = i + 1; k < j; k++) { + codeLines.add(lineAt(k)); } - final text = buffer.toString(); - final count = j - i; - // TODO(plugfox): Implement indentation for quotes - // Mike Matiunin , 16 June 2025 - pushBlock(MD$Quote( - indent: 1, // Indentation level for quotes - text: text, - spans: _parseInlineSpans(text), - )); - if (i + count == length) break; // Last line is quote - i = j - 1; // Skip the empty lines - continue; - } else if (line.startsWith('```')) { - // Parse code blocks - final language = line.length > 3 ? line.substring(3).trim() : ''; - var j = i + 1; - for (; j < length && !lines[j].startsWith('```'); j++) continue; - final codeText = lines.sublist(i + 1, j).join('\n'); - pushBlock(MD$Code( - text: codeText, - language: language, - )); - if (j == length - 1) break; // Last line is a code block - i = j; // Skip to the end of the code block - continue; - } else if (_listPattern.firstMatch(line) case RegExpMatch match - when match.namedGroup('indent')?.isEmpty == true) { - final marker = match.namedGroup('marker') ?? '*'; - final list = <({int intent, String marker, String text})>[ - ( - intent: 0, - marker: marker, - text: match.namedGroup('text')?.trim() ?? '', - ) - ]; - var j = i + 1; - for (; j < length; j++) { - final line = lines[j]; - final match = _listPattern.firstMatch(line); - final indent = match?.namedGroup('indent')?.length; - if (indent == null) break; - list.add(( - intent: indent, - marker: match?.namedGroup('marker') ?? '*', - text: match?.namedGroup('text')?.trim() ?? '', - )); + pushBlock(MD$Code(text: codeLines.join('\n'), language: language)); + + if (j == length - 1) break; + i = j; + } else { + // Code block not closed + hasOpenCodeBlock = true; + if (onCodeBlockOpen != null) { + final shouldContinue = onCodeBlockOpen(i, language); + if (!shouldContinue) { + // Add partial code block and break + final codeLines = []; + for (var k = i + 1; k < length; k++) { + codeLines.add(lineAt(k)); + } + pushBlock(MD$Code(text: codeLines.join('\n'), language: language)); + i = length; + break; + } + } else { + // Default behavior: treat as complete (for non-streaming) + final codeLines = []; + for (var k = i + 1; k < length; k++) { + codeLines.add(lineAt(k)); + } + pushBlock(MD$Code(text: codeLines.join('\n'), language: language)); + i = length - 1; } - // Convert to tree structure of [MD$ListItem]s - var offset = 0; - List traverse({int indent = 0}) { - final items = []; - for (; offset < list.length; offset++) { - final item = list[offset]; - if (item.intent == indent) { - // If the current item's indent matches, - // we create a new list item at this level. + } + continue; + } + + // --- List --- + if (_listPattern.firstMatch(line) case RegExpMatch match + when match.namedGroup('indent')?.isEmpty == true) { + final marker = match.namedGroup('marker') ?? '*'; + final list = <({int intent, String marker, String text})>[ + ( + intent: 0, + marker: marker, + text: match.namedGroup('text')?.trim() ?? '', + ) + ]; + + var j = i + 1; + for (; j < length; j++) { + final listLine = lineAt(j); + final listMatch = _listPattern.firstMatch(listLine); + final indent = listMatch?.namedGroup('indent')?.length; + if (indent == null) break; + list.add(( + intent: indent, + marker: listMatch?.namedGroup('marker') ?? '*', + text: listMatch?.namedGroup('text')?.trim() ?? '', + )); + } + + // Convert to tree structure + var offset = 0; + List traverse({int indent = 0}) { + final items = []; + for (; offset < list.length; offset++) { + final item = list[offset]; + if (item.intent == indent) { + items.add(MD$ListItem( + text: item.text, + marker: item.marker, + spans: _parseInlineSpans(item.text), + indent: item.intent, + )); + } else if (item.intent > indent) { + final children = traverse(indent: item.intent); + if (items.isNotEmpty) { + items.last = items.last.copyWith( + children: List.unmodifiable(children)); + } else { items.add(MD$ListItem( + marker: item.marker, text: item.text, - marker: item.marker, // '•', spans: _parseInlineSpans(item.text), indent: item.intent, + children: children, )); - } else if (item.intent > indent) { - // If the current item's indent is greater, - // we continue traversing deeper into the list. - final children = traverse(indent: item.intent); - if (items.isNotEmpty) { - // If we have a parent item, add children to it - items.last = items.last.copyWith( - children: List.unmodifiable(children)); - } else { - // If this is the first item, just add children - items.add(MD$ListItem( - marker: item.marker, // '•', - text: item.text, - spans: _parseInlineSpans(item.text), - indent: item.intent, - children: children, - )); - } - } else { - // If the indent is less, we stop traversing - offset--; // Step back to reprocess this item - break; } + } else { + offset--; + break; } - if (items.isEmpty) return const []; - return items; // Return the list of items at this level } + return items.isEmpty ? const [] : items; + } + + final count = j - i; + final listLines = []; + for (var k = i; k < j && k < length; k++) { + listLines.add(lineAt(k)); + } - // Create the list block with the items - final count = j - i; - final text = lines.sublist(i, j).join('\n'); - pushBlock(MD$List( + if (j < length) { + onBlockClosed?.call(i, j); + } + + pushBlock(MD$List(text: listLines.join('\n'), items: traverse())); + + if (i + count == length) break; + i = j - 1; + continue; + } + + // --- Table --- + if (line.startsWith('|')) { + MD$TableRow textToRow(String text) { + final cells = text.split('|'); + return MD$TableRow( text: text, - items: traverse(), - )); + cells: List>.unmodifiable(cells + .sublist(1, cells.length - 1) + .map((cell) => cell.trim()) + .map(_parseInlineSpans)), + ); + } - if (i + count == length) break; // Last line is a list item - i = j - 1; // Skip the list items - continue; - } else if (line.startsWith('|')) { - // Parse tables - MD$TableRow textToRow(String text) { - final cells = text.split('|'); - return MD$TableRow( - text: text, - cells: List>.unmodifiable(cells - .sublist(1, cells.length - 1) - .map((cell) => cell.trim()) - .map(_parseInlineSpans)), - ); + final header = textToRow(line); + final separator = length > i + 1 + ? RegExp(r'^\|[ -:]+[ -|:]*\|$').hasMatch(lineAt(i + 1)) + : false; + final rows = []; + var j = i + 2; + for (; j < length && lineAt(j).startsWith('|'); j++) { + rows.add(textToRow(lineAt(j))); + } + + final columns = header.cells.length; + if (columns > 0 && + separator && + rows.every((row) => row.cells.length == columns)) { + final tableLines = []; + for (var k = i; k < j && k < length; k++) { + tableLines.add(lineAt(k)); } - final header = textToRow(line); - final separator = lines.length > i + 1 - ? RegExp(r'^\|[ -:]+[ -|:]*\|$').hasMatch(lines[i + 1]) - : false; // Separator line for the table header - final rows = []; - var j = i + 2; // Skip the header and separator line - for (; j < length && lines[j].startsWith('|'); j++) - rows.add(textToRow(lines[j])); - // Validate - final columns = header.cells.length; - if (columns > 0 && - separator && - rows.every((row) => row.cells.length == columns)) { - // All rows have the same number of cells as the header - final text = lines.sublist(i, j).join('\n'); - pushBlock(MD$Table( - text: text, - header: header, - rows: List.unmodifiable(rows), - )); - } else { - // Table is malformed, treat it as a paragraph - if (paragraph.isNotEmpty) paragraph.writeln(); - paragraph.write(line); - continue; + if (j < length) { + onBlockClosed?.call(i, j); } - final count = j - i; - if (i + count == length) break; // Last line is a table row - i = j - 1; // Skip the table rows - continue; + pushBlock(MD$Table( + text: tableLines.join('\n'), + header: header, + rows: List.unmodifiable(rows), + )); } else { - // Parse paragraphs or other blocks + // Malformed table, treat as paragraph if (paragraph.isNotEmpty) paragraph.writeln(); paragraph.write(line); continue; } + + final count = j - i; + if (i + count == length) break; + i = j - 1; + continue; + } + + // --- Paragraph (default) --- + if (paragraph.isNotEmpty) paragraph.writeln(); + paragraph.write(line); + + // Check if paragraph can be closed + if (onBlockClosed != null && i + 1 < length) { + final nextLine = lineAt(i + 1); + if (nextLine.isEmpty || + _emptyPattern.hasMatch(nextLine) || + nextLine.startsWith('#') || + nextLine.startsWith('>') || + nextLine.startsWith('```') || + nextLine.startsWith('---') || + nextLine.startsWith('|') || + (_listPattern.firstMatch(nextLine)?.namedGroup('indent')?.isEmpty == + true)) { + onBlockClosed(i, i + 1); + maybeCommitParagraph(); + } } + } - // If there's any remaining text in the paragraph buffer, commit it - maybeCommitParagraph(); + maybeCommitParagraph(); + + return _ParseResult( + blocks: blocks, + nextIndex: i, + hasOpenCodeBlock: hasOpenCodeBlock, + ); +} + +// ============================================================================ +// MarkdownDecoder +// ============================================================================ + +/// {@template markdown_decoder} +/// A [Converter] that decodes Markdown formatted strings +/// into list of [MD$Block] objects. +/// This class is designed to parse Markdown syntax +/// and convert it into a structured format. +/// {@endtemplate} +class MarkdownDecoder extends Converter { + /// Creates a new instance of [MarkdownDecoder]. + /// {@macro markdown_decoder} + const MarkdownDecoder(); + + @override + Markdown convert(String input) { + final lines = LineSplitter.split(input).toList(growable: false); + if (lines.isEmpty) return const Markdown.empty(); + + final result = _parseMarkdownLines( + length: lines.length, + lineAt: (i) => lines[i], + ); return Markdown( markdown: input, - blocks: List.unmodifiable(blocks), + blocks: List.unmodifiable(result.blocks), ); } } @@ -589,3 +753,200 @@ List _parseInlineSpans(String text) { // For now, it returns an empty list as a placeholder. return spans; } + +// ============================================================================ +// Streaming Markdown Decoder +// ============================================================================ + +/// {@template streaming_markdown_decoder} +/// A streaming Markdown decoder optimized for LLM output scenarios. +/// +/// Unlike [MarkdownDecoder] which parses the entire input on each call, +/// this decoder maintains state across multiple [append] calls and only +/// re-parses lines that are still open (not yet closed). +/// +/// Uses the same core parsing logic as [MarkdownDecoder] via +/// [_parseMarkdownLines], but with line state tracking. +/// +/// Usage: +/// ```dart +/// final decoder = StreamingMarkdownDecoder(); +/// decoder.append('# Hello'); +/// decoder.append(' World\n\nSome text'); +/// final markdown = decoder.build(); +/// ``` +/// {@endtemplate} +class StreamingMarkdownDecoder { + /// Creates a new streaming Markdown decoder. + /// {@macro streaming_markdown_decoder} + StreamingMarkdownDecoder(); + + /// Parsed lines with their states. + /// The last line is always the "pending line" that may receive more content. + final List _lines = []; + + /// Accumulated blocks from parsing. + final List _blocks = []; + + /// Index of the first open line, used to skip closed lines during parsing. + int _firstOpenIndex = 0; + + /// Returns the current list of parsed blocks. + List get blocks => List.unmodifiable(_blocks); + + /// Returns the current number of lines. + int get lineCount => _lines.length; + + /// Returns the index of the first open line. + int get firstOpenIndex => _firstOpenIndex; + + /// Appends a chunk of text and triggers incremental parsing. + /// + /// The chunk may contain partial lines (no newline at the end), + /// which will be buffered until a newline is received. + Markdown append(String chunk) { + if (chunk.isEmpty) { + return Markdown( + markdown: _lines.map((l) => l.text).join('\n'), + blocks: List.unmodifiable(_blocks), + ); + } + + _processChunk(chunk); + _parse(); + + return Markdown( + markdown: _lines.map((l) => l.text).join('\n'), + blocks: List.unmodifiable(_blocks), + ); + } + + /// Processes a chunk of text, splitting into lines. + void _processChunk(String chunk) { + // Ensure we have a pending line to append to + if (_lines.isEmpty || _lines.last.state == LineState.closed) { + _lines.add(ParsedLine('', state: LineState.open)); + } + + // Append chunk to the pending line (last line) + final pendingIndex = _lines.length - 1; + final pending = _lines[pendingIndex]; + final combined = pending.text + chunk; + + // Use LineSplitter to split the combined text + final split = LineSplitter.split(combined).toList(growable: false); + + if (split.length > 1) { + // Multiple lines: we have newline characters + _lines[pendingIndex] = ParsedLine(split[0], state: LineState.open); + + for (var i = 1; i < split.length; i++) { + _lines.add(ParsedLine(split[i], state: LineState.open)); + } + + // If combined ends with newline, add empty pending line + if (combined.endsWith('\n') || combined.endsWith('\r')) { + _lines.add(ParsedLine('', state: LineState.open)); + } + } else { + _lines[pendingIndex] = ParsedLine(combined, state: LineState.open); + } + } + + /// Marks lines from [start] to [end] (exclusive) as closed. + void _closeLines(int start, int end) { + for (var k = start; k < end && k < _lines.length; k++) { + _lines[k].state = LineState.closed; + } + if (end <= _lines.length) { + _firstOpenIndex = end; + } + } + + /// Main parsing method using the shared [_parseMarkdownLines] function. + void _parse() { + // Remove blocks from _firstOpenIndex onwards (need to re-parse) + _truncateBlocks(); + + // Skip already closed lines + while (_firstOpenIndex < _lines.length && + _lines[_firstOpenIndex].state == LineState.closed) { + _firstOpenIndex++; + } + + final result = _parseMarkdownLines( + length: _lines.length, + lineAt: (i) => _lines[i].text, + startIndex: _firstOpenIndex, + existingBlocks: _blocks, + onBlockClosed: _closeLines, + onCodeBlockOpen: (startLine, language) { + // Don't continue parsing, break to wait for more input + return false; + }, + ); + + // Update _firstOpenIndex based on parse result + if (result.nextIndex > _firstOpenIndex && !result.hasOpenCodeBlock) { + // If we parsed some lines and no open code block, update index + for (var i = _firstOpenIndex; + i < result.nextIndex && i < _lines.length; + i++) { + if (_lines[i].state == LineState.closed) { + _firstOpenIndex = i + 1; + } + } + } + } + + /// Truncates blocks that need to be re-parsed. + void _truncateBlocks() { + // Count closed blocks by estimating line consumption + var count = 0; + var lineIdx = 0; + + for (final block in _blocks) { + if (lineIdx >= _firstOpenIndex) break; + final linesInBlock = _estimateBlockLines(block); + if (lineIdx + linesInBlock <= _firstOpenIndex) { + count++; + lineIdx += linesInBlock; + } else { + break; + } + } + + if (count < _blocks.length) { + _blocks.removeRange(count, _blocks.length); + } + } + + /// Estimates how many lines a block consumes. + int _estimateBlockLines(MD$Block block) { + return block.map( + paragraph: (p) => p.text.split('\n').length, + heading: (_) => 1, + quote: (q) => q.text.split('\n').length, + code: (c) => c.text.split('\n').length + 2, + list: (l) => l.text.split('\n').length, + divider: (_) => 1, + table: (t) => t.text.split('\n').length, + spacer: (s) => s.count, + ); + } + + /// Resets the decoder to its initial state. + void reset() { + _lines.clear(); + _blocks.clear(); + _firstOpenIndex = 0; + } + + /// Builds the final Markdown result. + Markdown build() { + return Markdown( + markdown: _lines.map((l) => l.text).join('\n'), + blocks: List.unmodifiable(_blocks), + ); + } +}