From 3ee604a1e6325a979274f81c7917c922a37788be Mon Sep 17 00:00:00 2001 From: Adsq Date: Sun, 7 Sep 2025 16:58:40 +0200 Subject: [PATCH 1/2] Improve Node Markdown parser finalize and console demo --- mark_to_html/parser.js | 284 +++++++++++++++++++++++++++++++++ mark_to_html/parser_console.js | 262 ++++++++++++++++++++++++++++++ 2 files changed, 546 insertions(+) create mode 100644 mark_to_html/parser.js create mode 100644 mark_to_html/parser_console.js diff --git a/mark_to_html/parser.js b/mark_to_html/parser.js new file mode 100644 index 0000000..e152e1d --- /dev/null +++ b/mark_to_html/parser.js @@ -0,0 +1,284 @@ +// Node version of streaming Markdown parser with finalize +const TAG = {"**":"strong","__":"strong","*":"em","_":"em","~~":"del"}; +const openTag = t => `<${t}>`; +const closeTag = t => ``; +function esc(s){ + return s.replace(/[&<>"']/g, ch => ({'&':'&','<':'<','>':'>','"':'"','\'':'''}[ch])); +} + +// tokenization +function* scanInline(src){ + let i=0,N=src.length; + while(i=0;i--){ + if(state.inlineStack[i].sig===sig){ + const above=state.inlineStack.splice(i+1); + let out=above.map(t=>closeTag(t.kind)).reverse().join(''); + const tok=state.inlineStack.pop(); out+=closeTag(tok.kind); + out+=above.map(t=>openTag(t.kind)).join(''); + return out; + } + } + return esc(sig); +} + +function handleInline(line,state){ + let out=""; + const tokens=[...scanInline(line)]; + + // A) settle single delimiter from previous chunk + if(state.boundaryHalf){ + const firstTok=tokens[0]; + const firstText=firstTok?.type==='text'?firstTok.value:(firstTok?.type==='delim'?firstTok.value:''); + const firstChar=firstText?firstText[0]:''; + const pair=state.boundaryHalf+firstChar; + if(pair==='**'||pair==='__'||pair==='~~'){ + const top=state.inlineStack[state.inlineStack.length-1]; + out+=(top&&top.sig===pair)?closeInlineDelim(state,pair):openInlineDelim(state,pair); + if(tokens.length && tokens[0].type==='text') tokens[0].value=tokens[0].value.slice(1); + } else { + out+=esc(state.boundaryHalf); + } + state.boundaryHalf=""; + } + + // B) settle pair delimiter from previous chunk + if(state.boundaryPair){ + const pair=state.boundaryPair; state.boundaryPair=""; + const top=state.inlineStack[state.inlineStack.length-1]; + out+=(top&&top.sig===pair)?closeInlineDelim(state,pair):openInlineDelim(state,pair); + } + + for(let i=0;i'; } return ''; } +function closeParagraphIfOpen(state){ if(state.block.inParagraph){ state.block.inParagraph=false; return '

'; } return ''; } + +function unwindListsToIndent(state,indent){ + let out=''; const stk=state.block.listStack; + while(stk.length){ + const t=stk[stk.length-1]; + if(t.type==='li' && t.indent>=indent){ stk.pop(); out+=''; continue; } + if((t.type==='ul'||t.type==='ol') && t.indent>indent){ stk.pop(); out+=``; continue; } + break; + } + return out; +} +function ensureListContainer(state,indent,listType){ + let html=''; html+=unwindListsToIndent(state,indent); + const stk=state.block.listStack; const top=stk[stk.length-1]; + if(top && (top.type==='ul'||top.type==='ol') && top.indent===indent && top.type!==listType){ stk.pop(); html+=``; } + const cur=stk[stk.length-1]; if(!cur||cur.type!==listType||cur.indent!==indent){ stk.push({type:listType,indent}); html+=`<${listType}>`; } + return html; +} +function handleListLine(line,state){ + const m=/^(\s*)([\*\+\-]|\d+\.)\s+(.*)$/.exec(line); if(!m) return null; + const indent=m[1].length, marker=m[2], rest=m[3]; const listType=/\d+\./.test(marker)?'ol':'ul'; + let html=''; + html+=closeParagraphIfOpen(state); + html+=ensureListContainer(state,indent,listType); + const stk=state.block.listStack; const top=stk[stk.length-1]; if(top && top.type==='li' && top.indent===indent){ stk.pop(); html+=''; } + stk.push({type:'li', parent:listType, indent}); html+='
  • '+handleInline(rest,state); + return html; +} +function closeAllLists(state){ let out=''; const stk=state.block.listStack; while(stk.length){ const t=stk.pop(); out+=(t.type==='li')?'
  • ':``; } return out; } + +// tables with safe splitting +function splitTableLineSafe(line){ + const cells=[]; let cur=""; let openRun=0; + for(let i=0;i!c || /^:?-{3,}:?$/.test(c)); } +function parseAligns(sepLine){ const parts=splitTableLineSafe(sepLine); return parts.map(s=>{ s=s.trim(); if(!s) return null; const L=s.startsWith(':'), R=s.endsWith(':'); if(L&&R) return 'center'; if(R) return 'right'; if(L) return 'left'; return null; }); } +function handleTableStart(lines,idx,state){ + const headerLine=lines[idx], sepLine=lines[idx+1]; + if(!headerLine?.includes('|') || !sepLine || !isTableSep(sepLine)) return null; + const header=splitTableLineSafe(headerLine); const aligns=parseAligns(sepLine); + let html=closeParagraphIfOpen(state); + html+=''+header.map((c,i)=>{ + const savedStack=state.inlineStack.slice(); + const savedTicks=state.pendingTicks; const savedHalf=state.boundaryHalf; const savedPair=state.boundaryPair; + const cellHTML=handleInline(c,state); + state.inlineStack=savedStack; state.pendingTicks=savedTicks; state.boundaryHalf=savedHalf; state.boundaryPair=savedPair; + return `${cellHTML}`; + }).join('')+''; + state.block.table={active:true, aligns}; + return {html, advance:2}; +} + +function fenceOpen(line){ const m=/^(\s*)(`{3,}|~{3,})(\s*\w+)?\s*$/.exec(line); if(!m) return null; const ticks=m[2][0]; const len=m[2].length; const lang=(m[3]||'').trim(); return {fence:ticks.repeat(len), ticks:len, lang:lang||''}; } + +function processLines(lines,state){ + let html=""; + for(let i=0;i`; continue; } + if(!state.block.table){ const t=handleTableStart(lines,i,state); if(t){ html+=t.html; i+=t.advance; continue; } } + else { + if(line.trim()==='' || !line.includes('|')){ html+='
    '; state.block.table=null; } + else { + const cells=splitTableLineSafe(line); const aligns=state.block.table.aligns; + html+=''+cells.map((c,ci)=>{ + const savedStack=state.inlineStack.slice(); const savedTicks=state.pendingTicks; const savedHalf=state.boundaryHalf; const savedPair=state.boundaryPair; + const cellHTML=handleInline(c,state); + state.inlineStack=savedStack; state.pendingTicks=savedTicks; state.boundaryHalf=savedHalf; state.boundaryPair=savedPair; + return `${cellHTML}`; + }).join('')+''; + continue; + } + } + const li=handleListLine(line,state); + if(li){ html+=li; continue; } + else if(line.trim()==='' && state.block.listStack.length){ html+=closeAllLists(state); continue; } + if(/^\s*(\*{3,}|-{3,}|_{3,})\s*$/.test(line)){ html+=closeParagraphIfOpen(state)+'
    '; continue; } + const qm=/^>\s?(.*)$/.exec(line); if(qm){ html+=closeParagraphIfOpen(state)+'
    '+handleInline(qm[1],state)+'
    '; continue; } + const hm=/^(#{1,6})\s+(.*)$/.exec(line); if(hm){ html+=closeAllLists(state)+closeParagraphIfOpen(state)+`${handleInline(hm[2],state)}`; continue; } + if(line.trim()===''){ html+=closeParagraphIfOpen(state); continue; } + html+=openParagraphIfNeeded(state)+handleInline(line,state); + } + return html; +} + +function processChunk(input,state){ state.lineBuffer+=input; const parts=state.lineBuffer.split('\n'); state.lineBuffer=parts.pop(); return processLines(parts,state); } + +function finalize(state){ + let html=''; + if(state.lineBuffer!==''){ html+=processLines([state.lineBuffer],state); state.lineBuffer=''; } + html+=handleInline('',state); + if(state.block.table){ html+=''; state.block.table=null; } + html+=closeAllLists(state); + html+=closeParagraphIfOpen(state); + if(state.block.fenced){ html+=''; state.block.fenced=null; } + return html; +} + +// demo chunks +const chunks=[ + {role:'model', content:'**witam'}, + {role:'model', content:' i *tu'}, + {role:'model', content:' jest** zamknięte*'}, + {role:'model', content:' zakończenie *'}, + {role:'model', content:'*pogrubienia'}, + {role:'model', content:' `ala `` nie zamyka'}, + {role:'model', content:'`` ala`` i koniec`'}, + {role:'model', content:'\n- UL A'}, + {role:'model', content:'\n - UL A.1'}, + {role:'model', content:'\n 1. OL A.2 (switch)'}, + {role:'model', content:'\n- UL B (powrót)'}, + {role:'model', content:'\nKol1 | Kol2 | Kol3'}, + {role:'model', content:'\n:--- | :--: | ---:'}, + {role:'model', content:'\n**A1** | _B1_ | C1'}, + {role:'model', content:'\nA2 | B2 z `code` | ~~C2~~'}, + {role:'model', content:'\nKolA | KolB'}, + {role:'model', content:'\n--- | ---'}, + {role:'model', content:'\n`x|y` | normal'}, + {role:'model', content:'\n'}, + {role:'model', content:'Tylko | piony | bez separatora'}, + {role:'model', content:'\nZwykły tekst po nim.'}, + {role:'model', content:'*witam**co następny tam * słychać**'}, + {role:'model', content:'\n# Nagłówek końcowy'} +]; + +let cumulativeHTML=""; let idx=0; +for(const chunk of chunks){ + const html=processChunk(chunk.content,S); + cumulativeHTML+=html; + console.log(`chunk ${++idx}:`); + console.log(cumulativeHTML); +} +cumulativeHTML+=finalize(S); + +const voidTags=new Set(['area','base','br','col','embed','hr','img','input','link','meta','param','source','track','wbr']); +function checkHtmlBalance(html){ + const stack=[]; const tagRegex=/<\/?([a-zA-Z0-9]+)(\s[^>]*)?>/g; let match; + while((match=tagRegex.exec(html))){ + const tag=match[1].toLowerCase(); + const isClosing=match[0][1]=='/'; + if(voidTags.has(tag)) continue; + if(isClosing){ if(stack.pop()!==tag) return false; } + else { stack.push(tag); } + } + return stack.length===0; +} +const balanced=checkHtmlBalance(cumulativeHTML); +console.log('Final HTML:'); +console.log(cumulativeHTML); +console.log('Balanced tags:', balanced); diff --git a/mark_to_html/parser_console.js b/mark_to_html/parser_console.js new file mode 100644 index 0000000..c609270 --- /dev/null +++ b/mark_to_html/parser_console.js @@ -0,0 +1,262 @@ +// Node version of streaming Markdown parser with finalize +const TAG = {"**":"strong","__":"strong","*":"em","_":"em","~~":"del"}; +const openTag = t => `<${t}>`; +const closeTag = t => ``; +function esc(s){ + return s.replace(/[&<>"']/g, ch => ({'&':'&','<':'<','>':'>','"':'"','\'':'''}[ch])); +} + +// tokenization +function* scanInline(src){ + let i=0,N=src.length; + while(i=0;i--){ + if(state.inlineStack[i].sig===sig){ + const above=state.inlineStack.splice(i+1); + let out=above.map(t=>closeTag(t.kind)).reverse().join(''); + const tok=state.inlineStack.pop(); out+=closeTag(tok.kind); + out+=above.map(t=>openTag(t.kind)).join(''); + return out; + } + } + return esc(sig); +} + +function handleInline(line,state){ + let out=""; + const tokens=[...scanInline(line)]; + + // A) settle single delimiter from previous chunk + if(state.boundaryHalf){ + const firstTok=tokens[0]; + const firstText=firstTok?.type==='text'?firstTok.value:(firstTok?.type==='delim'?firstTok.value:''); + const firstChar=firstText?firstText[0]:''; + const pair=state.boundaryHalf+firstChar; + if(pair==='**'||pair==='__'||pair==='~~'){ + const top=state.inlineStack[state.inlineStack.length-1]; + out+=(top&&top.sig===pair)?closeInlineDelim(state,pair):openInlineDelim(state,pair); + if(tokens.length && tokens[0].type==='text') tokens[0].value=tokens[0].value.slice(1); + } else { + out+=esc(state.boundaryHalf); + } + state.boundaryHalf=""; + } + + // B) settle pair delimiter from previous chunk + if(state.boundaryPair){ + const pair=state.boundaryPair; state.boundaryPair=""; + const top=state.inlineStack[state.inlineStack.length-1]; + out+=(top&&top.sig===pair)?closeInlineDelim(state,pair):openInlineDelim(state,pair); + } + + for(let i=0;i'; } return ''; } +function closeParagraphIfOpen(state){ if(state.block.inParagraph){ state.block.inParagraph=false; return '

    '; } return ''; } + +function unwindListsToIndent(state,indent){ + let out=''; const stk=state.block.listStack; + while(stk.length){ + const t=stk[stk.length-1]; + if(t.type==='li' && t.indent>=indent){ stk.pop(); out+=''; continue; } + if((t.type==='ul'||t.type==='ol') && t.indent>indent){ stk.pop(); out+=``; continue; } + break; + } + return out; +} +function ensureListContainer(state,indent,listType){ + let html=''; html+=unwindListsToIndent(state,indent); + const stk=state.block.listStack; const top=stk[stk.length-1]; + if(top && (top.type==='ul'||top.type==='ol') && top.indent===indent && top.type!==listType){ stk.pop(); html+=``; } + const cur=stk[stk.length-1]; if(!cur||cur.type!==listType||cur.indent!==indent){ stk.push({type:listType,indent}); html+=`<${listType}>`; } + return html; +} +function handleListLine(line,state){ + const m=/^(\s*)([\*\+\-]|\d+\.)\s+(.*)$/.exec(line); if(!m) return null; + const indent=m[1].length, marker=m[2], rest=m[3]; const listType=/\d+\./.test(marker)?'ol':'ul'; + let html=''; + html+=closeParagraphIfOpen(state); + html+=ensureListContainer(state,indent,listType); + const stk=state.block.listStack; const top=stk[stk.length-1]; if(top && top.type==='li' && top.indent===indent){ stk.pop(); html+=''; } + stk.push({type:'li', parent:listType, indent}); html+='
  • '+handleInline(rest,state); + return html; +} +function closeAllLists(state){ let out=''; const stk=state.block.listStack; while(stk.length){ const t=stk.pop(); out+=(t.type==='li')?'
  • ':``; } return out; } + +// tables with safe splitting +function splitTableLineSafe(line){ + const cells=[]; let cur=""; let openRun=0; + for(let i=0;i!c || /^:?-{3,}:?$/.test(c)); } +function parseAligns(sepLine){ const parts=splitTableLineSafe(sepLine); return parts.map(s=>{ s=s.trim(); if(!s) return null; const L=s.startsWith(':'), R=s.endsWith(':'); if(L&&R) return 'center'; if(R) return 'right'; if(L) return 'left'; return null; }); } +function handleTableStart(lines,idx,state){ + const headerLine=lines[idx], sepLine=lines[idx+1]; + if(!headerLine?.includes('|') || !sepLine || !isTableSep(sepLine)) return null; + const header=splitTableLineSafe(headerLine); const aligns=parseAligns(sepLine); + let html=closeParagraphIfOpen(state); + html+=''+header.map((c,i)=>{ + const savedStack=state.inlineStack.slice(); + const savedTicks=state.pendingTicks; const savedHalf=state.boundaryHalf; const savedPair=state.boundaryPair; + const cellHTML=handleInline(c,state); + state.inlineStack=savedStack; state.pendingTicks=savedTicks; state.boundaryHalf=savedHalf; state.boundaryPair=savedPair; + return `${cellHTML}`; + }).join('')+''; + state.block.table={active:true, aligns}; + return {html, advance:2}; +} + +function fenceOpen(line){ const m=/^(\s*)(`{3,}|~{3,})(\s*\w+)?\s*$/.exec(line); if(!m) return null; const ticks=m[2][0]; const len=m[2].length; const lang=(m[3]||'').trim(); return {fence:ticks.repeat(len), ticks:len, lang:lang||''}; } + +function processLines(lines,state){ + let html=""; + for(let i=0;i`; continue; } + if(!state.block.table){ const t=handleTableStart(lines,i,state); if(t){ html+=t.html; i+=t.advance; continue; } } + else { + if(line.trim()==='' || !line.includes('|')){ html+='
    '; state.block.table=null; } + else { + const cells=splitTableLineSafe(line); const aligns=state.block.table.aligns; + html+=''+cells.map((c,ci)=>{ + const savedStack=state.inlineStack.slice(); const savedTicks=state.pendingTicks; const savedHalf=state.boundaryHalf; const savedPair=state.boundaryPair; + const cellHTML=handleInline(c,state); + state.inlineStack=savedStack; state.pendingTicks=savedTicks; state.boundaryHalf=savedHalf; state.boundaryPair=savedPair; + return `${cellHTML}`; + }).join('')+''; + continue; + } + } + const li=handleListLine(line,state); + if(li){ html+=li; continue; } + else if(line.trim()==='' && state.block.listStack.length){ html+=closeAllLists(state); continue; } + if(/^\s*(\*{3,}|-{3,}|_{3,})\s*$/.test(line)){ html+=closeParagraphIfOpen(state)+'
    '; continue; } + const qm=/^>\s?(.*)$/.exec(line); if(qm){ html+=closeParagraphIfOpen(state)+'
    '+handleInline(qm[1],state)+'
    '; continue; } + const hm=/^(#{1,6})\s+(.*)$/.exec(line); if(hm){ html+=closeAllLists(state)+closeParagraphIfOpen(state)+`${handleInline(hm[2],state)}`; continue; } + if(line.trim()===''){ html+=closeParagraphIfOpen(state); continue; } + html+=openParagraphIfNeeded(state)+handleInline(line,state); + } + return html; +} + +function processChunk(input,state){ state.lineBuffer+=input; const parts=state.lineBuffer.split('\n'); state.lineBuffer=parts.pop(); return processLines(parts,state); } + +function finalize(state){ + let html=''; + if(state.lineBuffer!==''){ html+=processLines([state.lineBuffer],state); state.lineBuffer=''; } + html+=handleInline('',state); + if(state.block.table){ html+=''; state.block.table=null; } + html+=closeAllLists(state); + html+=closeParagraphIfOpen(state); + if(state.block.fenced){ html+=''; state.block.fenced=null; } + return html; +} + +// demo chunks +const chunks=[ + // simple demo: open in first chunk, close in second + {role:'model', content:'**witam'}, + {role:'model', content:' co tam**'} +]; + +let cumulativeHTML=""; let idx=0; +for(const chunk of chunks){ + const html=processChunk(chunk.content + '\n', S); + cumulativeHTML += html; + console.log(`chunk ${++idx}:`, cumulativeHTML); +} +cumulativeHTML += finalize(S); + +const voidTags=new Set(['area','base','br','col','embed','hr','img','input','link','meta','param','source','track','wbr']); +function checkHtmlBalance(html){ + const stack=[]; const tagRegex=/<\/?([a-zA-Z0-9]+)(\s[^>]*)?>/g; let match; + while((match=tagRegex.exec(html))){ + const tag=match[1].toLowerCase(); + const isClosing=match[0][1]=='/'; + if(voidTags.has(tag)) continue; + if(isClosing){ if(stack.pop()!==tag) return false; } + else { stack.push(tag); } + } + return stack.length===0; +} +const balanced=checkHtmlBalance(cumulativeHTML); +console.log('Final HTML:', cumulativeHTML); +console.log('Balanced tags:', balanced); From 91d2ebe1df43ab51d3697d3e9e8027b9dadbd081 Mon Sep 17 00:00:00 2001 From: Adsq Date: Sun, 7 Sep 2025 17:07:56 +0200 Subject: [PATCH 2/2] Show finalized HTML on last chunk --- mark_to_html/parser_console.js | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/mark_to_html/parser_console.js b/mark_to_html/parser_console.js index c609270..b0b3548 100644 --- a/mark_to_html/parser_console.js +++ b/mark_to_html/parser_console.js @@ -237,13 +237,16 @@ const chunks=[ {role:'model', content:' co tam**'} ]; -let cumulativeHTML=""; let idx=0; -for(const chunk of chunks){ - const html=processChunk(chunk.content + '\n', S); +let cumulativeHTML=""; +for (let i = 0; i < chunks.length; i++) { + const chunk = chunks[i]; + const html = processChunk(chunk.content + '\n', S); cumulativeHTML += html; - console.log(`chunk ${++idx}:`, cumulativeHTML); + if (i === chunks.length - 1) { + cumulativeHTML += finalize(S); + } + console.log(`chunk ${i + 1}:`, cumulativeHTML); } -cumulativeHTML += finalize(S); const voidTags=new Set(['area','base','br','col','embed','hr','img','input','link','meta','param','source','track','wbr']); function checkHtmlBalance(html){