From 2a6845d8849891608ff7e996d1768109d251b317 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89mile=20R=C3=A9?= Date: Fri, 3 Apr 2026 12:43:36 +0400 Subject: [PATCH] Reimplement ts converter using marked parser MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Émile Ré --- package-lock.json | 3 +- packages/prosemirror/package.json | 3 +- packages/prosemirror/src/markdown.ts | 664 +++++++++++++++++---------- 3 files changed, 419 insertions(+), 251 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7e83e5b5d..c75965758 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19709,7 +19709,8 @@ "version": "1.0.0", "license": "ISC", "dependencies": { - "@tiptap/pm": "^3.20.2" + "@tiptap/pm": "^3.20.2", + "marked": "^15.0.12" }, "devDependencies": { "typescript": "~5.8.3" diff --git a/packages/prosemirror/package.json b/packages/prosemirror/package.json index 6840edff4..8c1600a8f 100644 --- a/packages/prosemirror/package.json +++ b/packages/prosemirror/package.json @@ -10,7 +10,8 @@ "check": "tsc --noEmit" }, "dependencies": { - "@tiptap/pm": "^3.20.2" + "@tiptap/pm": "^3.20.2", + "marked": "^15.0.12" }, "devDependencies": { "typescript": "~5.8.3" diff --git a/packages/prosemirror/src/markdown.ts b/packages/prosemirror/src/markdown.ts index 9cd79fb3f..d400a0d98 100644 --- a/packages/prosemirror/src/markdown.ts +++ b/packages/prosemirror/src/markdown.ts @@ -2,287 +2,453 @@ // Use of this source code is governed by the ISC license // that can be found in the LICENSE file. -import type { Node as ProseMirrorNode, Schema } from "@tiptap/pm/model"; +import { lexer, type Token, type Tokens } from "marked"; +import type { + Mark as ProseMirrorMark, + Node as ProseMirrorNode, + Schema, +} from "@tiptap/pm/model"; -const codeBlockOpenPattern = /^```(\w*)$/; -const tableSeparatorPattern - = /^\|?\s*:?-{3,}:?\s*(\|\s*:?-{3,}:?\s*)*\|?\s*$/m; -const markdownLinePattern - = /(?:^```\w*$|^#{1,6}\s|^>\s|^[-+*]\s|^\d+\.\s|^(?:---|___|\*\*\*)\s*$)/m; +const tableSeparatorPattern = + /^\|?\s*:?-{3,}:?\s*(\|\s*:?-{3,}:?\s*)*\|?\s*$/m; +const markdownLinePattern = + /(?:^```\w*$|^#{1,6}\s|^>\s|^[-+*]\s|^\d+\.\s|^(?:---|___|\*\*\*)\s*$)/m; -function parseCells(line: string): string[] { - return line - .replace(/^\|/, "") - .replace(/\|\s*$/, "") - .split("|") - .map(cell => cell.trim()); +function unescapeHtml(html: string): string { + return html + .replace(/</g, "<") + .replace(/>/g, ">") + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/&/g, "&"); } -function applyMarks( - schema: Schema, - markNames: string[], - inner: string, -): ProseMirrorNode[] { - const marks = markNames.map(name => schema.marks[name].create()); - return parseInlineContent(schema, inner).map((node) => { - let combined = node.marks; - for (const mark of marks) { - combined = mark.addToSet(combined); +function safeLinkHref(href: string): string { + href = href.trim(); + if (!href) return "#"; + if (href[0] === "#") return href; + if (href.startsWith("/")) { + if (href.length > 1 && (href[1] === "/" || href[1] === "\\")) return "#"; + return href; + } + try { + const url = new URL(href); + const scheme = url.protocol.slice(0, -1).toLowerCase(); + switch (scheme) { + case "http": + case "https": + case "mailto": + case "tel": + return href; + default: + return "#"; } - return node.mark(combined); - }); + } catch { + return href; + } } -export function parseInlineContent( - schema: Schema, - text: string, -): ProseMirrorNode[] { - const nodes: ProseMirrorNode[] = []; - const pattern - = /`([^`]+)`|\*\*\*(.+?)\*\*\*|\*\*(.+?)\*\*|~~(.+?)~~|(.+?)<\/u>|\[([^\]]+)\]\(([^)]+)\)|\*(.+?)\*/g; - let lastIndex = 0; - let match; +type HtmlTagInfo = { + closing: boolean; + markName: string; + attrs?: Record; +}; - while ((match = pattern.exec(text)) !== null) { - if (match.index > lastIndex) { - nodes.push(schema.text(text.slice(lastIndex, match.index))); - } +const openTagPattern = /^<(u|b|i|s|em|strong|del|code|a)(\s[^>]*)?\s*\/?>$/i; +const closeTagPattern = /^<\/(u|b|i|s|em|strong|del|code|a)\s*>$/i; - if (match[1] !== undefined) { - nodes.push(schema.text(match[1], [schema.marks.code.create()])); - } else if (match[2] !== undefined) { - nodes.push(...applyMarks(schema, ["bold", "italic"], match[2])); - } else if (match[3] !== undefined) { - nodes.push(...applyMarks(schema, ["bold"], match[3])); - } else if (match[4] !== undefined) { - nodes.push(...applyMarks(schema, ["strike"], match[4])); - } else if (match[5] !== undefined) { - nodes.push(...applyMarks(schema, ["underline"], match[5])); - } else if (match[6] !== undefined) { - const linkMark = schema.marks.link.create({ href: match[7] }); - for (const node of parseInlineContent(schema, match[6])) { - nodes.push(node.mark(linkMark.addToSet(node.marks))); +function tagToMarkName(tag: string): string | null { + switch (tag.toLowerCase()) { + case "u": + return "underline"; + case "b": + case "strong": + return "bold"; + case "i": + case "em": + return "italic"; + case "s": + case "del": + return "strike"; + case "code": + return "code"; + case "a": + return "link"; + default: + return null; + } +} + +function parseHtmlTag(html: string): HtmlTagInfo | null { + const closeMatch = html.match(closeTagPattern); + if (closeMatch) { + const markName = tagToMarkName(closeMatch[1]); + if (markName) return { closing: true, markName }; + return null; + } + + const openMatch = html.match(openTagPattern); + if (openMatch) { + const markName = tagToMarkName(openMatch[1]); + if (!markName) return null; + + const info: HtmlTagInfo = { closing: false, markName }; + + if (openMatch[1].toLowerCase() === "a" && openMatch[2]) { + const hrefMatch = openMatch[2].match(/href=["']([^"']*)["']/i); + if (hrefMatch) { + info.attrs = { href: safeLinkHref(hrefMatch[1]) }; } - } else if (match[8] !== undefined) { - nodes.push(...applyMarks(schema, ["italic"], match[8])); } - lastIndex = match.index + match[0].length; + return info; } - if (lastIndex < text.length) { - nodes.push(schema.text(text.slice(lastIndex))); - } - - return nodes; + return null; } -function buildTable( - schema: Schema, - headerLine: string, - dataLines: string[], -): ProseMirrorNode { - const headers = parseCells(headerLine); - const columnCount = headers.length; +class Converter { + private schema: Schema; + private marks: ProseMirrorMark[]; - const headerCells = headers.map((h) => { - const content = h ? parseInlineContent(schema, h) : []; - return schema.nodes.tableHeader.create( + constructor(schema: Schema) { + this.schema = schema; + this.marks = []; + } + + private withMark( + mark: ProseMirrorMark, + fn: () => ProseMirrorNode[], + ): ProseMirrorNode[] { + this.marks.push(mark); + const result = fn(); + this.marks.pop(); + return result; + } + + private currentMarks(): readonly ProseMirrorMark[] { + let result: readonly ProseMirrorMark[] = []; + for (const mark of this.marks) { + result = mark.addToSet(result as ProseMirrorMark[]); + } + return result; + } + + private makeTextNode(text: string): ProseMirrorNode[] { + const unescaped = unescapeHtml(text); + if (!unescaped) return []; + const marks = this.currentMarks(); + return [this.schema.text(unescaped, marks.length > 0 ? marks : undefined)]; + } + + convertBlockTokens(tokens: Token[]): ProseMirrorNode[] { + const nodes: ProseMirrorNode[] = []; + for (const token of tokens) { + nodes.push(...this.convertBlockToken(token)); + } + return nodes; + } + + private convertBlockToken(token: Token): ProseMirrorNode[] { + switch (token.type) { + case "heading": + return this.convertHeading(token as Tokens.Heading); + case "paragraph": + return this.convertParagraph(token as Tokens.Paragraph); + case "blockquote": + return this.convertBlockquote(token as Tokens.Blockquote); + case "code": + return this.convertCodeBlock(token as Tokens.Code); + case "list": + return this.convertList(token as Tokens.List); + case "hr": + return [this.schema.nodes.horizontalRule.create()]; + case "table": + return this.convertTable(token as Tokens.Table); + case "text": + return this.convertBlockText(token as Tokens.Text); + case "html": + case "space": + case "def": + return []; + default: + return []; + } + } + + private convertHeading(token: Tokens.Heading): ProseMirrorNode[] { + const content = this.convertInlineTokens(token.tokens); + return [ + this.schema.nodes.heading.create( + { level: token.depth }, + content.length > 0 ? content : undefined, + ), + ]; + } + + private convertParagraph(token: Tokens.Paragraph): ProseMirrorNode[] { + const content = this.convertInlineTokens(token.tokens); + return [ + this.schema.nodes.paragraph.create( + null, + content.length > 0 ? content : undefined, + ), + ]; + } + + private convertBlockquote(token: Tokens.Blockquote): ProseMirrorNode[] { + const children = this.convertBlockTokens(token.tokens); + return [ + this.schema.nodes.blockquote.create( + null, + children.length > 0 ? children : undefined, + ), + ]; + } + + private convertCodeBlock(token: Tokens.Code): ProseMirrorNode[] { + const language = token.lang || null; + return [ + this.schema.nodes.codeBlock.create( + { language }, + token.text ? this.schema.text(token.text) : undefined, + ), + ]; + } + + private convertList(token: Tokens.List): ProseMirrorNode[] { + const children = token.items.flatMap(item => this.convertListItem(item)); + + if (token.ordered) { + const start = typeof token.start === "number" ? token.start : 1; + return [ + this.schema.nodes.orderedList.create( + { start }, + children.length > 0 ? children : undefined, + ), + ]; + } + + return [ + this.schema.nodes.bulletList.create( + null, + children.length > 0 ? children : undefined, + ), + ]; + } + + private convertListItem(token: Tokens.ListItem): ProseMirrorNode[] { + const children: ProseMirrorNode[] = []; + + for (const child of token.tokens) { + if (child.type === "text") { + const textToken = child as Tokens.Text; + const inlineContent = textToken.tokens + ? this.convertInlineTokens(textToken.tokens) + : this.makeTextNode(textToken.text); + children.push( + this.schema.nodes.paragraph.create( + null, + inlineContent.length > 0 ? inlineContent : undefined, + ), + ); + } else { + children.push(...this.convertBlockToken(child)); + } + } + + if (children.length === 0) { + children.push(this.schema.nodes.paragraph.create()); + } + + return [this.schema.nodes.listItem.create(null, children)]; + } + + private convertBlockText(token: Tokens.Text): ProseMirrorNode[] { + const content = token.tokens + ? this.convertInlineTokens(token.tokens) + : this.makeTextNode(token.text); + return [ + this.schema.nodes.paragraph.create( + null, + content.length > 0 ? content : undefined, + ), + ]; + } + + private convertTable(token: Tokens.Table): ProseMirrorNode[] { + const rows: ProseMirrorNode[] = []; + + const headerCells = token.header.map(cell => + this.convertTableCell(cell, true), + ); + rows.push(this.schema.nodes.tableRow.create(null, headerCells)); + + for (const row of token.rows) { + const dataCells = row.map(cell => this.convertTableCell(cell, false)); + rows.push(this.schema.nodes.tableRow.create(null, dataCells)); + } + + return [this.schema.nodes.table.create(null, rows)]; + } + + private convertTableCell( + cell: Tokens.TableCell, + isHeader: boolean, + ): ProseMirrorNode { + const content = this.convertInlineTokens(cell.tokens); + const nodeType = isHeader + ? this.schema.nodes.tableHeader + : this.schema.nodes.tableCell; + return nodeType.create( null, - schema.nodes.paragraph.create( + this.schema.nodes.paragraph.create( null, content.length > 0 ? content : undefined, ), ); - }); - - const tableRows: ProseMirrorNode[] = [ - schema.nodes.tableRow.create(null, headerCells), - ]; - - for (const line of dataLines) { - const cells = parseCells(line); - const rowCells: ProseMirrorNode[] = []; - for (let c = 0; c < columnCount; c++) { - const value = cells[c]?.trim() ?? ""; - const content = value ? parseInlineContent(schema, value) : []; - rowCells.push( - schema.nodes.tableCell.create( - null, - schema.nodes.paragraph.create( - null, - content.length > 0 ? content : undefined, - ), - ), - ); - } - tableRows.push(schema.nodes.tableRow.create(null, rowCells)); } - return schema.nodes.table.create(null, tableRows); + convertInlineTokens(tokens: Token[]): ProseMirrorNode[] { + const nodes: ProseMirrorNode[] = []; + + for (const token of tokens) { + switch (token.type) { + case "text": { + const textToken = token as Tokens.Text; + if (textToken.tokens && textToken.tokens.length > 0) { + nodes.push(...this.convertInlineTokens(textToken.tokens)); + } else { + nodes.push(...this.makeTextNode(textToken.text)); + } + break; + } + + case "strong": + nodes.push( + ...this.withMark(this.schema.marks.bold.create(), () => + this.convertInlineTokens((token as Tokens.Strong).tokens), + ), + ); + break; + + case "em": + nodes.push( + ...this.withMark(this.schema.marks.italic.create(), () => + this.convertInlineTokens((token as Tokens.Em).tokens), + ), + ); + break; + + case "del": + nodes.push( + ...this.withMark(this.schema.marks.strike.create(), () => + this.convertInlineTokens((token as Tokens.Del).tokens), + ), + ); + break; + + case "codespan": { + const codeText = unescapeHtml((token as Tokens.Codespan).text); + if (codeText) { + const codeMark = this.schema.marks.code.create(); + const marks = codeMark.addToSet( + this.currentMarks() as ProseMirrorMark[], + ); + nodes.push(this.schema.text(codeText, marks)); + } + break; + } + + case "link": { + const linkToken = token as Tokens.Link; + const attrs: Record = { + href: safeLinkHref(linkToken.href), + }; + if (linkToken.title) { + attrs.title = linkToken.title; + } + nodes.push( + ...this.withMark(this.schema.marks.link.create(attrs), () => + this.convertInlineTokens(linkToken.tokens), + ), + ); + break; + } + + case "image": { + const imgToken = token as Tokens.Image; + const attrs: Record = { src: imgToken.href }; + if (imgToken.title) attrs.title = imgToken.title; + if (imgToken.text) attrs.alt = unescapeHtml(imgToken.text); + nodes.push(this.schema.nodes.image.create(attrs)); + break; + } + + case "br": + nodes.push(this.schema.nodes.hardBreak.create()); + break; + + case "escape": + nodes.push(...this.makeTextNode((token as Tokens.Escape).text)); + break; + + case "html": { + const tagInfo = parseHtmlTag((token as Tokens.Tag).text); + if (tagInfo) { + if (tagInfo.closing) { + for (let j = this.marks.length - 1; j >= 0; j--) { + if (this.marks[j].type.name === tagInfo.markName) { + this.marks.splice(j, 1); + break; + } + } + } else { + const markType = this.schema.marks[tagInfo.markName]; + if (markType) { + this.marks.push(markType.create(tagInfo.attrs || null)); + } + } + } + break; + } + + default: + break; + } + } + + return nodes; + } } export function hasMarkdown(text: string): boolean { return markdownLinePattern.test(text) || tableSeparatorPattern.test(text); } +export function parseInlineContent( + schema: Schema, + text: string, +): ProseMirrorNode[] { + const tokens = lexer(text); + const converter = new Converter(schema); + + for (const token of tokens) { + if (token.type === "paragraph") { + return converter.convertInlineTokens( + (token as Tokens.Paragraph).tokens, + ); + } + } + + return []; +} + export function parseMarkdown( text: string, schema: Schema, ): ProseMirrorNode[] { - const lines = text.split("\n"); - const nodes: ProseMirrorNode[] = []; - let i = 0; - - while (i < lines.length) { - const line = lines[i]; - - const codeMatch = line.match(codeBlockOpenPattern); - if (codeMatch) { - const language = codeMatch[1] || null; - const contentLines: string[] = []; - i++; - while (i < lines.length && !lines[i].startsWith("```")) { - contentLines.push(lines[i]); - i++; - } - i++; - - const codeContent = contentLines.join("\n").replace(/\n$/, ""); - nodes.push( - schema.nodes.codeBlock.create( - { language }, - codeContent ? schema.text(codeContent) : undefined, - ), - ); - continue; - } - - if ( - line.includes("|") - && i + 1 < lines.length - && tableSeparatorPattern.test(lines[i + 1].trim()) - ) { - const headerLine = line; - i += 2; - const dataLines: string[] = []; - while (i < lines.length) { - const row = lines[i].trim(); - if (!row || !row.includes("|")) break; - dataLines.push(lines[i]); - i++; - } - nodes.push(buildTable(schema, headerLine, dataLines)); - continue; - } - - const trimmed = line.trim(); - if (trimmed === "---" || trimmed === "___" || trimmed === "***") { - nodes.push(schema.nodes.horizontalRule.create()); - i++; - continue; - } - - const headingMatch = line.match(/^(#{1,6})\s(.+)$/); - if (headingMatch) { - const level = headingMatch[1].length; - const content = parseInlineContent(schema, headingMatch[2]); - nodes.push( - schema.nodes.heading.create( - { level }, - content.length > 0 ? content : undefined, - ), - ); - i++; - continue; - } - - if (line.startsWith("> ")) { - const paragraphs: ProseMirrorNode[] = []; - while (i < lines.length) { - const bqMatch = lines[i].match(/^>\s(.*)$/); - if (!bqMatch) break; - const bqText = bqMatch[1]; - const content = bqText - ? parseInlineContent(schema, bqText) - : []; - paragraphs.push( - schema.nodes.paragraph.create( - null, - content.length > 0 ? content : undefined, - ), - ); - i++; - } - nodes.push(schema.nodes.blockquote.create(null, paragraphs)); - continue; - } - - if ( - line.startsWith("- ") - || line.startsWith("+ ") - || line.startsWith("* ") - ) { - const items: ProseMirrorNode[] = []; - while (i < lines.length) { - const blMatch = lines[i].match(/^[-+*]\s(.*)$/); - if (!blMatch) break; - const blText = blMatch[1]; - const content = blText - ? parseInlineContent(schema, blText) - : []; - items.push( - schema.nodes.listItem.create( - null, - schema.nodes.paragraph.create( - null, - content.length > 0 ? content : undefined, - ), - ), - ); - i++; - } - nodes.push(schema.nodes.bulletList.create(null, items)); - continue; - } - - const olMatch = line.match(/^(\d+)\.\s(.*)$/); - if (olMatch) { - const start = +olMatch[1]; - const items: ProseMirrorNode[] = []; - while (i < lines.length) { - const itemMatch = lines[i].match(/^\d+\.\s(.*)$/); - if (!itemMatch) break; - const olText = itemMatch[1]; - const content = olText - ? parseInlineContent(schema, olText) - : []; - items.push( - schema.nodes.listItem.create( - null, - schema.nodes.paragraph.create( - null, - content.length > 0 ? content : undefined, - ), - ), - ); - i++; - } - nodes.push(schema.nodes.orderedList.create({ start }, items)); - continue; - } - - if (line.trim()) { - const content = parseInlineContent(schema, line); - nodes.push( - schema.nodes.paragraph.create( - null, - content.length > 0 ? content : undefined, - ), - ); - } - i++; - } - - return nodes; + const tokens = lexer(text); + const converter = new Converter(schema); + return converter.convertBlockTokens(tokens); }