Reimplement ts converter using marked parser

Signed-off-by: Émile Ré <emile@getprobo.com>
This commit is contained in:
Émile Ré
2026-04-03 12:43:36 +04:00
parent a850b0f77c
commit 2a6845d884
3 changed files with 419 additions and 251 deletions

3
package-lock.json generated
View File

@@ -19709,7 +19709,8 @@
"version": "1.0.0",
"license": "ISC",
"dependencies": {
"@tiptap/pm": "^3.20.2"
"@tiptap/pm": "^3.20.2",
"marked": "^15.0.12"
},
"devDependencies": {
"typescript": "~5.8.3"

View File

@@ -10,7 +10,8 @@
"check": "tsc --noEmit"
},
"dependencies": {
"@tiptap/pm": "^3.20.2"
"@tiptap/pm": "^3.20.2",
"marked": "^15.0.12"
},
"devDependencies": {
"typescript": "~5.8.3"

View File

@@ -2,287 +2,453 @@
// Use of this source code is governed by the ISC license
// that can be found in the LICENSE file.
import type { Node as ProseMirrorNode, Schema } from "@tiptap/pm/model";
import { lexer, type Token, type Tokens } from "marked";
import type {
Mark as ProseMirrorMark,
Node as ProseMirrorNode,
Schema,
} from "@tiptap/pm/model";
const codeBlockOpenPattern = /^```(\w*)$/;
const tableSeparatorPattern
= /^\|?\s*:?-{3,}:?\s*(\|\s*:?-{3,}:?\s*)*\|?\s*$/m;
const markdownLinePattern
= /(?:^```\w*$|^#{1,6}\s|^>\s|^[-+*]\s|^\d+\.\s|^(?:---|___|\*\*\*)\s*$)/m;
const tableSeparatorPattern =
/^\|?\s*:?-{3,}:?\s*(\|\s*:?-{3,}:?\s*)*\|?\s*$/m;
const markdownLinePattern =
/(?:^```\w*$|^#{1,6}\s|^>\s|^[-+*]\s|^\d+\.\s|^(?:---|___|\*\*\*)\s*$)/m;
function parseCells(line: string): string[] {
return line
.replace(/^\|/, "")
.replace(/\|\s*$/, "")
.split("|")
.map(cell => cell.trim());
function unescapeHtml(html: string): string {
return html
.replace(/&lt;/g, "<")
.replace(/&gt;/g, ">")
.replace(/&quot;/g, '"')
.replace(/&#39;/g, "'")
.replace(/&amp;/g, "&");
}
function applyMarks(
schema: Schema,
markNames: string[],
inner: string,
): ProseMirrorNode[] {
const marks = markNames.map(name => schema.marks[name].create());
return parseInlineContent(schema, inner).map((node) => {
let combined = node.marks;
for (const mark of marks) {
combined = mark.addToSet(combined);
function safeLinkHref(href: string): string {
href = href.trim();
if (!href) return "#";
if (href[0] === "#") return href;
if (href.startsWith("/")) {
if (href.length > 1 && (href[1] === "/" || href[1] === "\\")) return "#";
return href;
}
try {
const url = new URL(href);
const scheme = url.protocol.slice(0, -1).toLowerCase();
switch (scheme) {
case "http":
case "https":
case "mailto":
case "tel":
return href;
default:
return "#";
}
return node.mark(combined);
});
} catch {
return href;
}
}
export function parseInlineContent(
schema: Schema,
text: string,
): ProseMirrorNode[] {
const nodes: ProseMirrorNode[] = [];
const pattern
= /`([^`]+)`|\*\*\*(.+?)\*\*\*|\*\*(.+?)\*\*|~~(.+?)~~|<u>(.+?)<\/u>|\[([^\]]+)\]\(([^)]+)\)|\*(.+?)\*/g;
let lastIndex = 0;
let match;
type HtmlTagInfo = {
closing: boolean;
markName: string;
attrs?: Record<string, unknown>;
};
while ((match = pattern.exec(text)) !== null) {
if (match.index > lastIndex) {
nodes.push(schema.text(text.slice(lastIndex, match.index)));
}
const openTagPattern = /^<(u|b|i|s|em|strong|del|code|a)(\s[^>]*)?\s*\/?>$/i;
const closeTagPattern = /^<\/(u|b|i|s|em|strong|del|code|a)\s*>$/i;
if (match[1] !== undefined) {
nodes.push(schema.text(match[1], [schema.marks.code.create()]));
} else if (match[2] !== undefined) {
nodes.push(...applyMarks(schema, ["bold", "italic"], match[2]));
} else if (match[3] !== undefined) {
nodes.push(...applyMarks(schema, ["bold"], match[3]));
} else if (match[4] !== undefined) {
nodes.push(...applyMarks(schema, ["strike"], match[4]));
} else if (match[5] !== undefined) {
nodes.push(...applyMarks(schema, ["underline"], match[5]));
} else if (match[6] !== undefined) {
const linkMark = schema.marks.link.create({ href: match[7] });
for (const node of parseInlineContent(schema, match[6])) {
nodes.push(node.mark(linkMark.addToSet(node.marks)));
function tagToMarkName(tag: string): string | null {
switch (tag.toLowerCase()) {
case "u":
return "underline";
case "b":
case "strong":
return "bold";
case "i":
case "em":
return "italic";
case "s":
case "del":
return "strike";
case "code":
return "code";
case "a":
return "link";
default:
return null;
}
}
function parseHtmlTag(html: string): HtmlTagInfo | null {
const closeMatch = html.match(closeTagPattern);
if (closeMatch) {
const markName = tagToMarkName(closeMatch[1]);
if (markName) return { closing: true, markName };
return null;
}
const openMatch = html.match(openTagPattern);
if (openMatch) {
const markName = tagToMarkName(openMatch[1]);
if (!markName) return null;
const info: HtmlTagInfo = { closing: false, markName };
if (openMatch[1].toLowerCase() === "a" && openMatch[2]) {
const hrefMatch = openMatch[2].match(/href=["']([^"']*)["']/i);
if (hrefMatch) {
info.attrs = { href: safeLinkHref(hrefMatch[1]) };
}
} else if (match[8] !== undefined) {
nodes.push(...applyMarks(schema, ["italic"], match[8]));
}
lastIndex = match.index + match[0].length;
return info;
}
if (lastIndex < text.length) {
nodes.push(schema.text(text.slice(lastIndex)));
}
return nodes;
return null;
}
function buildTable(
schema: Schema,
headerLine: string,
dataLines: string[],
): ProseMirrorNode {
const headers = parseCells(headerLine);
const columnCount = headers.length;
class Converter {
private schema: Schema;
private marks: ProseMirrorMark[];
const headerCells = headers.map((h) => {
const content = h ? parseInlineContent(schema, h) : [];
return schema.nodes.tableHeader.create(
constructor(schema: Schema) {
this.schema = schema;
this.marks = [];
}
private withMark(
mark: ProseMirrorMark,
fn: () => ProseMirrorNode[],
): ProseMirrorNode[] {
this.marks.push(mark);
const result = fn();
this.marks.pop();
return result;
}
private currentMarks(): readonly ProseMirrorMark[] {
let result: readonly ProseMirrorMark[] = [];
for (const mark of this.marks) {
result = mark.addToSet(result as ProseMirrorMark[]);
}
return result;
}
private makeTextNode(text: string): ProseMirrorNode[] {
const unescaped = unescapeHtml(text);
if (!unescaped) return [];
const marks = this.currentMarks();
return [this.schema.text(unescaped, marks.length > 0 ? marks : undefined)];
}
convertBlockTokens(tokens: Token[]): ProseMirrorNode[] {
const nodes: ProseMirrorNode[] = [];
for (const token of tokens) {
nodes.push(...this.convertBlockToken(token));
}
return nodes;
}
private convertBlockToken(token: Token): ProseMirrorNode[] {
switch (token.type) {
case "heading":
return this.convertHeading(token as Tokens.Heading);
case "paragraph":
return this.convertParagraph(token as Tokens.Paragraph);
case "blockquote":
return this.convertBlockquote(token as Tokens.Blockquote);
case "code":
return this.convertCodeBlock(token as Tokens.Code);
case "list":
return this.convertList(token as Tokens.List);
case "hr":
return [this.schema.nodes.horizontalRule.create()];
case "table":
return this.convertTable(token as Tokens.Table);
case "text":
return this.convertBlockText(token as Tokens.Text);
case "html":
case "space":
case "def":
return [];
default:
return [];
}
}
private convertHeading(token: Tokens.Heading): ProseMirrorNode[] {
const content = this.convertInlineTokens(token.tokens);
return [
this.schema.nodes.heading.create(
{ level: token.depth },
content.length > 0 ? content : undefined,
),
];
}
private convertParagraph(token: Tokens.Paragraph): ProseMirrorNode[] {
const content = this.convertInlineTokens(token.tokens);
return [
this.schema.nodes.paragraph.create(
null,
content.length > 0 ? content : undefined,
),
];
}
private convertBlockquote(token: Tokens.Blockquote): ProseMirrorNode[] {
const children = this.convertBlockTokens(token.tokens);
return [
this.schema.nodes.blockquote.create(
null,
children.length > 0 ? children : undefined,
),
];
}
private convertCodeBlock(token: Tokens.Code): ProseMirrorNode[] {
const language = token.lang || null;
return [
this.schema.nodes.codeBlock.create(
{ language },
token.text ? this.schema.text(token.text) : undefined,
),
];
}
private convertList(token: Tokens.List): ProseMirrorNode[] {
const children = token.items.flatMap(item => this.convertListItem(item));
if (token.ordered) {
const start = typeof token.start === "number" ? token.start : 1;
return [
this.schema.nodes.orderedList.create(
{ start },
children.length > 0 ? children : undefined,
),
];
}
return [
this.schema.nodes.bulletList.create(
null,
children.length > 0 ? children : undefined,
),
];
}
private convertListItem(token: Tokens.ListItem): ProseMirrorNode[] {
const children: ProseMirrorNode[] = [];
for (const child of token.tokens) {
if (child.type === "text") {
const textToken = child as Tokens.Text;
const inlineContent = textToken.tokens
? this.convertInlineTokens(textToken.tokens)
: this.makeTextNode(textToken.text);
children.push(
this.schema.nodes.paragraph.create(
null,
inlineContent.length > 0 ? inlineContent : undefined,
),
);
} else {
children.push(...this.convertBlockToken(child));
}
}
if (children.length === 0) {
children.push(this.schema.nodes.paragraph.create());
}
return [this.schema.nodes.listItem.create(null, children)];
}
private convertBlockText(token: Tokens.Text): ProseMirrorNode[] {
const content = token.tokens
? this.convertInlineTokens(token.tokens)
: this.makeTextNode(token.text);
return [
this.schema.nodes.paragraph.create(
null,
content.length > 0 ? content : undefined,
),
];
}
private convertTable(token: Tokens.Table): ProseMirrorNode[] {
const rows: ProseMirrorNode[] = [];
const headerCells = token.header.map(cell =>
this.convertTableCell(cell, true),
);
rows.push(this.schema.nodes.tableRow.create(null, headerCells));
for (const row of token.rows) {
const dataCells = row.map(cell => this.convertTableCell(cell, false));
rows.push(this.schema.nodes.tableRow.create(null, dataCells));
}
return [this.schema.nodes.table.create(null, rows)];
}
private convertTableCell(
cell: Tokens.TableCell,
isHeader: boolean,
): ProseMirrorNode {
const content = this.convertInlineTokens(cell.tokens);
const nodeType = isHeader
? this.schema.nodes.tableHeader
: this.schema.nodes.tableCell;
return nodeType.create(
null,
schema.nodes.paragraph.create(
this.schema.nodes.paragraph.create(
null,
content.length > 0 ? content : undefined,
),
);
});
const tableRows: ProseMirrorNode[] = [
schema.nodes.tableRow.create(null, headerCells),
];
for (const line of dataLines) {
const cells = parseCells(line);
const rowCells: ProseMirrorNode[] = [];
for (let c = 0; c < columnCount; c++) {
const value = cells[c]?.trim() ?? "";
const content = value ? parseInlineContent(schema, value) : [];
rowCells.push(
schema.nodes.tableCell.create(
null,
schema.nodes.paragraph.create(
null,
content.length > 0 ? content : undefined,
),
),
);
}
tableRows.push(schema.nodes.tableRow.create(null, rowCells));
}
return schema.nodes.table.create(null, tableRows);
convertInlineTokens(tokens: Token[]): ProseMirrorNode[] {
const nodes: ProseMirrorNode[] = [];
for (const token of tokens) {
switch (token.type) {
case "text": {
const textToken = token as Tokens.Text;
if (textToken.tokens && textToken.tokens.length > 0) {
nodes.push(...this.convertInlineTokens(textToken.tokens));
} else {
nodes.push(...this.makeTextNode(textToken.text));
}
break;
}
case "strong":
nodes.push(
...this.withMark(this.schema.marks.bold.create(), () =>
this.convertInlineTokens((token as Tokens.Strong).tokens),
),
);
break;
case "em":
nodes.push(
...this.withMark(this.schema.marks.italic.create(), () =>
this.convertInlineTokens((token as Tokens.Em).tokens),
),
);
break;
case "del":
nodes.push(
...this.withMark(this.schema.marks.strike.create(), () =>
this.convertInlineTokens((token as Tokens.Del).tokens),
),
);
break;
case "codespan": {
const codeText = unescapeHtml((token as Tokens.Codespan).text);
if (codeText) {
const codeMark = this.schema.marks.code.create();
const marks = codeMark.addToSet(
this.currentMarks() as ProseMirrorMark[],
);
nodes.push(this.schema.text(codeText, marks));
}
break;
}
case "link": {
const linkToken = token as Tokens.Link;
const attrs: Record<string, unknown> = {
href: safeLinkHref(linkToken.href),
};
if (linkToken.title) {
attrs.title = linkToken.title;
}
nodes.push(
...this.withMark(this.schema.marks.link.create(attrs), () =>
this.convertInlineTokens(linkToken.tokens),
),
);
break;
}
case "image": {
const imgToken = token as Tokens.Image;
const attrs: Record<string, unknown> = { src: imgToken.href };
if (imgToken.title) attrs.title = imgToken.title;
if (imgToken.text) attrs.alt = unescapeHtml(imgToken.text);
nodes.push(this.schema.nodes.image.create(attrs));
break;
}
case "br":
nodes.push(this.schema.nodes.hardBreak.create());
break;
case "escape":
nodes.push(...this.makeTextNode((token as Tokens.Escape).text));
break;
case "html": {
const tagInfo = parseHtmlTag((token as Tokens.Tag).text);
if (tagInfo) {
if (tagInfo.closing) {
for (let j = this.marks.length - 1; j >= 0; j--) {
if (this.marks[j].type.name === tagInfo.markName) {
this.marks.splice(j, 1);
break;
}
}
} else {
const markType = this.schema.marks[tagInfo.markName];
if (markType) {
this.marks.push(markType.create(tagInfo.attrs || null));
}
}
}
break;
}
default:
break;
}
}
return nodes;
}
}
export function hasMarkdown(text: string): boolean {
return markdownLinePattern.test(text) || tableSeparatorPattern.test(text);
}
export function parseInlineContent(
schema: Schema,
text: string,
): ProseMirrorNode[] {
const tokens = lexer(text);
const converter = new Converter(schema);
for (const token of tokens) {
if (token.type === "paragraph") {
return converter.convertInlineTokens(
(token as Tokens.Paragraph).tokens,
);
}
}
return [];
}
export function parseMarkdown(
text: string,
schema: Schema,
): ProseMirrorNode[] {
const lines = text.split("\n");
const nodes: ProseMirrorNode[] = [];
let i = 0;
while (i < lines.length) {
const line = lines[i];
const codeMatch = line.match(codeBlockOpenPattern);
if (codeMatch) {
const language = codeMatch[1] || null;
const contentLines: string[] = [];
i++;
while (i < lines.length && !lines[i].startsWith("```")) {
contentLines.push(lines[i]);
i++;
}
i++;
const codeContent = contentLines.join("\n").replace(/\n$/, "");
nodes.push(
schema.nodes.codeBlock.create(
{ language },
codeContent ? schema.text(codeContent) : undefined,
),
);
continue;
}
if (
line.includes("|")
&& i + 1 < lines.length
&& tableSeparatorPattern.test(lines[i + 1].trim())
) {
const headerLine = line;
i += 2;
const dataLines: string[] = [];
while (i < lines.length) {
const row = lines[i].trim();
if (!row || !row.includes("|")) break;
dataLines.push(lines[i]);
i++;
}
nodes.push(buildTable(schema, headerLine, dataLines));
continue;
}
const trimmed = line.trim();
if (trimmed === "---" || trimmed === "___" || trimmed === "***") {
nodes.push(schema.nodes.horizontalRule.create());
i++;
continue;
}
const headingMatch = line.match(/^(#{1,6})\s(.+)$/);
if (headingMatch) {
const level = headingMatch[1].length;
const content = parseInlineContent(schema, headingMatch[2]);
nodes.push(
schema.nodes.heading.create(
{ level },
content.length > 0 ? content : undefined,
),
);
i++;
continue;
}
if (line.startsWith("> ")) {
const paragraphs: ProseMirrorNode[] = [];
while (i < lines.length) {
const bqMatch = lines[i].match(/^>\s(.*)$/);
if (!bqMatch) break;
const bqText = bqMatch[1];
const content = bqText
? parseInlineContent(schema, bqText)
: [];
paragraphs.push(
schema.nodes.paragraph.create(
null,
content.length > 0 ? content : undefined,
),
);
i++;
}
nodes.push(schema.nodes.blockquote.create(null, paragraphs));
continue;
}
if (
line.startsWith("- ")
|| line.startsWith("+ ")
|| line.startsWith("* ")
) {
const items: ProseMirrorNode[] = [];
while (i < lines.length) {
const blMatch = lines[i].match(/^[-+*]\s(.*)$/);
if (!blMatch) break;
const blText = blMatch[1];
const content = blText
? parseInlineContent(schema, blText)
: [];
items.push(
schema.nodes.listItem.create(
null,
schema.nodes.paragraph.create(
null,
content.length > 0 ? content : undefined,
),
),
);
i++;
}
nodes.push(schema.nodes.bulletList.create(null, items));
continue;
}
const olMatch = line.match(/^(\d+)\.\s(.*)$/);
if (olMatch) {
const start = +olMatch[1];
const items: ProseMirrorNode[] = [];
while (i < lines.length) {
const itemMatch = lines[i].match(/^\d+\.\s(.*)$/);
if (!itemMatch) break;
const olText = itemMatch[1];
const content = olText
? parseInlineContent(schema, olText)
: [];
items.push(
schema.nodes.listItem.create(
null,
schema.nodes.paragraph.create(
null,
content.length > 0 ? content : undefined,
),
),
);
i++;
}
nodes.push(schema.nodes.orderedList.create({ start }, items));
continue;
}
if (line.trim()) {
const content = parseInlineContent(schema, line);
nodes.push(
schema.nodes.paragraph.create(
null,
content.length > 0 ? content : undefined,
),
);
}
i++;
}
return nodes;
const tokens = lexer(text);
const converter = new Converter(schema);
return converter.convertBlockTokens(tokens);
}