From 9781c7534f9390225e6695070e85537e8237345b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89mile=20R=C3=A9?= Date: Mon, 30 Mar 2026 16:58:01 +0400 Subject: [PATCH] Handle mardown html elements parsing with sanitization MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Émile Ré --- .../main.go | 6 +- go.mod | 5 +- go.sum | 6 + pkg/prosemirror/html_block.go | 600 ++++++++++++++++++ pkg/prosemirror/markdown.go | 118 +++- pkg/prosemirror/markdown_test.go | 158 +++++ 6 files changed, 876 insertions(+), 17 deletions(-) create mode 100644 pkg/prosemirror/html_block.go diff --git a/cmd/migrate-document-versions-markdown/main.go b/cmd/migrate-document-versions-markdown/main.go index 83e91add3..af01b444a 100644 --- a/cmd/migrate-document-versions-markdown/main.go +++ b/cmd/migrate-document-versions-markdown/main.go @@ -40,9 +40,9 @@ func main() { func run() error { var ( - pgDSN string - dryRun bool - continueOnError bool + pgDSN string + dryRun bool + continueOnError bool ) flag.StringVar( diff --git a/go.mod b/go.mod index 6c388bc85..262080242 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( github.com/go-chi/cors v1.2.2 github.com/jackc/pgx/v5 v5.8.0 github.com/jhillyerd/enmime v1.3.0 + github.com/microcosm-cc/bluemonday v1.0.27 github.com/modelcontextprotocol/go-sdk v1.4.1 github.com/muesli/termenv v0.16.0 github.com/openai/openai-go v1.12.0 @@ -50,6 +51,7 @@ require ( require ( github.com/atotto/clipboard v0.1.4 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/aymerick/douceur v0.2.0 // indirect github.com/catppuccin/go v0.3.0 // indirect github.com/charmbracelet/bubbles v0.21.1-0.20250623103423-23b8fd6302d7 // indirect github.com/charmbracelet/bubbletea v1.3.6 // indirect @@ -60,6 +62,7 @@ require ( github.com/charmbracelet/x/term v0.2.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect + github.com/gorilla/css v1.0.1 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/mattn/go-localereader v0.0.1 // indirect github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect @@ -163,7 +166,7 @@ require ( go.opentelemetry.io/proto/otlp v1.9.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect golang.org/x/mod v0.34.0 // indirect - golang.org/x/net v0.52.0 // indirect + golang.org/x/net v0.52.0 golang.org/x/sys v0.42.0 // indirect golang.org/x/term v0.41.0 golang.org/x/text v0.35.0 // indirect diff --git a/go.sum b/go.sum index a7c5df02a..768880b5b 100644 --- a/go.sum +++ b/go.sum @@ -52,6 +52,8 @@ github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiE github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/aymanbagabas/go-udiff v0.3.1 h1:LV+qyBQ2pqe0u42ZsUEtPiCaUoqgA9gYRDs3vj1nolY= github.com/aymanbagabas/go-udiff v0.3.1/go.mod h1:G0fsKmG+P6ylD0r6N/KgQD/nWzgfnl8ZBcNLgcbrw8E= +github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= +github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/beevik/etree v1.6.0 h1:u8Kwy8pp9D9XeITj2Z0XtA5qqZEmtJtuXZRQi+j03eE= github.com/beevik/etree v1.6.0/go.mod h1:bh4zJxiIr62SOf9pRzN7UUYaEDa9HEKafK25+sLc0Gc= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -184,6 +186,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.12 h1:Fg+zsqzYEs1Znvmczt github.com/googleapis/enterprise-certificate-proxy v0.3.12/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg= github.com/googleapis/gax-go/v2 v2.17.0 h1:RksgfBpxqff0EZkDWYuz9q/uWsTVz+kf43LsZ1J6SMc= github.com/googleapis/gax-go/v2 v2.17.0/go.mod h1:mzaqghpQp4JDh3HvADwrat+6M3MOIDp5YKHhb9PAgDY= +github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= +github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 h1:X+2YciYSxvMQK0UZ7sg45ZVabVZBeBuvMkmuI2V3Fak= @@ -235,6 +239,8 @@ github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+Ei github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= +github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4= github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= github.com/modelcontextprotocol/go-sdk v1.4.1 h1:M4x9GyIPj+HoIlHNGpK2hq5o3BFhC+78PkEaldQRphc= diff --git a/pkg/prosemirror/html_block.go b/pkg/prosemirror/html_block.go new file mode 100644 index 000000000..e7468856c --- /dev/null +++ b/pkg/prosemirror/html_block.go @@ -0,0 +1,600 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package prosemirror + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" + + "github.com/microcosm-cc/bluemonday" + "golang.org/x/net/html" +) + +// htmlBlockSanitizePolicy matches tags and attributes we can represent as +// ProseMirror/Tiptap JSON and strips scripts, event handlers, and unsafe URLs. +func htmlBlockSanitizePolicy() *bluemonday.Policy { + return bluemonday.UGCPolicy() +} + +func sanitizeHTMLBlockContent(s string) string { + return htmlBlockSanitizePolicy().Sanitize(s) +} + +// convertProseMirrorFromInlineHTML sanitizes inline raw HTML and maps it to +// ProseMirror paragraph-level children (text, hardBreak, image, marks). +func convertProseMirrorFromInlineHTML(raw string) ([]Node, error) { + sanitized := strings.TrimSpace(sanitizeHTMLBlockContent(raw)) + if sanitized == "" { + return nil, nil + } + + roots, err := parseHTMLFragmentRoots(sanitized) + if err != nil { + return nil, fmt.Errorf("cannot convert inline html: %w", err) + } + + c := &htmlBlockConverter{} + var out []Node + for _, root := range roots { + nodes, err := c.convertInlineNode(root) + if err != nil { + return nil, err + } + out = append(out, nodes...) + } + if len(out) > 0 { + return out, nil + } + + plain := strings.TrimSpace(plainTextFromHTMLFragment(sanitized)) + if plain == "" { + return nil, nil + } + return []Node{{Type: NodeText, Text: &plain}}, nil +} + +func convertProseMirrorFromHTMLBlock(raw string) ([]Node, error) { + sanitized := strings.TrimSpace(sanitizeHTMLBlockContent(raw)) + if sanitized == "" { + return nil, nil + } + + nodes, err := htmlFragmentToProseMirrorBlocks(sanitized) + if err != nil { + return nil, fmt.Errorf("cannot convert html block to prosemirror: %w", err) + } + if len(nodes) > 0 { + return nodes, nil + } + + plain := strings.TrimSpace(plainTextFromHTMLFragment(sanitized)) + if plain == "" { + return nil, nil + } + return []Node{paragraphWithPlainText(plain)}, nil +} + +func paragraphWithPlainText(s string) Node { + return Node{ + Type: NodeParagraph, + Content: []Node{{ + Type: NodeText, + Text: &s, + }}, + } +} + +func parseHTMLFragmentRoots(htmlStr string) ([]*html.Node, error) { + return html.ParseFragmentWithOptions( + strings.NewReader(htmlStr), + nil, + html.ParseOptionEnableScripting(false), + ) +} + +func plainTextFromHTMLFragment(htmlStr string) string { + roots, err := parseHTMLFragmentRoots(htmlStr) + if err != nil { + return "" + } + var b strings.Builder + var walk func(*html.Node) + walk = func(n *html.Node) { + if n.Type == html.TextNode { + b.WriteString(n.Data) + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + walk(c) + } + } + for _, root := range roots { + walk(root) + } + return b.String() +} + +func htmlFragmentToProseMirrorBlocks(htmlStr string) ([]Node, error) { + roots, err := parseHTMLFragmentRoots(htmlStr) + if err != nil { + return nil, err + } + + c := &htmlBlockConverter{} + var out []Node + for _, root := range roots { + nodes, err := c.convertTopLevel(root) + if err != nil { + return nil, err + } + out = append(out, nodes...) + } + return out, nil +} + +type htmlBlockConverter struct { + marks []Mark +} + +func (c *htmlBlockConverter) convertTopLevel(n *html.Node) ([]Node, error) { + switch n.Type { + case html.TextNode: + t := strings.TrimSpace(n.Data) + if t == "" { + return nil, nil + } + return []Node{paragraphWithPlainText(t)}, nil + case html.ElementNode: + return c.convertBlockElement(n) + default: + return nil, nil + } +} + +func (c *htmlBlockConverter) convertBlockElement(n *html.Node) ([]Node, error) { + switch n.Data { + case "p": + inlines, err := c.convertInlineFragments(n) + if err != nil { + return nil, err + } + return []Node{{Type: NodeParagraph, Content: inlines}}, nil + case "h1", "h2", "h3", "h4", "h5", "h6": + level := int(n.Data[1] - '0') + inlines, err := c.convertInlineFragments(n) + if err != nil { + return nil, err + } + attrs, err := json.Marshal(HeadingAttrs{Level: level}) + if err != nil { + return nil, fmt.Errorf("cannot marshal heading attrs: %w", err) + } + return []Node{{ + Type: NodeHeading, + Attrs: attrs, + Content: inlines, + }}, nil + case "blockquote": + return c.convertBlockquote(n) + case "pre": + return c.convertPre(n) + case "hr": + return []Node{{Type: NodeHorizontalRule}}, nil + case "ul": + return c.convertList(n, false) + case "ol": + return c.convertList(n, true) + case "table": + return c.convertTable(n) + case "br": + return []Node{{ + Type: NodeParagraph, + Content: []Node{{Type: NodeHardBreak}}, + }}, nil + case "img": + img, err := c.convertImageElement(n) + if err != nil { + return nil, err + } + if img == nil { + return nil, nil + } + return []Node{{ + Type: NodeParagraph, + Content: []Node{*img}, + }}, nil + case "div", "section", "article", "aside", "main", "header", "footer", "nav", + "center", "figure", "body", "html", "span": + return c.unwrapBlockElement(n) + default: + return c.unwrapBlockElement(n) + } +} + +func (c *htmlBlockConverter) unwrapBlockElement(n *html.Node) ([]Node, error) { + return c.convertBlockChildren(n) +} + +func (c *htmlBlockConverter) convertBlockChildren(n *html.Node) ([]Node, error) { + var out []Node + for ch := n.FirstChild; ch != nil; ch = ch.NextSibling { + nodes, err := c.convertTopLevel(ch) + if err != nil { + return nil, err + } + out = append(out, nodes...) + } + return out, nil +} + +func (c *htmlBlockConverter) convertBlockquote(n *html.Node) ([]Node, error) { + if hasBlockElementChild(n) { + inner, err := c.convertBlockChildren(n) + if err != nil { + return nil, err + } + return []Node{{Type: NodeBlockquote, Content: inner}}, nil + } + inlines, err := c.convertInlineFragments(n) + if err != nil { + return nil, err + } + var content []Node + if len(inlines) > 0 { + content = []Node{{Type: NodeParagraph, Content: inlines}} + } + return []Node{{Type: NodeBlockquote, Content: content}}, nil +} + +func (c *htmlBlockConverter) convertPre(n *html.Node) ([]Node, error) { + var lang *string + var textBuf strings.Builder + for ch := n.FirstChild; ch != nil; ch = ch.NextSibling { + if ch.Type == html.ElementNode && ch.Data == "code" { + lang = codeLanguageFromClass(attrVal(ch, "class")) + var walkText func(*html.Node) + walkText = func(x *html.Node) { + if x.Type == html.TextNode { + textBuf.WriteString(x.Data) + } + for cc := x.FirstChild; cc != nil; cc = cc.NextSibling { + walkText(cc) + } + } + walkText(ch) + break + } + } + if textBuf.Len() == 0 { + var walkText func(*html.Node) + walkText = func(x *html.Node) { + if x.Type == html.TextNode { + textBuf.WriteString(x.Data) + } + for cc := x.FirstChild; cc != nil; cc = cc.NextSibling { + walkText(cc) + } + } + walkText(n) + } + + content := textBuf.String() + attrs, err := json.Marshal(CodeBlockAttrs{Language: lang}) + if err != nil { + return nil, fmt.Errorf("cannot marshal code block attrs: %w", err) + } + + var textNodes []Node + if content != "" { + textNodes = []Node{{Type: NodeText, Text: &content}} + } + return []Node{{ + Type: NodeCodeBlock, + Attrs: attrs, + Content: textNodes, + }}, nil +} + +func codeLanguageFromClass(class string) *string { + const prefix = "language-" + for _, part := range strings.Fields(class) { + if strings.HasPrefix(part, prefix) { + lang := strings.TrimPrefix(part, prefix) + if lang != "" { + return &lang + } + } + } + return nil +} + +func (c *htmlBlockConverter) convertList(n *html.Node, ordered bool) ([]Node, error) { + var items []Node + for li := n.FirstChild; li != nil; li = li.NextSibling { + if li.Type != html.ElementNode || li.Data != "li" { + continue + } + body, err := c.convertListItem(li) + if err != nil { + return nil, err + } + if len(body) == 0 { + continue + } + items = append(items, Node{Type: NodeListItem, Content: body}) + } + if len(items) == 0 { + return nil, nil + } + if ordered { + start := parseOlStart(n) + attrs, err := json.Marshal(OrderedListAttrs{Start: start}) + if err != nil { + return nil, fmt.Errorf("cannot marshal ordered list attrs: %w", err) + } + return []Node{{ + Type: NodeOrderedList, + Attrs: attrs, + Content: items, + }}, nil + } + return []Node{{ + Type: NodeBulletList, + Content: items, + }}, nil +} + +func parseOlStart(n *html.Node) int { + s := attrVal(n, "start") + if s == "" { + return 1 + } + v, err := strconv.Atoi(strings.TrimSpace(s)) + if err != nil || v < 1 { + return 1 + } + return v +} + +func (c *htmlBlockConverter) convertListItem(li *html.Node) ([]Node, error) { + if hasBlockElementChild(li) { + return c.convertBlockChildren(li) + } + inlines, err := c.convertInlineFragments(li) + if err != nil { + return nil, err + } + if len(inlines) == 0 { + return nil, nil + } + return []Node{{Type: NodeParagraph, Content: inlines}}, nil +} + +func hasBlockElementChild(n *html.Node) bool { + for ch := n.FirstChild; ch != nil; ch = ch.NextSibling { + if ch.Type == html.ElementNode && blockTagName(ch.Data) { + return true + } + } + return false +} + +func blockTagName(name string) bool { + switch name { + case "p", "div", "blockquote", "pre", "ul", "ol", "table", + "h1", "h2", "h3", "h4", "h5", "h6", "hr": + return true + default: + return false + } +} + +func (c *htmlBlockConverter) convertTable(n *html.Node) ([]Node, error) { + var rows []*html.Node + var collect func(*html.Node) + collect = func(x *html.Node) { + if x.Type == html.ElementNode && x.Data == "tr" { + rows = append(rows, x) + return + } + for ch := x.FirstChild; ch != nil; ch = ch.NextSibling { + collect(ch) + } + } + collect(n) + + var rowNodes []Node + for _, tr := range rows { + row, err := c.convertTableRow(tr) + if err != nil { + return nil, err + } + if row != nil { + rowNodes = append(rowNodes, *row) + } + } + if len(rowNodes) == 0 { + return nil, nil + } + return []Node{{Type: NodeTable, Content: rowNodes}}, nil +} + +func (c *htmlBlockConverter) convertTableRow(tr *html.Node) (*Node, error) { + var cells []Node + for ch := tr.FirstChild; ch != nil; ch = ch.NextSibling { + if ch.Type != html.ElementNode { + continue + } + var cell *Node + var err error + switch ch.Data { + case "th": + cell, err = c.convertTableCell(ch, NodeTableHeader) + case "td": + cell, err = c.convertTableCell(ch, NodeTableCell) + default: + continue + } + if err != nil { + return nil, err + } + if cell != nil { + cells = append(cells, *cell) + } + } + if len(cells) == 0 { + return nil, nil + } + return &Node{Type: NodeTableRow, Content: cells}, nil +} + +func (c *htmlBlockConverter) convertTableCell(n *html.Node, typ NodeType) (*Node, error) { + attrs, err := json.Marshal(TableCellAttrs{}) + if err != nil { + return nil, fmt.Errorf("cannot marshal table cell attrs: %w", err) + } + inlines, err := c.convertInlineFragments(n) + if err != nil { + return nil, err + } + content := []Node{{Type: NodeParagraph, Content: inlines}} + return &Node{Type: typ, Attrs: attrs, Content: content}, nil +} + +func (c *htmlBlockConverter) convertInlineFragments(parent *html.Node) ([]Node, error) { + var out []Node + for ch := parent.FirstChild; ch != nil; ch = ch.NextSibling { + nodes, err := c.convertInlineNode(ch) + if err != nil { + return nil, err + } + out = append(out, nodes...) + } + return out, nil +} + +func (c *htmlBlockConverter) convertInlineNode(n *html.Node) ([]Node, error) { + switch n.Type { + case html.TextNode: + if n.Data == "" { + return nil, nil + } + t := n.Data + return []Node{{ + Type: NodeText, + Text: &t, + Marks: copyMarks(c.marks), + }}, nil + case html.ElementNode: + return c.convertInlineElement(n) + default: + return nil, nil + } +} + +func (c *htmlBlockConverter) convertInlineElement(n *html.Node) ([]Node, error) { + switch n.Data { + case "p", "div", "section", "article", "aside", "main", "header", "footer", "nav", + "center", "figure", "h1", "h2", "h3", "h4", "h5", "h6": + // Block-ish tags inside inline HTML: unwrap to children only. + return c.convertInlineFragments(n) + case "br": + return []Node{{Type: NodeHardBreak}}, nil + case "strong", "b": + return c.withMark(Mark{Type: MarkStrong}, n) + case "em", "i": + return c.withMark(Mark{Type: MarkEm}, n) + case "s", "strike", "del": + return c.withMark(Mark{Type: MarkStrike}, n) + case "u": + return c.withMark(Mark{Type: MarkUnderline}, n) + case "code": + return c.withMark(Mark{Type: MarkCode}, n) + case "a": + return c.convertAnchor(n) + case "img": + img, err := c.convertImageElement(n) + if err != nil || img == nil { + return nil, err + } + return []Node{*img}, nil + case "span": + return c.convertInlineFragments(n) + default: + return c.convertInlineFragments(n) + } +} + +func (c *htmlBlockConverter) withMark(m Mark, n *html.Node) ([]Node, error) { + c.marks = append(c.marks, m) + nodes, err := c.convertInlineFragments(n) + c.marks = c.marks[:len(c.marks)-1] + if err != nil { + return nil, err + } + return nodes, nil +} + +func (c *htmlBlockConverter) convertAnchor(n *html.Node) ([]Node, error) { + href := attrVal(n, "href") + if href == "" { + return c.convertInlineFragments(n) + } + var title *string + if t := attrVal(n, "title"); t != "" { + title = &t + } + attrs, err := json.Marshal(LinkAttrs{Href: href, Title: title}) + if err != nil { + return nil, fmt.Errorf("cannot marshal link attrs: %w", err) + } + m := Mark{Type: MarkLink, Attrs: attrs} + c.marks = append(c.marks, m) + nodes, err := c.convertInlineFragments(n) + c.marks = c.marks[:len(c.marks)-1] + if err != nil { + return nil, err + } + return nodes, nil +} + +func (c *htmlBlockConverter) convertImageElement(n *html.Node) (*Node, error) { + src := attrVal(n, "src") + if src == "" { + return nil, nil + } + var alt, title *string + if a := attrVal(n, "alt"); a != "" { + alt = &a + } + if t := attrVal(n, "title"); t != "" { + title = &t + } + attrs, err := json.Marshal(ImageAttrs{Src: src, Alt: alt, Title: title}) + if err != nil { + return nil, fmt.Errorf("cannot marshal image attrs: %w", err) + } + return &Node{Type: NodeImage, Attrs: attrs}, nil +} + +func attrVal(n *html.Node, key string) string { + for _, a := range n.Attr { + if a.Key == key { + return a.Val + } + } + return "" +} diff --git a/pkg/prosemirror/markdown.go b/pkg/prosemirror/markdown.go index 01531177c..e22bf2427 100644 --- a/pkg/prosemirror/markdown.go +++ b/pkg/prosemirror/markdown.go @@ -18,6 +18,7 @@ import ( "bytes" "encoding/json" "fmt" + "strings" "github.com/yuin/goldmark" "github.com/yuin/goldmark/ast" @@ -70,6 +71,62 @@ func (c *converter) convertChildren(n ast.Node) ([]Node, error) { return nodes, nil } +// convertInlineChildren walks inline content and merges adjacent RawHTML + Text +// segments so tags split by goldmark (e.g. , b, ) form one HTML +// fragment for sanitization and conversion. +func (c *converter) convertInlineChildren(n ast.Node) ([]Node, error) { + var nodes []Node + + for ch := n.FirstChild(); ch != nil; { + if ch.Kind() == ast.KindRawHTML { + run, next := c.collectRawHTMLRun(ch) + inodes, err := convertProseMirrorFromInlineHTML(run) + if err != nil { + return nil, err + } + nodes = append(nodes, prependOuterMarks(copyMarks(c.marks), inodes)...) + ch = next + continue + } + + converted, err := c.convertNode(ch) + if err != nil { + return nil, err + } + nodes = append(nodes, converted...) + ch = ch.NextSibling() + } + + return nodes, nil +} + +// collectRawHTMLRun concatenates a leading RawHTML node and following Text/String +// and RawHTML siblings until a different node kind is seen. next is the first +// sibling not consumed (or nil). +func (c *converter) collectRawHTMLRun(start ast.Node) (run string, next ast.Node) { + var buf bytes.Buffer + ch := start + for ch != nil { + switch ch.Kind() { + case ast.KindRawHTML: + raw := ch.(*ast.RawHTML) + for i := 0; i < raw.Segments.Len(); i++ { + seg := raw.Segments.At(i) + buf.Write(seg.Value(c.source)) + } + case ast.KindText: + t := ch.(*ast.Text) + buf.Write(t.Segment.Value(c.source)) + case ast.KindString: + buf.Write(ch.(*ast.String).Value) + default: + return buf.String(), ch + } + ch = ch.NextSibling() + } + return buf.String(), nil +} + func (c *converter) convertNode(n ast.Node) ([]Node, error) { switch n.Kind() { case ast.KindHeading: @@ -107,7 +164,7 @@ func (c *converter) convertNode(n ast.Node) ([]Node, error) { case ast.KindRawHTML: return c.convertRawHTML(n) case ast.KindHTMLBlock: - return nil, nil + return c.convertHTMLBlock(n.(*ast.HTMLBlock)) default: if n.Kind() == goldmarkast.KindStrikethrough { return c.convertStrikethrough(n) @@ -117,7 +174,7 @@ func (c *converter) convertNode(n ast.Node) ([]Node, error) { } func (c *converter) convertHeading(n *ast.Heading) ([]Node, error) { - children, err := c.convertChildren(n) + children, err := c.convertInlineChildren(n) if err != nil { return nil, err } @@ -135,7 +192,7 @@ func (c *converter) convertHeading(n *ast.Heading) ([]Node, error) { } func (c *converter) convertParagraph(n ast.Node) ([]Node, error) { - children, err := c.convertChildren(n) + children, err := c.convertInlineChildren(n) if err != nil { return nil, err } @@ -334,7 +391,7 @@ func (c *converter) convertEmphasis(n *ast.Emphasis) ([]Node, error) { } c.marks = append(c.marks, mark) - children, err := c.convertChildren(n) + children, err := c.convertInlineChildren(n) c.marks = c.marks[:len(c.marks)-1] if err != nil { return nil, err @@ -383,7 +440,7 @@ func (c *converter) convertLink(n *ast.Link) ([]Node, error) { } c.marks = append(c.marks, Mark{Type: MarkLink, Attrs: attrs}) - children, err := c.convertChildren(n) + children, err := c.convertInlineChildren(n) c.marks = c.marks[:len(c.marks)-1] if err != nil { return nil, err @@ -411,28 +468,48 @@ func (c *converter) convertAutoLink(n *ast.AutoLink) ([]Node, error) { } func (c *converter) convertRawHTML(n ast.Node) ([]Node, error) { + raw, ok := n.(*ast.RawHTML) + if !ok { + return nil, fmt.Errorf("cannot convert raw html: unexpected node type %T", n) + } + var buf bytes.Buffer + for i := 0; i < raw.Segments.Len(); i++ { + seg := raw.Segments.At(i) + buf.Write(seg.Value(c.source)) + } + run := buf.String() + if run == "" { + return nil, nil + } + nodes, err := convertProseMirrorFromInlineHTML(run) + if err != nil { + return nil, err + } + return prependOuterMarks(copyMarks(c.marks), nodes), nil +} + +func (c *converter) convertHTMLBlock(n *ast.HTMLBlock) ([]Node, error) { var buf bytes.Buffer for i := 0; i < n.Lines().Len(); i++ { line := n.Lines().At(i) buf.Write(line.Value(c.source)) } + if n.HasClosure() { + buf.Write(n.ClosureLine.Value(c.source)) + } - content := buf.String() - if content == "" { + raw := buf.String() + if strings.TrimSpace(raw) == "" { return nil, nil } - return []Node{{ - Type: NodeText, - Text: &content, - Marks: copyMarks(c.marks), - }}, nil + return convertProseMirrorFromHTMLBlock(raw) } func (c *converter) convertStrikethrough(n ast.Node) ([]Node, error) { c.marks = append(c.marks, Mark{Type: MarkStrike}) - children, err := c.convertChildren(n) + children, err := c.convertInlineChildren(n) c.marks = c.marks[:len(c.marks)-1] if err != nil { return nil, err @@ -451,3 +528,18 @@ func copyMarks(marks []Mark) []Mark { return cp } + +// prependOuterMarks applies markdown inline context marks (e.g. emphasis around +// raw HTML) to nodes produced from sanitized HTML. Images are left unchanged. +func prependOuterMarks(outer []Mark, nodes []Node) []Node { + if len(outer) == 0 { + return nodes + } + for i := range nodes { + if nodes[i].Type == NodeImage { + continue + } + nodes[i].Marks = append(copyMarks(outer), nodes[i].Marks...) + } + return nodes +} diff --git a/pkg/prosemirror/markdown_test.go b/pkg/prosemirror/markdown_test.go index 5b9c4f9ae..400341015 100644 --- a/pkg/prosemirror/markdown_test.go +++ b/pkg/prosemirror/markdown_test.go @@ -366,6 +366,164 @@ func TestParseMarkdown_MixedContent(t *testing.T) { assert.Equal(t, MarkStrong, p.Content[1].Marks[0].Type) } +func TestParseMarkdown_BlockHTML(t *testing.T) { + t.Parallel() + + doc, err := ParseMarkdown("
block
\n") + require.NoError(t, err) + require.Len(t, doc.Content, 1) + assert.Equal(t, NodeParagraph, doc.Content[0].Type) + require.Len(t, doc.Content[0].Content, 1) + assert.Equal(t, NodeText, doc.Content[0].Content[0].Type) + require.NotNil(t, doc.Content[0].Content[0].Text) + assert.Equal(t, "block", *doc.Content[0].Content[0].Text) +} + +func TestParseMarkdown_BlockHTMLWithClosureLine(t *testing.T) { + t.Parallel() + + // Type 1 HTML block: closing tag is stored on ClosureLine, not in Lines. + // Script is stripped by the HTML sanitizer; nothing safe remains. + md := "\n" + doc, err := ParseMarkdown(md) + require.NoError(t, err) + assert.Empty(t, doc.Content) +} + +func TestParseMarkdown_BlockHTMLParagraphAndHeading(t *testing.T) { + t.Parallel() + + doc, err := ParseMarkdown("

a

\n

Title

\n") + require.NoError(t, err) + require.Len(t, doc.Content, 2) + assert.Equal(t, NodeParagraph, doc.Content[0].Type) + require.Len(t, doc.Content[0].Content, 1) + assert.Equal(t, "a", *doc.Content[0].Content[0].Text) + assert.Equal(t, NodeHeading, doc.Content[1].Type) + attrs, err := doc.Content[1].HeadingAttrs() + require.NoError(t, err) + assert.Equal(t, 2, attrs.Level) +} + +func TestParseMarkdown_BlockHTMLListAndBlockquote(t *testing.T) { + t.Parallel() + + doc, err := ParseMarkdown("
  • one
\n

q

\n") + require.NoError(t, err) + require.Len(t, doc.Content, 2) + assert.Equal(t, NodeBulletList, doc.Content[0].Type) + assert.Equal(t, NodeBlockquote, doc.Content[1].Type) +} + +func TestParseMarkdown_BlockHTMLTable(t *testing.T) { + t.Parallel() + + md := "
AB
\n" + doc, err := ParseMarkdown(md) + require.NoError(t, err) + require.Len(t, doc.Content, 1) + assert.Equal(t, NodeTable, doc.Content[0].Type) + require.Len(t, doc.Content[0].Content, 1) + assert.Equal(t, NodeTableRow, doc.Content[0].Content[0].Type) + require.Len(t, doc.Content[0].Content[0].Content, 2) + assert.Equal(t, NodeTableHeader, doc.Content[0].Content[0].Content[0].Type) + assert.Equal(t, NodeTableCell, doc.Content[0].Content[0].Content[1].Type) +} + +func TestParseMarkdown_BlockHTMLScriptRemovedKeepsSafeContent(t *testing.T) { + t.Parallel() + + md := "

ok

\n\n" + doc, err := ParseMarkdown(md) + require.NoError(t, err) + require.Len(t, doc.Content, 1) + assert.Equal(t, NodeParagraph, doc.Content[0].Type) + assert.Equal(t, "ok", *doc.Content[0].Content[0].Text) +} + +func TestParseMarkdown_InlineRawHTML(t *testing.T) { + t.Parallel() + + doc, err := ParseMarkdown("before x after") + require.NoError(t, err) + require.Len(t, doc.Content, 1) + p := doc.Content[0] + require.Equal(t, NodeParagraph, p.Type) + + var joined string + for _, ch := range p.Content { + require.Equal(t, NodeText, ch.Type) + require.NotNil(t, ch.Text) + joined += *ch.Text + } + // Sanitized HTML: span is unwrapped to plain text content. + assert.Equal(t, "before x after", joined) +} + +func TestParseMarkdown_InlineRawHTMLStrong(t *testing.T) { + t.Parallel() + + doc, err := ParseMarkdown(`a b c`) + require.NoError(t, err) + require.Len(t, doc.Content, 1) + p := doc.Content[0] + require.Len(t, p.Content, 3) + assert.Equal(t, "a ", *p.Content[0].Text) + assert.Equal(t, "b", *p.Content[1].Text) + require.Len(t, p.Content[1].Marks, 1) + assert.Equal(t, MarkStrong, p.Content[1].Marks[0].Type) + assert.Equal(t, " c", *p.Content[2].Text) +} + +func TestParseMarkdown_InlineRawHTMLScriptStripped(t *testing.T) { + t.Parallel() + + doc, err := ParseMarkdown(`hi there`) + require.NoError(t, err) + require.Len(t, doc.Content, 1) + p := doc.Content[0] + var joined string + for _, ch := range p.Content { + if ch.Type == NodeText && ch.Text != nil { + joined += *ch.Text + } + } + assert.NotContains(t, joined, "script") + assert.NotContains(t, joined, "evil") + assert.Contains(t, joined, "hi") + assert.Contains(t, joined, "there") +} + +func TestParseMarkdown_InlineRawHTMLWithOuterBold(t *testing.T) { + t.Parallel() + + doc, err := ParseMarkdown(`**a b c**`) + require.NoError(t, err) + require.Len(t, doc.Content, 1) + p := doc.Content[0] + require.GreaterOrEqual(t, len(p.Content), 3) + + var joined string + for _, ch := range p.Content { + require.Equal(t, NodeText, ch.Type) + require.NotNil(t, ch.Text) + joined += *ch.Text + } + assert.Equal(t, "a b c", joined) + + var mid *Node + for i := range p.Content { + if p.Content[i].Text != nil && *p.Content[i].Text == "b" { + mid = &p.Content[i] + break + } + } + require.NotNil(t, mid, "expected inner as text node b") + require.GreaterOrEqual(t, len(mid.Marks), 2) + assert.Equal(t, MarkStrong, mid.Marks[0].Type) + assert.Equal(t, MarkEm, mid.Marks[1].Type) +} + func TestParseMarkdown_JSONRoundTrip(t *testing.T) { t.Parallel()