rohit/probo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Handle mardown html elements parsing with sanitization

Browse Source 
Signed-off-by: Émile Ré <emile@getprobo.com>
This commit is contained in:
Émile Ré
2026-03-30 16:58:01 +04:00
parent 4bb009f25b
commit 9781c7534f
6 changed files with 876 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
cmd/migrate-document-versions-markdown/main.go
View File

@@ -40,9 +40,9 @@ func main() {
func run() error {
var (
pgDSN string
dryRun bool
continueOnError bool
pgDSN string
dryRun bool
continueOnError bool
)
flag.StringVar(

5
go.mod
View File

@@ -22,6 +22,7 @@ require (
github.com/go-chi/cors v1.2.2
github.com/jackc/pgx/v5 v5.8.0
github.com/jhillyerd/enmime v1.3.0
github.com/microcosm-cc/bluemonday v1.0.27
github.com/modelcontextprotocol/go-sdk v1.4.1
github.com/muesli/termenv v0.16.0
github.com/openai/openai-go v1.12.0
@@ -50,6 +51,7 @@ require (
require (
github.com/atotto/clipboard v0.1.4 // indirect
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
github.com/aymerick/douceur v0.2.0 // indirect
github.com/catppuccin/go v0.3.0 // indirect
github.com/charmbracelet/bubbles v0.21.1-0.20250623103423-23b8fd6302d7 // indirect
github.com/charmbracelet/bubbletea v1.3.6 // indirect
@@ -60,6 +62,7 @@ require (
github.com/charmbracelet/x/term v0.2.1 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
github.com/gorilla/css v1.0.1 // indirect
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
github.com/mattn/go-localereader v0.0.1 // indirect
github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect
@@ -163,7 +166,7 @@ require (
go.opentelemetry.io/proto/otlp v1.9.0 // indirect
go.yaml.in/yaml/v2 v2.4.3 // indirect
golang.org/x/mod v0.34.0 // indirect
golang.org/x/net v0.52.0 // indirect
golang.org/x/net v0.52.0
golang.org/x/sys v0.42.0 // indirect
golang.org/x/term v0.41.0
golang.org/x/text v0.35.0 // indirect

6
go.sum
View File

@@ -52,6 +52,8 @@ github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiE
github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
github.com/aymanbagabas/go-udiff v0.3.1 h1:LV+qyBQ2pqe0u42ZsUEtPiCaUoqgA9gYRDs3vj1nolY=
github.com/aymanbagabas/go-udiff v0.3.1/go.mod h1:G0fsKmG+P6ylD0r6N/KgQD/nWzgfnl8ZBcNLgcbrw8E=
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
github.com/beevik/etree v1.6.0 h1:u8Kwy8pp9D9XeITj2Z0XtA5qqZEmtJtuXZRQi+j03eE=
github.com/beevik/etree v1.6.0/go.mod h1:bh4zJxiIr62SOf9pRzN7UUYaEDa9HEKafK25+sLc0Gc=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -184,6 +186,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.12 h1:Fg+zsqzYEs1Znvmczt
github.com/googleapis/enterprise-certificate-proxy v0.3.12/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg=
github.com/googleapis/gax-go/v2 v2.17.0 h1:RksgfBpxqff0EZkDWYuz9q/uWsTVz+kf43LsZ1J6SMc=
github.com/googleapis/gax-go/v2 v2.17.0/go.mod h1:mzaqghpQp4JDh3HvADwrat+6M3MOIDp5YKHhb9PAgDY=
github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8=
github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0=
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 h1:X+2YciYSxvMQK0UZ7sg45ZVabVZBeBuvMkmuI2V3Fak=
@@ -235,6 +239,8 @@ github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+Ei
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw=
github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk=
github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA=
github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4=
github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE=
github.com/modelcontextprotocol/go-sdk v1.4.1 h1:M4x9GyIPj+HoIlHNGpK2hq5o3BFhC+78PkEaldQRphc=

600
pkg/prosemirror/html_block.go Normal file
View File

@@ -0,0 +1,600 @@
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package prosemirror
import (
"encoding/json"
"fmt"
"strconv"
"strings"
"github.com/microcosm-cc/bluemonday"
"golang.org/x/net/html"
)
// htmlBlockSanitizePolicy matches tags and attributes we can represent as
// ProseMirror/Tiptap JSON and strips scripts, event handlers, and unsafe URLs.
func htmlBlockSanitizePolicy() *bluemonday.Policy {
return bluemonday.UGCPolicy()
}
func sanitizeHTMLBlockContent(s string) string {
return htmlBlockSanitizePolicy().Sanitize(s)
}
// convertProseMirrorFromInlineHTML sanitizes inline raw HTML and maps it to
// ProseMirror paragraph-level children (text, hardBreak, image, marks).
func convertProseMirrorFromInlineHTML(raw string) ([]Node, error) {
sanitized := strings.TrimSpace(sanitizeHTMLBlockContent(raw))
if sanitized == "" {
return nil, nil
}
roots, err := parseHTMLFragmentRoots(sanitized)
if err != nil {
return nil, fmt.Errorf("cannot convert inline html: %w", err)
}
c := &htmlBlockConverter{}
var out []Node
for _, root := range roots {
nodes, err := c.convertInlineNode(root)
if err != nil {
return nil, err
}
out = append(out, nodes...)
}
if len(out) > 0 {
return out, nil
}
plain := strings.TrimSpace(plainTextFromHTMLFragment(sanitized))
if plain == "" {
return nil, nil
}
return []Node{{Type: NodeText, Text: &plain}}, nil
}
func convertProseMirrorFromHTMLBlock(raw string) ([]Node, error) {
sanitized := strings.TrimSpace(sanitizeHTMLBlockContent(raw))
if sanitized == "" {
return nil, nil
}
nodes, err := htmlFragmentToProseMirrorBlocks(sanitized)
if err != nil {
return nil, fmt.Errorf("cannot convert html block to prosemirror: %w", err)
}
if len(nodes) > 0 {
return nodes, nil
}
plain := strings.TrimSpace(plainTextFromHTMLFragment(sanitized))
if plain == "" {
return nil, nil
}
return []Node{paragraphWithPlainText(plain)}, nil
}
func paragraphWithPlainText(s string) Node {
return Node{
Type: NodeParagraph,
Content: []Node{{
Type: NodeText,
Text: &s,
}},
}
}
func parseHTMLFragmentRoots(htmlStr string) ([]*html.Node, error) {
return html.ParseFragmentWithOptions(
strings.NewReader(htmlStr),
nil,
html.ParseOptionEnableScripting(false),
)
}
func plainTextFromHTMLFragment(htmlStr string) string {
roots, err := parseHTMLFragmentRoots(htmlStr)
if err != nil {
return ""
}
var b strings.Builder
var walk func(*html.Node)
walk = func(n *html.Node) {
if n.Type == html.TextNode {
b.WriteString(n.Data)
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
walk(c)
}
}
for _, root := range roots {
walk(root)
}
return b.String()
}
func htmlFragmentToProseMirrorBlocks(htmlStr string) ([]Node, error) {
roots, err := parseHTMLFragmentRoots(htmlStr)
if err != nil {
return nil, err
}
c := &htmlBlockConverter{}
var out []Node
for _, root := range roots {
nodes, err := c.convertTopLevel(root)
if err != nil {
return nil, err
}
out = append(out, nodes...)
}
return out, nil
}
type htmlBlockConverter struct {
marks []Mark
}
func (c *htmlBlockConverter) convertTopLevel(n *html.Node) ([]Node, error) {
switch n.Type {
case html.TextNode:
t := strings.TrimSpace(n.Data)
if t == "" {
return nil, nil
}
return []Node{paragraphWithPlainText(t)}, nil
case html.ElementNode:
return c.convertBlockElement(n)
default:
return nil, nil
}
}
func (c *htmlBlockConverter) convertBlockElement(n *html.Node) ([]Node, error) {
switch n.Data {
case "p":
inlines, err := c.convertInlineFragments(n)
if err != nil {
return nil, err
}
return []Node{{Type: NodeParagraph, Content: inlines}}, nil
case "h1", "h2", "h3", "h4", "h5", "h6":
level := int(n.Data[1] - '0')
inlines, err := c.convertInlineFragments(n)
if err != nil {
return nil, err
}
attrs, err := json.Marshal(HeadingAttrs{Level: level})
if err != nil {
return nil, fmt.Errorf("cannot marshal heading attrs: %w", err)
}
return []Node{{
Type: NodeHeading,
Attrs: attrs,
Content: inlines,
}}, nil
case "blockquote":
return c.convertBlockquote(n)
case "pre":
return c.convertPre(n)
case "hr":
return []Node{{Type: NodeHorizontalRule}}, nil
case "ul":
return c.convertList(n, false)
case "ol":
return c.convertList(n, true)
case "table":
return c.convertTable(n)
case "br":
return []Node{{
Type: NodeParagraph,
Content: []Node{{Type: NodeHardBreak}},
}}, nil
case "img":
img, err := c.convertImageElement(n)
if err != nil {
return nil, err
}
if img == nil {
return nil, nil
}
return []Node{{
Type: NodeParagraph,
Content: []Node{*img},
}}, nil
case "div", "section", "article", "aside", "main", "header", "footer", "nav",
"center", "figure", "body", "html", "span":
return c.unwrapBlockElement(n)
default:
return c.unwrapBlockElement(n)
}
}
func (c *htmlBlockConverter) unwrapBlockElement(n *html.Node) ([]Node, error) {
return c.convertBlockChildren(n)
}
func (c *htmlBlockConverter) convertBlockChildren(n *html.Node) ([]Node, error) {
var out []Node
for ch := n.FirstChild; ch != nil; ch = ch.NextSibling {
nodes, err := c.convertTopLevel(ch)
if err != nil {
return nil, err
}
out = append(out, nodes...)
}
return out, nil
}
func (c *htmlBlockConverter) convertBlockquote(n *html.Node) ([]Node, error) {
if hasBlockElementChild(n) {
inner, err := c.convertBlockChildren(n)
if err != nil {
return nil, err
}
return []Node{{Type: NodeBlockquote, Content: inner}}, nil
}
inlines, err := c.convertInlineFragments(n)
if err != nil {
return nil, err
}
var content []Node
if len(inlines) > 0 {
content = []Node{{Type: NodeParagraph, Content: inlines}}
}
return []Node{{Type: NodeBlockquote, Content: content}}, nil
}
func (c *htmlBlockConverter) convertPre(n *html.Node) ([]Node, error) {
var lang *string
var textBuf strings.Builder
for ch := n.FirstChild; ch != nil; ch = ch.NextSibling {
if ch.Type == html.ElementNode && ch.Data == "code" {
lang = codeLanguageFromClass(attrVal(ch, "class"))
var walkText func(*html.Node)
walkText = func(x *html.Node) {
if x.Type == html.TextNode {
textBuf.WriteString(x.Data)
}
for cc := x.FirstChild; cc != nil; cc = cc.NextSibling {
walkText(cc)
}
}
walkText(ch)
break
}
}
if textBuf.Len() == 0 {
var walkText func(*html.Node)
walkText = func(x *html.Node) {
if x.Type == html.TextNode {
textBuf.WriteString(x.Data)
}
for cc := x.FirstChild; cc != nil; cc = cc.NextSibling {
walkText(cc)
}
}
walkText(n)
}
content := textBuf.String()
attrs, err := json.Marshal(CodeBlockAttrs{Language: lang})
if err != nil {
return nil, fmt.Errorf("cannot marshal code block attrs: %w", err)
}
var textNodes []Node
if content != "" {
textNodes = []Node{{Type: NodeText, Text: &content}}
}
return []Node{{
Type: NodeCodeBlock,
Attrs: attrs,
Content: textNodes,
}}, nil
}
func codeLanguageFromClass(class string) *string {
const prefix = "language-"
for _, part := range strings.Fields(class) {
if strings.HasPrefix(part, prefix) {
lang := strings.TrimPrefix(part, prefix)
if lang != "" {
return &lang
}
}
}
return nil
}
func (c *htmlBlockConverter) convertList(n *html.Node, ordered bool) ([]Node, error) {
var items []Node
for li := n.FirstChild; li != nil; li = li.NextSibling {
if li.Type != html.ElementNode || li.Data != "li" {
continue
}
body, err := c.convertListItem(li)
if err != nil {
return nil, err
}
if len(body) == 0 {
continue
}
items = append(items, Node{Type: NodeListItem, Content: body})
}
if len(items) == 0 {
return nil, nil
}
if ordered {
start := parseOlStart(n)
attrs, err := json.Marshal(OrderedListAttrs{Start: start})
if err != nil {
return nil, fmt.Errorf("cannot marshal ordered list attrs: %w", err)
}
return []Node{{
Type: NodeOrderedList,
Attrs: attrs,
Content: items,
}}, nil
}
return []Node{{
Type: NodeBulletList,
Content: items,
}}, nil
}
func parseOlStart(n *html.Node) int {
s := attrVal(n, "start")
if s == "" {
return 1
}
v, err := strconv.Atoi(strings.TrimSpace(s))
if err != nil || v < 1 {
return 1
}
return v
}
func (c *htmlBlockConverter) convertListItem(li *html.Node) ([]Node, error) {
if hasBlockElementChild(li) {
return c.convertBlockChildren(li)
}
inlines, err := c.convertInlineFragments(li)
if err != nil {
return nil, err
}
if len(inlines) == 0 {
return nil, nil
}
return []Node{{Type: NodeParagraph, Content: inlines}}, nil
}
func hasBlockElementChild(n *html.Node) bool {
for ch := n.FirstChild; ch != nil; ch = ch.NextSibling {
if ch.Type == html.ElementNode && blockTagName(ch.Data) {
return true
}
}
return false
}
func blockTagName(name string) bool {
switch name {
case "p", "div", "blockquote", "pre", "ul", "ol", "table",
"h1", "h2", "h3", "h4", "h5", "h6", "hr":
return true
default:
return false
}
}
func (c *htmlBlockConverter) convertTable(n *html.Node) ([]Node, error) {
var rows []*html.Node
var collect func(*html.Node)
collect = func(x *html.Node) {
if x.Type == html.ElementNode && x.Data == "tr" {
rows = append(rows, x)
return
}
for ch := x.FirstChild; ch != nil; ch = ch.NextSibling {
collect(ch)
}
}
collect(n)
var rowNodes []Node
for _, tr := range rows {
row, err := c.convertTableRow(tr)
if err != nil {
return nil, err
}
if row != nil {
rowNodes = append(rowNodes, *row)
}
}
if len(rowNodes) == 0 {
return nil, nil
}
return []Node{{Type: NodeTable, Content: rowNodes}}, nil
}
func (c *htmlBlockConverter) convertTableRow(tr *html.Node) (*Node, error) {
var cells []Node
for ch := tr.FirstChild; ch != nil; ch = ch.NextSibling {
if ch.Type != html.ElementNode {
continue
}
var cell *Node
var err error
switch ch.Data {
case "th":
cell, err = c.convertTableCell(ch, NodeTableHeader)
case "td":
cell, err = c.convertTableCell(ch, NodeTableCell)
default:
continue
}
if err != nil {
return nil, err
}
if cell != nil {
cells = append(cells, *cell)
}
}
if len(cells) == 0 {
return nil, nil
}
return &Node{Type: NodeTableRow, Content: cells}, nil
}
func (c *htmlBlockConverter) convertTableCell(n *html.Node, typ NodeType) (*Node, error) {
attrs, err := json.Marshal(TableCellAttrs{})
if err != nil {
return nil, fmt.Errorf("cannot marshal table cell attrs: %w", err)
}
inlines, err := c.convertInlineFragments(n)
if err != nil {
return nil, err
}
content := []Node{{Type: NodeParagraph, Content: inlines}}
return &Node{Type: typ, Attrs: attrs, Content: content}, nil
}
func (c *htmlBlockConverter) convertInlineFragments(parent *html.Node) ([]Node, error) {
var out []Node
for ch := parent.FirstChild; ch != nil; ch = ch.NextSibling {
nodes, err := c.convertInlineNode(ch)
if err != nil {
return nil, err
}
out = append(out, nodes...)
}
return out, nil
}
func (c *htmlBlockConverter) convertInlineNode(n *html.Node) ([]Node, error) {
switch n.Type {
case html.TextNode:
if n.Data == "" {
return nil, nil
}
t := n.Data
return []Node{{
Type: NodeText,
Text: &t,
Marks: copyMarks(c.marks),
}}, nil
case html.ElementNode:
return c.convertInlineElement(n)
default:
return nil, nil
}
}
func (c *htmlBlockConverter) convertInlineElement(n *html.Node) ([]Node, error) {
switch n.Data {
case "p", "div", "section", "article", "aside", "main", "header", "footer", "nav",
"center", "figure", "h1", "h2", "h3", "h4", "h5", "h6":
// Block-ish tags inside inline HTML: unwrap to children only.
return c.convertInlineFragments(n)
case "br":
return []Node{{Type: NodeHardBreak}}, nil
case "strong", "b":
return c.withMark(Mark{Type: MarkStrong}, n)
case "em", "i":
return c.withMark(Mark{Type: MarkEm}, n)
case "s", "strike", "del":
return c.withMark(Mark{Type: MarkStrike}, n)
case "u":
return c.withMark(Mark{Type: MarkUnderline}, n)
case "code":
return c.withMark(Mark{Type: MarkCode}, n)
case "a":
return c.convertAnchor(n)
case "img":
img, err := c.convertImageElement(n)
if err != nil || img == nil {
return nil, err
}
return []Node{*img}, nil
case "span":
return c.convertInlineFragments(n)
default:
return c.convertInlineFragments(n)
}
}
func (c *htmlBlockConverter) withMark(m Mark, n *html.Node) ([]Node, error) {
c.marks = append(c.marks, m)
nodes, err := c.convertInlineFragments(n)
c.marks = c.marks[:len(c.marks)-1]
if err != nil {
return nil, err
}
return nodes, nil
}
func (c *htmlBlockConverter) convertAnchor(n *html.Node) ([]Node, error) {
href := attrVal(n, "href")
if href == "" {
return c.convertInlineFragments(n)
}
var title *string
if t := attrVal(n, "title"); t != "" {
title = &t
}
attrs, err := json.Marshal(LinkAttrs{Href: href, Title: title})
if err != nil {
return nil, fmt.Errorf("cannot marshal link attrs: %w", err)
}
m := Mark{Type: MarkLink, Attrs: attrs}
c.marks = append(c.marks, m)
nodes, err := c.convertInlineFragments(n)
c.marks = c.marks[:len(c.marks)-1]
if err != nil {
return nil, err
}
return nodes, nil
}
func (c *htmlBlockConverter) convertImageElement(n *html.Node) (*Node, error) {
src := attrVal(n, "src")
if src == "" {
return nil, nil
}
var alt, title *string
if a := attrVal(n, "alt"); a != "" {
alt = &a
}
if t := attrVal(n, "title"); t != "" {
title = &t
}
attrs, err := json.Marshal(ImageAttrs{Src: src, Alt: alt, Title: title})
if err != nil {
return nil, fmt.Errorf("cannot marshal image attrs: %w", err)
}
return &Node{Type: NodeImage, Attrs: attrs}, nil
}
func attrVal(n *html.Node, key string) string {
for _, a := range n.Attr {
if a.Key == key {
return a.Val
}
}
return ""
}

118
pkg/prosemirror/markdown.go
View File

@@ -18,6 +18,7 @@ import (
"bytes"
"encoding/json"
"fmt"
"strings"
"github.com/yuin/goldmark"
"github.com/yuin/goldmark/ast"
@@ -70,6 +71,62 @@ func (c *converter) convertChildren(n ast.Node) ([]Node, error) {
return nodes, nil
}
// convertInlineChildren walks inline content and merges adjacent RawHTML + Text
// segments so tags split by goldmark (e.g. <strong>, b, </strong>) form one HTML
// fragment for sanitization and conversion.
func (c *converter) convertInlineChildren(n ast.Node) ([]Node, error) {
var nodes []Node
for ch := n.FirstChild(); ch != nil; {
if ch.Kind() == ast.KindRawHTML {
run, next := c.collectRawHTMLRun(ch)
inodes, err := convertProseMirrorFromInlineHTML(run)
if err != nil {
return nil, err
}
nodes = append(nodes, prependOuterMarks(copyMarks(c.marks), inodes)...)
ch = next
continue
}
converted, err := c.convertNode(ch)
if err != nil {
return nil, err
}
nodes = append(nodes, converted...)
ch = ch.NextSibling()
}
return nodes, nil
}
// collectRawHTMLRun concatenates a leading RawHTML node and following Text/String
// and RawHTML siblings until a different node kind is seen. next is the first
// sibling not consumed (or nil).
func (c *converter) collectRawHTMLRun(start ast.Node) (run string, next ast.Node) {
var buf bytes.Buffer
ch := start
for ch != nil {
switch ch.Kind() {
case ast.KindRawHTML:
raw := ch.(*ast.RawHTML)
for i := 0; i < raw.Segments.Len(); i++ {
seg := raw.Segments.At(i)
buf.Write(seg.Value(c.source))
}
case ast.KindText:
t := ch.(*ast.Text)
buf.Write(t.Segment.Value(c.source))
case ast.KindString:
buf.Write(ch.(*ast.String).Value)
default:
return buf.String(), ch
}
ch = ch.NextSibling()
}
return buf.String(), nil
}
func (c *converter) convertNode(n ast.Node) ([]Node, error) {
switch n.Kind() {
case ast.KindHeading:
@@ -107,7 +164,7 @@ func (c *converter) convertNode(n ast.Node) ([]Node, error) {
case ast.KindRawHTML:
return c.convertRawHTML(n)
case ast.KindHTMLBlock:
return nil, nil
return c.convertHTMLBlock(n.(*ast.HTMLBlock))
default:
if n.Kind() == goldmarkast.KindStrikethrough {
return c.convertStrikethrough(n)
@@ -117,7 +174,7 @@ func (c *converter) convertNode(n ast.Node) ([]Node, error) {
}
func (c *converter) convertHeading(n *ast.Heading) ([]Node, error) {
children, err := c.convertChildren(n)
children, err := c.convertInlineChildren(n)
if err != nil {
return nil, err
}
@@ -135,7 +192,7 @@ func (c *converter) convertHeading(n *ast.Heading) ([]Node, error) {
}
func (c *converter) convertParagraph(n ast.Node) ([]Node, error) {
children, err := c.convertChildren(n)
children, err := c.convertInlineChildren(n)
if err != nil {
return nil, err
}
@@ -334,7 +391,7 @@ func (c *converter) convertEmphasis(n *ast.Emphasis) ([]Node, error) {
}
c.marks = append(c.marks, mark)
children, err := c.convertChildren(n)
children, err := c.convertInlineChildren(n)
c.marks = c.marks[:len(c.marks)-1]
if err != nil {
return nil, err
@@ -383,7 +440,7 @@ func (c *converter) convertLink(n *ast.Link) ([]Node, error) {
}
c.marks = append(c.marks, Mark{Type: MarkLink, Attrs: attrs})
children, err := c.convertChildren(n)
children, err := c.convertInlineChildren(n)
c.marks = c.marks[:len(c.marks)-1]
if err != nil {
return nil, err
@@ -411,28 +468,48 @@ func (c *converter) convertAutoLink(n *ast.AutoLink) ([]Node, error) {
}
func (c *converter) convertRawHTML(n ast.Node) ([]Node, error) {
raw, ok := n.(*ast.RawHTML)
if !ok {
return nil, fmt.Errorf("cannot convert raw html: unexpected node type %T", n)
}
var buf bytes.Buffer
for i := 0; i < raw.Segments.Len(); i++ {
seg := raw.Segments.At(i)
buf.Write(seg.Value(c.source))
}
run := buf.String()
if run == "" {
return nil, nil
}
nodes, err := convertProseMirrorFromInlineHTML(run)
if err != nil {
return nil, err
}
return prependOuterMarks(copyMarks(c.marks), nodes), nil
}
func (c *converter) convertHTMLBlock(n *ast.HTMLBlock) ([]Node, error) {
var buf bytes.Buffer
for i := 0; i < n.Lines().Len(); i++ {
line := n.Lines().At(i)
buf.Write(line.Value(c.source))
}
if n.HasClosure() {
buf.Write(n.ClosureLine.Value(c.source))
}
content := buf.String()
if content == "" {
raw := buf.String()
if strings.TrimSpace(raw) == "" {
return nil, nil
}
return []Node{{
Type: NodeText,
Text: &content,
Marks: copyMarks(c.marks),
}}, nil
return convertProseMirrorFromHTMLBlock(raw)
}
func (c *converter) convertStrikethrough(n ast.Node) ([]Node, error) {
c.marks = append(c.marks, Mark{Type: MarkStrike})
children, err := c.convertChildren(n)
children, err := c.convertInlineChildren(n)
c.marks = c.marks[:len(c.marks)-1]
if err != nil {
return nil, err
@@ -451,3 +528,18 @@ func copyMarks(marks []Mark) []Mark {
return cp
}
// prependOuterMarks applies markdown inline context marks (e.g. emphasis around
// raw HTML) to nodes produced from sanitized HTML. Images are left unchanged.
func prependOuterMarks(outer []Mark, nodes []Node) []Node {
if len(outer) == 0 {
return nodes
}
for i := range nodes {
if nodes[i].Type == NodeImage {
continue
}
nodes[i].Marks = append(copyMarks(outer), nodes[i].Marks...)
}
return nodes
}

158
pkg/prosemirror/markdown_test.go
View File

@@ -366,6 +366,164 @@ func TestParseMarkdown_MixedContent(t *testing.T) {
assert.Equal(t, MarkStrong, p.Content[1].Marks[0].Type)
}
func TestParseMarkdown_BlockHTML(t *testing.T) {
t.Parallel()
doc, err := ParseMarkdown("<div>block</div>\n")
require.NoError(t, err)
require.Len(t, doc.Content, 1)
assert.Equal(t, NodeParagraph, doc.Content[0].Type)
require.Len(t, doc.Content[0].Content, 1)
assert.Equal(t, NodeText, doc.Content[0].Content[0].Type)
require.NotNil(t, doc.Content[0].Content[0].Text)
assert.Equal(t, "block", *doc.Content[0].Content[0].Text)
}
func TestParseMarkdown_BlockHTMLWithClosureLine(t *testing.T) {
t.Parallel()
// Type 1 HTML block: closing tag is stored on ClosureLine, not in Lines.
// Script is stripped by the HTML sanitizer; nothing safe remains.
md := "<script>\nconsole.log(1)\n</script>\n"
doc, err := ParseMarkdown(md)
require.NoError(t, err)
assert.Empty(t, doc.Content)
}
func TestParseMarkdown_BlockHTMLParagraphAndHeading(t *testing.T) {
t.Parallel()
doc, err := ParseMarkdown("<p>a</p>\n<h2>Title</h2>\n")
require.NoError(t, err)
require.Len(t, doc.Content, 2)
assert.Equal(t, NodeParagraph, doc.Content[0].Type)
require.Len(t, doc.Content[0].Content, 1)
assert.Equal(t, "a", *doc.Content[0].Content[0].Text)
assert.Equal(t, NodeHeading, doc.Content[1].Type)
attrs, err := doc.Content[1].HeadingAttrs()
require.NoError(t, err)
assert.Equal(t, 2, attrs.Level)
}
func TestParseMarkdown_BlockHTMLListAndBlockquote(t *testing.T) {
t.Parallel()
doc, err := ParseMarkdown("<ul><li>one</li></ul>\n<blockquote><p>q</p></blockquote>\n")
require.NoError(t, err)
require.Len(t, doc.Content, 2)
assert.Equal(t, NodeBulletList, doc.Content[0].Type)
assert.Equal(t, NodeBlockquote, doc.Content[1].Type)
}
func TestParseMarkdown_BlockHTMLTable(t *testing.T) {
t.Parallel()
md := "<table><tr><th>A</th><td>B</td></tr></table>\n"
doc, err := ParseMarkdown(md)
require.NoError(t, err)
require.Len(t, doc.Content, 1)
assert.Equal(t, NodeTable, doc.Content[0].Type)
require.Len(t, doc.Content[0].Content, 1)
assert.Equal(t, NodeTableRow, doc.Content[0].Content[0].Type)
require.Len(t, doc.Content[0].Content[0].Content, 2)
assert.Equal(t, NodeTableHeader, doc.Content[0].Content[0].Content[0].Type)
assert.Equal(t, NodeTableCell, doc.Content[0].Content[0].Content[1].Type)
}
func TestParseMarkdown_BlockHTMLScriptRemovedKeepsSafeContent(t *testing.T) {
t.Parallel()
md := "<p>ok</p>\n<script>bad()</script>\n"
doc, err := ParseMarkdown(md)
require.NoError(t, err)
require.Len(t, doc.Content, 1)
assert.Equal(t, NodeParagraph, doc.Content[0].Type)
assert.Equal(t, "ok", *doc.Content[0].Content[0].Text)
}
func TestParseMarkdown_InlineRawHTML(t *testing.T) {
t.Parallel()
doc, err := ParseMarkdown("before <span>x</span> after")
require.NoError(t, err)
require.Len(t, doc.Content, 1)
p := doc.Content[0]
require.Equal(t, NodeParagraph, p.Type)
var joined string
for _, ch := range p.Content {
require.Equal(t, NodeText, ch.Type)
require.NotNil(t, ch.Text)
joined += *ch.Text
}
// Sanitized HTML: span is unwrapped to plain text content.
assert.Equal(t, "before x after", joined)
}
func TestParseMarkdown_InlineRawHTMLStrong(t *testing.T) {
t.Parallel()
doc, err := ParseMarkdown(`a <strong>b</strong> c`)
require.NoError(t, err)
require.Len(t, doc.Content, 1)
p := doc.Content[0]
require.Len(t, p.Content, 3)
assert.Equal(t, "a ", *p.Content[0].Text)
assert.Equal(t, "b", *p.Content[1].Text)
require.Len(t, p.Content[1].Marks, 1)
assert.Equal(t, MarkStrong, p.Content[1].Marks[0].Type)
assert.Equal(t, " c", *p.Content[2].Text)
}
func TestParseMarkdown_InlineRawHTMLScriptStripped(t *testing.T) {
t.Parallel()
doc, err := ParseMarkdown(`hi <script>evil()</script> there`)
require.NoError(t, err)
require.Len(t, doc.Content, 1)
p := doc.Content[0]
var joined string
for _, ch := range p.Content {
if ch.Type == NodeText && ch.Text != nil {
joined += *ch.Text
}
}
assert.NotContains(t, joined, "script")
assert.NotContains(t, joined, "evil")
assert.Contains(t, joined, "hi")
assert.Contains(t, joined, "there")
}
func TestParseMarkdown_InlineRawHTMLWithOuterBold(t *testing.T) {
t.Parallel()
doc, err := ParseMarkdown(`**a <em>b</em> c**`)
require.NoError(t, err)
require.Len(t, doc.Content, 1)
p := doc.Content[0]
require.GreaterOrEqual(t, len(p.Content), 3)
var joined string
for _, ch := range p.Content {
require.Equal(t, NodeText, ch.Type)
require.NotNil(t, ch.Text)
joined += *ch.Text
}
assert.Equal(t, "a b c", joined)
var mid *Node
for i := range p.Content {
if p.Content[i].Text != nil && *p.Content[i].Text == "b" {
mid = &p.Content[i]
break
}
}
require.NotNil(t, mid, "expected inner <em> as text node b")
require.GreaterOrEqual(t, len(mid.Marks), 2)
assert.Equal(t, MarkStrong, mid.Marks[0].Type)
assert.Equal(t, MarkEm, mid.Marks[1].Type)
}
func TestParseMarkdown_JSONRoundTrip(t *testing.T) {
t.Parallel()