// Copyright (c) 2026 Probo Inc . // // Permission to use, copy, modify, and/or distribute this software for any // purpose with or without fee is hereby granted, provided that the above // copyright notice and this permission notice appear in all copies. // // THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH // REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY // AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, // INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM // LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR // OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR // PERFORMANCE OF THIS SOFTWARE. package browser import ( "compress/gzip" "context" "encoding/xml" "fmt" "io" "net/http" "strings" "time" "go.probo.inc/probo/pkg/agent" ) type ( sitemapParams struct { URL string `json:"url" jsonschema:"The full URL of the sitemap to fetch (e.g. https://example.com/sitemap.xml)"` } sitemapResult struct { Found bool `json:"found"` URLs []string `json:"urls,omitempty"` URLCount int `json:"url_count"` ErrorDetail string `json:"error_detail,omitempty"` } ) const ( maxSitemapURLs = 200 ) func FetchSitemapTool() agent.Tool { client := &http.Client{Timeout: 15 * time.Second} return agent.FunctionTool( "fetch_sitemap", "Fetch and parse a sitemap XML file. Returns discovered URLs which can reveal pages not linked from the main navigation (trust centers, legal docs, status pages).", func(ctx context.Context, p sitemapParams) (agent.ToolResult, error) { if err := validatePublicURL(p.URL); err != nil { return agent.ResultJSON(sitemapResult{ Found: false, ErrorDetail: fmt.Sprintf("URL not allowed: %s", err), }), nil } req, err := http.NewRequestWithContext(ctx, http.MethodGet, p.URL, nil) if err != nil { return agent.ResultJSON(sitemapResult{ Found: false, ErrorDetail: fmt.Sprintf("cannot create request: %s", err), }), nil } resp, err := client.Do(req) if err != nil { return agent.ResultJSON(sitemapResult{ Found: false, ErrorDetail: fmt.Sprintf("cannot fetch sitemap: %s", err), }), nil } defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { return agent.ResultJSON(sitemapResult{ Found: false, ErrorDetail: fmt.Sprintf("sitemap returned status %d", resp.StatusCode), }), nil } var reader io.Reader = resp.Body if strings.HasSuffix(strings.ToLower(p.URL), ".gz") || resp.Header.Get("Content-Encoding") == "gzip" { gz, err := gzip.NewReader(resp.Body) if err != nil { return agent.ResultJSON(sitemapResult{ Found: false, ErrorDetail: fmt.Sprintf("cannot decompress gzipped sitemap: %s", err), }), nil } defer func() { _ = gz.Close() }() reader = gz } // Limit read to 5MB. reader = io.LimitReader(reader, 5*1024*1024) urls, err := parseSitemapXML(reader) if err != nil { return agent.ResultJSON(sitemapResult{ Found: false, ErrorDetail: fmt.Sprintf("cannot parse sitemap XML: %s", err), }), nil } result := sitemapResult{ Found: true, URLCount: len(urls), } if len(urls) > maxSitemapURLs { result.URLs = urls[:maxSitemapURLs] } else { result.URLs = urls } return agent.ResultJSON(result), nil }, ) } func parseSitemapXML(r io.Reader) ([]string, error) { var urls []string decoder := xml.NewDecoder(r) for { tok, err := decoder.Token() if err == io.EOF { break } if err != nil { return urls, err } if se, ok := tok.(xml.StartElement); ok && se.Name.Local == "loc" { var loc string if err := decoder.DecodeElement(&loc, &se); err == nil { loc = strings.TrimSpace(loc) if loc != "" { urls = append(urls, loc) } } } } return urls, nil }