Enforce Go style rules across codebase
Apply five style rules: convert iota string enums to typed string constants, replace errors.As with errors.AsType, merge three-group imports into two groups, fix multiline parameter/argument formatting, and replace fmt.Sprintf URL construction with net/url. Signed-off-by: Émile Ré <emile@probo.com>
This commit is contained in:
@@ -54,56 +54,70 @@ func DownloadPDFTool() agent.Tool {
|
||||
"Download a PDF document from a URL and extract its text content. Use this for DPAs, SOC 2 reports, privacy policies, and other documents hosted as PDFs.",
|
||||
func(ctx context.Context, p downloadPDFParams) (agent.ToolResult, error) {
|
||||
if err := validatePublicURL(p.URL); err != nil {
|
||||
return agent.ResultJSON(downloadPDFResult{
|
||||
ErrorDetail: fmt.Sprintf("URL not allowed: %s", err),
|
||||
}), nil
|
||||
return agent.ResultJSON(
|
||||
downloadPDFResult{
|
||||
ErrorDetail: fmt.Sprintf("URL not allowed: %s", err),
|
||||
},
|
||||
), nil
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, p.URL, nil)
|
||||
if err != nil {
|
||||
return agent.ResultJSON(downloadPDFResult{
|
||||
ErrorDetail: fmt.Sprintf("cannot create request: %s", err),
|
||||
}), nil
|
||||
return agent.ResultJSON(
|
||||
downloadPDFResult{
|
||||
ErrorDetail: fmt.Sprintf("cannot create request: %s", err),
|
||||
},
|
||||
), nil
|
||||
}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return agent.ResultJSON(downloadPDFResult{
|
||||
ErrorDetail: fmt.Sprintf("cannot download PDF: %s", err),
|
||||
}), nil
|
||||
return agent.ResultJSON(
|
||||
downloadPDFResult{
|
||||
ErrorDetail: fmt.Sprintf("cannot download PDF: %s", err),
|
||||
},
|
||||
), nil
|
||||
}
|
||||
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return agent.ResultJSON(downloadPDFResult{
|
||||
ErrorDetail: fmt.Sprintf("PDF download returned status %d", resp.StatusCode),
|
||||
}), nil
|
||||
return agent.ResultJSON(
|
||||
downloadPDFResult{
|
||||
ErrorDetail: fmt.Sprintf("PDF download returned status %d", resp.StatusCode),
|
||||
},
|
||||
), nil
|
||||
}
|
||||
|
||||
// Read PDF into memory (max 20MB).
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 20*1024*1024))
|
||||
if err != nil {
|
||||
return agent.ResultJSON(downloadPDFResult{
|
||||
ErrorDetail: fmt.Sprintf("cannot read PDF body: %s", err),
|
||||
}), nil
|
||||
return agent.ResultJSON(
|
||||
downloadPDFResult{
|
||||
ErrorDetail: fmt.Sprintf("cannot read PDF body: %s", err),
|
||||
},
|
||||
), nil
|
||||
}
|
||||
|
||||
// Write to temp file for pdfcpu.
|
||||
tmpDir, err := os.MkdirTemp("", "pdf-extract-*")
|
||||
if err != nil {
|
||||
return agent.ResultJSON(downloadPDFResult{
|
||||
ErrorDetail: fmt.Sprintf("cannot create temp dir: %s", err),
|
||||
}), nil
|
||||
return agent.ResultJSON(
|
||||
downloadPDFResult{
|
||||
ErrorDetail: fmt.Sprintf("cannot create temp dir: %s", err),
|
||||
},
|
||||
), nil
|
||||
}
|
||||
|
||||
defer func() { _ = os.RemoveAll(tmpDir) }()
|
||||
|
||||
tmpFile := filepath.Join(tmpDir, "input.pdf")
|
||||
if err := os.WriteFile(tmpFile, body, 0o600); err != nil {
|
||||
return agent.ResultJSON(downloadPDFResult{
|
||||
ErrorDetail: fmt.Sprintf("cannot write temp file: %s", err),
|
||||
}), nil
|
||||
return agent.ResultJSON(
|
||||
downloadPDFResult{
|
||||
ErrorDetail: fmt.Sprintf("cannot write temp file: %s", err),
|
||||
},
|
||||
), nil
|
||||
}
|
||||
|
||||
// Get page count.
|
||||
@@ -111,24 +125,30 @@ func DownloadPDFTool() agent.Tool {
|
||||
|
||||
pageCount, err := api.PageCountFile(tmpFile)
|
||||
if err != nil {
|
||||
return agent.ResultJSON(downloadPDFResult{
|
||||
ErrorDetail: fmt.Sprintf("cannot read PDF: %s", err),
|
||||
}), nil
|
||||
return agent.ResultJSON(
|
||||
downloadPDFResult{
|
||||
ErrorDetail: fmt.Sprintf("cannot read PDF: %s", err),
|
||||
},
|
||||
), nil
|
||||
}
|
||||
|
||||
// Extract content to output dir.
|
||||
outDir := filepath.Join(tmpDir, "out")
|
||||
if err := os.MkdirAll(outDir, 0o700); err != nil {
|
||||
return agent.ResultJSON(downloadPDFResult{
|
||||
ErrorDetail: fmt.Sprintf("cannot create output dir: %s", err),
|
||||
}), nil
|
||||
return agent.ResultJSON(
|
||||
downloadPDFResult{
|
||||
ErrorDetail: fmt.Sprintf("cannot create output dir: %s", err),
|
||||
},
|
||||
), nil
|
||||
}
|
||||
|
||||
reader := bytes.NewReader(body)
|
||||
if err := api.ExtractContent(reader, outDir, "content", nil, conf); err != nil {
|
||||
return agent.ResultJSON(downloadPDFResult{
|
||||
ErrorDetail: fmt.Sprintf("cannot extract PDF content: %s", err),
|
||||
}), nil
|
||||
return agent.ResultJSON(
|
||||
downloadPDFResult{
|
||||
ErrorDetail: fmt.Sprintf("cannot extract PDF content: %s", err),
|
||||
},
|
||||
), nil
|
||||
}
|
||||
|
||||
// Read all extracted content files.
|
||||
@@ -154,10 +174,12 @@ func DownloadPDFTool() agent.Tool {
|
||||
text = text[:maxTextLength] + "\n[... truncated]"
|
||||
}
|
||||
|
||||
return agent.ResultJSON(downloadPDFResult{
|
||||
Text: text,
|
||||
PageCount: pageCount,
|
||||
}), nil
|
||||
return agent.ResultJSON(
|
||||
downloadPDFResult{
|
||||
Text: text,
|
||||
PageCount: pageCount,
|
||||
},
|
||||
), nil
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -46,37 +47,49 @@ func FetchRobotsTxtTool() agent.Tool {
|
||||
"Fetch and parse the robots.txt file for a domain. Returns sitemap URLs and disallowed paths, which can reveal hidden pages the crawler might miss.",
|
||||
func(ctx context.Context, p robotsParams) (agent.ToolResult, error) {
|
||||
if err := validatePublicDomain(p.Domain); err != nil {
|
||||
return agent.ResultJSON(robotsResult{
|
||||
Found: false,
|
||||
ErrorDetail: fmt.Sprintf("domain not allowed: %s", err),
|
||||
}), nil
|
||||
return agent.ResultJSON(
|
||||
robotsResult{
|
||||
Found: false,
|
||||
ErrorDetail: fmt.Sprintf("domain not allowed: %s", err),
|
||||
},
|
||||
), nil
|
||||
}
|
||||
|
||||
u := "https://" + p.Domain + "/robots.txt"
|
||||
u := &url.URL{
|
||||
Scheme: "https",
|
||||
Host: p.Domain,
|
||||
Path: "/robots.txt",
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
|
||||
if err != nil {
|
||||
return agent.ResultJSON(robotsResult{
|
||||
Found: false,
|
||||
ErrorDetail: fmt.Sprintf("cannot create request: %s", err),
|
||||
}), nil
|
||||
return agent.ResultJSON(
|
||||
robotsResult{
|
||||
Found: false,
|
||||
ErrorDetail: fmt.Sprintf("cannot create request: %s", err),
|
||||
},
|
||||
), nil
|
||||
}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return agent.ResultJSON(robotsResult{
|
||||
Found: false,
|
||||
ErrorDetail: fmt.Sprintf("cannot fetch robots.txt: %s", err),
|
||||
}), nil
|
||||
return agent.ResultJSON(
|
||||
robotsResult{
|
||||
Found: false,
|
||||
ErrorDetail: fmt.Sprintf("cannot fetch robots.txt: %s", err),
|
||||
},
|
||||
), nil
|
||||
}
|
||||
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return agent.ResultJSON(robotsResult{
|
||||
Found: false,
|
||||
ErrorDetail: fmt.Sprintf("robots.txt returned status %d", resp.StatusCode),
|
||||
}), nil
|
||||
return agent.ResultJSON(
|
||||
robotsResult{
|
||||
Found: false,
|
||||
ErrorDetail: fmt.Sprintf("robots.txt returned status %d", resp.StatusCode),
|
||||
},
|
||||
), nil
|
||||
}
|
||||
|
||||
var result robotsResult
|
||||
|
||||
@@ -52,35 +52,43 @@ func FetchSitemapTool() agent.Tool {
|
||||
"Fetch and parse a sitemap XML file. Returns discovered URLs which can reveal pages not linked from the main navigation (trust centers, legal docs, status pages).",
|
||||
func(ctx context.Context, p sitemapParams) (agent.ToolResult, error) {
|
||||
if err := validatePublicURL(p.URL); err != nil {
|
||||
return agent.ResultJSON(sitemapResult{
|
||||
Found: false,
|
||||
ErrorDetail: fmt.Sprintf("URL not allowed: %s", err),
|
||||
}), nil
|
||||
return agent.ResultJSON(
|
||||
sitemapResult{
|
||||
Found: false,
|
||||
ErrorDetail: fmt.Sprintf("URL not allowed: %s", err),
|
||||
},
|
||||
), nil
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, p.URL, nil)
|
||||
if err != nil {
|
||||
return agent.ResultJSON(sitemapResult{
|
||||
Found: false,
|
||||
ErrorDetail: fmt.Sprintf("cannot create request: %s", err),
|
||||
}), nil
|
||||
return agent.ResultJSON(
|
||||
sitemapResult{
|
||||
Found: false,
|
||||
ErrorDetail: fmt.Sprintf("cannot create request: %s", err),
|
||||
},
|
||||
), nil
|
||||
}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return agent.ResultJSON(sitemapResult{
|
||||
Found: false,
|
||||
ErrorDetail: fmt.Sprintf("cannot fetch sitemap: %s", err),
|
||||
}), nil
|
||||
return agent.ResultJSON(
|
||||
sitemapResult{
|
||||
Found: false,
|
||||
ErrorDetail: fmt.Sprintf("cannot fetch sitemap: %s", err),
|
||||
},
|
||||
), nil
|
||||
}
|
||||
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return agent.ResultJSON(sitemapResult{
|
||||
Found: false,
|
||||
ErrorDetail: fmt.Sprintf("sitemap returned status %d", resp.StatusCode),
|
||||
}), nil
|
||||
return agent.ResultJSON(
|
||||
sitemapResult{
|
||||
Found: false,
|
||||
ErrorDetail: fmt.Sprintf("sitemap returned status %d", resp.StatusCode),
|
||||
},
|
||||
), nil
|
||||
}
|
||||
|
||||
var reader io.Reader = resp.Body
|
||||
@@ -88,10 +96,12 @@ func FetchSitemapTool() agent.Tool {
|
||||
resp.Header.Get("Content-Encoding") == "gzip" {
|
||||
gz, err := gzip.NewReader(resp.Body)
|
||||
if err != nil {
|
||||
return agent.ResultJSON(sitemapResult{
|
||||
Found: false,
|
||||
ErrorDetail: fmt.Sprintf("cannot decompress gzipped sitemap: %s", err),
|
||||
}), nil
|
||||
return agent.ResultJSON(
|
||||
sitemapResult{
|
||||
Found: false,
|
||||
ErrorDetail: fmt.Sprintf("cannot decompress gzipped sitemap: %s", err),
|
||||
},
|
||||
), nil
|
||||
}
|
||||
|
||||
defer func() { _ = gz.Close() }()
|
||||
@@ -104,10 +114,12 @@ func FetchSitemapTool() agent.Tool {
|
||||
|
||||
urls, err := parseSitemapXML(reader)
|
||||
if err != nil {
|
||||
return agent.ResultJSON(sitemapResult{
|
||||
Found: false,
|
||||
ErrorDetail: fmt.Sprintf("cannot parse sitemap XML: %s", err),
|
||||
}), nil
|
||||
return agent.ResultJSON(
|
||||
sitemapResult{
|
||||
Found: false,
|
||||
ErrorDetail: fmt.Sprintf("cannot parse sitemap XML: %s", err),
|
||||
},
|
||||
), nil
|
||||
}
|
||||
|
||||
result := sitemapResult{
|
||||
|
||||
@@ -80,11 +80,13 @@ func NavigateToURLTool(b *Browser) agent.Tool {
|
||||
return agent.ResultError(b.classifyError(ctx, p.URL, err)), nil
|
||||
}
|
||||
|
||||
return agent.ResultJSON(navigateResult{
|
||||
Title: title,
|
||||
Description: description,
|
||||
FinalURL: finalURL,
|
||||
}), nil
|
||||
return agent.ResultJSON(
|
||||
navigateResult{
|
||||
Title: title,
|
||||
Description: description,
|
||||
FinalURL: finalURL,
|
||||
},
|
||||
), nil
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user