Add vendor assessment agent
Signed-off-by: Aurélien Sibiril <81782+aureliensibiril@users.noreply.github.com>
This commit is contained in:
156
pkg/agent/tools/search/diff_documents.go
Normal file
156
pkg/agent/tools/search/diff_documents.go
Normal file
@@ -0,0 +1,156 @@
|
||||
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||
//
|
||||
// Permission to use, copy, modify, and/or distribute this software for any
|
||||
// purpose with or without fee is hereby granted, provided that the above
|
||||
// copyright notice and this permission notice appear in all copies.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||
// PERFORMANCE OF THIS SOFTWARE.
|
||||
|
||||
package search
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"go.probo.inc/probo/pkg/agent"
|
||||
)
|
||||
|
||||
type (
|
||||
diffParams struct {
|
||||
TextA string `json:"text_a" jsonschema:"The first document text to compare"`
|
||||
TextB string `json:"text_b" jsonschema:"The second document text to compare"`
|
||||
LabelA string `json:"label_a" jsonschema:"Label for the first document (e.g. 'current version')"`
|
||||
LabelB string `json:"label_b" jsonschema:"Label for the second document (e.g. 'archived version')"`
|
||||
}
|
||||
|
||||
diffResult struct {
|
||||
HasDifferences bool `json:"has_differences"`
|
||||
UnifiedDiff string `json:"unified_diff,omitempty"`
|
||||
AddedLines int `json:"added_lines"`
|
||||
RemovedLines int `json:"removed_lines"`
|
||||
ErrorDetail string `json:"error_detail,omitempty"`
|
||||
}
|
||||
)
|
||||
|
||||
const (
|
||||
maxDiffOutput = 16000
|
||||
)
|
||||
|
||||
func DiffDocumentsTool() agent.Tool {
|
||||
return agent.FunctionTool(
|
||||
"diff_documents",
|
||||
"Compare two document texts and return a unified diff showing the differences. Useful for comparing current vs. archived versions of privacy policies, terms of service, or other legal documents.",
|
||||
func(ctx context.Context, p diffParams) (agent.ToolResult, error) {
|
||||
labelA := p.LabelA
|
||||
if labelA == "" {
|
||||
labelA = "document_a"
|
||||
}
|
||||
labelB := p.LabelB
|
||||
if labelB == "" {
|
||||
labelB = "document_b"
|
||||
}
|
||||
|
||||
linesA := strings.Split(p.TextA, "\n")
|
||||
linesB := strings.Split(p.TextB, "\n")
|
||||
|
||||
diff := computeDiff(linesA, linesB, labelA, labelB)
|
||||
|
||||
if diff.tooLarge {
|
||||
return agent.ResultJSON(diffResult{
|
||||
HasDifferences: true,
|
||||
ErrorDetail: diff.output,
|
||||
}), nil
|
||||
}
|
||||
|
||||
result := diffResult{
|
||||
HasDifferences: diff.added > 0 || diff.removed > 0,
|
||||
AddedLines: diff.added,
|
||||
RemovedLines: diff.removed,
|
||||
}
|
||||
|
||||
if result.HasDifferences {
|
||||
output := diff.output
|
||||
if len(output) > maxDiffOutput {
|
||||
output = output[:maxDiffOutput] + "\n[... diff truncated]"
|
||||
}
|
||||
result.UnifiedDiff = output
|
||||
}
|
||||
|
||||
return agent.ResultJSON(result), nil
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
type (
|
||||
diffOutput struct {
|
||||
output string
|
||||
added int
|
||||
removed int
|
||||
tooLarge bool
|
||||
}
|
||||
)
|
||||
|
||||
func computeDiff(linesA, linesB []string, labelA, labelB string) diffOutput {
|
||||
// Simple line-by-line LCS-based diff.
|
||||
m, n := len(linesA), len(linesB)
|
||||
|
||||
// Build LCS table (bounded to prevent excessive memory for very large docs).
|
||||
if m > 5000 || n > 5000 {
|
||||
return diffOutput{
|
||||
output: "documents too large for detailed diff (limit 5000 lines per side)",
|
||||
tooLarge: true,
|
||||
}
|
||||
}
|
||||
|
||||
// LCS length table.
|
||||
dp := make([][]int, m+1)
|
||||
for i := range dp {
|
||||
dp[i] = make([]int, n+1)
|
||||
}
|
||||
for i := m - 1; i >= 0; i-- {
|
||||
for j := n - 1; j >= 0; j-- {
|
||||
if linesA[i] == linesB[j] {
|
||||
dp[i][j] = dp[i+1][j+1] + 1
|
||||
} else if dp[i+1][j] >= dp[i][j+1] {
|
||||
dp[i][j] = dp[i+1][j]
|
||||
} else {
|
||||
dp[i][j] = dp[i][j+1]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Walk the LCS table to produce diff hunks.
|
||||
var sb strings.Builder
|
||||
fmt.Fprintf(&sb, "--- %s\n+++ %s\n", labelA, labelB)
|
||||
|
||||
var added, removed int
|
||||
i, j := 0, 0
|
||||
for i < m || j < n {
|
||||
if i < m && j < n && linesA[i] == linesB[j] {
|
||||
// Context line — only emit near changes.
|
||||
i++
|
||||
j++
|
||||
} else if j < n && (i >= m || dp[i][j+1] >= dp[i+1][j]) {
|
||||
sb.WriteString("+ " + linesB[j] + "\n")
|
||||
added++
|
||||
j++
|
||||
} else if i < m {
|
||||
sb.WriteString("- " + linesA[i] + "\n")
|
||||
removed++
|
||||
i++
|
||||
}
|
||||
}
|
||||
|
||||
return diffOutput{
|
||||
output: sb.String(),
|
||||
added: added,
|
||||
removed: removed,
|
||||
}
|
||||
}
|
||||
203
pkg/agent/tools/search/diff_test.go
Normal file
203
pkg/agent/tools/search/diff_test.go
Normal file
@@ -0,0 +1,203 @@
|
||||
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||
//
|
||||
// Permission to use, copy, modify, and/or distribute this software for any
|
||||
// purpose with or without fee is hereby granted, provided that the above
|
||||
// copyright notice and this permission notice appear in all copies.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||
// PERFORMANCE OF THIS SOFTWARE.
|
||||
|
||||
package search
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestComputeDiff(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
t.Run(
|
||||
"identical documents have no changes",
|
||||
func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
lines := []string{"line one", "line two", "line three"}
|
||||
diff := computeDiff(lines, lines, "a", "b")
|
||||
|
||||
assert.Equal(t, 0, diff.added)
|
||||
assert.Equal(t, 0, diff.removed)
|
||||
},
|
||||
)
|
||||
|
||||
t.Run(
|
||||
"completely different documents",
|
||||
func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
linesA := []string{"alpha", "beta"}
|
||||
linesB := []string{"gamma", "delta"}
|
||||
diff := computeDiff(linesA, linesB, "a", "b")
|
||||
|
||||
assert.Equal(t, 2, diff.added)
|
||||
assert.Equal(t, 2, diff.removed)
|
||||
assert.Contains(t, diff.output, "- alpha")
|
||||
assert.Contains(t, diff.output, "- beta")
|
||||
assert.Contains(t, diff.output, "+ gamma")
|
||||
assert.Contains(t, diff.output, "+ delta")
|
||||
},
|
||||
)
|
||||
|
||||
t.Run(
|
||||
"added lines only",
|
||||
func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
linesA := []string{"line one"}
|
||||
linesB := []string{"line one", "line two", "line three"}
|
||||
diff := computeDiff(linesA, linesB, "a", "b")
|
||||
|
||||
assert.Equal(t, 2, diff.added)
|
||||
assert.Equal(t, 0, diff.removed)
|
||||
assert.Contains(t, diff.output, "+ line two")
|
||||
assert.Contains(t, diff.output, "+ line three")
|
||||
},
|
||||
)
|
||||
|
||||
t.Run(
|
||||
"removed lines only",
|
||||
func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
linesA := []string{"line one", "line two", "line three"}
|
||||
linesB := []string{"line one"}
|
||||
diff := computeDiff(linesA, linesB, "a", "b")
|
||||
|
||||
assert.Equal(t, 0, diff.added)
|
||||
assert.Equal(t, 2, diff.removed)
|
||||
assert.Contains(t, diff.output, "- line two")
|
||||
assert.Contains(t, diff.output, "- line three")
|
||||
},
|
||||
)
|
||||
|
||||
t.Run(
|
||||
"mixed changes",
|
||||
func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
linesA := []string{"keep", "remove me", "also keep"}
|
||||
linesB := []string{"keep", "add me", "also keep"}
|
||||
diff := computeDiff(linesA, linesB, "a", "b")
|
||||
|
||||
assert.Equal(t, 1, diff.added)
|
||||
assert.Equal(t, 1, diff.removed)
|
||||
assert.Contains(t, diff.output, "- remove me")
|
||||
assert.Contains(t, diff.output, "+ add me")
|
||||
},
|
||||
)
|
||||
|
||||
t.Run(
|
||||
"both inputs empty",
|
||||
func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
diff := computeDiff([]string{}, []string{}, "a", "b")
|
||||
|
||||
assert.Equal(t, 0, diff.added)
|
||||
assert.Equal(t, 0, diff.removed)
|
||||
},
|
||||
)
|
||||
|
||||
t.Run(
|
||||
"first input empty",
|
||||
func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
linesB := []string{"new line"}
|
||||
diff := computeDiff([]string{}, linesB, "a", "b")
|
||||
|
||||
assert.Equal(t, 1, diff.added)
|
||||
assert.Equal(t, 0, diff.removed)
|
||||
assert.Contains(t, diff.output, "+ new line")
|
||||
},
|
||||
)
|
||||
|
||||
t.Run(
|
||||
"second input empty",
|
||||
func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
linesA := []string{"old line"}
|
||||
diff := computeDiff(linesA, []string{}, "a", "b")
|
||||
|
||||
assert.Equal(t, 0, diff.added)
|
||||
assert.Equal(t, 1, diff.removed)
|
||||
assert.Contains(t, diff.output, "- old line")
|
||||
},
|
||||
)
|
||||
|
||||
t.Run(
|
||||
"single line documents identical",
|
||||
func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
diff := computeDiff([]string{"same"}, []string{"same"}, "a", "b")
|
||||
|
||||
assert.Equal(t, 0, diff.added)
|
||||
assert.Equal(t, 0, diff.removed)
|
||||
},
|
||||
)
|
||||
|
||||
t.Run(
|
||||
"single line documents different",
|
||||
func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
diff := computeDiff([]string{"old"}, []string{"new"}, "a", "b")
|
||||
|
||||
assert.Equal(t, 1, diff.added)
|
||||
assert.Equal(t, 1, diff.removed)
|
||||
},
|
||||
)
|
||||
|
||||
t.Run(
|
||||
"output contains labels",
|
||||
func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
diff := computeDiff(
|
||||
[]string{"a"},
|
||||
[]string{"b"},
|
||||
"current version",
|
||||
"archived version",
|
||||
)
|
||||
|
||||
assert.True(t, strings.HasPrefix(diff.output, "--- current version\n+++ archived version\n"))
|
||||
},
|
||||
)
|
||||
|
||||
t.Run(
|
||||
"documents too large returns bounded message",
|
||||
func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
large := make([]string, 5001)
|
||||
for i := range large {
|
||||
large[i] = "line"
|
||||
}
|
||||
|
||||
diff := computeDiff(large, []string{"small"}, "a", "b")
|
||||
|
||||
assert.Equal(t, 0, diff.added)
|
||||
assert.Equal(t, 0, diff.removed)
|
||||
assert.Contains(t, diff.output, "too large")
|
||||
},
|
||||
)
|
||||
}
|
||||
164
pkg/agent/tools/search/government_db.go
Normal file
164
pkg/agent/tools/search/government_db.go
Normal file
@@ -0,0 +1,164 @@
|
||||
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||
//
|
||||
// Permission to use, copy, modify, and/or distribute this software for any
|
||||
// purpose with or without fee is hereby granted, provided that the above
|
||||
// copyright notice and this permission notice appear in all copies.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||
// PERFORMANCE OF THIS SOFTWARE.
|
||||
|
||||
package search
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"go.probo.inc/probo/pkg/agent"
|
||||
)
|
||||
|
||||
type (
|
||||
govDBParams struct {
|
||||
CompanyName string `json:"company_name" jsonschema:"The company name to search for in government databases"`
|
||||
Domain string `json:"domain" jsonschema:"The company domain for additional search context (optional)"`
|
||||
}
|
||||
|
||||
govDBEntry struct {
|
||||
Source string `json:"source"`
|
||||
Title string `json:"title"`
|
||||
URL string `json:"url"`
|
||||
Snippet string `json:"snippet,omitempty"`
|
||||
}
|
||||
|
||||
govDBResult struct {
|
||||
SECFilings []govDBEntry `json:"sec_filings,omitempty"`
|
||||
FTCActions []govDBEntry `json:"ftc_actions,omitempty"`
|
||||
GDPRFines []govDBEntry `json:"gdpr_fines,omitempty"`
|
||||
OtherActions []govDBEntry `json:"other_regulatory_actions,omitempty"`
|
||||
ErrorDetail string `json:"error_detail,omitempty"`
|
||||
}
|
||||
)
|
||||
|
||||
func CheckGovernmentDBTool(searchEndpoint string) agent.Tool {
|
||||
client := &http.Client{Timeout: 15 * time.Second}
|
||||
|
||||
return agent.FunctionTool(
|
||||
"check_government_databases",
|
||||
"Search government and regulatory databases for enforcement actions, SEC filings, FTC actions, and GDPR fines related to a company.",
|
||||
func(ctx context.Context, p govDBParams) (agent.ToolResult, error) {
|
||||
var result govDBResult
|
||||
|
||||
name := p.CompanyName
|
||||
if p.Domain != "" {
|
||||
name = name + " " + p.Domain
|
||||
}
|
||||
|
||||
type searchSpec struct {
|
||||
query string
|
||||
source string
|
||||
target *[]govDBEntry
|
||||
}
|
||||
|
||||
searches := []searchSpec{
|
||||
{
|
||||
query: fmt.Sprintf(`site:sec.gov "%s"`, p.CompanyName),
|
||||
source: "SEC",
|
||||
target: &result.SECFilings,
|
||||
},
|
||||
{
|
||||
query: fmt.Sprintf(`site:ftc.gov "%s"`, p.CompanyName),
|
||||
source: "FTC",
|
||||
target: &result.FTCActions,
|
||||
},
|
||||
{
|
||||
query: fmt.Sprintf(`site:enforcementtracker.com "%s"`, p.CompanyName),
|
||||
source: "GDPR Enforcement Tracker",
|
||||
target: &result.GDPRFines,
|
||||
},
|
||||
{
|
||||
query: fmt.Sprintf(`"%s" regulatory action OR enforcement OR fine OR penalty OR sanction`, name),
|
||||
source: "General",
|
||||
target: &result.OtherActions,
|
||||
},
|
||||
}
|
||||
|
||||
for _, s := range searches {
|
||||
entries, err := searxngSearch(ctx, client, searchEndpoint, s.query, 3)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, e := range entries {
|
||||
*s.target = append(*s.target, govDBEntry{
|
||||
Source: s.source,
|
||||
Title: e.Title,
|
||||
URL: e.URL,
|
||||
Snippet: e.Snippet,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return agent.ResultJSON(result), nil
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func searxngSearch(ctx context.Context, client *http.Client, endpoint, query string, maxResults int) ([]searchResult, error) {
|
||||
u, err := url.Parse(endpoint + "/search")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
q := u.Query()
|
||||
q.Set("q", query)
|
||||
q.Set("format", "json")
|
||||
q.Set("categories", "general")
|
||||
u.RawQuery = q.Encode()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("search returned status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
var searxResp searxngResponse
|
||||
if err := json.Unmarshal(body, &searxResp); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
results := make([]searchResult, 0, maxResults)
|
||||
for i, r := range searxResp.Results {
|
||||
if i >= maxResults {
|
||||
break
|
||||
}
|
||||
results = append(results, searchResult{
|
||||
Title: r.Title,
|
||||
URL: r.URL,
|
||||
Snippet: r.Content,
|
||||
})
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
38
pkg/agent/tools/search/search.go
Normal file
38
pkg/agent/tools/search/search.go
Normal file
@@ -0,0 +1,38 @@
|
||||
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||
//
|
||||
// Permission to use, copy, modify, and/or distribute this software for any
|
||||
// purpose with or without fee is hereby granted, provided that the above
|
||||
// copyright notice and this permission notice appear in all copies.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||
// PERFORMANCE OF THIS SOFTWARE.
|
||||
|
||||
package search
|
||||
|
||||
import (
|
||||
"go.probo.inc/probo/pkg/agent"
|
||||
)
|
||||
|
||||
// Toolset provides web search tools.
|
||||
type Toolset struct {
|
||||
endpoint string
|
||||
}
|
||||
|
||||
// NewToolset creates a search toolset with the given SearXNG endpoint.
|
||||
func NewToolset(endpoint string) *Toolset {
|
||||
return &Toolset{endpoint: endpoint}
|
||||
}
|
||||
|
||||
func (t *Toolset) Tools() []agent.Tool {
|
||||
return []agent.Tool{
|
||||
WebSearchTool(t.endpoint),
|
||||
CheckGovernmentDBTool(t.endpoint),
|
||||
CheckWaybackTool(),
|
||||
DiffDocumentsTool(),
|
||||
}
|
||||
}
|
||||
147
pkg/agent/tools/search/wayback.go
Normal file
147
pkg/agent/tools/search/wayback.go
Normal file
@@ -0,0 +1,147 @@
|
||||
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||
//
|
||||
// Permission to use, copy, modify, and/or distribute this software for any
|
||||
// purpose with or without fee is hereby granted, provided that the above
|
||||
// copyright notice and this permission notice appear in all copies.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||
// PERFORMANCE OF THIS SOFTWARE.
|
||||
|
||||
package search
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"go.probo.inc/probo/pkg/agent"
|
||||
)
|
||||
|
||||
type (
|
||||
waybackParams struct {
|
||||
URL string `json:"url" jsonschema:"The URL to check in the Wayback Machine (e.g. https://example.com/privacy)"`
|
||||
}
|
||||
|
||||
waybackSnapshot struct {
|
||||
Timestamp string `json:"timestamp"`
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
waybackResult struct {
|
||||
Available bool `json:"available"`
|
||||
OldestSnapshot *waybackSnapshot `json:"oldest_snapshot,omitempty"`
|
||||
NewestSnapshot *waybackSnapshot `json:"newest_snapshot,omitempty"`
|
||||
ErrorDetail string `json:"error_detail,omitempty"`
|
||||
}
|
||||
|
||||
waybackAvailabilityResponse struct {
|
||||
ArchivedSnapshots struct {
|
||||
Closest struct {
|
||||
Available bool `json:"available"`
|
||||
URL string `json:"url"`
|
||||
Timestamp string `json:"timestamp"`
|
||||
} `json:"closest"`
|
||||
} `json:"archived_snapshots"`
|
||||
}
|
||||
|
||||
waybackCDXResponse = [][]string
|
||||
)
|
||||
|
||||
func CheckWaybackTool() agent.Tool {
|
||||
client := &http.Client{Timeout: 15 * time.Second}
|
||||
|
||||
return agent.FunctionTool(
|
||||
"check_wayback",
|
||||
"Check the Internet Archive Wayback Machine for archived versions of a URL. Useful for detecting changes in privacy policies, trust pages, or terms of service over time.",
|
||||
func(ctx context.Context, p waybackParams) (agent.ToolResult, error) {
|
||||
var result waybackResult
|
||||
|
||||
// Check availability.
|
||||
availURL := "https://archive.org/wayback/available?url=" + url.QueryEscape(p.URL)
|
||||
body, err := httpGet(ctx, client, availURL)
|
||||
if err != nil {
|
||||
result.ErrorDetail = fmt.Sprintf("cannot check Wayback Machine availability: %s", err)
|
||||
return agent.ResultJSON(result), nil
|
||||
}
|
||||
|
||||
var avail waybackAvailabilityResponse
|
||||
if err := json.Unmarshal(body, &avail); err == nil {
|
||||
result.Available = avail.ArchivedSnapshots.Closest.Available
|
||||
}
|
||||
|
||||
if !result.Available {
|
||||
return agent.ResultJSON(result), nil
|
||||
}
|
||||
|
||||
// Get oldest snapshot.
|
||||
oldestURL := fmt.Sprintf(
|
||||
"https://web.archive.org/cdx/search/cdx?url=%s&output=json&fl=timestamp,original&limit=1",
|
||||
url.QueryEscape(p.URL),
|
||||
)
|
||||
if body, err := httpGet(ctx, client, oldestURL); err == nil {
|
||||
if snap := parseCDXSnapshot(body); snap != nil {
|
||||
result.OldestSnapshot = snap
|
||||
}
|
||||
}
|
||||
|
||||
// Get newest snapshot.
|
||||
newestURL := fmt.Sprintf(
|
||||
"https://web.archive.org/cdx/search/cdx?url=%s&output=json&fl=timestamp,original&limit=1&sort=reverse",
|
||||
url.QueryEscape(p.URL),
|
||||
)
|
||||
if body, err := httpGet(ctx, client, newestURL); err == nil {
|
||||
if snap := parseCDXSnapshot(body); snap != nil {
|
||||
result.NewestSnapshot = snap
|
||||
}
|
||||
}
|
||||
|
||||
return agent.ResultJSON(result), nil
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func httpGet(ctx context.Context, client *http.Client, rawURL string) ([]byte, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
return io.ReadAll(io.LimitReader(resp.Body, 1*1024*1024))
|
||||
}
|
||||
|
||||
func parseCDXSnapshot(body []byte) *waybackSnapshot {
|
||||
var rows waybackCDXResponse
|
||||
if err := json.Unmarshal(body, &rows); err != nil || len(rows) < 2 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// First row is headers ["timestamp", "original"], data starts at row 1.
|
||||
row := rows[1]
|
||||
if len(row) < 2 {
|
||||
return nil
|
||||
}
|
||||
|
||||
return &waybackSnapshot{
|
||||
Timestamp: row[0],
|
||||
URL: row[1],
|
||||
}
|
||||
}
|
||||
109
pkg/agent/tools/search/wayback_test.go
Normal file
109
pkg/agent/tools/search/wayback_test.go
Normal file
@@ -0,0 +1,109 @@
|
||||
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||
//
|
||||
// Permission to use, copy, modify, and/or distribute this software for any
|
||||
// purpose with or without fee is hereby granted, provided that the above
|
||||
// copyright notice and this permission notice appear in all copies.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||
// PERFORMANCE OF THIS SOFTWARE.
|
||||
|
||||
package search
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestParseCDXSnapshot(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
t.Run(
|
||||
"valid JSON array response",
|
||||
func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
body := []byte(`[["timestamp","original"],["20200115120000","https://example.com/privacy"]]`)
|
||||
|
||||
snap := parseCDXSnapshot(body)
|
||||
|
||||
require.NotNil(t, snap)
|
||||
assert.Equal(t, "20200115120000", snap.Timestamp)
|
||||
assert.Equal(t, "https://example.com/privacy", snap.URL)
|
||||
},
|
||||
)
|
||||
|
||||
t.Run(
|
||||
"empty array returns nil",
|
||||
func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
body := []byte(`[]`)
|
||||
|
||||
assert.Nil(t, parseCDXSnapshot(body))
|
||||
},
|
||||
)
|
||||
|
||||
t.Run(
|
||||
"single row header only returns nil",
|
||||
func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
body := []byte(`[["timestamp","original"]]`)
|
||||
|
||||
assert.Nil(t, parseCDXSnapshot(body))
|
||||
},
|
||||
)
|
||||
|
||||
t.Run(
|
||||
"malformed JSON returns nil",
|
||||
func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
body := []byte(`not valid json`)
|
||||
|
||||
assert.Nil(t, parseCDXSnapshot(body))
|
||||
},
|
||||
)
|
||||
|
||||
t.Run(
|
||||
"data row with insufficient fields returns nil",
|
||||
func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
body := []byte(`[["timestamp","original"],["20200115120000"]]`)
|
||||
|
||||
assert.Nil(t, parseCDXSnapshot(body))
|
||||
},
|
||||
)
|
||||
|
||||
t.Run(
|
||||
"empty body returns nil",
|
||||
func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
assert.Nil(t, parseCDXSnapshot([]byte{}))
|
||||
},
|
||||
)
|
||||
|
||||
t.Run(
|
||||
"response with extra fields uses first two",
|
||||
func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
body := []byte(`[["timestamp","original","extra"],["20210601000000","https://example.com/tos","200"]]`)
|
||||
|
||||
snap := parseCDXSnapshot(body)
|
||||
|
||||
require.NotNil(t, snap)
|
||||
assert.Equal(t, "20210601000000", snap.Timestamp)
|
||||
assert.Equal(t, "https://example.com/tos", snap.URL)
|
||||
},
|
||||
)
|
||||
}
|
||||
74
pkg/agent/tools/search/web_search.go
Normal file
74
pkg/agent/tools/search/web_search.go
Normal file
@@ -0,0 +1,74 @@
|
||||
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||
//
|
||||
// Permission to use, copy, modify, and/or distribute this software for any
|
||||
// purpose with or without fee is hereby granted, provided that the above
|
||||
// copyright notice and this permission notice appear in all copies.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||
// PERFORMANCE OF THIS SOFTWARE.
|
||||
|
||||
package search
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"go.probo.inc/probo/pkg/agent"
|
||||
)
|
||||
|
||||
type (
|
||||
searchParams struct {
|
||||
Query string `json:"query" jsonschema:"The search query to execute"`
|
||||
MaxResults int `json:"max_results" jsonschema:"Maximum number of results to return (default 5, max 10)"`
|
||||
}
|
||||
|
||||
searchResult struct {
|
||||
Title string `json:"title"`
|
||||
URL string `json:"url"`
|
||||
Snippet string `json:"snippet"`
|
||||
}
|
||||
|
||||
searxngResponse struct {
|
||||
Results []searxngResult `json:"results"`
|
||||
}
|
||||
|
||||
searxngResult struct {
|
||||
Title string `json:"title"`
|
||||
URL string `json:"url"`
|
||||
Content string `json:"content"`
|
||||
}
|
||||
)
|
||||
|
||||
// WebSearchTool creates a tool that searches the web using a SearXNG instance.
|
||||
// The endpoint should be the base URL of the SearXNG instance (e.g.
|
||||
// "http://localhost:8888").
|
||||
func WebSearchTool(endpoint string) agent.Tool {
|
||||
client := &http.Client{Timeout: 15 * time.Second}
|
||||
|
||||
return agent.FunctionTool(
|
||||
"web_search",
|
||||
"Search the web for information about a topic. Returns a list of results with title, URL, and snippet. Use this to find news, reviews, breach reports, regulatory actions, and other external information about a vendor.",
|
||||
func(ctx context.Context, p searchParams) (agent.ToolResult, error) {
|
||||
maxResults := p.MaxResults
|
||||
if maxResults <= 0 {
|
||||
maxResults = 5
|
||||
}
|
||||
if maxResults > 10 {
|
||||
maxResults = 10
|
||||
}
|
||||
|
||||
results, err := searxngSearch(ctx, client, endpoint, p.Query, maxResults)
|
||||
if err != nil {
|
||||
return agent.ResultErrorf("search request failed: %s", err), nil
|
||||
}
|
||||
|
||||
return agent.ResultJSON(results), nil
|
||||
},
|
||||
)
|
||||
}
|
||||
Reference in New Issue
Block a user