Guard vetting agent HTTP tools against SSRF

The third-party vetting agent runs a suite of HTTP "security" tools on
the internal worker network against a caller-supplied URL that is only
validated for length and charset, not host. Several tools reached
internal, loopback, and link-local addresses:

  - analyze_csp used a bare http.Client with no host validation, no
    redirect control, and no rebinding-safe transport, reflecting the
    target's CSP header back to the caller.
  - check_security_headers, fetch_robots_txt, and fetch_sitemap
    validated only the initial host, then followed 3xx redirects with an
    ordinary client, yielding full-read SSRF via a redirect to an
    internal address.
  - check_cors validated the URL but still dialed through an ordinary
    transport, leaving it exposed to DNS-rebinding TOCTOU.

Route every one of these clients through the house-standard
httpclient.DefaultPooledClient(WithSSRFProtection()), which rejects
dials to loopback, private, CGNAT, link-local, ULA, IPv4-mapped, and
reserved ranges on the resolved peer IP at connect time (defeating DNS
rebinding on every redirect hop) and refuses cross-origin redirects.
download_pdf moves onto the same client, and the now-unused local
netcheck.NewPinnedTransport is removed. analyze_csp also gains an
up-front ValidatePublicURL check for a clean early error and scheme
enforcement.

Signed-off-by: Sacha Al Himdani <sacha@probo.com>

pkg/agent/tools/security/cors.go

@@ -21,6 +21,7 @@ import (
 	"strings"
 	"time"
 
+	"go.gearno.de/kit/httpclient"
 	"go.probo.inc/probo/pkg/agent"
 	"go.probo.inc/probo/pkg/agent/tools/internal/netcheck"
 )
@@ -63,6 +64,12 @@ func splitTrimmed(s, sep string) []string {
 }
 
 func CheckCORSTool() agent.Tool {
+	client := httpclient.DefaultPooledClient(httpclient.WithSSRFProtection())
+	client.Timeout = 10 * time.Second
+	client.CheckRedirect = func(_ *http.Request, _ []*http.Request) error {
+		return http.ErrUseLastResponse
+	}
+
 	return agent.FunctionTool(
 		"check_cors",
 		"Send a CORS preflight (OPTIONS) request to a URL with a given Origin and analyze the Access-Control-* response headers, flagging wildcard origins and origin reflection.",
@@ -75,13 +82,6 @@ func CheckCORSTool() agent.Tool {
 				), nil
 			}
 
-			client := &http.Client{
-				Timeout: 10 * time.Second,
-				CheckRedirect: func(_ *http.Request, _ []*http.Request) error {
-					return http.ErrUseLastResponse
-				},
-			}
-
 			req, err := http.NewRequestWithContext(
 				ctx,
 				http.MethodOptions,

pkg/agent/tools/security/csp.go

@@ -21,7 +21,9 @@ import (
 	"strings"
 	"time"
 
+	"go.gearno.de/kit/httpclient"
 	"go.probo.inc/probo/pkg/agent"
+	"go.probo.inc/probo/pkg/agent/tools/internal/netcheck"
 )
 
 type (
@@ -73,11 +75,20 @@ func parseCSPDirectives(raw string) []cspDirective {
 }
 
 func AnalyzeCSPTool() agent.Tool {
+	client := httpclient.DefaultPooledClient(httpclient.WithSSRFProtection())
+	client.Timeout = 10 * time.Second
+
 	return agent.FunctionTool(
 		"analyze_csp",
 		"Analyze the Content-Security-Policy header for a URL, parsing directives and flagging unsafe patterns like unsafe-eval, unsafe-inline, and wildcard sources.",
 		func(ctx context.Context, p cspParams) (agent.ToolResult, error) {
-			client := &http.Client{Timeout: 10 * time.Second}
+			if err := netcheck.ValidatePublicURL(p.URL); err != nil {
+				return agent.ResultJSON(
+					cspResult{
+						ErrorDetail: fmt.Sprintf("URL not allowed: %s", err),
+					},
+				), nil
+			}
 
 			req, err := http.NewRequestWithContext(ctx, http.MethodGet, p.URL, nil)
 			if err != nil {

pkg/agent/tools/security/headers.go

@@ -22,6 +22,7 @@ import (
 	"strings"
 	"time"
 
+	"go.gearno.de/kit/httpclient"
 	"go.probo.inc/probo/pkg/agent"
 	"go.probo.inc/probo/pkg/agent/tools/internal/netcheck"
 )
@@ -75,6 +76,15 @@ func headersFromResponse(resp *http.Response) headersResult {
 }
 
 func CheckSecurityHeadersTool() agent.Tool {
+	client := httpclient.DefaultPooledClient(httpclient.WithSSRFProtection())
+	client.Timeout = 10 * time.Second
+	client.CheckRedirect = func(_ *http.Request, _ []*http.Request) error {
+		return http.ErrUseLastResponse
+	}
+
+	followClient := httpclient.DefaultPooledClient(httpclient.WithSSRFProtection())
+	followClient.Timeout = 10 * time.Second
+
 	return agent.FunctionTool(
 		"check_security_headers",
 		"Check security-related HTTP headers for a URL (HSTS, CSP, X-Frame-Options, X-Content-Type-Options, Referrer-Policy, Permissions-Policy, Cross-Origin-*-Policy). Also checks if HTTP redirects to HTTPS.",
@@ -87,13 +97,6 @@ func CheckSecurityHeadersTool() agent.Tool {
 				), nil
 			}
 
-			client := &http.Client{
-				Timeout: 10 * time.Second,
-				CheckRedirect: func(_ *http.Request, _ []*http.Request) error {
-					return http.ErrUseLastResponse
-				},
-			}
-
 			// First check the HTTP version to detect HTTP→HTTPS redirect.
 			redirectsToHTTPS := false
 
@@ -129,8 +132,6 @@ func CheckSecurityHeadersTool() agent.Tool {
 			httpsParsed.Scheme = "https"
 			httpsURL := httpsParsed.String()
 
-			followClient := &http.Client{Timeout: 10 * time.Second}
-
 			httpsReq, err := http.NewRequestWithContext(ctx, http.MethodGet, httpsURL, nil)
 			if err != nil {
 				return agent.ResultJSON(