Add Firecrawl web search tool for tracker mapping

Firecrawl provides higher quality search results than SearXNG.
When configured (firecrawl-endpoint + firecrawl-api-key), the
tracker-mapping agent and search toolset prefer it over the
SearXNG backend. Also improves the tracker identification prompt
with multi-strategy search queries that leverage domain signals
and adapt to tracker type.

Signed-off-by: Émile Ré <emile@probo.com>

pkg/cookiebanner/prompts/tracker_identification.txt

@@ -17,7 +17,12 @@ Return a structured JSON response with:
 
 2. If search_tracker_patterns returns results with a third party name, use the search_third_parties tool to confirm the exact name in the database and get its category.
 
-3. Only use web_search as a last resort if the internal tools return nothing useful. Search for "what is cookie [name]" or "[name] cookie tracker".
+3. Only use web_search as a last resort if the internal tools return nothing useful. Try up to 3 different queries, adapting your strategy based on the available signals:
+   - If the tracker has a recognizable prefix or name, start with: "[prefix] cookie tracking" (e.g. "_ce.s cookie tracking").
+   - If observed domains are available and the name is opaque, search by domain instead: "[domain] cookies tracking privacy" (e.g. "clarity.ms cookies tracking privacy").
+   - For localStorage keys, include the type: "[name] localStorage tracking script".
+   - If the first query returns nothing useful, broaden: "[name] web tracker" or "site:[domain] cookie documentation".
+   - Stop searching once you get a confident match; do not exhaust all query slots if the first one succeeds.
 
 4. Common cookie naming conventions to recognize:
    - _ga*, _gid, _gat*: Google Analytics

pkg/cookiebanner/tracker_mapping_worker.go

@@ -51,9 +51,11 @@ type trackerMappingHandler struct {
 }
 
 type TrackerMappingConfig struct {
-	LLMClient      *llm.Client
-	Model          string
-	SearchEndpoint string
+	LLMClient         *llm.Client
+	Model             string
+	SearchEndpoint    string
+	FirecrawlEndpoint string
+	FirecrawlAPIKey   string
 }
 
 func NewTrackerMappingWorker(
@@ -68,13 +70,7 @@ func NewTrackerMappingWorker(
 	}
 
 	if cfg.LLMClient != nil {
-		h.agent = buildTrackerMappingAgent(
-			cfg.LLMClient,
-			cfg.Model,
-			cfg.SearchEndpoint,
-			pgClient,
-			logger,
-		)
+		h.agent = buildTrackerMappingAgent(cfg, pgClient, logger)
 	}
 
 	return worker.New(
@@ -86,9 +82,7 @@ func NewTrackerMappingWorker(
 }
 
 func buildTrackerMappingAgent(
-	llmClient *llm.Client,
-	model string,
-	searchEndpoint string,
+	cfg TrackerMappingConfig,
 	pgClient *pg.Client,
 	logger *log.Logger,
 ) *agent.Agent {
@@ -97,8 +91,10 @@ func buildTrackerMappingAgent(
 		searchThirdPartiesTool(pgClient),
 	}
 
-	if searchEndpoint != "" {
-		tools = append(tools, search.WebSearchTool(searchEndpoint))
+	if cfg.FirecrawlEndpoint != "" && cfg.FirecrawlAPIKey != "" {
+		tools = append(tools, search.FirecrawlSearchTool(cfg.FirecrawlEndpoint, cfg.FirecrawlAPIKey))
+	} else if cfg.SearchEndpoint != "" {
+		tools = append(tools, search.WebSearchTool(cfg.SearchEndpoint))
 	}
 
 	outputType, err := agent.NewOutputType[TrackerIdentification]("tracker_identification")
@@ -108,9 +104,9 @@ func buildTrackerMappingAgent(
 
 	return agent.New(
 		"tracker-mapping",
-		llmClient,
+		cfg.LLMClient,
 		agent.WithInstructions(trackerIdentificationPrompt),
-		agent.WithModel(model),
+		agent.WithModel(cfg.Model),
 		agent.WithTools(tools...),
 		agent.WithOutputType(outputType),
 		agent.WithMaxTurns(agentMaxTurns),