Add common third party enricher worker

Introduce a poll-based worker that fills the global common_third_parties catalog (URLs, headquarter address, legal name, certifications, logo) so each tenant no longer starts from sparse, name-only rows. Enrichment is requested at row creation by ResolveOrCreateCommonThirdParty; curated seed rows are not enqueued, to avoid a re-seed storm. The pipeline uses two specialized agents plus a deterministic logo step. Agent A (company profile) resolves legal name, headquarter address, and the canonical website over web search; its website and legal name feed Agent B and the logo step. Agent B (compliance docs) resolves the legal document URLs, trust/security/status pages, and certifications using the browser read-only toolset (gated on ChromeDPAddr) plus web search. The logo step restores pkg/webinspect as a pure deterministic package and stores the discovered icon in S3, linked via logo_file_id. Each agent returns per-field value/confidence/source_url. The worker writes a column only when confidence clears a configurable threshold and the field is not externally owned (seed or human), and always records full per-field provenance in a new enrichment JSONB column so re-runs fill only gaps and human edits are never clobbered. New bookkeeping columns (enrichment_requested_at, enrichment, enrichment_attempts) back the claim queue and stale recovery; agents run outside transactions and results persist in one final transaction. The worker is opt-in: it no-ops unless its agent provider is configured. Signed-off-by: Émile Ré <emile@probo.com>
2026-06-10 13:49:56 +02:00
parent 99d568d07d
commit 229c6b99c6
23 changed files with 2325 additions and 15 deletions
--- a/pkg/webinspect/parse.go
+++ b/pkg/webinspect/parse.go
@@ -0,0 +1,117 @@
+// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
+// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+// Package webinspect parses a web page's static HTML to extract metadata
+// from its <head> (icons/logos, meta tags, link relations). It is a pure,
+// deterministic helper: callers supply an http.Client (e.g. one with SSRF
+// protection) so the package itself makes no policy decisions about which
+// hosts are reachable. The logo discovery is reused by the common
+// third-party enricher to populate logo_file_id without an LLM.
+package webinspect
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+
+	"golang.org/x/net/html"
+)
+
+type PageInfo struct {
+	URL  *url.URL
+	Root *html.Node
+}
+
+func Parse(ctx context.Context, client *http.Client, websiteURL string) (*PageInfo, error) {
+	parsed, err := url.Parse(websiteURL)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse website URL: %w", err)
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, websiteURL, nil)
+	if err != nil {
+		return nil, fmt.Errorf("cannot create request: %w", err)
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("cannot fetch page: %w", err)
+	}
+	defer func() { _ = resp.Body.Close() }()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("cannot fetch page: status %d", resp.StatusCode)
+	}
+
+	const maxHTMLSize = 10 << 20 // 10 MiB
+	return ParseHTML(parsed, io.LimitReader(resp.Body, maxHTMLSize))
+}
+
+func ParseHTML(baseURL *url.URL, r io.Reader) (*PageInfo, error) {
+	root, err := html.Parse(r)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse HTML: %w", err)
+	}
+
+	return &PageInfo{URL: baseURL, Root: root}, nil
+}
+
+func (p *PageInfo) ResolveHref(href string) string {
+	ref, err := url.Parse(href)
+	if err != nil {
+		return href
+	}
+
+	return p.URL.ResolveReference(ref).String()
+}
+
+func findElement(n *html.Node, tag string) *html.Node {
+	if n.Type == html.ElementNode && n.Data == tag {
+		return n
+	}
+
+	for c := n.FirstChild; c != nil; c = c.NextSibling {
+		if found := findElement(c, tag); found != nil {
+			return found
+		}
+	}
+
+	return nil
+}
+
+func findAllIn(parent *html.Node, tag string) []*html.Node {
+	var nodes []*html.Node
+
+	for c := parent.FirstChild; c != nil; c = c.NextSibling {
+		if c.Type == html.ElementNode && c.Data == tag {
+			nodes = append(nodes, c)
+		}
+
+		nodes = append(nodes, findAllIn(c, tag)...)
+	}
+
+	return nodes
+}
+
+func attrVal(n *html.Node, key string) string {
+	for _, a := range n.Attr {
+		if a.Key == key {
+			return a.Val
+		}
+	}
+
+	return ""
+}