Move tracker mapping worker to cookiebanner and rename worker files

Move tracker_mapping_worker.go from pkg/probo to pkg/cookiebanner and rename worker.go to pattern_analysis_worker.go to reflect the worker name. Signed-off-by: Émile Ré <emile@probo.com>
2026-05-15 11:30:23 +04:00
parent 57f39163d1
commit 81707fe429
4 changed files with 2 additions and 2 deletions
--- a/pkg/cookiebanner/pattern_analysis_worker.go
+++ b/pkg/cookiebanner/pattern_analysis_worker.go
@@ -0,0 +1,593 @@
+// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
+// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+package cookiebanner
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+
+	"go.gearno.de/kit/log"
+	"go.gearno.de/kit/pg"
+	"go.gearno.de/kit/worker"
+	"go.probo.inc/probo/pkg/coredata"
+	"go.probo.inc/probo/pkg/gid"
+)
+
+const patternMergeThreshold = 3
+
+// durationUnits mirrors the snap table from cookie-utils.ts. The same
+// tracker observed across different clients can have jitter in its
+// max-age (e.g. an "Expires" header computed from Date.now() yields
+// slightly different seconds each time). Snapping to the nearest
+// human-meaningful unit absorbs that jitter so the patterns still
+// merge. This is compliant because the resulting bucket matches the
+// duration shown to end users in the cookie banner — two cookies that
+// display the same human-readable lifetime will merge, two that
+// display differently will not.
+var durationUnits = [...]struct {
+	seconds int
+	snap    int
+}{
+	{365 * 24 * 3600, 21 * 24 * 3600}, // years, snap +-21 days
+	{30 * 24 * 3600, 2 * 24 * 3600},   // months, snap +-2 days
+	{7 * 24 * 3600, 12 * 3600},        // weeks, snap +-12 hours
+	{24 * 3600, 2 * 3600},             // days, snap +-2 hours
+	{3600, 5 * 60},                    // hours, snap +-5 minutes
+	{60, 5},                           // minutes, snap +-5 seconds
+	{1, 0},                            // seconds, no snap
+}
+
+func durationBucket(maxAge *int) int {
+	if maxAge == nil || *maxAge <= 0 {
+		return -1
+	}
+
+	remaining := *maxAge
+	total := 0
+	for _, u := range durationUnits {
+		if remaining >= u.seconds-u.snap {
+			count := remaining / u.seconds
+			leftover := remaining - count*u.seconds
+			if leftover >= u.seconds-u.snap {
+				count++
+				remaining = 0
+			} else if leftover <= u.snap {
+				remaining = 0
+			} else {
+				remaining = leftover
+			}
+			total += count * u.seconds
+		}
+	}
+	return total
+}
+
+type patternAnalysisHandler struct {
+	svc    *Service
+	pg     *pg.Client
+	logger *log.Logger
+}
+
+func NewPatternAnalysisWorker(
+	svc *Service,
+	pgClient *pg.Client,
+	logger *log.Logger,
+	opts ...worker.Option,
+) *worker.Worker[coredata.CookieBanner] {
+	h := &patternAnalysisHandler{
+		svc:    svc,
+		pg:     pgClient,
+		logger: logger,
+	}
+
+	return worker.New(
+		"tracker-pattern-analysis-worker",
+		h,
+		logger,
+		opts...,
+	)
+}
+
+func (h *patternAnalysisHandler) Claim(ctx context.Context) (coredata.CookieBanner, error) {
+	var banner coredata.CookieBanner
+
+	if err := h.pg.WithTx(
+		ctx,
+		func(ctx context.Context, tx pg.Tx) error {
+			if err := banner.LoadNextForPatternAnalysisForUpdateSkipLocked(ctx, tx); err != nil {
+				return err
+			}
+
+			return banner.ClearPatternAnalysisRequestedAt(ctx, tx)
+		},
+	); err != nil {
+		if errors.Is(err, coredata.ErrResourceNotFound) {
+			return coredata.CookieBanner{}, worker.ErrNoTask
+		}
+		return coredata.CookieBanner{}, fmt.Errorf("cannot claim pattern analysis task: %w", err)
+	}
+
+	return banner, nil
+}
+
+func (h *patternAnalysisHandler) Process(ctx context.Context, banner coredata.CookieBanner) error {
+	return h.pg.WithTx(
+		ctx,
+		func(ctx context.Context, tx pg.Tx) error {
+			scope := coredata.NewScopeFromObjectID(banner.ID)
+
+			var uncategorised coredata.CookieCategory
+			hasUncategorised := true
+			if err := uncategorised.LoadUncategorisedByCookieBannerID(ctx, tx, scope, banner.ID); err != nil {
+				if !errors.Is(err, coredata.ErrResourceNotFound) {
+					return fmt.Errorf("cannot load uncategorised category: %w", err)
+				}
+				hasUncategorised = false
+			}
+
+			var exactPatterns coredata.TrackerPatterns
+			if err := exactPatterns.LoadAllByCookieBannerID(
+				ctx,
+				tx,
+				scope,
+				banner.ID,
+				coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeExact), nil, new(false)),
+				nil,
+			); err != nil {
+				return fmt.Errorf("cannot load exact patterns: %w", err)
+			}
+
+			mergeGroups := findMergeGroups(exactPatterns, patternMergeThreshold)
+
+			consentChanged := false
+			for key, group := range mergeGroups {
+				var maxAge *int
+				if key.durationBucket >= 0 {
+					v := key.durationBucket
+					maxAge = &v
+				}
+
+				source := bestSource(group)
+
+				globPattern := &coredata.TrackerPattern{
+					ID:               gid.New(banner.ID.TenantID(), coredata.TrackerPatternEntityType),
+					OrganizationID:   group[0].OrganizationID,
+					CookieBannerID:   banner.ID,
+					CookieCategoryID: key.categoryID,
+					TrackerType:      key.trackerType,
+					Pattern:          key.template,
+					MatchType:        coredata.TrackerPatternMatchTypeGlob,
+					DisplayName:      key.template,
+					MaxAgeSeconds:    maxAge,
+					Description:      "",
+					Source:           source,
+					CreatedAt:        time.Now(),
+					UpdatedAt:        time.Now(),
+				}
+
+				inserted, err := globPattern.InsertIfNotExists(ctx, tx, scope)
+				if err != nil {
+					return fmt.Errorf("cannot insert glob pattern %q: %w", key.template, err)
+				}
+				if !inserted {
+					if err := globPattern.LoadByBannerIDTypeAndPattern(ctx, tx, scope, banner.ID, key.trackerType, key.template, maxAge); err != nil {
+						return fmt.Errorf("cannot load existing glob pattern %q: %w", key.template, err)
+					}
+
+					if globPattern.CookieCategoryID != key.categoryID || globPattern.MatchType != coredata.TrackerPatternMatchTypeGlob {
+						continue
+					}
+				}
+
+				for _, exactPattern := range group {
+					var trackers coredata.DetectedTrackers
+					if err := trackers.RelinkByTrackerPatternID(ctx, tx, scope, exactPattern.ID, globPattern.ID); err != nil {
+						return fmt.Errorf("cannot relink detected trackers from pattern %q: %w", exactPattern.Pattern, err)
+					}
+
+					if err := exactPattern.Delete(ctx, tx, scope); err != nil {
+						return fmt.Errorf("cannot delete orphaned exact pattern %q: %w", exactPattern.Pattern, err)
+					}
+				}
+
+				if !hasUncategorised || key.categoryID != uncategorised.ID {
+					consentChanged = true
+				}
+
+				h.logger.InfoCtx(
+					ctx,
+					"merged exact patterns into glob pattern",
+					log.String("template", key.template),
+					log.Int("count", len(group)),
+					log.String("banner_id", banner.ID.String()),
+				)
+			}
+
+			if _, err := h.adoptUncategorisedPatterns(ctx, tx, scope, banner); err != nil {
+				return fmt.Errorf("cannot adopt uncategorised patterns: %w", err)
+			}
+
+			var patterns coredata.TrackerPatterns
+			if err := patterns.RefreshLastMatchedAtByCookieBannerID(ctx, tx, scope, banner.ID); err != nil {
+				return fmt.Errorf("cannot refresh last_matched_at: %w", err)
+			}
+
+			if consentChanged {
+				if _, err := h.svc.ensureDraftVersionForBanner(ctx, tx, scope, banner.ID); err != nil {
+					return fmt.Errorf("cannot ensure draft version: %w", err)
+				}
+			}
+
+			return nil
+		},
+	)
+}
+
+type mergeGroupKey struct {
+	categoryID     gid.GID
+	trackerType    coredata.TrackerType
+	template       string
+	durationBucket int
+}
+
+func findMergeGroups(
+	patterns coredata.TrackerPatterns,
+	threshold int,
+) map[mergeGroupKey][]*coredata.TrackerPattern {
+	type memberKey struct {
+		groupKey mergeGroupKey
+		pattern  *coredata.TrackerPattern
+	}
+
+	templateCounts := make(map[mergeGroupKey][]*coredata.TrackerPattern)
+	heuristicKeys := make(map[mergeGroupKey]bool)
+	seen := make(map[memberKey]bool)
+
+	for _, p := range patterns {
+		bucket := durationBucket(p.MaxAgeSeconds)
+
+		if tmpl, ok := heuristicTemplate(p.Pattern); ok {
+			key := mergeGroupKey{categoryID: p.CookieCategoryID, trackerType: p.TrackerType, template: tmpl, durationBucket: bucket}
+			mk := memberKey{key, p}
+			if !seen[mk] {
+				seen[mk] = true
+				templateCounts[key] = append(templateCounts[key], p)
+			}
+			heuristicKeys[key] = true
+		}
+
+		for _, tmpl := range templateCandidates(p.Pattern) {
+			key := mergeGroupKey{categoryID: p.CookieCategoryID, trackerType: p.TrackerType, template: tmpl, durationBucket: bucket}
+			mk := memberKey{key, p}
+			if !seen[mk] {
+				seen[mk] = true
+				templateCounts[key] = append(templateCounts[key], p)
+			}
+		}
+	}
+
+	type candidate struct {
+		key         mergeGroupKey
+		fixedChars  int
+		isHeuristic bool
+		patterns    []*coredata.TrackerPattern
+	}
+
+	var candidates []candidate
+	for key, pats := range templateCounts {
+		isH := heuristicKeys[key]
+		effectiveThreshold := threshold
+		if isH {
+			effectiveThreshold = 1
+		}
+		if len(pats) >= effectiveThreshold {
+			candidates = append(candidates, candidate{key, len(strings.ReplaceAll(key.template, "*", "")), isH, pats})
+		}
+	}
+
+	// Sort: heuristic first, then descending specificity (more fixed
+	// characters), then descending coverage, then template name for a
+	// fully deterministic order.
+	sort.Slice(candidates, func(i, j int) bool {
+		if candidates[i].isHeuristic != candidates[j].isHeuristic {
+			return candidates[i].isHeuristic
+		}
+		if candidates[i].fixedChars != candidates[j].fixedChars {
+			return candidates[i].fixedChars > candidates[j].fixedChars
+		}
+		if len(candidates[i].patterns) != len(candidates[j].patterns) {
+			return len(candidates[i].patterns) > len(candidates[j].patterns)
+		}
+		return candidates[i].key.template < candidates[j].key.template
+	})
+
+	assigned := make(map[*coredata.TrackerPattern]bool)
+	groups := make(map[mergeGroupKey][]*coredata.TrackerPattern)
+
+	for _, c := range candidates {
+		effectiveThreshold := threshold
+		if c.isHeuristic {
+			effectiveThreshold = 1
+		}
+
+		var unassigned []*coredata.TrackerPattern
+		for _, p := range c.patterns {
+			if !assigned[p] {
+				unassigned = append(unassigned, p)
+			}
+		}
+
+		if len(unassigned) < effectiveThreshold {
+			continue
+		}
+
+		groups[c.key] = unassigned
+		for _, p := range unassigned {
+			assigned[p] = true
+		}
+	}
+
+	return groups
+}
+
+func heuristicTemplate(name string) (string, bool) {
+	tokens, sep := splitTokens(name)
+	if sep == 0 {
+		return "", false
+	}
+
+	s := string(sep)
+	changed := false
+	var result []string
+	for _, t := range tokens {
+		if looksVariable(t) {
+			changed = true
+			if len(result) == 0 || result[len(result)-1] != "*" {
+				result = append(result, "*")
+			}
+		} else {
+			result = append(result, t)
+		}
+	}
+
+	if !changed {
+		return "", false
+	}
+
+	return strings.Join(result, s), true
+}
+
+func templateCandidates(name string) []string {
+	var candidates []string
+
+	for i, ch := range name {
+		if ch == '_' || ch == '-' {
+			candidates = append(candidates, name[:i+1]+"*")
+		}
+	}
+
+	tokens, sep := splitTokens(name)
+	if len(tokens) >= 3 && sep != 0 {
+		s := string(sep)
+		for pos := 1; pos < len(tokens)-1; pos++ {
+			tmpl := strings.Join(tokens[:pos], s) + s + "*" + s + strings.Join(tokens[pos+1:], s)
+			candidates = append(candidates, tmpl)
+		}
+	}
+
+	return candidates
+}
+
+func looksVariable(token string) bool {
+	if len(token) == 0 {
+		return false
+	}
+
+	hasDigit := false
+	hasLetter := false
+	allHex := true
+	allDigits := true
+	for _, ch := range token {
+		switch {
+		case ch >= '0' && ch <= '9':
+			hasDigit = true
+		case ch >= 'a' && ch <= 'f', ch >= 'A' && ch <= 'F':
+			hasLetter = true
+			allDigits = false
+		case ch >= 'g' && ch <= 'z', ch >= 'G' && ch <= 'Z':
+			hasLetter = true
+			allHex = false
+			allDigits = false
+		case ch == '-':
+			allHex = false
+			allDigits = false
+		default:
+			allHex = false
+			allDigits = false
+		}
+	}
+
+	if len(token) >= 8 && hasDigit && hasLetter {
+		return true
+	}
+
+	if len(token) >= 16 && allHex && hasDigit {
+		return true
+	}
+
+	if isUUIDShape(token) {
+		return true
+	}
+
+	if len(token) >= 8 && allDigits {
+		return true
+	}
+
+	return false
+}
+
+func isUUIDShape(s string) bool {
+	if len(s) != 36 {
+		return false
+	}
+	for i, ch := range s {
+		if i == 8 || i == 13 || i == 18 || i == 23 {
+			if ch != '-' {
+				return false
+			}
+			continue
+		}
+		if (ch < '0' || ch > '9') && (ch < 'a' || ch > 'f') && (ch < 'A' || ch > 'F') {
+			return false
+		}
+	}
+	return true
+}
+
+func splitTokens(name string) ([]string, byte) {
+	if found := strings.Contains(name, "_"); found {
+		return strings.Split(name, "_"), '_'
+	}
+	if found := strings.Contains(name, "-"); found {
+		return strings.Split(name, "-"), '-'
+	}
+	return []string{name}, 0
+}
+
+func globMatch(pattern, name string) bool {
+	parts := strings.Split(pattern, "*")
+	if len(parts) == 1 {
+		return pattern == name
+	}
+
+	if !strings.HasPrefix(name, parts[0]) {
+		return false
+	}
+	name = name[len(parts[0]):]
+
+	last := parts[len(parts)-1]
+	if !strings.HasSuffix(name, last) {
+		return false
+	}
+	name = name[:len(name)-len(last)]
+
+	for _, part := range parts[1 : len(parts)-1] {
+		idx := strings.Index(name, part)
+		if idx == -1 {
+			return false
+		}
+		name = name[idx+len(part):]
+	}
+
+	return true
+}
+
+func bestSource(patterns []*coredata.TrackerPattern) *coredata.CookieSource {
+	for _, p := range patterns {
+		if p.Source != nil && *p.Source == coredata.CookieSourceScript {
+			return p.Source
+		}
+	}
+	src := coredata.CookieSourcePreExisting
+	return &src
+}
+
+func (h *patternAnalysisHandler) adoptUncategorisedPatterns(
+	ctx context.Context,
+	tx pg.Tx,
+	scope coredata.Scoper,
+	banner coredata.CookieBanner,
+) (bool, error) {
+	var uncategorised coredata.CookieCategory
+	if err := uncategorised.LoadUncategorisedByCookieBannerID(ctx, tx, scope, banner.ID); err != nil {
+		if errors.Is(err, coredata.ErrResourceNotFound) {
+			return false, nil
+		}
+		return false, fmt.Errorf("cannot load uncategorised category: %w", err)
+	}
+
+	var globPatterns coredata.TrackerPatterns
+	if err := globPatterns.LoadAllByCookieBannerID(
+		ctx,
+		tx,
+		scope,
+		banner.ID,
+		coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeGlob), nil, new(false)),
+		nil,
+	); err != nil {
+		return false, fmt.Errorf("cannot load glob patterns: %w", err)
+	}
+
+	if len(globPatterns) == 0 {
+		return false, nil
+	}
+
+	sort.Slice(globPatterns, func(i, j int) bool {
+		return len(globPatterns[i].Pattern) > len(globPatterns[j].Pattern)
+	})
+
+	exactMatchType := coredata.TrackerPatternMatchTypeExact
+	var uncategorisedExact coredata.TrackerPatterns
+	if err := uncategorisedExact.LoadAllByCookieBannerID(
+		ctx,
+		tx,
+		scope,
+		banner.ID,
+		coredata.NewTrackerPatternFilter(&exactMatchType, &uncategorised.ID, new(false)),
+		nil,
+	); err != nil {
+		return false, fmt.Errorf("cannot load uncategorised exact patterns: %w", err)
+	}
+
+	adopted := false
+	for _, ep := range uncategorisedExact {
+		var match *coredata.TrackerPattern
+		epBucket := durationBucket(ep.MaxAgeSeconds)
+		for _, gp := range globPatterns {
+			if ep.TrackerType == gp.TrackerType && globMatch(gp.Pattern, ep.Pattern) && durationBucket(gp.MaxAgeSeconds) == epBucket {
+				match = gp
+				break
+			}
+		}
+
+		if match == nil {
+			continue
+		}
+
+		var trackers coredata.DetectedTrackers
+		if err := trackers.RelinkByTrackerPatternID(ctx, tx, scope, ep.ID, match.ID); err != nil {
+			return false, fmt.Errorf("cannot relink detected trackers from pattern %q: %w", ep.Pattern, err)
+		}
+
+		if err := ep.Delete(ctx, tx, scope); err != nil {
+			return false, fmt.Errorf("cannot delete adopted exact pattern %q: %w", ep.Pattern, err)
+		}
+
+		adopted = true
+		h.logger.InfoCtx(
+			ctx,
+			"adopted uncategorised exact pattern into glob pattern",
+			log.String("exact_pattern", ep.Pattern),
+			log.String("glob_pattern", match.Pattern),
+			log.String("banner_id", banner.ID.String()),
+		)
+	}
+
+	return adopted, nil
+}