rohit/probo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add proboctl catalog and banner reset commands

Browse Source 
Add operator commands to proboctl for iterating on the cookie-banner
agents.

The global catalog groups (common-tracker-pattern, common-third-party)
list/filter/sort/show the catalogs using the shared coredata cursor
layer, and common-tracker-pattern reenrich re-describes selected rows by
running the enricher in-process (so it completes synchronously rather
than racing the async queue); a --cfg-file flag reuses probod's config
to wire the agent. --linked-banner/--linked-org target exactly the
catalog rows a banner or org depends on.

The cookie-banner reset-trackers command is tenant-scoped (it derives a
coredata.Scope from the banner/org GID) and rebuilds a banner's
uncategorised, non-excluded patterns from detected_trackers, decomposing
derived globs back into exacts, then re-arms the analysis and mapping
workers. --mapping-only skips the rebuild. A DB-backed test covers the
rebuild, link clearing, and preservation of categorised/excluded
patterns.

Signed-off-by: Émile Ré <emile@probo.com>
This commit is contained in:
Émile Ré
2026-06-03 13:37:48 +02:00
parent 662c0ae428
commit c763f83b13
16 changed files with 2142 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

205
pkg/cookiebanner/reset_trackers.go Normal file
View File

@@ -0,0 +1,205 @@
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package cookiebanner
import (
"context"
"fmt"
"time"
"go.gearno.de/kit/pg"
"go.probo.inc/probo/pkg/coredata"
"go.probo.inc/probo/pkg/gid"
)
// ResetTrackersResult summarizes what a banner reset changed.
type ResetTrackersResult struct {
PatternsReset int64
GlobsDecomposed int
ExactsCreated int
DetectionsRelinked int
AnalysisRequested bool
}
// ResetBannerTrackers re-arms the tracker pipeline for a banner's
// uncategorised, non-excluded patterns. It is an operator action
// (proboctl), tenant-scoped via the provided Scoper.
//
// With mappingOnly, it only clears each pattern's catalog/vendor links
// and re-arms mapping, for iterating on the mapping agent without
// touching analysis.
//
// The full reset additionally rebuilds the raw exact patterns from the
// surviving detected_trackers and re-arms pattern analysis, so the
// analysis worker re-derives globs from scratch: the pattern-analysis
// worker consumes (deletes) exact patterns when it merges them into
// globs, so the only way to re-run analysis is to reconstruct the exacts
// from detections. Each uncategorised, non-excluded glob is decomposed -
// every detection it covers becomes (or rejoins) an exact pattern keyed
// by its identifier - and the now-empty glob is deleted. User-categorised
// and excluded patterns are never touched.
func ResetBannerTrackers(
ctx context.Context,
pgClient *pg.Client,
scope coredata.Scoper,
bannerID gid.GID,
mappingOnly bool,
) (ResetTrackersResult, error) {
var result ResetTrackersResult
err := pgClient.WithTx(
ctx,
func(ctx context.Context, tx pg.Tx) error {
var uncategorised coredata.CookieCategory
if err := uncategorised.LoadUncategorisedByCookieBannerID(ctx, tx, scope, bannerID); err != nil {
return fmt.Errorf("cannot load uncategorised category: %w", err)
}
if !mappingOnly {
if err := decomposeGlobs(ctx, tx, scope, bannerID, uncategorised.ID, &result); err != nil {
return err
}
}
var patterns coredata.TrackerPatterns
reset, err := patterns.ResetAndRequestMappingByCookieCategoryID(ctx, tx, scope, uncategorised.ID)
if err != nil {
return fmt.Errorf("cannot reset and request mapping: %w", err)
}
result.PatternsReset = reset
if !mappingOnly {
banner := coredata.CookieBanner{ID: bannerID}
if err := banner.SetPatternAnalysisRequested(ctx, tx); err != nil {
return fmt.Errorf("cannot request pattern analysis: %w", err)
}
result.AnalysisRequested = true
}
return nil
},
)
if err != nil {
return ResetTrackersResult{}, err
}
return result, nil
}
// decomposeGlobs turns every uncategorised, non-excluded glob pattern of
// the banner back into exact patterns derived from its detected trackers,
// relinking each detection to its exact and deleting the emptied glob.
func decomposeGlobs(
ctx context.Context,
tx pg.Tx,
scope coredata.Scoper,
bannerID gid.GID,
uncategorisedID gid.GID,
result *ResetTrackersResult,
) error {
globMatchType := coredata.TrackerPatternMatchTypeGlob
notExcluded := false
var globs coredata.TrackerPatterns
if err := globs.LoadAllByCookieBannerID(
ctx,
tx,
scope,
bannerID,
coredata.NewTrackerPatternFilter(&globMatchType, &uncategorisedID, &notExcluded),
nil,
); err != nil {
return fmt.Errorf("cannot load glob patterns: %w", err)
}
for _, glob := range globs {
var detections coredata.DetectedTrackers
if err := detections.LoadAllByTrackerPatternID(ctx, tx, scope, glob.ID); err != nil {
return fmt.Errorf("cannot load detections for glob %q: %w", glob.Pattern, err)
}
for _, detection := range detections {
exactID, created, err := ensureExactPattern(ctx, tx, scope, glob, uncategorisedID, detection)
if err != nil {
return err
}
if created {
result.ExactsCreated++
}
detection.TrackerPatternID = &exactID
if err := detection.UpdateTrackerPatternID(ctx, tx, scope); err != nil {
return fmt.Errorf("cannot relink detection %s: %w", detection.ID, err)
}
result.DetectionsRelinked++
}
if err := glob.Delete(ctx, tx, scope); err != nil {
return fmt.Errorf("cannot delete glob pattern %q: %w", glob.Pattern, err)
}
result.GlobsDecomposed++
}
return nil
}
// ensureExactPattern finds or creates the exact pattern for a detection
// (keyed by banner, tracker type, identifier, and max-age) in the
// uncategorised category, returning its id and whether it was created.
func ensureExactPattern(
ctx context.Context,
tx pg.Tx,
scope coredata.Scoper,
glob *coredata.TrackerPattern,
uncategorisedID gid.GID,
detection *coredata.DetectedTracker,
) (gid.GID, bool, error) {
now := time.Now()
exact := &coredata.TrackerPattern{
ID: gid.New(glob.CookieBannerID.TenantID(), coredata.TrackerPatternEntityType),
OrganizationID: glob.OrganizationID,
CookieBannerID: glob.CookieBannerID,
CookieCategoryID: uncategorisedID,
TrackerType: detection.TrackerType,
Pattern: detection.Identifier,
MatchType: coredata.TrackerPatternMatchTypeExact,
DisplayName: detection.Identifier,
MaxAgeSeconds: detection.MaxAgeSeconds,
Source: detection.Source,
MappingRequestedAt: &now,
CreatedAt: now,
UpdatedAt: now,
}
created, err := exact.InsertIfNotExists(ctx, tx, scope)
if err != nil {
return gid.GID{}, false, fmt.Errorf("cannot insert exact pattern %q: %w", detection.Identifier, err)
}
if !created {
if err := exact.LoadByBannerIDTypeAndPattern(ctx, tx, scope, glob.CookieBannerID, detection.TrackerType, detection.Identifier, detection.MaxAgeSeconds); err != nil {
return gid.GID{}, false, fmt.Errorf("cannot load existing exact pattern %q: %w", detection.Identifier, err)
}
}
return exact.ID, created, nil
}

179
pkg/cookiebanner/reset_trackers_test.go Normal file
View File

@@ -0,0 +1,179 @@
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package cookiebanner
import (
"context"
"testing"
"time"
"github.com/stretchr/testify/require"
"go.gearno.de/kit/pg"
"go.probo.inc/probo/pkg/coredata"
"go.probo.inc/probo/pkg/gid"
)
// TestResetBannerTrackers_FullRebuild seeds a banner with an
// uncategorised glob covering two detections, an uncategorised exact
// carrying catalog/vendor links, a categorised exact, and an excluded
// exact. A full reset must: decompose the glob into per-identifier
// exacts and relink its detections, clear links on the surviving
// uncategorised exact and re-arm its mapping, preserve the categorised
// and excluded patterns, and arm pattern analysis on the banner.
func TestResetBannerTrackers_FullRebuild(t *testing.T) {
t.Parallel()
client := newTestPgClient(t)
ctx := context.Background()
fx := seedWorkerFixture(t, ctx, client)
thirdPartyID := seedThirdParty(t, ctx, client, fx, "Reset Vendor")
commonPatternID := seedCommonTrackerPattern(t, ctx, client, "ga_linked")
glob := newGlobInCategory(fx, "_ga_*", fx.uncategorisedID, coredata.CookieSourceScript, nil)
linkedExact := newExactPattern(fx, "linked_cookie", fx.uncategorisedID, coredata.CookieSourcePreExisting, nil)
linkedExact.CommonTrackerPatternID = &commonPatternID
linkedExact.ThirdPartyID = &thirdPartyID
linkedExact.Description = "stale description"
categorised := newExactPattern(fx, "categorised_cookie", fx.normalCategoryID, coredata.CookieSourceScript, nil)
excluded := newExactPattern(fx, "excluded_cookie", fx.uncategorisedID, coredata.CookieSourceScript, nil)
excluded.Excluded = true
now := time.Now().UTC().Truncate(time.Microsecond)
require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error {
for _, p := range []*coredata.TrackerPattern{glob, linkedExact, categorised, excluded} {
if err := p.Insert(ctx, tx, fx.scope); err != nil {
return err
}
}
for _, identifier := range []string{"_ga_ABC", "_ga_DEF"} {
detection := &coredata.DetectedTracker{
ID: gid.New(fx.scope.GetTenantID(), coredata.DetectedTrackerEntityType),
CookieBannerID: fx.banner.ID,
TrackerPatternID: &glob.ID,
TrackerType: coredata.TrackerTypeCookie,
Identifier: identifier,
Source: ref(coredata.CookieSourceScript),
LastDetectedAt: now,
CreatedAt: now,
UpdatedAt: now,
}
if _, err := detection.Upsert(ctx, tx, fx.scope); err != nil {
return err
}
}
return nil
}))
result, err := ResetBannerTrackers(ctx, client, fx.scope, fx.banner.ID, false)
require.NoError(t, err)
require.Equal(t, 1, result.GlobsDecomposed)
require.Equal(t, 2, result.ExactsCreated)
require.Equal(t, 2, result.DetectionsRelinked)
require.True(t, result.AnalysisRequested)
// linked_cookie + _ga_ABC + _ga_DEF (excluded and categorised are untouched).
require.Equal(t, int64(3), result.PatternsReset)
require.NoError(t, client.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error {
// The glob is gone.
var goneGlob coredata.TrackerPattern
err := goneGlob.LoadByBannerIDTypeAndPattern(ctx, conn, fx.scope, fx.banner.ID, coredata.TrackerTypeCookie, "_ga_*", nil)
require.ErrorIs(t, err, coredata.ErrResourceNotFound)
// Each detection identifier is now its own exact, with mapping armed
// and no links, and the detection relinked to it.
for _, identifier := range []string{"_ga_ABC", "_ga_DEF"} {
var exact coredata.TrackerPattern
require.NoError(t, exact.LoadByBannerIDTypeAndPattern(ctx, conn, fx.scope, fx.banner.ID, coredata.TrackerTypeCookie, identifier, nil))
require.Equal(t, coredata.TrackerPatternMatchTypeExact, exact.MatchType)
require.Equal(t, fx.uncategorisedID, exact.CookieCategoryID)
require.Nil(t, exact.CommonTrackerPatternID)
require.Nil(t, exact.ThirdPartyID)
require.NotNil(t, exact.MappingRequestedAt)
var detections coredata.DetectedTrackers
require.NoError(t, detections.LoadAllByTrackerPatternID(ctx, conn, fx.scope, exact.ID))
require.Len(t, detections, 1)
require.Equal(t, identifier, detections[0].Identifier)
}
// The surviving uncategorised exact had its links and copied
// description cleared and mapping re-armed.
var survivor coredata.TrackerPattern
require.NoError(t, survivor.LoadByBannerIDTypeAndPattern(ctx, conn, fx.scope, fx.banner.ID, coredata.TrackerTypeCookie, "linked_cookie", nil))
require.Nil(t, survivor.CommonTrackerPatternID)
require.Nil(t, survivor.ThirdPartyID)
require.Empty(t, survivor.Description)
require.NotNil(t, survivor.MappingRequestedAt)
// The categorised pattern is untouched.
var categorisedRow coredata.TrackerPattern
require.NoError(t, categorisedRow.LoadByBannerIDTypeAndPattern(ctx, conn, fx.scope, fx.banner.ID, coredata.TrackerTypeCookie, "categorised_cookie", nil))
require.Equal(t, fx.normalCategoryID, categorisedRow.CookieCategoryID)
// The excluded pattern is preserved.
var excludedRow coredata.TrackerPattern
require.NoError(t, excludedRow.LoadByBannerIDTypeAndPattern(ctx, conn, fx.scope, fx.banner.ID, coredata.TrackerTypeCookie, "excluded_cookie", nil))
require.True(t, excludedRow.Excluded)
// Pattern analysis is armed on the banner.
var banner coredata.CookieBanner
require.NoError(t, banner.LoadByID(ctx, conn, fx.scope, fx.banner.ID))
require.NotNil(t, banner.PatternAnalysisRequestedAt)
return nil
}))
}
func ref[T any](v T) *T {
return &v
}
func seedCommonTrackerPattern(t *testing.T, ctx context.Context, client *pg.Client, pattern string) gid.GID {
t.Helper()
now := time.Now().UTC().Truncate(time.Microsecond)
cp := coredata.CommonTrackerPattern{
ID: gid.New(gid.NilTenant, coredata.CommonTrackerPatternEntityType),
TrackerType: coredata.TrackerTypeCookie,
Pattern: pattern,
MatchType: coredata.TrackerPatternMatchTypeExact,
Description: "seeded",
Confidence: 1,
CreatedAt: now,
UpdatedAt: now,
}
require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error {
return cp.Insert(ctx, tx)
}))
t.Cleanup(func() {
_ = client.WithTx(context.Background(), func(ctx context.Context, tx pg.Tx) error {
_, err := tx.Exec(ctx, `DELETE FROM common_tracker_patterns WHERE id = $1`, cp.ID)
return err
})
})
return cp.ID
}