Files
probo/pkg/cookiebanner/reset_trackers_test.go
Sacha Al Himdani 9ab8ea2085 Refacto load all functions
Unbounded LoadAll* loaders materialised an entire result set in one
query with no ceiling. A table that is small in development can grow
without bound in production, so these loaders were a latent memory
and query-time hazard.

Remove the LoadAll* methods from pkg/coredata and walk the cursor-
paginated LoadBy* siblings instead through a shared page.LoadAll
helper. The helper advances a MaxCursorSize forward cursor until the
result set is exhausted and concatenates the pages. It caps a single
call at MaxLoadAllPages (20) batches of 500 rows and errors past that
rather than materialising an unbounded set, so a runaway caller fails
loudly instead of exhausting memory.

Callers that genuinely need every row now express that explicitly,
and the coredata load-naming rule and docs are updated to discourage
new unbounded loaders.

Signed-off-by: Sacha Al Himdani <sacha@probo.com>
2026-06-16 14:35:16 +02:00

193 lines
7.4 KiB
Go

// Copyright (c) 2026 Probo Inc <hello@probo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package cookiebanner
import (
"context"
"testing"
"time"
"github.com/stretchr/testify/require"
"go.gearno.de/kit/pg"
"go.probo.inc/probo/internal/test"
"go.probo.inc/probo/pkg/coredata"
"go.probo.inc/probo/pkg/gid"
"go.probo.inc/probo/pkg/page"
)
// TestResetBannerTrackers_FullRebuild seeds a banner with an
// uncategorised glob covering two detections, an uncategorised exact
// carrying catalog/vendor links, a categorised exact, and an excluded
// exact. A full reset must: decompose the glob into per-identifier
// exacts and relink its detections, clear links on the surviving
// uncategorised exact and re-arm its mapping, preserve the categorised
// and excluded patterns, and arm pattern analysis on the banner.
func TestResetBannerTrackers_FullRebuild(t *testing.T) {
t.Parallel()
client := test.PGClient(t)
ctx := context.Background()
fx := seedWorkerFixture(t, ctx, client)
thirdPartyID := seedThirdParty(t, ctx, client, fx, "Reset Vendor")
commonPatternID := seedCommonTrackerPattern(t, ctx, client, "ga_linked")
glob := newGlobInCategory(fx, "_ga_*", fx.uncategorisedID, coredata.CookieSourceScript, nil)
linkedExact := newExactPattern(fx, "linked_cookie", fx.uncategorisedID, coredata.CookieSourcePreExisting, nil)
linkedExact.CommonTrackerPatternID = &commonPatternID
linkedExact.ThirdPartyID = &thirdPartyID
linkedExact.Description = "stale description"
categorised := newExactPattern(fx, "categorised_cookie", fx.normalCategoryID, coredata.CookieSourceScript, nil)
excluded := newExactPattern(fx, "excluded_cookie", fx.uncategorisedID, coredata.CookieSourceScript, nil)
excluded.Excluded = true
now := time.Now().UTC().Truncate(time.Microsecond)
require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error {
for _, p := range []*coredata.TrackerPattern{glob, linkedExact, categorised, excluded} {
if err := p.Insert(ctx, tx, fx.scope); err != nil {
return err
}
}
for _, identifier := range []string{"_ga_ABC", "_ga_DEF"} {
detection := &coredata.DetectedTracker{
ID: gid.New(fx.scope.GetTenantID(), coredata.DetectedTrackerEntityType),
CookieBannerID: fx.banner.ID,
TrackerPatternID: &glob.ID,
TrackerType: coredata.TrackerTypeCookie,
Identifier: identifier,
Source: new(coredata.CookieSourceScript),
LastDetectedAt: now,
CreatedAt: now,
UpdatedAt: now,
}
if _, err := detection.Upsert(ctx, tx, fx.scope); err != nil {
return err
}
}
return nil
}))
result, err := ResetBannerTrackers(ctx, client, fx.scope, fx.banner.ID, false, nil)
require.NoError(t, err)
require.Equal(t, 1, result.GlobsDecomposed)
require.Equal(t, 2, result.ExactsCreated)
require.Equal(t, 2, result.DetectionsRelinked)
require.True(t, result.AnalysisRequested)
// linked_cookie + _ga_ABC + _ga_DEF (excluded and categorised are untouched).
require.Equal(t, int64(3), result.PatternsReset)
require.NoError(t, client.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error {
// The glob is gone.
var goneGlob coredata.TrackerPattern
err := goneGlob.LoadByBannerIDTypeAndPattern(ctx, conn, fx.scope, fx.banner.ID, coredata.TrackerTypeCookie, "_ga_*", nil)
require.ErrorIs(t, err, coredata.ErrResourceNotFound)
// Each detection identifier is now its own exact, with mapping armed
// and no links, and the detection relinked to it.
for _, identifier := range []string{"_ga_ABC", "_ga_DEF"} {
var exact coredata.TrackerPattern
require.NoError(t, exact.LoadByBannerIDTypeAndPattern(ctx, conn, fx.scope, fx.banner.ID, coredata.TrackerTypeCookie, identifier, nil))
require.Equal(t, coredata.TrackerPatternMatchTypeExact, exact.MatchType)
require.Equal(t, fx.uncategorisedID, exact.CookieCategoryID)
require.Nil(t, exact.CommonTrackerPatternID)
require.Nil(t, exact.ThirdPartyID)
require.NotNil(t, exact.MappingRequestedAt)
detections, err := page.LoadAll(
ctx,
page.OrderBy[coredata.DetectedTrackerOrderField]{
Field: coredata.DetectedTrackerOrderFieldLastDetectedAt,
Direction: page.OrderDirectionAsc,
},
func(ctx context.Context, cursor *page.Cursor[coredata.DetectedTrackerOrderField]) ([]*coredata.DetectedTracker, error) {
var batch coredata.DetectedTrackers
if err := batch.LoadByTrackerPatternID(ctx, conn, fx.scope, exact.ID, cursor); err != nil {
return nil, err
}
return batch, nil
},
)
require.NoError(t, err)
require.Len(t, detections, 1)
require.Equal(t, identifier, detections[0].Identifier)
}
// The surviving uncategorised exact had its links and copied
// description cleared and mapping re-armed.
var survivor coredata.TrackerPattern
require.NoError(t, survivor.LoadByBannerIDTypeAndPattern(ctx, conn, fx.scope, fx.banner.ID, coredata.TrackerTypeCookie, "linked_cookie", nil))
require.Nil(t, survivor.CommonTrackerPatternID)
require.Nil(t, survivor.ThirdPartyID)
require.Empty(t, survivor.Description)
require.NotNil(t, survivor.MappingRequestedAt)
// The categorised pattern is untouched.
var categorisedRow coredata.TrackerPattern
require.NoError(t, categorisedRow.LoadByBannerIDTypeAndPattern(ctx, conn, fx.scope, fx.banner.ID, coredata.TrackerTypeCookie, "categorised_cookie", nil))
require.Equal(t, fx.normalCategoryID, categorisedRow.CookieCategoryID)
// The excluded pattern is preserved.
var excludedRow coredata.TrackerPattern
require.NoError(t, excludedRow.LoadByBannerIDTypeAndPattern(ctx, conn, fx.scope, fx.banner.ID, coredata.TrackerTypeCookie, "excluded_cookie", nil))
require.True(t, excludedRow.Excluded)
// Pattern analysis is armed on the banner.
var banner coredata.CookieBanner
require.NoError(t, banner.LoadByID(ctx, conn, fx.scope, fx.banner.ID))
require.NotNil(t, banner.PatternAnalysisRequestedAt)
return nil
}))
}
func seedCommonTrackerPattern(t *testing.T, ctx context.Context, client *pg.Client, pattern string) gid.GID {
t.Helper()
now := time.Now().UTC().Truncate(time.Microsecond)
cp := coredata.CommonTrackerPattern{
ID: gid.New(gid.NilTenant, coredata.CommonTrackerPatternEntityType),
TrackerType: coredata.TrackerTypeCookie,
Pattern: pattern,
MatchType: coredata.TrackerPatternMatchTypeExact,
Description: "seeded",
Confidence: 1,
CreatedAt: now,
UpdatedAt: now,
}
require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error {
return cp.Insert(ctx, tx)
}))
t.Cleanup(func() {
_ = client.WithTx(context.Background(), func(ctx context.Context, tx pg.Tx) error {
_, err := tx.Exec(ctx, `DELETE FROM common_tracker_patterns WHERE id = $1`, cp.ID)
return err
})
})
return cp.ID
}