Refacto load all functions

Unbounded LoadAll* loaders materialised an entire result set in one
query with no ceiling. A table that is small in development can grow
without bound in production, so these loaders were a latent memory
and query-time hazard.

Remove the LoadAll* methods from pkg/coredata and walk the cursor-
paginated LoadBy* siblings instead through a shared page.LoadAll
helper. The helper advances a MaxCursorSize forward cursor until the
result set is exhausted and concatenates the pages. It caps a single
call at MaxLoadAllPages (20) batches of 500 rows and errors past that
rather than materialising an unbounded set, so a runaway caller fails
loudly instead of exhausting memory.

Callers that genuinely need every row now express that explicitly,
and the coredata load-naming rule and docs are updated to discourage
new unbounded loaders.

Signed-off-by: Sacha Al Himdani <sacha@probo.com>
This commit is contained in:
Sacha Al Himdani
2026-06-09 19:33:36 +02:00
committed by Sacha Al Himdani
parent 853f2404a6
commit 9ab8ea2085
46 changed files with 1218 additions and 1674 deletions

View File

@@ -27,6 +27,7 @@ import (
"go.gearno.de/kit/worker"
"go.probo.inc/probo/pkg/coredata"
"go.probo.inc/probo/pkg/gid"
"go.probo.inc/probo/pkg/page"
)
const (
@@ -147,16 +148,23 @@ func (h *patternAnalysisHandler) Process(ctx context.Context, banner coredata.Co
func(ctx context.Context, tx pg.Tx) error {
scope := coredata.NewScopeFromObjectID(banner.ID)
var exactPatterns coredata.TrackerPatterns
if err := exactPatterns.LoadAllByCookieBannerID(
exactPatterns, err := page.LoadAll(
ctx,
tx,
scope,
banner.ID,
coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeExact), nil, new(false)),
nil,
); err != nil {
return fmt.Errorf("cannot load exact patterns: %w", err)
page.OrderBy[coredata.TrackerPatternOrderField]{
Field: coredata.TrackerPatternOrderFieldCreatedAt,
Direction: page.OrderDirectionAsc,
},
func(ctx context.Context, cursor *page.Cursor[coredata.TrackerPatternOrderField]) ([]*coredata.TrackerPattern, error) {
var batch coredata.TrackerPatterns
if err := batch.LoadByCookieBannerID(ctx, tx, scope, banner.ID, cursor, coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeExact), nil, new(false))); err != nil {
return nil, fmt.Errorf("cannot load exact patterns: %w", err)
}
return batch, nil
},
)
if err != nil {
return err
}
mergeGroups := findMergeGroups(exactPatterns, patternMergeThreshold)
@@ -816,16 +824,23 @@ func (h *patternAnalysisHandler) adoptUncategorisedPatterns(
return false, fmt.Errorf("cannot load uncategorised category: %w", err)
}
var globPatterns coredata.TrackerPatterns
if err := globPatterns.LoadAllByCookieBannerID(
globPatterns, err := page.LoadAll(
ctx,
tx,
scope,
banner.ID,
coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeGlob), nil, new(false)),
nil,
); err != nil {
return false, fmt.Errorf("cannot load glob patterns: %w", err)
page.OrderBy[coredata.TrackerPatternOrderField]{
Field: coredata.TrackerPatternOrderFieldCreatedAt,
Direction: page.OrderDirectionAsc,
},
func(ctx context.Context, cursor *page.Cursor[coredata.TrackerPatternOrderField]) ([]*coredata.TrackerPattern, error) {
var batch coredata.TrackerPatterns
if err := batch.LoadByCookieBannerID(ctx, tx, scope, banner.ID, cursor, coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeGlob), nil, new(false))); err != nil {
return nil, fmt.Errorf("cannot load glob patterns: %w", err)
}
return batch, nil
},
)
if err != nil {
return false, err
}
if len(globPatterns) == 0 {
@@ -841,16 +856,23 @@ func (h *patternAnalysisHandler) adoptUncategorisedPatterns(
exactMatchType := coredata.TrackerPatternMatchTypeExact
var uncategorisedExact coredata.TrackerPatterns
if err := uncategorisedExact.LoadAllByCookieBannerID(
uncategorisedExact, err := page.LoadAll(
ctx,
tx,
scope,
banner.ID,
coredata.NewTrackerPatternFilter(&exactMatchType, &uncategorised.ID, new(false)),
nil,
); err != nil {
return false, fmt.Errorf("cannot load uncategorised exact patterns: %w", err)
page.OrderBy[coredata.TrackerPatternOrderField]{
Field: coredata.TrackerPatternOrderFieldCreatedAt,
Direction: page.OrderDirectionAsc,
},
func(ctx context.Context, cursor *page.Cursor[coredata.TrackerPatternOrderField]) ([]*coredata.TrackerPattern, error) {
var batch coredata.TrackerPatterns
if err := batch.LoadByCookieBannerID(ctx, tx, scope, banner.ID, cursor, coredata.NewTrackerPatternFilter(&exactMatchType, &uncategorised.ID, new(false))); err != nil {
return nil, fmt.Errorf("cannot load uncategorised exact patterns: %w", err)
}
return batch, nil
},
)
if err != nil {
return false, err
}
adopted := false

View File

@@ -28,6 +28,7 @@ import (
"go.probo.inc/probo/internal/test"
"go.probo.inc/probo/pkg/coredata"
"go.probo.inc/probo/pkg/gid"
"go.probo.inc/probo/pkg/page"
)
// workerFixture bootstraps the parent rows the worker's transaction
@@ -324,14 +325,28 @@ func TestPatternAnalysisWorker_PromotesSourceOnExistingGlob(t *testing.T) {
var remainingExacts coredata.TrackerPatterns
require.NoError(t, client.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error {
return remainingExacts.LoadAllByCookieBannerID(
loaded, err := page.LoadAll(
ctx,
conn,
fx.scope,
fx.banner.ID,
coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeExact), nil, new(false)),
nil,
page.OrderBy[coredata.TrackerPatternOrderField]{
Field: coredata.TrackerPatternOrderFieldCreatedAt,
Direction: page.OrderDirectionAsc,
},
func(ctx context.Context, cursor *page.Cursor[coredata.TrackerPatternOrderField]) ([]*coredata.TrackerPattern, error) {
var batch coredata.TrackerPatterns
if err := batch.LoadByCookieBannerID(ctx, conn, fx.scope, fx.banner.ID, cursor, coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeExact), nil, new(false))); err != nil {
return nil, err
}
return batch, nil
},
)
if err != nil {
return err
}
remainingExacts = loaded
return nil
}))
assert.Empty(t, remainingExacts, "all three exacts must be relinked and deleted")
}
@@ -401,14 +416,28 @@ func TestPatternAnalysisWorker_AdoptionTriggersDraftVersion(t *testing.T) {
var remainingExacts coredata.TrackerPatterns
require.NoError(t, client.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error {
return remainingExacts.LoadAllByCookieBannerID(
loaded, err := page.LoadAll(
ctx,
conn,
fx.scope,
fx.banner.ID,
coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeExact), nil, new(false)),
nil,
page.OrderBy[coredata.TrackerPatternOrderField]{
Field: coredata.TrackerPatternOrderFieldCreatedAt,
Direction: page.OrderDirectionAsc,
},
func(ctx context.Context, cursor *page.Cursor[coredata.TrackerPatternOrderField]) ([]*coredata.TrackerPattern, error) {
var batch coredata.TrackerPatterns
if err := batch.LoadByCookieBannerID(ctx, conn, fx.scope, fx.banner.ID, cursor, coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeExact), nil, new(false))); err != nil {
return nil, err
}
return batch, nil
},
)
if err != nil {
return err
}
remainingExacts = loaded
return nil
}))
assert.Empty(t, remainingExacts, "adoptUncategorisedPatterns must absorb the uncategorised exacts into the existing glob")
@@ -486,14 +515,28 @@ func TestPatternAnalysisWorker_AdoptionPromotesSourceCrossCategory(t *testing.T)
var remainingExacts coredata.TrackerPatterns
require.NoError(t, client.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error {
return remainingExacts.LoadAllByCookieBannerID(
loaded, err := page.LoadAll(
ctx,
conn,
fx.scope,
fx.banner.ID,
coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeExact), nil, new(false)),
nil,
page.OrderBy[coredata.TrackerPatternOrderField]{
Field: coredata.TrackerPatternOrderFieldCreatedAt,
Direction: page.OrderDirectionAsc,
},
func(ctx context.Context, cursor *page.Cursor[coredata.TrackerPatternOrderField]) ([]*coredata.TrackerPattern, error) {
var batch coredata.TrackerPatterns
if err := batch.LoadByCookieBannerID(ctx, conn, fx.scope, fx.banner.ID, cursor, coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeExact), nil, new(false))); err != nil {
return nil, err
}
return batch, nil
},
)
if err != nil {
return err
}
remainingExacts = loaded
return nil
}))
assert.Empty(t, remainingExacts, "the uncategorised exact must be adopted into the existing glob")
}
@@ -559,14 +602,28 @@ func TestReportDetectedTrackers_PromotesSourceOnExistingGlob(t *testing.T) {
var exacts coredata.TrackerPatterns
require.NoError(t, client.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error {
return exacts.LoadAllByCookieBannerID(
loaded, err := page.LoadAll(
ctx,
conn,
fx.scope,
fx.banner.ID,
coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeExact), nil, new(false)),
nil,
page.OrderBy[coredata.TrackerPatternOrderField]{
Field: coredata.TrackerPatternOrderFieldCreatedAt,
Direction: page.OrderDirectionAsc,
},
func(ctx context.Context, cursor *page.Cursor[coredata.TrackerPatternOrderField]) ([]*coredata.TrackerPattern, error) {
var batch coredata.TrackerPatterns
if err := batch.LoadByCookieBannerID(ctx, conn, fx.scope, fx.banner.ID, cursor, coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeExact), nil, new(false))); err != nil {
return nil, err
}
return batch, nil
},
)
if err != nil {
return err
}
exacts = loaded
return nil
}))
assert.Empty(t, exacts, "the detected cookie globMatches the existing glob; no exact pattern must be created")
}
@@ -608,14 +665,28 @@ func TestPatternAnalysisWorker_MergeWithoutAdoptionSkipsDraftVersion(t *testing.
var globs coredata.TrackerPatterns
require.NoError(t, client.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error {
return globs.LoadAllByCookieBannerID(
loaded, err := page.LoadAll(
ctx,
conn,
fx.scope,
fx.banner.ID,
coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeGlob), nil, new(false)),
nil,
page.OrderBy[coredata.TrackerPatternOrderField]{
Field: coredata.TrackerPatternOrderFieldCreatedAt,
Direction: page.OrderDirectionAsc,
},
func(ctx context.Context, cursor *page.Cursor[coredata.TrackerPatternOrderField]) ([]*coredata.TrackerPattern, error) {
var batch coredata.TrackerPatterns
if err := batch.LoadByCookieBannerID(ctx, conn, fx.scope, fx.banner.ID, cursor, coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeGlob), nil, new(false))); err != nil {
return nil, err
}
return batch, nil
},
)
if err != nil {
return err
}
globs = loaded
return nil
}))
require.Len(t, globs, 1, "the three exacts must consolidate into a single glob")
assert.Equal(t, "_ga_*", globs[0].Pattern)

View File

@@ -22,6 +22,7 @@ import (
"go.gearno.de/kit/pg"
"go.probo.inc/probo/pkg/coredata"
"go.probo.inc/probo/pkg/gid"
"go.probo.inc/probo/pkg/page"
)
// ResetTrackersResult summarizes what a banner reset changed.
@@ -122,22 +123,43 @@ func decomposeGlobs(
globMatchType := coredata.TrackerPatternMatchTypeGlob
notExcluded := false
var globs coredata.TrackerPatterns
if err := globs.LoadAllByCookieBannerID(
globs, err := page.LoadAll(
ctx,
tx,
scope,
bannerID,
coredata.NewTrackerPatternFilter(&globMatchType, &uncategorisedID, &notExcluded).WithPatternKeyword(keyword),
nil,
); err != nil {
return fmt.Errorf("cannot load glob patterns: %w", err)
page.OrderBy[coredata.TrackerPatternOrderField]{
Field: coredata.TrackerPatternOrderFieldCreatedAt,
Direction: page.OrderDirectionAsc,
},
func(ctx context.Context, cursor *page.Cursor[coredata.TrackerPatternOrderField]) ([]*coredata.TrackerPattern, error) {
var batch coredata.TrackerPatterns
if err := batch.LoadByCookieBannerID(ctx, tx, scope, bannerID, cursor, coredata.NewTrackerPatternFilter(&globMatchType, &uncategorisedID, &notExcluded).WithPatternKeyword(keyword)); err != nil {
return nil, fmt.Errorf("cannot load glob patterns: %w", err)
}
return batch, nil
},
)
if err != nil {
return err
}
for _, glob := range globs {
var detections coredata.DetectedTrackers
if err := detections.LoadAllByTrackerPatternID(ctx, tx, scope, glob.ID); err != nil {
return fmt.Errorf("cannot load detections for glob %q: %w", glob.Pattern, err)
detections, err := page.LoadAll(
ctx,
page.OrderBy[coredata.DetectedTrackerOrderField]{
Field: coredata.DetectedTrackerOrderFieldLastDetectedAt,
Direction: page.OrderDirectionAsc,
},
func(ctx context.Context, cursor *page.Cursor[coredata.DetectedTrackerOrderField]) ([]*coredata.DetectedTracker, error) {
var batch coredata.DetectedTrackers
if err := batch.LoadByTrackerPatternID(ctx, tx, scope, glob.ID, cursor); err != nil {
return nil, fmt.Errorf("cannot load detections for glob %q: %w", glob.Pattern, err)
}
return batch, nil
},
)
if err != nil {
return err
}
for _, detection := range detections {

View File

@@ -24,6 +24,7 @@ import (
"go.probo.inc/probo/internal/test"
"go.probo.inc/probo/pkg/coredata"
"go.probo.inc/probo/pkg/gid"
"go.probo.inc/probo/pkg/page"
)
// TestResetBannerTrackers_FullRebuild seeds a banner with an
@@ -113,8 +114,22 @@ func TestResetBannerTrackers_FullRebuild(t *testing.T) {
require.Nil(t, exact.ThirdPartyID)
require.NotNil(t, exact.MappingRequestedAt)
var detections coredata.DetectedTrackers
require.NoError(t, detections.LoadAllByTrackerPatternID(ctx, conn, fx.scope, exact.ID))
detections, err := page.LoadAll(
ctx,
page.OrderBy[coredata.DetectedTrackerOrderField]{
Field: coredata.DetectedTrackerOrderFieldLastDetectedAt,
Direction: page.OrderDirectionAsc,
},
func(ctx context.Context, cursor *page.Cursor[coredata.DetectedTrackerOrderField]) ([]*coredata.DetectedTracker, error) {
var batch coredata.DetectedTrackers
if err := batch.LoadByTrackerPatternID(ctx, conn, fx.scope, exact.ID, cursor); err != nil {
return nil, err
}
return batch, nil
},
)
require.NoError(t, err)
require.Len(t, detections, 1)
require.Equal(t, identifier, detections[0].Identifier)
}

View File

@@ -553,21 +553,42 @@ func (s *Service) ensureDraftVersionForBanner(
consentFilter := coredata.NewCookieCategoryFilter(new(coredata.CookieCategoryKindUncategorised))
var categories coredata.CookieCategories
if err := categories.LoadAllByCookieBannerID(ctx, tx, scope, bannerID, consentFilter); err != nil {
return nil, fmt.Errorf("cannot load cookie categories: %w", err)
categories, err := page.LoadAll(
ctx,
page.OrderBy[coredata.CookieCategoryOrderField]{
Field: coredata.CookieCategoryOrderFieldRank,
Direction: page.OrderDirectionAsc,
},
func(ctx context.Context, cursor *page.Cursor[coredata.CookieCategoryOrderField]) ([]*coredata.CookieCategory, error) {
var batch coredata.CookieCategories
if err := batch.LoadByCookieBannerID(ctx, tx, scope, bannerID, cursor, consentFilter); err != nil {
return nil, fmt.Errorf("cannot load cookie categories: %w", err)
}
return batch, nil
},
)
if err != nil {
return nil, err
}
var allPatterns coredata.TrackerPatterns
if err := allPatterns.LoadAllByCookieBannerID(
allPatterns, err := page.LoadAll(
ctx,
tx,
scope,
bannerID,
coredata.NewTrackerPatternFilter(nil, nil, new(false)),
nil,
); err != nil {
return nil, fmt.Errorf("cannot load tracker patterns: %w", err)
page.OrderBy[coredata.TrackerPatternOrderField]{
Field: coredata.TrackerPatternOrderFieldCreatedAt,
Direction: page.OrderDirectionAsc,
},
func(ctx context.Context, cursor *page.Cursor[coredata.TrackerPatternOrderField]) ([]*coredata.TrackerPattern, error) {
var batch coredata.TrackerPatterns
if err := batch.LoadByCookieBannerID(ctx, tx, scope, bannerID, cursor, coredata.NewTrackerPatternFilter(nil, nil, new(false))); err != nil {
return nil, fmt.Errorf("cannot load tracker patterns: %w", err)
}
return batch, nil
},
)
if err != nil {
return nil, err
}
return s.ensureDraftVersion(ctx, tx, scope, &banner, categories, allPatterns)
@@ -1695,14 +1716,42 @@ func (s *Service) GetActiveBannerConfig(
consentFilter := coredata.NewCookieCategoryFilter(new(coredata.CookieCategoryKindUncategorised))
var categories coredata.CookieCategories
if err := categories.LoadAllByCookieBannerID(ctx, conn, scope, banner.ID, consentFilter); err != nil {
return fmt.Errorf("cannot load cookie categories: %w", err)
categories, err := page.LoadAll(
ctx,
page.OrderBy[coredata.CookieCategoryOrderField]{
Field: coredata.CookieCategoryOrderFieldRank,
Direction: page.OrderDirectionAsc,
},
func(ctx context.Context, cursor *page.Cursor[coredata.CookieCategoryOrderField]) ([]*coredata.CookieCategory, error) {
var batch coredata.CookieCategories
if err := batch.LoadByCookieBannerID(ctx, conn, scope, banner.ID, cursor, consentFilter); err != nil {
return nil, fmt.Errorf("cannot load cookie categories: %w", err)
}
return batch, nil
},
)
if err != nil {
return err
}
var translations coredata.CookieBannerTranslations
if err := translations.LoadAllByCookieBannerID(ctx, conn, scope, banner.ID); err != nil {
return fmt.Errorf("cannot load cookie banner translations: %w", err)
translations, err := page.LoadAll(
ctx,
page.OrderBy[coredata.CookieBannerTranslationOrderField]{
Field: coredata.CookieBannerTranslationOrderFieldLanguage,
Direction: page.OrderDirectionAsc,
},
func(ctx context.Context, cursor *page.Cursor[coredata.CookieBannerTranslationOrderField]) ([]*coredata.CookieBannerTranslation, error) {
var batch coredata.CookieBannerTranslations
if err := batch.LoadByCookieBannerID(ctx, conn, scope, banner.ID, cursor); err != nil {
return nil, fmt.Errorf("cannot load cookie banner translations: %w", err)
}
return batch, nil
},
)
if err != nil {
return err
}
resolved := resolveTranslations(translations, categories)
@@ -1959,7 +2008,28 @@ func (s *Service) ListCookieBannerTranslations(
err := s.pg.WithConn(
ctx,
func(ctx context.Context, conn pg.Querier) error {
return translations.LoadAllByCookieBannerID(ctx, conn, scope, cookieBannerID)
loaded, err := page.LoadAll(
ctx,
page.OrderBy[coredata.CookieBannerTranslationOrderField]{
Field: coredata.CookieBannerTranslationOrderFieldLanguage,
Direction: page.OrderDirectionAsc,
},
func(ctx context.Context, cursor *page.Cursor[coredata.CookieBannerTranslationOrderField]) ([]*coredata.CookieBannerTranslation, error) {
var batch coredata.CookieBannerTranslations
if err := batch.LoadByCookieBannerID(ctx, conn, scope, cookieBannerID, cursor); err != nil {
return nil, fmt.Errorf("cannot load cookie banner translations: %w", err)
}
return batch, nil
},
)
if err != nil {
return err
}
translations = loaded
return nil
},
)
if err != nil {

View File

@@ -29,6 +29,7 @@ import (
"go.probo.inc/probo/pkg/coredata"
"go.probo.inc/probo/pkg/gid"
"go.probo.inc/probo/pkg/llm"
"go.probo.inc/probo/pkg/page"
"go.probo.inc/probo/pkg/stringsx"
"go.probo.inc/probo/pkg/thirdparty"
"go.probo.inc/probo/pkg/uri"
@@ -1216,15 +1217,23 @@ func (h *trackerMappingHandler) prepareOrgThirdParty(
firstLevel := 1
var orgThirdParties coredata.ThirdParties
if err := orgThirdParties.LoadAllByOrganizationID(
orgThirdParties, err := page.LoadAll(
ctx,
conn,
scope,
tp.OrganizationID,
coredata.NewThirdPartyFilter(nil, &firstLevel, nil),
); err != nil {
return prep, fmt.Errorf("cannot load org third parties: %w", err)
page.OrderBy[coredata.ThirdPartyOrderField]{
Field: coredata.ThirdPartyOrderFieldName,
Direction: page.OrderDirectionAsc,
},
func(ctx context.Context, cursor *page.Cursor[coredata.ThirdPartyOrderField]) ([]*coredata.ThirdParty, error) {
var batch coredata.ThirdParties
if err := batch.LoadByOrganizationID(ctx, conn, scope, tp.OrganizationID, cursor, coredata.NewThirdPartyFilter(nil, &firstLevel, nil)); err != nil {
return nil, fmt.Errorf("cannot load org third parties: %w", err)
}
return batch, nil
},
)
if err != nil {
return prep, err
}
ranked := thirdparty.RankCandidates(prep.commonParty, prep.commonDomains, orgThirdParties)