rohit/probo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refacto load all functions

Browse Source 
Unbounded LoadAll* loaders materialised an entire result set in one
query with no ceiling. A table that is small in development can grow
without bound in production, so these loaders were a latent memory
and query-time hazard.

Remove the LoadAll* methods from pkg/coredata and walk the cursor-
paginated LoadBy* siblings instead through a shared page.LoadAll
helper. The helper advances a MaxCursorSize forward cursor until the
result set is exhausted and concatenates the pages. It caps a single
call at MaxLoadAllPages (20) batches of 500 rows and errors past that
rather than materialising an unbounded set, so a runaway caller fails
loudly instead of exhausting memory.

Callers that genuinely need every row now express that explicitly,
and the coredata load-naming rule and docs are updated to discourage
new unbounded loaders.

Signed-off-by: Sacha Al Himdani <sacha@probo.com>
This commit is contained in:
Sacha Al Himdani
2026-06-09 19:33:36 +02:00
committed by Sacha Al Himdani
parent 853f2404a6
commit 9ab8ea2085
46 changed files with 1218 additions and 1674 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
pkg/cookiebanner/pattern_analysis_worker.go
View File

@@ -27,6 +27,7 @@ import (
"go.gearno.de/kit/worker"
"go.probo.inc/probo/pkg/coredata"
"go.probo.inc/probo/pkg/gid"
"go.probo.inc/probo/pkg/page"
)
const (
@@ -147,16 +148,23 @@ func (h *patternAnalysisHandler) Process(ctx context.Context, banner coredata.Co
func(ctx context.Context, tx pg.Tx) error {
scope := coredata.NewScopeFromObjectID(banner.ID)
var exactPatterns coredata.TrackerPatterns
if err := exactPatterns.LoadAllByCookieBannerID(
exactPatterns, err := page.LoadAll(
ctx,
tx,
scope,
banner.ID,
coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeExact), nil, new(false)),
nil,
); err != nil {
return fmt.Errorf("cannot load exact patterns: %w", err)
page.OrderBy[coredata.TrackerPatternOrderField]{
Field: coredata.TrackerPatternOrderFieldCreatedAt,
Direction: page.OrderDirectionAsc,
},
func(ctx context.Context, cursor *page.Cursor[coredata.TrackerPatternOrderField]) ([]*coredata.TrackerPattern, error) {
var batch coredata.TrackerPatterns
if err := batch.LoadByCookieBannerID(ctx, tx, scope, banner.ID, cursor, coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeExact), nil, new(false))); err != nil {
return nil, fmt.Errorf("cannot load exact patterns: %w", err)
}
return batch, nil
},
)
if err != nil {
return err
}
mergeGroups := findMergeGroups(exactPatterns, patternMergeThreshold)
@@ -816,16 +824,23 @@ func (h *patternAnalysisHandler) adoptUncategorisedPatterns(
return false, fmt.Errorf("cannot load uncategorised category: %w", err)
}
var globPatterns coredata.TrackerPatterns
if err := globPatterns.LoadAllByCookieBannerID(
globPatterns, err := page.LoadAll(
ctx,
tx,
scope,
banner.ID,
coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeGlob), nil, new(false)),
nil,
); err != nil {
return false, fmt.Errorf("cannot load glob patterns: %w", err)
page.OrderBy[coredata.TrackerPatternOrderField]{
Field: coredata.TrackerPatternOrderFieldCreatedAt,
Direction: page.OrderDirectionAsc,
},
func(ctx context.Context, cursor *page.Cursor[coredata.TrackerPatternOrderField]) ([]*coredata.TrackerPattern, error) {
var batch coredata.TrackerPatterns
if err := batch.LoadByCookieBannerID(ctx, tx, scope, banner.ID, cursor, coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeGlob), nil, new(false))); err != nil {
return nil, fmt.Errorf("cannot load glob patterns: %w", err)
}
return batch, nil
},
)
if err != nil {
return false, err
}
if len(globPatterns) == 0 {
@@ -841,16 +856,23 @@ func (h *patternAnalysisHandler) adoptUncategorisedPatterns(
exactMatchType := coredata.TrackerPatternMatchTypeExact
var uncategorisedExact coredata.TrackerPatterns
if err := uncategorisedExact.LoadAllByCookieBannerID(
uncategorisedExact, err := page.LoadAll(
ctx,
tx,
scope,
banner.ID,
coredata.NewTrackerPatternFilter(&exactMatchType, &uncategorised.ID, new(false)),
nil,
); err != nil {
return false, fmt.Errorf("cannot load uncategorised exact patterns: %w", err)
page.OrderBy[coredata.TrackerPatternOrderField]{
Field: coredata.TrackerPatternOrderFieldCreatedAt,
Direction: page.OrderDirectionAsc,
},
func(ctx context.Context, cursor *page.Cursor[coredata.TrackerPatternOrderField]) ([]*coredata.TrackerPattern, error) {
var batch coredata.TrackerPatterns
if err := batch.LoadByCookieBannerID(ctx, tx, scope, banner.ID, cursor, coredata.NewTrackerPatternFilter(&exactMatchType, &uncategorised.ID, new(false))); err != nil {
return nil, fmt.Errorf("cannot load uncategorised exact patterns: %w", err)
}
return batch, nil
},
)
if err != nil {
return false, err
}
adopted := false