Refacto load all functions
Unbounded LoadAll* loaders materialised an entire result set in one query with no ceiling. A table that is small in development can grow without bound in production, so these loaders were a latent memory and query-time hazard. Remove the LoadAll* methods from pkg/coredata and walk the cursor- paginated LoadBy* siblings instead through a shared page.LoadAll helper. The helper advances a MaxCursorSize forward cursor until the result set is exhausted and concatenates the pages. It caps a single call at MaxLoadAllPages (20) batches of 500 rows and errors past that rather than materialising an unbounded set, so a runaway caller fails loudly instead of exhausting memory. Callers that genuinely need every row now express that explicitly, and the coredata load-naming rule and docs are updated to discourage new unbounded loaders. Signed-off-by: Sacha Al Himdani <sacha@probo.com>
This commit is contained in:
committed by
Sacha Al Himdani
parent
853f2404a6
commit
9ab8ea2085
@@ -27,6 +27,7 @@ import (
|
||||
"go.gearno.de/kit/worker"
|
||||
"go.probo.inc/probo/pkg/coredata"
|
||||
"go.probo.inc/probo/pkg/gid"
|
||||
"go.probo.inc/probo/pkg/page"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -147,16 +148,23 @@ func (h *patternAnalysisHandler) Process(ctx context.Context, banner coredata.Co
|
||||
func(ctx context.Context, tx pg.Tx) error {
|
||||
scope := coredata.NewScopeFromObjectID(banner.ID)
|
||||
|
||||
var exactPatterns coredata.TrackerPatterns
|
||||
if err := exactPatterns.LoadAllByCookieBannerID(
|
||||
exactPatterns, err := page.LoadAll(
|
||||
ctx,
|
||||
tx,
|
||||
scope,
|
||||
banner.ID,
|
||||
coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeExact), nil, new(false)),
|
||||
nil,
|
||||
); err != nil {
|
||||
return fmt.Errorf("cannot load exact patterns: %w", err)
|
||||
page.OrderBy[coredata.TrackerPatternOrderField]{
|
||||
Field: coredata.TrackerPatternOrderFieldCreatedAt,
|
||||
Direction: page.OrderDirectionAsc,
|
||||
},
|
||||
func(ctx context.Context, cursor *page.Cursor[coredata.TrackerPatternOrderField]) ([]*coredata.TrackerPattern, error) {
|
||||
var batch coredata.TrackerPatterns
|
||||
if err := batch.LoadByCookieBannerID(ctx, tx, scope, banner.ID, cursor, coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeExact), nil, new(false))); err != nil {
|
||||
return nil, fmt.Errorf("cannot load exact patterns: %w", err)
|
||||
}
|
||||
|
||||
return batch, nil
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
mergeGroups := findMergeGroups(exactPatterns, patternMergeThreshold)
|
||||
@@ -816,16 +824,23 @@ func (h *patternAnalysisHandler) adoptUncategorisedPatterns(
|
||||
return false, fmt.Errorf("cannot load uncategorised category: %w", err)
|
||||
}
|
||||
|
||||
var globPatterns coredata.TrackerPatterns
|
||||
if err := globPatterns.LoadAllByCookieBannerID(
|
||||
globPatterns, err := page.LoadAll(
|
||||
ctx,
|
||||
tx,
|
||||
scope,
|
||||
banner.ID,
|
||||
coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeGlob), nil, new(false)),
|
||||
nil,
|
||||
); err != nil {
|
||||
return false, fmt.Errorf("cannot load glob patterns: %w", err)
|
||||
page.OrderBy[coredata.TrackerPatternOrderField]{
|
||||
Field: coredata.TrackerPatternOrderFieldCreatedAt,
|
||||
Direction: page.OrderDirectionAsc,
|
||||
},
|
||||
func(ctx context.Context, cursor *page.Cursor[coredata.TrackerPatternOrderField]) ([]*coredata.TrackerPattern, error) {
|
||||
var batch coredata.TrackerPatterns
|
||||
if err := batch.LoadByCookieBannerID(ctx, tx, scope, banner.ID, cursor, coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeGlob), nil, new(false))); err != nil {
|
||||
return nil, fmt.Errorf("cannot load glob patterns: %w", err)
|
||||
}
|
||||
|
||||
return batch, nil
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
if len(globPatterns) == 0 {
|
||||
@@ -841,16 +856,23 @@ func (h *patternAnalysisHandler) adoptUncategorisedPatterns(
|
||||
|
||||
exactMatchType := coredata.TrackerPatternMatchTypeExact
|
||||
|
||||
var uncategorisedExact coredata.TrackerPatterns
|
||||
if err := uncategorisedExact.LoadAllByCookieBannerID(
|
||||
uncategorisedExact, err := page.LoadAll(
|
||||
ctx,
|
||||
tx,
|
||||
scope,
|
||||
banner.ID,
|
||||
coredata.NewTrackerPatternFilter(&exactMatchType, &uncategorised.ID, new(false)),
|
||||
nil,
|
||||
); err != nil {
|
||||
return false, fmt.Errorf("cannot load uncategorised exact patterns: %w", err)
|
||||
page.OrderBy[coredata.TrackerPatternOrderField]{
|
||||
Field: coredata.TrackerPatternOrderFieldCreatedAt,
|
||||
Direction: page.OrderDirectionAsc,
|
||||
},
|
||||
func(ctx context.Context, cursor *page.Cursor[coredata.TrackerPatternOrderField]) ([]*coredata.TrackerPattern, error) {
|
||||
var batch coredata.TrackerPatterns
|
||||
if err := batch.LoadByCookieBannerID(ctx, tx, scope, banner.ID, cursor, coredata.NewTrackerPatternFilter(&exactMatchType, &uncategorised.ID, new(false))); err != nil {
|
||||
return nil, fmt.Errorf("cannot load uncategorised exact patterns: %w", err)
|
||||
}
|
||||
|
||||
return batch, nil
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
adopted := false
|
||||
|
||||
@@ -28,6 +28,7 @@ import (
|
||||
"go.probo.inc/probo/internal/test"
|
||||
"go.probo.inc/probo/pkg/coredata"
|
||||
"go.probo.inc/probo/pkg/gid"
|
||||
"go.probo.inc/probo/pkg/page"
|
||||
)
|
||||
|
||||
// workerFixture bootstraps the parent rows the worker's transaction
|
||||
@@ -324,14 +325,28 @@ func TestPatternAnalysisWorker_PromotesSourceOnExistingGlob(t *testing.T) {
|
||||
var remainingExacts coredata.TrackerPatterns
|
||||
|
||||
require.NoError(t, client.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error {
|
||||
return remainingExacts.LoadAllByCookieBannerID(
|
||||
loaded, err := page.LoadAll(
|
||||
ctx,
|
||||
conn,
|
||||
fx.scope,
|
||||
fx.banner.ID,
|
||||
coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeExact), nil, new(false)),
|
||||
nil,
|
||||
page.OrderBy[coredata.TrackerPatternOrderField]{
|
||||
Field: coredata.TrackerPatternOrderFieldCreatedAt,
|
||||
Direction: page.OrderDirectionAsc,
|
||||
},
|
||||
func(ctx context.Context, cursor *page.Cursor[coredata.TrackerPatternOrderField]) ([]*coredata.TrackerPattern, error) {
|
||||
var batch coredata.TrackerPatterns
|
||||
if err := batch.LoadByCookieBannerID(ctx, conn, fx.scope, fx.banner.ID, cursor, coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeExact), nil, new(false))); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return batch, nil
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
remainingExacts = loaded
|
||||
|
||||
return nil
|
||||
}))
|
||||
assert.Empty(t, remainingExacts, "all three exacts must be relinked and deleted")
|
||||
}
|
||||
@@ -401,14 +416,28 @@ func TestPatternAnalysisWorker_AdoptionTriggersDraftVersion(t *testing.T) {
|
||||
var remainingExacts coredata.TrackerPatterns
|
||||
|
||||
require.NoError(t, client.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error {
|
||||
return remainingExacts.LoadAllByCookieBannerID(
|
||||
loaded, err := page.LoadAll(
|
||||
ctx,
|
||||
conn,
|
||||
fx.scope,
|
||||
fx.banner.ID,
|
||||
coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeExact), nil, new(false)),
|
||||
nil,
|
||||
page.OrderBy[coredata.TrackerPatternOrderField]{
|
||||
Field: coredata.TrackerPatternOrderFieldCreatedAt,
|
||||
Direction: page.OrderDirectionAsc,
|
||||
},
|
||||
func(ctx context.Context, cursor *page.Cursor[coredata.TrackerPatternOrderField]) ([]*coredata.TrackerPattern, error) {
|
||||
var batch coredata.TrackerPatterns
|
||||
if err := batch.LoadByCookieBannerID(ctx, conn, fx.scope, fx.banner.ID, cursor, coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeExact), nil, new(false))); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return batch, nil
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
remainingExacts = loaded
|
||||
|
||||
return nil
|
||||
}))
|
||||
assert.Empty(t, remainingExacts, "adoptUncategorisedPatterns must absorb the uncategorised exacts into the existing glob")
|
||||
|
||||
@@ -486,14 +515,28 @@ func TestPatternAnalysisWorker_AdoptionPromotesSourceCrossCategory(t *testing.T)
|
||||
var remainingExacts coredata.TrackerPatterns
|
||||
|
||||
require.NoError(t, client.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error {
|
||||
return remainingExacts.LoadAllByCookieBannerID(
|
||||
loaded, err := page.LoadAll(
|
||||
ctx,
|
||||
conn,
|
||||
fx.scope,
|
||||
fx.banner.ID,
|
||||
coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeExact), nil, new(false)),
|
||||
nil,
|
||||
page.OrderBy[coredata.TrackerPatternOrderField]{
|
||||
Field: coredata.TrackerPatternOrderFieldCreatedAt,
|
||||
Direction: page.OrderDirectionAsc,
|
||||
},
|
||||
func(ctx context.Context, cursor *page.Cursor[coredata.TrackerPatternOrderField]) ([]*coredata.TrackerPattern, error) {
|
||||
var batch coredata.TrackerPatterns
|
||||
if err := batch.LoadByCookieBannerID(ctx, conn, fx.scope, fx.banner.ID, cursor, coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeExact), nil, new(false))); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return batch, nil
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
remainingExacts = loaded
|
||||
|
||||
return nil
|
||||
}))
|
||||
assert.Empty(t, remainingExacts, "the uncategorised exact must be adopted into the existing glob")
|
||||
}
|
||||
@@ -559,14 +602,28 @@ func TestReportDetectedTrackers_PromotesSourceOnExistingGlob(t *testing.T) {
|
||||
var exacts coredata.TrackerPatterns
|
||||
|
||||
require.NoError(t, client.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error {
|
||||
return exacts.LoadAllByCookieBannerID(
|
||||
loaded, err := page.LoadAll(
|
||||
ctx,
|
||||
conn,
|
||||
fx.scope,
|
||||
fx.banner.ID,
|
||||
coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeExact), nil, new(false)),
|
||||
nil,
|
||||
page.OrderBy[coredata.TrackerPatternOrderField]{
|
||||
Field: coredata.TrackerPatternOrderFieldCreatedAt,
|
||||
Direction: page.OrderDirectionAsc,
|
||||
},
|
||||
func(ctx context.Context, cursor *page.Cursor[coredata.TrackerPatternOrderField]) ([]*coredata.TrackerPattern, error) {
|
||||
var batch coredata.TrackerPatterns
|
||||
if err := batch.LoadByCookieBannerID(ctx, conn, fx.scope, fx.banner.ID, cursor, coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeExact), nil, new(false))); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return batch, nil
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
exacts = loaded
|
||||
|
||||
return nil
|
||||
}))
|
||||
assert.Empty(t, exacts, "the detected cookie globMatches the existing glob; no exact pattern must be created")
|
||||
}
|
||||
@@ -608,14 +665,28 @@ func TestPatternAnalysisWorker_MergeWithoutAdoptionSkipsDraftVersion(t *testing.
|
||||
var globs coredata.TrackerPatterns
|
||||
|
||||
require.NoError(t, client.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error {
|
||||
return globs.LoadAllByCookieBannerID(
|
||||
loaded, err := page.LoadAll(
|
||||
ctx,
|
||||
conn,
|
||||
fx.scope,
|
||||
fx.banner.ID,
|
||||
coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeGlob), nil, new(false)),
|
||||
nil,
|
||||
page.OrderBy[coredata.TrackerPatternOrderField]{
|
||||
Field: coredata.TrackerPatternOrderFieldCreatedAt,
|
||||
Direction: page.OrderDirectionAsc,
|
||||
},
|
||||
func(ctx context.Context, cursor *page.Cursor[coredata.TrackerPatternOrderField]) ([]*coredata.TrackerPattern, error) {
|
||||
var batch coredata.TrackerPatterns
|
||||
if err := batch.LoadByCookieBannerID(ctx, conn, fx.scope, fx.banner.ID, cursor, coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeGlob), nil, new(false))); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return batch, nil
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
globs = loaded
|
||||
|
||||
return nil
|
||||
}))
|
||||
require.Len(t, globs, 1, "the three exacts must consolidate into a single glob")
|
||||
assert.Equal(t, "_ga_*", globs[0].Pattern)
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"go.gearno.de/kit/pg"
|
||||
"go.probo.inc/probo/pkg/coredata"
|
||||
"go.probo.inc/probo/pkg/gid"
|
||||
"go.probo.inc/probo/pkg/page"
|
||||
)
|
||||
|
||||
// ResetTrackersResult summarizes what a banner reset changed.
|
||||
@@ -122,22 +123,43 @@ func decomposeGlobs(
|
||||
globMatchType := coredata.TrackerPatternMatchTypeGlob
|
||||
notExcluded := false
|
||||
|
||||
var globs coredata.TrackerPatterns
|
||||
if err := globs.LoadAllByCookieBannerID(
|
||||
globs, err := page.LoadAll(
|
||||
ctx,
|
||||
tx,
|
||||
scope,
|
||||
bannerID,
|
||||
coredata.NewTrackerPatternFilter(&globMatchType, &uncategorisedID, ¬Excluded).WithPatternKeyword(keyword),
|
||||
nil,
|
||||
); err != nil {
|
||||
return fmt.Errorf("cannot load glob patterns: %w", err)
|
||||
page.OrderBy[coredata.TrackerPatternOrderField]{
|
||||
Field: coredata.TrackerPatternOrderFieldCreatedAt,
|
||||
Direction: page.OrderDirectionAsc,
|
||||
},
|
||||
func(ctx context.Context, cursor *page.Cursor[coredata.TrackerPatternOrderField]) ([]*coredata.TrackerPattern, error) {
|
||||
var batch coredata.TrackerPatterns
|
||||
if err := batch.LoadByCookieBannerID(ctx, tx, scope, bannerID, cursor, coredata.NewTrackerPatternFilter(&globMatchType, &uncategorisedID, ¬Excluded).WithPatternKeyword(keyword)); err != nil {
|
||||
return nil, fmt.Errorf("cannot load glob patterns: %w", err)
|
||||
}
|
||||
|
||||
return batch, nil
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, glob := range globs {
|
||||
var detections coredata.DetectedTrackers
|
||||
if err := detections.LoadAllByTrackerPatternID(ctx, tx, scope, glob.ID); err != nil {
|
||||
return fmt.Errorf("cannot load detections for glob %q: %w", glob.Pattern, err)
|
||||
detections, err := page.LoadAll(
|
||||
ctx,
|
||||
page.OrderBy[coredata.DetectedTrackerOrderField]{
|
||||
Field: coredata.DetectedTrackerOrderFieldLastDetectedAt,
|
||||
Direction: page.OrderDirectionAsc,
|
||||
},
|
||||
func(ctx context.Context, cursor *page.Cursor[coredata.DetectedTrackerOrderField]) ([]*coredata.DetectedTracker, error) {
|
||||
var batch coredata.DetectedTrackers
|
||||
if err := batch.LoadByTrackerPatternID(ctx, tx, scope, glob.ID, cursor); err != nil {
|
||||
return nil, fmt.Errorf("cannot load detections for glob %q: %w", glob.Pattern, err)
|
||||
}
|
||||
|
||||
return batch, nil
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, detection := range detections {
|
||||
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
"go.probo.inc/probo/internal/test"
|
||||
"go.probo.inc/probo/pkg/coredata"
|
||||
"go.probo.inc/probo/pkg/gid"
|
||||
"go.probo.inc/probo/pkg/page"
|
||||
)
|
||||
|
||||
// TestResetBannerTrackers_FullRebuild seeds a banner with an
|
||||
@@ -113,8 +114,22 @@ func TestResetBannerTrackers_FullRebuild(t *testing.T) {
|
||||
require.Nil(t, exact.ThirdPartyID)
|
||||
require.NotNil(t, exact.MappingRequestedAt)
|
||||
|
||||
var detections coredata.DetectedTrackers
|
||||
require.NoError(t, detections.LoadAllByTrackerPatternID(ctx, conn, fx.scope, exact.ID))
|
||||
detections, err := page.LoadAll(
|
||||
ctx,
|
||||
page.OrderBy[coredata.DetectedTrackerOrderField]{
|
||||
Field: coredata.DetectedTrackerOrderFieldLastDetectedAt,
|
||||
Direction: page.OrderDirectionAsc,
|
||||
},
|
||||
func(ctx context.Context, cursor *page.Cursor[coredata.DetectedTrackerOrderField]) ([]*coredata.DetectedTracker, error) {
|
||||
var batch coredata.DetectedTrackers
|
||||
if err := batch.LoadByTrackerPatternID(ctx, conn, fx.scope, exact.ID, cursor); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return batch, nil
|
||||
},
|
||||
)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, detections, 1)
|
||||
require.Equal(t, identifier, detections[0].Identifier)
|
||||
}
|
||||
|
||||
@@ -553,21 +553,42 @@ func (s *Service) ensureDraftVersionForBanner(
|
||||
|
||||
consentFilter := coredata.NewCookieCategoryFilter(new(coredata.CookieCategoryKindUncategorised))
|
||||
|
||||
var categories coredata.CookieCategories
|
||||
if err := categories.LoadAllByCookieBannerID(ctx, tx, scope, bannerID, consentFilter); err != nil {
|
||||
return nil, fmt.Errorf("cannot load cookie categories: %w", err)
|
||||
categories, err := page.LoadAll(
|
||||
ctx,
|
||||
page.OrderBy[coredata.CookieCategoryOrderField]{
|
||||
Field: coredata.CookieCategoryOrderFieldRank,
|
||||
Direction: page.OrderDirectionAsc,
|
||||
},
|
||||
func(ctx context.Context, cursor *page.Cursor[coredata.CookieCategoryOrderField]) ([]*coredata.CookieCategory, error) {
|
||||
var batch coredata.CookieCategories
|
||||
if err := batch.LoadByCookieBannerID(ctx, tx, scope, bannerID, cursor, consentFilter); err != nil {
|
||||
return nil, fmt.Errorf("cannot load cookie categories: %w", err)
|
||||
}
|
||||
|
||||
return batch, nil
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var allPatterns coredata.TrackerPatterns
|
||||
if err := allPatterns.LoadAllByCookieBannerID(
|
||||
allPatterns, err := page.LoadAll(
|
||||
ctx,
|
||||
tx,
|
||||
scope,
|
||||
bannerID,
|
||||
coredata.NewTrackerPatternFilter(nil, nil, new(false)),
|
||||
nil,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("cannot load tracker patterns: %w", err)
|
||||
page.OrderBy[coredata.TrackerPatternOrderField]{
|
||||
Field: coredata.TrackerPatternOrderFieldCreatedAt,
|
||||
Direction: page.OrderDirectionAsc,
|
||||
},
|
||||
func(ctx context.Context, cursor *page.Cursor[coredata.TrackerPatternOrderField]) ([]*coredata.TrackerPattern, error) {
|
||||
var batch coredata.TrackerPatterns
|
||||
if err := batch.LoadByCookieBannerID(ctx, tx, scope, bannerID, cursor, coredata.NewTrackerPatternFilter(nil, nil, new(false))); err != nil {
|
||||
return nil, fmt.Errorf("cannot load tracker patterns: %w", err)
|
||||
}
|
||||
|
||||
return batch, nil
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return s.ensureDraftVersion(ctx, tx, scope, &banner, categories, allPatterns)
|
||||
@@ -1695,14 +1716,42 @@ func (s *Service) GetActiveBannerConfig(
|
||||
|
||||
consentFilter := coredata.NewCookieCategoryFilter(new(coredata.CookieCategoryKindUncategorised))
|
||||
|
||||
var categories coredata.CookieCategories
|
||||
if err := categories.LoadAllByCookieBannerID(ctx, conn, scope, banner.ID, consentFilter); err != nil {
|
||||
return fmt.Errorf("cannot load cookie categories: %w", err)
|
||||
categories, err := page.LoadAll(
|
||||
ctx,
|
||||
page.OrderBy[coredata.CookieCategoryOrderField]{
|
||||
Field: coredata.CookieCategoryOrderFieldRank,
|
||||
Direction: page.OrderDirectionAsc,
|
||||
},
|
||||
func(ctx context.Context, cursor *page.Cursor[coredata.CookieCategoryOrderField]) ([]*coredata.CookieCategory, error) {
|
||||
var batch coredata.CookieCategories
|
||||
if err := batch.LoadByCookieBannerID(ctx, conn, scope, banner.ID, cursor, consentFilter); err != nil {
|
||||
return nil, fmt.Errorf("cannot load cookie categories: %w", err)
|
||||
}
|
||||
|
||||
return batch, nil
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var translations coredata.CookieBannerTranslations
|
||||
if err := translations.LoadAllByCookieBannerID(ctx, conn, scope, banner.ID); err != nil {
|
||||
return fmt.Errorf("cannot load cookie banner translations: %w", err)
|
||||
translations, err := page.LoadAll(
|
||||
ctx,
|
||||
page.OrderBy[coredata.CookieBannerTranslationOrderField]{
|
||||
Field: coredata.CookieBannerTranslationOrderFieldLanguage,
|
||||
Direction: page.OrderDirectionAsc,
|
||||
},
|
||||
func(ctx context.Context, cursor *page.Cursor[coredata.CookieBannerTranslationOrderField]) ([]*coredata.CookieBannerTranslation, error) {
|
||||
var batch coredata.CookieBannerTranslations
|
||||
if err := batch.LoadByCookieBannerID(ctx, conn, scope, banner.ID, cursor); err != nil {
|
||||
return nil, fmt.Errorf("cannot load cookie banner translations: %w", err)
|
||||
}
|
||||
|
||||
return batch, nil
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resolved := resolveTranslations(translations, categories)
|
||||
@@ -1959,7 +2008,28 @@ func (s *Service) ListCookieBannerTranslations(
|
||||
err := s.pg.WithConn(
|
||||
ctx,
|
||||
func(ctx context.Context, conn pg.Querier) error {
|
||||
return translations.LoadAllByCookieBannerID(ctx, conn, scope, cookieBannerID)
|
||||
loaded, err := page.LoadAll(
|
||||
ctx,
|
||||
page.OrderBy[coredata.CookieBannerTranslationOrderField]{
|
||||
Field: coredata.CookieBannerTranslationOrderFieldLanguage,
|
||||
Direction: page.OrderDirectionAsc,
|
||||
},
|
||||
func(ctx context.Context, cursor *page.Cursor[coredata.CookieBannerTranslationOrderField]) ([]*coredata.CookieBannerTranslation, error) {
|
||||
var batch coredata.CookieBannerTranslations
|
||||
if err := batch.LoadByCookieBannerID(ctx, conn, scope, cookieBannerID, cursor); err != nil {
|
||||
return nil, fmt.Errorf("cannot load cookie banner translations: %w", err)
|
||||
}
|
||||
|
||||
return batch, nil
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
translations = loaded
|
||||
|
||||
return nil
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
|
||||
@@ -29,6 +29,7 @@ import (
|
||||
"go.probo.inc/probo/pkg/coredata"
|
||||
"go.probo.inc/probo/pkg/gid"
|
||||
"go.probo.inc/probo/pkg/llm"
|
||||
"go.probo.inc/probo/pkg/page"
|
||||
"go.probo.inc/probo/pkg/stringsx"
|
||||
"go.probo.inc/probo/pkg/thirdparty"
|
||||
"go.probo.inc/probo/pkg/uri"
|
||||
@@ -1216,15 +1217,23 @@ func (h *trackerMappingHandler) prepareOrgThirdParty(
|
||||
|
||||
firstLevel := 1
|
||||
|
||||
var orgThirdParties coredata.ThirdParties
|
||||
if err := orgThirdParties.LoadAllByOrganizationID(
|
||||
orgThirdParties, err := page.LoadAll(
|
||||
ctx,
|
||||
conn,
|
||||
scope,
|
||||
tp.OrganizationID,
|
||||
coredata.NewThirdPartyFilter(nil, &firstLevel, nil),
|
||||
); err != nil {
|
||||
return prep, fmt.Errorf("cannot load org third parties: %w", err)
|
||||
page.OrderBy[coredata.ThirdPartyOrderField]{
|
||||
Field: coredata.ThirdPartyOrderFieldName,
|
||||
Direction: page.OrderDirectionAsc,
|
||||
},
|
||||
func(ctx context.Context, cursor *page.Cursor[coredata.ThirdPartyOrderField]) ([]*coredata.ThirdParty, error) {
|
||||
var batch coredata.ThirdParties
|
||||
if err := batch.LoadByOrganizationID(ctx, conn, scope, tp.OrganizationID, cursor, coredata.NewThirdPartyFilter(nil, &firstLevel, nil)); err != nil {
|
||||
return nil, fmt.Errorf("cannot load org third parties: %w", err)
|
||||
}
|
||||
|
||||
return batch, nil
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return prep, err
|
||||
}
|
||||
|
||||
ranked := thirdparty.RankCandidates(prep.commonParty, prep.commonDomains, orgThirdParties)
|
||||
|
||||
Reference in New Issue
Block a user