diff --git a/.cursor/rules/go-coredata-load-naming.mdc b/.cursor/rules/go-coredata-load-naming.mdc index 456d85c0e..14b6cc2ee 100644 --- a/.cursor/rules/go-coredata-load-naming.mdc +++ b/.cursor/rules/go-coredata-load-naming.mdc @@ -6,49 +6,32 @@ alwaysApply: false # Coredata Load vs LoadAll naming +Do **not** add new unbounded `LoadAll*` loaders that materialise an entire +result set with one query. They have no ceiling: a table that is small in +dev can grow without bound in production, blowing up memory and query time. +Expose a cursor-paginated `LoadBy*` instead, and when a caller genuinely +needs every row, walk that method with the `page.LoadAll` helper (see +[`pkg/page/load_all.go`](../../pkg/page/load_all.go)). + The method name signals whether the result set is bounded: -- **`LoadBy*` with a `cursor` param** — paginated list; the cursor provides limit and ordering. -- **`Load` / `LoadBy*` with a `limit int` param** — filtered list with explicit limit, when cursor pagination is not needed but the caller controls the result count. -- **`LoadAllBy*`** — returns all matching rows, no limit or cursor. -- **`LoadAll`** — same as `LoadAllBy*` but without a parent key; returns all rows matching a filter. +- **`LoadBy*` with a `cursor` param** — paginated list; the cursor provides + the limit and ordering. This is the primary list shape; prefer it. +- **`Load` / `LoadBy*` with a `limit int` param** — filtered list with an + explicit, hard-capped limit (e.g. `... LIMIT 20`), when the caller controls + a small bounded result count. +- **`LoadAllBy*` / `LoadAll`** — legacy unbounded loaders. Do not add new + ones. The few that remain are deliberate exceptions: tiny per-parent sets + (handful of rows), `[]gid.GID` / map projections, or hard-`LIMIT` search + helpers. When in doubt, use `LoadBy*` + `page.LoadAll`. -`LoadAll*` methods must **never** accept a cursor or limit parameter — `All` means the entire matching set is returned. The codebase has some legacy `LoadAllBy*` methods that accept a cursor; do not follow that pattern — new code must use `LoadBy*` for paginated queries. +`LoadAll*` methods must **never** accept a cursor or limit parameter — the +`All` suffix means the entire matching set is returned. The codebase has +some legacy `LoadAllBy*` methods that accept a cursor; do not follow that +pattern. ```go -// GOOD — explicit limit, named Load -func (ds *Things) Load( - ctx context.Context, - conn pg.Querier, - limit int, - filter *ThingFilter, -) error { - -// GOOD — no limit, named LoadAll -func (ds *Things) LoadAll( - ctx context.Context, - conn pg.Querier, - filter *ThingFilter, -) error { - -// BAD — LoadAll with a limit -func (ds *Things) LoadAll( - ctx context.Context, - conn pg.Querier, - limit int, - filter *ThingFilter, -) error { - -// BAD — LoadAllBy with a cursor (legacy pattern, do not use) -func (ds *Things) LoadAllByParentID( - ctx context.Context, - conn pg.Querier, - scope Scoper, - parentID gid.GID, - cursor *page.Cursor[ThingOrderField], -) error { - -// GOOD — paginated query uses LoadBy, not LoadAll +// GOOD — paginated query uses LoadBy with a cursor func (ds *Things) LoadByParentID( ctx context.Context, conn pg.Querier, @@ -56,8 +39,53 @@ func (ds *Things) LoadByParentID( parentID gid.GID, cursor *page.Cursor[ThingOrderField], ) error { + +// GOOD — caller that needs every row walks the paginated method +things, err := page.LoadAll( + ctx, + page.OrderBy[coredata.ThingOrderField]{ + Field: coredata.ThingOrderFieldCreatedAt, + Direction: page.OrderDirectionAsc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.ThingOrderField]) ([]*coredata.Thing, error) { + var batch coredata.Things + if err := batch.LoadByParentID(ctx, conn, scope, parentID, cursor); err != nil { + return nil, err + } + + return batch, nil + }, +) + +// GOOD — explicit hard-capped limit, named Load +func (ds *Things) Load( + ctx context.Context, + conn pg.Querier, + limit int, + filter *ThingFilter, +) error { + +// BAD — new unbounded loader; add LoadByParentID + page.LoadAll instead +func (ds *Things) LoadAllByParentID( + ctx context.Context, + conn pg.Querier, + scope Scoper, + parentID gid.GID, +) error { + +// BAD — LoadAll with a limit, or LoadAllBy with a cursor (legacy, do not use) +func (ds *Things) LoadAll( + ctx context.Context, + conn pg.Querier, + limit int, + filter *ThingFilter, +) error { ``` +Whatever order field the loop uses must have a `CursorKey` case on the +entity — `CursorKey` panics at runtime (not compile time) on an unhandled +field, so add the case when introducing the order field. + # Cross-entity table references Each entity file in `pkg/coredata` queries its own table. Never JOIN two diff --git a/contrib/claude/coredata.md b/contrib/claude/coredata.md index fdb3b4ed2..6f1a34609 100644 --- a/contrib/claude/coredata.md +++ b/contrib/claude/coredata.md @@ -102,8 +102,8 @@ This ensures the compiler catches renamed or removed enum values instead of sile | `LoadBy*(ctx, conn, scope, key)` | `*Entity` | `error` | Single entity by unique key | | `LoadBy*(ctx, conn, scope, parentID, cursor, filter)` | `*Entities` | `error` | Paginated list (cursor provides limit) | | `Load(ctx, conn, limit, filter)` | `*Entities` | `error` | Filtered list with explicit limit | -| `LoadAllBy*(ctx, conn, scope, parentID)` | `*Entities` | `error` | All matching rows (never cursor/limit) | -| `LoadAll(ctx, conn, filter)` | `*Entities` | `error` | All matching rows with filter (never cursor/limit) | +| `LoadAllBy*(ctx, conn, scope, parentID)` | `*Entities` | `error` | Legacy unbounded loader — do not add new ones (use `LoadBy*` + `page.LoadAll`) | +| `LoadAll(ctx, conn, filter)` | `*Entities` | `error` | Legacy unbounded loader — do not add new ones (use `LoadBy*` + `page.LoadAll`) | | `CountBy*(ctx, conn, scope, parentID, filter)` | `*Entities` | `(int, error)` | Count matching rows | | `Insert(ctx, conn, scope)` | `*Entity` | `error` | Insert, uses `scope.GetTenantID()` | | `Update(ctx, conn, scope)` | `*Entity` | `error` | Update via `Exec` (no `RETURNING`) | @@ -115,12 +115,35 @@ This ensures the compiler catches renamed or removed enum values instead of sile The method name signals whether the result set is bounded: -- **`LoadBy*` with a `cursor` param** — paginated list tied to a GraphQL connection; the cursor provides the limit and ordering. Example: `Assets.LoadByOrganizationID(ctx, conn, scope, orgID, cursor)`. -- **`Load` / `LoadBy*` with a `limit int` param** — filtered list with an explicit limit, used when cursor pagination is not needed but the caller controls the result count. Example: `CommonThirdPartyDomains.Load(ctx, conn, 1, filter)`. -- **`LoadAllBy*`** — returns all matching rows with no limit or cursor. Use only when the full set is needed (e.g. all categories for a banner). -- **`LoadAll`** — same as `LoadAllBy*` but without a parent key; returns all rows matching a filter. Example: `CommonThirdParties.LoadAll(ctx, conn, filter)`. +- **`LoadBy*` with a `cursor` param** — paginated list tied to a GraphQL connection; the cursor provides the limit and ordering. Example: `Assets.LoadByOrganizationID(ctx, conn, scope, orgID, cursor)`. This is the primary list shape; prefer it. +- **`Load` / `LoadBy*` with a `limit int` param** — filtered list with an explicit, hard-capped limit (e.g. `... LIMIT 20`), used when the caller controls a small bounded result count. Example: `CommonThirdPartyDomains.Load(ctx, conn, 1, filter)`. +- **`LoadAllBy*` / `LoadAll`** — legacy unbounded loaders. **Do not add new ones.** They run one query with no ceiling, so a table that is small in dev can blow up memory and query time in production. The few that remain are deliberate exceptions: tiny per-parent sets, `[]gid.GID` / map projections, or hard-`LIMIT` search helpers. -`LoadAll*` methods must **never** accept a cursor or limit parameter — the `All` suffix means the entire matching set is returned. If a bounded result is needed, use `Load*` or `LoadBy*` with a `cursor` or `limit int` instead. The codebase has some legacy `LoadAllBy*` methods that accept a cursor; do not follow that pattern — new code must use `LoadBy*` for paginated queries. +`LoadAll*` methods must **never** accept a cursor or limit parameter — the `All` suffix means the entire matching set is returned. + +### Loading every row without an unbounded query + +When a caller genuinely needs all rows, expose a cursor-paginated `LoadBy*` and walk it with the generic `page.LoadAll` helper ([`pkg/page/load_all.go`](../../pkg/page/load_all.go)). It repeatedly fetches forward pages of `MaxCursorSize` until the result is exhausted, and returns an error past `MaxLoadAllPages` (20) pages so a genuinely unbounded set fails loudly instead of exhausting memory. + +```go +things, err := page.LoadAll( + ctx, + page.OrderBy[coredata.ThingOrderField]{ + Field: coredata.ThingOrderFieldCreatedAt, + Direction: page.OrderDirectionAsc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.ThingOrderField]) ([]*coredata.Thing, error) { + var batch coredata.Things + if err := batch.LoadByParentID(ctx, conn, scope, parentID, cursor); err != nil { + return nil, err + } + + return batch, nil + }, +) +``` + +The order field passed to `page.LoadAll` must have a `CursorKey` case on the entity. `CursorKey` panics at runtime (not compile time) on an unhandled field, so add the case when introducing the order field. ## No cross-entity JOINs diff --git a/pkg/cookiebanner/pattern_analysis_worker.go b/pkg/cookiebanner/pattern_analysis_worker.go index 3fadbae0a..d3ce012fb 100644 --- a/pkg/cookiebanner/pattern_analysis_worker.go +++ b/pkg/cookiebanner/pattern_analysis_worker.go @@ -27,6 +27,7 @@ import ( "go.gearno.de/kit/worker" "go.probo.inc/probo/pkg/coredata" "go.probo.inc/probo/pkg/gid" + "go.probo.inc/probo/pkg/page" ) const ( @@ -147,16 +148,23 @@ func (h *patternAnalysisHandler) Process(ctx context.Context, banner coredata.Co func(ctx context.Context, tx pg.Tx) error { scope := coredata.NewScopeFromObjectID(banner.ID) - var exactPatterns coredata.TrackerPatterns - if err := exactPatterns.LoadAllByCookieBannerID( + exactPatterns, err := page.LoadAll( ctx, - tx, - scope, - banner.ID, - coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeExact), nil, new(false)), - nil, - ); err != nil { - return fmt.Errorf("cannot load exact patterns: %w", err) + page.OrderBy[coredata.TrackerPatternOrderField]{ + Field: coredata.TrackerPatternOrderFieldCreatedAt, + Direction: page.OrderDirectionAsc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.TrackerPatternOrderField]) ([]*coredata.TrackerPattern, error) { + var batch coredata.TrackerPatterns + if err := batch.LoadByCookieBannerID(ctx, tx, scope, banner.ID, cursor, coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeExact), nil, new(false))); err != nil { + return nil, fmt.Errorf("cannot load exact patterns: %w", err) + } + + return batch, nil + }, + ) + if err != nil { + return err } mergeGroups := findMergeGroups(exactPatterns, patternMergeThreshold) @@ -816,16 +824,23 @@ func (h *patternAnalysisHandler) adoptUncategorisedPatterns( return false, fmt.Errorf("cannot load uncategorised category: %w", err) } - var globPatterns coredata.TrackerPatterns - if err := globPatterns.LoadAllByCookieBannerID( + globPatterns, err := page.LoadAll( ctx, - tx, - scope, - banner.ID, - coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeGlob), nil, new(false)), - nil, - ); err != nil { - return false, fmt.Errorf("cannot load glob patterns: %w", err) + page.OrderBy[coredata.TrackerPatternOrderField]{ + Field: coredata.TrackerPatternOrderFieldCreatedAt, + Direction: page.OrderDirectionAsc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.TrackerPatternOrderField]) ([]*coredata.TrackerPattern, error) { + var batch coredata.TrackerPatterns + if err := batch.LoadByCookieBannerID(ctx, tx, scope, banner.ID, cursor, coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeGlob), nil, new(false))); err != nil { + return nil, fmt.Errorf("cannot load glob patterns: %w", err) + } + + return batch, nil + }, + ) + if err != nil { + return false, err } if len(globPatterns) == 0 { @@ -841,16 +856,23 @@ func (h *patternAnalysisHandler) adoptUncategorisedPatterns( exactMatchType := coredata.TrackerPatternMatchTypeExact - var uncategorisedExact coredata.TrackerPatterns - if err := uncategorisedExact.LoadAllByCookieBannerID( + uncategorisedExact, err := page.LoadAll( ctx, - tx, - scope, - banner.ID, - coredata.NewTrackerPatternFilter(&exactMatchType, &uncategorised.ID, new(false)), - nil, - ); err != nil { - return false, fmt.Errorf("cannot load uncategorised exact patterns: %w", err) + page.OrderBy[coredata.TrackerPatternOrderField]{ + Field: coredata.TrackerPatternOrderFieldCreatedAt, + Direction: page.OrderDirectionAsc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.TrackerPatternOrderField]) ([]*coredata.TrackerPattern, error) { + var batch coredata.TrackerPatterns + if err := batch.LoadByCookieBannerID(ctx, tx, scope, banner.ID, cursor, coredata.NewTrackerPatternFilter(&exactMatchType, &uncategorised.ID, new(false))); err != nil { + return nil, fmt.Errorf("cannot load uncategorised exact patterns: %w", err) + } + + return batch, nil + }, + ) + if err != nil { + return false, err } adopted := false diff --git a/pkg/cookiebanner/pattern_analysis_worker_process_test.go b/pkg/cookiebanner/pattern_analysis_worker_process_test.go index d93115221..fec2eacb0 100644 --- a/pkg/cookiebanner/pattern_analysis_worker_process_test.go +++ b/pkg/cookiebanner/pattern_analysis_worker_process_test.go @@ -28,6 +28,7 @@ import ( "go.probo.inc/probo/internal/test" "go.probo.inc/probo/pkg/coredata" "go.probo.inc/probo/pkg/gid" + "go.probo.inc/probo/pkg/page" ) // workerFixture bootstraps the parent rows the worker's transaction @@ -324,14 +325,28 @@ func TestPatternAnalysisWorker_PromotesSourceOnExistingGlob(t *testing.T) { var remainingExacts coredata.TrackerPatterns require.NoError(t, client.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error { - return remainingExacts.LoadAllByCookieBannerID( + loaded, err := page.LoadAll( ctx, - conn, - fx.scope, - fx.banner.ID, - coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeExact), nil, new(false)), - nil, + page.OrderBy[coredata.TrackerPatternOrderField]{ + Field: coredata.TrackerPatternOrderFieldCreatedAt, + Direction: page.OrderDirectionAsc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.TrackerPatternOrderField]) ([]*coredata.TrackerPattern, error) { + var batch coredata.TrackerPatterns + if err := batch.LoadByCookieBannerID(ctx, conn, fx.scope, fx.banner.ID, cursor, coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeExact), nil, new(false))); err != nil { + return nil, err + } + + return batch, nil + }, ) + if err != nil { + return err + } + + remainingExacts = loaded + + return nil })) assert.Empty(t, remainingExacts, "all three exacts must be relinked and deleted") } @@ -401,14 +416,28 @@ func TestPatternAnalysisWorker_AdoptionTriggersDraftVersion(t *testing.T) { var remainingExacts coredata.TrackerPatterns require.NoError(t, client.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error { - return remainingExacts.LoadAllByCookieBannerID( + loaded, err := page.LoadAll( ctx, - conn, - fx.scope, - fx.banner.ID, - coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeExact), nil, new(false)), - nil, + page.OrderBy[coredata.TrackerPatternOrderField]{ + Field: coredata.TrackerPatternOrderFieldCreatedAt, + Direction: page.OrderDirectionAsc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.TrackerPatternOrderField]) ([]*coredata.TrackerPattern, error) { + var batch coredata.TrackerPatterns + if err := batch.LoadByCookieBannerID(ctx, conn, fx.scope, fx.banner.ID, cursor, coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeExact), nil, new(false))); err != nil { + return nil, err + } + + return batch, nil + }, ) + if err != nil { + return err + } + + remainingExacts = loaded + + return nil })) assert.Empty(t, remainingExacts, "adoptUncategorisedPatterns must absorb the uncategorised exacts into the existing glob") @@ -486,14 +515,28 @@ func TestPatternAnalysisWorker_AdoptionPromotesSourceCrossCategory(t *testing.T) var remainingExacts coredata.TrackerPatterns require.NoError(t, client.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error { - return remainingExacts.LoadAllByCookieBannerID( + loaded, err := page.LoadAll( ctx, - conn, - fx.scope, - fx.banner.ID, - coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeExact), nil, new(false)), - nil, + page.OrderBy[coredata.TrackerPatternOrderField]{ + Field: coredata.TrackerPatternOrderFieldCreatedAt, + Direction: page.OrderDirectionAsc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.TrackerPatternOrderField]) ([]*coredata.TrackerPattern, error) { + var batch coredata.TrackerPatterns + if err := batch.LoadByCookieBannerID(ctx, conn, fx.scope, fx.banner.ID, cursor, coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeExact), nil, new(false))); err != nil { + return nil, err + } + + return batch, nil + }, ) + if err != nil { + return err + } + + remainingExacts = loaded + + return nil })) assert.Empty(t, remainingExacts, "the uncategorised exact must be adopted into the existing glob") } @@ -559,14 +602,28 @@ func TestReportDetectedTrackers_PromotesSourceOnExistingGlob(t *testing.T) { var exacts coredata.TrackerPatterns require.NoError(t, client.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error { - return exacts.LoadAllByCookieBannerID( + loaded, err := page.LoadAll( ctx, - conn, - fx.scope, - fx.banner.ID, - coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeExact), nil, new(false)), - nil, + page.OrderBy[coredata.TrackerPatternOrderField]{ + Field: coredata.TrackerPatternOrderFieldCreatedAt, + Direction: page.OrderDirectionAsc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.TrackerPatternOrderField]) ([]*coredata.TrackerPattern, error) { + var batch coredata.TrackerPatterns + if err := batch.LoadByCookieBannerID(ctx, conn, fx.scope, fx.banner.ID, cursor, coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeExact), nil, new(false))); err != nil { + return nil, err + } + + return batch, nil + }, ) + if err != nil { + return err + } + + exacts = loaded + + return nil })) assert.Empty(t, exacts, "the detected cookie globMatches the existing glob; no exact pattern must be created") } @@ -608,14 +665,28 @@ func TestPatternAnalysisWorker_MergeWithoutAdoptionSkipsDraftVersion(t *testing. var globs coredata.TrackerPatterns require.NoError(t, client.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error { - return globs.LoadAllByCookieBannerID( + loaded, err := page.LoadAll( ctx, - conn, - fx.scope, - fx.banner.ID, - coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeGlob), nil, new(false)), - nil, + page.OrderBy[coredata.TrackerPatternOrderField]{ + Field: coredata.TrackerPatternOrderFieldCreatedAt, + Direction: page.OrderDirectionAsc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.TrackerPatternOrderField]) ([]*coredata.TrackerPattern, error) { + var batch coredata.TrackerPatterns + if err := batch.LoadByCookieBannerID(ctx, conn, fx.scope, fx.banner.ID, cursor, coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeGlob), nil, new(false))); err != nil { + return nil, err + } + + return batch, nil + }, ) + if err != nil { + return err + } + + globs = loaded + + return nil })) require.Len(t, globs, 1, "the three exacts must consolidate into a single glob") assert.Equal(t, "_ga_*", globs[0].Pattern) diff --git a/pkg/cookiebanner/reset_trackers.go b/pkg/cookiebanner/reset_trackers.go index 01097799c..10fb6e172 100644 --- a/pkg/cookiebanner/reset_trackers.go +++ b/pkg/cookiebanner/reset_trackers.go @@ -22,6 +22,7 @@ import ( "go.gearno.de/kit/pg" "go.probo.inc/probo/pkg/coredata" "go.probo.inc/probo/pkg/gid" + "go.probo.inc/probo/pkg/page" ) // ResetTrackersResult summarizes what a banner reset changed. @@ -122,22 +123,43 @@ func decomposeGlobs( globMatchType := coredata.TrackerPatternMatchTypeGlob notExcluded := false - var globs coredata.TrackerPatterns - if err := globs.LoadAllByCookieBannerID( + globs, err := page.LoadAll( ctx, - tx, - scope, - bannerID, - coredata.NewTrackerPatternFilter(&globMatchType, &uncategorisedID, ¬Excluded).WithPatternKeyword(keyword), - nil, - ); err != nil { - return fmt.Errorf("cannot load glob patterns: %w", err) + page.OrderBy[coredata.TrackerPatternOrderField]{ + Field: coredata.TrackerPatternOrderFieldCreatedAt, + Direction: page.OrderDirectionAsc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.TrackerPatternOrderField]) ([]*coredata.TrackerPattern, error) { + var batch coredata.TrackerPatterns + if err := batch.LoadByCookieBannerID(ctx, tx, scope, bannerID, cursor, coredata.NewTrackerPatternFilter(&globMatchType, &uncategorisedID, ¬Excluded).WithPatternKeyword(keyword)); err != nil { + return nil, fmt.Errorf("cannot load glob patterns: %w", err) + } + + return batch, nil + }, + ) + if err != nil { + return err } for _, glob := range globs { - var detections coredata.DetectedTrackers - if err := detections.LoadAllByTrackerPatternID(ctx, tx, scope, glob.ID); err != nil { - return fmt.Errorf("cannot load detections for glob %q: %w", glob.Pattern, err) + detections, err := page.LoadAll( + ctx, + page.OrderBy[coredata.DetectedTrackerOrderField]{ + Field: coredata.DetectedTrackerOrderFieldLastDetectedAt, + Direction: page.OrderDirectionAsc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.DetectedTrackerOrderField]) ([]*coredata.DetectedTracker, error) { + var batch coredata.DetectedTrackers + if err := batch.LoadByTrackerPatternID(ctx, tx, scope, glob.ID, cursor); err != nil { + return nil, fmt.Errorf("cannot load detections for glob %q: %w", glob.Pattern, err) + } + + return batch, nil + }, + ) + if err != nil { + return err } for _, detection := range detections { diff --git a/pkg/cookiebanner/reset_trackers_test.go b/pkg/cookiebanner/reset_trackers_test.go index d2fc0b52d..817db9cca 100644 --- a/pkg/cookiebanner/reset_trackers_test.go +++ b/pkg/cookiebanner/reset_trackers_test.go @@ -24,6 +24,7 @@ import ( "go.probo.inc/probo/internal/test" "go.probo.inc/probo/pkg/coredata" "go.probo.inc/probo/pkg/gid" + "go.probo.inc/probo/pkg/page" ) // TestResetBannerTrackers_FullRebuild seeds a banner with an @@ -113,8 +114,22 @@ func TestResetBannerTrackers_FullRebuild(t *testing.T) { require.Nil(t, exact.ThirdPartyID) require.NotNil(t, exact.MappingRequestedAt) - var detections coredata.DetectedTrackers - require.NoError(t, detections.LoadAllByTrackerPatternID(ctx, conn, fx.scope, exact.ID)) + detections, err := page.LoadAll( + ctx, + page.OrderBy[coredata.DetectedTrackerOrderField]{ + Field: coredata.DetectedTrackerOrderFieldLastDetectedAt, + Direction: page.OrderDirectionAsc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.DetectedTrackerOrderField]) ([]*coredata.DetectedTracker, error) { + var batch coredata.DetectedTrackers + if err := batch.LoadByTrackerPatternID(ctx, conn, fx.scope, exact.ID, cursor); err != nil { + return nil, err + } + + return batch, nil + }, + ) + require.NoError(t, err) require.Len(t, detections, 1) require.Equal(t, identifier, detections[0].Identifier) } diff --git a/pkg/cookiebanner/service.go b/pkg/cookiebanner/service.go index 99d11218d..fba86eb1f 100644 --- a/pkg/cookiebanner/service.go +++ b/pkg/cookiebanner/service.go @@ -553,21 +553,42 @@ func (s *Service) ensureDraftVersionForBanner( consentFilter := coredata.NewCookieCategoryFilter(new(coredata.CookieCategoryKindUncategorised)) - var categories coredata.CookieCategories - if err := categories.LoadAllByCookieBannerID(ctx, tx, scope, bannerID, consentFilter); err != nil { - return nil, fmt.Errorf("cannot load cookie categories: %w", err) + categories, err := page.LoadAll( + ctx, + page.OrderBy[coredata.CookieCategoryOrderField]{ + Field: coredata.CookieCategoryOrderFieldRank, + Direction: page.OrderDirectionAsc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.CookieCategoryOrderField]) ([]*coredata.CookieCategory, error) { + var batch coredata.CookieCategories + if err := batch.LoadByCookieBannerID(ctx, tx, scope, bannerID, cursor, consentFilter); err != nil { + return nil, fmt.Errorf("cannot load cookie categories: %w", err) + } + + return batch, nil + }, + ) + if err != nil { + return nil, err } - var allPatterns coredata.TrackerPatterns - if err := allPatterns.LoadAllByCookieBannerID( + allPatterns, err := page.LoadAll( ctx, - tx, - scope, - bannerID, - coredata.NewTrackerPatternFilter(nil, nil, new(false)), - nil, - ); err != nil { - return nil, fmt.Errorf("cannot load tracker patterns: %w", err) + page.OrderBy[coredata.TrackerPatternOrderField]{ + Field: coredata.TrackerPatternOrderFieldCreatedAt, + Direction: page.OrderDirectionAsc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.TrackerPatternOrderField]) ([]*coredata.TrackerPattern, error) { + var batch coredata.TrackerPatterns + if err := batch.LoadByCookieBannerID(ctx, tx, scope, bannerID, cursor, coredata.NewTrackerPatternFilter(nil, nil, new(false))); err != nil { + return nil, fmt.Errorf("cannot load tracker patterns: %w", err) + } + + return batch, nil + }, + ) + if err != nil { + return nil, err } return s.ensureDraftVersion(ctx, tx, scope, &banner, categories, allPatterns) @@ -1695,14 +1716,42 @@ func (s *Service) GetActiveBannerConfig( consentFilter := coredata.NewCookieCategoryFilter(new(coredata.CookieCategoryKindUncategorised)) - var categories coredata.CookieCategories - if err := categories.LoadAllByCookieBannerID(ctx, conn, scope, banner.ID, consentFilter); err != nil { - return fmt.Errorf("cannot load cookie categories: %w", err) + categories, err := page.LoadAll( + ctx, + page.OrderBy[coredata.CookieCategoryOrderField]{ + Field: coredata.CookieCategoryOrderFieldRank, + Direction: page.OrderDirectionAsc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.CookieCategoryOrderField]) ([]*coredata.CookieCategory, error) { + var batch coredata.CookieCategories + if err := batch.LoadByCookieBannerID(ctx, conn, scope, banner.ID, cursor, consentFilter); err != nil { + return nil, fmt.Errorf("cannot load cookie categories: %w", err) + } + + return batch, nil + }, + ) + if err != nil { + return err } - var translations coredata.CookieBannerTranslations - if err := translations.LoadAllByCookieBannerID(ctx, conn, scope, banner.ID); err != nil { - return fmt.Errorf("cannot load cookie banner translations: %w", err) + translations, err := page.LoadAll( + ctx, + page.OrderBy[coredata.CookieBannerTranslationOrderField]{ + Field: coredata.CookieBannerTranslationOrderFieldLanguage, + Direction: page.OrderDirectionAsc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.CookieBannerTranslationOrderField]) ([]*coredata.CookieBannerTranslation, error) { + var batch coredata.CookieBannerTranslations + if err := batch.LoadByCookieBannerID(ctx, conn, scope, banner.ID, cursor); err != nil { + return nil, fmt.Errorf("cannot load cookie banner translations: %w", err) + } + + return batch, nil + }, + ) + if err != nil { + return err } resolved := resolveTranslations(translations, categories) @@ -1959,7 +2008,28 @@ func (s *Service) ListCookieBannerTranslations( err := s.pg.WithConn( ctx, func(ctx context.Context, conn pg.Querier) error { - return translations.LoadAllByCookieBannerID(ctx, conn, scope, cookieBannerID) + loaded, err := page.LoadAll( + ctx, + page.OrderBy[coredata.CookieBannerTranslationOrderField]{ + Field: coredata.CookieBannerTranslationOrderFieldLanguage, + Direction: page.OrderDirectionAsc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.CookieBannerTranslationOrderField]) ([]*coredata.CookieBannerTranslation, error) { + var batch coredata.CookieBannerTranslations + if err := batch.LoadByCookieBannerID(ctx, conn, scope, cookieBannerID, cursor); err != nil { + return nil, fmt.Errorf("cannot load cookie banner translations: %w", err) + } + + return batch, nil + }, + ) + if err != nil { + return err + } + + translations = loaded + + return nil }, ) if err != nil { diff --git a/pkg/cookiebanner/tracker_mapping_worker.go b/pkg/cookiebanner/tracker_mapping_worker.go index 83f5eaed3..e792fb36a 100644 --- a/pkg/cookiebanner/tracker_mapping_worker.go +++ b/pkg/cookiebanner/tracker_mapping_worker.go @@ -29,6 +29,7 @@ import ( "go.probo.inc/probo/pkg/coredata" "go.probo.inc/probo/pkg/gid" "go.probo.inc/probo/pkg/llm" + "go.probo.inc/probo/pkg/page" "go.probo.inc/probo/pkg/stringsx" "go.probo.inc/probo/pkg/thirdparty" "go.probo.inc/probo/pkg/uri" @@ -1216,15 +1217,23 @@ func (h *trackerMappingHandler) prepareOrgThirdParty( firstLevel := 1 - var orgThirdParties coredata.ThirdParties - if err := orgThirdParties.LoadAllByOrganizationID( + orgThirdParties, err := page.LoadAll( ctx, - conn, - scope, - tp.OrganizationID, - coredata.NewThirdPartyFilter(nil, &firstLevel, nil), - ); err != nil { - return prep, fmt.Errorf("cannot load org third parties: %w", err) + page.OrderBy[coredata.ThirdPartyOrderField]{ + Field: coredata.ThirdPartyOrderFieldName, + Direction: page.OrderDirectionAsc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.ThirdPartyOrderField]) ([]*coredata.ThirdParty, error) { + var batch coredata.ThirdParties + if err := batch.LoadByOrganizationID(ctx, conn, scope, tp.OrganizationID, cursor, coredata.NewThirdPartyFilter(nil, &firstLevel, nil)); err != nil { + return nil, fmt.Errorf("cannot load org third parties: %w", err) + } + + return batch, nil + }, + ) + if err != nil { + return prep, err } ranked := thirdparty.RankCandidates(prep.commonParty, prep.commonDomains, orgThirdParties) diff --git a/pkg/coredata/access_review_campaign_source_fetch_attempt.go b/pkg/coredata/access_review_campaign_source_fetch_attempt.go index d9f551d16..c2571b5a0 100644 --- a/pkg/coredata/access_review_campaign_source_fetch_attempt.go +++ b/pkg/coredata/access_review_campaign_source_fetch_attempt.go @@ -386,54 +386,6 @@ WHERE return nil } -// LoadAllByCampaignSourceID returns the full attempt history for a snapshot, -// newest first. -func (attempts *AccessReviewCampaignSourceFetchAttempts) LoadAllByCampaignSourceID( - ctx context.Context, - conn pg.Querier, - scope Scoper, - campaignSourceID gid.GID, -) error { - q := ` -SELECT - id, - organization_id, - tenant_id, - access_review_campaign_source_id, - attempt_number, - status, - fetched_accounts_count, - error, - started_at, - completed_at, - created_at, - updated_at -FROM access_review_campaign_source_fetch_attempts -WHERE - %s - AND access_review_campaign_source_id = @access_review_campaign_source_id -ORDER BY attempt_number DESC -` - q = fmt.Sprintf(q, scope.SQLFragment()) - - args := pgx.StrictNamedArgs{"access_review_campaign_source_id": campaignSourceID} - maps.Copy(args, scope.SQLArguments()) - - rows, err := conn.Query(ctx, q, args) - if err != nil { - return fmt.Errorf("cannot query fetch attempts: %w", err) - } - - result, err := pgx.CollectRows(rows, pgx.RowToAddrOfStructByName[AccessReviewCampaignSourceFetchAttempt]) - if err != nil { - return fmt.Errorf("cannot collect fetch attempts: %w", err) - } - - *attempts = result - - return nil -} - func (attempts *AccessReviewCampaignSourceFetchAttempts) CountByCampaignSourceID( ctx context.Context, conn pg.Querier, diff --git a/pkg/coredata/access_review_campaign_source_test.go b/pkg/coredata/access_review_campaign_source_test.go index d6507c6c0..01dc061cb 100644 --- a/pkg/coredata/access_review_campaign_source_test.go +++ b/pkg/coredata/access_review_campaign_source_test.go @@ -16,6 +16,7 @@ package coredata_test import ( "context" + "fmt" "testing" "time" @@ -25,6 +26,7 @@ import ( "go.probo.inc/probo/internal/test" "go.probo.inc/probo/pkg/coredata" "go.probo.inc/probo/pkg/gid" + "go.probo.inc/probo/pkg/page" ) func insertAccessReviewEntry(t *testing.T, ctx context.Context, client *pg.Client, fx accessEntryFixture, accountKey string) gid.GID { @@ -143,7 +145,28 @@ func TestSourceFetchAttempts_AppendOnly(t *testing.T) { var history coredata.AccessReviewCampaignSourceFetchAttempts require.NoError(t, client.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error { - return history.LoadAllByCampaignSourceID(ctx, conn, fx.scope, fx.campaignSourceID) + loaded, err := page.LoadAll( + ctx, + page.OrderBy[coredata.AccessReviewCampaignSourceFetchAttemptOrderField]{ + Field: coredata.AccessReviewCampaignSourceFetchAttemptOrderFieldCreatedAt, + Direction: page.OrderDirectionDesc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.AccessReviewCampaignSourceFetchAttemptOrderField]) ([]*coredata.AccessReviewCampaignSourceFetchAttempt, error) { + var batch coredata.AccessReviewCampaignSourceFetchAttempts + if err := batch.LoadByCampaignSourceID(ctx, conn, fx.scope, fx.campaignSourceID, cursor); err != nil { + return nil, fmt.Errorf("cannot load fetch attempts: %w", err) + } + + return batch, nil + }, + ) + if err != nil { + return err + } + + history = loaded + + return nil })) require.Len(t, history, 2, "both attempts must be retained") assert.Equal(t, coredata.AccessReviewCampaignSourceFetchStatusSuccess, history[0].Status, "history is newest first") diff --git a/pkg/coredata/applicability_statement.go b/pkg/coredata/applicability_statement.go index b89fc99f1..5d6d9e603 100644 --- a/pkg/coredata/applicability_statement.go +++ b/pkg/coredata/applicability_statement.go @@ -482,57 +482,6 @@ WHERE return nil } -func (sacs *ApplicabilityStatements) LoadAllByStatementOfApplicabilityID( - ctx context.Context, - conn pg.Querier, - scope Scoper, - statementOfApplicabilityID gid.GID, -) error { - q := ` -SELECT - a.id, - a.statement_of_applicability_id, - a.control_id, - a.organization_id, - a.applicability, - a.justification, - a.created_at, - a.updated_at, - f.name || ' - ' || c.section_title AS section_title -FROM - applicability_statements a -INNER JOIN - controls c ON c.id = a.control_id -INNER JOIN - frameworks f ON f.id = c.framework_id -WHERE - a.%s - AND a.statement_of_applicability_id = @statement_of_applicability_id -ORDER BY - section_title_sort_key(f.name || ' - ' || c.section_title) ASC; -` - q = fmt.Sprintf(q, scope.SQLFragment()) - - args := pgx.StrictNamedArgs{ - "statement_of_applicability_id": statementOfApplicabilityID, - } - maps.Copy(args, scope.SQLArguments()) - - rows, err := conn.Query(ctx, q, args) - if err != nil { - return fmt.Errorf("cannot query applicability_statements: %w", err) - } - - controls, err := pgx.CollectRows(rows, pgx.RowToAddrOfStructByName[ApplicabilityStatement]) - if err != nil { - return fmt.Errorf("cannot collect applicability_statements: %w", err) - } - - *sacs = controls - - return nil -} - func (sacs *ApplicabilityStatements) CountByStatementOfApplicabilityID( ctx context.Context, conn pg.Querier, diff --git a/pkg/coredata/asset.go b/pkg/coredata/asset.go index 6277dd408..27cbb4b6e 100644 --- a/pkg/coredata/asset.go +++ b/pkg/coredata/asset.go @@ -50,6 +50,8 @@ func (a *Asset) CursorKey(field AssetOrderField) page.CursorKey { return page.NewCursorKey(a.ID, a.CreatedAt) case AssetOrderFieldAmount: return page.NewCursorKey(a.ID, a.Amount) + case AssetOrderFieldName: + return page.NewCursorKey(a.ID, a.Name) } panic(fmt.Sprintf("unsupported order by: %s", field)) @@ -270,52 +272,6 @@ WHERE return nil } -func (a *Assets) LoadAllByOrganizationID( - ctx context.Context, - conn pg.Querier, - scope Scoper, - organizationID gid.GID, -) error { - q := ` -SELECT - id, - name, - organization_id, - owner_profile_id, - amount, - asset_type, - data_types_stored, - created_at, - updated_at -FROM - assets -WHERE - %s - AND organization_id = @organization_id -ORDER BY - name ASC -` - - q = fmt.Sprintf(q, scope.SQLFragment()) - - args := pgx.StrictNamedArgs{"organization_id": organizationID} - maps.Copy(args, scope.SQLArguments()) - - rows, err := conn.Query(ctx, q, args) - if err != nil { - return fmt.Errorf("cannot query assets: %w", err) - } - - assets, err := pgx.CollectRows(rows, pgx.RowToAddrOfStructByName[Asset]) - if err != nil { - return fmt.Errorf("cannot collect assets: %w", err) - } - - *a = assets - - return nil -} - func (a *Asset) Insert( ctx context.Context, conn pg.Tx, diff --git a/pkg/coredata/audit.go b/pkg/coredata/audit.go index ca1dbb36f..d7b1ee012 100644 --- a/pkg/coredata/audit.go +++ b/pkg/coredata/audit.go @@ -237,56 +237,6 @@ WHERE return nil } -func (a *Audits) LoadAllByOrganizationID( - ctx context.Context, - conn pg.Querier, - scope Scoper, - organizationID gid.GID, - filter *AuditFilter, -) error { - q := ` -SELECT - id, - name, - organization_id, - framework_id, - report_file_id, - valid_from, - valid_until, - state, - trust_center_visibility, - created_at, - updated_at -FROM - audits -WHERE - %s - AND organization_id = @organization_id - AND %s -ORDER BY valid_from DESC -` - - q = fmt.Sprintf(q, scope.SQLFragment(), filter.SQLFragment()) - - args := pgx.StrictNamedArgs{"organization_id": organizationID} - maps.Copy(args, scope.SQLArguments()) - maps.Copy(args, filter.SQLArguments()) - - rows, err := conn.Query(ctx, q, args) - if err != nil { - return fmt.Errorf("cannot query audits: %w", err) - } - - audits, err := pgx.CollectRows(rows, pgx.RowToAddrOfStructByName[Audit]) - if err != nil { - return fmt.Errorf("cannot collect audits: %w", err) - } - - *a = audits - - return nil -} - func (a *Audit) Insert( ctx context.Context, conn pg.Tx, diff --git a/pkg/coredata/cookie_banner_translation.go b/pkg/coredata/cookie_banner_translation.go index 83d703a0c..c62e28570 100644 --- a/pkg/coredata/cookie_banner_translation.go +++ b/pkg/coredata/cookie_banner_translation.go @@ -27,6 +27,7 @@ import ( "go.gearno.de/kit/pg" "go.probo.inc/probo/pkg/gid" "go.probo.inc/probo/pkg/iam/policy" + "go.probo.inc/probo/pkg/page" ) type ( @@ -43,6 +44,17 @@ type ( CookieBannerTranslations []*CookieBannerTranslation ) +func (t CookieBannerTranslation) CursorKey(field CookieBannerTranslationOrderField) page.CursorKey { + switch field { + case CookieBannerTranslationOrderFieldLanguage: + return page.NewCursorKey(t.ID, t.Language) + case CookieBannerTranslationOrderFieldCreatedAt: + return page.NewCursorKey(t.ID, t.CreatedAt) + } + + panic(fmt.Sprintf("unsupported order by: %s", field)) +} + func (t *CookieBannerTranslation) AuthorizationAttributes( ctx context.Context, conn pg.Querier, @@ -181,11 +193,12 @@ LIMIT 1; return nil } -func (t *CookieBannerTranslations) LoadAllByCookieBannerID( +func (t *CookieBannerTranslations) LoadByCookieBannerID( ctx context.Context, conn pg.Querier, scope Scoper, cookieBannerID gid.GID, + cursor *page.Cursor[CookieBannerTranslationOrderField], ) error { q := ` SELECT @@ -201,14 +214,14 @@ FROM WHERE %s AND cookie_banner_id = @cookie_banner_id -ORDER BY - language ASC; + AND %s ` - q = fmt.Sprintf(q, scope.SQLFragment()) + q = fmt.Sprintf(q, scope.SQLFragment(), cursor.SQLFragment()) args := pgx.StrictNamedArgs{"cookie_banner_id": cookieBannerID} maps.Copy(args, scope.SQLArguments()) + maps.Copy(args, cursor.SQLArguments()) rows, err := conn.Query(ctx, q, args) if err != nil { diff --git a/pkg/coredata/cookie_banner_translation_order_field.go b/pkg/coredata/cookie_banner_translation_order_field.go new file mode 100644 index 000000000..f9ec7bbc1 --- /dev/null +++ b/pkg/coredata/cookie_banner_translation_order_field.go @@ -0,0 +1,79 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package coredata + +import ( + "encoding" + "fmt" + + "go.probo.inc/probo/pkg/page" +) + +type ( + CookieBannerTranslationOrderField string +) + +const ( + CookieBannerTranslationOrderFieldLanguage CookieBannerTranslationOrderField = "LANGUAGE" + CookieBannerTranslationOrderFieldCreatedAt CookieBannerTranslationOrderField = "CREATED_AT" +) + +var ( + _ page.OrderField = CookieBannerTranslationOrderField("") + _ fmt.Stringer = CookieBannerTranslationOrderField("") + _ encoding.TextMarshaler = CookieBannerTranslationOrderField("") + _ encoding.TextUnmarshaler = (*CookieBannerTranslationOrderField)(nil) +) + +func CookieBannerTranslationOrderFields() []CookieBannerTranslationOrderField { + return []CookieBannerTranslationOrderField{ + CookieBannerTranslationOrderFieldLanguage, + CookieBannerTranslationOrderFieldCreatedAt, + } +} + +func (v CookieBannerTranslationOrderField) IsValid() bool { + switch v { + case + CookieBannerTranslationOrderFieldLanguage, + CookieBannerTranslationOrderFieldCreatedAt: + return true + } + + return false +} + +func (v CookieBannerTranslationOrderField) String() string { + return string(v) +} + +func (v CookieBannerTranslationOrderField) MarshalText() ([]byte, error) { + return []byte(v.String()), nil +} + +func (v *CookieBannerTranslationOrderField) UnmarshalText(text []byte) error { + val := CookieBannerTranslationOrderField(text) + if !val.IsValid() { + return fmt.Errorf("invalid CookieBannerTranslationOrderField value: %q", string(text)) + } + + *v = val + + return nil +} + +func (p CookieBannerTranslationOrderField) Column() string { + return string(p) +} diff --git a/pkg/coredata/cookie_category.go b/pkg/coredata/cookie_category.go index 821dcc617..182ad7821 100644 --- a/pkg/coredata/cookie_category.go +++ b/pkg/coredata/cookie_category.go @@ -386,58 +386,6 @@ WHERE return count, nil } -func (c *CookieCategories) LoadAllByCookieBannerID( - ctx context.Context, - conn pg.Querier, - scope Scoper, - cookieBannerID gid.GID, - filter *CookieCategoryFilter, -) error { - q := ` -SELECT - id, - organization_id, - cookie_banner_id, - name, - slug, - description, - kind, - rank, - gcm_consent_types, - posthog_consent, - created_at, - updated_at -FROM - cookie_categories -WHERE - %s - AND cookie_banner_id = @cookie_banner_id - AND %s -ORDER BY - rank ASC, id ASC; -` - - q = fmt.Sprintf(q, scope.SQLFragment(), filter.SQLFragment()) - - args := pgx.StrictNamedArgs{"cookie_banner_id": cookieBannerID} - maps.Copy(args, scope.SQLArguments()) - maps.Copy(args, filter.SQLArguments()) - - rows, err := conn.Query(ctx, q, args) - if err != nil { - return fmt.Errorf("cannot query cookie categories: %w", err) - } - - categories, err := pgx.CollectRows(rows, pgx.RowToAddrOfStructByName[CookieCategory]) - if err != nil { - return fmt.Errorf("cannot collect cookie categories: %w", err) - } - - *c = categories - - return nil -} - func (c *CookieCategory) Insert( ctx context.Context, tx pg.Tx, diff --git a/pkg/coredata/data_protection_impact_assessment.go b/pkg/coredata/data_protection_impact_assessment.go index 5cde14edb..97b82b656 100644 --- a/pkg/coredata/data_protection_impact_assessment.go +++ b/pkg/coredata/data_protection_impact_assessment.go @@ -284,51 +284,6 @@ WHERE return nil } -func (dpias *DataProtectionImpactAssessments) LoadAllByOrganizationID( - ctx context.Context, - conn pg.Querier, - scope Scoper, - organizationID gid.GID, -) error { - q := ` -SELECT - id, - organization_id, - processing_activity_id, - description, - necessity_and_proportionality, - potential_risk, - mitigations, - residual_risk, - created_at, - updated_at -FROM - processing_activity_data_protection_impact_assessments -WHERE - %s - AND organization_id = @organization_id -` - - q = fmt.Sprintf(q, scope.SQLFragment()) - - args := pgx.StrictNamedArgs{"organization_id": organizationID} - maps.Copy(args, scope.SQLArguments()) - - rows, err := conn.Query(ctx, q, args) - if err != nil { - return fmt.Errorf("cannot query data protection impact assessments: %w", err) - } - - results, err := pgx.CollectRows(rows, pgx.RowToAddrOfStructByName[DataProtectionImpactAssessment]) - if err != nil { - return fmt.Errorf("cannot collect data protection impact assessments: %w", err) - } - - *dpias = results - - return nil -} - func (dpia *DataProtectionImpactAssessment) LoadByID( ctx context.Context, conn pg.Querier, diff --git a/pkg/coredata/datum.go b/pkg/coredata/datum.go index f493708f7..4a4e98b15 100644 --- a/pkg/coredata/datum.go +++ b/pkg/coredata/datum.go @@ -258,50 +258,6 @@ WHERE return nil } -func (d *Data) LoadAllByOrganizationID( - ctx context.Context, - conn pg.Querier, - scope Scoper, - organizationID gid.GID, -) error { - q := ` -SELECT - id, - name, - organization_id, - owner_profile_id, - data_classification, - created_at, - updated_at -FROM - data -WHERE - %s - AND organization_id = @organization_id -ORDER BY - name ASC -` - - q = fmt.Sprintf(q, scope.SQLFragment()) - - args := pgx.StrictNamedArgs{"organization_id": organizationID} - maps.Copy(args, scope.SQLArguments()) - - rows, err := conn.Query(ctx, q, args) - if err != nil { - return fmt.Errorf("cannot query data: %w", err) - } - - data, err := pgx.CollectRows(rows, pgx.RowToAddrOfStructByName[Datum]) - if err != nil { - return fmt.Errorf("cannot collect data: %w", err) - } - - *d = data - - return nil -} - func (d *Datum) Insert( ctx context.Context, conn pg.Tx, diff --git a/pkg/coredata/detected_tracker.go b/pkg/coredata/detected_tracker.go index 80aecf241..c806d0b55 100644 --- a/pkg/coredata/detected_tracker.go +++ b/pkg/coredata/detected_tracker.go @@ -295,59 +295,6 @@ LIMIT @limit; return ids, nil } -// LoadAllByTrackerPatternID returns every detected tracker linked to the -// pattern, with no pagination. It backs the banner-reset rebuild, which -// recreates exact patterns from a glob's detections. -func (dts *DetectedTrackers) LoadAllByTrackerPatternID( - ctx context.Context, - conn pg.Querier, - scope Scoper, - trackerPatternID gid.GID, -) error { - q := ` -SELECT - id, - cookie_banner_id, - tracker_pattern_id, - tracker_type, - identifier, - max_age_seconds, - source, - value_size, - initiator_url, - initiator_domain, - last_detected_at, - created_at, - updated_at -FROM - detected_trackers -WHERE - %s - AND tracker_pattern_id = @tracker_pattern_id -ORDER BY - identifier ASC, id ASC -` - - q = fmt.Sprintf(q, scope.SQLFragment()) - - args := pgx.StrictNamedArgs{"tracker_pattern_id": trackerPatternID} - maps.Copy(args, scope.SQLArguments()) - - rows, err := conn.Query(ctx, q, args) - if err != nil { - return fmt.Errorf("cannot query detected trackers: %w", err) - } - - trackers, err := pgx.CollectRows(rows, pgx.RowToAddrOfStructByName[DetectedTracker]) - if err != nil { - return fmt.Errorf("cannot collect detected trackers: %w", err) - } - - *dts = trackers - - return nil -} - // UpdateTrackerPatternID repoints a single detected tracker at another // pattern. It is the per-row counterpart of RelinkByTrackerPatternID, // used by the banner-reset rebuild where each detection of a glob moves diff --git a/pkg/coredata/document.go b/pkg/coredata/document.go index c8fab3abe..b6e5e9e2e 100644 --- a/pkg/coredata/document.go +++ b/pkg/coredata/document.go @@ -377,64 +377,6 @@ SELECT * FROM base WHERE %s return nil } -func (p *Documents) LoadAllByOrganizationID( - ctx context.Context, - conn pg.Querier, - scope Scoper, - organizationID gid.GID, - filter *DocumentFilter, -) error { - q := ` -WITH latest_versions AS ( - SELECT DISTINCT ON (document_id) document_id, title, document_type - FROM document_versions - ORDER BY document_id, major DESC, minor DESC -) -SELECT - documents.id, - documents.organization_id, - documents.current_published_major, - documents.current_published_minor, - documents.write_mode, - documents.trust_center_visibility, - documents.status, - documents.archived_at, - documents.created_at, - documents.updated_at, - COALESCE(lv.title, '') AS title, - COALESCE(lv.document_type, 'OTHER') AS document_type -FROM - documents -LEFT JOIN latest_versions lv ON lv.document_id = documents.id -WHERE - %s - AND documents.deleted_at IS NULL - AND documents.organization_id = @organization_id - AND %s -ORDER BY title ASC -` - - q = fmt.Sprintf(q, scope.SQLFragment(), filter.SQLFragment()) - - args := pgx.NamedArgs{"organization_id": organizationID} - maps.Copy(args, scope.SQLArguments()) - maps.Copy(args, filter.SQLArguments()) - - rows, err := conn.Query(ctx, q, args) - if err != nil { - return fmt.Errorf("cannot query documents: %w", err) - } - - documents, err := pgx.CollectRows(rows, pgx.RowToAddrOfStructByName[Document]) - if err != nil { - return fmt.Errorf("cannot collect documents: %w", err) - } - - *p = documents - - return nil -} - func (p *Documents) LoadPublishedByOrganizationID( ctx context.Context, conn pg.Querier, diff --git a/pkg/coredata/finding.go b/pkg/coredata/finding.go index 6836839b6..012d41626 100644 --- a/pkg/coredata/finding.go +++ b/pkg/coredata/finding.go @@ -560,60 +560,6 @@ WHERE return count, nil } -func (fs *Findings) LoadAllByOrganizationID( - ctx context.Context, - conn pg.Querier, - scope Scoper, - organizationID gid.GID, -) error { - q := ` -SELECT - id, - organization_id, - kind, - reference_id, - description, - source, - identified_on, - root_cause, - corrective_action, - owner_id, - due_date, - status, - priority, - risk_id, - effectiveness_check, - created_at, - updated_at -FROM - findings -WHERE - %s - AND organization_id = @organization_id -ORDER BY - reference_id ASC -` - - q = fmt.Sprintf(q, scope.SQLFragment()) - - args := pgx.StrictNamedArgs{"organization_id": organizationID} - maps.Copy(args, scope.SQLArguments()) - - rows, err := conn.Query(ctx, q, args) - if err != nil { - return fmt.Errorf("cannot query findings: %w", err) - } - - findings, err := pgx.CollectRows(rows, pgx.RowToAddrOfStructByName[Finding]) - if err != nil { - return fmt.Errorf("cannot collect findings: %w", err) - } - - *fs = findings - - return nil -} - func (f Finding) GetGeneratedDocumentID( ctx context.Context, conn pg.Querier, diff --git a/pkg/coredata/mailing_list_subscriber.go b/pkg/coredata/mailing_list_subscriber.go index fe0652850..f7bb03597 100644 --- a/pkg/coredata/mailing_list_subscriber.go +++ b/pkg/coredata/mailing_list_subscriber.go @@ -344,11 +344,12 @@ WHERE return count, nil } -func (cnss *MailingListSubscribers) LoadAllConfirmedByMailingListID( +func (cnss *MailingListSubscribers) LoadConfirmedByMailingListID( ctx context.Context, conn pg.Querier, scope Scoper, mailingListID gid.GID, + cursor *page.Cursor[MailingListSubscriberOrderField], ) error { q := ` SELECT @@ -366,12 +367,14 @@ WHERE %s AND mailing_list_id = @mailing_list_id AND status = 'CONFIRMED' + AND %s ` - q = fmt.Sprintf(q, scope.SQLFragment()) + q = fmt.Sprintf(q, scope.SQLFragment(), cursor.SQLFragment()) args := pgx.StrictNamedArgs{"mailing_list_id": mailingListID} maps.Copy(args, scope.SQLArguments()) + maps.Copy(args, cursor.SQLArguments()) rows, err := conn.Query(ctx, q, args) if err != nil { diff --git a/pkg/coredata/membership_profile.go b/pkg/coredata/membership_profile.go index 6e37e6732..888a021f4 100644 --- a/pkg/coredata/membership_profile.go +++ b/pkg/coredata/membership_profile.go @@ -566,118 +566,6 @@ WHERE return nil } -func (p *MembershipProfiles) LoadAllByOrganizationID( - ctx context.Context, - conn pg.Querier, - scope Scoper, - organizationID gid.GID, - filter *MembershipProfileFilter, -) error { - q := ` -WITH profiles AS ( - SELECT - p.id, - p.identity_id, - p.organization_id, - i.email_address, - p.source, - p.state, - p.full_name, - p.kind, - p.additional_email_addresses, - p.position, - p.contract_start_date, - p.contract_end_date, - p.user_name, - p.external_id, - p.nickname, - p.locale, - p.timezone, - p.profile_url, - p.preferred_language, - p.given_name, - p.family_name, - p.formatted_name, - p.middle_name, - p.honorific_prefix, - p.honorific_suffix, - p.employee_number, - p.department, - p.cost_center, - p.enterprise_organization, - p.division, - p.manager_value, - p.created_at, - p.updated_at - FROM - iam_membership_profiles p - INNER JOIN identities i ON i.id = p.identity_id - WHERE - p.%s - AND p.organization_id = @organization_id - AND %s -) -SELECT - id, - identity_id, - organization_id, - email_address, - source, - state, - full_name, - kind, - additional_email_addresses, - position, - contract_start_date, - contract_end_date, - '' AS organization_name, - user_name, - external_id, - nickname, - locale, - timezone, - profile_url, - preferred_language, - given_name, - family_name, - formatted_name, - middle_name, - honorific_prefix, - honorific_suffix, - employee_number, - department, - cost_center, - enterprise_organization, - division, - manager_value, - created_at, - updated_at -FROM profiles -` - - q = fmt.Sprintf(q, scope.SQLFragment(), filter.SQLFragment()) - - args := pgx.NamedArgs{ - "organization_id": organizationID, - } - maps.Copy(args, scope.SQLArguments()) - maps.Copy(args, filter.SQLArguments()) - - rows, err := conn.Query(ctx, q, args) - if err != nil { - return fmt.Errorf("cannot query profiles: %w", err) - } - - profiles, err := pgx.CollectRows(rows, pgx.RowToAddrOfStructByName[MembershipProfile]) - if err != nil { - return fmt.Errorf("cannot collect profiles: %w", err) - } - - *p = profiles - - return nil -} - func (p *MembershipProfiles) LoadByIdentityID( ctx context.Context, conn pg.Querier, diff --git a/pkg/coredata/obligation.go b/pkg/coredata/obligation.go index ddf3a47f6..be8f2d9ef 100644 --- a/pkg/coredata/obligation.go +++ b/pkg/coredata/obligation.go @@ -612,57 +612,6 @@ WHERE return nil } -func (os *Obligations) LoadAllByOrganizationID( - ctx context.Context, - conn pg.Querier, - scope Scoper, - organizationID gid.GID, -) error { - q := ` -SELECT - id, - organization_id, - area, - source, - requirement, - actions_to_be_implemented, - regulator, - owner_profile_id, - last_review_date, - due_date, - status, - type, - created_at, - updated_at -FROM - obligations -WHERE - %s - AND organization_id = @organization_id -ORDER BY - created_at ASC -` - - q = fmt.Sprintf(q, scope.SQLFragment()) - - args := pgx.StrictNamedArgs{"organization_id": organizationID} - maps.Copy(args, scope.SQLArguments()) - - rows, err := conn.Query(ctx, q, args) - if err != nil { - return fmt.Errorf("cannot query obligations: %w", err) - } - - obligations, err := pgx.CollectRows(rows, pgx.RowToAddrOfStructByName[Obligation]) - if err != nil { - return fmt.Errorf("cannot collect obligations: %w", err) - } - - *os = obligations - - return nil -} - func (o Obligation) GetGeneratedDocumentID( ctx context.Context, conn pg.Querier, diff --git a/pkg/coredata/organization.go b/pkg/coredata/organization.go index 9fdf400d5..ca98594ef 100644 --- a/pkg/coredata/organization.go +++ b/pkg/coredata/organization.go @@ -255,11 +255,12 @@ WHERE return nil } -func (o *Organizations) LoadAllByIdentityIDWithPendingInvitation( +func (o *Organizations) LoadByIdentityIDWithPendingInvitation( ctx context.Context, conn pg.Querier, scope Scoper, identityID gid.GID, + cursor *page.Cursor[OrganizationOrderField], ) error { q := ` WITH invited_org AS ( @@ -292,13 +293,14 @@ INNER JOIN invited_org ON organizations.id = invited_org.organization_id WHERE %s -ORDER BY name ASC + AND %s ` - q = fmt.Sprintf(q, scope.SQLFragment()) + q = fmt.Sprintf(q, scope.SQLFragment(), cursor.SQLFragment()) args := pgx.StrictNamedArgs{"identity_id": identityID} maps.Copy(args, scope.SQLArguments()) + maps.Copy(args, cursor.SQLArguments()) rows, err := conn.Query(ctx, q, args) if err != nil { diff --git a/pkg/coredata/processing_activities.go b/pkg/coredata/processing_activities.go index 0b4d599d5..ff234c200 100644 --- a/pkg/coredata/processing_activities.go +++ b/pkg/coredata/processing_activities.go @@ -437,65 +437,6 @@ WHERE return nil } -func (p *ProcessingActivities) LoadAllByOrganizationID( - ctx context.Context, - conn pg.Querier, - scope Scoper, - organizationID gid.GID, -) error { - q := ` -SELECT - id, - organization_id, - name, - purpose, - data_subject_category, - personal_data_category, - special_or_criminal_data, - consent_evidence_link, - lawful_basis, - recipients, - location, - international_transfers, - transfer_safeguards, - retention_period, - security_measures, - data_protection_impact_assessment_needed, - transfer_impact_assessment_needed, - last_review_date, - next_review_date, - role, - dpo_profile_id, - created_at, - updated_at -FROM - processing_activities -WHERE - %s - AND organization_id = @organization_id -ORDER BY created_at DESC -` - - q = fmt.Sprintf(q, scope.SQLFragment()) - - args := pgx.StrictNamedArgs{"organization_id": organizationID} - maps.Copy(args, scope.SQLArguments()) - - rows, err := conn.Query(ctx, q, args) - if err != nil { - return fmt.Errorf("cannot query processing activities: %w", err) - } - - processingActivities, err := pgx.CollectRows(rows, pgx.RowToAddrOfStructByName[ProcessingActivity]) - if err != nil { - return fmt.Errorf("cannot collect processing activities: %w", err) - } - - *p = processingActivities - - return nil -} - func (p *ProcessingActivity) Insert( ctx context.Context, conn pg.Tx, diff --git a/pkg/coredata/risk.go b/pkg/coredata/risk.go index c7060be4a..24a64679d 100644 --- a/pkg/coredata/risk.go +++ b/pkg/coredata/risk.go @@ -464,57 +464,6 @@ WHERE %s return nil } -func (r *Risks) LoadAllByOrganizationID( - ctx context.Context, - conn pg.Querier, - scope Scoper, - organizationID gid.GID, -) error { - q := ` -SELECT - r.id, - r.organization_id, - r.name, - r.description, - r.category, - r.owner_profile_id, - NULL as owner_full_name, - r.treatment, - r.note, - r.inherent_likelihood, - r.inherent_impact, - r.inherent_risk_score, - r.residual_likelihood, - r.residual_impact, - r.residual_risk_score, - r.created_at, - r.updated_at -FROM - risks r -WHERE %s - AND r.organization_id = @organization_id -ORDER BY r.name ASC, r.id ASC -` - q = fmt.Sprintf(q, scope.SQLFragment()) - - args := pgx.StrictNamedArgs{"organization_id": organizationID} - maps.Copy(args, scope.SQLArguments()) - - rows, err := conn.Query(ctx, q, args) - if err != nil { - return fmt.Errorf("cannot query risks: %w", err) - } - - risks, err := pgx.CollectRows(rows, pgx.RowToAddrOfStructByName[Risk]) - if err != nil { - return fmt.Errorf("cannot collect risks: %w", err) - } - - *r = risks - - return nil -} - func (r *Risk) LoadByID( ctx context.Context, conn pg.Querier, diff --git a/pkg/coredata/risk_assessment_boundary.go b/pkg/coredata/risk_assessment_boundary.go index ac9465e09..e79b97162 100644 --- a/pkg/coredata/risk_assessment_boundary.go +++ b/pkg/coredata/risk_assessment_boundary.go @@ -136,48 +136,6 @@ WHERE return nil } -func (bs *RiskAssessmentBoundaries) LoadAllByRiskAssessmentScopeID( - ctx context.Context, - conn pg.Querier, - scope Scoper, - riskAssessmentScopeID gid.GID, -) error { - q := ` -SELECT - id, - organization_id, - risk_assessment_scope_id, - parent_boundary_id, - name, - created_at, - updated_at -FROM - risk_assessment_boundaries -WHERE - %s - AND risk_assessment_scope_id = @risk_assessment_scope_id -ORDER BY - created_at ASC, id ASC -` - q = fmt.Sprintf(q, scope.SQLFragment()) - args := pgx.NamedArgs{"risk_assessment_scope_id": riskAssessmentScopeID} - maps.Copy(args, scope.SQLArguments()) - - rows, err := conn.Query(ctx, q, args) - if err != nil { - return fmt.Errorf("cannot query risk assessment boundaries: %w", err) - } - - results, err := pgx.CollectRows(rows, pgx.RowToAddrOfStructByName[RiskAssessmentBoundary]) - if err != nil { - return fmt.Errorf("cannot collect risk assessment boundaries: %w", err) - } - - *bs = results - - return nil -} - func (bs *RiskAssessmentBoundaries) CountByRiskAssessmentScopeID( ctx context.Context, conn pg.Querier, diff --git a/pkg/coredata/risk_assessment_node.go b/pkg/coredata/risk_assessment_node.go index 14d0ff7fd..1343cca56 100644 --- a/pkg/coredata/risk_assessment_node.go +++ b/pkg/coredata/risk_assessment_node.go @@ -138,49 +138,6 @@ WHERE return nil } -func (ns *RiskAssessmentNodes) LoadAllByRiskAssessmentScopeID( - ctx context.Context, - conn pg.Querier, - scope Scoper, - riskAssessmentScopeID gid.GID, -) error { - q := ` -SELECT - id, - organization_id, - risk_assessment_scope_id, - boundary_id, - node_type, - name, - created_at, - updated_at -FROM - risk_assessment_nodes -WHERE - %s - AND risk_assessment_scope_id = @risk_assessment_scope_id -ORDER BY - created_at ASC, id ASC -` - q = fmt.Sprintf(q, scope.SQLFragment()) - args := pgx.NamedArgs{"risk_assessment_scope_id": riskAssessmentScopeID} - maps.Copy(args, scope.SQLArguments()) - - rows, err := conn.Query(ctx, q, args) - if err != nil { - return fmt.Errorf("cannot query risk assessment nodes: %w", err) - } - - results, err := pgx.CollectRows(rows, pgx.RowToAddrOfStructByName[RiskAssessmentNode]) - if err != nil { - return fmt.Errorf("cannot collect risk assessment nodes: %w", err) - } - - *ns = results - - return nil -} - func (ns *RiskAssessmentNodes) CountByRiskAssessmentScopeID( ctx context.Context, conn pg.Querier, diff --git a/pkg/coredata/risk_assessment_process.go b/pkg/coredata/risk_assessment_process.go index c1ffa8035..ebfa6460a 100644 --- a/pkg/coredata/risk_assessment_process.go +++ b/pkg/coredata/risk_assessment_process.go @@ -138,49 +138,6 @@ WHERE return nil } -func (ps *RiskAssessmentProcesses) LoadAllByRiskAssessmentScopeID( - ctx context.Context, - conn pg.Querier, - scope Scoper, - riskAssessmentScopeID gid.GID, -) error { - q := ` -SELECT - id, - organization_id, - risk_assessment_scope_id, - source_node_id, - target_node_id, - name, - created_at, - updated_at -FROM - risk_assessment_processes -WHERE - %s - AND risk_assessment_scope_id = @risk_assessment_scope_id -ORDER BY - created_at ASC, id ASC -` - q = fmt.Sprintf(q, scope.SQLFragment()) - args := pgx.NamedArgs{"risk_assessment_scope_id": riskAssessmentScopeID} - maps.Copy(args, scope.SQLArguments()) - - rows, err := conn.Query(ctx, q, args) - if err != nil { - return fmt.Errorf("cannot query risk assessment processes: %w", err) - } - - results, err := pgx.CollectRows(rows, pgx.RowToAddrOfStructByName[RiskAssessmentProcess]) - if err != nil { - return fmt.Errorf("cannot collect risk assessment processes: %w", err) - } - - *ps = results - - return nil -} - func (ps *RiskAssessmentProcesses) CountByRiskAssessmentScopeID( ctx context.Context, conn pg.Querier, diff --git a/pkg/coredata/risk_assessment_threat.go b/pkg/coredata/risk_assessment_threat.go index d234622ca..e14195484 100644 --- a/pkg/coredata/risk_assessment_threat.go +++ b/pkg/coredata/risk_assessment_threat.go @@ -138,49 +138,6 @@ WHERE return nil } -func (ts *RiskAssessmentThreats) LoadAllByRiskAssessmentScopeID( - ctx context.Context, - conn pg.Querier, - scope Scoper, - riskAssessmentScopeID gid.GID, -) error { - q := ` -SELECT - id, - organization_id, - risk_assessment_scope_id, - process_id, - name, - category, - created_at, - updated_at -FROM - risk_assessment_threats -WHERE - %s - AND risk_assessment_scope_id = @risk_assessment_scope_id -ORDER BY - created_at ASC, id ASC -` - q = fmt.Sprintf(q, scope.SQLFragment()) - args := pgx.NamedArgs{"risk_assessment_scope_id": riskAssessmentScopeID} - maps.Copy(args, scope.SQLArguments()) - - rows, err := conn.Query(ctx, q, args) - if err != nil { - return fmt.Errorf("cannot query risk threats: %w", err) - } - - results, err := pgx.CollectRows(rows, pgx.RowToAddrOfStructByName[RiskAssessmentThreat]) - if err != nil { - return fmt.Errorf("cannot collect risk threats: %w", err) - } - - *ts = results - - return nil -} - func (ts *RiskAssessmentThreats) CountByRiskAssessmentScopeID( ctx context.Context, conn pg.Querier, diff --git a/pkg/coredata/third_party.go b/pkg/coredata/third_party.go index 6b8eb7001..91f794d8f 100644 --- a/pkg/coredata/third_party.go +++ b/pkg/coredata/third_party.go @@ -776,76 +776,6 @@ WHERE return count, nil } -func (v *ThirdParties) LoadAllByOrganizationID( - ctx context.Context, - conn pg.Querier, - scope Scoper, - organizationID gid.GID, - filter *ThirdPartyFilter, -) error { - q := ` -SELECT - id, - organization_id, - parent_third_party_id, - common_third_party_id, - name, - description, - category, - headquarter_address, - legal_name, - website_url, - privacy_policy_url, - service_level_agreement_url, - data_processing_agreement_url, - business_associate_agreement_url, - subprocessors_list_url, - certifications, - countries, - business_owner_profile_id, - security_owner_profile_id, - status_page_url, - terms_of_service_url, - security_page_url, - trust_page_url, - show_on_trust_center, - level, - vetting_status, - vetting_website_url, - vetting_procedure, - vetting_processing_started_at, - vetting_error_message, - created_at, - updated_at -FROM - third_parties -WHERE - %s - AND organization_id = @organization_id - AND %s -ORDER BY name ASC -` - q = fmt.Sprintf(q, scope.SQLFragment(), filter.SQLFragment()) - - args := pgx.StrictNamedArgs{"organization_id": organizationID} - maps.Copy(args, scope.SQLArguments()) - maps.Copy(args, filter.SQLArguments()) - - rows, err := conn.Query(ctx, q, args) - if err != nil { - return fmt.Errorf("cannot query thirdParties: %w", err) - } - - thirdParties, err := pgx.CollectRows(rows, pgx.RowToAddrOfStructByName[ThirdParty]) - if err != nil { - return fmt.Errorf("cannot collect thirdParties: %w", err) - } - - *v = thirdParties - - return nil -} - func (v *ThirdParties) LoadByOrganizationID( ctx context.Context, conn pg.Querier, @@ -1230,113 +1160,6 @@ WHERE %s return count, nil } -func (vs *ThirdParties) LoadAllByDatumID( - ctx context.Context, - conn pg.Querier, - scope Scoper, - datumID gid.GID, -) error { - q := ` -WITH vend AS ( - SELECT - v.id, - v.tenant_id, - v.organization_id, - v.parent_third_party_id, - v.common_third_party_id, - v.name, - v.description, - v.category, - v.headquarter_address, - v.legal_name, - v.website_url, - v.privacy_policy_url, - v.service_level_agreement_url, - v.data_processing_agreement_url, - v.business_associate_agreement_url, - v.subprocessors_list_url, - v.certifications, - v.countries, - v.business_owner_profile_id, - v.security_owner_profile_id, - v.status_page_url, - v.terms_of_service_url, - v.security_page_url, - v.trust_page_url, - v.show_on_trust_center, - v.level, - v.vetting_status, - v.vetting_website_url, - v.vetting_procedure, - v.vetting_processing_started_at, - v.vetting_error_message, - v.created_at, - v.updated_at - FROM - third_parties v - INNER JOIN - data_third_parties dv ON v.id = dv.third_party_id - WHERE - dv.datum_id = @datum_id -) -SELECT - id, - organization_id, - parent_third_party_id, - common_third_party_id, - name, - description, - category, - headquarter_address, - legal_name, - website_url, - privacy_policy_url, - service_level_agreement_url, - data_processing_agreement_url, - business_associate_agreement_url, - subprocessors_list_url, - certifications, - countries, - business_owner_profile_id, - security_owner_profile_id, - status_page_url, - terms_of_service_url, - security_page_url, - trust_page_url, - show_on_trust_center, - level, - vetting_status, - vetting_website_url, - vetting_procedure, - vetting_processing_started_at, - vetting_error_message, - created_at, - updated_at -FROM - vend -WHERE %s -ORDER BY name ASC -` - q = fmt.Sprintf(q, scope.SQLFragment()) - - args := pgx.StrictNamedArgs{"datum_id": datumID} - maps.Copy(args, scope.SQLArguments()) - - rows, err := conn.Query(ctx, q, args) - if err != nil { - return fmt.Errorf("cannot query thirdParties: %w", err) - } - - thirdParties, err := pgx.CollectRows(rows, pgx.RowToAddrOfStructByName[ThirdParty]) - if err != nil { - return fmt.Errorf("cannot collect thirdParties: %w", err) - } - - *vs = thirdParties - - return nil -} - func (vs *ThirdParties) LoadByDatumID( ctx context.Context, conn pg.Querier, @@ -1624,113 +1447,6 @@ ORDER BY return thirdPartyMap, nil } -func (vs *ThirdParties) LoadAllByAssetID( - ctx context.Context, - conn pg.Querier, - scope Scoper, - assetID gid.GID, -) error { - q := ` -WITH vend AS ( - SELECT - v.id, - v.tenant_id, - v.organization_id, - v.parent_third_party_id, - v.common_third_party_id, - v.name, - v.description, - v.category, - v.headquarter_address, - v.legal_name, - v.website_url, - v.privacy_policy_url, - v.service_level_agreement_url, - v.data_processing_agreement_url, - v.business_associate_agreement_url, - v.subprocessors_list_url, - v.certifications, - v.countries, - v.business_owner_profile_id, - v.security_owner_profile_id, - v.status_page_url, - v.terms_of_service_url, - v.security_page_url, - v.trust_page_url, - v.show_on_trust_center, - v.level, - v.vetting_status, - v.vetting_website_url, - v.vetting_procedure, - v.vetting_processing_started_at, - v.vetting_error_message, - v.created_at, - v.updated_at - FROM - third_parties v - INNER JOIN - asset_third_parties av ON v.id = av.third_party_id - WHERE - av.asset_id = @asset_id -) -SELECT - id, - organization_id, - parent_third_party_id, - common_third_party_id, - name, - description, - category, - headquarter_address, - legal_name, - website_url, - privacy_policy_url, - service_level_agreement_url, - data_processing_agreement_url, - business_associate_agreement_url, - subprocessors_list_url, - certifications, - countries, - business_owner_profile_id, - security_owner_profile_id, - status_page_url, - terms_of_service_url, - security_page_url, - trust_page_url, - show_on_trust_center, - level, - vetting_status, - vetting_website_url, - vetting_procedure, - vetting_processing_started_at, - vetting_error_message, - created_at, - updated_at -FROM - vend -WHERE %s -ORDER BY name ASC -` - q = fmt.Sprintf(q, scope.SQLFragment()) - - args := pgx.StrictNamedArgs{"asset_id": assetID} - maps.Copy(args, scope.SQLArguments()) - - rows, err := conn.Query(ctx, q, args) - if err != nil { - return fmt.Errorf("cannot query thirdParties: %w", err) - } - - thirdParties, err := pgx.CollectRows(rows, pgx.RowToAddrOfStructByName[ThirdParty]) - if err != nil { - return fmt.Errorf("cannot collect thirdParties: %w", err) - } - - *vs = thirdParties - - return nil -} - func (v *ThirdParty) LoadByOrganizationIDAndCommonThirdPartyID( ctx context.Context, conn pg.Querier, diff --git a/pkg/coredata/tracker_pattern.go b/pkg/coredata/tracker_pattern.go index d8c714d64..5793b25a6 100644 --- a/pkg/coredata/tracker_pattern.go +++ b/pkg/coredata/tracker_pattern.go @@ -634,75 +634,6 @@ WHERE return nil } -func (tps *TrackerPatterns) LoadAllByCookieBannerID( - ctx context.Context, - conn pg.Querier, - scope Scoper, - cookieBannerID gid.GID, - filter *TrackerPatternFilter, - trackerType *TrackerType, -) error { - trackerTypeFragment := "TRUE" - if trackerType != nil { - trackerTypeFragment = "tracker_type = @tracker_type" - } - - q := ` -SELECT - id, - organization_id, - cookie_banner_id, - cookie_category_id, - common_tracker_pattern_id, - third_party_id, - tracker_type, - pattern, - match_type, - display_name, - description, - excluded, - max_age_seconds, - source, - last_matched_at, - mapping_requested_at, - created_at, - updated_at -FROM - tracker_patterns -WHERE - %s - AND cookie_banner_id = @cookie_banner_id - AND %s - AND %s -ORDER BY - created_at ASC, id ASC; -` - - q = fmt.Sprintf(q, scope.SQLFragment(), trackerTypeFragment, filter.SQLFragment()) - - args := pgx.StrictNamedArgs{"cookie_banner_id": cookieBannerID} - maps.Copy(args, scope.SQLArguments()) - maps.Copy(args, filter.SQLArguments()) - - if trackerType != nil { - args["tracker_type"] = *trackerType - } - - rows, err := conn.Query(ctx, q, args) - if err != nil { - return fmt.Errorf("cannot query tracker patterns: %w", err) - } - - patterns, err := pgx.CollectRows(rows, pgx.RowToAddrOfStructByName[TrackerPattern]) - if err != nil { - return fmt.Errorf("cannot collect tracker patterns: %w", err) - } - - *tps = patterns - - return nil -} - func (tps *TrackerPatterns) RefreshLastMatchedAtByCookieBannerID( ctx context.Context, tx pg.Tx, diff --git a/pkg/coredata/tracker_resource.go b/pkg/coredata/tracker_resource.go index 958c6335a..1240832de 100644 --- a/pkg/coredata/tracker_resource.go +++ b/pkg/coredata/tracker_resource.go @@ -457,59 +457,6 @@ WHERE return nil } -func (trs *TrackerResources) LoadAllByCookieBannerID( - ctx context.Context, - conn pg.Querier, - scope Scoper, - cookieBannerID gid.GID, - filter *TrackerResourceFilter, -) error { - q := ` -SELECT - id, - organization_id, - cookie_banner_id, - cookie_category_id, - resource_type, - origin, - path, - display_name, - description, - excluded, - last_detected_at, - created_at, - updated_at -FROM - tracker_resources -WHERE - %s - AND cookie_banner_id = @cookie_banner_id - AND %s -ORDER BY - created_at ASC, id ASC; -` - - q = fmt.Sprintf(q, scope.SQLFragment(), filter.SQLFragment()) - - args := pgx.StrictNamedArgs{"cookie_banner_id": cookieBannerID} - maps.Copy(args, scope.SQLArguments()) - maps.Copy(args, filter.SQLArguments()) - - rows, err := conn.Query(ctx, q, args) - if err != nil { - return fmt.Errorf("cannot query tracker resources: %w", err) - } - - resources, err := pgx.CollectRows(rows, pgx.RowToAddrOfStructByName[TrackerResource]) - if err != nil { - return fmt.Errorf("cannot collect tracker resources: %w", err) - } - - *trs = resources - - return nil -} - func (trs *TrackerResources) LoadUncategorisedByCookieBannerID( ctx context.Context, conn pg.Querier, diff --git a/pkg/coredata/transfer_impact_assessment.go b/pkg/coredata/transfer_impact_assessment.go index 5509ef725..dc606b4fb 100644 --- a/pkg/coredata/transfer_impact_assessment.go +++ b/pkg/coredata/transfer_impact_assessment.go @@ -283,51 +283,6 @@ WHERE return nil } -func (tias *TransferImpactAssessments) LoadAllByOrganizationID( - ctx context.Context, - conn pg.Querier, - scope Scoper, - organizationID gid.GID, -) error { - q := ` -SELECT - id, - organization_id, - processing_activity_id, - data_subjects, - legal_mechanism, - transfer, - local_law_risk, - supplementary_measures, - created_at, - updated_at -FROM - processing_activity_transfer_impact_assessments -WHERE - %s - AND organization_id = @organization_id -` - - q = fmt.Sprintf(q, scope.SQLFragment()) - - args := pgx.StrictNamedArgs{"organization_id": organizationID} - maps.Copy(args, scope.SQLArguments()) - - rows, err := conn.Query(ctx, q, args) - if err != nil { - return fmt.Errorf("cannot query transfer impact assessments: %w", err) - } - - results, err := pgx.CollectRows(rows, pgx.RowToAddrOfStructByName[TransferImpactAssessment]) - if err != nil { - return fmt.Errorf("cannot collect transfer impact assessments: %w", err) - } - - *tias = results - - return nil -} - func (tia *TransferImpactAssessment) LoadByID( ctx context.Context, conn pg.Querier, diff --git a/pkg/coredata/trust_center_document_access.go b/pkg/coredata/trust_center_document_access.go index 9b09004a6..54676adb5 100644 --- a/pkg/coredata/trust_center_document_access.go +++ b/pkg/coredata/trust_center_document_access.go @@ -594,11 +594,12 @@ WHERE %s return nil } -func (tcdas *TrustCenterDocumentAccesses) LoadAllByTrustCenterAccessID( +func (tcdas *TrustCenterDocumentAccesses) LoadByTrustCenterAccessID( ctx context.Context, conn pg.Querier, scope Scoper, trustCenterAccessID gid.GID, + cursor *page.Cursor[TrustCenterDocumentAccessOrderField], ) error { q := ` SELECT @@ -616,15 +617,16 @@ FROM WHERE %s AND trust_center_access_id = @trust_center_access_id -ORDER BY id ASC + AND %s ` - q = fmt.Sprintf(q, scope.SQLFragment()) + q = fmt.Sprintf(q, scope.SQLFragment(), cursor.SQLFragment()) args := pgx.StrictNamedArgs{ "trust_center_access_id": trustCenterAccessID, } maps.Copy(args, scope.SQLArguments()) + maps.Copy(args, cursor.SQLArguments()) rows, err := conn.Query(ctx, q, args) if err != nil { diff --git a/pkg/coredata/trust_center_file.go b/pkg/coredata/trust_center_file.go index e36229b45..48a537d50 100644 --- a/pkg/coredata/trust_center_file.go +++ b/pkg/coredata/trust_center_file.go @@ -389,51 +389,3 @@ WHERE return count, nil } - -func (t *TrustCenterFiles) LoadAllByOrganizationID( - ctx context.Context, - conn pg.Querier, - scope Scoper, - organizationID gid.GID, - filter *TrustCenterFileFilter, -) error { - q := ` -SELECT - id, - organization_id, - name, - category, - file_id, - trust_center_visibility, - created_at, - updated_at -FROM - trust_center_files -WHERE - %s - AND %s - AND organization_id = @organization_id -ORDER BY - created_at DESC -` - - q = fmt.Sprintf(q, scope.SQLFragment(), filter.SQLFragment()) - - args := pgx.StrictNamedArgs{"organization_id": organizationID} - maps.Copy(args, scope.SQLArguments()) - maps.Copy(args, filter.SQLArguments()) - - rows, err := conn.Query(ctx, q, args) - if err != nil { - return fmt.Errorf("cannot query trust center files: %w", err) - } - - files, err := pgx.CollectRows(rows, pgx.RowToAddrOfStructByName[TrustCenterFile]) - if err != nil { - return fmt.Errorf("cannot collect trust center files: %w", err) - } - - *t = files - - return nil -} diff --git a/pkg/iam/account_service.go b/pkg/iam/account_service.go index cd08a39ac..ae84302c4 100644 --- a/pkg/iam/account_service.go +++ b/pkg/iam/account_service.go @@ -606,11 +606,27 @@ func (s AccountService) ListInvitingOrganizations(ctx context.Context, identityI err := s.pg.WithConn( ctx, func(ctx context.Context, conn pg.Querier) error { - err := organizations.LoadAllByIdentityIDWithPendingInvitation(ctx, conn, coredata.NewNoScope(), identityID) + loaded, err := page.LoadAll( + ctx, + page.OrderBy[coredata.OrganizationOrderField]{ + Field: coredata.OrganizationOrderFieldName, + Direction: page.OrderDirectionAsc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.OrganizationOrderField]) ([]*coredata.Organization, error) { + var batch coredata.Organizations + if err := batch.LoadByIdentityIDWithPendingInvitation(ctx, conn, coredata.NewNoScope(), identityID, cursor); err != nil { + return nil, fmt.Errorf("cannot load inviting organizations: %w", err) + } + + return batch, nil + }, + ) if err != nil { - return fmt.Errorf("cannot load inviting organizations: %w", err) + return err } + organizations = loaded + return nil }, ) diff --git a/pkg/iam/scim/service.go b/pkg/iam/scim/service.go index c5bb506b4..5f5426ab6 100644 --- a/pkg/iam/scim/service.go +++ b/pkg/iam/scim/service.go @@ -38,6 +38,7 @@ import ( "go.probo.inc/probo/pkg/crypto/rand" "go.probo.inc/probo/pkg/gid" "go.probo.inc/probo/pkg/mail" + "go.probo.inc/probo/pkg/page" "go.probo.inc/probo/pkg/webhook" webhooktypes "go.probo.inc/probo/pkg/webhook/types" ) @@ -407,10 +408,27 @@ func (s *Service) ListUsers( err = s.pg.WithConn( ctx, func(ctx context.Context, conn pg.Querier) error { - if err := profiles.LoadAllByOrganizationID(ctx, conn, scope, config.OrganizationID, filter); err != nil { - return fmt.Errorf("cannot load profiles: %w", err) + loaded, err := page.LoadAll( + ctx, + page.OrderBy[coredata.MembershipProfileOrderField]{ + Field: coredata.MembershipProfileOrderFieldCreatedAt, + Direction: page.OrderDirectionAsc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.MembershipProfileOrderField]) ([]*coredata.MembershipProfile, error) { + var batch coredata.MembershipProfiles + if err := batch.LoadByOrganizationID(ctx, conn, scope, config.OrganizationID, cursor, filter); err != nil { + return nil, fmt.Errorf("cannot load profiles: %w", err) + } + + return batch, nil + }, + ) + if err != nil { + return err } + profiles = loaded + return nil }, ) diff --git a/pkg/mailman/service.go b/pkg/mailman/service.go index 509d2ef01..5d2b86cda 100644 --- a/pkg/mailman/service.go +++ b/pkg/mailman/service.go @@ -706,9 +706,23 @@ func (s *Service) CreateUpdateEmails( return s.pg.WithTx( ctx, func(ctx context.Context, tx pg.Tx) error { - var subscribers coredata.MailingListSubscribers - if err := subscribers.LoadAllConfirmedByMailingListID(ctx, tx, scope, mailingListID); err != nil { - return fmt.Errorf("cannot load confirmed subscribers: %w", err) + subscribers, err := page.LoadAll( + ctx, + page.OrderBy[coredata.MailingListSubscriberOrderField]{ + Field: coredata.MailingListSubscriberOrderFieldCreatedAt, + Direction: page.OrderDirectionAsc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.MailingListSubscriberOrderField]) ([]*coredata.MailingListSubscriber, error) { + var batch coredata.MailingListSubscribers + if err := batch.LoadConfirmedByMailingListID(ctx, tx, scope, mailingListID, cursor); err != nil { + return nil, fmt.Errorf("cannot load confirmed subscribers: %w", err) + } + + return batch, nil + }, + ) + if err != nil { + return err } if len(subscribers) == 0 { diff --git a/pkg/page/load_all.go b/pkg/page/load_all.go new file mode 100644 index 000000000..800d00632 --- /dev/null +++ b/pkg/page/load_all.go @@ -0,0 +1,76 @@ +// Copyright (c) 2025-2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package page + +import ( + "context" + "fmt" +) + +// MaxLoadAllPages caps how many pages LoadAll walks, bounding a single +// call to MaxLoadAllPages*MaxCursorSize rows. Past that, LoadAll errors +// rather than materialising an unbounded set. +const MaxLoadAllPages = 20 + +// Loader runs one paginated query for the given cursor and returns the +// rows it loaded. Callers bind the connection, scope, parent key and +// filter in a closure, exposing only ctx and cursor (typically a coredata +// LoadBy* on a fresh receiver). +type Loader[T Paginable[U], U OrderField] func(ctx context.Context, cursor *Cursor[U]) ([]T, error) + +// LoadAll walks every matching row via keyset pagination, advancing a +// MaxCursorSize forward cursor until no rows remain, and returns them +// concatenated. fetch runs the paginated query for the cursor. It errors +// past MaxLoadAllPages pages. +func LoadAll[T Paginable[U], U OrderField]( + ctx context.Context, + orderBy OrderBy[U], + fetch Loader[T, U], +) ([]T, error) { + var ( + all []T + key *CursorKey + ) + + for page := 0; ; page++ { + if page >= MaxLoadAllPages { + return nil, fmt.Errorf( + "cannot load all rows: result set exceeds %d rows (%d pages of %d)", + MaxLoadAllPages*MaxCursorSize, + MaxLoadAllPages, + MaxCursorSize, + ) + } + + cursor := NewCursor(MaxCursorSize, key, Head, orderBy) + + rows, err := fetch(ctx, cursor) + if err != nil { + return nil, fmt.Errorf("cannot load all rows: %w", err) + } + + p := NewPage(rows, cursor) + all = append(all, p.Data...) + + if !p.Info.HasNext { + break + } + + k := p.Last().CursorKey(orderBy.Field) + key = &k + } + + return all, nil +} diff --git a/pkg/page/load_all_test.go b/pkg/page/load_all_test.go new file mode 100644 index 000000000..a7e51ac49 --- /dev/null +++ b/pkg/page/load_all_test.go @@ -0,0 +1,179 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package page + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.probo.inc/probo/pkg/gid" +) + +// loadAllItem is a minimal Paginable used to exercise LoadAll without a +// database. Values are unique so ordering is fully determined by value and +// the id tiebreak is never decisive. +type loadAllItem struct { + id gid.GID + value int +} + +func (i *loadAllItem) CursorKey(_ testOrderField) CursorKey { + return CursorKey{ID: i.id, Value: i.value} +} + +// newLoadAllStore builds n items pre-sorted ascending by value (value == i). +func newLoadAllStore(n int) []*loadAllItem { + tenantID := gid.NewTenantID() + + store := make([]*loadAllItem, n) + for i := range store { + store[i] = &loadAllItem{id: gid.New(tenantID, 1), value: i} + } + + return store +} + +// keysetPage mimics the SQL a coredata LoadBy* method runs for a forward +// (ascending, Head) cursor: it returns the over-fetched window the cursor +// asks for (Size+1 rows with no key, Size+2 once a key is set, including the +// boundary row) so NewPage can trim it exactly as it would in production. +func keysetPage(store []*loadAllItem, cursor *Cursor[testOrderField]) []*loadAllItem { + limit := cursor.Size + 1 + if cursor.Key != nil { + limit = cursor.Size + 2 + } + + var out []*loadAllItem + + for _, it := range store { + if cursor.Key != nil { + keyValue := cursor.Key.Value.(int) + if it.value < keyValue { + continue + } + + if it.value == keyValue && it.id.String() < cursor.Key.ID.String() { + continue + } + } + + out = append(out, it) + if len(out) == limit { + break + } + } + + return out +} + +func loadAllValues(items []*loadAllItem) []int { + values := make([]int, len(items)) + for i, it := range items { + values[i] = it.value + } + + return values +} + +func ascOrderBy() OrderBy[testOrderField] { + return OrderBy[testOrderField]{ + Field: testOrderField("value"), + Direction: OrderDirectionAsc, + } +} + +func TestLoadAll(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + count int + expectedFetchs int + }{ + {name: "empty result set", count: 0, expectedFetchs: 1}, + {name: "single short page", count: 10, expectedFetchs: 1}, + {name: "exactly one full page", count: MaxCursorSize, expectedFetchs: 1}, + {name: "two pages", count: MaxCursorSize + 1, expectedFetchs: 2}, + {name: "several pages", count: 2*MaxCursorSize + 7, expectedFetchs: 3}, + {name: "at the page-count limit", count: MaxLoadAllPages * MaxCursorSize, expectedFetchs: MaxLoadAllPages}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + store := newLoadAllStore(tt.count) + + fetchs := 0 + got, err := LoadAll( + context.Background(), + ascOrderBy(), + func(_ context.Context, cursor *Cursor[testOrderField]) ([]*loadAllItem, error) { + fetchs++ + return keysetPage(store, cursor), nil + }, + ) + + require.NoError(t, err) + require.Len(t, got, tt.count) + assert.Equal(t, loadAllValues(store), loadAllValues(got), "every row returned exactly once, in order") + assert.Equal(t, tt.expectedFetchs, fetchs, "page count") + }) + } +} + +func TestLoadAllPropagatesFetchError(t *testing.T) { + t.Parallel() + + sentinel := errors.New("boom") + + got, err := LoadAll( + context.Background(), + ascOrderBy(), + func(_ context.Context, _ *Cursor[testOrderField]) ([]*loadAllItem, error) { + return nil, sentinel + }, + ) + + require.ErrorIs(t, err, sentinel) + assert.Nil(t, got) +} + +func TestLoadAllRefusesUnboundedResultSet(t *testing.T) { + t.Parallel() + + // One row past MaxLoadAllPages full pages keeps HasNext true after the + // last allowed page, so LoadAll must bail instead of looping forever. + store := newLoadAllStore(MaxLoadAllPages*MaxCursorSize + 1) + + fetchs := 0 + got, err := LoadAll( + context.Background(), + ascOrderBy(), + func(_ context.Context, cursor *Cursor[testOrderField]) ([]*loadAllItem, error) { + fetchs++ + return keysetPage(store, cursor), nil + }, + ) + + require.Error(t, err) + assert.Nil(t, got) + assert.Contains(t, err.Error(), "result set exceeds") + assert.Equal(t, MaxLoadAllPages, fetchs, "stops after walking the max number of pages") +} diff --git a/pkg/probo/generated_document_service.go b/pkg/probo/generated_document_service.go index 2b80703bb..bfb9dab84 100644 --- a/pkg/probo/generated_document_service.go +++ b/pkg/probo/generated_document_service.go @@ -28,6 +28,7 @@ import ( "go.probo.inc/probo/pkg/coredata" "go.probo.inc/probo/pkg/docgen" "go.probo.inc/probo/pkg/gid" + "go.probo.inc/probo/pkg/page" ) type GeneratedDocumentService struct { @@ -148,9 +149,23 @@ func (s *GeneratedDocumentService) buildStatementOfApplicabilityDocumentData( return docgen.StatementOfApplicabilityData{}, fmt.Errorf("cannot load organization: %w", err) } - var applicabilityStatements coredata.ApplicabilityStatements - if err := applicabilityStatements.LoadAllByStatementOfApplicabilityID(ctx, conn, scope, statementOfApplicability.ID); err != nil { - return docgen.StatementOfApplicabilityData{}, fmt.Errorf("cannot load applicability statements: %w", err) + applicabilityStatements, err := page.LoadAll( + ctx, + page.OrderBy[coredata.ApplicabilityStatementOrderField]{ + Field: coredata.ApplicabilityStatementOrderFieldControlSectionTitle, + Direction: page.OrderDirectionAsc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.ApplicabilityStatementOrderField]) ([]*coredata.ApplicabilityStatement, error) { + var batch coredata.ApplicabilityStatements + if err := batch.LoadByStatementOfApplicabilityID(ctx, conn, scope, statementOfApplicability.ID, cursor); err != nil { + return nil, fmt.Errorf("cannot load applicability statements: %w", err) + } + + return batch, nil + }, + ) + if err != nil { + return docgen.StatementOfApplicabilityData{}, err } if len(applicabilityStatements) == 0 { @@ -421,9 +436,23 @@ func (s *GeneratedDocumentService) buildDataListDocumentData( conn pg.Querier, organization *coredata.Organization, ) (docgen.DataListData, error) { - var data coredata.Data - if err := data.LoadAllByOrganizationID(ctx, conn, scope, organization.ID); err != nil { - return docgen.DataListData{}, fmt.Errorf("cannot load data: %w", err) + data, err := page.LoadAll( + ctx, + page.OrderBy[coredata.DatumOrderField]{ + Field: coredata.DatumOrderFieldName, + Direction: page.OrderDirectionAsc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.DatumOrderField]) ([]*coredata.Datum, error) { + var batch coredata.Data + if err := batch.LoadByOrganizationID(ctx, conn, scope, organization.ID, cursor); err != nil { + return nil, fmt.Errorf("cannot load data: %w", err) + } + + return batch, nil + }, + ) + if err != nil { + return docgen.DataListData{}, err } if len(data) == 0 { @@ -462,9 +491,23 @@ func (s *GeneratedDocumentService) buildDataListDocumentData( ownerName = p.FullName } - var thirdParties coredata.ThirdParties - if err := thirdParties.LoadAllByDatumID(ctx, conn, scope, d.ID); err != nil { - return docgen.DataListData{}, fmt.Errorf("cannot load thirdParties for datum %s: %w", d.ID, err) + thirdParties, err := page.LoadAll( + ctx, + page.OrderBy[coredata.ThirdPartyOrderField]{ + Field: coredata.ThirdPartyOrderFieldName, + Direction: page.OrderDirectionAsc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.ThirdPartyOrderField]) ([]*coredata.ThirdParty, error) { + var batch coredata.ThirdParties + if err := batch.LoadByDatumID(ctx, conn, scope, d.ID, cursor); err != nil { + return nil, fmt.Errorf("cannot load thirdParties for datum %s: %w", d.ID, err) + } + + return batch, nil + }, + ) + if err != nil { + return docgen.DataListData{}, err } thirdPartyNames := make([]string, 0, len(thirdParties)) @@ -665,9 +708,23 @@ func (s *GeneratedDocumentService) buildAssetListDocumentData( conn pg.Querier, organization *coredata.Organization, ) (docgen.AssetListData, error) { - var assets coredata.Assets - if err := assets.LoadAllByOrganizationID(ctx, conn, scope, organization.ID); err != nil { - return docgen.AssetListData{}, fmt.Errorf("cannot load assets: %w", err) + assets, err := page.LoadAll( + ctx, + page.OrderBy[coredata.AssetOrderField]{ + Field: coredata.AssetOrderFieldName, + Direction: page.OrderDirectionAsc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.AssetOrderField]) ([]*coredata.Asset, error) { + var batch coredata.Assets + if err := batch.LoadByOrganizationID(ctx, conn, scope, organization.ID, cursor); err != nil { + return nil, fmt.Errorf("cannot load assets: %w", err) + } + + return batch, nil + }, + ) + if err != nil { + return docgen.AssetListData{}, err } if len(assets) == 0 { @@ -706,9 +763,23 @@ func (s *GeneratedDocumentService) buildAssetListDocumentData( ownerName = p.FullName } - var thirdParties coredata.ThirdParties - if err := thirdParties.LoadAllByAssetID(ctx, conn, scope, a.ID); err != nil { - return docgen.AssetListData{}, fmt.Errorf("cannot load thirdParties for asset %s: %w", a.ID, err) + thirdParties, err := page.LoadAll( + ctx, + page.OrderBy[coredata.ThirdPartyOrderField]{ + Field: coredata.ThirdPartyOrderFieldName, + Direction: page.OrderDirectionAsc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.ThirdPartyOrderField]) ([]*coredata.ThirdParty, error) { + var batch coredata.ThirdParties + if err := batch.LoadByAssetID(ctx, conn, scope, a.ID, cursor); err != nil { + return nil, fmt.Errorf("cannot load thirdParties for asset %s: %w", a.ID, err) + } + + return batch, nil + }, + ) + if err != nil { + return docgen.AssetListData{}, err } thirdPartyNames := make([]string, 0, len(thirdParties)) @@ -932,9 +1003,23 @@ func (s *GeneratedDocumentService) buildFindingListDocumentData( conn pg.Querier, organization *coredata.Organization, ) (docgen.FindingListData, error) { - var findings coredata.Findings - if err := findings.LoadAllByOrganizationID(ctx, conn, scope, organization.ID); err != nil { - return docgen.FindingListData{}, fmt.Errorf("cannot load findings: %w", err) + findings, err := page.LoadAll( + ctx, + page.OrderBy[coredata.FindingOrderField]{ + Field: coredata.FindingOrderFieldReferenceId, + Direction: page.OrderDirectionAsc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.FindingOrderField]) ([]*coredata.Finding, error) { + var batch coredata.Findings + if err := batch.LoadByOrganizationID(ctx, conn, scope, organization.ID, cursor, coredata.NewFindingFilter(nil, nil, nil, nil)); err != nil { + return nil, fmt.Errorf("cannot load findings: %w", err) + } + + return batch, nil + }, + ) + if err != nil { + return docgen.FindingListData{}, err } if len(findings) == 0 { @@ -1244,9 +1329,23 @@ func (s *GeneratedDocumentService) buildObligationListDocumentData( conn pg.Querier, organization *coredata.Organization, ) (docgen.ObligationListData, error) { - var obligations coredata.Obligations - if err := obligations.LoadAllByOrganizationID(ctx, conn, scope, organization.ID); err != nil { - return docgen.ObligationListData{}, fmt.Errorf("cannot load obligations: %w", err) + obligations, err := page.LoadAll( + ctx, + page.OrderBy[coredata.ObligationOrderField]{ + Field: coredata.ObligationOrderFieldCreatedAt, + Direction: page.OrderDirectionAsc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.ObligationOrderField]) ([]*coredata.Obligation, error) { + var batch coredata.Obligations + if err := batch.LoadByOrganizationID(ctx, conn, scope, organization.ID, cursor); err != nil { + return nil, fmt.Errorf("cannot load obligations: %w", err) + } + + return batch, nil + }, + ) + if err != nil { + return docgen.ObligationListData{}, err } if len(obligations) == 0 { @@ -1524,9 +1623,23 @@ func (s *GeneratedDocumentService) buildProcessingActivityListDocumentData( conn pg.Querier, organization *coredata.Organization, ) (docgen.ProcessingActivityListData, error) { - var processingActivities coredata.ProcessingActivities - if err := processingActivities.LoadAllByOrganizationID(ctx, conn, scope, organization.ID); err != nil { - return docgen.ProcessingActivityListData{}, fmt.Errorf("cannot load processing activities: %w", err) + processingActivities, err := page.LoadAll( + ctx, + page.OrderBy[coredata.ProcessingActivityOrderField]{ + Field: coredata.ProcessingActivityOrderFieldCreatedAt, + Direction: page.OrderDirectionDesc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.ProcessingActivityOrderField]) ([]*coredata.ProcessingActivity, error) { + var batch coredata.ProcessingActivities + if err := batch.LoadByOrganizationID(ctx, conn, scope, organization.ID, cursor); err != nil { + return nil, fmt.Errorf("cannot load processing activities: %w", err) + } + + return batch, nil + }, + ) + if err != nil { + return docgen.ProcessingActivityListData{}, err } if len(processingActivities) == 0 { @@ -1905,9 +2018,23 @@ func (s *GeneratedDocumentService) buildDataProtectionImpactAssessmentListDocume conn pg.Querier, organization *coredata.Organization, ) (docgen.DataProtectionImpactAssessmentListData, error) { - var assessments coredata.DataProtectionImpactAssessments - if err := assessments.LoadAllByOrganizationID(ctx, conn, scope, organization.ID); err != nil { - return docgen.DataProtectionImpactAssessmentListData{}, fmt.Errorf("cannot load DPIAs: %w", err) + assessments, err := page.LoadAll( + ctx, + page.OrderBy[coredata.DataProtectionImpactAssessmentOrderField]{ + Field: coredata.DataProtectionImpactAssessmentOrderFieldCreatedAt, + Direction: page.OrderDirectionAsc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.DataProtectionImpactAssessmentOrderField]) ([]*coredata.DataProtectionImpactAssessment, error) { + var batch coredata.DataProtectionImpactAssessments + if err := batch.LoadByOrganizationID(ctx, conn, scope, organization.ID, cursor); err != nil { + return nil, fmt.Errorf("cannot load DPIAs: %w", err) + } + + return batch, nil + }, + ) + if err != nil { + return docgen.DataProtectionImpactAssessmentListData{}, err } if len(assessments) == 0 { @@ -2123,9 +2250,23 @@ func (s *GeneratedDocumentService) buildTransferImpactAssessmentListDocumentData conn pg.Querier, organization *coredata.Organization, ) (docgen.TransferImpactAssessmentListData, error) { - var assessments coredata.TransferImpactAssessments - if err := assessments.LoadAllByOrganizationID(ctx, conn, scope, organization.ID); err != nil { - return docgen.TransferImpactAssessmentListData{}, fmt.Errorf("cannot load TIAs: %w", err) + assessments, err := page.LoadAll( + ctx, + page.OrderBy[coredata.TransferImpactAssessmentOrderField]{ + Field: coredata.TransferImpactAssessmentOrderFieldCreatedAt, + Direction: page.OrderDirectionAsc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.TransferImpactAssessmentOrderField]) ([]*coredata.TransferImpactAssessment, error) { + var batch coredata.TransferImpactAssessments + if err := batch.LoadByOrganizationID(ctx, conn, scope, organization.ID, cursor); err != nil { + return nil, fmt.Errorf("cannot load TIAs: %w", err) + } + + return batch, nil + }, + ) + if err != nil { + return docgen.TransferImpactAssessmentListData{}, err } if len(assessments) == 0 { @@ -2359,15 +2500,23 @@ func (s *GeneratedDocumentService) buildThirdPartyListDocumentData( ) (docgen.ThirdPartyListData, error) { firstLevel := 1 - var thirdParties coredata.ThirdParties - if err := thirdParties.LoadAllByOrganizationID( + thirdParties, err := page.LoadAll( ctx, - conn, - scope, - organization.ID, - coredata.NewThirdPartyFilter(nil, &firstLevel, nil), - ); err != nil { - return docgen.ThirdPartyListData{}, fmt.Errorf("cannot load thirdParties: %w", err) + page.OrderBy[coredata.ThirdPartyOrderField]{ + Field: coredata.ThirdPartyOrderFieldName, + Direction: page.OrderDirectionAsc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.ThirdPartyOrderField]) ([]*coredata.ThirdParty, error) { + var batch coredata.ThirdParties + if err := batch.LoadByOrganizationID(ctx, conn, scope, organization.ID, cursor, coredata.NewThirdPartyFilter(nil, &firstLevel, nil)); err != nil { + return nil, fmt.Errorf("cannot load thirdParties: %w", err) + } + + return batch, nil + }, + ) + if err != nil { + return docgen.ThirdPartyListData{}, err } if len(thirdParties) == 0 { @@ -2900,9 +3049,23 @@ func (s *GeneratedDocumentService) buildRiskListDocumentData( conn pg.Querier, organization *coredata.Organization, ) (docgen.RiskListData, error) { - var risks coredata.Risks - if err := risks.LoadAllByOrganizationID(ctx, conn, scope, organization.ID); err != nil { - return docgen.RiskListData{}, fmt.Errorf("cannot load risks: %w", err) + risks, err := page.LoadAll( + ctx, + page.OrderBy[coredata.RiskOrderField]{ + Field: coredata.RiskOrderFieldName, + Direction: page.OrderDirectionAsc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.RiskOrderField]) ([]*coredata.Risk, error) { + var batch coredata.Risks + if err := batch.LoadByOrganizationID(ctx, conn, scope, organization.ID, cursor, coredata.NewRiskFilter(nil)); err != nil { + return nil, fmt.Errorf("cannot load risks: %w", err) + } + + return batch, nil + }, + ) + if err != nil { + return docgen.RiskListData{}, err } if len(risks) == 0 { diff --git a/pkg/riskmanagement/mermaid.go b/pkg/riskmanagement/mermaid.go index 0aee213ac..2f0bef814 100644 --- a/pkg/riskmanagement/mermaid.go +++ b/pkg/riskmanagement/mermaid.go @@ -22,6 +22,7 @@ import ( "go.gearno.de/kit/pg" "go.probo.inc/probo/pkg/coredata" "go.probo.inc/probo/pkg/gid" + "go.probo.inc/probo/pkg/page" ) func (s *Service) BuildScopeMermaidChart(ctx context.Context, scope coredata.Scoper, scopeID gid.GID) (string, error) { @@ -33,22 +34,90 @@ func (s *Service) BuildScopeMermaidChart(ctx context.Context, scope coredata.Sco ) err := s.pg.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error { - if err := nodes.LoadAllByRiskAssessmentScopeID(ctx, conn, scope, scopeID); err != nil { - return fmt.Errorf("cannot load nodes: %w", err) + loadedNodes, err := page.LoadAll( + ctx, + page.OrderBy[coredata.RiskAssessmentNodeOrderField]{ + Field: coredata.RiskAssessmentNodeOrderFieldCreatedAt, + Direction: page.OrderDirectionAsc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.RiskAssessmentNodeOrderField]) ([]*coredata.RiskAssessmentNode, error) { + var batch coredata.RiskAssessmentNodes + if err := batch.LoadByRiskAssessmentScopeID(ctx, conn, scope, scopeID, cursor); err != nil { + return nil, fmt.Errorf("cannot load nodes: %w", err) + } + + return batch, nil + }, + ) + if err != nil { + return err } - if err := boundaries.LoadAllByRiskAssessmentScopeID(ctx, conn, scope, scopeID); err != nil { - return fmt.Errorf("cannot load boundaries: %w", err) + nodes = loadedNodes + + loadedBoundaries, err := page.LoadAll( + ctx, + page.OrderBy[coredata.RiskAssessmentBoundaryOrderField]{ + Field: coredata.RiskAssessmentBoundaryOrderFieldCreatedAt, + Direction: page.OrderDirectionAsc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.RiskAssessmentBoundaryOrderField]) ([]*coredata.RiskAssessmentBoundary, error) { + var batch coredata.RiskAssessmentBoundaries + if err := batch.LoadByRiskAssessmentScopeID(ctx, conn, scope, scopeID, cursor); err != nil { + return nil, fmt.Errorf("cannot load boundaries: %w", err) + } + + return batch, nil + }, + ) + if err != nil { + return err } - if err := processes.LoadAllByRiskAssessmentScopeID(ctx, conn, scope, scopeID); err != nil { - return fmt.Errorf("cannot load processes: %w", err) + boundaries = loadedBoundaries + + loadedProcesses, err := page.LoadAll( + ctx, + page.OrderBy[coredata.RiskAssessmentProcessOrderField]{ + Field: coredata.RiskAssessmentProcessOrderFieldCreatedAt, + Direction: page.OrderDirectionAsc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.RiskAssessmentProcessOrderField]) ([]*coredata.RiskAssessmentProcess, error) { + var batch coredata.RiskAssessmentProcesses + if err := batch.LoadByRiskAssessmentScopeID(ctx, conn, scope, scopeID, cursor); err != nil { + return nil, fmt.Errorf("cannot load processes: %w", err) + } + + return batch, nil + }, + ) + if err != nil { + return err } - if err := threats.LoadAllByRiskAssessmentScopeID(ctx, conn, scope, scopeID); err != nil { - return fmt.Errorf("cannot load threats: %w", err) + processes = loadedProcesses + + loadedThreats, err := page.LoadAll( + ctx, + page.OrderBy[coredata.RiskAssessmentThreatOrderField]{ + Field: coredata.RiskAssessmentThreatOrderFieldCreatedAt, + Direction: page.OrderDirectionAsc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.RiskAssessmentThreatOrderField]) ([]*coredata.RiskAssessmentThreat, error) { + var batch coredata.RiskAssessmentThreats + if err := batch.LoadByRiskAssessmentScopeID(ctx, conn, scope, scopeID, cursor); err != nil { + return nil, fmt.Errorf("cannot load threats: %w", err) + } + + return batch, nil + }, + ) + if err != nil { + return err } + threats = loadedThreats + return nil }) if err != nil { diff --git a/pkg/slack/slack_message_service.go b/pkg/slack/slack_message_service.go index c69725dab..85935136a 100644 --- a/pkg/slack/slack_message_service.go +++ b/pkg/slack/slack_message_service.go @@ -27,6 +27,7 @@ import ( "go.probo.inc/probo/pkg/coredata" "go.probo.inc/probo/pkg/gid" "go.probo.inc/probo/pkg/mail" + "go.probo.inc/probo/pkg/page" ) const ( @@ -327,9 +328,23 @@ func (s *Service) loadDocumentsReportsAndFilesFromAccesses( reports = []SlackMessageReport{} files = []SlackMessageFile{} - var accesses coredata.TrustCenterDocumentAccesses - if err := accesses.LoadAllByTrustCenterAccessID(ctx, conn, scope, trustCenterAccessID); err != nil { - return nil, nil, nil, fmt.Errorf("cannot load trust center document accesses: %w", err) + accesses, err := page.LoadAll( + ctx, + page.OrderBy[coredata.TrustCenterDocumentAccessOrderField]{ + Field: coredata.TrustCenterDocumentAccessOrderFieldCreatedAt, + Direction: page.OrderDirectionAsc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.TrustCenterDocumentAccessOrderField]) ([]*coredata.TrustCenterDocumentAccess, error) { + var batch coredata.TrustCenterDocumentAccesses + if err := batch.LoadByTrustCenterAccessID(ctx, conn, scope, trustCenterAccessID, cursor); err != nil { + return nil, fmt.Errorf("cannot load trust center document accesses: %w", err) + } + + return batch, nil + }, + ) + if err != nil { + return nil, nil, nil, err } for _, access := range accesses { diff --git a/pkg/trust/trust_center_access_service.go b/pkg/trust/trust_center_access_service.go index 6d4c5b106..6f2199f07 100644 --- a/pkg/trust/trust_center_access_service.go +++ b/pkg/trust/trust_center_access_service.go @@ -27,6 +27,7 @@ import ( "go.probo.inc/probo/pkg/gid" "go.probo.inc/probo/pkg/iam" "go.probo.inc/probo/pkg/mail" + "go.probo.inc/probo/pkg/page" ) type ( @@ -76,12 +77,25 @@ func (s TrustCenterAccessService) Request( documentIDs := req.DocumentIDs if req.DocumentIDs == nil { - var allDocuments coredata.Documents - filter := coredata.NewDocumentTrustCenterFilter() - if err := allDocuments.LoadAllByOrganizationID(ctx, tx, scope, organizationID, filter); err != nil { - return fmt.Errorf("cannot list documents: %w", err) + allDocuments, err := page.LoadAll( + ctx, + page.OrderBy[coredata.DocumentOrderField]{ + Field: coredata.DocumentOrderFieldTitle, + Direction: page.OrderDirectionAsc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.DocumentOrderField]) ([]*coredata.Document, error) { + var batch coredata.Documents + if err := batch.LoadByOrganizationID(ctx, tx, scope, organizationID, cursor, filter); err != nil { + return nil, fmt.Errorf("cannot list documents: %w", err) + } + + return batch, nil + }, + ) + if err != nil { + return err } for _, doc := range allDocuments { @@ -91,12 +105,25 @@ func (s TrustCenterAccessService) Request( reportIDs := req.ReportIDs if req.ReportIDs == nil { - var allAudits coredata.Audits - auditFilter := coredata.NewAuditTrustCenterFilter() - if err := allAudits.LoadAllByOrganizationID(ctx, tx, scope, organizationID, auditFilter); err != nil { - return fmt.Errorf("cannot list audits: %w", err) + allAudits, err := page.LoadAll( + ctx, + page.OrderBy[coredata.AuditOrderField]{ + Field: coredata.AuditOrderFieldCreatedAt, + Direction: page.OrderDirectionAsc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.AuditOrderField]) ([]*coredata.Audit, error) { + var batch coredata.Audits + if err := batch.LoadByOrganizationID(ctx, tx, scope, organizationID, cursor, auditFilter); err != nil { + return nil, fmt.Errorf("cannot list audits: %w", err) + } + + return batch, nil + }, + ) + if err != nil { + return err } for _, audit := range allAudits { @@ -108,14 +135,27 @@ func (s TrustCenterAccessService) Request( trustCenterFileIDs := req.TrustCenterFileIDs if req.TrustCenterFileIDs == nil { - var allTrustCenterFiles coredata.TrustCenterFiles - filter := coredata.NewTrustCenterFileFilter( coredata.WithTrustCenterFileVisibilities(coredata.TrustCenterVisibilityPrivate, coredata.TrustCenterVisibilityNone), ) - if err := allTrustCenterFiles.LoadAllByOrganizationID(ctx, tx, scope, organizationID, filter); err != nil { - return fmt.Errorf("cannot list trust center files: %w", err) + allTrustCenterFiles, err := page.LoadAll( + ctx, + page.OrderBy[coredata.TrustCenterFileOrderField]{ + Field: coredata.TrustCenterFileOrderFieldCreatedAt, + Direction: page.OrderDirectionDesc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.TrustCenterFileOrderField]) ([]*coredata.TrustCenterFile, error) { + var batch coredata.TrustCenterFiles + if err := batch.LoadByOrganizationID(ctx, tx, scope, organizationID, cursor, filter); err != nil { + return nil, fmt.Errorf("cannot list trust center files: %w", err) + } + + return batch, nil + }, + ) + if err != nil { + return err } for _, file := range allTrustCenterFiles { @@ -123,9 +163,23 @@ func (s TrustCenterAccessService) Request( } } - var existingAccesses coredata.TrustCenterDocumentAccesses - if err := existingAccesses.LoadAllByTrustCenterAccessID(ctx, tx, scope, access.ID); err != nil { - return fmt.Errorf("cannot load existing access records: %w", err) + existingAccesses, err := page.LoadAll( + ctx, + page.OrderBy[coredata.TrustCenterDocumentAccessOrderField]{ + Field: coredata.TrustCenterDocumentAccessOrderFieldCreatedAt, + Direction: page.OrderDirectionAsc, + }, + func(ctx context.Context, cursor *page.Cursor[coredata.TrustCenterDocumentAccessOrderField]) ([]*coredata.TrustCenterDocumentAccess, error) { + var batch coredata.TrustCenterDocumentAccesses + if err := batch.LoadByTrustCenterAccessID(ctx, tx, scope, access.ID, cursor); err != nil { + return nil, fmt.Errorf("cannot load existing access records: %w", err) + } + + return batch, nil + }, + ) + if err != nil { + return err } existingDocumentIDs, existingReportIDs, existingTrustCenterFileIDs := extractExistingIDs(existingAccesses)