Let WalkAll use coredata collection slice types
WalkAll now accepts loaders and callbacks typed as S ~[]T so exports can use AuditLogEntries and SCIMEvents directly. Pass the trimmed page slice to walk, not the over-fetched loader result. Signed-off-by: Cursor Agent <cursoragent@cursor.com> Co-authored-by: Bryan FRIMIN <bryan@frimin.fr>
This commit is contained in:
@@ -95,7 +95,7 @@ func (s *LogExportService) streamAuditLogCSV(
|
|||||||
Field: coredata.AuditLogEntryOrderFieldCreatedAt,
|
Field: coredata.AuditLogEntryOrderFieldCreatedAt,
|
||||||
Direction: page.OrderDirectionAsc,
|
Direction: page.OrderDirectionAsc,
|
||||||
},
|
},
|
||||||
func(ctx context.Context, cursor *page.Cursor[coredata.AuditLogEntryOrderField]) ([]*coredata.AuditLogEntry, error) {
|
func(ctx context.Context, cursor *page.Cursor[coredata.AuditLogEntryOrderField]) (coredata.AuditLogEntries, error) {
|
||||||
var logs coredata.AuditLogEntries
|
var logs coredata.AuditLogEntries
|
||||||
if err := logs.LoadByOrganizationID(
|
if err := logs.LoadByOrganizationID(
|
||||||
ctx,
|
ctx,
|
||||||
@@ -110,8 +110,7 @@ func (s *LogExportService) streamAuditLogCSV(
|
|||||||
|
|
||||||
return logs, nil
|
return logs, nil
|
||||||
},
|
},
|
||||||
func(pageEntries []*coredata.AuditLogEntry) error {
|
func(entries coredata.AuditLogEntries) error {
|
||||||
entries := coredata.AuditLogEntries(pageEntries)
|
|
||||||
actorsByID, err := loadAuditLogActorExportInfo(ctx, conn, entries)
|
actorsByID, err := loadAuditLogActorExportInfo(ctx, conn, entries)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -154,7 +153,7 @@ func (s *LogExportService) streamSCIMEventCSV(
|
|||||||
Field: coredata.SCIMEventOrderFieldCreatedAt,
|
Field: coredata.SCIMEventOrderFieldCreatedAt,
|
||||||
Direction: page.OrderDirectionAsc,
|
Direction: page.OrderDirectionAsc,
|
||||||
},
|
},
|
||||||
func(ctx context.Context, cursor *page.Cursor[coredata.SCIMEventOrderField]) ([]*coredata.SCIMEvent, error) {
|
func(ctx context.Context, cursor *page.Cursor[coredata.SCIMEventOrderField]) (coredata.SCIMEvents, error) {
|
||||||
var events coredata.SCIMEvents
|
var events coredata.SCIMEvents
|
||||||
if err := events.LoadByOrganizationID(
|
if err := events.LoadByOrganizationID(
|
||||||
ctx,
|
ctx,
|
||||||
@@ -169,8 +168,7 @@ func (s *LogExportService) streamSCIMEventCSV(
|
|||||||
|
|
||||||
return events, nil
|
return events, nil
|
||||||
},
|
},
|
||||||
func(pageEvents []*coredata.SCIMEvent) error {
|
func(events coredata.SCIMEvents) error {
|
||||||
events := coredata.SCIMEvents(pageEvents)
|
|
||||||
profilesByUserName, err := loadSCIMProfileExportInfo(
|
profilesByUserName, err := loadSCIMProfileExportInfo(
|
||||||
ctx,
|
ctx,
|
||||||
conn,
|
conn,
|
||||||
|
|||||||
@@ -38,14 +38,18 @@ const MaxLoadAllPages = 20
|
|||||||
// LoadBy* on a fresh receiver).
|
// LoadBy* on a fresh receiver).
|
||||||
type Loader[T Paginable[U], U OrderField] func(ctx context.Context, cursor *Cursor[U]) ([]T, error)
|
type Loader[T Paginable[U], U OrderField] func(ctx context.Context, cursor *Cursor[U]) ([]T, error)
|
||||||
|
|
||||||
|
// WalkLoader is like Loader but allows coredata collection types (e.g.
|
||||||
|
// AuditLogEntries) whose underlying type is []T.
|
||||||
|
type WalkLoader[T Paginable[U], U OrderField, S ~[]T] func(ctx context.Context, cursor *Cursor[U]) (S, error)
|
||||||
|
|
||||||
// WalkAll walks every matching row via keyset pagination, advancing a
|
// WalkAll walks every matching row via keyset pagination, advancing a
|
||||||
// MaxCursorSize forward cursor until no rows remain, and invokes walk with
|
// MaxCursorSize forward cursor until no rows remain, and invokes walk with
|
||||||
// every page of rows. Unlike LoadAll, it does not apply MaxLoadAllPages.
|
// every page of rows. Unlike LoadAll, it does not apply MaxLoadAllPages.
|
||||||
func WalkAll[T Paginable[U], U OrderField](
|
func WalkAll[T Paginable[U], U OrderField, S ~[]T](
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
orderBy OrderBy[U],
|
orderBy OrderBy[U],
|
||||||
fetch Loader[T, U],
|
fetch WalkLoader[T, U, S],
|
||||||
walk func(rows []T) error,
|
walk func(rows S) error,
|
||||||
) error {
|
) error {
|
||||||
var key *CursorKey
|
var key *CursorKey
|
||||||
|
|
||||||
@@ -57,8 +61,8 @@ func WalkAll[T Paginable[U], U OrderField](
|
|||||||
return fmt.Errorf("cannot load all rows: %w", err)
|
return fmt.Errorf("cannot load all rows: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
p := NewPage(rows, cursor)
|
p := NewPage([]T(rows), cursor)
|
||||||
if err := walk(p.Data); err != nil {
|
if err := walk(S(p.Data)); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -212,3 +212,29 @@ func TestWalkAllHasNoPageCap(t *testing.T) {
|
|||||||
assert.Equal(t, loadAllValues(store), loadAllValues(got))
|
assert.Equal(t, loadAllValues(store), loadAllValues(got))
|
||||||
assert.Greater(t, fetchs, MaxLoadAllPages)
|
assert.Greater(t, fetchs, MaxLoadAllPages)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type loadAllItems []*loadAllItem
|
||||||
|
|
||||||
|
func TestWalkAllNamedSliceType(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
store := newLoadAllStore(MaxCursorSize + 1)
|
||||||
|
|
||||||
|
var got loadAllItems
|
||||||
|
|
||||||
|
err := WalkAll(
|
||||||
|
context.Background(),
|
||||||
|
ascOrderBy(),
|
||||||
|
func(_ context.Context, cursor *Cursor[testOrderField]) (loadAllItems, error) {
|
||||||
|
return loadAllItems(keysetPage(store, cursor)), nil
|
||||||
|
},
|
||||||
|
func(rows loadAllItems) error {
|
||||||
|
got = append(got, rows...)
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Len(t, got, len(store))
|
||||||
|
assert.Equal(t, loadAllValues(store), loadAllValues(got))
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user