diff --git a/pkg/proboctl/cmdutil/paginate.go b/pkg/proboctl/cmdutil/paginate.go index c7354fc69..27a91aabd 100644 --- a/pkg/proboctl/cmdutil/paginate.go +++ b/pkg/proboctl/cmdutil/paginate.go @@ -16,63 +16,172 @@ package cmdutil import ( "context" + "fmt" + "io" + "github.com/spf13/cobra" "go.probo.inc/probo/pkg/page" ) -// paginatePageSize is the per-request fetch size used to walk a cursor -// connection. It is independent of the caller's --limit, which only -// bounds the total returned. -const paginatePageSize = 100 +// defaultPageSize is the forward page size used when neither --first nor +// --last is provided. +const defaultPageSize = 50 -// Paginate walks a cursor-paginated coredata Load into a single slice, -// following the keyset cursor until the caller's limit is reached or the -// connection is exhausted. A limit <= 0 returns every matching row. This -// is the proboctl-side counterpart of the API's connection resolvers: it -// drives page.NewCursor + page.NewPage against the same coredata Load -// methods, so a future proboctl API can reuse the data layer untouched. -func Paginate[E page.Paginable[F], F page.OrderField]( - ctx context.Context, - orderBy page.OrderBy[F], - limit int, - load func(ctx context.Context, cursor *page.Cursor[F]) ([]E, error), -) ([]E, error) { - var ( - result []E - from *page.CursorKey - ) - - for { - size := paginatePageSize - - if limit > 0 { - remaining := limit - len(result) - if remaining <= 0 { - break - } - - if remaining < size { - size = remaining - } - } - - cursor := page.NewCursor(size, from, page.Head, orderBy) - - rows, err := load(ctx, cursor) - if err != nil { - return nil, err - } - - p := page.NewPage(rows, cursor) - result = append(result, p.Data...) - - if !p.Info.HasNext || len(p.Data) == 0 { - break - } - - key := p.Data[len(p.Data)-1].CursorKey(orderBy.Field) - from = &key +type ( + // PageFlags holds the cursor-pagination flag values registered by + // AddPageFlags. They mirror the GraphQL connection arguments: --first + // with --after walks forward, --last with --before walks backward. + PageFlags struct { + First int + Last int + After string + Before string } - return result, nil + // PageInfo is the proboctl-side counterpart of the API's PageInfo. The + // cursors are the base64 page.CursorKey scalars, so they round-trip + // straight back into --after / --before. + PageInfo struct { + HasNextPage bool `json:"hasNextPage"` + HasPreviousPage bool `json:"hasPreviousPage"` + StartCursor *string `json:"startCursor"` + EndCursor *string `json:"endCursor"` + } + + // PageOutput wraps a flat slice of results with its page info for JSON + // output. + PageOutput struct { + Items any `json:"items"` + PageInfo PageInfo `json:"pageInfo"` + } +) + +// AddPageFlags registers the cursor-pagination flags on cmd and returns a +// pointer to the bound values. +func AddPageFlags(cmd *cobra.Command) *PageFlags { + pf := &PageFlags{} + + cmd.Flags().IntVar(&pf.First, "first", 0, fmt.Sprintf("Return the first N rows after --after (default %d)", defaultPageSize)) + cmd.Flags().IntVar(&pf.Last, "last", 0, fmt.Sprintf("Return the last N rows before --before (default %d)", defaultPageSize)) + cmd.Flags().StringVar(&pf.After, "after", "", "Cursor to page forward from (use with --first)") + cmd.Flags().StringVar(&pf.Before, "before", "", "Cursor to page backward from (use with --last)") + + return pf +} + +// NewCursorFromFlags builds a keyset cursor from the pagination flags, +// mirroring the GraphQL API's first/after vs last/before semantics. +func NewCursorFromFlags[F page.OrderField]( + pf *PageFlags, + orderBy page.OrderBy[F], +) (*page.Cursor[F], error) { + if pf.First > 0 && pf.Last > 0 { + return nil, fmt.Errorf("--first and --last are mutually exclusive") + } + + if pf.After != "" && pf.Before != "" { + return nil, fmt.Errorf("--after and --before are mutually exclusive") + } + + if pf.After != "" && pf.Last > 0 { + return nil, fmt.Errorf("--after cannot be combined with --last") + } + + var ( + size int + from *page.CursorKey + position page.Position + ) + + switch { + case pf.Last > 0 || pf.Before != "": + size = pf.Last + if size == 0 { + size = defaultPageSize + } + + position = page.Tail + + if pf.Before != "" { + ck, err := page.ParseCursorKey(pf.Before) + if err != nil { + return nil, fmt.Errorf("invalid --before cursor: %w", err) + } + + from = &ck + } + default: + size = pf.First + if size == 0 { + size = defaultPageSize + } + + position = page.Head + + if pf.After != "" { + ck, err := page.ParseCursorKey(pf.After) + if err != nil { + return nil, fmt.Errorf("invalid --after cursor: %w", err) + } + + from = &ck + } + } + + return page.NewCursor(size, from, position, orderBy), nil +} + +// FetchPage loads a single keyset page using cursor and wraps the rows into a +// page.Page, which trims the over-fetch and computes HasNext / HasPrev. It is +// the proboctl-side counterpart of the API's connection resolvers. +func FetchPage[E page.Paginable[F], F page.OrderField]( + ctx context.Context, + cursor *page.Cursor[F], + load func(ctx context.Context, cursor *page.Cursor[F]) ([]E, error), +) (*page.Page[E, F], error) { + rows, err := load(ctx, cursor) + if err != nil { + return nil, err + } + + return page.NewPage(rows, cursor), nil +} + +// NewPageInfo derives the proboctl PageInfo (cursors as base64 strings) from a +// loaded page. +func NewPageInfo[T page.Paginable[F], F page.OrderField](p *page.Page[T, F]) PageInfo { + pi := PageInfo{ + HasNextPage: p.Info.HasNext, + HasPreviousPage: p.Info.HasPrev, + } + + if len(p.Data) > 0 { + start := p.First().CursorKey(p.Cursor.OrderBy.Field).String() + end := p.Last().CursorKey(p.Cursor.OrderBy.Field).String() + pi.StartCursor = &start + pi.EndCursor = &end + } + + return pi +} + +// PrintPageInfo writes the page info as a footer below a rendered table. +func PrintPageInfo(out io.Writer, pi PageInfo) { + var start, end string + if pi.StartCursor != nil { + start = *pi.StartCursor + } + + if pi.EndCursor != nil { + end = *pi.EndCursor + } + + _, _ = fmt.Fprintf( + out, + "hasNextPage: %t hasPreviousPage: %t\nstartCursor: %s\nendCursor: %s\n", + pi.HasNextPage, + pi.HasPreviousPage, + start, + end, + ) } diff --git a/pkg/proboctl/commonthirdparty/list.go b/pkg/proboctl/commonthirdparty/list.go index c240c18da..36804ba3c 100644 --- a/pkg/proboctl/commonthirdparty/list.go +++ b/pkg/proboctl/commonthirdparty/list.go @@ -33,7 +33,6 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command { flagKeyword string flagSort string flagOrder string - flagLimit int ) cmd := &cobra.Command{ @@ -49,7 +48,8 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command { cmd.Flags().StringVar(&flagKeyword, "keyword", "", "Filter by name/slug substring") cmd.Flags().StringVar(&flagSort, "sort", "name", "Sort field: name, created, updated") cmd.Flags().StringVar(&flagOrder, "order", "", "Sort order: asc, desc (default depends on field)") - cmd.Flags().IntVarP(&flagLimit, "limit", "L", 50, "Maximum rows to return (0 for all)") + + pageFlags := cmdutil.AddPageFlags(cmd) cmd.RunE = func(cmd *cobra.Command, args []string) error { if err := clicmdutil.ValidateOutputFlag(output); err != nil { @@ -61,6 +61,11 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command { return err } + cursor, err := cmdutil.NewCursorFromFlags(pageFlags, orderBy) + if err != nil { + return err + } + filter := coredata.NewCommonThirdPartyFilter(optionalString(flagName)) if flagCategory != "" { @@ -81,15 +86,17 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command { return err } - var parties coredata.CommonThirdParties + var ( + parties coredata.CommonThirdParties + pageInfo cmdutil.PageInfo + ) if err := pgClient.WithConn( cmd.Context(), func(ctx context.Context, conn pg.Querier) error { - rows, err := cmdutil.Paginate( + p, err := cmdutil.FetchPage( ctx, - orderBy, - flagLimit, + cursor, func(ctx context.Context, cursor *page.Cursor[coredata.CommonThirdPartyOrderField]) ([]*coredata.CommonThirdParty, error) { var ts coredata.CommonThirdParties if err := ts.Load(ctx, conn, cursor, filter); err != nil { @@ -103,7 +110,8 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command { return err } - parties = rows + parties = p.Data + pageInfo = cmdutil.NewPageInfo(p) return nil }, @@ -112,7 +120,7 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command { } if *output == clicmdutil.OutputJSON { - return clicmdutil.PrintJSON(f.IOStreams.Out, parties) + return clicmdutil.PrintJSON(f.IOStreams.Out, cmdutil.PageOutput{Items: parties, PageInfo: pageInfo}) } if len(parties) == 0 { @@ -133,6 +141,7 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command { } _, _ = fmt.Fprintln(f.IOStreams.Out, table.Render()) + cmdutil.PrintPageInfo(f.IOStreams.Out, pageInfo) _, _ = fmt.Fprintf(f.IOStreams.ErrOut, "Showing %d common third parties.\n", len(parties)) return nil diff --git a/pkg/proboctl/commontrackerpattern/list.go b/pkg/proboctl/commontrackerpattern/list.go index 4a95dbad1..e5741e174 100644 --- a/pkg/proboctl/commontrackerpattern/list.go +++ b/pkg/proboctl/commontrackerpattern/list.go @@ -40,7 +40,6 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command { flagWithoutDescription bool flagSort string flagOrder string - flagLimit int ) cmd := &cobra.Command{ @@ -62,7 +61,8 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command { cmd.Flags().BoolVar(&flagWithoutDescription, "without-description", false, "Only patterns with a blank description") cmd.Flags().StringVar(&flagSort, "sort", "confidence", "Sort field: pattern, confidence, created, updated, enriched") cmd.Flags().StringVar(&flagOrder, "order", "", "Sort order: asc, desc (default depends on field)") - cmd.Flags().IntVarP(&flagLimit, "limit", "L", 50, "Maximum rows to return (0 for all)") + + pageFlags := cmdutil.AddPageFlags(cmd) cmd.RunE = func(cmd *cobra.Command, args []string) error { if err := clicmdutil.ValidateOutputFlag(output); err != nil { @@ -78,6 +78,11 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command { return err } + cursor, err := cmdutil.NewCursorFromFlags(pageFlags, orderBy) + if err != nil { + return err + } + var withCommonThirdParty *bool if cmd.Flags().Changed("with-common-third-party") { withCommonThirdParty = &flagWithCommonThirdParty @@ -100,7 +105,10 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command { ctx := cmd.Context() - var patterns coredata.CommonTrackerPatterns + var ( + patterns coredata.CommonTrackerPatterns + pageInfo cmdutil.PageInfo + ) if err := pgClient.WithConn( ctx, @@ -153,10 +161,9 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command { filter.WithIDs(linkedIDs) } - rows, err := cmdutil.Paginate( + p, err := cmdutil.FetchPage( ctx, - orderBy, - flagLimit, + cursor, func(ctx context.Context, cursor *page.Cursor[coredata.CommonTrackerPatternOrderField]) ([]*coredata.CommonTrackerPattern, error) { var ps coredata.CommonTrackerPatterns if err := ps.Load(ctx, conn, cursor, filter); err != nil { @@ -170,7 +177,8 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command { return err } - patterns = rows + patterns = p.Data + pageInfo = cmdutil.NewPageInfo(p) return nil }, @@ -179,16 +187,16 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command { } if *output == clicmdutil.OutputJSON { - return clicmdutil.PrintJSON(f.IOStreams.Out, patterns) + return clicmdutil.PrintJSON(f.IOStreams.Out, cmdutil.PageOutput{Items: patterns, PageInfo: pageInfo}) } - return renderPatternTable(cmd, f, patterns) + return renderPatternTable(cmd, f, patterns, pageInfo) } return cmd } -func renderPatternTable(cmd *cobra.Command, f *cmdutil.Factory, patterns coredata.CommonTrackerPatterns) error { +func renderPatternTable(cmd *cobra.Command, f *cmdutil.Factory, patterns coredata.CommonTrackerPatterns, pageInfo cmdutil.PageInfo) error { out := f.IOStreams.Out if len(patterns) == 0 { @@ -243,6 +251,7 @@ func renderPatternTable(cmd *cobra.Command, f *cmdutil.Factory, patterns coredat } _, _ = fmt.Fprintln(out, table.Render()) + cmdutil.PrintPageInfo(out, pageInfo) _, _ = fmt.Fprintf(f.IOStreams.ErrOut, "Showing %d common tracker patterns.\n", len(patterns)) return nil diff --git a/pkg/proboctl/commontrackerpattern/stats.go b/pkg/proboctl/commontrackerpattern/stats.go index 3f9336261..7054f3829 100644 --- a/pkg/proboctl/commontrackerpattern/stats.go +++ b/pkg/proboctl/commontrackerpattern/stats.go @@ -55,7 +55,8 @@ func newCmdStats(f *cmdutil.Factory) *cobra.Command { }{ {"total", coredata.NewCommonTrackerPatternFilter()}, {"queued", coredata.NewCommonTrackerPatternFilter().WithState(new(coredata.CommonTrackerPatternEnrichmentStateQueued))}, - {"enriched", coredata.NewCommonTrackerPatternFilter().WithState(new(coredata.CommonTrackerPatternEnrichmentStateEnriched))}, + {"enriched", coredata.NewCommonTrackerPatternFilter().WithState(new(coredata.CommonTrackerPatternEnrichmentStateEnriched)).WithDescribed(new(true))}, + {"enriched (no description)", coredata.NewCommonTrackerPatternFilter().WithState(new(coredata.CommonTrackerPatternEnrichmentStateEnriched)).WithDescribed(new(false))}, {"unenriched", coredata.NewCommonTrackerPatternFilter().WithState(new(coredata.CommonTrackerPatternEnrichmentStateUnenriched))}, {"linked", coredata.NewCommonTrackerPatternFilter().WithLinked(new(true))}, {"unlinked", coredata.NewCommonTrackerPatternFilter().WithLinked(new(false))}, @@ -83,7 +84,7 @@ func newCmdStats(f *cmdutil.Factory) *cobra.Command { } table := clicmdutil.NewTable("METRIC", "COUNT") - for _, key := range []string{"total", "queued", "enriched", "unenriched", "linked", "unlinked"} { + for _, key := range []string{"total", "queued", "enriched", "enriched (no description)", "unenriched", "linked", "unlinked"} { table.Row(key, fmt.Sprintf("%d", stats[key])) }