Add cursor pagination flags to proboctl list commands

Replace the limit-driven auto-walking Paginate helper with explicit
cursor-pagination flags (--first/--after, --last/--before) that mirror
the GraphQL connection arguments. List commands now return a single
keyset page with its page info, and emit cursors so callers can page
forward and backward. --before no longer requires --last: both --first
and --last default to 50 when omitted.

Also split the tracker-pattern stats into enriched with and without a
description so the enrichment backlog is visible at a glance.

Signed-off-by: Émile Ré <emile@probo.com>

pkg/proboctl/cmdutil/paginate.go

@@ -16,63 +16,172 @@ package cmdutil
 
 import (
 	"context"
+	"fmt"
+	"io"
 
+	"github.com/spf13/cobra"
 	"go.probo.inc/probo/pkg/page"
 )
 
-// paginatePageSize is the per-request fetch size used to walk a cursor
+// defaultPageSize is the forward page size used when neither --first nor
-// connection. It is independent of the caller's --limit, which only
+// --last is provided.
-// bounds the total returned.
+const defaultPageSize = 50
-const paginatePageSize = 100
 
-// Paginate walks a cursor-paginated coredata Load into a single slice,
+type (
-// following the keyset cursor until the caller's limit is reached or the
+	// PageFlags holds the cursor-pagination flag values registered by
-// connection is exhausted. A limit <= 0 returns every matching row. This
+	// AddPageFlags. They mirror the GraphQL connection arguments: --first
-// is the proboctl-side counterpart of the API's connection resolvers: it
+	// with --after walks forward, --last with --before walks backward.
-// drives page.NewCursor + page.NewPage against the same coredata Load
+	PageFlags struct {
-// methods, so a future proboctl API can reuse the data layer untouched.
+		First  int
-func Paginate[E page.Paginable[F], F page.OrderField](
+		Last   int
-	ctx context.Context,
+		After  string
-	orderBy page.OrderBy[F],
+		Before string
-	limit int,
-	load func(ctx context.Context, cursor *page.Cursor[F]) ([]E, error),
-) ([]E, error) {
-	var (
-		result []E
-		from   *page.CursorKey
-	)
-
-	for {
-		size := paginatePageSize
-
-		if limit > 0 {
-			remaining := limit - len(result)
-			if remaining <= 0 {
-				break
-			}
-
-			if remaining < size {
-				size = remaining
-			}
-		}
-
-		cursor := page.NewCursor(size, from, page.Head, orderBy)
-
-		rows, err := load(ctx, cursor)
-		if err != nil {
-			return nil, err
-		}
-
-		p := page.NewPage(rows, cursor)
-		result = append(result, p.Data...)
-
-		if !p.Info.HasNext || len(p.Data) == 0 {
-			break
-		}
-
-		key := p.Data[len(p.Data)-1].CursorKey(orderBy.Field)
-		from = &key
 	}
 
-	return result, nil
+	// PageInfo is the proboctl-side counterpart of the API's PageInfo. The
+	// cursors are the base64 page.CursorKey scalars, so they round-trip
+	// straight back into --after / --before.
+	PageInfo struct {
+		HasNextPage     bool    `json:"hasNextPage"`
+		HasPreviousPage bool    `json:"hasPreviousPage"`
+		StartCursor     *string `json:"startCursor"`
+		EndCursor       *string `json:"endCursor"`
+	}
+
+	// PageOutput wraps a flat slice of results with its page info for JSON
+	// output.
+	PageOutput struct {
+		Items    any      `json:"items"`
+		PageInfo PageInfo `json:"pageInfo"`
+	}
+)
+
+// AddPageFlags registers the cursor-pagination flags on cmd and returns a
+// pointer to the bound values.
+func AddPageFlags(cmd *cobra.Command) *PageFlags {
+	pf := &PageFlags{}
+
+	cmd.Flags().IntVar(&pf.First, "first", 0, fmt.Sprintf("Return the first N rows after --after (default %d)", defaultPageSize))
+	cmd.Flags().IntVar(&pf.Last, "last", 0, fmt.Sprintf("Return the last N rows before --before (default %d)", defaultPageSize))
+	cmd.Flags().StringVar(&pf.After, "after", "", "Cursor to page forward from (use with --first)")
+	cmd.Flags().StringVar(&pf.Before, "before", "", "Cursor to page backward from (use with --last)")
+
+	return pf
+}
+
+// NewCursorFromFlags builds a keyset cursor from the pagination flags,
+// mirroring the GraphQL API's first/after vs last/before semantics.
+func NewCursorFromFlags[F page.OrderField](
+	pf *PageFlags,
+	orderBy page.OrderBy[F],
+) (*page.Cursor[F], error) {
+	if pf.First > 0 && pf.Last > 0 {
+		return nil, fmt.Errorf("--first and --last are mutually exclusive")
+	}
+
+	if pf.After != "" && pf.Before != "" {
+		return nil, fmt.Errorf("--after and --before are mutually exclusive")
+	}
+
+	if pf.After != "" && pf.Last > 0 {
+		return nil, fmt.Errorf("--after cannot be combined with --last")
+	}
+
+	var (
+		size     int
+		from     *page.CursorKey
+		position page.Position
+	)
+
+	switch {
+	case pf.Last > 0 || pf.Before != "":
+		size = pf.Last
+		if size == 0 {
+			size = defaultPageSize
+		}
+
+		position = page.Tail
+
+		if pf.Before != "" {
+			ck, err := page.ParseCursorKey(pf.Before)
+			if err != nil {
+				return nil, fmt.Errorf("invalid --before cursor: %w", err)
+			}
+
+			from = &ck
+		}
+	default:
+		size = pf.First
+		if size == 0 {
+			size = defaultPageSize
+		}
+
+		position = page.Head
+
+		if pf.After != "" {
+			ck, err := page.ParseCursorKey(pf.After)
+			if err != nil {
+				return nil, fmt.Errorf("invalid --after cursor: %w", err)
+			}
+
+			from = &ck
+		}
+	}
+
+	return page.NewCursor(size, from, position, orderBy), nil
+}
+
+// FetchPage loads a single keyset page using cursor and wraps the rows into a
+// page.Page, which trims the over-fetch and computes HasNext / HasPrev. It is
+// the proboctl-side counterpart of the API's connection resolvers.
+func FetchPage[E page.Paginable[F], F page.OrderField](
+	ctx context.Context,
+	cursor *page.Cursor[F],
+	load func(ctx context.Context, cursor *page.Cursor[F]) ([]E, error),
+) (*page.Page[E, F], error) {
+	rows, err := load(ctx, cursor)
+	if err != nil {
+		return nil, err
+	}
+
+	return page.NewPage(rows, cursor), nil
+}
+
+// NewPageInfo derives the proboctl PageInfo (cursors as base64 strings) from a
+// loaded page.
+func NewPageInfo[T page.Paginable[F], F page.OrderField](p *page.Page[T, F]) PageInfo {
+	pi := PageInfo{
+		HasNextPage:     p.Info.HasNext,
+		HasPreviousPage: p.Info.HasPrev,
+	}
+
+	if len(p.Data) > 0 {
+		start := p.First().CursorKey(p.Cursor.OrderBy.Field).String()
+		end := p.Last().CursorKey(p.Cursor.OrderBy.Field).String()
+		pi.StartCursor = &start
+		pi.EndCursor = &end
+	}
+
+	return pi
+}
+
+// PrintPageInfo writes the page info as a footer below a rendered table.
+func PrintPageInfo(out io.Writer, pi PageInfo) {
+	var start, end string
+	if pi.StartCursor != nil {
+		start = *pi.StartCursor
+	}
+
+	if pi.EndCursor != nil {
+		end = *pi.EndCursor
+	}
+
+	_, _ = fmt.Fprintf(
+		out,
+		"hasNextPage: %t  hasPreviousPage: %t\nstartCursor: %s\nendCursor: %s\n",
+		pi.HasNextPage,
+		pi.HasPreviousPage,
+		start,
+		end,
+	)
 }

pkg/proboctl/commonthirdparty/list.go

@@ -33,7 +33,6 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
 		flagKeyword  string
 		flagSort     string
 		flagOrder    string
-		flagLimit    int
 	)
 
 	cmd := &cobra.Command{
@@ -49,7 +48,8 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
 	cmd.Flags().StringVar(&flagKeyword, "keyword", "", "Filter by name/slug substring")
 	cmd.Flags().StringVar(&flagSort, "sort", "name", "Sort field: name, created, updated")
 	cmd.Flags().StringVar(&flagOrder, "order", "", "Sort order: asc, desc (default depends on field)")
-	cmd.Flags().IntVarP(&flagLimit, "limit", "L", 50, "Maximum rows to return (0 for all)")
+
+	pageFlags := cmdutil.AddPageFlags(cmd)
 
 	cmd.RunE = func(cmd *cobra.Command, args []string) error {
 		if err := clicmdutil.ValidateOutputFlag(output); err != nil {
@@ -61,6 +61,11 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
 			return err
 		}
 
+		cursor, err := cmdutil.NewCursorFromFlags(pageFlags, orderBy)
+		if err != nil {
+			return err
+		}
+
 		filter := coredata.NewCommonThirdPartyFilter(optionalString(flagName))
 
 		if flagCategory != "" {
@@ -81,15 +86,17 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
 			return err
 		}
 
-		var parties coredata.CommonThirdParties
+		var (
+			parties  coredata.CommonThirdParties
+			pageInfo cmdutil.PageInfo
+		)
 
 		if err := pgClient.WithConn(
 			cmd.Context(),
 			func(ctx context.Context, conn pg.Querier) error {
-				rows, err := cmdutil.Paginate(
+				p, err := cmdutil.FetchPage(
 					ctx,
-					orderBy,
+					cursor,
-					flagLimit,
 					func(ctx context.Context, cursor *page.Cursor[coredata.CommonThirdPartyOrderField]) ([]*coredata.CommonThirdParty, error) {
 						var ts coredata.CommonThirdParties
 						if err := ts.Load(ctx, conn, cursor, filter); err != nil {
@@ -103,7 +110,8 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
 					return err
 				}
 
-				parties = rows
+				parties = p.Data
+				pageInfo = cmdutil.NewPageInfo(p)
 
 				return nil
 			},
@@ -112,7 +120,7 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
 		}
 
 		if *output == clicmdutil.OutputJSON {
-			return clicmdutil.PrintJSON(f.IOStreams.Out, parties)
+			return clicmdutil.PrintJSON(f.IOStreams.Out, cmdutil.PageOutput{Items: parties, PageInfo: pageInfo})
 		}
 
 		if len(parties) == 0 {
@@ -133,6 +141,7 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
 		}
 
 		_, _ = fmt.Fprintln(f.IOStreams.Out, table.Render())
+		cmdutil.PrintPageInfo(f.IOStreams.Out, pageInfo)
 		_, _ = fmt.Fprintf(f.IOStreams.ErrOut, "Showing %d common third parties.\n", len(parties))
 
 		return nil

pkg/proboctl/commontrackerpattern/list.go

@@ -40,7 +40,6 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
 		flagWithoutDescription   bool
 		flagSort                 string
 		flagOrder                string
-		flagLimit                int
 	)
 
 	cmd := &cobra.Command{
@@ -62,7 +61,8 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
 	cmd.Flags().BoolVar(&flagWithoutDescription, "without-description", false, "Only patterns with a blank description")
 	cmd.Flags().StringVar(&flagSort, "sort", "confidence", "Sort field: pattern, confidence, created, updated, enriched")
 	cmd.Flags().StringVar(&flagOrder, "order", "", "Sort order: asc, desc (default depends on field)")
-	cmd.Flags().IntVarP(&flagLimit, "limit", "L", 50, "Maximum rows to return (0 for all)")
+
+	pageFlags := cmdutil.AddPageFlags(cmd)
 
 	cmd.RunE = func(cmd *cobra.Command, args []string) error {
 		if err := clicmdutil.ValidateOutputFlag(output); err != nil {
@@ -78,6 +78,11 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
 			return err
 		}
 
+		cursor, err := cmdutil.NewCursorFromFlags(pageFlags, orderBy)
+		if err != nil {
+			return err
+		}
+
 		var withCommonThirdParty *bool
 		if cmd.Flags().Changed("with-common-third-party") {
 			withCommonThirdParty = &flagWithCommonThirdParty
@@ -100,7 +105,10 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
 
 		ctx := cmd.Context()
 
-		var patterns coredata.CommonTrackerPatterns
+		var (
+			patterns coredata.CommonTrackerPatterns
+			pageInfo cmdutil.PageInfo
+		)
 
 		if err := pgClient.WithConn(
 			ctx,
@@ -153,10 +161,9 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
 					filter.WithIDs(linkedIDs)
 				}
 
-				rows, err := cmdutil.Paginate(
+				p, err := cmdutil.FetchPage(
 					ctx,
-					orderBy,
+					cursor,
-					flagLimit,
 					func(ctx context.Context, cursor *page.Cursor[coredata.CommonTrackerPatternOrderField]) ([]*coredata.CommonTrackerPattern, error) {
 						var ps coredata.CommonTrackerPatterns
 						if err := ps.Load(ctx, conn, cursor, filter); err != nil {
@@ -170,7 +177,8 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
 					return err
 				}
 
-				patterns = rows
+				patterns = p.Data
+				pageInfo = cmdutil.NewPageInfo(p)
 
 				return nil
 			},
@@ -179,16 +187,16 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
 		}
 
 		if *output == clicmdutil.OutputJSON {
-			return clicmdutil.PrintJSON(f.IOStreams.Out, patterns)
+			return clicmdutil.PrintJSON(f.IOStreams.Out, cmdutil.PageOutput{Items: patterns, PageInfo: pageInfo})
 		}
 
-		return renderPatternTable(cmd, f, patterns)
+		return renderPatternTable(cmd, f, patterns, pageInfo)
 	}
 
 	return cmd
 }
 
-func renderPatternTable(cmd *cobra.Command, f *cmdutil.Factory, patterns coredata.CommonTrackerPatterns) error {
+func renderPatternTable(cmd *cobra.Command, f *cmdutil.Factory, patterns coredata.CommonTrackerPatterns, pageInfo cmdutil.PageInfo) error {
 	out := f.IOStreams.Out
 
 	if len(patterns) == 0 {
@@ -243,6 +251,7 @@ func renderPatternTable(cmd *cobra.Command, f *cmdutil.Factory, patterns coredat
 	}
 
 	_, _ = fmt.Fprintln(out, table.Render())
+	cmdutil.PrintPageInfo(out, pageInfo)
 	_, _ = fmt.Fprintf(f.IOStreams.ErrOut, "Showing %d common tracker patterns.\n", len(patterns))
 
 	return nil

pkg/proboctl/commontrackerpattern/stats.go

@@ -55,7 +55,8 @@ func newCmdStats(f *cmdutil.Factory) *cobra.Command {
 				}{
 					{"total", coredata.NewCommonTrackerPatternFilter()},
 					{"queued", coredata.NewCommonTrackerPatternFilter().WithState(new(coredata.CommonTrackerPatternEnrichmentStateQueued))},
-					{"enriched", coredata.NewCommonTrackerPatternFilter().WithState(new(coredata.CommonTrackerPatternEnrichmentStateEnriched))},
+					{"enriched", coredata.NewCommonTrackerPatternFilter().WithState(new(coredata.CommonTrackerPatternEnrichmentStateEnriched)).WithDescribed(new(true))},
+					{"enriched (no description)", coredata.NewCommonTrackerPatternFilter().WithState(new(coredata.CommonTrackerPatternEnrichmentStateEnriched)).WithDescribed(new(false))},
 					{"unenriched", coredata.NewCommonTrackerPatternFilter().WithState(new(coredata.CommonTrackerPatternEnrichmentStateUnenriched))},
 					{"linked", coredata.NewCommonTrackerPatternFilter().WithLinked(new(true))},
 					{"unlinked", coredata.NewCommonTrackerPatternFilter().WithLinked(new(false))},
@@ -83,7 +84,7 @@ func newCmdStats(f *cmdutil.Factory) *cobra.Command {
 		}
 
 		table := clicmdutil.NewTable("METRIC", "COUNT")
-		for _, key := range []string{"total", "queued", "enriched", "unenriched", "linked", "unlinked"} {
+		for _, key := range []string{"total", "queued", "enriched", "enriched (no description)", "unenriched", "linked", "unlinked"} {
 			table.Row(key, fmt.Sprintf("%d", stats[key]))
 		}