rohit/probo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add cursor pagination flags to proboctl list commands

Browse Source 
Replace the limit-driven auto-walking Paginate helper with explicit
cursor-pagination flags (--first/--after, --last/--before) that mirror
the GraphQL connection arguments. List commands now return a single
keyset page with its page info, and emit cursors so callers can page
forward and backward. --before no longer requires --last: both --first
and --last default to 50 when omitted.

Also split the tracker-pattern stats into enriched with and without a
description so the enrichment backlog is visible at a glance.

Signed-off-by: Émile Ré <emile@probo.com>
This commit is contained in:
Émile Ré
2026-06-09 13:10:12 +02:00
parent 4f6fcb42f9
commit ab3750fca4
4 changed files with 200 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

213
pkg/proboctl/cmdutil/paginate.go
View File

@@ -16,63 +16,172 @@ package cmdutil
import (
"context"
"fmt"
"io"
"github.com/spf13/cobra"
"go.probo.inc/probo/pkg/page"
)
// paginatePageSize is the per-request fetch size used to walk a cursor
// connection. It is independent of the caller's --limit, which only
// bounds the total returned.
const paginatePageSize = 100
// defaultPageSize is the forward page size used when neither --first nor
// --last is provided.
const defaultPageSize = 50
// Paginate walks a cursor-paginated coredata Load into a single slice,
// following the keyset cursor until the caller's limit is reached or the
// connection is exhausted. A limit <= 0 returns every matching row. This
// is the proboctl-side counterpart of the API's connection resolvers: it
// drives page.NewCursor + page.NewPage against the same coredata Load
// methods, so a future proboctl API can reuse the data layer untouched.
func Paginate[E page.Paginable[F], F page.OrderField](
ctx context.Context,
orderBy page.OrderBy[F],
limit int,
load func(ctx context.Context, cursor *page.Cursor[F]) ([]E, error),
) ([]E, error) {
var (
result []E
from *page.CursorKey
)
for {
size := paginatePageSize
if limit > 0 {
remaining := limit - len(result)
if remaining <= 0 {
break
}
if remaining < size {
size = remaining
}
}
cursor := page.NewCursor(size, from, page.Head, orderBy)
rows, err := load(ctx, cursor)
if err != nil {
return nil, err
}
p := page.NewPage(rows, cursor)
result = append(result, p.Data...)
if !p.Info.HasNext || len(p.Data) == 0 {
break
}
key := p.Data[len(p.Data)-1].CursorKey(orderBy.Field)
from = &key
type (
// PageFlags holds the cursor-pagination flag values registered by
// AddPageFlags. They mirror the GraphQL connection arguments: --first
// with --after walks forward, --last with --before walks backward.
PageFlags struct {
First int
Last int
After string
Before string
}
return result, nil
// PageInfo is the proboctl-side counterpart of the API's PageInfo. The
// cursors are the base64 page.CursorKey scalars, so they round-trip
// straight back into --after / --before.
PageInfo struct {
HasNextPage bool `json:"hasNextPage"`
HasPreviousPage bool `json:"hasPreviousPage"`
StartCursor *string `json:"startCursor"`
EndCursor *string `json:"endCursor"`
}
// PageOutput wraps a flat slice of results with its page info for JSON
// output.
PageOutput struct {
Items any `json:"items"`
PageInfo PageInfo `json:"pageInfo"`
}
)
// AddPageFlags registers the cursor-pagination flags on cmd and returns a
// pointer to the bound values.
func AddPageFlags(cmd *cobra.Command) *PageFlags {
pf := &PageFlags{}
cmd.Flags().IntVar(&pf.First, "first", 0, fmt.Sprintf("Return the first N rows after --after (default %d)", defaultPageSize))
cmd.Flags().IntVar(&pf.Last, "last", 0, fmt.Sprintf("Return the last N rows before --before (default %d)", defaultPageSize))
cmd.Flags().StringVar(&pf.After, "after", "", "Cursor to page forward from (use with --first)")
cmd.Flags().StringVar(&pf.Before, "before", "", "Cursor to page backward from (use with --last)")
return pf
}
// NewCursorFromFlags builds a keyset cursor from the pagination flags,
// mirroring the GraphQL API's first/after vs last/before semantics.
func NewCursorFromFlags[F page.OrderField](
pf *PageFlags,
orderBy page.OrderBy[F],
) (*page.Cursor[F], error) {
if pf.First > 0 && pf.Last > 0 {
return nil, fmt.Errorf("--first and --last are mutually exclusive")
}
if pf.After != "" && pf.Before != "" {
return nil, fmt.Errorf("--after and --before are mutually exclusive")
}
if pf.After != "" && pf.Last > 0 {
return nil, fmt.Errorf("--after cannot be combined with --last")
}
var (
size int
from *page.CursorKey
position page.Position
)
switch {
case pf.Last > 0 || pf.Before != "":
size = pf.Last
if size == 0 {
size = defaultPageSize
}
position = page.Tail
if pf.Before != "" {
ck, err := page.ParseCursorKey(pf.Before)
if err != nil {
return nil, fmt.Errorf("invalid --before cursor: %w", err)
}
from = &ck
}
default:
size = pf.First
if size == 0 {
size = defaultPageSize
}
position = page.Head
if pf.After != "" {
ck, err := page.ParseCursorKey(pf.After)
if err != nil {
return nil, fmt.Errorf("invalid --after cursor: %w", err)
}
from = &ck
}
}
return page.NewCursor(size, from, position, orderBy), nil
}
// FetchPage loads a single keyset page using cursor and wraps the rows into a
// page.Page, which trims the over-fetch and computes HasNext / HasPrev. It is
// the proboctl-side counterpart of the API's connection resolvers.
func FetchPage[E page.Paginable[F], F page.OrderField](
ctx context.Context,
cursor *page.Cursor[F],
load func(ctx context.Context, cursor *page.Cursor[F]) ([]E, error),
) (*page.Page[E, F], error) {
rows, err := load(ctx, cursor)
if err != nil {
return nil, err
}
return page.NewPage(rows, cursor), nil
}
// NewPageInfo derives the proboctl PageInfo (cursors as base64 strings) from a
// loaded page.
func NewPageInfo[T page.Paginable[F], F page.OrderField](p *page.Page[T, F]) PageInfo {
pi := PageInfo{
HasNextPage: p.Info.HasNext,
HasPreviousPage: p.Info.HasPrev,
}
if len(p.Data) > 0 {
start := p.First().CursorKey(p.Cursor.OrderBy.Field).String()
end := p.Last().CursorKey(p.Cursor.OrderBy.Field).String()
pi.StartCursor = &start
pi.EndCursor = &end
}
return pi
}
// PrintPageInfo writes the page info as a footer below a rendered table.
func PrintPageInfo(out io.Writer, pi PageInfo) {
var start, end string
if pi.StartCursor != nil {
start = *pi.StartCursor
}
if pi.EndCursor != nil {
end = *pi.EndCursor
}
_, _ = fmt.Fprintf(
out,
"hasNextPage: %t hasPreviousPage: %t\nstartCursor: %s\nendCursor: %s\n",
pi.HasNextPage,
pi.HasPreviousPage,
start,
end,
)
}

25
pkg/proboctl/commonthirdparty/list.go vendored
View File

@@ -33,7 +33,6 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
flagKeyword string
flagSort string
flagOrder string
flagLimit int
)
cmd := &cobra.Command{
@@ -49,7 +48,8 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
cmd.Flags().StringVar(&flagKeyword, "keyword", "", "Filter by name/slug substring")
cmd.Flags().StringVar(&flagSort, "sort", "name", "Sort field: name, created, updated")
cmd.Flags().StringVar(&flagOrder, "order", "", "Sort order: asc, desc (default depends on field)")
cmd.Flags().IntVarP(&flagLimit, "limit", "L", 50, "Maximum rows to return (0 for all)")
pageFlags := cmdutil.AddPageFlags(cmd)
cmd.RunE = func(cmd *cobra.Command, args []string) error {
if err := clicmdutil.ValidateOutputFlag(output); err != nil {
@@ -61,6 +61,11 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
return err
}
cursor, err := cmdutil.NewCursorFromFlags(pageFlags, orderBy)
if err != nil {
return err
}
filter := coredata.NewCommonThirdPartyFilter(optionalString(flagName))
if flagCategory != "" {
@@ -81,15 +86,17 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
return err
}
var parties coredata.CommonThirdParties
var (
parties coredata.CommonThirdParties
pageInfo cmdutil.PageInfo
)
if err := pgClient.WithConn(
cmd.Context(),
func(ctx context.Context, conn pg.Querier) error {
rows, err := cmdutil.Paginate(
p, err := cmdutil.FetchPage(
ctx,
orderBy,
flagLimit,
cursor,
func(ctx context.Context, cursor *page.Cursor[coredata.CommonThirdPartyOrderField]) ([]*coredata.CommonThirdParty, error) {
var ts coredata.CommonThirdParties
if err := ts.Load(ctx, conn, cursor, filter); err != nil {
@@ -103,7 +110,8 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
return err
}
parties = rows
parties = p.Data
pageInfo = cmdutil.NewPageInfo(p)
return nil
},
@@ -112,7 +120,7 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
}
if *output == clicmdutil.OutputJSON {
return clicmdutil.PrintJSON(f.IOStreams.Out, parties)
return clicmdutil.PrintJSON(f.IOStreams.Out, cmdutil.PageOutput{Items: parties, PageInfo: pageInfo})
}
if len(parties) == 0 {
@@ -133,6 +141,7 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
}
_, _ = fmt.Fprintln(f.IOStreams.Out, table.Render())
cmdutil.PrintPageInfo(f.IOStreams.Out, pageInfo)
_, _ = fmt.Fprintf(f.IOStreams.ErrOut, "Showing %d common third parties.\n", len(parties))
return nil

29
pkg/proboctl/commontrackerpattern/list.go
View File

@@ -40,7 +40,6 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
flagWithoutDescription bool
flagSort string
flagOrder string
flagLimit int
)
cmd := &cobra.Command{
@@ -62,7 +61,8 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
cmd.Flags().BoolVar(&flagWithoutDescription, "without-description", false, "Only patterns with a blank description")
cmd.Flags().StringVar(&flagSort, "sort", "confidence", "Sort field: pattern, confidence, created, updated, enriched")
cmd.Flags().StringVar(&flagOrder, "order", "", "Sort order: asc, desc (default depends on field)")
cmd.Flags().IntVarP(&flagLimit, "limit", "L", 50, "Maximum rows to return (0 for all)")
pageFlags := cmdutil.AddPageFlags(cmd)
cmd.RunE = func(cmd *cobra.Command, args []string) error {
if err := clicmdutil.ValidateOutputFlag(output); err != nil {
@@ -78,6 +78,11 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
return err
}
cursor, err := cmdutil.NewCursorFromFlags(pageFlags, orderBy)
if err != nil {
return err
}
var withCommonThirdParty *bool
if cmd.Flags().Changed("with-common-third-party") {
withCommonThirdParty = &flagWithCommonThirdParty
@@ -100,7 +105,10 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
ctx := cmd.Context()
var patterns coredata.CommonTrackerPatterns
var (
patterns coredata.CommonTrackerPatterns
pageInfo cmdutil.PageInfo
)
if err := pgClient.WithConn(
ctx,
@@ -153,10 +161,9 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
filter.WithIDs(linkedIDs)
}
rows, err := cmdutil.Paginate(
p, err := cmdutil.FetchPage(
ctx,
orderBy,
flagLimit,
cursor,
func(ctx context.Context, cursor *page.Cursor[coredata.CommonTrackerPatternOrderField]) ([]*coredata.CommonTrackerPattern, error) {
var ps coredata.CommonTrackerPatterns
if err := ps.Load(ctx, conn, cursor, filter); err != nil {
@@ -170,7 +177,8 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
return err
}
patterns = rows
patterns = p.Data
pageInfo = cmdutil.NewPageInfo(p)
return nil
},
@@ -179,16 +187,16 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
}
if *output == clicmdutil.OutputJSON {
return clicmdutil.PrintJSON(f.IOStreams.Out, patterns)
return clicmdutil.PrintJSON(f.IOStreams.Out, cmdutil.PageOutput{Items: patterns, PageInfo: pageInfo})
}
return renderPatternTable(cmd, f, patterns)
return renderPatternTable(cmd, f, patterns, pageInfo)
}
return cmd
}
func renderPatternTable(cmd *cobra.Command, f *cmdutil.Factory, patterns coredata.CommonTrackerPatterns) error {
func renderPatternTable(cmd *cobra.Command, f *cmdutil.Factory, patterns coredata.CommonTrackerPatterns, pageInfo cmdutil.PageInfo) error {
out := f.IOStreams.Out
if len(patterns) == 0 {
@@ -243,6 +251,7 @@ func renderPatternTable(cmd *cobra.Command, f *cmdutil.Factory, patterns coredat
}
_, _ = fmt.Fprintln(out, table.Render())
cmdutil.PrintPageInfo(out, pageInfo)
_, _ = fmt.Fprintf(f.IOStreams.ErrOut, "Showing %d common tracker patterns.\n", len(patterns))
return nil

5
pkg/proboctl/commontrackerpattern/stats.go
View File

@@ -55,7 +55,8 @@ func newCmdStats(f *cmdutil.Factory) *cobra.Command {
}{
{"total", coredata.NewCommonTrackerPatternFilter()},
{"queued", coredata.NewCommonTrackerPatternFilter().WithState(new(coredata.CommonTrackerPatternEnrichmentStateQueued))},
{"enriched", coredata.NewCommonTrackerPatternFilter().WithState(new(coredata.CommonTrackerPatternEnrichmentStateEnriched))},
{"enriched", coredata.NewCommonTrackerPatternFilter().WithState(new(coredata.CommonTrackerPatternEnrichmentStateEnriched)).WithDescribed(new(true))},
{"enriched (no description)", coredata.NewCommonTrackerPatternFilter().WithState(new(coredata.CommonTrackerPatternEnrichmentStateEnriched)).WithDescribed(new(false))},
{"unenriched", coredata.NewCommonTrackerPatternFilter().WithState(new(coredata.CommonTrackerPatternEnrichmentStateUnenriched))},
{"linked", coredata.NewCommonTrackerPatternFilter().WithLinked(new(true))},
{"unlinked", coredata.NewCommonTrackerPatternFilter().WithLinked(new(false))},
@@ -83,7 +84,7 @@ func newCmdStats(f *cmdutil.Factory) *cobra.Command {
}
table := clicmdutil.NewTable("METRIC", "COUNT")
for _, key := range []string{"total", "queued", "enriched", "unenriched", "linked", "unlinked"} {
for _, key := range []string{"total", "queued", "enriched", "enriched (no description)", "unenriched", "linked", "unlinked"} {
table.Row(key, fmt.Sprintf("%d", stats[key]))
}