From 709b3363c592b6e7f306c167543d4ecee74eeb9f Mon Sep 17 00:00:00 2001 From: gearnode Date: Tue, 18 Mar 2025 16:12:04 +0100 Subject: [PATCH] Refactor pagination system with flexible ordering support This commit overhauls the pagination system to support flexible ordering beyond the default created_at timestamp: - Change CursorKey from fixed byte array to structured object with ID and Value - Add OrderField and OrderDirection types to support different sorting options - Update SQLFragment to dynamically generate SQL based on field and direction - Modify cursor navigation logic to consider custom ordering - Simplify Position type to use string constants instead of numeric values - Update Page struct to include reference to Cursor These changes allow paginated queries to be ordered by different fields (created_at, updated_at, name) in either ascending or descending order while maintaining consistent cursor-based pagination behavior. Signed-off-by: gearnode --- pkg/page/cursor.go | 99 +++++++++++----------- pkg/page/cursorkey.go | 164 +++++++++++++++++++++++++----------- pkg/page/order_direction.go | 49 +++++++++++ pkg/page/order_field.go | 71 ++++++++++++++++ pkg/page/page.go | 16 ++-- 5 files changed, 292 insertions(+), 107 deletions(-) create mode 100644 pkg/page/order_direction.go create mode 100644 pkg/page/order_field.go diff --git a/pkg/page/cursor.go b/pkg/page/cursor.go index b529d95dc..2af52fedf 100644 --- a/pkg/page/cursor.go +++ b/pkg/page/cursor.go @@ -15,80 +15,80 @@ package page import ( - "fmt" - "github.com/jackc/pgx/v5" ) type ( Cursor struct { Size int - Key *CursorKey Position Position + Key *CursorKey + OrderBy OrderBy } - Position int8 + Position string + + OrderBy struct { + Field OrderField + Direction OrderDirection + } ) const ( DefaultCursorSize = 25 - Tail Position = iota - Head + Tail Position = "TAIL" + Head Position = "HEAD" ) -func (p Position) ToDirection() string { - switch p { - case Tail: - return "ASC" - case Head: - return "DESC" - default: - panic(fmt.Errorf("unknown direction: %d", p)) - } -} - -func NewCursor(size int, from *CursorKey, pos Position) *Cursor { +func NewCursor(size int, from *CursorKey, pos Position, orderBy *OrderBy) *Cursor { if size == 0 { size = DefaultCursorSize } + if orderBy == nil { + orderBy = &OrderBy{ + Field: OrderFieldCreatedAt, + Direction: OrderDirectionDesc, + } + } + return &Cursor{ Size: size, Key: from, Position: pos, + OrderBy: *orderBy, } } func (c *Cursor) SQLFragment() string { - return ` - CASE - WHEN @cursor_order = 'DESC' AND @cursor_from_id::TEXT IS NOT NULL THEN ( - (created_at <= @cursor_from_ts) AND NOT (created_at = @cursor_from_ts AND id > @cursor_from_id) - ) - WHEN @cursor_order = 'ASC' AND @cursor_from_id::TEXT IS NOT NULL THEN ( - (created_at >= @cursor_from_ts) AND NOT (created_at = @cursor_from_ts AND id < @cursor_from_id) - ) - ELSE TRUE - END -ORDER BY - CASE - WHEN @cursor_order = 'ASC' THEN created_at - END ASC, - CASE - WHEN @cursor_order = 'ASC' THEN id - END ASC, - CASE - WHEN @cursor_order = 'DESC' THEN created_at - END DESC, - CASE - WHEN @cursor_order = 'DESC' THEN id - END DESC -LIMIT @cursor_limit -` + fieldName := c.OrderBy.Field.Column() + + var orderDirection string + switch { + case c.OrderBy.Direction == OrderDirectionAsc && c.Position == Head: + orderDirection = "ASC" + case c.OrderBy.Direction == OrderDirectionDesc && c.Position == Head: + orderDirection = "DESC" + case c.OrderBy.Direction == OrderDirectionAsc && c.Position == Tail: + orderDirection = "DESC" + case c.OrderBy.Direction == OrderDirectionDesc && c.Position == Tail: + orderDirection = "ASC" + } + + whereClause := "TRUE" + if c.Key != nil && orderDirection == "DESC" { + whereClause = "(" + fieldName + " <= @cursor_field_value) AND NOT (" + fieldName + " = @cursor_field_value AND id > @cursor_id)" + } else if c.Key != nil && orderDirection == "ASC" { + whereClause = "(" + fieldName + " >= @cursor_field_value) AND NOT (" + fieldName + " = @cursor_field_value AND id < @cursor_id)" + } + + orderByClause := fieldName + " " + orderDirection + ", id " + orderDirection + + return whereClause + " ORDER BY " + orderByClause + " LIMIT @cursor_limit" } -func (c *Cursor) SQLArguments() pgx.StrictNamedArgs { +func (c *Cursor) SQLArguments() pgx.NamedArgs { var size = c.Size if c.Key == nil { size += 1 @@ -96,16 +96,13 @@ func (c *Cursor) SQLArguments() pgx.StrictNamedArgs { size += 2 } - arguments := pgx.StrictNamedArgs{ - "cursor_order": c.Position.ToDirection(), - "cursor_limit": size, - "cursor_from_id": nil, - "cursor_from_ts": nil, + arguments := pgx.NamedArgs{ + "cursor_limit": size, } if c.Key != nil { - arguments["cursor_from_id"] = c.Key.ID() - arguments["cursor_from_ts"] = c.Key.Timestamp() + arguments["cursor_id"] = c.Key.ID + arguments["cursor_field_value"] = c.Key.Value } return arguments diff --git a/pkg/page/cursorkey.go b/pkg/page/cursorkey.go index fb809a800..978282e66 100644 --- a/pkg/page/cursorkey.go +++ b/pkg/page/cursorkey.go @@ -16,16 +16,16 @@ package page import ( "encoding/base64" - "encoding/binary" + "encoding/json" "errors" - "time" "github.com/getprobo/probo/pkg/gid" ) -type ( - CursorKey [byteLength]byte -) +type CursorKey struct { + ID gid.GID + Value any +} var ( CursorKeyNil CursorKey @@ -33,63 +33,64 @@ var ( ErrInvalidFormat = errors.New("invalid format") ) -const ( - byteLength = 24 -) - func ParseCursorKey(s string) (CursorKey, error) { - b, err := base64.RawURLEncoding.DecodeString(s) + data, err := base64.RawURLEncoding.DecodeString(s) if err != nil { return CursorKeyNil, ErrInvalidFormat } - ck, err := CursorKeyFromBytes(b) + var arr []json.RawMessage + if err := json.Unmarshal(data, &arr); err != nil { + return CursorKeyNil, ErrInvalidFormat + } + + if len(arr) != 2 { + return CursorKeyNil, ErrInvalidFormat + } + + var idStr string + if err := json.Unmarshal(arr[0], &idStr); err != nil { + return CursorKeyNil, ErrInvalidFormat + } + + id, err := gid.ParseGID(idStr) if err != nil { return CursorKeyNil, ErrInvalidFormat } - return ck, nil -} - -func CursorKeyFromBytes(b []byte) (CursorKey, error) { - var ck CursorKey - - if len(b) != byteLength { + var value any + if err := json.Unmarshal(arr[1], &value); err != nil { return CursorKeyNil, ErrInvalidFormat } - copy(ck[:], b) - - return ck, nil + return CursorKey{ + ID: id, + Value: value, + }, nil } -func NewCursorKey(id gid.GID, t time.Time) CursorKey { - var cursorKey CursorKey - copy(cursorKey[:16], id[:]) - _ = binary.PutVarint(cursorKey[16:], t.UnixMicro()) - - return cursorKey +func NewCursorKey(id gid.GID, value any) CursorKey { + return CursorKey{ + ID: id, + Value: value, + } } func (ck CursorKey) Bytes() []byte { - return ck[:] + data, _ := ck.MarshalBinary() + return data } func (ck CursorKey) String() string { - return base64.RawURLEncoding.EncodeToString(ck.Bytes()) + data, err := ck.MarshalBinary() + if err != nil { + return "" + } + return base64.RawURLEncoding.EncodeToString(data) } -func (ck CursorKey) Timestamp() time.Time { - unixMicro, _ := binary.Varint(ck[16:]) - - seconds := unixMicro / 1e6 - nanoseconds := (unixMicro % 1e6) * 1e3 - - return time.Unix(seconds, nanoseconds) -} - -func (ck CursorKey) ID() gid.GID { - return gid.GID(ck[:16]) +func (ck CursorKey) FieldValue() any { + return ck.Value } func (ck CursorKey) MarshalText() ([]byte, error) { @@ -97,27 +98,92 @@ func (ck CursorKey) MarshalText() ([]byte, error) { } func (ck *CursorKey) UnmarshalText(data []byte) error { - ck2, err := ParseCursorKey(string(data)) + newCk, err := ParseCursorKey(string(data)) if err != nil { return err } - - *ck = ck2 - + *ck = newCk return nil } func (ck CursorKey) MarshalBinary() ([]byte, error) { - return ck.Bytes(), nil + arr := []any{ck.ID.String(), ck.Value} + return json.Marshal(arr) } -func (ck *CursorKey) UnmarshalBinary(b []byte) error { - ck2, err := CursorKeyFromBytes(b) - if err != nil { +func (ck *CursorKey) UnmarshalBinary(data []byte) error { + var arr []json.RawMessage + if err := json.Unmarshal(data, &arr); err != nil { return err } - *ck = ck2 + if len(arr) != 2 { + return ErrInvalidFormat + } + // Parse the ID + var idStr string + if err := json.Unmarshal(arr[0], &idStr); err != nil { + return ErrInvalidFormat + } + + id, err := gid.ParseGID(idStr) + if err != nil { + return ErrInvalidFormat + } + + var value any + if err := json.Unmarshal(arr[1], &value); err != nil { + return ErrInvalidFormat + } + + ck.ID = id + ck.Value = value + return nil +} + +func (ck CursorKey) MarshalJSON() ([]byte, error) { + arr := []any{ck.ID.String(), ck.Value} + return json.Marshal(arr) +} + +func (ck *CursorKey) UnmarshalJSON(data []byte) error { + var arr []json.RawMessage + if err := json.Unmarshal(data, &arr); err != nil { + var s string + if err := json.Unmarshal(data, &s); err != nil { + return err + } + + parsed, err := ParseCursorKey(s) + if err != nil { + return err + } + + *ck = parsed + return nil + } + + if len(arr) != 2 { + return ErrInvalidFormat + } + + var idStr string + if err := json.Unmarshal(arr[0], &idStr); err != nil { + return ErrInvalidFormat + } + + id, err := gid.ParseGID(idStr) + if err != nil { + return ErrInvalidFormat + } + + var value any + if err := json.Unmarshal(arr[1], &value); err != nil { + return ErrInvalidFormat + } + + ck.ID = id + ck.Value = value return nil } diff --git a/pkg/page/order_direction.go b/pkg/page/order_direction.go new file mode 100644 index 000000000..e48dc8bb3 --- /dev/null +++ b/pkg/page/order_direction.go @@ -0,0 +1,49 @@ +// Copyright (c) 2025 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package page + +import ( + "fmt" +) + +type OrderDirection string + +const ( + OrderDirectionAsc OrderDirection = "ASC" + OrderDirectionDesc OrderDirection = "DESC" +) + +func (od OrderDirection) MarshalText() ([]byte, error) { + return []byte(od.String()), nil +} + +func (od *OrderDirection) UnmarshalText(data []byte) error { + val := string(data) + + switch val { + case OrderDirectionAsc.String(): + *od = OrderDirectionAsc + case OrderDirectionDesc.String(): + *od = OrderDirectionDesc + default: + return fmt.Errorf("invalid OrderDirection value: %q", val) + } + + return nil +} + +func (od OrderDirection) String() string { + return string(od) +} diff --git a/pkg/page/order_field.go b/pkg/page/order_field.go new file mode 100644 index 000000000..3a6869c32 --- /dev/null +++ b/pkg/page/order_field.go @@ -0,0 +1,71 @@ +// Copyright (c) 2025 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package page + +import ( + "fmt" +) + +type ( + OrderField interface { + Column() string + } + + BaseOrderField string +) + +const ( + OrderFieldCreatedAt BaseOrderField = "CREATED_AT" + OrderFieldUpdatedAt BaseOrderField = "UPDATED_AT" + OrderFieldName BaseOrderField = "NAME" +) + +func (of BaseOrderField) String() string { + return string(of) +} + +func (of BaseOrderField) Column() string { + switch of { + case OrderFieldCreatedAt: + return "created_at" + case OrderFieldUpdatedAt: + return "updated_at" + case OrderFieldName: + return "name" + default: + return "" + } +} + +func (of BaseOrderField) MarshalText() ([]byte, error) { + return []byte(of.String()), nil +} + +func (of *BaseOrderField) UnmarshalText(data []byte) error { + val := string(data) + + switch val { + case OrderFieldCreatedAt.String(): + *of = OrderFieldCreatedAt + case OrderFieldUpdatedAt.String(): + *of = OrderFieldUpdatedAt + case OrderFieldName.String(): + *of = OrderFieldName + default: + return fmt.Errorf("invalid BaseOrderField value: %q", val) + } + + return nil +} diff --git a/pkg/page/page.go b/pkg/page/page.go index 5c8dd8eb0..01b63c274 100644 --- a/pkg/page/page.go +++ b/pkg/page/page.go @@ -16,7 +16,7 @@ package page type ( Paginable interface { - CursorKey() CursorKey + CursorKey(orderBy OrderField) CursorKey } PageInfo struct { @@ -25,8 +25,9 @@ type ( } Page[T Paginable] struct { - Info *PageInfo - Data []T + Info *PageInfo + Cursor *Cursor + Data []T } ) @@ -73,7 +74,7 @@ func NewPage[T Paginable](data []T, c *Cursor) *Page[T] { edges = edges[0 : len(edges)-1] } - if c.Key != nil && *c.Key == firstFromData.CursorKey() { + if c.Key != nil && c.Key.String() == firstFromData.CursorKey(c.OrderBy.Field).String() { pi.HasPrev = true } @@ -97,7 +98,7 @@ func NewPage[T Paginable](data []T, c *Cursor) *Page[T] { edges = edges[1:] } - if c.Key != nil && *c.Key == firstFromData.CursorKey() { + if c.Key != nil && c.Key.String() == firstFromData.CursorKey(c.OrderBy.Field).String() { pi.HasNext = true } @@ -109,7 +110,8 @@ func NewPage[T Paginable](data []T, c *Cursor) *Page[T] { } return &Page[T]{ - Info: pi, - Data: edges, + Info: pi, + Cursor: c, + Data: edges, } }