Add proboctl catalog and banner reset commands
Add operator commands to proboctl for iterating on the cookie-banner agents. The global catalog groups (common-tracker-pattern, common-third-party) list/filter/sort/show the catalogs using the shared coredata cursor layer, and common-tracker-pattern reenrich re-describes selected rows by running the enricher in-process (so it completes synchronously rather than racing the async queue); a --cfg-file flag reuses probod's config to wire the agent. --linked-banner/--linked-org target exactly the catalog rows a banner or org depends on. The cookie-banner reset-trackers command is tenant-scoped (it derives a coredata.Scope from the banner/org GID) and rebuilds a banner's uncategorised, non-excluded patterns from detected_trackers, decomposing derived globs back into exacts, then re-arms the analysis and mapping workers. --mapping-only skips the rebuild. A DB-backed test covers the rebuild, link clearing, and preservation of categorised/excluded patterns. Signed-off-by: Émile Ré <emile@probo.com>
This commit is contained in:
@@ -16,16 +16,25 @@ package cmdutil
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"go.gearno.de/kit/log"
|
||||
"go.gearno.de/kit/pg"
|
||||
"go.opentelemetry.io/otel/trace/noop"
|
||||
"go.probo.inc/probo/pkg/agentsbuild"
|
||||
"go.probo.inc/probo/pkg/cmd/iostreams"
|
||||
"go.probo.inc/probo/pkg/cookiebanner"
|
||||
"go.probo.inc/probo/pkg/proboctl/pgconn"
|
||||
"go.probo.inc/probo/pkg/probodconfig"
|
||||
"sigs.k8s.io/yaml"
|
||||
)
|
||||
|
||||
type Factory struct {
|
||||
IOStreams *iostreams.IOStreams
|
||||
Version string
|
||||
PgDSN string
|
||||
CfgFile string
|
||||
}
|
||||
|
||||
func (f *Factory) PgClient() (*pg.Client, error) {
|
||||
@@ -35,3 +44,56 @@ func (f *Factory) PgClient() (*pg.Client, error) {
|
||||
|
||||
return pgconn.NewPgClientFromDSN(f.PgDSN)
|
||||
}
|
||||
|
||||
// ProbodConfig loads the shared probod configuration file (--cfg-file).
|
||||
// It reuses the exact file, struct, and json-tagged (un)marshaling probod
|
||||
// uses, so proboctl and probod stay consistent.
|
||||
func (f *Factory) ProbodConfig() (probodconfig.Config, error) {
|
||||
if f.CfgFile == "" {
|
||||
return probodconfig.Config{}, fmt.Errorf("set --cfg-file to the probod config file")
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(f.CfgFile)
|
||||
if err != nil {
|
||||
return probodconfig.Config{}, fmt.Errorf("cannot read config file %q: %w", f.CfgFile, err)
|
||||
}
|
||||
|
||||
var full probodconfig.FullConfig
|
||||
if err := yaml.Unmarshal(data, &full); err != nil {
|
||||
return probodconfig.Config{}, fmt.Errorf("cannot parse config file %q: %w", f.CfgFile, err)
|
||||
}
|
||||
|
||||
return full.Probod, nil
|
||||
}
|
||||
|
||||
// TrackerAgentsConfig builds the tracker-agents config (LLM client +
|
||||
// Firecrawl key) from the shared probod config for in-process agent
|
||||
// execution, e.g. synchronous common-pattern re-enrichment. It errors
|
||||
// when no LLM provider is configured.
|
||||
func (f *Factory) TrackerAgentsConfig() (cookiebanner.TrackerAgentsConfig, error) {
|
||||
cfg, err := f.ProbodConfig()
|
||||
if err != nil {
|
||||
return cookiebanner.TrackerAgentsConfig{}, err
|
||||
}
|
||||
|
||||
logger := log.NewLogger(
|
||||
log.WithName("proboctl"),
|
||||
log.WithOutput(f.IOStreams.ErrOut),
|
||||
)
|
||||
|
||||
trackerCfg, _, err := agentsbuild.BuildTrackerAgentsConfig(
|
||||
cfg,
|
||||
logger,
|
||||
noop.NewTracerProvider(),
|
||||
prometheus.NewRegistry(),
|
||||
)
|
||||
if err != nil {
|
||||
return cookiebanner.TrackerAgentsConfig{}, fmt.Errorf("cannot build tracker agents config: %w", err)
|
||||
}
|
||||
|
||||
if trackerCfg.LLMClient == nil {
|
||||
return cookiebanner.TrackerAgentsConfig{}, fmt.Errorf("no LLM provider configured; set llm.tracker-mapping.provider in %q", f.CfgFile)
|
||||
}
|
||||
|
||||
return trackerCfg, nil
|
||||
}
|
||||
|
||||
77
pkg/proboctl/cmdutil/paginate.go
Normal file
77
pkg/proboctl/cmdutil/paginate.go
Normal file
@@ -0,0 +1,77 @@
|
||||
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||
//
|
||||
// Permission to use, copy, modify, and/or distribute this software for any
|
||||
// purpose with or without fee is hereby granted, provided that the above
|
||||
// copyright notice and this permission notice appear in all copies.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||
// PERFORMANCE OF THIS SOFTWARE.
|
||||
|
||||
package cmdutil
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"go.probo.inc/probo/pkg/page"
|
||||
)
|
||||
|
||||
// paginatePageSize is the per-request fetch size used to walk a cursor
|
||||
// connection. It is independent of the caller's --limit, which only
|
||||
// bounds the total returned.
|
||||
const paginatePageSize = 100
|
||||
|
||||
// Paginate walks a cursor-paginated coredata Load into a single slice,
|
||||
// following the keyset cursor until the caller's limit is reached or the
|
||||
// connection is exhausted. A limit <= 0 returns every matching row. This
|
||||
// is the proboctl-side counterpart of the API's connection resolvers: it
|
||||
// drives page.NewCursor + page.NewPage against the same coredata Load
|
||||
// methods, so a future proboctl API can reuse the data layer untouched.
|
||||
func Paginate[E page.Paginable[F], F page.OrderField](
|
||||
ctx context.Context,
|
||||
orderBy page.OrderBy[F],
|
||||
limit int,
|
||||
load func(ctx context.Context, cursor *page.Cursor[F]) ([]E, error),
|
||||
) ([]E, error) {
|
||||
var (
|
||||
result []E
|
||||
from *page.CursorKey
|
||||
)
|
||||
|
||||
for {
|
||||
size := paginatePageSize
|
||||
if limit > 0 {
|
||||
remaining := limit - len(result)
|
||||
if remaining <= 0 {
|
||||
break
|
||||
}
|
||||
|
||||
if remaining < size {
|
||||
size = remaining
|
||||
}
|
||||
}
|
||||
|
||||
cursor := page.NewCursor(size, from, page.Head, orderBy)
|
||||
|
||||
rows, err := load(ctx, cursor)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
p := page.NewPage(rows, cursor)
|
||||
result = append(result, p.Data...)
|
||||
|
||||
if !p.Info.HasNext || len(p.Data) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
key := p.Data[len(p.Data)-1].CursorKey(orderBy.Field)
|
||||
from = &key
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
Reference in New Issue
Block a user