Add proboctl catalog and banner reset commands
Add operator commands to proboctl for iterating on the cookie-banner agents. The global catalog groups (common-tracker-pattern, common-third-party) list/filter/sort/show the catalogs using the shared coredata cursor layer, and common-tracker-pattern reenrich re-describes selected rows by running the enricher in-process (so it completes synchronously rather than racing the async queue); a --cfg-file flag reuses probod's config to wire the agent. --linked-banner/--linked-org target exactly the catalog rows a banner or org depends on. The cookie-banner reset-trackers command is tenant-scoped (it derives a coredata.Scope from the banner/org GID) and rebuilds a banner's uncategorised, non-excluded patterns from detected_trackers, decomposing derived globs back into exacts, then re-arms the analysis and mapping workers. --mapping-only skips the rebuild. A DB-backed test covers the rebuild, link clearing, and preservation of categorised/excluded patterns. Signed-off-by: Émile Ré <emile@probo.com>
This commit is contained in:
97
pkg/proboctl/commontrackerpattern/commontrackerpattern.go
Normal file
97
pkg/proboctl/commontrackerpattern/commontrackerpattern.go
Normal file
@@ -0,0 +1,97 @@
|
||||
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||
//
|
||||
// Permission to use, copy, modify, and/or distribute this software for any
|
||||
// purpose with or without fee is hereby granted, provided that the above
|
||||
// copyright notice and this permission notice appear in all copies.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||
// PERFORMANCE OF THIS SOFTWARE.
|
||||
|
||||
package commontrackerpattern
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"go.gearno.de/kit/pg"
|
||||
"go.probo.inc/probo/pkg/coredata"
|
||||
"go.probo.inc/probo/pkg/gid"
|
||||
"go.probo.inc/probo/pkg/proboctl/cmdutil"
|
||||
)
|
||||
|
||||
// NewCmdCommonTrackerPattern is the entry point for inspecting and
|
||||
// re-enriching the global common tracker pattern catalog.
|
||||
func NewCmdCommonTrackerPattern(f *cmdutil.Factory) *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "common-tracker-pattern <command>",
|
||||
Aliases: []string{"ctp"},
|
||||
Short: "Inspect and re-enrich the global common tracker pattern catalog",
|
||||
}
|
||||
|
||||
cmd.AddCommand(newCmdList(f))
|
||||
cmd.AddCommand(newCmdShow(f))
|
||||
cmd.AddCommand(newCmdReenrich(f))
|
||||
cmd.AddCommand(newCmdStats(f))
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
// enrichmentState classifies a pattern's position in the enrichment
|
||||
// lifecycle for display.
|
||||
func enrichmentState(p *coredata.CommonTrackerPattern) string {
|
||||
switch {
|
||||
case p.EnrichmentRequestedAt != nil:
|
||||
return "queued"
|
||||
case p.EnrichedAt != nil:
|
||||
return "enriched"
|
||||
default:
|
||||
return "unenriched"
|
||||
}
|
||||
}
|
||||
|
||||
// resolveCommonThirdPartyID accepts either a common third party GID or a
|
||||
// slug and returns the corresponding id.
|
||||
func resolveCommonThirdPartyID(ctx context.Context, conn pg.Querier, value string) (gid.GID, error) {
|
||||
if id, err := gid.ParseGID(value); err == nil {
|
||||
return id, nil
|
||||
}
|
||||
|
||||
var party coredata.CommonThirdParty
|
||||
if err := party.LoadBySlug(ctx, conn, value); err != nil {
|
||||
if errors.Is(err, coredata.ErrResourceNotFound) {
|
||||
return gid.GID{}, fmt.Errorf("no common third party found for %q (pass a slug or GID)", value)
|
||||
}
|
||||
|
||||
return gid.GID{}, fmt.Errorf("cannot resolve common third party %q: %w", value, err)
|
||||
}
|
||||
|
||||
return party.ID, nil
|
||||
}
|
||||
|
||||
// thirdPartyNamesByID loads display names for the given common third
|
||||
// party ids, skipping nil/empty inputs. It is used to render the linked
|
||||
// vendor column without per-row queries.
|
||||
func thirdPartyNamesByID(ctx context.Context, conn pg.Querier, ids []gid.GID) (map[gid.GID]string, error) {
|
||||
names := make(map[gid.GID]string)
|
||||
if len(ids) == 0 {
|
||||
return names, nil
|
||||
}
|
||||
|
||||
var parties coredata.CommonThirdParties
|
||||
if err := parties.LoadByIDs(ctx, conn, ids); err != nil {
|
||||
return nil, fmt.Errorf("cannot load common third parties: %w", err)
|
||||
}
|
||||
|
||||
for _, p := range parties {
|
||||
names[p.ID] = p.Name
|
||||
}
|
||||
|
||||
return names, nil
|
||||
}
|
||||
299
pkg/proboctl/commontrackerpattern/list.go
Normal file
299
pkg/proboctl/commontrackerpattern/list.go
Normal file
@@ -0,0 +1,299 @@
|
||||
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||
//
|
||||
// Permission to use, copy, modify, and/or distribute this software for any
|
||||
// purpose with or without fee is hereby granted, provided that the above
|
||||
// copyright notice and this permission notice appear in all copies.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||
// PERFORMANCE OF THIS SOFTWARE.
|
||||
|
||||
package commontrackerpattern
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"go.gearno.de/kit/pg"
|
||||
clicmdutil "go.probo.inc/probo/pkg/cmd/cmdutil"
|
||||
"go.probo.inc/probo/pkg/coredata"
|
||||
"go.probo.inc/probo/pkg/gid"
|
||||
"go.probo.inc/probo/pkg/page"
|
||||
"go.probo.inc/probo/pkg/proboctl/cmdutil"
|
||||
)
|
||||
|
||||
func newCmdList(f *cmdutil.Factory) *cobra.Command {
|
||||
var (
|
||||
flagTrackerType string
|
||||
flagMatchType string
|
||||
flagThirdParty string
|
||||
flagKeyword string
|
||||
flagState string
|
||||
flagLinked bool
|
||||
flagUnlinked bool
|
||||
flagSort string
|
||||
flagOrder string
|
||||
flagLimit int
|
||||
)
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "list",
|
||||
Short: "List common tracker patterns with filters and sorting",
|
||||
Args: cobra.NoArgs,
|
||||
}
|
||||
|
||||
output := clicmdutil.AddOutputFlag(cmd)
|
||||
|
||||
cmd.Flags().StringVar(&flagTrackerType, "tracker-type", "", "Filter by tracker type (COOKIE, LOCAL_STORAGE, SESSION_STORAGE, INDEXED_DB)")
|
||||
cmd.Flags().StringVar(&flagMatchType, "match-type", "", "Filter by match type (EXACT, GLOB, PREFIX)")
|
||||
cmd.Flags().StringVar(&flagThirdParty, "third-party", "", "Filter by linked common third party (slug or GID)")
|
||||
cmd.Flags().StringVar(&flagKeyword, "keyword", "", "Filter by pattern/description substring")
|
||||
cmd.Flags().StringVar(&flagState, "state", "", "Filter by enrichment state (queued, enriched, unenriched)")
|
||||
cmd.Flags().BoolVar(&flagLinked, "linked", false, "Only patterns linked to a common third party")
|
||||
cmd.Flags().BoolVar(&flagUnlinked, "unlinked", false, "Only patterns not linked to a common third party")
|
||||
cmd.Flags().StringVar(&flagSort, "sort", "confidence", "Sort field: pattern, confidence, created, updated, enriched")
|
||||
cmd.Flags().StringVar(&flagOrder, "order", "", "Sort order: asc, desc (default depends on field)")
|
||||
cmd.Flags().IntVarP(&flagLimit, "limit", "L", 50, "Maximum rows to return (0 for all)")
|
||||
|
||||
cmd.RunE = func(cmd *cobra.Command, args []string) error {
|
||||
if err := clicmdutil.ValidateOutputFlag(output); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if flagLinked && flagUnlinked {
|
||||
return fmt.Errorf("--linked and --unlinked are mutually exclusive")
|
||||
}
|
||||
|
||||
orderBy, err := parseOrderBy(flagSort, flagOrder)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
filter, err := buildListFilter(flagTrackerType, flagMatchType, flagKeyword, flagState, flagLinked, flagUnlinked)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
pgClient, err := f.PgClient()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ctx := cmd.Context()
|
||||
|
||||
var patterns coredata.CommonTrackerPatterns
|
||||
|
||||
if err := pgClient.WithConn(
|
||||
ctx,
|
||||
func(ctx context.Context, conn pg.Querier) error {
|
||||
if flagThirdParty != "" {
|
||||
id, err := resolveCommonThirdPartyID(ctx, conn, flagThirdParty)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
filter.WithCommonThirdPartyID(&id)
|
||||
}
|
||||
|
||||
rows, err := cmdutil.Paginate(
|
||||
ctx,
|
||||
orderBy,
|
||||
flagLimit,
|
||||
func(ctx context.Context, cursor *page.Cursor[coredata.CommonTrackerPatternOrderField]) ([]*coredata.CommonTrackerPattern, error) {
|
||||
var ps coredata.CommonTrackerPatterns
|
||||
if err := ps.Load(ctx, conn, cursor, filter); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return ps, nil
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
patterns = rows
|
||||
|
||||
return nil
|
||||
},
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if *output == clicmdutil.OutputJSON {
|
||||
return clicmdutil.PrintJSON(f.IOStreams.Out, patterns)
|
||||
}
|
||||
|
||||
return renderPatternTable(cmd, f, patterns)
|
||||
}
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
func renderPatternTable(cmd *cobra.Command, f *cmdutil.Factory, patterns coredata.CommonTrackerPatterns) error {
|
||||
out := f.IOStreams.Out
|
||||
|
||||
if len(patterns) == 0 {
|
||||
_, _ = fmt.Fprintln(out, "No common tracker patterns found.")
|
||||
return nil
|
||||
}
|
||||
|
||||
var linkedIDs []gid.GID
|
||||
for _, p := range patterns {
|
||||
if p.CommonThirdPartyID != nil {
|
||||
linkedIDs = append(linkedIDs, *p.CommonThirdPartyID)
|
||||
}
|
||||
}
|
||||
|
||||
pgClient, err := f.PgClient()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var names map[gid.GID]string
|
||||
|
||||
if err := pgClient.WithConn(
|
||||
cmd.Context(),
|
||||
func(ctx context.Context, conn pg.Querier) error {
|
||||
names, err = thirdPartyNamesByID(ctx, conn, linkedIDs)
|
||||
return err
|
||||
},
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
table := clicmdutil.NewTable("ID", "TYPE", "MATCH", "PATTERN", "CONF", "STATE", "THIRD PARTY")
|
||||
|
||||
for _, p := range patterns {
|
||||
thirdParty := ""
|
||||
if p.CommonThirdPartyID != nil {
|
||||
thirdParty = names[*p.CommonThirdPartyID]
|
||||
}
|
||||
|
||||
table.Row(
|
||||
p.ID.String(),
|
||||
string(p.TrackerType),
|
||||
string(p.MatchType),
|
||||
p.Pattern,
|
||||
fmt.Sprintf("%.2f", p.Confidence),
|
||||
enrichmentState(p),
|
||||
thirdParty,
|
||||
)
|
||||
}
|
||||
|
||||
_, _ = fmt.Fprintln(out, table.Render())
|
||||
_, _ = fmt.Fprintf(f.IOStreams.ErrOut, "Showing %d common tracker patterns.\n", len(patterns))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// parseOrderBy maps the --sort/--order flags to a page.OrderBy. When
|
||||
// --order is empty it defaults to descending for the time/score fields
|
||||
// and ascending for pattern.
|
||||
func parseOrderBy(sort, order string) (page.OrderBy[coredata.CommonTrackerPatternOrderField], error) {
|
||||
var (
|
||||
field coredata.CommonTrackerPatternOrderField
|
||||
defaultDesc bool
|
||||
zeroOrderBy page.OrderBy[coredata.CommonTrackerPatternOrderField]
|
||||
)
|
||||
|
||||
switch sort {
|
||||
case "pattern":
|
||||
field = coredata.CommonTrackerPatternOrderFieldPattern
|
||||
case "confidence":
|
||||
field, defaultDesc = coredata.CommonTrackerPatternOrderFieldConfidence, true
|
||||
case "created":
|
||||
field, defaultDesc = coredata.CommonTrackerPatternOrderFieldCreatedAt, true
|
||||
case "updated":
|
||||
field, defaultDesc = coredata.CommonTrackerPatternOrderFieldUpdatedAt, true
|
||||
case "enriched":
|
||||
field, defaultDesc = coredata.CommonTrackerPatternOrderFieldEnrichedAt, true
|
||||
default:
|
||||
return zeroOrderBy, fmt.Errorf("invalid --sort value %q: valid values are pattern, confidence, created, updated, enriched", sort)
|
||||
}
|
||||
|
||||
direction := page.OrderDirectionAsc
|
||||
if defaultDesc {
|
||||
direction = page.OrderDirectionDesc
|
||||
}
|
||||
|
||||
switch order {
|
||||
case "":
|
||||
// keep field default
|
||||
case "asc":
|
||||
direction = page.OrderDirectionAsc
|
||||
case "desc":
|
||||
direction = page.OrderDirectionDesc
|
||||
default:
|
||||
return zeroOrderBy, fmt.Errorf("invalid --order value %q: valid values are asc, desc", order)
|
||||
}
|
||||
|
||||
return page.OrderBy[coredata.CommonTrackerPatternOrderField]{Field: field, Direction: direction}, nil
|
||||
}
|
||||
|
||||
func buildListFilter(
|
||||
trackerType, matchType, keyword, state string,
|
||||
linked, unlinked bool,
|
||||
) (*coredata.CommonTrackerPatternFilter, error) {
|
||||
filter := coredata.NewCommonTrackerPatternFilter()
|
||||
|
||||
if trackerType != "" {
|
||||
tt := coredata.TrackerType(trackerType)
|
||||
if !tt.IsValid() {
|
||||
return nil, fmt.Errorf("invalid --tracker-type value %q", trackerType)
|
||||
}
|
||||
|
||||
filter.WithTrackerType(&tt)
|
||||
}
|
||||
|
||||
if matchType != "" {
|
||||
mt := coredata.TrackerPatternMatchType(matchType)
|
||||
if !mt.IsValid() {
|
||||
return nil, fmt.Errorf("invalid --match-type value %q", matchType)
|
||||
}
|
||||
|
||||
filter.WithMatchType(&mt)
|
||||
}
|
||||
|
||||
if keyword != "" {
|
||||
filter.WithKeyword(&keyword)
|
||||
}
|
||||
|
||||
if state != "" {
|
||||
st, err := parseEnrichmentState(state)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
filter.WithState(&st)
|
||||
}
|
||||
|
||||
switch {
|
||||
case linked:
|
||||
v := true
|
||||
filter.WithLinked(&v)
|
||||
case unlinked:
|
||||
v := false
|
||||
filter.WithLinked(&v)
|
||||
}
|
||||
|
||||
return filter, nil
|
||||
}
|
||||
|
||||
func parseEnrichmentState(value string) (coredata.CommonTrackerPatternEnrichmentState, error) {
|
||||
switch value {
|
||||
case "queued":
|
||||
return coredata.CommonTrackerPatternEnrichmentStateQueued, nil
|
||||
case "enriched":
|
||||
return coredata.CommonTrackerPatternEnrichmentStateEnriched, nil
|
||||
case "unenriched":
|
||||
return coredata.CommonTrackerPatternEnrichmentStateUnenriched, nil
|
||||
default:
|
||||
return "", fmt.Errorf("invalid --state value %q: valid values are queued, enriched, unenriched", value)
|
||||
}
|
||||
}
|
||||
291
pkg/proboctl/commontrackerpattern/reenrich.go
Normal file
291
pkg/proboctl/commontrackerpattern/reenrich.go
Normal file
@@ -0,0 +1,291 @@
|
||||
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||
//
|
||||
// Permission to use, copy, modify, and/or distribute this software for any
|
||||
// purpose with or without fee is hereby granted, provided that the above
|
||||
// copyright notice and this permission notice appear in all copies.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||
// PERFORMANCE OF THIS SOFTWARE.
|
||||
|
||||
package commontrackerpattern
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"go.gearno.de/kit/log"
|
||||
"go.gearno.de/kit/pg"
|
||||
"go.probo.inc/probo/pkg/cookiebanner"
|
||||
"go.probo.inc/probo/pkg/coredata"
|
||||
"go.probo.inc/probo/pkg/gid"
|
||||
"go.probo.inc/probo/pkg/proboctl/cmdutil"
|
||||
)
|
||||
|
||||
func newCmdReenrich(f *cmdutil.Factory) *cobra.Command {
|
||||
var (
|
||||
flagIDs []string
|
||||
flagThirdParty string
|
||||
flagTrackerType string
|
||||
flagKeyword string
|
||||
flagState string
|
||||
flagAll bool
|
||||
flagLinkedBanner string
|
||||
flagLinkedOrg string
|
||||
flagConcurrency int
|
||||
flagResetEnriched bool
|
||||
flagDryRun bool
|
||||
flagYes bool
|
||||
flagEnqueue bool
|
||||
)
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "reenrich",
|
||||
Short: "Re-describe common tracker patterns by running the enrichment agent",
|
||||
Long: "Re-describe selected common tracker patterns. By default the enrichment " +
|
||||
"agent runs in-process and the command returns only when the work is done " +
|
||||
"(requires --cfg-file with an LLM provider). Use --enqueue to instead arm the " +
|
||||
"async enrichment worker. Re-describe a banner's catalog rows with --linked-banner " +
|
||||
"before running 'cookie-banner reset-trackers' so fresh descriptions copy down.",
|
||||
Args: cobra.NoArgs,
|
||||
}
|
||||
|
||||
cmd.Flags().StringSliceVar(&flagIDs, "id", nil, "Common tracker pattern GID(s) to re-enrich (repeatable)")
|
||||
cmd.Flags().StringVar(&flagThirdParty, "third-party", "", "Select patterns linked to a common third party (slug or GID)")
|
||||
cmd.Flags().StringVar(&flagTrackerType, "tracker-type", "", "Select patterns of a tracker type")
|
||||
cmd.Flags().StringVar(&flagKeyword, "keyword", "", "Select patterns matching a pattern/description substring")
|
||||
cmd.Flags().StringVar(&flagState, "state", "", "Select by enrichment state (queued, enriched, unenriched)")
|
||||
cmd.Flags().BoolVar(&flagAll, "all", false, "Select every common tracker pattern")
|
||||
cmd.Flags().StringVar(&flagLinkedBanner, "linked-banner", "", "Select catalog rows linked to a cookie banner's patterns (GID)")
|
||||
cmd.Flags().StringVar(&flagLinkedOrg, "linked-org", "", "Select catalog rows linked to an organization's patterns (GID)")
|
||||
cmd.Flags().IntVar(&flagConcurrency, "concurrency", 4, "Number of patterns to enrich in parallel (sync mode)")
|
||||
cmd.Flags().BoolVar(&flagResetEnriched, "reset-enriched", true, "Clear enriched_at so terminal rows are re-processed")
|
||||
cmd.Flags().BoolVar(&flagDryRun, "dry-run", false, "Print the selected patterns without enriching")
|
||||
cmd.Flags().BoolVar(&flagYes, "yes", false, "Skip confirmation")
|
||||
cmd.Flags().BoolVar(&flagEnqueue, "enqueue", false, "Arm the async enrichment worker instead of running the agent in-process")
|
||||
|
||||
cmd.RunE = func(cmd *cobra.Command, args []string) error {
|
||||
ctx := cmd.Context()
|
||||
|
||||
pgClient, err := f.PgClient()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ids, err := resolveReenrichIDs(
|
||||
ctx,
|
||||
pgClient,
|
||||
flagIDs,
|
||||
flagThirdParty,
|
||||
flagTrackerType,
|
||||
flagKeyword,
|
||||
flagState,
|
||||
flagAll,
|
||||
flagLinkedBanner,
|
||||
flagLinkedOrg,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
out := f.IOStreams.Out
|
||||
|
||||
if len(ids) == 0 {
|
||||
_, _ = fmt.Fprintln(out, "No common tracker patterns matched the selection.")
|
||||
return nil
|
||||
}
|
||||
|
||||
if flagDryRun {
|
||||
_, _ = fmt.Fprintf(out, "Would re-enrich %d common tracker pattern(s).\n", len(ids))
|
||||
printSample(out, ids)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
if !flagYes {
|
||||
return fmt.Errorf("about to re-enrich %d pattern(s); pass --yes to proceed or --dry-run to preview", len(ids))
|
||||
}
|
||||
|
||||
if flagEnqueue {
|
||||
var requeued int64
|
||||
|
||||
if err := pgClient.WithTx(
|
||||
ctx,
|
||||
func(ctx context.Context, tx pg.Tx) error {
|
||||
var ps coredata.CommonTrackerPatterns
|
||||
requeued, err = ps.RequestEnrichmentByIDs(ctx, tx, ids, flagResetEnriched)
|
||||
|
||||
return err
|
||||
},
|
||||
); err != nil {
|
||||
return fmt.Errorf("cannot enqueue enrichment: %w", err)
|
||||
}
|
||||
|
||||
_, _ = fmt.Fprintf(out, "Queued %d common tracker pattern(s) for the enrichment worker.\n", requeued)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
cfg, err := f.TrackerAgentsConfig()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
logger := log.NewLogger(
|
||||
log.WithName("proboctl"),
|
||||
log.WithOutput(f.IOStreams.ErrOut),
|
||||
)
|
||||
|
||||
enricher := cookiebanner.NewCommonPatternEnricher(pgClient, logger, cfg)
|
||||
|
||||
enriched, err := enricher.EnrichByIDs(ctx, ids, flagConcurrency)
|
||||
|
||||
_, _ = fmt.Fprintf(out, "Enriched %d of %d common tracker pattern(s).\n", enriched, len(ids))
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("enrichment did not complete for all patterns: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
func resolveReenrichIDs(
|
||||
ctx context.Context,
|
||||
pgClient *pg.Client,
|
||||
rawIDs []string,
|
||||
thirdParty, trackerType, keyword, state string,
|
||||
all bool,
|
||||
linkedBanner, linkedOrg string,
|
||||
) ([]gid.GID, error) {
|
||||
if len(rawIDs) > 0 {
|
||||
ids := make([]gid.GID, 0, len(rawIDs))
|
||||
|
||||
for _, raw := range rawIDs {
|
||||
id, err := gid.ParseGID(raw)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid --id value %q: %w", raw, err)
|
||||
}
|
||||
|
||||
ids = append(ids, id)
|
||||
}
|
||||
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
var ids []gid.GID
|
||||
|
||||
err := pgClient.WithConn(
|
||||
ctx,
|
||||
func(ctx context.Context, conn pg.Querier) error {
|
||||
switch {
|
||||
case linkedBanner != "":
|
||||
bannerID, err := gid.ParseGID(linkedBanner)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid --linked-banner GID %q: %w", linkedBanner, err)
|
||||
}
|
||||
|
||||
var tps coredata.TrackerPatterns
|
||||
ids, err = tps.LoadAllLinkedCommonTrackerPatternIDsByCookieBannerID(ctx, conn, coredata.NewScopeFromObjectID(bannerID), bannerID)
|
||||
|
||||
return err
|
||||
case linkedOrg != "":
|
||||
orgID, err := gid.ParseGID(linkedOrg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid --linked-org GID %q: %w", linkedOrg, err)
|
||||
}
|
||||
|
||||
var tps coredata.TrackerPatterns
|
||||
ids, err = tps.LoadAllLinkedCommonTrackerPatternIDsByOrganizationID(ctx, conn, coredata.NewScopeFromObjectID(orgID), orgID)
|
||||
|
||||
return err
|
||||
default:
|
||||
filter, hasSelector, err := buildReenrichFilter(ctx, conn, thirdParty, trackerType, keyword, state)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !hasSelector && !all {
|
||||
return fmt.Errorf("specify a selector (--id, --third-party, --tracker-type, --state, --keyword, --linked-banner, --linked-org) or --all")
|
||||
}
|
||||
|
||||
var ps coredata.CommonTrackerPatterns
|
||||
ids, err = ps.LoadAllIDs(ctx, conn, filter)
|
||||
|
||||
return err
|
||||
}
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
func buildReenrichFilter(
|
||||
ctx context.Context,
|
||||
conn pg.Querier,
|
||||
thirdParty, trackerType, keyword, state string,
|
||||
) (*coredata.CommonTrackerPatternFilter, bool, error) {
|
||||
filter := coredata.NewCommonTrackerPatternFilter()
|
||||
hasSelector := false
|
||||
|
||||
if thirdParty != "" {
|
||||
id, err := resolveCommonThirdPartyID(ctx, conn, thirdParty)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
filter.WithCommonThirdPartyID(&id)
|
||||
hasSelector = true
|
||||
}
|
||||
|
||||
if trackerType != "" {
|
||||
tt := coredata.TrackerType(trackerType)
|
||||
if !tt.IsValid() {
|
||||
return nil, false, fmt.Errorf("invalid --tracker-type value %q", trackerType)
|
||||
}
|
||||
|
||||
filter.WithTrackerType(&tt)
|
||||
hasSelector = true
|
||||
}
|
||||
|
||||
if keyword != "" {
|
||||
filter.WithKeyword(&keyword)
|
||||
hasSelector = true
|
||||
}
|
||||
|
||||
if state != "" {
|
||||
st, err := parseEnrichmentState(state)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
filter.WithState(&st)
|
||||
hasSelector = true
|
||||
}
|
||||
|
||||
return filter, hasSelector, nil
|
||||
}
|
||||
|
||||
func printSample(out io.Writer, ids []gid.GID) {
|
||||
const sampleSize = 10
|
||||
|
||||
for i, id := range ids {
|
||||
if i >= sampleSize {
|
||||
_, _ = fmt.Fprintf(out, " ... and %d more\n", len(ids)-sampleSize)
|
||||
break
|
||||
}
|
||||
|
||||
_, _ = fmt.Fprintf(out, " %s\n", id.String())
|
||||
}
|
||||
}
|
||||
138
pkg/proboctl/commontrackerpattern/show.go
Normal file
138
pkg/proboctl/commontrackerpattern/show.go
Normal file
@@ -0,0 +1,138 @@
|
||||
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||
//
|
||||
// Permission to use, copy, modify, and/or distribute this software for any
|
||||
// purpose with or without fee is hereby granted, provided that the above
|
||||
// copyright notice and this permission notice appear in all copies.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||
// PERFORMANCE OF THIS SOFTWARE.
|
||||
|
||||
package commontrackerpattern
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/charmbracelet/lipgloss"
|
||||
"github.com/spf13/cobra"
|
||||
"go.gearno.de/kit/pg"
|
||||
clicmdutil "go.probo.inc/probo/pkg/cmd/cmdutil"
|
||||
"go.probo.inc/probo/pkg/coredata"
|
||||
"go.probo.inc/probo/pkg/gid"
|
||||
"go.probo.inc/probo/pkg/proboctl/cmdutil"
|
||||
)
|
||||
|
||||
func newCmdShow(f *cmdutil.Factory) *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "show <gid>",
|
||||
Short: "Show a single common tracker pattern by GID",
|
||||
Args: cobra.ExactArgs(1),
|
||||
}
|
||||
|
||||
output := clicmdutil.AddOutputFlag(cmd)
|
||||
|
||||
cmd.RunE = func(cmd *cobra.Command, args []string) error {
|
||||
if err := clicmdutil.ValidateOutputFlag(output); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
id, err := gid.ParseGID(args[0])
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid GID %q: %w", args[0], err)
|
||||
}
|
||||
|
||||
pgClient, err := f.PgClient()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var (
|
||||
pattern coredata.CommonTrackerPattern
|
||||
thirdPartyName string
|
||||
)
|
||||
|
||||
if err := pgClient.WithConn(
|
||||
cmd.Context(),
|
||||
func(ctx context.Context, conn pg.Querier) error {
|
||||
if err := pattern.LoadByID(ctx, conn, id); err != nil {
|
||||
if errors.Is(err, coredata.ErrResourceNotFound) {
|
||||
return fmt.Errorf("no common tracker pattern found for %q", args[0])
|
||||
}
|
||||
|
||||
return fmt.Errorf("cannot load common tracker pattern: %w", err)
|
||||
}
|
||||
|
||||
if pattern.CommonThirdPartyID != nil {
|
||||
var party coredata.CommonThirdParty
|
||||
if err := party.LoadByID(ctx, conn, *pattern.CommonThirdPartyID); err == nil {
|
||||
thirdPartyName = party.Name
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if *output == clicmdutil.OutputJSON {
|
||||
return clicmdutil.PrintJSON(f.IOStreams.Out, pattern)
|
||||
}
|
||||
|
||||
return renderPatternDetail(f, pattern, thirdPartyName)
|
||||
}
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
func renderPatternDetail(f *cmdutil.Factory, p coredata.CommonTrackerPattern, thirdPartyName string) error {
|
||||
out := f.IOStreams.Out
|
||||
label := lipgloss.NewStyle().Foreground(lipgloss.Color("242")).Width(20)
|
||||
|
||||
row := func(name, value string) {
|
||||
_, _ = fmt.Fprintf(out, "%s%s\n", label.Render(name), value)
|
||||
}
|
||||
|
||||
row("ID:", p.ID.String())
|
||||
row("Tracker type:", string(p.TrackerType))
|
||||
row("Match type:", string(p.MatchType))
|
||||
row("Pattern:", p.Pattern)
|
||||
row("Confidence:", fmt.Sprintf("%.2f", p.Confidence))
|
||||
row("State:", enrichmentState(&p))
|
||||
|
||||
if p.MaxAgeSeconds != nil {
|
||||
row("Max age (s):", fmt.Sprintf("%d", *p.MaxAgeSeconds))
|
||||
}
|
||||
|
||||
if p.CommonThirdPartyID != nil {
|
||||
row("Third party:", fmt.Sprintf("%s (%s)", thirdPartyName, p.CommonThirdPartyID.String()))
|
||||
} else {
|
||||
row("Third party:", "(unlinked)")
|
||||
}
|
||||
|
||||
description := p.Description
|
||||
if description == "" {
|
||||
description = "(none)"
|
||||
}
|
||||
|
||||
row("Description:", description)
|
||||
|
||||
if p.EnrichmentRequestedAt != nil {
|
||||
row("Enrichment queued:", p.EnrichmentRequestedAt.Format("2006-01-02 15:04:05"))
|
||||
}
|
||||
|
||||
if p.EnrichedAt != nil {
|
||||
row("Enriched at:", p.EnrichedAt.Format("2006-01-02 15:04:05"))
|
||||
}
|
||||
|
||||
row("Created:", p.CreatedAt.Format("2006-01-02 15:04:05"))
|
||||
row("Updated:", p.UpdatedAt.Format("2006-01-02 15:04:05"))
|
||||
|
||||
return nil
|
||||
}
|
||||
104
pkg/proboctl/commontrackerpattern/stats.go
Normal file
104
pkg/proboctl/commontrackerpattern/stats.go
Normal file
@@ -0,0 +1,104 @@
|
||||
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||
//
|
||||
// Permission to use, copy, modify, and/or distribute this software for any
|
||||
// purpose with or without fee is hereby granted, provided that the above
|
||||
// copyright notice and this permission notice appear in all copies.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||
// PERFORMANCE OF THIS SOFTWARE.
|
||||
|
||||
package commontrackerpattern
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"go.gearno.de/kit/pg"
|
||||
clicmdutil "go.probo.inc/probo/pkg/cmd/cmdutil"
|
||||
"go.probo.inc/probo/pkg/coredata"
|
||||
"go.probo.inc/probo/pkg/proboctl/cmdutil"
|
||||
)
|
||||
|
||||
func newCmdStats(f *cmdutil.Factory) *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "stats",
|
||||
Short: "Summarize the common tracker pattern catalog by enrichment and link state",
|
||||
Args: cobra.NoArgs,
|
||||
}
|
||||
|
||||
output := clicmdutil.AddOutputFlag(cmd)
|
||||
|
||||
cmd.RunE = func(cmd *cobra.Command, args []string) error {
|
||||
if err := clicmdutil.ValidateOutputFlag(output); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
pgClient, err := f.PgClient()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
stats := map[string]int{}
|
||||
|
||||
if err := pgClient.WithConn(
|
||||
cmd.Context(),
|
||||
func(ctx context.Context, conn pg.Querier) error {
|
||||
counts := []struct {
|
||||
key string
|
||||
filter *coredata.CommonTrackerPatternFilter
|
||||
}{
|
||||
{"total", coredata.NewCommonTrackerPatternFilter()},
|
||||
{"queued", coredata.NewCommonTrackerPatternFilter().WithState(refState(coredata.CommonTrackerPatternEnrichmentStateQueued))},
|
||||
{"enriched", coredata.NewCommonTrackerPatternFilter().WithState(refState(coredata.CommonTrackerPatternEnrichmentStateEnriched))},
|
||||
{"unenriched", coredata.NewCommonTrackerPatternFilter().WithState(refState(coredata.CommonTrackerPatternEnrichmentStateUnenriched))},
|
||||
{"linked", coredata.NewCommonTrackerPatternFilter().WithLinked(refBool(true))},
|
||||
{"unlinked", coredata.NewCommonTrackerPatternFilter().WithLinked(refBool(false))},
|
||||
}
|
||||
|
||||
for _, c := range counts {
|
||||
var ps coredata.CommonTrackerPatterns
|
||||
|
||||
n, err := ps.CountAll(ctx, conn, c.filter)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
stats[c.key] = n
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if *output == clicmdutil.OutputJSON {
|
||||
return clicmdutil.PrintJSON(f.IOStreams.Out, stats)
|
||||
}
|
||||
|
||||
table := clicmdutil.NewTable("METRIC", "COUNT")
|
||||
for _, key := range []string{"total", "queued", "enriched", "unenriched", "linked", "unlinked"} {
|
||||
table.Row(key, fmt.Sprintf("%d", stats[key]))
|
||||
}
|
||||
|
||||
_, _ = fmt.Fprintln(f.IOStreams.Out, table.Render())
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
func refState(s coredata.CommonTrackerPatternEnrichmentState) *coredata.CommonTrackerPatternEnrichmentState {
|
||||
return &s
|
||||
}
|
||||
|
||||
func refBool(b bool) *bool {
|
||||
return &b
|
||||
}
|
||||
Reference in New Issue
Block a user