Add proboctl catalog and banner reset commands
Add operator commands to proboctl for iterating on the cookie-banner agents. The global catalog groups (common-tracker-pattern, common-third-party) list/filter/sort/show the catalogs using the shared coredata cursor layer, and common-tracker-pattern reenrich re-describes selected rows by running the enricher in-process (so it completes synchronously rather than racing the async queue); a --cfg-file flag reuses probod's config to wire the agent. --linked-banner/--linked-org target exactly the catalog rows a banner or org depends on. The cookie-banner reset-trackers command is tenant-scoped (it derives a coredata.Scope from the banner/org GID) and rebuilds a banner's uncategorised, non-excluded patterns from detected_trackers, decomposing derived globs back into exacts, then re-arms the analysis and mapping workers. --mapping-only skips the rebuild. A DB-backed test covers the rebuild, link clearing, and preservation of categorised/excluded patterns. Signed-off-by: Émile Ré <emile@probo.com>
This commit is contained in:
205
pkg/cookiebanner/reset_trackers.go
Normal file
205
pkg/cookiebanner/reset_trackers.go
Normal file
@@ -0,0 +1,205 @@
|
|||||||
|
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||||
|
//
|
||||||
|
// Permission to use, copy, modify, and/or distribute this software for any
|
||||||
|
// purpose with or without fee is hereby granted, provided that the above
|
||||||
|
// copyright notice and this permission notice appear in all copies.
|
||||||
|
//
|
||||||
|
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||||
|
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||||
|
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||||
|
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||||
|
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||||
|
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||||
|
// PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
|
||||||
|
package cookiebanner
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"go.gearno.de/kit/pg"
|
||||||
|
"go.probo.inc/probo/pkg/coredata"
|
||||||
|
"go.probo.inc/probo/pkg/gid"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ResetTrackersResult summarizes what a banner reset changed.
|
||||||
|
type ResetTrackersResult struct {
|
||||||
|
PatternsReset int64
|
||||||
|
GlobsDecomposed int
|
||||||
|
ExactsCreated int
|
||||||
|
DetectionsRelinked int
|
||||||
|
AnalysisRequested bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// ResetBannerTrackers re-arms the tracker pipeline for a banner's
|
||||||
|
// uncategorised, non-excluded patterns. It is an operator action
|
||||||
|
// (proboctl), tenant-scoped via the provided Scoper.
|
||||||
|
//
|
||||||
|
// With mappingOnly, it only clears each pattern's catalog/vendor links
|
||||||
|
// and re-arms mapping, for iterating on the mapping agent without
|
||||||
|
// touching analysis.
|
||||||
|
//
|
||||||
|
// The full reset additionally rebuilds the raw exact patterns from the
|
||||||
|
// surviving detected_trackers and re-arms pattern analysis, so the
|
||||||
|
// analysis worker re-derives globs from scratch: the pattern-analysis
|
||||||
|
// worker consumes (deletes) exact patterns when it merges them into
|
||||||
|
// globs, so the only way to re-run analysis is to reconstruct the exacts
|
||||||
|
// from detections. Each uncategorised, non-excluded glob is decomposed -
|
||||||
|
// every detection it covers becomes (or rejoins) an exact pattern keyed
|
||||||
|
// by its identifier - and the now-empty glob is deleted. User-categorised
|
||||||
|
// and excluded patterns are never touched.
|
||||||
|
func ResetBannerTrackers(
|
||||||
|
ctx context.Context,
|
||||||
|
pgClient *pg.Client,
|
||||||
|
scope coredata.Scoper,
|
||||||
|
bannerID gid.GID,
|
||||||
|
mappingOnly bool,
|
||||||
|
) (ResetTrackersResult, error) {
|
||||||
|
var result ResetTrackersResult
|
||||||
|
|
||||||
|
err := pgClient.WithTx(
|
||||||
|
ctx,
|
||||||
|
func(ctx context.Context, tx pg.Tx) error {
|
||||||
|
var uncategorised coredata.CookieCategory
|
||||||
|
if err := uncategorised.LoadUncategorisedByCookieBannerID(ctx, tx, scope, bannerID); err != nil {
|
||||||
|
return fmt.Errorf("cannot load uncategorised category: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !mappingOnly {
|
||||||
|
if err := decomposeGlobs(ctx, tx, scope, bannerID, uncategorised.ID, &result); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var patterns coredata.TrackerPatterns
|
||||||
|
|
||||||
|
reset, err := patterns.ResetAndRequestMappingByCookieCategoryID(ctx, tx, scope, uncategorised.ID)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("cannot reset and request mapping: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
result.PatternsReset = reset
|
||||||
|
|
||||||
|
if !mappingOnly {
|
||||||
|
banner := coredata.CookieBanner{ID: bannerID}
|
||||||
|
if err := banner.SetPatternAnalysisRequested(ctx, tx); err != nil {
|
||||||
|
return fmt.Errorf("cannot request pattern analysis: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
result.AnalysisRequested = true
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return ResetTrackersResult{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// decomposeGlobs turns every uncategorised, non-excluded glob pattern of
|
||||||
|
// the banner back into exact patterns derived from its detected trackers,
|
||||||
|
// relinking each detection to its exact and deleting the emptied glob.
|
||||||
|
func decomposeGlobs(
|
||||||
|
ctx context.Context,
|
||||||
|
tx pg.Tx,
|
||||||
|
scope coredata.Scoper,
|
||||||
|
bannerID gid.GID,
|
||||||
|
uncategorisedID gid.GID,
|
||||||
|
result *ResetTrackersResult,
|
||||||
|
) error {
|
||||||
|
globMatchType := coredata.TrackerPatternMatchTypeGlob
|
||||||
|
notExcluded := false
|
||||||
|
|
||||||
|
var globs coredata.TrackerPatterns
|
||||||
|
if err := globs.LoadAllByCookieBannerID(
|
||||||
|
ctx,
|
||||||
|
tx,
|
||||||
|
scope,
|
||||||
|
bannerID,
|
||||||
|
coredata.NewTrackerPatternFilter(&globMatchType, &uncategorisedID, ¬Excluded),
|
||||||
|
nil,
|
||||||
|
); err != nil {
|
||||||
|
return fmt.Errorf("cannot load glob patterns: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, glob := range globs {
|
||||||
|
var detections coredata.DetectedTrackers
|
||||||
|
if err := detections.LoadAllByTrackerPatternID(ctx, tx, scope, glob.ID); err != nil {
|
||||||
|
return fmt.Errorf("cannot load detections for glob %q: %w", glob.Pattern, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, detection := range detections {
|
||||||
|
exactID, created, err := ensureExactPattern(ctx, tx, scope, glob, uncategorisedID, detection)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if created {
|
||||||
|
result.ExactsCreated++
|
||||||
|
}
|
||||||
|
|
||||||
|
detection.TrackerPatternID = &exactID
|
||||||
|
if err := detection.UpdateTrackerPatternID(ctx, tx, scope); err != nil {
|
||||||
|
return fmt.Errorf("cannot relink detection %s: %w", detection.ID, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
result.DetectionsRelinked++
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := glob.Delete(ctx, tx, scope); err != nil {
|
||||||
|
return fmt.Errorf("cannot delete glob pattern %q: %w", glob.Pattern, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
result.GlobsDecomposed++
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ensureExactPattern finds or creates the exact pattern for a detection
|
||||||
|
// (keyed by banner, tracker type, identifier, and max-age) in the
|
||||||
|
// uncategorised category, returning its id and whether it was created.
|
||||||
|
func ensureExactPattern(
|
||||||
|
ctx context.Context,
|
||||||
|
tx pg.Tx,
|
||||||
|
scope coredata.Scoper,
|
||||||
|
glob *coredata.TrackerPattern,
|
||||||
|
uncategorisedID gid.GID,
|
||||||
|
detection *coredata.DetectedTracker,
|
||||||
|
) (gid.GID, bool, error) {
|
||||||
|
now := time.Now()
|
||||||
|
|
||||||
|
exact := &coredata.TrackerPattern{
|
||||||
|
ID: gid.New(glob.CookieBannerID.TenantID(), coredata.TrackerPatternEntityType),
|
||||||
|
OrganizationID: glob.OrganizationID,
|
||||||
|
CookieBannerID: glob.CookieBannerID,
|
||||||
|
CookieCategoryID: uncategorisedID,
|
||||||
|
TrackerType: detection.TrackerType,
|
||||||
|
Pattern: detection.Identifier,
|
||||||
|
MatchType: coredata.TrackerPatternMatchTypeExact,
|
||||||
|
DisplayName: detection.Identifier,
|
||||||
|
MaxAgeSeconds: detection.MaxAgeSeconds,
|
||||||
|
Source: detection.Source,
|
||||||
|
MappingRequestedAt: &now,
|
||||||
|
CreatedAt: now,
|
||||||
|
UpdatedAt: now,
|
||||||
|
}
|
||||||
|
|
||||||
|
created, err := exact.InsertIfNotExists(ctx, tx, scope)
|
||||||
|
if err != nil {
|
||||||
|
return gid.GID{}, false, fmt.Errorf("cannot insert exact pattern %q: %w", detection.Identifier, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !created {
|
||||||
|
if err := exact.LoadByBannerIDTypeAndPattern(ctx, tx, scope, glob.CookieBannerID, detection.TrackerType, detection.Identifier, detection.MaxAgeSeconds); err != nil {
|
||||||
|
return gid.GID{}, false, fmt.Errorf("cannot load existing exact pattern %q: %w", detection.Identifier, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return exact.ID, created, nil
|
||||||
|
}
|
||||||
179
pkg/cookiebanner/reset_trackers_test.go
Normal file
179
pkg/cookiebanner/reset_trackers_test.go
Normal file
@@ -0,0 +1,179 @@
|
|||||||
|
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||||
|
//
|
||||||
|
// Permission to use, copy, modify, and/or distribute this software for any
|
||||||
|
// purpose with or without fee is hereby granted, provided that the above
|
||||||
|
// copyright notice and this permission notice appear in all copies.
|
||||||
|
//
|
||||||
|
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||||
|
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||||
|
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||||
|
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||||
|
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||||
|
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||||
|
// PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
|
||||||
|
package cookiebanner
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"go.gearno.de/kit/pg"
|
||||||
|
"go.probo.inc/probo/pkg/coredata"
|
||||||
|
"go.probo.inc/probo/pkg/gid"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestResetBannerTrackers_FullRebuild seeds a banner with an
|
||||||
|
// uncategorised glob covering two detections, an uncategorised exact
|
||||||
|
// carrying catalog/vendor links, a categorised exact, and an excluded
|
||||||
|
// exact. A full reset must: decompose the glob into per-identifier
|
||||||
|
// exacts and relink its detections, clear links on the surviving
|
||||||
|
// uncategorised exact and re-arm its mapping, preserve the categorised
|
||||||
|
// and excluded patterns, and arm pattern analysis on the banner.
|
||||||
|
func TestResetBannerTrackers_FullRebuild(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
client := newTestPgClient(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
fx := seedWorkerFixture(t, ctx, client)
|
||||||
|
|
||||||
|
thirdPartyID := seedThirdParty(t, ctx, client, fx, "Reset Vendor")
|
||||||
|
commonPatternID := seedCommonTrackerPattern(t, ctx, client, "ga_linked")
|
||||||
|
|
||||||
|
glob := newGlobInCategory(fx, "_ga_*", fx.uncategorisedID, coredata.CookieSourceScript, nil)
|
||||||
|
|
||||||
|
linkedExact := newExactPattern(fx, "linked_cookie", fx.uncategorisedID, coredata.CookieSourcePreExisting, nil)
|
||||||
|
linkedExact.CommonTrackerPatternID = &commonPatternID
|
||||||
|
linkedExact.ThirdPartyID = &thirdPartyID
|
||||||
|
linkedExact.Description = "stale description"
|
||||||
|
|
||||||
|
categorised := newExactPattern(fx, "categorised_cookie", fx.normalCategoryID, coredata.CookieSourceScript, nil)
|
||||||
|
|
||||||
|
excluded := newExactPattern(fx, "excluded_cookie", fx.uncategorisedID, coredata.CookieSourceScript, nil)
|
||||||
|
excluded.Excluded = true
|
||||||
|
|
||||||
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
||||||
|
|
||||||
|
require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error {
|
||||||
|
for _, p := range []*coredata.TrackerPattern{glob, linkedExact, categorised, excluded} {
|
||||||
|
if err := p.Insert(ctx, tx, fx.scope); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, identifier := range []string{"_ga_ABC", "_ga_DEF"} {
|
||||||
|
detection := &coredata.DetectedTracker{
|
||||||
|
ID: gid.New(fx.scope.GetTenantID(), coredata.DetectedTrackerEntityType),
|
||||||
|
CookieBannerID: fx.banner.ID,
|
||||||
|
TrackerPatternID: &glob.ID,
|
||||||
|
TrackerType: coredata.TrackerTypeCookie,
|
||||||
|
Identifier: identifier,
|
||||||
|
Source: ref(coredata.CookieSourceScript),
|
||||||
|
LastDetectedAt: now,
|
||||||
|
CreatedAt: now,
|
||||||
|
UpdatedAt: now,
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := detection.Upsert(ctx, tx, fx.scope); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}))
|
||||||
|
|
||||||
|
result, err := ResetBannerTrackers(ctx, client, fx.scope, fx.banner.ID, false)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
require.Equal(t, 1, result.GlobsDecomposed)
|
||||||
|
require.Equal(t, 2, result.ExactsCreated)
|
||||||
|
require.Equal(t, 2, result.DetectionsRelinked)
|
||||||
|
require.True(t, result.AnalysisRequested)
|
||||||
|
// linked_cookie + _ga_ABC + _ga_DEF (excluded and categorised are untouched).
|
||||||
|
require.Equal(t, int64(3), result.PatternsReset)
|
||||||
|
|
||||||
|
require.NoError(t, client.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error {
|
||||||
|
// The glob is gone.
|
||||||
|
var goneGlob coredata.TrackerPattern
|
||||||
|
err := goneGlob.LoadByBannerIDTypeAndPattern(ctx, conn, fx.scope, fx.banner.ID, coredata.TrackerTypeCookie, "_ga_*", nil)
|
||||||
|
require.ErrorIs(t, err, coredata.ErrResourceNotFound)
|
||||||
|
|
||||||
|
// Each detection identifier is now its own exact, with mapping armed
|
||||||
|
// and no links, and the detection relinked to it.
|
||||||
|
for _, identifier := range []string{"_ga_ABC", "_ga_DEF"} {
|
||||||
|
var exact coredata.TrackerPattern
|
||||||
|
require.NoError(t, exact.LoadByBannerIDTypeAndPattern(ctx, conn, fx.scope, fx.banner.ID, coredata.TrackerTypeCookie, identifier, nil))
|
||||||
|
require.Equal(t, coredata.TrackerPatternMatchTypeExact, exact.MatchType)
|
||||||
|
require.Equal(t, fx.uncategorisedID, exact.CookieCategoryID)
|
||||||
|
require.Nil(t, exact.CommonTrackerPatternID)
|
||||||
|
require.Nil(t, exact.ThirdPartyID)
|
||||||
|
require.NotNil(t, exact.MappingRequestedAt)
|
||||||
|
|
||||||
|
var detections coredata.DetectedTrackers
|
||||||
|
require.NoError(t, detections.LoadAllByTrackerPatternID(ctx, conn, fx.scope, exact.ID))
|
||||||
|
require.Len(t, detections, 1)
|
||||||
|
require.Equal(t, identifier, detections[0].Identifier)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The surviving uncategorised exact had its links and copied
|
||||||
|
// description cleared and mapping re-armed.
|
||||||
|
var survivor coredata.TrackerPattern
|
||||||
|
require.NoError(t, survivor.LoadByBannerIDTypeAndPattern(ctx, conn, fx.scope, fx.banner.ID, coredata.TrackerTypeCookie, "linked_cookie", nil))
|
||||||
|
require.Nil(t, survivor.CommonTrackerPatternID)
|
||||||
|
require.Nil(t, survivor.ThirdPartyID)
|
||||||
|
require.Empty(t, survivor.Description)
|
||||||
|
require.NotNil(t, survivor.MappingRequestedAt)
|
||||||
|
|
||||||
|
// The categorised pattern is untouched.
|
||||||
|
var categorisedRow coredata.TrackerPattern
|
||||||
|
require.NoError(t, categorisedRow.LoadByBannerIDTypeAndPattern(ctx, conn, fx.scope, fx.banner.ID, coredata.TrackerTypeCookie, "categorised_cookie", nil))
|
||||||
|
require.Equal(t, fx.normalCategoryID, categorisedRow.CookieCategoryID)
|
||||||
|
|
||||||
|
// The excluded pattern is preserved.
|
||||||
|
var excludedRow coredata.TrackerPattern
|
||||||
|
require.NoError(t, excludedRow.LoadByBannerIDTypeAndPattern(ctx, conn, fx.scope, fx.banner.ID, coredata.TrackerTypeCookie, "excluded_cookie", nil))
|
||||||
|
require.True(t, excludedRow.Excluded)
|
||||||
|
|
||||||
|
// Pattern analysis is armed on the banner.
|
||||||
|
var banner coredata.CookieBanner
|
||||||
|
require.NoError(t, banner.LoadByID(ctx, conn, fx.scope, fx.banner.ID))
|
||||||
|
require.NotNil(t, banner.PatternAnalysisRequestedAt)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
func ref[T any](v T) *T {
|
||||||
|
return &v
|
||||||
|
}
|
||||||
|
|
||||||
|
func seedCommonTrackerPattern(t *testing.T, ctx context.Context, client *pg.Client, pattern string) gid.GID {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
||||||
|
cp := coredata.CommonTrackerPattern{
|
||||||
|
ID: gid.New(gid.NilTenant, coredata.CommonTrackerPatternEntityType),
|
||||||
|
TrackerType: coredata.TrackerTypeCookie,
|
||||||
|
Pattern: pattern,
|
||||||
|
MatchType: coredata.TrackerPatternMatchTypeExact,
|
||||||
|
Description: "seeded",
|
||||||
|
Confidence: 1,
|
||||||
|
CreatedAt: now,
|
||||||
|
UpdatedAt: now,
|
||||||
|
}
|
||||||
|
|
||||||
|
require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error {
|
||||||
|
return cp.Insert(ctx, tx)
|
||||||
|
}))
|
||||||
|
|
||||||
|
t.Cleanup(func() {
|
||||||
|
_ = client.WithTx(context.Background(), func(ctx context.Context, tx pg.Tx) error {
|
||||||
|
_, err := tx.Exec(ctx, `DELETE FROM common_tracker_patterns WHERE id = $1`, cp.ID)
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
return cp.ID
|
||||||
|
}
|
||||||
@@ -16,16 +16,25 @@ package cmdutil
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
|
"go.gearno.de/kit/log"
|
||||||
"go.gearno.de/kit/pg"
|
"go.gearno.de/kit/pg"
|
||||||
|
"go.opentelemetry.io/otel/trace/noop"
|
||||||
|
"go.probo.inc/probo/pkg/agentsbuild"
|
||||||
"go.probo.inc/probo/pkg/cmd/iostreams"
|
"go.probo.inc/probo/pkg/cmd/iostreams"
|
||||||
|
"go.probo.inc/probo/pkg/cookiebanner"
|
||||||
"go.probo.inc/probo/pkg/proboctl/pgconn"
|
"go.probo.inc/probo/pkg/proboctl/pgconn"
|
||||||
|
"go.probo.inc/probo/pkg/probodconfig"
|
||||||
|
"sigs.k8s.io/yaml"
|
||||||
)
|
)
|
||||||
|
|
||||||
type Factory struct {
|
type Factory struct {
|
||||||
IOStreams *iostreams.IOStreams
|
IOStreams *iostreams.IOStreams
|
||||||
Version string
|
Version string
|
||||||
PgDSN string
|
PgDSN string
|
||||||
|
CfgFile string
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *Factory) PgClient() (*pg.Client, error) {
|
func (f *Factory) PgClient() (*pg.Client, error) {
|
||||||
@@ -35,3 +44,56 @@ func (f *Factory) PgClient() (*pg.Client, error) {
|
|||||||
|
|
||||||
return pgconn.NewPgClientFromDSN(f.PgDSN)
|
return pgconn.NewPgClientFromDSN(f.PgDSN)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ProbodConfig loads the shared probod configuration file (--cfg-file).
|
||||||
|
// It reuses the exact file, struct, and json-tagged (un)marshaling probod
|
||||||
|
// uses, so proboctl and probod stay consistent.
|
||||||
|
func (f *Factory) ProbodConfig() (probodconfig.Config, error) {
|
||||||
|
if f.CfgFile == "" {
|
||||||
|
return probodconfig.Config{}, fmt.Errorf("set --cfg-file to the probod config file")
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := os.ReadFile(f.CfgFile)
|
||||||
|
if err != nil {
|
||||||
|
return probodconfig.Config{}, fmt.Errorf("cannot read config file %q: %w", f.CfgFile, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var full probodconfig.FullConfig
|
||||||
|
if err := yaml.Unmarshal(data, &full); err != nil {
|
||||||
|
return probodconfig.Config{}, fmt.Errorf("cannot parse config file %q: %w", f.CfgFile, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return full.Probod, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// TrackerAgentsConfig builds the tracker-agents config (LLM client +
|
||||||
|
// Firecrawl key) from the shared probod config for in-process agent
|
||||||
|
// execution, e.g. synchronous common-pattern re-enrichment. It errors
|
||||||
|
// when no LLM provider is configured.
|
||||||
|
func (f *Factory) TrackerAgentsConfig() (cookiebanner.TrackerAgentsConfig, error) {
|
||||||
|
cfg, err := f.ProbodConfig()
|
||||||
|
if err != nil {
|
||||||
|
return cookiebanner.TrackerAgentsConfig{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
logger := log.NewLogger(
|
||||||
|
log.WithName("proboctl"),
|
||||||
|
log.WithOutput(f.IOStreams.ErrOut),
|
||||||
|
)
|
||||||
|
|
||||||
|
trackerCfg, _, err := agentsbuild.BuildTrackerAgentsConfig(
|
||||||
|
cfg,
|
||||||
|
logger,
|
||||||
|
noop.NewTracerProvider(),
|
||||||
|
prometheus.NewRegistry(),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return cookiebanner.TrackerAgentsConfig{}, fmt.Errorf("cannot build tracker agents config: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if trackerCfg.LLMClient == nil {
|
||||||
|
return cookiebanner.TrackerAgentsConfig{}, fmt.Errorf("no LLM provider configured; set llm.tracker-mapping.provider in %q", f.CfgFile)
|
||||||
|
}
|
||||||
|
|
||||||
|
return trackerCfg, nil
|
||||||
|
}
|
||||||
|
|||||||
77
pkg/proboctl/cmdutil/paginate.go
Normal file
77
pkg/proboctl/cmdutil/paginate.go
Normal file
@@ -0,0 +1,77 @@
|
|||||||
|
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||||
|
//
|
||||||
|
// Permission to use, copy, modify, and/or distribute this software for any
|
||||||
|
// purpose with or without fee is hereby granted, provided that the above
|
||||||
|
// copyright notice and this permission notice appear in all copies.
|
||||||
|
//
|
||||||
|
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||||
|
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||||
|
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||||
|
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||||
|
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||||
|
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||||
|
// PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
|
||||||
|
package cmdutil
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
|
"go.probo.inc/probo/pkg/page"
|
||||||
|
)
|
||||||
|
|
||||||
|
// paginatePageSize is the per-request fetch size used to walk a cursor
|
||||||
|
// connection. It is independent of the caller's --limit, which only
|
||||||
|
// bounds the total returned.
|
||||||
|
const paginatePageSize = 100
|
||||||
|
|
||||||
|
// Paginate walks a cursor-paginated coredata Load into a single slice,
|
||||||
|
// following the keyset cursor until the caller's limit is reached or the
|
||||||
|
// connection is exhausted. A limit <= 0 returns every matching row. This
|
||||||
|
// is the proboctl-side counterpart of the API's connection resolvers: it
|
||||||
|
// drives page.NewCursor + page.NewPage against the same coredata Load
|
||||||
|
// methods, so a future proboctl API can reuse the data layer untouched.
|
||||||
|
func Paginate[E page.Paginable[F], F page.OrderField](
|
||||||
|
ctx context.Context,
|
||||||
|
orderBy page.OrderBy[F],
|
||||||
|
limit int,
|
||||||
|
load func(ctx context.Context, cursor *page.Cursor[F]) ([]E, error),
|
||||||
|
) ([]E, error) {
|
||||||
|
var (
|
||||||
|
result []E
|
||||||
|
from *page.CursorKey
|
||||||
|
)
|
||||||
|
|
||||||
|
for {
|
||||||
|
size := paginatePageSize
|
||||||
|
if limit > 0 {
|
||||||
|
remaining := limit - len(result)
|
||||||
|
if remaining <= 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
if remaining < size {
|
||||||
|
size = remaining
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
cursor := page.NewCursor(size, from, page.Head, orderBy)
|
||||||
|
|
||||||
|
rows, err := load(ctx, cursor)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
p := page.NewPage(rows, cursor)
|
||||||
|
result = append(result, p.Data...)
|
||||||
|
|
||||||
|
if !p.Info.HasNext || len(p.Data) == 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
key := p.Data[len(p.Data)-1].CursorKey(orderBy.Field)
|
||||||
|
from = &key
|
||||||
|
}
|
||||||
|
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
109
pkg/proboctl/commonthirdparty/commonthirdparty.go
vendored
Normal file
109
pkg/proboctl/commonthirdparty/commonthirdparty.go
vendored
Normal file
@@ -0,0 +1,109 @@
|
|||||||
|
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||||
|
//
|
||||||
|
// Permission to use, copy, modify, and/or distribute this software for any
|
||||||
|
// purpose with or without fee is hereby granted, provided that the above
|
||||||
|
// copyright notice and this permission notice appear in all copies.
|
||||||
|
//
|
||||||
|
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||||
|
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||||
|
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||||
|
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||||
|
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||||
|
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||||
|
// PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
|
||||||
|
package commonthirdparty
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
"go.gearno.de/kit/pg"
|
||||||
|
"go.probo.inc/probo/pkg/coredata"
|
||||||
|
"go.probo.inc/probo/pkg/gid"
|
||||||
|
"go.probo.inc/probo/pkg/page"
|
||||||
|
"go.probo.inc/probo/pkg/proboctl/cmdutil"
|
||||||
|
)
|
||||||
|
|
||||||
|
// NewCmdCommonThirdParty is the entry point for inspecting the global
|
||||||
|
// common third party catalog.
|
||||||
|
func NewCmdCommonThirdParty(f *cmdutil.Factory) *cobra.Command {
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: "common-third-party <command>",
|
||||||
|
Aliases: []string{"ctp3"},
|
||||||
|
Short: "Inspect the global common third party catalog",
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd.AddCommand(newCmdList(f))
|
||||||
|
cmd.AddCommand(newCmdShow(f))
|
||||||
|
cmd.AddCommand(newCmdDomains(f))
|
||||||
|
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolveCommonThirdParty loads a common third party by GID or slug.
|
||||||
|
func resolveCommonThirdParty(ctx context.Context, conn pg.Querier, value string) (coredata.CommonThirdParty, error) {
|
||||||
|
var party coredata.CommonThirdParty
|
||||||
|
|
||||||
|
if id, err := gid.ParseGID(value); err == nil {
|
||||||
|
if err := party.LoadByID(ctx, conn, id); err != nil {
|
||||||
|
if errors.Is(err, coredata.ErrResourceNotFound) {
|
||||||
|
return party, fmt.Errorf("no common third party found for %q", value)
|
||||||
|
}
|
||||||
|
|
||||||
|
return party, fmt.Errorf("cannot load common third party: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return party, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := party.LoadBySlug(ctx, conn, value); err != nil {
|
||||||
|
if errors.Is(err, coredata.ErrResourceNotFound) {
|
||||||
|
return party, fmt.Errorf("no common third party found for %q (pass a slug or GID)", value)
|
||||||
|
}
|
||||||
|
|
||||||
|
return party, fmt.Errorf("cannot load common third party: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return party, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseOrderBy maps the --sort/--order flags to a page.OrderBy. Name
|
||||||
|
// defaults to ascending; the time fields default to descending.
|
||||||
|
func parseOrderBy(sort, order string) (page.OrderBy[coredata.CommonThirdPartyOrderField], error) {
|
||||||
|
var (
|
||||||
|
field coredata.CommonThirdPartyOrderField
|
||||||
|
defaultDesc bool
|
||||||
|
zero page.OrderBy[coredata.CommonThirdPartyOrderField]
|
||||||
|
)
|
||||||
|
|
||||||
|
switch sort {
|
||||||
|
case "name":
|
||||||
|
field = coredata.CommonThirdPartyOrderFieldName
|
||||||
|
case "created":
|
||||||
|
field, defaultDesc = coredata.CommonThirdPartyOrderFieldCreatedAt, true
|
||||||
|
case "updated":
|
||||||
|
field, defaultDesc = coredata.CommonThirdPartyOrderFieldUpdatedAt, true
|
||||||
|
default:
|
||||||
|
return zero, fmt.Errorf("invalid --sort value %q: valid values are name, created, updated", sort)
|
||||||
|
}
|
||||||
|
|
||||||
|
direction := page.OrderDirectionAsc
|
||||||
|
if defaultDesc {
|
||||||
|
direction = page.OrderDirectionDesc
|
||||||
|
}
|
||||||
|
|
||||||
|
switch order {
|
||||||
|
case "":
|
||||||
|
case "asc":
|
||||||
|
direction = page.OrderDirectionAsc
|
||||||
|
case "desc":
|
||||||
|
direction = page.OrderDirectionDesc
|
||||||
|
default:
|
||||||
|
return zero, fmt.Errorf("invalid --order value %q: valid values are asc, desc", order)
|
||||||
|
}
|
||||||
|
|
||||||
|
return page.OrderBy[coredata.CommonThirdPartyOrderField]{Field: field, Direction: direction}, nil
|
||||||
|
}
|
||||||
87
pkg/proboctl/commonthirdparty/domains.go
vendored
Normal file
87
pkg/proboctl/commonthirdparty/domains.go
vendored
Normal file
@@ -0,0 +1,87 @@
|
|||||||
|
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||||
|
//
|
||||||
|
// Permission to use, copy, modify, and/or distribute this software for any
|
||||||
|
// purpose with or without fee is hereby granted, provided that the above
|
||||||
|
// copyright notice and this permission notice appear in all copies.
|
||||||
|
//
|
||||||
|
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||||
|
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||||
|
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||||
|
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||||
|
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||||
|
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||||
|
// PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
|
||||||
|
package commonthirdparty
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
"go.gearno.de/kit/pg"
|
||||||
|
clicmdutil "go.probo.inc/probo/pkg/cmd/cmdutil"
|
||||||
|
"go.probo.inc/probo/pkg/coredata"
|
||||||
|
"go.probo.inc/probo/pkg/proboctl/cmdutil"
|
||||||
|
)
|
||||||
|
|
||||||
|
func newCmdDomains(f *cmdutil.Factory) *cobra.Command {
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: "domains <gid|slug>",
|
||||||
|
Short: "List the domains of a common third party",
|
||||||
|
Args: cobra.ExactArgs(1),
|
||||||
|
}
|
||||||
|
|
||||||
|
output := clicmdutil.AddOutputFlag(cmd)
|
||||||
|
|
||||||
|
cmd.RunE = func(cmd *cobra.Command, args []string) error {
|
||||||
|
if err := clicmdutil.ValidateOutputFlag(output); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
pgClient, err := f.PgClient()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
var domains coredata.CommonThirdPartyDomains
|
||||||
|
|
||||||
|
if err := pgClient.WithConn(
|
||||||
|
cmd.Context(),
|
||||||
|
func(ctx context.Context, conn pg.Querier) error {
|
||||||
|
party, err := resolveCommonThirdParty(ctx, conn, args[0])
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := domains.LoadByCommonThirdPartyID(ctx, conn, party.ID); err != nil {
|
||||||
|
return fmt.Errorf("cannot load domains: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if *output == clicmdutil.OutputJSON {
|
||||||
|
return clicmdutil.PrintJSON(f.IOStreams.Out, domains)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(domains) == 0 {
|
||||||
|
_, _ = fmt.Fprintln(f.IOStreams.Out, "No domains found.")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
table := clicmdutil.NewTable("DOMAIN", "ID")
|
||||||
|
for _, d := range domains {
|
||||||
|
table.Row(d.Domain, d.ID.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
_, _ = fmt.Fprintln(f.IOStreams.Out, table.Render())
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
143
pkg/proboctl/commonthirdparty/list.go
vendored
Normal file
143
pkg/proboctl/commonthirdparty/list.go
vendored
Normal file
@@ -0,0 +1,143 @@
|
|||||||
|
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||||
|
//
|
||||||
|
// Permission to use, copy, modify, and/or distribute this software for any
|
||||||
|
// purpose with or without fee is hereby granted, provided that the above
|
||||||
|
// copyright notice and this permission notice appear in all copies.
|
||||||
|
//
|
||||||
|
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||||
|
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||||
|
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||||
|
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||||
|
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||||
|
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||||
|
// PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
|
||||||
|
package commonthirdparty
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
"go.gearno.de/kit/pg"
|
||||||
|
clicmdutil "go.probo.inc/probo/pkg/cmd/cmdutil"
|
||||||
|
"go.probo.inc/probo/pkg/coredata"
|
||||||
|
"go.probo.inc/probo/pkg/page"
|
||||||
|
"go.probo.inc/probo/pkg/proboctl/cmdutil"
|
||||||
|
)
|
||||||
|
|
||||||
|
func newCmdList(f *cmdutil.Factory) *cobra.Command {
|
||||||
|
var (
|
||||||
|
flagName string
|
||||||
|
flagCategory string
|
||||||
|
flagKeyword string
|
||||||
|
flagSort string
|
||||||
|
flagOrder string
|
||||||
|
flagLimit int
|
||||||
|
)
|
||||||
|
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: "list",
|
||||||
|
Short: "List common third parties with filters and sorting",
|
||||||
|
Args: cobra.NoArgs,
|
||||||
|
}
|
||||||
|
|
||||||
|
output := clicmdutil.AddOutputFlag(cmd)
|
||||||
|
|
||||||
|
cmd.Flags().StringVar(&flagName, "name", "", "Filter by name substring")
|
||||||
|
cmd.Flags().StringVar(&flagCategory, "category", "", "Filter by category")
|
||||||
|
cmd.Flags().StringVar(&flagKeyword, "keyword", "", "Filter by name/slug substring")
|
||||||
|
cmd.Flags().StringVar(&flagSort, "sort", "name", "Sort field: name, created, updated")
|
||||||
|
cmd.Flags().StringVar(&flagOrder, "order", "", "Sort order: asc, desc (default depends on field)")
|
||||||
|
cmd.Flags().IntVarP(&flagLimit, "limit", "L", 50, "Maximum rows to return (0 for all)")
|
||||||
|
|
||||||
|
cmd.RunE = func(cmd *cobra.Command, args []string) error {
|
||||||
|
if err := clicmdutil.ValidateOutputFlag(output); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
orderBy, err := parseOrderBy(flagSort, flagOrder)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
filter := coredata.NewCommonThirdPartyFilter(optionalString(flagName))
|
||||||
|
|
||||||
|
if flagCategory != "" {
|
||||||
|
cat := coredata.ThirdPartyCategory(flagCategory)
|
||||||
|
if !cat.IsValid() {
|
||||||
|
return fmt.Errorf("invalid --category value %q", flagCategory)
|
||||||
|
}
|
||||||
|
|
||||||
|
filter.WithCategory(&cat)
|
||||||
|
}
|
||||||
|
|
||||||
|
if flagKeyword != "" {
|
||||||
|
filter.WithKeyword(&flagKeyword)
|
||||||
|
}
|
||||||
|
|
||||||
|
pgClient, err := f.PgClient()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
var parties coredata.CommonThirdParties
|
||||||
|
|
||||||
|
if err := pgClient.WithConn(
|
||||||
|
cmd.Context(),
|
||||||
|
func(ctx context.Context, conn pg.Querier) error {
|
||||||
|
rows, err := cmdutil.Paginate(
|
||||||
|
ctx,
|
||||||
|
orderBy,
|
||||||
|
flagLimit,
|
||||||
|
func(ctx context.Context, cursor *page.Cursor[coredata.CommonThirdPartyOrderField]) ([]*coredata.CommonThirdParty, error) {
|
||||||
|
var ts coredata.CommonThirdParties
|
||||||
|
if err := ts.Load(ctx, conn, cursor, filter); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return ts, nil
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
parties = rows
|
||||||
|
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if *output == clicmdutil.OutputJSON {
|
||||||
|
return clicmdutil.PrintJSON(f.IOStreams.Out, parties)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(parties) == 0 {
|
||||||
|
_, _ = fmt.Fprintln(f.IOStreams.Out, "No common third parties found.")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
table := clicmdutil.NewTable("ID", "NAME", "SLUG", "CATEGORY")
|
||||||
|
for _, p := range parties {
|
||||||
|
table.Row(p.ID.String(), p.Name, p.Slug, string(p.Category))
|
||||||
|
}
|
||||||
|
|
||||||
|
_, _ = fmt.Fprintln(f.IOStreams.Out, table.Render())
|
||||||
|
_, _ = fmt.Fprintf(f.IOStreams.ErrOut, "Showing %d common third parties.\n", len(parties))
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
func optionalString(s string) *string {
|
||||||
|
if s == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return &s
|
||||||
|
}
|
||||||
122
pkg/proboctl/commonthirdparty/show.go
vendored
Normal file
122
pkg/proboctl/commonthirdparty/show.go
vendored
Normal file
@@ -0,0 +1,122 @@
|
|||||||
|
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||||
|
//
|
||||||
|
// Permission to use, copy, modify, and/or distribute this software for any
|
||||||
|
// purpose with or without fee is hereby granted, provided that the above
|
||||||
|
// copyright notice and this permission notice appear in all copies.
|
||||||
|
//
|
||||||
|
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||||
|
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||||
|
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||||
|
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||||
|
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||||
|
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||||
|
// PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
|
||||||
|
package commonthirdparty
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/charmbracelet/lipgloss"
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
"go.gearno.de/kit/pg"
|
||||||
|
clicmdutil "go.probo.inc/probo/pkg/cmd/cmdutil"
|
||||||
|
"go.probo.inc/probo/pkg/coredata"
|
||||||
|
"go.probo.inc/probo/pkg/proboctl/cmdutil"
|
||||||
|
)
|
||||||
|
|
||||||
|
func newCmdShow(f *cmdutil.Factory) *cobra.Command {
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: "show <gid|slug>",
|
||||||
|
Short: "Show a single common third party with its domains and linked pattern count",
|
||||||
|
Args: cobra.ExactArgs(1),
|
||||||
|
}
|
||||||
|
|
||||||
|
output := clicmdutil.AddOutputFlag(cmd)
|
||||||
|
|
||||||
|
cmd.RunE = func(cmd *cobra.Command, args []string) error {
|
||||||
|
if err := clicmdutil.ValidateOutputFlag(output); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
pgClient, err := f.PgClient()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
party coredata.CommonThirdParty
|
||||||
|
domains coredata.CommonThirdPartyDomains
|
||||||
|
patternCount int
|
||||||
|
)
|
||||||
|
|
||||||
|
if err := pgClient.WithConn(
|
||||||
|
cmd.Context(),
|
||||||
|
func(ctx context.Context, conn pg.Querier) error {
|
||||||
|
party, err = resolveCommonThirdParty(ctx, conn, args[0])
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := domains.LoadByCommonThirdPartyID(ctx, conn, party.ID); err != nil {
|
||||||
|
return fmt.Errorf("cannot load domains: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var patterns coredata.CommonTrackerPatterns
|
||||||
|
if err := patterns.LoadByCommonThirdPartyID(ctx, conn, party.ID); err != nil {
|
||||||
|
return fmt.Errorf("cannot load linked patterns: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
patternCount = len(patterns)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if *output == clicmdutil.OutputJSON {
|
||||||
|
return clicmdutil.PrintJSON(f.IOStreams.Out, map[string]any{
|
||||||
|
"thirdParty": party,
|
||||||
|
"domains": domains,
|
||||||
|
"linkedPatternCount": patternCount,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
out := f.IOStreams.Out
|
||||||
|
label := lipgloss.NewStyle().Foreground(lipgloss.Color("242")).Width(20)
|
||||||
|
row := func(name, value string) {
|
||||||
|
_, _ = fmt.Fprintf(out, "%s%s\n", label.Render(name), value)
|
||||||
|
}
|
||||||
|
|
||||||
|
row("ID:", party.ID.String())
|
||||||
|
row("Name:", party.Name)
|
||||||
|
row("Slug:", party.Slug)
|
||||||
|
row("Category:", string(party.Category))
|
||||||
|
|
||||||
|
if party.WebsiteURL != nil {
|
||||||
|
row("Website:", *party.WebsiteURL)
|
||||||
|
}
|
||||||
|
|
||||||
|
domainNames := make([]string, 0, len(domains))
|
||||||
|
for _, d := range domains {
|
||||||
|
domainNames = append(domainNames, d.Domain)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(domainNames) > 0 {
|
||||||
|
row("Domains:", strings.Join(domainNames, ", "))
|
||||||
|
} else {
|
||||||
|
row("Domains:", "(none)")
|
||||||
|
}
|
||||||
|
|
||||||
|
row("Linked patterns:", fmt.Sprintf("%d", patternCount))
|
||||||
|
row("Created:", party.CreatedAt.Format("2006-01-02 15:04:05"))
|
||||||
|
row("Updated:", party.UpdatedAt.Format("2006-01-02 15:04:05"))
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
97
pkg/proboctl/commontrackerpattern/commontrackerpattern.go
Normal file
97
pkg/proboctl/commontrackerpattern/commontrackerpattern.go
Normal file
@@ -0,0 +1,97 @@
|
|||||||
|
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||||
|
//
|
||||||
|
// Permission to use, copy, modify, and/or distribute this software for any
|
||||||
|
// purpose with or without fee is hereby granted, provided that the above
|
||||||
|
// copyright notice and this permission notice appear in all copies.
|
||||||
|
//
|
||||||
|
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||||
|
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||||
|
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||||
|
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||||
|
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||||
|
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||||
|
// PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
|
||||||
|
package commontrackerpattern
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
"go.gearno.de/kit/pg"
|
||||||
|
"go.probo.inc/probo/pkg/coredata"
|
||||||
|
"go.probo.inc/probo/pkg/gid"
|
||||||
|
"go.probo.inc/probo/pkg/proboctl/cmdutil"
|
||||||
|
)
|
||||||
|
|
||||||
|
// NewCmdCommonTrackerPattern is the entry point for inspecting and
|
||||||
|
// re-enriching the global common tracker pattern catalog.
|
||||||
|
func NewCmdCommonTrackerPattern(f *cmdutil.Factory) *cobra.Command {
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: "common-tracker-pattern <command>",
|
||||||
|
Aliases: []string{"ctp"},
|
||||||
|
Short: "Inspect and re-enrich the global common tracker pattern catalog",
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd.AddCommand(newCmdList(f))
|
||||||
|
cmd.AddCommand(newCmdShow(f))
|
||||||
|
cmd.AddCommand(newCmdReenrich(f))
|
||||||
|
cmd.AddCommand(newCmdStats(f))
|
||||||
|
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
// enrichmentState classifies a pattern's position in the enrichment
|
||||||
|
// lifecycle for display.
|
||||||
|
func enrichmentState(p *coredata.CommonTrackerPattern) string {
|
||||||
|
switch {
|
||||||
|
case p.EnrichmentRequestedAt != nil:
|
||||||
|
return "queued"
|
||||||
|
case p.EnrichedAt != nil:
|
||||||
|
return "enriched"
|
||||||
|
default:
|
||||||
|
return "unenriched"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolveCommonThirdPartyID accepts either a common third party GID or a
|
||||||
|
// slug and returns the corresponding id.
|
||||||
|
func resolveCommonThirdPartyID(ctx context.Context, conn pg.Querier, value string) (gid.GID, error) {
|
||||||
|
if id, err := gid.ParseGID(value); err == nil {
|
||||||
|
return id, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var party coredata.CommonThirdParty
|
||||||
|
if err := party.LoadBySlug(ctx, conn, value); err != nil {
|
||||||
|
if errors.Is(err, coredata.ErrResourceNotFound) {
|
||||||
|
return gid.GID{}, fmt.Errorf("no common third party found for %q (pass a slug or GID)", value)
|
||||||
|
}
|
||||||
|
|
||||||
|
return gid.GID{}, fmt.Errorf("cannot resolve common third party %q: %w", value, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return party.ID, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// thirdPartyNamesByID loads display names for the given common third
|
||||||
|
// party ids, skipping nil/empty inputs. It is used to render the linked
|
||||||
|
// vendor column without per-row queries.
|
||||||
|
func thirdPartyNamesByID(ctx context.Context, conn pg.Querier, ids []gid.GID) (map[gid.GID]string, error) {
|
||||||
|
names := make(map[gid.GID]string)
|
||||||
|
if len(ids) == 0 {
|
||||||
|
return names, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var parties coredata.CommonThirdParties
|
||||||
|
if err := parties.LoadByIDs(ctx, conn, ids); err != nil {
|
||||||
|
return nil, fmt.Errorf("cannot load common third parties: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, p := range parties {
|
||||||
|
names[p.ID] = p.Name
|
||||||
|
}
|
||||||
|
|
||||||
|
return names, nil
|
||||||
|
}
|
||||||
299
pkg/proboctl/commontrackerpattern/list.go
Normal file
299
pkg/proboctl/commontrackerpattern/list.go
Normal file
@@ -0,0 +1,299 @@
|
|||||||
|
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||||
|
//
|
||||||
|
// Permission to use, copy, modify, and/or distribute this software for any
|
||||||
|
// purpose with or without fee is hereby granted, provided that the above
|
||||||
|
// copyright notice and this permission notice appear in all copies.
|
||||||
|
//
|
||||||
|
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||||
|
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||||
|
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||||
|
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||||
|
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||||
|
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||||
|
// PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
|
||||||
|
package commontrackerpattern
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
"go.gearno.de/kit/pg"
|
||||||
|
clicmdutil "go.probo.inc/probo/pkg/cmd/cmdutil"
|
||||||
|
"go.probo.inc/probo/pkg/coredata"
|
||||||
|
"go.probo.inc/probo/pkg/gid"
|
||||||
|
"go.probo.inc/probo/pkg/page"
|
||||||
|
"go.probo.inc/probo/pkg/proboctl/cmdutil"
|
||||||
|
)
|
||||||
|
|
||||||
|
func newCmdList(f *cmdutil.Factory) *cobra.Command {
|
||||||
|
var (
|
||||||
|
flagTrackerType string
|
||||||
|
flagMatchType string
|
||||||
|
flagThirdParty string
|
||||||
|
flagKeyword string
|
||||||
|
flagState string
|
||||||
|
flagLinked bool
|
||||||
|
flagUnlinked bool
|
||||||
|
flagSort string
|
||||||
|
flagOrder string
|
||||||
|
flagLimit int
|
||||||
|
)
|
||||||
|
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: "list",
|
||||||
|
Short: "List common tracker patterns with filters and sorting",
|
||||||
|
Args: cobra.NoArgs,
|
||||||
|
}
|
||||||
|
|
||||||
|
output := clicmdutil.AddOutputFlag(cmd)
|
||||||
|
|
||||||
|
cmd.Flags().StringVar(&flagTrackerType, "tracker-type", "", "Filter by tracker type (COOKIE, LOCAL_STORAGE, SESSION_STORAGE, INDEXED_DB)")
|
||||||
|
cmd.Flags().StringVar(&flagMatchType, "match-type", "", "Filter by match type (EXACT, GLOB, PREFIX)")
|
||||||
|
cmd.Flags().StringVar(&flagThirdParty, "third-party", "", "Filter by linked common third party (slug or GID)")
|
||||||
|
cmd.Flags().StringVar(&flagKeyword, "keyword", "", "Filter by pattern/description substring")
|
||||||
|
cmd.Flags().StringVar(&flagState, "state", "", "Filter by enrichment state (queued, enriched, unenriched)")
|
||||||
|
cmd.Flags().BoolVar(&flagLinked, "linked", false, "Only patterns linked to a common third party")
|
||||||
|
cmd.Flags().BoolVar(&flagUnlinked, "unlinked", false, "Only patterns not linked to a common third party")
|
||||||
|
cmd.Flags().StringVar(&flagSort, "sort", "confidence", "Sort field: pattern, confidence, created, updated, enriched")
|
||||||
|
cmd.Flags().StringVar(&flagOrder, "order", "", "Sort order: asc, desc (default depends on field)")
|
||||||
|
cmd.Flags().IntVarP(&flagLimit, "limit", "L", 50, "Maximum rows to return (0 for all)")
|
||||||
|
|
||||||
|
cmd.RunE = func(cmd *cobra.Command, args []string) error {
|
||||||
|
if err := clicmdutil.ValidateOutputFlag(output); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if flagLinked && flagUnlinked {
|
||||||
|
return fmt.Errorf("--linked and --unlinked are mutually exclusive")
|
||||||
|
}
|
||||||
|
|
||||||
|
orderBy, err := parseOrderBy(flagSort, flagOrder)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
filter, err := buildListFilter(flagTrackerType, flagMatchType, flagKeyword, flagState, flagLinked, flagUnlinked)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
pgClient, err := f.PgClient()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx := cmd.Context()
|
||||||
|
|
||||||
|
var patterns coredata.CommonTrackerPatterns
|
||||||
|
|
||||||
|
if err := pgClient.WithConn(
|
||||||
|
ctx,
|
||||||
|
func(ctx context.Context, conn pg.Querier) error {
|
||||||
|
if flagThirdParty != "" {
|
||||||
|
id, err := resolveCommonThirdPartyID(ctx, conn, flagThirdParty)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
filter.WithCommonThirdPartyID(&id)
|
||||||
|
}
|
||||||
|
|
||||||
|
rows, err := cmdutil.Paginate(
|
||||||
|
ctx,
|
||||||
|
orderBy,
|
||||||
|
flagLimit,
|
||||||
|
func(ctx context.Context, cursor *page.Cursor[coredata.CommonTrackerPatternOrderField]) ([]*coredata.CommonTrackerPattern, error) {
|
||||||
|
var ps coredata.CommonTrackerPatterns
|
||||||
|
if err := ps.Load(ctx, conn, cursor, filter); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return ps, nil
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
patterns = rows
|
||||||
|
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if *output == clicmdutil.OutputJSON {
|
||||||
|
return clicmdutil.PrintJSON(f.IOStreams.Out, patterns)
|
||||||
|
}
|
||||||
|
|
||||||
|
return renderPatternTable(cmd, f, patterns)
|
||||||
|
}
|
||||||
|
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
func renderPatternTable(cmd *cobra.Command, f *cmdutil.Factory, patterns coredata.CommonTrackerPatterns) error {
|
||||||
|
out := f.IOStreams.Out
|
||||||
|
|
||||||
|
if len(patterns) == 0 {
|
||||||
|
_, _ = fmt.Fprintln(out, "No common tracker patterns found.")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var linkedIDs []gid.GID
|
||||||
|
for _, p := range patterns {
|
||||||
|
if p.CommonThirdPartyID != nil {
|
||||||
|
linkedIDs = append(linkedIDs, *p.CommonThirdPartyID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pgClient, err := f.PgClient()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
var names map[gid.GID]string
|
||||||
|
|
||||||
|
if err := pgClient.WithConn(
|
||||||
|
cmd.Context(),
|
||||||
|
func(ctx context.Context, conn pg.Querier) error {
|
||||||
|
names, err = thirdPartyNamesByID(ctx, conn, linkedIDs)
|
||||||
|
return err
|
||||||
|
},
|
||||||
|
); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
table := clicmdutil.NewTable("ID", "TYPE", "MATCH", "PATTERN", "CONF", "STATE", "THIRD PARTY")
|
||||||
|
|
||||||
|
for _, p := range patterns {
|
||||||
|
thirdParty := ""
|
||||||
|
if p.CommonThirdPartyID != nil {
|
||||||
|
thirdParty = names[*p.CommonThirdPartyID]
|
||||||
|
}
|
||||||
|
|
||||||
|
table.Row(
|
||||||
|
p.ID.String(),
|
||||||
|
string(p.TrackerType),
|
||||||
|
string(p.MatchType),
|
||||||
|
p.Pattern,
|
||||||
|
fmt.Sprintf("%.2f", p.Confidence),
|
||||||
|
enrichmentState(p),
|
||||||
|
thirdParty,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, _ = fmt.Fprintln(out, table.Render())
|
||||||
|
_, _ = fmt.Fprintf(f.IOStreams.ErrOut, "Showing %d common tracker patterns.\n", len(patterns))
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseOrderBy maps the --sort/--order flags to a page.OrderBy. When
|
||||||
|
// --order is empty it defaults to descending for the time/score fields
|
||||||
|
// and ascending for pattern.
|
||||||
|
func parseOrderBy(sort, order string) (page.OrderBy[coredata.CommonTrackerPatternOrderField], error) {
|
||||||
|
var (
|
||||||
|
field coredata.CommonTrackerPatternOrderField
|
||||||
|
defaultDesc bool
|
||||||
|
zeroOrderBy page.OrderBy[coredata.CommonTrackerPatternOrderField]
|
||||||
|
)
|
||||||
|
|
||||||
|
switch sort {
|
||||||
|
case "pattern":
|
||||||
|
field = coredata.CommonTrackerPatternOrderFieldPattern
|
||||||
|
case "confidence":
|
||||||
|
field, defaultDesc = coredata.CommonTrackerPatternOrderFieldConfidence, true
|
||||||
|
case "created":
|
||||||
|
field, defaultDesc = coredata.CommonTrackerPatternOrderFieldCreatedAt, true
|
||||||
|
case "updated":
|
||||||
|
field, defaultDesc = coredata.CommonTrackerPatternOrderFieldUpdatedAt, true
|
||||||
|
case "enriched":
|
||||||
|
field, defaultDesc = coredata.CommonTrackerPatternOrderFieldEnrichedAt, true
|
||||||
|
default:
|
||||||
|
return zeroOrderBy, fmt.Errorf("invalid --sort value %q: valid values are pattern, confidence, created, updated, enriched", sort)
|
||||||
|
}
|
||||||
|
|
||||||
|
direction := page.OrderDirectionAsc
|
||||||
|
if defaultDesc {
|
||||||
|
direction = page.OrderDirectionDesc
|
||||||
|
}
|
||||||
|
|
||||||
|
switch order {
|
||||||
|
case "":
|
||||||
|
// keep field default
|
||||||
|
case "asc":
|
||||||
|
direction = page.OrderDirectionAsc
|
||||||
|
case "desc":
|
||||||
|
direction = page.OrderDirectionDesc
|
||||||
|
default:
|
||||||
|
return zeroOrderBy, fmt.Errorf("invalid --order value %q: valid values are asc, desc", order)
|
||||||
|
}
|
||||||
|
|
||||||
|
return page.OrderBy[coredata.CommonTrackerPatternOrderField]{Field: field, Direction: direction}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildListFilter(
|
||||||
|
trackerType, matchType, keyword, state string,
|
||||||
|
linked, unlinked bool,
|
||||||
|
) (*coredata.CommonTrackerPatternFilter, error) {
|
||||||
|
filter := coredata.NewCommonTrackerPatternFilter()
|
||||||
|
|
||||||
|
if trackerType != "" {
|
||||||
|
tt := coredata.TrackerType(trackerType)
|
||||||
|
if !tt.IsValid() {
|
||||||
|
return nil, fmt.Errorf("invalid --tracker-type value %q", trackerType)
|
||||||
|
}
|
||||||
|
|
||||||
|
filter.WithTrackerType(&tt)
|
||||||
|
}
|
||||||
|
|
||||||
|
if matchType != "" {
|
||||||
|
mt := coredata.TrackerPatternMatchType(matchType)
|
||||||
|
if !mt.IsValid() {
|
||||||
|
return nil, fmt.Errorf("invalid --match-type value %q", matchType)
|
||||||
|
}
|
||||||
|
|
||||||
|
filter.WithMatchType(&mt)
|
||||||
|
}
|
||||||
|
|
||||||
|
if keyword != "" {
|
||||||
|
filter.WithKeyword(&keyword)
|
||||||
|
}
|
||||||
|
|
||||||
|
if state != "" {
|
||||||
|
st, err := parseEnrichmentState(state)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
filter.WithState(&st)
|
||||||
|
}
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case linked:
|
||||||
|
v := true
|
||||||
|
filter.WithLinked(&v)
|
||||||
|
case unlinked:
|
||||||
|
v := false
|
||||||
|
filter.WithLinked(&v)
|
||||||
|
}
|
||||||
|
|
||||||
|
return filter, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseEnrichmentState(value string) (coredata.CommonTrackerPatternEnrichmentState, error) {
|
||||||
|
switch value {
|
||||||
|
case "queued":
|
||||||
|
return coredata.CommonTrackerPatternEnrichmentStateQueued, nil
|
||||||
|
case "enriched":
|
||||||
|
return coredata.CommonTrackerPatternEnrichmentStateEnriched, nil
|
||||||
|
case "unenriched":
|
||||||
|
return coredata.CommonTrackerPatternEnrichmentStateUnenriched, nil
|
||||||
|
default:
|
||||||
|
return "", fmt.Errorf("invalid --state value %q: valid values are queued, enriched, unenriched", value)
|
||||||
|
}
|
||||||
|
}
|
||||||
291
pkg/proboctl/commontrackerpattern/reenrich.go
Normal file
291
pkg/proboctl/commontrackerpattern/reenrich.go
Normal file
@@ -0,0 +1,291 @@
|
|||||||
|
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||||
|
//
|
||||||
|
// Permission to use, copy, modify, and/or distribute this software for any
|
||||||
|
// purpose with or without fee is hereby granted, provided that the above
|
||||||
|
// copyright notice and this permission notice appear in all copies.
|
||||||
|
//
|
||||||
|
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||||
|
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||||
|
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||||
|
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||||
|
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||||
|
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||||
|
// PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
|
||||||
|
package commontrackerpattern
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
"go.gearno.de/kit/log"
|
||||||
|
"go.gearno.de/kit/pg"
|
||||||
|
"go.probo.inc/probo/pkg/cookiebanner"
|
||||||
|
"go.probo.inc/probo/pkg/coredata"
|
||||||
|
"go.probo.inc/probo/pkg/gid"
|
||||||
|
"go.probo.inc/probo/pkg/proboctl/cmdutil"
|
||||||
|
)
|
||||||
|
|
||||||
|
func newCmdReenrich(f *cmdutil.Factory) *cobra.Command {
|
||||||
|
var (
|
||||||
|
flagIDs []string
|
||||||
|
flagThirdParty string
|
||||||
|
flagTrackerType string
|
||||||
|
flagKeyword string
|
||||||
|
flagState string
|
||||||
|
flagAll bool
|
||||||
|
flagLinkedBanner string
|
||||||
|
flagLinkedOrg string
|
||||||
|
flagConcurrency int
|
||||||
|
flagResetEnriched bool
|
||||||
|
flagDryRun bool
|
||||||
|
flagYes bool
|
||||||
|
flagEnqueue bool
|
||||||
|
)
|
||||||
|
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: "reenrich",
|
||||||
|
Short: "Re-describe common tracker patterns by running the enrichment agent",
|
||||||
|
Long: "Re-describe selected common tracker patterns. By default the enrichment " +
|
||||||
|
"agent runs in-process and the command returns only when the work is done " +
|
||||||
|
"(requires --cfg-file with an LLM provider). Use --enqueue to instead arm the " +
|
||||||
|
"async enrichment worker. Re-describe a banner's catalog rows with --linked-banner " +
|
||||||
|
"before running 'cookie-banner reset-trackers' so fresh descriptions copy down.",
|
||||||
|
Args: cobra.NoArgs,
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd.Flags().StringSliceVar(&flagIDs, "id", nil, "Common tracker pattern GID(s) to re-enrich (repeatable)")
|
||||||
|
cmd.Flags().StringVar(&flagThirdParty, "third-party", "", "Select patterns linked to a common third party (slug or GID)")
|
||||||
|
cmd.Flags().StringVar(&flagTrackerType, "tracker-type", "", "Select patterns of a tracker type")
|
||||||
|
cmd.Flags().StringVar(&flagKeyword, "keyword", "", "Select patterns matching a pattern/description substring")
|
||||||
|
cmd.Flags().StringVar(&flagState, "state", "", "Select by enrichment state (queued, enriched, unenriched)")
|
||||||
|
cmd.Flags().BoolVar(&flagAll, "all", false, "Select every common tracker pattern")
|
||||||
|
cmd.Flags().StringVar(&flagLinkedBanner, "linked-banner", "", "Select catalog rows linked to a cookie banner's patterns (GID)")
|
||||||
|
cmd.Flags().StringVar(&flagLinkedOrg, "linked-org", "", "Select catalog rows linked to an organization's patterns (GID)")
|
||||||
|
cmd.Flags().IntVar(&flagConcurrency, "concurrency", 4, "Number of patterns to enrich in parallel (sync mode)")
|
||||||
|
cmd.Flags().BoolVar(&flagResetEnriched, "reset-enriched", true, "Clear enriched_at so terminal rows are re-processed")
|
||||||
|
cmd.Flags().BoolVar(&flagDryRun, "dry-run", false, "Print the selected patterns without enriching")
|
||||||
|
cmd.Flags().BoolVar(&flagYes, "yes", false, "Skip confirmation")
|
||||||
|
cmd.Flags().BoolVar(&flagEnqueue, "enqueue", false, "Arm the async enrichment worker instead of running the agent in-process")
|
||||||
|
|
||||||
|
cmd.RunE = func(cmd *cobra.Command, args []string) error {
|
||||||
|
ctx := cmd.Context()
|
||||||
|
|
||||||
|
pgClient, err := f.PgClient()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
ids, err := resolveReenrichIDs(
|
||||||
|
ctx,
|
||||||
|
pgClient,
|
||||||
|
flagIDs,
|
||||||
|
flagThirdParty,
|
||||||
|
flagTrackerType,
|
||||||
|
flagKeyword,
|
||||||
|
flagState,
|
||||||
|
flagAll,
|
||||||
|
flagLinkedBanner,
|
||||||
|
flagLinkedOrg,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
out := f.IOStreams.Out
|
||||||
|
|
||||||
|
if len(ids) == 0 {
|
||||||
|
_, _ = fmt.Fprintln(out, "No common tracker patterns matched the selection.")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if flagDryRun {
|
||||||
|
_, _ = fmt.Fprintf(out, "Would re-enrich %d common tracker pattern(s).\n", len(ids))
|
||||||
|
printSample(out, ids)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if !flagYes {
|
||||||
|
return fmt.Errorf("about to re-enrich %d pattern(s); pass --yes to proceed or --dry-run to preview", len(ids))
|
||||||
|
}
|
||||||
|
|
||||||
|
if flagEnqueue {
|
||||||
|
var requeued int64
|
||||||
|
|
||||||
|
if err := pgClient.WithTx(
|
||||||
|
ctx,
|
||||||
|
func(ctx context.Context, tx pg.Tx) error {
|
||||||
|
var ps coredata.CommonTrackerPatterns
|
||||||
|
requeued, err = ps.RequestEnrichmentByIDs(ctx, tx, ids, flagResetEnriched)
|
||||||
|
|
||||||
|
return err
|
||||||
|
},
|
||||||
|
); err != nil {
|
||||||
|
return fmt.Errorf("cannot enqueue enrichment: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, _ = fmt.Fprintf(out, "Queued %d common tracker pattern(s) for the enrichment worker.\n", requeued)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg, err := f.TrackerAgentsConfig()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
logger := log.NewLogger(
|
||||||
|
log.WithName("proboctl"),
|
||||||
|
log.WithOutput(f.IOStreams.ErrOut),
|
||||||
|
)
|
||||||
|
|
||||||
|
enricher := cookiebanner.NewCommonPatternEnricher(pgClient, logger, cfg)
|
||||||
|
|
||||||
|
enriched, err := enricher.EnrichByIDs(ctx, ids, flagConcurrency)
|
||||||
|
|
||||||
|
_, _ = fmt.Fprintf(out, "Enriched %d of %d common tracker pattern(s).\n", enriched, len(ids))
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("enrichment did not complete for all patterns: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveReenrichIDs(
|
||||||
|
ctx context.Context,
|
||||||
|
pgClient *pg.Client,
|
||||||
|
rawIDs []string,
|
||||||
|
thirdParty, trackerType, keyword, state string,
|
||||||
|
all bool,
|
||||||
|
linkedBanner, linkedOrg string,
|
||||||
|
) ([]gid.GID, error) {
|
||||||
|
if len(rawIDs) > 0 {
|
||||||
|
ids := make([]gid.GID, 0, len(rawIDs))
|
||||||
|
|
||||||
|
for _, raw := range rawIDs {
|
||||||
|
id, err := gid.ParseGID(raw)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("invalid --id value %q: %w", raw, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ids = append(ids, id)
|
||||||
|
}
|
||||||
|
|
||||||
|
return ids, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var ids []gid.GID
|
||||||
|
|
||||||
|
err := pgClient.WithConn(
|
||||||
|
ctx,
|
||||||
|
func(ctx context.Context, conn pg.Querier) error {
|
||||||
|
switch {
|
||||||
|
case linkedBanner != "":
|
||||||
|
bannerID, err := gid.ParseGID(linkedBanner)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("invalid --linked-banner GID %q: %w", linkedBanner, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var tps coredata.TrackerPatterns
|
||||||
|
ids, err = tps.LoadAllLinkedCommonTrackerPatternIDsByCookieBannerID(ctx, conn, coredata.NewScopeFromObjectID(bannerID), bannerID)
|
||||||
|
|
||||||
|
return err
|
||||||
|
case linkedOrg != "":
|
||||||
|
orgID, err := gid.ParseGID(linkedOrg)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("invalid --linked-org GID %q: %w", linkedOrg, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var tps coredata.TrackerPatterns
|
||||||
|
ids, err = tps.LoadAllLinkedCommonTrackerPatternIDsByOrganizationID(ctx, conn, coredata.NewScopeFromObjectID(orgID), orgID)
|
||||||
|
|
||||||
|
return err
|
||||||
|
default:
|
||||||
|
filter, hasSelector, err := buildReenrichFilter(ctx, conn, thirdParty, trackerType, keyword, state)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if !hasSelector && !all {
|
||||||
|
return fmt.Errorf("specify a selector (--id, --third-party, --tracker-type, --state, --keyword, --linked-banner, --linked-org) or --all")
|
||||||
|
}
|
||||||
|
|
||||||
|
var ps coredata.CommonTrackerPatterns
|
||||||
|
ids, err = ps.LoadAllIDs(ctx, conn, filter)
|
||||||
|
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return ids, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildReenrichFilter(
|
||||||
|
ctx context.Context,
|
||||||
|
conn pg.Querier,
|
||||||
|
thirdParty, trackerType, keyword, state string,
|
||||||
|
) (*coredata.CommonTrackerPatternFilter, bool, error) {
|
||||||
|
filter := coredata.NewCommonTrackerPatternFilter()
|
||||||
|
hasSelector := false
|
||||||
|
|
||||||
|
if thirdParty != "" {
|
||||||
|
id, err := resolveCommonThirdPartyID(ctx, conn, thirdParty)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
|
||||||
|
filter.WithCommonThirdPartyID(&id)
|
||||||
|
hasSelector = true
|
||||||
|
}
|
||||||
|
|
||||||
|
if trackerType != "" {
|
||||||
|
tt := coredata.TrackerType(trackerType)
|
||||||
|
if !tt.IsValid() {
|
||||||
|
return nil, false, fmt.Errorf("invalid --tracker-type value %q", trackerType)
|
||||||
|
}
|
||||||
|
|
||||||
|
filter.WithTrackerType(&tt)
|
||||||
|
hasSelector = true
|
||||||
|
}
|
||||||
|
|
||||||
|
if keyword != "" {
|
||||||
|
filter.WithKeyword(&keyword)
|
||||||
|
hasSelector = true
|
||||||
|
}
|
||||||
|
|
||||||
|
if state != "" {
|
||||||
|
st, err := parseEnrichmentState(state)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
|
||||||
|
filter.WithState(&st)
|
||||||
|
hasSelector = true
|
||||||
|
}
|
||||||
|
|
||||||
|
return filter, hasSelector, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func printSample(out io.Writer, ids []gid.GID) {
|
||||||
|
const sampleSize = 10
|
||||||
|
|
||||||
|
for i, id := range ids {
|
||||||
|
if i >= sampleSize {
|
||||||
|
_, _ = fmt.Fprintf(out, " ... and %d more\n", len(ids)-sampleSize)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
_, _ = fmt.Fprintf(out, " %s\n", id.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
138
pkg/proboctl/commontrackerpattern/show.go
Normal file
138
pkg/proboctl/commontrackerpattern/show.go
Normal file
@@ -0,0 +1,138 @@
|
|||||||
|
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||||
|
//
|
||||||
|
// Permission to use, copy, modify, and/or distribute this software for any
|
||||||
|
// purpose with or without fee is hereby granted, provided that the above
|
||||||
|
// copyright notice and this permission notice appear in all copies.
|
||||||
|
//
|
||||||
|
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||||
|
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||||
|
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||||
|
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||||
|
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||||
|
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||||
|
// PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
|
||||||
|
package commontrackerpattern
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/charmbracelet/lipgloss"
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
"go.gearno.de/kit/pg"
|
||||||
|
clicmdutil "go.probo.inc/probo/pkg/cmd/cmdutil"
|
||||||
|
"go.probo.inc/probo/pkg/coredata"
|
||||||
|
"go.probo.inc/probo/pkg/gid"
|
||||||
|
"go.probo.inc/probo/pkg/proboctl/cmdutil"
|
||||||
|
)
|
||||||
|
|
||||||
|
func newCmdShow(f *cmdutil.Factory) *cobra.Command {
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: "show <gid>",
|
||||||
|
Short: "Show a single common tracker pattern by GID",
|
||||||
|
Args: cobra.ExactArgs(1),
|
||||||
|
}
|
||||||
|
|
||||||
|
output := clicmdutil.AddOutputFlag(cmd)
|
||||||
|
|
||||||
|
cmd.RunE = func(cmd *cobra.Command, args []string) error {
|
||||||
|
if err := clicmdutil.ValidateOutputFlag(output); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
id, err := gid.ParseGID(args[0])
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("invalid GID %q: %w", args[0], err)
|
||||||
|
}
|
||||||
|
|
||||||
|
pgClient, err := f.PgClient()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
pattern coredata.CommonTrackerPattern
|
||||||
|
thirdPartyName string
|
||||||
|
)
|
||||||
|
|
||||||
|
if err := pgClient.WithConn(
|
||||||
|
cmd.Context(),
|
||||||
|
func(ctx context.Context, conn pg.Querier) error {
|
||||||
|
if err := pattern.LoadByID(ctx, conn, id); err != nil {
|
||||||
|
if errors.Is(err, coredata.ErrResourceNotFound) {
|
||||||
|
return fmt.Errorf("no common tracker pattern found for %q", args[0])
|
||||||
|
}
|
||||||
|
|
||||||
|
return fmt.Errorf("cannot load common tracker pattern: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if pattern.CommonThirdPartyID != nil {
|
||||||
|
var party coredata.CommonThirdParty
|
||||||
|
if err := party.LoadByID(ctx, conn, *pattern.CommonThirdPartyID); err == nil {
|
||||||
|
thirdPartyName = party.Name
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if *output == clicmdutil.OutputJSON {
|
||||||
|
return clicmdutil.PrintJSON(f.IOStreams.Out, pattern)
|
||||||
|
}
|
||||||
|
|
||||||
|
return renderPatternDetail(f, pattern, thirdPartyName)
|
||||||
|
}
|
||||||
|
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
func renderPatternDetail(f *cmdutil.Factory, p coredata.CommonTrackerPattern, thirdPartyName string) error {
|
||||||
|
out := f.IOStreams.Out
|
||||||
|
label := lipgloss.NewStyle().Foreground(lipgloss.Color("242")).Width(20)
|
||||||
|
|
||||||
|
row := func(name, value string) {
|
||||||
|
_, _ = fmt.Fprintf(out, "%s%s\n", label.Render(name), value)
|
||||||
|
}
|
||||||
|
|
||||||
|
row("ID:", p.ID.String())
|
||||||
|
row("Tracker type:", string(p.TrackerType))
|
||||||
|
row("Match type:", string(p.MatchType))
|
||||||
|
row("Pattern:", p.Pattern)
|
||||||
|
row("Confidence:", fmt.Sprintf("%.2f", p.Confidence))
|
||||||
|
row("State:", enrichmentState(&p))
|
||||||
|
|
||||||
|
if p.MaxAgeSeconds != nil {
|
||||||
|
row("Max age (s):", fmt.Sprintf("%d", *p.MaxAgeSeconds))
|
||||||
|
}
|
||||||
|
|
||||||
|
if p.CommonThirdPartyID != nil {
|
||||||
|
row("Third party:", fmt.Sprintf("%s (%s)", thirdPartyName, p.CommonThirdPartyID.String()))
|
||||||
|
} else {
|
||||||
|
row("Third party:", "(unlinked)")
|
||||||
|
}
|
||||||
|
|
||||||
|
description := p.Description
|
||||||
|
if description == "" {
|
||||||
|
description = "(none)"
|
||||||
|
}
|
||||||
|
|
||||||
|
row("Description:", description)
|
||||||
|
|
||||||
|
if p.EnrichmentRequestedAt != nil {
|
||||||
|
row("Enrichment queued:", p.EnrichmentRequestedAt.Format("2006-01-02 15:04:05"))
|
||||||
|
}
|
||||||
|
|
||||||
|
if p.EnrichedAt != nil {
|
||||||
|
row("Enriched at:", p.EnrichedAt.Format("2006-01-02 15:04:05"))
|
||||||
|
}
|
||||||
|
|
||||||
|
row("Created:", p.CreatedAt.Format("2006-01-02 15:04:05"))
|
||||||
|
row("Updated:", p.UpdatedAt.Format("2006-01-02 15:04:05"))
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
104
pkg/proboctl/commontrackerpattern/stats.go
Normal file
104
pkg/proboctl/commontrackerpattern/stats.go
Normal file
@@ -0,0 +1,104 @@
|
|||||||
|
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||||
|
//
|
||||||
|
// Permission to use, copy, modify, and/or distribute this software for any
|
||||||
|
// purpose with or without fee is hereby granted, provided that the above
|
||||||
|
// copyright notice and this permission notice appear in all copies.
|
||||||
|
//
|
||||||
|
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||||
|
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||||
|
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||||
|
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||||
|
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||||
|
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||||
|
// PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
|
||||||
|
package commontrackerpattern
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
"go.gearno.de/kit/pg"
|
||||||
|
clicmdutil "go.probo.inc/probo/pkg/cmd/cmdutil"
|
||||||
|
"go.probo.inc/probo/pkg/coredata"
|
||||||
|
"go.probo.inc/probo/pkg/proboctl/cmdutil"
|
||||||
|
)
|
||||||
|
|
||||||
|
func newCmdStats(f *cmdutil.Factory) *cobra.Command {
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: "stats",
|
||||||
|
Short: "Summarize the common tracker pattern catalog by enrichment and link state",
|
||||||
|
Args: cobra.NoArgs,
|
||||||
|
}
|
||||||
|
|
||||||
|
output := clicmdutil.AddOutputFlag(cmd)
|
||||||
|
|
||||||
|
cmd.RunE = func(cmd *cobra.Command, args []string) error {
|
||||||
|
if err := clicmdutil.ValidateOutputFlag(output); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
pgClient, err := f.PgClient()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
stats := map[string]int{}
|
||||||
|
|
||||||
|
if err := pgClient.WithConn(
|
||||||
|
cmd.Context(),
|
||||||
|
func(ctx context.Context, conn pg.Querier) error {
|
||||||
|
counts := []struct {
|
||||||
|
key string
|
||||||
|
filter *coredata.CommonTrackerPatternFilter
|
||||||
|
}{
|
||||||
|
{"total", coredata.NewCommonTrackerPatternFilter()},
|
||||||
|
{"queued", coredata.NewCommonTrackerPatternFilter().WithState(refState(coredata.CommonTrackerPatternEnrichmentStateQueued))},
|
||||||
|
{"enriched", coredata.NewCommonTrackerPatternFilter().WithState(refState(coredata.CommonTrackerPatternEnrichmentStateEnriched))},
|
||||||
|
{"unenriched", coredata.NewCommonTrackerPatternFilter().WithState(refState(coredata.CommonTrackerPatternEnrichmentStateUnenriched))},
|
||||||
|
{"linked", coredata.NewCommonTrackerPatternFilter().WithLinked(refBool(true))},
|
||||||
|
{"unlinked", coredata.NewCommonTrackerPatternFilter().WithLinked(refBool(false))},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, c := range counts {
|
||||||
|
var ps coredata.CommonTrackerPatterns
|
||||||
|
|
||||||
|
n, err := ps.CountAll(ctx, conn, c.filter)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
stats[c.key] = n
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if *output == clicmdutil.OutputJSON {
|
||||||
|
return clicmdutil.PrintJSON(f.IOStreams.Out, stats)
|
||||||
|
}
|
||||||
|
|
||||||
|
table := clicmdutil.NewTable("METRIC", "COUNT")
|
||||||
|
for _, key := range []string{"total", "queued", "enriched", "unenriched", "linked", "unlinked"} {
|
||||||
|
table.Row(key, fmt.Sprintf("%d", stats[key]))
|
||||||
|
}
|
||||||
|
|
||||||
|
_, _ = fmt.Fprintln(f.IOStreams.Out, table.Render())
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
func refState(s coredata.CommonTrackerPatternEnrichmentState) *coredata.CommonTrackerPatternEnrichmentState {
|
||||||
|
return &s
|
||||||
|
}
|
||||||
|
|
||||||
|
func refBool(b bool) *bool {
|
||||||
|
return &b
|
||||||
|
}
|
||||||
34
pkg/proboctl/cookiebanner/cookiebanner.go
Normal file
34
pkg/proboctl/cookiebanner/cookiebanner.go
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||||
|
//
|
||||||
|
// Permission to use, copy, modify, and/or distribute this software for any
|
||||||
|
// purpose with or without fee is hereby granted, provided that the above
|
||||||
|
// copyright notice and this permission notice appear in all copies.
|
||||||
|
//
|
||||||
|
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||||
|
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||||
|
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||||
|
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||||
|
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||||
|
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||||
|
// PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
|
||||||
|
package cookiebanner
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
"go.probo.inc/probo/pkg/proboctl/cmdutil"
|
||||||
|
)
|
||||||
|
|
||||||
|
// NewCmdCookieBanner groups operator commands acting on a tenant's cookie
|
||||||
|
// banners. Unlike the global catalog commands, these are tenant-scoped:
|
||||||
|
// every write derives a coredata.Scope from the banner/org GID.
|
||||||
|
func NewCmdCookieBanner(f *cmdutil.Factory) *cobra.Command {
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: "cookie-banner <command>",
|
||||||
|
Short: "Operator commands for a tenant's cookie banners",
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd.AddCommand(newCmdResetTrackers(f))
|
||||||
|
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
182
pkg/proboctl/cookiebanner/reset_trackers.go
Normal file
182
pkg/proboctl/cookiebanner/reset_trackers.go
Normal file
@@ -0,0 +1,182 @@
|
|||||||
|
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||||
|
//
|
||||||
|
// Permission to use, copy, modify, and/or distribute this software for any
|
||||||
|
// purpose with or without fee is hereby granted, provided that the above
|
||||||
|
// copyright notice and this permission notice appear in all copies.
|
||||||
|
//
|
||||||
|
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||||
|
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||||
|
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||||
|
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||||
|
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||||
|
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||||
|
// PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
|
||||||
|
package cookiebanner
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
"go.gearno.de/kit/pg"
|
||||||
|
"go.probo.inc/probo/pkg/cookiebanner"
|
||||||
|
"go.probo.inc/probo/pkg/coredata"
|
||||||
|
"go.probo.inc/probo/pkg/gid"
|
||||||
|
"go.probo.inc/probo/pkg/page"
|
||||||
|
"go.probo.inc/probo/pkg/proboctl/cmdutil"
|
||||||
|
)
|
||||||
|
|
||||||
|
func newCmdResetTrackers(f *cmdutil.Factory) *cobra.Command {
|
||||||
|
var (
|
||||||
|
flagBanner string
|
||||||
|
flagOrg string
|
||||||
|
flagMappingOnly bool
|
||||||
|
flagDryRun bool
|
||||||
|
flagYes bool
|
||||||
|
)
|
||||||
|
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: "reset-trackers",
|
||||||
|
Short: "Rebuild a banner's tracker patterns from detections and re-arm the analysis + mapping workers",
|
||||||
|
Long: "Destructive, tenant-scoped operator action. For a banner's uncategorised, " +
|
||||||
|
"non-excluded patterns it clears catalog/vendor links, rebuilds the raw exact " +
|
||||||
|
"patterns from detected_trackers (decomposing derived globs), and re-arms the " +
|
||||||
|
"pattern-analysis and mapping workers. User-categorised and excluded patterns are " +
|
||||||
|
"preserved. With --mapping-only it skips the rebuild and only re-arms mapping.",
|
||||||
|
Args: cobra.NoArgs,
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd.Flags().StringVar(&flagBanner, "banner", "", "Cookie banner GID to reset")
|
||||||
|
cmd.Flags().StringVar(&flagOrg, "org", "", "Organization GID: reset every banner of the org")
|
||||||
|
cmd.Flags().BoolVar(&flagMappingOnly, "mapping-only", false, "Only re-arm mapping (skip the detection rebuild and analysis)")
|
||||||
|
cmd.Flags().BoolVar(&flagDryRun, "dry-run", false, "Print the target banners without writing")
|
||||||
|
cmd.Flags().BoolVar(&flagYes, "yes", false, "Skip confirmation")
|
||||||
|
|
||||||
|
cmd.RunE = func(cmd *cobra.Command, args []string) error {
|
||||||
|
if (flagBanner == "") == (flagOrg == "") {
|
||||||
|
return fmt.Errorf("exactly one of --banner or --org is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx := cmd.Context()
|
||||||
|
|
||||||
|
pgClient, err := f.PgClient()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
bannerIDs, scope, err := resolveTargetBanners(ctx, pgClient, flagBanner, flagOrg)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
out := f.IOStreams.Out
|
||||||
|
|
||||||
|
if len(bannerIDs) == 0 {
|
||||||
|
_, _ = fmt.Fprintln(out, "No cookie banners matched.")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
mode := "full reset"
|
||||||
|
if flagMappingOnly {
|
||||||
|
mode = "mapping-only reset"
|
||||||
|
}
|
||||||
|
|
||||||
|
if flagDryRun {
|
||||||
|
_, _ = fmt.Fprintf(out, "Would run %s on %d banner(s):\n", mode, len(bannerIDs))
|
||||||
|
for _, id := range bannerIDs {
|
||||||
|
_, _ = fmt.Fprintf(out, " %s\n", id.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if !flagYes {
|
||||||
|
return fmt.Errorf("about to run %s on %d banner(s); pass --yes to proceed or --dry-run to preview", mode, len(bannerIDs))
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, id := range bannerIDs {
|
||||||
|
result, err := cookiebanner.ResetBannerTrackers(ctx, pgClient, scope, id, flagMappingOnly)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("cannot reset banner %s: %w", id, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, _ = fmt.Fprintf(
|
||||||
|
out,
|
||||||
|
"%s: reset %d pattern(s), decomposed %d glob(s) into %d exact(s), relinked %d detection(s), analysis_requested=%t\n",
|
||||||
|
id.String(),
|
||||||
|
result.PatternsReset,
|
||||||
|
result.GlobsDecomposed,
|
||||||
|
result.ExactsCreated,
|
||||||
|
result.DetectionsRelinked,
|
||||||
|
result.AnalysisRequested,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolveTargetBanners returns the banner ids to reset and a tenant scope
|
||||||
|
// derived from the provided GID. The scope is keyed off the banner or org
|
||||||
|
// GID so every downstream write stays tenant-isolated.
|
||||||
|
func resolveTargetBanners(
|
||||||
|
ctx context.Context,
|
||||||
|
pgClient *pg.Client,
|
||||||
|
bannerFlag, orgFlag string,
|
||||||
|
) ([]gid.GID, coredata.Scoper, error) {
|
||||||
|
if bannerFlag != "" {
|
||||||
|
id, err := gid.ParseGID(bannerFlag)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("invalid --banner GID %q: %w", bannerFlag, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return []gid.GID{id}, coredata.NewScopeFromObjectID(id), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
orgID, err := gid.ParseGID(orgFlag)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("invalid --org GID %q: %w", orgFlag, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
scope := coredata.NewScopeFromObjectID(orgID)
|
||||||
|
|
||||||
|
var ids []gid.GID
|
||||||
|
|
||||||
|
if err := pgClient.WithConn(
|
||||||
|
ctx,
|
||||||
|
func(ctx context.Context, conn pg.Querier) error {
|
||||||
|
banners, err := cmdutil.Paginate(
|
||||||
|
ctx,
|
||||||
|
page.OrderBy[coredata.CookieBannerOrderField]{
|
||||||
|
Field: coredata.CookieBannerOrderFieldCreatedAt,
|
||||||
|
Direction: page.OrderDirectionAsc,
|
||||||
|
},
|
||||||
|
0,
|
||||||
|
func(ctx context.Context, cursor *page.Cursor[coredata.CookieBannerOrderField]) ([]*coredata.CookieBanner, error) {
|
||||||
|
var bs coredata.CookieBanners
|
||||||
|
if err := bs.LoadByOrganizationID(ctx, conn, scope, orgID, cursor, coredata.NewCookieBannerFilter(nil)); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return bs, nil
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, b := range banners {
|
||||||
|
ids = append(ids, b.ID)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
); err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return ids, scope, nil
|
||||||
|
}
|
||||||
@@ -19,6 +19,9 @@ import (
|
|||||||
|
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
"go.probo.inc/probo/pkg/proboctl/cmdutil"
|
"go.probo.inc/probo/pkg/proboctl/cmdutil"
|
||||||
|
"go.probo.inc/probo/pkg/proboctl/commonthirdparty"
|
||||||
|
"go.probo.inc/probo/pkg/proboctl/commontrackerpattern"
|
||||||
|
proboctlcookiebanner "go.probo.inc/probo/pkg/proboctl/cookiebanner"
|
||||||
"go.probo.inc/probo/pkg/proboctl/seed"
|
"go.probo.inc/probo/pkg/proboctl/seed"
|
||||||
"go.probo.inc/probo/pkg/proboctl/version"
|
"go.probo.inc/probo/pkg/proboctl/version"
|
||||||
)
|
)
|
||||||
@@ -38,7 +41,17 @@ func NewCmdRoot(f *cmdutil.Factory) *cobra.Command {
|
|||||||
"PostgreSQL connection URL (default: DATABASE_URL env)",
|
"PostgreSQL connection URL (default: DATABASE_URL env)",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
cmd.PersistentFlags().StringVar(
|
||||||
|
&f.CfgFile,
|
||||||
|
"cfg-file",
|
||||||
|
os.Getenv("PROBOD_CFG_FILE"),
|
||||||
|
"Path to the probod config file (default: PROBOD_CFG_FILE env); required for agent-backed commands",
|
||||||
|
)
|
||||||
|
|
||||||
cmd.AddCommand(seed.NewCmdSeed(f))
|
cmd.AddCommand(seed.NewCmdSeed(f))
|
||||||
|
cmd.AddCommand(commontrackerpattern.NewCmdCommonTrackerPattern(f))
|
||||||
|
cmd.AddCommand(commonthirdparty.NewCmdCommonThirdParty(f))
|
||||||
|
cmd.AddCommand(proboctlcookiebanner.NewCmdCookieBanner(f))
|
||||||
cmd.AddCommand(version.NewCmdVersion(f))
|
cmd.AddCommand(version.NewCmdVersion(f))
|
||||||
|
|
||||||
return cmd
|
return cmd
|
||||||
|
|||||||
Reference in New Issue
Block a user