rohit/probo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add proboctl CLI and move seed commands into it

Browse Source 
Introduce a new proboctl Cobra CLI for Probo instance management
that connects directly to PostgreSQL. Move the standalone
common-third-parties-import and common-tracker-patterns-import
commands into proboctl as `proboctl seed common-third-parties`
and `proboctl seed common-tracker-patterns`, replacing flag-based
PG connection with a shared factory pattern.

Signed-off-by: Émile Ré <emile@probo.com>
This commit is contained in:
Émile Ré
2026-05-14 18:37:42 +04:00
parent 844817543d
commit 48a494461e
11 changed files with 520 additions and 386 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
pkg/proboctl/cmdutil/cmdutil.go Normal file
View File

@@ -0,0 +1,36 @@
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package cmdutil
import (
"fmt"
"go.gearno.de/kit/pg"
"go.probo.inc/probo/pkg/cmd/iostreams"
"go.probo.inc/probo/pkg/proboctl/pgconn"
)
type Factory struct {
IOStreams *iostreams.IOStreams
Version string
PgDSN string
}
func (f *Factory) PgClient() (*pg.Client, error) {
if f.PgDSN == "" {
return nil, fmt.Errorf("set --pg-dsn or DATABASE_URL")
}
return pgconn.NewPgClientFromDSN(f.PgDSN)
}

66
pkg/proboctl/pgconn/pgconn.go Normal file
View File

@@ -0,0 +1,66 @@
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package pgconn
import (
"fmt"
"net"
"net/url"
"go.gearno.de/kit/pg"
)
func NewPgClientFromDSN(dsn string) (*pg.Client, error) {
u, err := url.Parse(dsn)
if err != nil {
return nil, fmt.Errorf("cannot parse DSN (check URL format)")
}
var opts []pg.Option
switch u.Query().Get("sslmode") {
case "", "disable":
case "require":
opts = append(opts, pg.WithUnsecureTLS())
case "prefer":
return nil, fmt.Errorf(
"unsupported sslmode %q (prefer fallback semantics are not supported)",
u.Query().Get("sslmode"),
)
default:
return nil, fmt.Errorf("unsupported sslmode %q", u.Query().Get("sslmode"))
}
if u.Host != "" {
host := u.Host
if u.Port() == "" {
host = net.JoinHostPort(u.Hostname(), "5432")
}
opts = append(opts, pg.WithAddr(host))
}
if u.User != nil {
opts = append(opts, pg.WithUser(u.User.Username()))
if password, ok := u.User.Password(); ok {
opts = append(opts, pg.WithPassword(password))
}
}
if len(u.Path) > 1 {
opts = append(opts, pg.WithDatabase(u.Path[1:]))
}
return pg.NewClient(opts...)
}

45
pkg/proboctl/root/root.go Normal file
View File

@@ -0,0 +1,45 @@
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package root
import (
"os"
"github.com/spf13/cobra"
"go.probo.inc/probo/pkg/proboctl/cmdutil"
"go.probo.inc/probo/pkg/proboctl/seed"
"go.probo.inc/probo/pkg/proboctl/version"
)
func NewCmdRoot(f *cmdutil.Factory) *cobra.Command {
cmd := &cobra.Command{
Use: "proboctl <command> [flags]",
Short: "Probo instance management CLI",
SilenceUsage: true,
SilenceErrors: true,
}
cmd.PersistentFlags().StringVar(
&f.PgDSN,
"pg-dsn",
os.Getenv("DATABASE_URL"),
"PostgreSQL connection URL (default: DATABASE_URL env)",
)
cmd.AddCommand(seed.NewCmdSeed(f))
cmd.AddCommand(version.NewCmdVersion(f))
return cmd
}

428
pkg/proboctl/seed/common-third-parties/common_third_parties.go Normal file
View File

@@ -0,0 +1,428 @@
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package commonthirdparties
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"os"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/spf13/cobra"
"go.gearno.de/crypto/uuid"
"go.gearno.de/kit/httpclient"
"go.gearno.de/kit/pg"
"go.probo.inc/probo/pkg/coredata"
"go.probo.inc/probo/pkg/filemanager"
"go.probo.inc/probo/pkg/gid"
"go.probo.inc/probo/pkg/proboctl/cmdutil"
"go.probo.inc/probo/pkg/slug"
"go.probo.inc/probo/pkg/version"
"go.probo.inc/probo/pkg/webinspect"
)
type thirdPartyData struct {
Name string `json:"name"`
Category *string `json:"category,omitempty"`
HeadquarterAddress *string `json:"headquarterAddress,omitempty"`
LegalName *string `json:"legalName,omitempty"`
WebsiteURL *string `json:"websiteUrl,omitempty"`
PrivacyPolicyURL *string `json:"privacyPolicyUrl,omitempty"`
ServiceLevelAgreementURL *string `json:"serviceLevelAgreementUrl,omitempty"`
ServiceSoftwareAgreementURL *string `json:"serviceSoftwareAgreementUrl,omitempty"`
DataProcessingAgreementURL *string `json:"dataProcessingAgreementUrl,omitempty"`
BusinessAssociateAgreementURL *string `json:"businessAssociateAgreementUrl,omitempty"`
SubprocessorsListURL *string `json:"subprocessorsListUrl,omitempty"`
Certifications []string `json:"certifications,omitempty"`
StatusPageURL *string `json:"statusPageUrl,omitempty"`
TermsOfServiceURL *string `json:"termsOfServiceUrl,omitempty"`
SecurityPageURL *string `json:"securityPageUrl,omitempty"`
TrustPageURL *string `json:"trustPageUrl,omitempty"`
Domains []string `json:"domains,omitempty"`
}
func NewCmdCommonThirdParties(f *cmdutil.Factory) *cobra.Command {
var (
flagData string
flagFetchLogos bool
flagS3Bucket string
flagS3Endpoint string
flagS3Region string
flagS3AccessKey string
flagS3SecretKey string
flagS3UsePathStyle bool
)
cmd := &cobra.Command{
Use: "common-third-parties",
Short: "Seed common third parties from a data.json file",
Long: "Seed the common_third_parties table from a JSON file. " +
"Re-running is safe: existing rows are upserted on conflict (slug) " +
"so ids and created_at are preserved.",
RunE: func(cmd *cobra.Command, args []string) error {
out := f.IOStreams.Out
errOut := f.IOStreams.ErrOut
if flagFetchLogos && flagS3Bucket == "" {
return fmt.Errorf("set --s3-bucket or AWS_S3_BUCKET when using --fetch-logos")
}
ctx := cmd.Context()
thirdParties, err := loadThirdParties(flagData)
if err != nil {
return fmt.Errorf("cannot load third-party data: %w", err)
}
pgClient, err := f.PgClient()
if err != nil {
return fmt.Errorf("cannot create pg client: %w", err)
}
_, _ = fmt.Fprintf(out, "seeding %d common third parties from %s\n", len(thirdParties), flagData)
var inserted, updated, domainsInserted, domainsUpdated int
if err := pgClient.WithTx(
ctx,
func(ctx context.Context, tx pg.Tx) error {
now := time.Now()
for _, tp := range thirdParties {
party := coredata.CommonThirdParty{
ID: gid.New(gid.NilTenant, coredata.CommonThirdPartyEntityType),
Name: tp.Name,
Slug: slug.Make(tp.Name),
Category: parseCategory(errOut, tp),
HeadquarterAddress: tp.HeadquarterAddress,
LegalName: tp.LegalName,
WebsiteURL: tp.WebsiteURL,
PrivacyPolicyURL: tp.PrivacyPolicyURL,
ServiceLevelAgreementURL: tp.ServiceLevelAgreementURL,
ServiceSoftwareAgreementURL: tp.ServiceSoftwareAgreementURL,
DataProcessingAgreementURL: tp.DataProcessingAgreementURL,
BusinessAssociateAgreementURL: tp.BusinessAssociateAgreementURL,
SubprocessorsListURL: tp.SubprocessorsListURL,
Certifications: tp.Certifications,
StatusPageURL: tp.StatusPageURL,
TermsOfServiceURL: tp.TermsOfServiceURL,
SecurityPageURL: tp.SecurityPageURL,
TrustPageURL: tp.TrustPageURL,
CreatedAt: now,
UpdatedAt: now,
}
wasInserted, err := party.Upsert(ctx, tx)
if err != nil {
return fmt.Errorf("cannot upsert common third party %q: %w", tp.Name, err)
}
if wasInserted {
inserted++
} else {
updated++
if err := party.LoadByName(ctx, tx, tp.Name); err != nil {
return fmt.Errorf("cannot reload common third party %q: %w", tp.Name, err)
}
}
for _, domain := range tp.Domains {
d := coredata.CommonThirdPartyDomain{
ID: gid.New(gid.NilTenant, coredata.CommonThirdPartyDomainEntityType),
CommonThirdPartyID: party.ID,
Domain: domain,
CreatedAt: now,
UpdatedAt: now,
}
domainInserted, err := d.Upsert(ctx, tx)
if err != nil {
return fmt.Errorf("cannot upsert domain %q for %q: %w", domain, tp.Name, err)
}
if domainInserted {
domainsInserted++
} else {
domainsUpdated++
}
}
}
return nil
},
); err != nil {
return err
}
_, _ = fmt.Fprintf(out, "seeded %d third parties (%d inserted, %d updated)\n", len(thirdParties), inserted, updated)
_, _ = fmt.Fprintf(out, "seeded %d domains (%d inserted, %d updated)\n", domainsInserted+domainsUpdated, domainsInserted, domainsUpdated)
if flagFetchLogos {
if err := fetchAndStoreLogos(
ctx, out, errOut, pgClient, thirdParties,
flagS3Bucket, flagS3Endpoint, flagS3Region, flagS3AccessKey, flagS3SecretKey, flagS3UsePathStyle,
); err != nil {
return fmt.Errorf("cannot fetch logos: %w", err)
}
}
return nil
},
}
cmd.Flags().StringVar(&flagData, "data", "", "Path to the third-party data.json file")
_ = cmd.MarkFlagRequired("data")
cmd.Flags().BoolVar(&flagFetchLogos, "fetch-logos", false, "Fetch favicons and store them in S3")
cmd.Flags().StringVar(&flagS3Bucket, "s3-bucket", os.Getenv("AWS_S3_BUCKET"), "S3 bucket name (default: AWS_S3_BUCKET env)")
cmd.Flags().StringVar(&flagS3Endpoint, "s3-endpoint", os.Getenv("AWS_ENDPOINT_URL"), "S3 endpoint URL (default: AWS_ENDPOINT_URL env)")
cmd.Flags().StringVar(&flagS3Region, "s3-region", os.Getenv("AWS_REGION"), "S3 region (default: AWS_REGION env)")
cmd.Flags().StringVar(&flagS3AccessKey, "s3-access-key", os.Getenv("AWS_ACCESS_KEY_ID"), "S3 access key ID (default: AWS_ACCESS_KEY_ID env)")
cmd.Flags().StringVar(&flagS3SecretKey, "s3-secret-key", os.Getenv("AWS_SECRET_ACCESS_KEY"), "S3 secret access key (default: AWS_SECRET_ACCESS_KEY env)")
cmd.Flags().BoolVar(&flagS3UsePathStyle, "s3-path-style", false, "Use S3 path-style addressing")
return cmd
}
func fetchAndStoreLogos(
ctx context.Context,
out, errOut io.Writer,
pgClient *pg.Client,
thirdParties []thirdPartyData,
bucket, endpoint, region, accessKey, secretKey string,
usePathStyle bool,
) error {
s3Client := newS3Client(endpoint, region, accessKey, secretKey, usePathStyle)
fileMgr := filemanager.NewService(s3Client)
httpClient := httpclient.DefaultPooledClient(httpclient.WithSSRFProtection())
httpClient.Transport = &userAgentTransport{
next: httpClient.Transport,
ua: version.UserAgent("proboctl"),
}
scope := coredata.NewScope(gid.NilTenant)
var fetched, skipped, failed int
for _, tp := range thirdParties {
if tp.WebsiteURL == nil || *tp.WebsiteURL == "" {
skipped++
continue
}
var party coredata.CommonThirdParty
if err := pgClient.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error {
return party.LoadByName(ctx, conn, tp.Name)
}); err != nil {
_, _ = fmt.Fprintf(errOut, "warning: cannot load %q, skipping logo: %v\n", tp.Name, err)
failed++
continue
}
if party.LogoFileID != nil {
skipped++
continue
}
var logoURL string
pageInfo, err := webinspect.Parse(ctx, httpClient, *tp.WebsiteURL)
if err != nil {
_, _ = fmt.Fprintf(errOut, "warning: cannot inspect page for %q, trying default apple-touch-icon: %v\n", tp.Name, err)
} else {
logoURL, err = webinspect.FindLogoURL(pageInfo)
if err != nil {
_, _ = fmt.Fprintf(errOut, "warning: cannot find logo for %q, trying default apple-touch-icon: %v\n", tp.Name, err)
}
}
parsed, err := url.Parse(*tp.WebsiteURL)
if err != nil {
_, _ = fmt.Fprintf(errOut, "warning: cannot parse URL for %q, skipping logo: %v\n", tp.Name, err)
failed++
continue
}
var candidateURLs []string
if logoURL != "" {
candidateURLs = append(candidateURLs, logoURL)
}
base := fmt.Sprintf("%s://%s", parsed.Scheme, parsed.Host)
candidateURLs = append(
candidateURLs,
base+"/apple-touch-icon.png",
base+"/apple-touch-icon-precomposed.png",
"https://logo.debounce.com/"+parsed.Host,
)
var (
body []byte
contentType string
)
for _, candidate := range candidateURLs {
resp, err := httpClient.Get(candidate)
if err != nil {
continue
}
b, err := io.ReadAll(resp.Body)
_ = resp.Body.Close()
if err != nil || resp.StatusCode != http.StatusOK || len(b) == 0 {
continue
}
body = b
contentType = resp.Header.Get("Content-Type")
break
}
if len(body) == 0 {
_, _ = fmt.Fprintf(errOut, "warning: cannot fetch logo for %q from any candidate URL\n", tp.Name)
failed++
continue
}
if contentType == "" {
contentType = "image/png"
}
objectKey, err := uuid.NewV7()
if err != nil {
return fmt.Errorf("cannot generate object key: %w", err)
}
now := time.Now()
fileID := gid.New(gid.NilTenant, coredata.FileEntityType)
fileRecord := &coredata.File{
ID: fileID,
OrganizationID: gid.Nil,
BucketName: bucket,
MimeType: contentType,
FileName: tp.Name + "-logo" + webinspect.ExtensionForMIME(contentType),
FileKey: objectKey.String(),
FileSize: int64(len(body)),
Visibility: coredata.FileVisibilityPublic,
CreatedAt: now,
UpdatedAt: now,
}
if _, err := fileMgr.PutFile(ctx, fileRecord, bytes.NewReader(body), map[string]string{
"type": "common-third-party-logo",
"common-third-party-id": party.ID.String(),
}); err != nil {
_, _ = fmt.Fprintf(errOut, "warning: cannot upload logo for %q to S3: %v\n", tp.Name, err)
failed++
continue
}
if err := pgClient.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error {
if err := fileRecord.Insert(ctx, tx, scope); err != nil {
return fmt.Errorf("cannot insert file record: %w", err)
}
party.LogoFileID = &fileID
party.UpdatedAt = now
if err := party.UpdateLogoFileID(ctx, tx); err != nil {
return fmt.Errorf("cannot update logo_file_id: %w", err)
}
return nil
}); err != nil {
_, _ = fmt.Fprintf(errOut, "warning: cannot store logo for %q: %v\n", tp.Name, err)
failed++
continue
}
fetched++
_, _ = fmt.Fprintf(out, " fetched logo for %q\n", tp.Name)
}
_, _ = fmt.Fprintf(out, "logos: %d fetched, %d skipped, %d failed\n", fetched, skipped, failed)
return nil
}
func newS3Client(endpoint, region, accessKey, secretKey string, usePathStyle bool) *s3.Client {
if region == "" {
region = "us-east-2"
}
cfg := aws.Config{
Region: region,
}
if accessKey != "" && secretKey != "" {
cfg.Credentials = credentials.NewStaticCredentialsProvider(accessKey, secretKey, "")
}
if endpoint != "" {
cfg.BaseEndpoint = &endpoint
}
return s3.NewFromConfig(cfg, func(o *s3.Options) {
o.UsePathStyle = usePathStyle
})
}
func loadThirdParties(path string) ([]thirdPartyData, error) {
f, err := os.Open(path)
if err != nil {
return nil, fmt.Errorf("cannot open %s: %w", path, err)
}
defer func() { _ = f.Close() }()
var thirdParties []thirdPartyData
dec := json.NewDecoder(f)
dec.DisallowUnknownFields()
if err := dec.Decode(&thirdParties); err != nil {
return nil, fmt.Errorf("cannot decode %s: %w", path, err)
}
return thirdParties, nil
}
func parseCategory(errOut io.Writer, tp thirdPartyData) coredata.ThirdPartyCategory {
if tp.Category == nil || *tp.Category == "" {
return coredata.ThirdPartyCategoryOther
}
var c coredata.ThirdPartyCategory
if err := c.Scan(*tp.Category); err != nil {
_, _ = fmt.Fprintf(errOut, "warning: third party %q has unknown category %q, falling back to OTHER\n", tp.Name, *tp.Category)
return coredata.ThirdPartyCategoryOther
}
return c
}
type userAgentTransport struct {
next http.RoundTripper
ua string
}
func (t *userAgentTransport) RoundTrip(req *http.Request) (*http.Response, error) {
req = req.Clone(req.Context())
req.Header.Set("User-Agent", t.ua)
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8")
req.Header.Set("Accept-Language", "en-US,en;q=0.5")
return t.next.RoundTrip(req)
}

468
pkg/proboctl/seed/common-tracker-patterns/common_tracker_patterns.go Normal file
View File

@@ -0,0 +1,468 @@
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package commontrackerpatterns
import (
"context"
"encoding/json"
"errors"
"fmt"
"math"
"os"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"time"
git "github.com/go-git/go-git/v5"
"github.com/spf13/cobra"
"go.gearno.de/kit/pg"
"go.probo.inc/probo/pkg/coredata"
"go.probo.inc/probo/pkg/gid"
"go.probo.inc/probo/pkg/proboctl/cmdutil"
"go.probo.inc/probo/pkg/slug"
)
const (
ocdRepoURL = "https://github.com/jkwakman/Open-Cookie-Database.git"
ocdJSONFile = "open-cookie-database.json"
)
type (
ocdEntry struct {
ID string `json:"id"`
Category string `json:"category"`
Cookie string `json:"cookie"`
Domain string `json:"domain"`
Description string `json:"description"`
RetentionPeriod string `json:"retentionPeriod"`
DataController string `json:"dataController"`
PrivacyLink string `json:"privacyLink"`
WildcardMatch string `json:"wildcardMatch"`
}
trackerPatternData struct {
Pattern string
TrackerType string
MatchType string
ThirdPartyName *string
Domain string
Category string
Description string
MaxAgeSeconds *int
Confidence float32
}
)
func NewCmdCommonTrackerPatterns(f *cmdutil.Factory) *cobra.Command {
return &cobra.Command{
Use: "common-tracker-patterns",
Short: "Seed common tracker patterns from the Open Cookie Database",
Long: "Seed the common_tracker_patterns table from the Open Cookie Database " +
"(https://github.com/jkwakman/Open-Cookie-Database). " +
"The repository is cloned into a temporary directory. " +
"Re-running is safe: existing rows are upserted so ids and created_at are preserved. " +
"Entries are linked to matching common_third_parties rows via slug lookup, " +
"domain lookup, then auto-create.",
RunE: func(cmd *cobra.Command, args []string) error {
out := f.IOStreams.Out
errOut := f.IOStreams.ErrOut
ctx := cmd.Context()
_, _ = fmt.Fprintf(out, "cloning %s\n", ocdRepoURL)
tmpDir, cleanup, err := cloneRepo()
if err != nil {
return fmt.Errorf("cannot clone repository: %w", err)
}
defer cleanup()
patterns, err := loadPatternsFromOCD(tmpDir)
if err != nil {
return fmt.Errorf("cannot load tracker pattern data: %w", err)
}
pgClient, err := f.PgClient()
if err != nil {
return fmt.Errorf("cannot create pg client: %w", err)
}
_, _ = fmt.Fprintf(out, "seeding %d common tracker patterns from Open Cookie Database\n", len(patterns))
var inserted, updated, skipped int
var partiesCreated int
if err := pgClient.WithTx(
ctx,
func(ctx context.Context, tx pg.Tx) error {
now := time.Now()
thirdPartyCache := make(map[string]*gid.GID)
for _, p := range patterns {
thirdPartyID, err := resolveThirdParty(ctx, tx, p, thirdPartyCache, now, &partiesCreated)
if err != nil {
return err
}
trackerType, err := parseTrackerType(p.TrackerType)
if err != nil {
_, _ = fmt.Fprintf(errOut, "warning: %v, skipping pattern %q\n", err, p.Pattern)
skipped++
continue
}
matchType, err := parseMatchType(p.MatchType)
if err != nil {
_, _ = fmt.Fprintf(errOut, "warning: %v, skipping pattern %q\n", err, p.Pattern)
skipped++
continue
}
pattern := coredata.CommonTrackerPattern{
ID: gid.New(gid.NilTenant, coredata.CommonTrackerPatternEntityType),
CommonThirdPartyID: thirdPartyID,
TrackerType: trackerType,
Pattern: p.Pattern,
MatchType: matchType,
Description: p.Description,
MaxAgeSeconds: p.MaxAgeSeconds,
Confidence: p.Confidence,
CreatedAt: now,
UpdatedAt: now,
}
_, wasInserted, err := pattern.Upsert(ctx, tx)
if err != nil {
return fmt.Errorf("cannot upsert common tracker pattern %q: %w", p.Pattern, err)
}
if wasInserted {
inserted++
} else {
updated++
}
}
return nil
},
); err != nil {
return err
}
_, _ = fmt.Fprintf(
out,
"seeded %d patterns (%d inserted, %d updated, %d skipped, %d third parties auto-created)\n",
len(patterns)-skipped,
inserted,
updated,
skipped,
partiesCreated,
)
return nil
},
}
}
func cloneRepo() (string, func(), error) {
tmpDir, err := os.MkdirTemp("", "ocd-*")
if err != nil {
return "", nil, fmt.Errorf("cannot create temp dir: %w", err)
}
cleanup := func() { _ = os.RemoveAll(tmpDir) }
_, err = git.PlainClone(
tmpDir,
false,
&git.CloneOptions{
URL: ocdRepoURL,
Depth: 1,
},
)
if err != nil {
cleanup()
return "", nil, fmt.Errorf("cannot clone %s: %w", ocdRepoURL, err)
}
return tmpDir, cleanup, nil
}
func loadPatternsFromOCD(dir string) ([]trackerPatternData, error) {
f, err := os.Open(filepath.Join(dir, ocdJSONFile))
if err != nil {
return nil, fmt.Errorf("cannot open %s: %w", ocdJSONFile, err)
}
defer func() { _ = f.Close() }()
var db map[string][]ocdEntry
if err := json.NewDecoder(f).Decode(&db); err != nil {
return nil, fmt.Errorf("cannot decode %s: %w", ocdJSONFile, err)
}
platforms := make([]string, 0, len(db))
for k := range db {
platforms = append(platforms, k)
}
sort.Strings(platforms)
var patterns []trackerPatternData
for _, platform := range platforms {
for _, e := range db[platform] {
if e.Cookie == "" {
continue
}
matchType := "EXACT"
if e.WildcardMatch == "1" {
matchType = "GLOB"
}
cookiePattern := e.Cookie
if matchType == "GLOB" && !strings.ContainsAny(cookiePattern, "*?") {
cookiePattern += "*"
}
patterns = append(
patterns,
trackerPatternData{
Pattern: cookiePattern,
TrackerType: "COOKIE",
MatchType: matchType,
ThirdPartyName: new(platform),
Domain: e.Domain,
Category: e.Category,
Description: e.Description,
MaxAgeSeconds: parseRetentionPeriod(e.RetentionPeriod),
Confidence: 1.0,
},
)
}
}
return patterns, nil
}
func resolveThirdParty(
ctx context.Context,
tx pg.Tx,
p trackerPatternData,
cache map[string]*gid.GID,
now time.Time,
created *int,
) (*gid.GID, error) {
if p.ThirdPartyName == nil || *p.ThirdPartyName == "" {
return nil, nil
}
platformSlug := slug.Make(*p.ThirdPartyName)
if platformSlug == "" {
return nil, nil
}
if cached, ok := cache[platformSlug]; ok {
return cached, nil
}
var party coredata.CommonThirdParty
if err := party.LoadBySlug(ctx, tx, platformSlug); err != nil {
if !errors.Is(err, coredata.ErrResourceNotFound) {
return nil, fmt.Errorf("cannot look up common third party by slug %q: %w", platformSlug, err)
}
} else {
cache[platformSlug] = &party.ID
return &party.ID, nil
}
domain := normalizeDomain(p.Domain)
if domain != "" {
var domainRow coredata.CommonThirdPartyDomain
if err := domainRow.LoadByDomain(ctx, tx, domain); err != nil {
if !errors.Is(err, coredata.ErrResourceNotFound) {
return nil, fmt.Errorf("cannot look up common third party by domain %q: %w", domain, err)
}
} else {
if err := party.LoadByID(ctx, tx, domainRow.CommonThirdPartyID); err != nil {
return nil, fmt.Errorf("cannot load common third party by ID %s: %w", domainRow.CommonThirdPartyID, err)
}
cache[platformSlug] = &party.ID
return &party.ID, nil
}
}
party = coredata.CommonThirdParty{
ID: gid.New(gid.NilTenant, coredata.CommonThirdPartyEntityType),
Name: *p.ThirdPartyName,
Slug: platformSlug,
Category: mapOCDCategory(p.Category),
Certifications: []string{},
CreatedAt: now,
UpdatedAt: now,
}
if _, err := party.Upsert(ctx, tx); err != nil {
return nil, fmt.Errorf("cannot auto-create common third party %q: %w", *p.ThirdPartyName, err)
}
if err := party.LoadBySlug(ctx, tx, platformSlug); err != nil {
return nil, fmt.Errorf("cannot reload auto-created common third party %q: %w", *p.ThirdPartyName, err)
}
if domain != "" {
d := coredata.CommonThirdPartyDomain{
ID: gid.New(gid.NilTenant, coredata.CommonThirdPartyDomainEntityType),
CommonThirdPartyID: party.ID,
Domain: domain,
CreatedAt: now,
UpdatedAt: now,
}
if _, err := d.Upsert(ctx, tx); err != nil {
return nil, fmt.Errorf("cannot upsert domain %q for %q: %w", domain, *p.ThirdPartyName, err)
}
}
*created++
cache[platformSlug] = &party.ID
return &party.ID, nil
}
var domainValidRe = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9.-]*[a-zA-Z]$`)
func normalizeDomain(s string) string {
s = strings.TrimSpace(s)
s = strings.Map(func(r rune) rune {
if r == '\u200b' || r == '\ufeff' {
return -1
}
return r
}, s)
if idx := strings.Index(s, " or "); idx != -1 {
s = s[:idx]
}
s = strings.TrimPrefix(s, "or ")
if idx := strings.IndexByte(s, '('); idx != -1 {
s = strings.TrimSpace(s[:idx])
}
if strings.ContainsAny(s, "[]") {
return ""
}
s = strings.TrimPrefix(s, ".")
s = strings.TrimSuffix(s, ".")
s = strings.TrimSpace(s)
if s == "" || !domainValidRe.MatchString(s) {
return ""
}
return s
}
func mapOCDCategory(s string) coredata.ThirdPartyCategory {
switch strings.ToLower(strings.TrimSpace(s)) {
case "analytics":
return coredata.ThirdPartyCategoryAnalytics
case "marketing":
return coredata.ThirdPartyCategoryMarketing
default:
return coredata.ThirdPartyCategoryOther
}
}
var retentionRe = regexp.MustCompile(`(?i)^(\d+)\s+(second|seconds|sec|secs|minute|minutes|mins|min|hour|hours|day|days|week|weeks|month|months|year|years)`)
func parseRetentionPeriod(s string) *int {
s = strings.TrimSpace(s)
if s == "" {
return nil
}
lower := strings.ToLower(s)
switch {
case lower == "session" || lower == "sessions" || lower == "seesion" ||
lower == "session cookie" || strings.HasPrefix(lower, "end of session"):
return nil
case lower == "varies" || lower == "various" || lower == "unknown" ||
lower == "undefined" || lower == "persistent" || lower == "permanent" ||
lower == "forever" || lower == "unlimited" || lower == "no expiration" ||
lower == "local storage":
return nil
}
m := retentionRe.FindStringSubmatch(s)
if m == nil {
return nil
}
n, err := strconv.Atoi(m[1])
if err != nil {
return nil
}
var multiplier int
switch strings.ToLower(m[2]) {
case "second", "seconds", "sec", "secs":
multiplier = 1
case "minute", "minutes", "mins", "min":
multiplier = 60
case "hour", "hours":
multiplier = 3600
case "day", "days":
multiplier = 86400
case "week", "weeks":
multiplier = 604800
case "month", "months":
multiplier = 2592000
case "year", "years":
multiplier = 31536000
default:
return nil
}
result := min(n*multiplier, math.MaxInt32)
return &result
}
func parseTrackerType(s string) (coredata.TrackerType, error) {
switch s {
case "COOKIE":
return coredata.TrackerTypeCookie, nil
case "LOCAL_STORAGE":
return coredata.TrackerTypeLocalStorage, nil
case "SESSION_STORAGE":
return coredata.TrackerTypeSessionStorage, nil
case "INDEXED_DB":
return coredata.TrackerTypeIndexedDB, nil
default:
return "", fmt.Errorf("unknown tracker type %q", s)
}
}
func parseMatchType(s string) (coredata.TrackerPatternMatchType, error) {
switch s {
case "EXACT":
return coredata.TrackerPatternMatchTypeExact, nil
case "GLOB":
return coredata.TrackerPatternMatchTypeGlob, nil
case "PREFIX":
return coredata.TrackerPatternMatchTypePrefix, nil
default:
return "", fmt.Errorf("unknown match type %q", s)
}
}

34
pkg/proboctl/seed/seed.go Normal file
View File

@@ -0,0 +1,34 @@
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package seed
import (
"github.com/spf13/cobra"
"go.probo.inc/probo/pkg/proboctl/cmdutil"
commonthirdparties "go.probo.inc/probo/pkg/proboctl/seed/common-third-parties"
commontrackerpatterns "go.probo.inc/probo/pkg/proboctl/seed/common-tracker-patterns"
)
func NewCmdSeed(f *cmdutil.Factory) *cobra.Command {
cmd := &cobra.Command{
Use: "seed <command>",
Short: "Seed common reference data",
}
cmd.AddCommand(commonthirdparties.NewCmdCommonThirdParties(f))
cmd.AddCommand(commontrackerpatterns.NewCmdCommonTrackerPatterns(f))
return cmd
}

49
pkg/proboctl/version/version.go Normal file
View File

@@ -0,0 +1,49 @@
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package version
import (
"fmt"
"github.com/spf13/cobra"
"go.probo.inc/probo/pkg/proboctl/cmdutil"
"go.probo.inc/probo/pkg/version"
)
func NewCmdVersion(f *cmdutil.Factory) *cobra.Command {
return &cobra.Command{
Use: "version",
Short: "Print the version of proboctl",
RunE: func(cmd *cobra.Command, args []string) error {
info := version.GetBuildInfo()
v := f.Version
if v == "" || v == "unknown" {
v = info.Version
}
_, _ = fmt.Fprintf(
f.IOStreams.Out,
"proboctl version %s (commit: %s, built: %s, go: %s)\n",
v,
info.Commit,
info.BuildDate,
info.GoVersion,
)
return nil
},
}
}