Add proboctl CLI and move seed commands into it

Introduce a new proboctl Cobra CLI for Probo instance management
that connects directly to PostgreSQL. Move the standalone
common-third-parties-import and common-tracker-patterns-import
commands into proboctl as `proboctl seed common-third-parties`
and `proboctl seed common-tracker-patterns`, replacing flag-based
PG connection with a shared factory pattern.

Signed-off-by: Émile Ré <emile@probo.com>
This commit is contained in:
Émile Ré
2026-05-14 18:37:42 +04:00
parent 844817543d
commit 48a494461e
11 changed files with 520 additions and 386 deletions

View File

@@ -25,10 +25,12 @@ DOCKER_COMPOSE= $(DOCKER) compose -f compose.yaml $(DOCKER_COMPOSE_FLAGS)
PRB_VERSION= $(shell cat cmd/prb/VERSION) PRB_VERSION= $(shell cat cmd/prb/VERSION)
PROBOD_VERSION= $(shell cat cmd/probod/VERSION) PROBOD_VERSION= $(shell cat cmd/probod/VERSION)
PROBOD_BOOTSTRAP_VERSION=$(shell cat cmd/probod-bootstrap/VERSION) PROBOD_BOOTSTRAP_VERSION=$(shell cat cmd/probod-bootstrap/VERSION)
PROBOCTL_VERSION= $(shell cat cmd/proboctl/VERSION)
PRB_LDFLAGS= -ldflags "-X 'main.version=$(PRB_VERSION)'" PRB_LDFLAGS= -ldflags "-X 'main.version=$(PRB_VERSION)'"
PROBOD_LDFLAGS= -ldflags "-X 'main.version=$(PROBOD_VERSION)' -X 'main.env=prod'" PROBOD_LDFLAGS= -ldflags "-X 'main.version=$(PROBOD_VERSION)' -X 'main.env=prod'"
PROBOD_BOOTSTRAP_LDFLAGS=-ldflags "-X 'main.version=$(PROBOD_BOOTSTRAP_VERSION)'" PROBOD_BOOTSTRAP_LDFLAGS=-ldflags "-X 'main.version=$(PROBOD_BOOTSTRAP_VERSION)'"
PROBOCTL_LDFLAGS= -ldflags "-X 'main.version=$(PROBOCTL_VERSION)'"
GCFLAGS= -gcflags="-e" GCFLAGS= -gcflags="-e"
@@ -73,6 +75,9 @@ PRB_SRC= cmd/prb/main.go
PROBOD_BOOTSTRAP_BIN= bin/probod-bootstrap PROBOD_BOOTSTRAP_BIN= bin/probod-bootstrap
PROBOD_BOOTSTRAP_SRC= cmd/probod-bootstrap/main.go PROBOD_BOOTSTRAP_SRC= cmd/probod-bootstrap/main.go
PROBOCTL_BIN= bin/proboctl
PROBOCTL_SRC= cmd/proboctl/main.go
ifdef WITH_APPS ifdef WITH_APPS
GENERATED += relay GENERATED += relay
EMBEDDED += \ EMBEDDED += \
@@ -172,7 +177,7 @@ coverage-combined: coverage-report test-e2e-coverage ## Generate combined covera
$(GO) tool cover -html=coverage-combined.out -o=coverage-combined.html $(GO) tool cover -html=coverage-combined.out -o=coverage-combined.html
.PHONY: build .PHONY: build
build: $(PROBOD_BIN) bin/prb bin/probod-bootstrap build: $(PROBOD_BIN) bin/prb bin/probod-bootstrap bin/proboctl
CFG_DEV_OAUTH2_KEY = cfg/.dev-oauth2-signing-key.pem CFG_DEV_OAUTH2_KEY = cfg/.dev-oauth2-signing-key.pem
DEV_ENV = .env DEV_ENV = .env
@@ -252,6 +257,10 @@ bin/prb:
$(PROBOD_BOOTSTRAP_BIN): $(PROBOD_BOOTSTRAP_BIN):
$(GO_BUILD) $(PROBOD_BOOTSTRAP_LDFLAGS) -o $(PROBOD_BOOTSTRAP_BIN) $(PROBOD_BOOTSTRAP_SRC) $(GO_BUILD) $(PROBOD_BOOTSTRAP_LDFLAGS) -o $(PROBOD_BOOTSTRAP_BIN) $(PROBOD_BOOTSTRAP_SRC)
.PHONY: bin/proboctl
bin/proboctl:
$(GO_BUILD) $(PROBOCTL_LDFLAGS) -o $(PROBOCTL_BIN) $(PROBOCTL_SRC)
.PHONY: @probo/emails .PHONY: @probo/emails
@probo/emails: @probo/emails:
$(NPM) --workspace $@ run build $(NPM) --workspace $@ run build

View File

@@ -0,0 +1 @@
# Changelog

1
cmd/proboctl/VERSION Normal file
View File

@@ -0,0 +1 @@
0.0.0

42
cmd/proboctl/main.go Normal file
View File

@@ -0,0 +1,42 @@
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package main
import (
"fmt"
"os"
"go.probo.inc/probo/pkg/cmd/iostreams"
"go.probo.inc/probo/pkg/proboctl/cmdutil"
"go.probo.inc/probo/pkg/proboctl/root"
)
var version string = "unknown"
func main() {
ios := iostreams.System()
f := &cmdutil.Factory{
IOStreams: ios,
Version: version,
}
cmd := root.NewCmdRoot(f)
if err := cmd.Execute(); err != nil {
fmt.Fprintf(os.Stderr, "Error: %s\n", err)
os.Exit(1)
}
}

View File

@@ -0,0 +1,36 @@
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package cmdutil
import (
"fmt"
"go.gearno.de/kit/pg"
"go.probo.inc/probo/pkg/cmd/iostreams"
"go.probo.inc/probo/pkg/proboctl/pgconn"
)
type Factory struct {
IOStreams *iostreams.IOStreams
Version string
PgDSN string
}
func (f *Factory) PgClient() (*pg.Client, error) {
if f.PgDSN == "" {
return nil, fmt.Errorf("set --pg-dsn or DATABASE_URL")
}
return pgconn.NewPgClientFromDSN(f.PgDSN)
}

View File

@@ -0,0 +1,66 @@
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package pgconn
import (
"fmt"
"net"
"net/url"
"go.gearno.de/kit/pg"
)
func NewPgClientFromDSN(dsn string) (*pg.Client, error) {
u, err := url.Parse(dsn)
if err != nil {
return nil, fmt.Errorf("cannot parse DSN (check URL format)")
}
var opts []pg.Option
switch u.Query().Get("sslmode") {
case "", "disable":
case "require":
opts = append(opts, pg.WithUnsecureTLS())
case "prefer":
return nil, fmt.Errorf(
"unsupported sslmode %q (prefer fallback semantics are not supported)",
u.Query().Get("sslmode"),
)
default:
return nil, fmt.Errorf("unsupported sslmode %q", u.Query().Get("sslmode"))
}
if u.Host != "" {
host := u.Host
if u.Port() == "" {
host = net.JoinHostPort(u.Hostname(), "5432")
}
opts = append(opts, pg.WithAddr(host))
}
if u.User != nil {
opts = append(opts, pg.WithUser(u.User.Username()))
if password, ok := u.User.Password(); ok {
opts = append(opts, pg.WithPassword(password))
}
}
if len(u.Path) > 1 {
opts = append(opts, pg.WithDatabase(u.Path[1:]))
}
return pg.NewClient(opts...)
}

45
pkg/proboctl/root/root.go Normal file
View File

@@ -0,0 +1,45 @@
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package root
import (
"os"
"github.com/spf13/cobra"
"go.probo.inc/probo/pkg/proboctl/cmdutil"
"go.probo.inc/probo/pkg/proboctl/seed"
"go.probo.inc/probo/pkg/proboctl/version"
)
func NewCmdRoot(f *cmdutil.Factory) *cobra.Command {
cmd := &cobra.Command{
Use: "proboctl <command> [flags]",
Short: "Probo instance management CLI",
SilenceUsage: true,
SilenceErrors: true,
}
cmd.PersistentFlags().StringVar(
&f.PgDSN,
"pg-dsn",
os.Getenv("DATABASE_URL"),
"PostgreSQL connection URL (default: DATABASE_URL env)",
)
cmd.AddCommand(seed.NewCmdSeed(f))
cmd.AddCommand(version.NewCmdVersion(f))
return cmd
}

View File

@@ -12,24 +12,14 @@
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR // OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE. // PERFORMANCE OF THIS SOFTWARE.
// Command common-third-parties-import seeds the common_third_parties table from package commonthirdparties
// packages/thirdParties/data.json. It is idempotent: re-running upserts on conflict
// (slug) so existing rows keep their id and created_at.
//
// When -fetch-logos is set, the tool inspects each third party's website to
// find the best available logo (SVG icon, apple-touch-icon, etc.) and stores
// it in S3 as a public file, linking it to each common third party via
// logo_file_id.
package main
import ( import (
"bytes" "bytes"
"context" "context"
"encoding/json" "encoding/json"
"flag"
"fmt" "fmt"
"io" "io"
"net"
"net/http" "net/http"
"net/url" "net/url"
"os" "os"
@@ -38,12 +28,14 @@ import (
"github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/spf13/cobra"
"go.gearno.de/crypto/uuid" "go.gearno.de/crypto/uuid"
"go.gearno.de/kit/httpclient" "go.gearno.de/kit/httpclient"
"go.gearno.de/kit/pg" "go.gearno.de/kit/pg"
"go.probo.inc/probo/pkg/coredata" "go.probo.inc/probo/pkg/coredata"
"go.probo.inc/probo/pkg/filemanager" "go.probo.inc/probo/pkg/filemanager"
"go.probo.inc/probo/pkg/gid" "go.probo.inc/probo/pkg/gid"
"go.probo.inc/probo/pkg/proboctl/cmdutil"
"go.probo.inc/probo/pkg/slug" "go.probo.inc/probo/pkg/slug"
"go.probo.inc/probo/pkg/version" "go.probo.inc/probo/pkg/version"
"go.probo.inc/probo/pkg/webinspect" "go.probo.inc/probo/pkg/webinspect"
@@ -69,191 +61,151 @@ type thirdPartyData struct {
Domains []string `json:"domains,omitempty"` Domains []string `json:"domains,omitempty"`
} }
func main() { func NewCmdCommonThirdParties(f *cmdutil.Factory) *cobra.Command {
if err := run(); err != nil {
fmt.Fprintf(os.Stderr, "error: %v\n", err)
os.Exit(1)
}
}
func run() error {
var ( var (
pgDSN string flagData string
dataPath string flagFetchLogos bool
fetchLogos bool flagS3Bucket string
s3Bucket string flagS3Endpoint string
s3Endpoint string flagS3Region string
s3Region string flagS3AccessKey string
s3AccessKey string flagS3SecretKey string
s3SecretKey string flagS3UsePathStyle bool
s3UsePathStyle bool
) )
flag.StringVar( cmd := &cobra.Command{
&pgDSN, Use: "common-third-parties",
"pg-dsn", Short: "Seed common third parties from a data.json file",
os.Getenv("DATABASE_URL"), Long: "Seed the common_third_parties table from a JSON file. " +
"PostgreSQL connection URL (default: DATABASE_URL env)", "Re-running is safe: existing rows are upserted on conflict (slug) " +
) "so ids and created_at are preserved.",
flag.StringVar( RunE: func(cmd *cobra.Command, args []string) error {
&dataPath, out := f.IOStreams.Out
"data", errOut := f.IOStreams.ErrOut
"",
"Path to the third-party data.json file",
)
flag.BoolVar(
&fetchLogos,
"fetch-logos",
false,
"Fetch favicons from Google and store them in S3",
)
flag.StringVar(
&s3Bucket,
"s3-bucket",
os.Getenv("AWS_S3_BUCKET"),
"S3 bucket name (default: AWS_S3_BUCKET env)",
)
flag.StringVar(
&s3Endpoint,
"s3-endpoint",
os.Getenv("AWS_ENDPOINT_URL"),
"S3 endpoint URL (default: AWS_ENDPOINT_URL env)",
)
flag.StringVar(
&s3Region,
"s3-region",
os.Getenv("AWS_REGION"),
"S3 region (default: AWS_REGION env)",
)
flag.StringVar(
&s3AccessKey,
"s3-access-key",
os.Getenv("AWS_ACCESS_KEY_ID"),
"S3 access key ID (default: AWS_ACCESS_KEY_ID env)",
)
flag.StringVar(
&s3SecretKey,
"s3-secret-key",
os.Getenv("AWS_SECRET_ACCESS_KEY"),
"S3 secret access key (default: AWS_SECRET_ACCESS_KEY env)",
)
flag.BoolVar(
&s3UsePathStyle,
"s3-path-style",
false,
"Use S3 path-style addressing",
)
flag.Parse()
if pgDSN == "" { if flagFetchLogos && flagS3Bucket == "" {
return fmt.Errorf("set -pg-dsn or DATABASE_URL") return fmt.Errorf("set --s3-bucket or AWS_S3_BUCKET when using --fetch-logos")
} }
if fetchLogos && s3Bucket == "" { ctx := cmd.Context()
return fmt.Errorf("set -s3-bucket or AWS_S3_BUCKET when using -fetch-logos")
}
ctx := context.Background() thirdParties, err := loadThirdParties(flagData)
if err != nil {
return fmt.Errorf("cannot load third-party data: %w", err)
}
thirdParties, err := loadThirdParties(dataPath) pgClient, err := f.PgClient()
if err != nil { if err != nil {
return fmt.Errorf("cannot load third-party data: %w", err) return fmt.Errorf("cannot create pg client: %w", err)
} }
pgClient, err := newPgClientFromDSN(pgDSN) _, _ = fmt.Fprintf(out, "seeding %d common third parties from %s\n", len(thirdParties), flagData)
if err != nil {
return fmt.Errorf("cannot create pg client: %w", err)
}
fmt.Printf("importing %d common third parties from %s\n", len(thirdParties), dataPath) var inserted, updated, domainsInserted, domainsUpdated int
var inserted, updated, domainsInserted, domainsUpdated int if err := pgClient.WithTx(
ctx,
func(ctx context.Context, tx pg.Tx) error {
now := time.Now()
if err := pgClient.WithTx( for _, tp := range thirdParties {
ctx, party := coredata.CommonThirdParty{
func(ctx context.Context, tx pg.Tx) error { ID: gid.New(gid.NilTenant, coredata.CommonThirdPartyEntityType),
now := time.Now() Name: tp.Name,
Slug: slug.Make(tp.Name),
Category: parseCategory(errOut, tp),
HeadquarterAddress: tp.HeadquarterAddress,
LegalName: tp.LegalName,
WebsiteURL: tp.WebsiteURL,
PrivacyPolicyURL: tp.PrivacyPolicyURL,
ServiceLevelAgreementURL: tp.ServiceLevelAgreementURL,
ServiceSoftwareAgreementURL: tp.ServiceSoftwareAgreementURL,
DataProcessingAgreementURL: tp.DataProcessingAgreementURL,
BusinessAssociateAgreementURL: tp.BusinessAssociateAgreementURL,
SubprocessorsListURL: tp.SubprocessorsListURL,
Certifications: tp.Certifications,
StatusPageURL: tp.StatusPageURL,
TermsOfServiceURL: tp.TermsOfServiceURL,
SecurityPageURL: tp.SecurityPageURL,
TrustPageURL: tp.TrustPageURL,
CreatedAt: now,
UpdatedAt: now,
}
for _, tp := range thirdParties { wasInserted, err := party.Upsert(ctx, tx)
party := coredata.CommonThirdParty{ if err != nil {
ID: gid.New(gid.NilTenant, coredata.CommonThirdPartyEntityType), return fmt.Errorf("cannot upsert common third party %q: %w", tp.Name, err)
Name: tp.Name, }
Slug: slug.Make(tp.Name),
Category: parseCategory(tp),
HeadquarterAddress: tp.HeadquarterAddress,
LegalName: tp.LegalName,
WebsiteURL: tp.WebsiteURL,
PrivacyPolicyURL: tp.PrivacyPolicyURL,
ServiceLevelAgreementURL: tp.ServiceLevelAgreementURL,
ServiceSoftwareAgreementURL: tp.ServiceSoftwareAgreementURL,
DataProcessingAgreementURL: tp.DataProcessingAgreementURL,
BusinessAssociateAgreementURL: tp.BusinessAssociateAgreementURL,
SubprocessorsListURL: tp.SubprocessorsListURL,
Certifications: tp.Certifications,
StatusPageURL: tp.StatusPageURL,
TermsOfServiceURL: tp.TermsOfServiceURL,
SecurityPageURL: tp.SecurityPageURL,
TrustPageURL: tp.TrustPageURL,
CreatedAt: now,
UpdatedAt: now,
}
wasInserted, err := party.Upsert(ctx, tx) if wasInserted {
if err != nil { inserted++
return fmt.Errorf("cannot upsert common third party %q: %w", tp.Name, err) } else {
} updated++
if err := party.LoadByName(ctx, tx, tp.Name); err != nil {
return fmt.Errorf("cannot reload common third party %q: %w", tp.Name, err)
}
}
if wasInserted { for _, domain := range tp.Domains {
inserted++ d := coredata.CommonThirdPartyDomain{
} else { ID: gid.New(gid.NilTenant, coredata.CommonThirdPartyDomainEntityType),
updated++ CommonThirdPartyID: party.ID,
if err := party.LoadByName(ctx, tx, tp.Name); err != nil { Domain: domain,
return fmt.Errorf("cannot reload common third party %q: %w", tp.Name, err) CreatedAt: now,
} UpdatedAt: now,
} }
for _, domain := range tp.Domains { domainInserted, err := d.Upsert(ctx, tx)
d := coredata.CommonThirdPartyDomain{ if err != nil {
ID: gid.New(gid.NilTenant, coredata.CommonThirdPartyDomainEntityType), return fmt.Errorf("cannot upsert domain %q for %q: %w", domain, tp.Name, err)
CommonThirdPartyID: party.ID, }
Domain: domain,
CreatedAt: now, if domainInserted {
UpdatedAt: now, domainsInserted++
} else {
domainsUpdated++
}
}
} }
domainInserted, err := d.Upsert(ctx, tx) return nil
if err != nil { },
return fmt.Errorf("cannot upsert domain %q for %q: %w", domain, tp.Name, err) ); err != nil {
} return err
}
if domainInserted { _, _ = fmt.Fprintf(out, "seeded %d third parties (%d inserted, %d updated)\n", len(thirdParties), inserted, updated)
domainsInserted++ _, _ = fmt.Fprintf(out, "seeded %d domains (%d inserted, %d updated)\n", domainsInserted+domainsUpdated, domainsInserted, domainsUpdated)
} else {
domainsUpdated++ if flagFetchLogos {
} if err := fetchAndStoreLogos(
ctx, out, errOut, pgClient, thirdParties,
flagS3Bucket, flagS3Endpoint, flagS3Region, flagS3AccessKey, flagS3SecretKey, flagS3UsePathStyle,
); err != nil {
return fmt.Errorf("cannot fetch logos: %w", err)
} }
} }
return nil return nil
}, },
); err != nil {
return err
} }
fmt.Printf("imported %d third parties (%d inserted, %d updated)\n", len(thirdParties), inserted, updated) cmd.Flags().StringVar(&flagData, "data", "", "Path to the third-party data.json file")
fmt.Printf("imported %d domains (%d inserted, %d updated)\n", domainsInserted+domainsUpdated, domainsInserted, domainsUpdated) _ = cmd.MarkFlagRequired("data")
cmd.Flags().BoolVar(&flagFetchLogos, "fetch-logos", false, "Fetch favicons and store them in S3")
cmd.Flags().StringVar(&flagS3Bucket, "s3-bucket", os.Getenv("AWS_S3_BUCKET"), "S3 bucket name (default: AWS_S3_BUCKET env)")
cmd.Flags().StringVar(&flagS3Endpoint, "s3-endpoint", os.Getenv("AWS_ENDPOINT_URL"), "S3 endpoint URL (default: AWS_ENDPOINT_URL env)")
cmd.Flags().StringVar(&flagS3Region, "s3-region", os.Getenv("AWS_REGION"), "S3 region (default: AWS_REGION env)")
cmd.Flags().StringVar(&flagS3AccessKey, "s3-access-key", os.Getenv("AWS_ACCESS_KEY_ID"), "S3 access key ID (default: AWS_ACCESS_KEY_ID env)")
cmd.Flags().StringVar(&flagS3SecretKey, "s3-secret-key", os.Getenv("AWS_SECRET_ACCESS_KEY"), "S3 secret access key (default: AWS_SECRET_ACCESS_KEY env)")
cmd.Flags().BoolVar(&flagS3UsePathStyle, "s3-path-style", false, "Use S3 path-style addressing")
if fetchLogos { return cmd
if err := fetchAndStoreLogos(ctx, pgClient, thirdParties, s3Bucket, s3Endpoint, s3Region, s3AccessKey, s3SecretKey, s3UsePathStyle); err != nil {
return fmt.Errorf("cannot fetch logos: %w", err)
}
}
return nil
} }
func fetchAndStoreLogos( func fetchAndStoreLogos(
ctx context.Context, ctx context.Context,
out, errOut io.Writer,
pgClient *pg.Client, pgClient *pg.Client,
thirdParties []thirdPartyData, thirdParties []thirdPartyData,
bucket, endpoint, region, accessKey, secretKey string, bucket, endpoint, region, accessKey, secretKey string,
@@ -264,7 +216,7 @@ func fetchAndStoreLogos(
httpClient := httpclient.DefaultPooledClient(httpclient.WithSSRFProtection()) httpClient := httpclient.DefaultPooledClient(httpclient.WithSSRFProtection())
httpClient.Transport = &userAgentTransport{ httpClient.Transport = &userAgentTransport{
next: httpClient.Transport, next: httpClient.Transport,
ua: version.UserAgent("common-third-parties-import"), ua: version.UserAgent("proboctl"),
} }
scope := coredata.NewScope(gid.NilTenant) scope := coredata.NewScope(gid.NilTenant)
@@ -280,7 +232,7 @@ func fetchAndStoreLogos(
if err := pgClient.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error { if err := pgClient.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error {
return party.LoadByName(ctx, conn, tp.Name) return party.LoadByName(ctx, conn, tp.Name)
}); err != nil { }); err != nil {
fmt.Fprintf(os.Stderr, "warning: cannot load %q, skipping logo: %v\n", tp.Name, err) _, _ = fmt.Fprintf(errOut, "warning: cannot load %q, skipping logo: %v\n", tp.Name, err)
failed++ failed++
continue continue
} }
@@ -293,17 +245,17 @@ func fetchAndStoreLogos(
var logoURL string var logoURL string
pageInfo, err := webinspect.Parse(ctx, httpClient, *tp.WebsiteURL) pageInfo, err := webinspect.Parse(ctx, httpClient, *tp.WebsiteURL)
if err != nil { if err != nil {
fmt.Fprintf(os.Stderr, "warning: cannot inspect page for %q, trying default apple-touch-icon: %v\n", tp.Name, err) _, _ = fmt.Fprintf(errOut, "warning: cannot inspect page for %q, trying default apple-touch-icon: %v\n", tp.Name, err)
} else { } else {
logoURL, err = webinspect.FindLogoURL(pageInfo) logoURL, err = webinspect.FindLogoURL(pageInfo)
if err != nil { if err != nil {
fmt.Fprintf(os.Stderr, "warning: cannot find logo for %q, trying default apple-touch-icon: %v\n", tp.Name, err) _, _ = fmt.Fprintf(errOut, "warning: cannot find logo for %q, trying default apple-touch-icon: %v\n", tp.Name, err)
} }
} }
parsed, err := url.Parse(*tp.WebsiteURL) parsed, err := url.Parse(*tp.WebsiteURL)
if err != nil { if err != nil {
fmt.Fprintf(os.Stderr, "warning: cannot parse URL for %q, skipping logo: %v\n", tp.Name, err) _, _ = fmt.Fprintf(errOut, "warning: cannot parse URL for %q, skipping logo: %v\n", tp.Name, err)
failed++ failed++
continue continue
} }
@@ -313,7 +265,8 @@ func fetchAndStoreLogos(
candidateURLs = append(candidateURLs, logoURL) candidateURLs = append(candidateURLs, logoURL)
} }
base := fmt.Sprintf("%s://%s", parsed.Scheme, parsed.Host) base := fmt.Sprintf("%s://%s", parsed.Scheme, parsed.Host)
candidateURLs = append(candidateURLs, candidateURLs = append(
candidateURLs,
base+"/apple-touch-icon.png", base+"/apple-touch-icon.png",
base+"/apple-touch-icon-precomposed.png", base+"/apple-touch-icon-precomposed.png",
"https://logo.debounce.com/"+parsed.Host, "https://logo.debounce.com/"+parsed.Host,
@@ -342,7 +295,7 @@ func fetchAndStoreLogos(
} }
if len(body) == 0 { if len(body) == 0 {
fmt.Fprintf(os.Stderr, "warning: cannot fetch logo for %q from any candidate URL\n", tp.Name) _, _ = fmt.Fprintf(errOut, "warning: cannot fetch logo for %q from any candidate URL\n", tp.Name)
failed++ failed++
continue continue
} }
@@ -376,7 +329,7 @@ func fetchAndStoreLogos(
"type": "common-third-party-logo", "type": "common-third-party-logo",
"common-third-party-id": party.ID.String(), "common-third-party-id": party.ID.String(),
}); err != nil { }); err != nil {
fmt.Fprintf(os.Stderr, "warning: cannot upload logo for %q to S3: %v\n", tp.Name, err) _, _ = fmt.Fprintf(errOut, "warning: cannot upload logo for %q to S3: %v\n", tp.Name, err)
failed++ failed++
continue continue
} }
@@ -394,16 +347,16 @@ func fetchAndStoreLogos(
return nil return nil
}); err != nil { }); err != nil {
fmt.Fprintf(os.Stderr, "warning: cannot store logo for %q: %v\n", tp.Name, err) _, _ = fmt.Fprintf(errOut, "warning: cannot store logo for %q: %v\n", tp.Name, err)
failed++ failed++
continue continue
} }
fetched++ fetched++
fmt.Printf(" fetched logo for %q\n", tp.Name) _, _ = fmt.Fprintf(out, " fetched logo for %q\n", tp.Name)
} }
fmt.Printf("logos: %d fetched, %d skipped, %d failed\n", fetched, skipped, failed) _, _ = fmt.Fprintf(out, "logos: %d fetched, %d skipped, %d failed\n", fetched, skipped, failed)
return nil return nil
} }
@@ -447,61 +400,20 @@ func loadThirdParties(path string) ([]thirdPartyData, error) {
return thirdParties, nil return thirdParties, nil
} }
func parseCategory(tp thirdPartyData) coredata.ThirdPartyCategory { func parseCategory(errOut io.Writer, tp thirdPartyData) coredata.ThirdPartyCategory {
if tp.Category == nil || *tp.Category == "" { if tp.Category == nil || *tp.Category == "" {
return coredata.ThirdPartyCategoryOther return coredata.ThirdPartyCategoryOther
} }
var c coredata.ThirdPartyCategory var c coredata.ThirdPartyCategory
if err := c.Scan(*tp.Category); err != nil { if err := c.Scan(*tp.Category); err != nil {
fmt.Fprintf(os.Stderr, "warning: third party %q has unknown category %q, falling back to OTHER\n", tp.Name, *tp.Category) _, _ = fmt.Fprintf(errOut, "warning: third party %q has unknown category %q, falling back to OTHER\n", tp.Name, *tp.Category)
return coredata.ThirdPartyCategoryOther return coredata.ThirdPartyCategoryOther
} }
return c return c
} }
func newPgClientFromDSN(dsn string) (*pg.Client, error) {
u, err := url.Parse(dsn)
if err != nil {
return nil, fmt.Errorf("cannot parse DSN (check URL format)")
}
var opts []pg.Option
switch u.Query().Get("sslmode") {
case "", "disable":
// plain connection, no TLS
case "require":
opts = append(opts, pg.WithUnsecureTLS())
case "prefer":
return nil, fmt.Errorf("unsupported sslmode %q (prefer fallback semantics are not supported)", u.Query().Get("sslmode"))
default:
return nil, fmt.Errorf("unsupported sslmode %q", u.Query().Get("sslmode"))
}
if u.Host != "" {
host := u.Host
if u.Port() == "" {
host = net.JoinHostPort(u.Hostname(), "5432")
}
opts = append(opts, pg.WithAddr(host))
}
if u.User != nil {
opts = append(opts, pg.WithUser(u.User.Username()))
if password, ok := u.User.Password(); ok {
opts = append(opts, pg.WithPassword(password))
}
}
if len(u.Path) > 1 {
opts = append(opts, pg.WithDatabase(u.Path[1:]))
}
return pg.NewClient(opts...)
}
type userAgentTransport struct { type userAgentTransport struct {
next http.RoundTripper next http.RoundTripper
ua string ua string

View File

@@ -12,26 +12,14 @@
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR // OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE. // PERFORMANCE OF THIS SOFTWARE.
// Command common-tracker-patterns-import seeds the common_tracker_patterns package commontrackerpatterns
// table from the Open Cookie Database (https://github.com/jkwakman/Open-Cookie-Database).
// It clones the repository into a temporary directory, reads
// open-cookie-database.json, and upserts each entry. Re-running is safe:
// existing rows are updated on the unique constraint so ids and created_at
// are preserved.
//
// Entries are linked to the matching common_third_parties row via a
// three-step cascade: slug lookup, domain lookup, then auto-create.
package main
import ( import (
"context" "context"
"encoding/json" "encoding/json"
"errors" "errors"
"flag"
"fmt" "fmt"
"math" "math"
"net"
"net/url"
"os" "os"
"path/filepath" "path/filepath"
"regexp" "regexp"
@@ -41,9 +29,11 @@ import (
"time" "time"
git "github.com/go-git/go-git/v5" git "github.com/go-git/go-git/v5"
"github.com/spf13/cobra"
"go.gearno.de/kit/pg" "go.gearno.de/kit/pg"
"go.probo.inc/probo/pkg/coredata" "go.probo.inc/probo/pkg/coredata"
"go.probo.inc/probo/pkg/gid" "go.probo.inc/probo/pkg/gid"
"go.probo.inc/probo/pkg/proboctl/cmdutil"
"go.probo.inc/probo/pkg/slug" "go.probo.inc/probo/pkg/slug"
) )
@@ -78,120 +68,114 @@ type (
} }
) )
func main() { func NewCmdCommonTrackerPatterns(f *cmdutil.Factory) *cobra.Command {
if err := run(); err != nil { return &cobra.Command{
fmt.Fprintf(os.Stderr, "error: %v\n", err) Use: "common-tracker-patterns",
os.Exit(1) Short: "Seed common tracker patterns from the Open Cookie Database",
} Long: "Seed the common_tracker_patterns table from the Open Cookie Database " +
} "(https://github.com/jkwakman/Open-Cookie-Database). " +
"The repository is cloned into a temporary directory. " +
"Re-running is safe: existing rows are upserted so ids and created_at are preserved. " +
"Entries are linked to matching common_third_parties rows via slug lookup, " +
"domain lookup, then auto-create.",
RunE: func(cmd *cobra.Command, args []string) error {
out := f.IOStreams.Out
errOut := f.IOStreams.ErrOut
ctx := cmd.Context()
func run() error { _, _ = fmt.Fprintf(out, "cloning %s\n", ocdRepoURL)
var pgDSN string
flag.StringVar( tmpDir, cleanup, err := cloneRepo()
&pgDSN, if err != nil {
"pg-dsn", return fmt.Errorf("cannot clone repository: %w", err)
os.Getenv("DATABASE_URL"),
"PostgreSQL connection URL (default: DATABASE_URL env)",
)
flag.Parse()
if pgDSN == "" {
return fmt.Errorf("set -pg-dsn or DATABASE_URL")
}
ctx := context.Background()
fmt.Printf("cloning %s\n", ocdRepoURL)
tmpDir, cleanup, err := cloneRepo()
if err != nil {
return fmt.Errorf("cannot clone repository: %w", err)
}
defer cleanup()
patterns, err := loadPatternsFromOCD(tmpDir)
if err != nil {
return fmt.Errorf("cannot load tracker pattern data: %w", err)
}
pgClient, err := newPgClientFromDSN(pgDSN)
if err != nil {
return fmt.Errorf("cannot create pg client: %w", err)
}
fmt.Printf("importing %d common tracker patterns from Open Cookie Database\n", len(patterns))
var inserted, updated, skipped int
var partiesCreated int
if err := pgClient.WithTx(
ctx,
func(ctx context.Context, tx pg.Tx) error {
now := time.Now()
thirdPartyCache := make(map[string]*gid.GID)
for _, p := range patterns {
thirdPartyID, err := resolveThirdParty(ctx, tx, p, thirdPartyCache, now, &partiesCreated)
if err != nil {
return err
}
trackerType, err := parseTrackerType(p.TrackerType)
if err != nil {
fmt.Fprintf(os.Stderr, "warning: %v, skipping pattern %q\n", err, p.Pattern)
skipped++
continue
}
matchType, err := parseMatchType(p.MatchType)
if err != nil {
fmt.Fprintf(os.Stderr, "warning: %v, skipping pattern %q\n", err, p.Pattern)
skipped++
continue
}
pattern := coredata.CommonTrackerPattern{
ID: gid.New(gid.NilTenant, coredata.CommonTrackerPatternEntityType),
CommonThirdPartyID: thirdPartyID,
TrackerType: trackerType,
Pattern: p.Pattern,
MatchType: matchType,
Description: p.Description,
MaxAgeSeconds: p.MaxAgeSeconds,
Confidence: p.Confidence,
CreatedAt: now,
UpdatedAt: now,
}
_, wasInserted, err := pattern.Upsert(ctx, tx)
if err != nil {
return fmt.Errorf("cannot upsert common tracker pattern %q: %w", p.Pattern, err)
}
if wasInserted {
inserted++
} else {
updated++
}
} }
defer cleanup()
patterns, err := loadPatternsFromOCD(tmpDir)
if err != nil {
return fmt.Errorf("cannot load tracker pattern data: %w", err)
}
pgClient, err := f.PgClient()
if err != nil {
return fmt.Errorf("cannot create pg client: %w", err)
}
_, _ = fmt.Fprintf(out, "seeding %d common tracker patterns from Open Cookie Database\n", len(patterns))
var inserted, updated, skipped int
var partiesCreated int
if err := pgClient.WithTx(
ctx,
func(ctx context.Context, tx pg.Tx) error {
now := time.Now()
thirdPartyCache := make(map[string]*gid.GID)
for _, p := range patterns {
thirdPartyID, err := resolveThirdParty(ctx, tx, p, thirdPartyCache, now, &partiesCreated)
if err != nil {
return err
}
trackerType, err := parseTrackerType(p.TrackerType)
if err != nil {
_, _ = fmt.Fprintf(errOut, "warning: %v, skipping pattern %q\n", err, p.Pattern)
skipped++
continue
}
matchType, err := parseMatchType(p.MatchType)
if err != nil {
_, _ = fmt.Fprintf(errOut, "warning: %v, skipping pattern %q\n", err, p.Pattern)
skipped++
continue
}
pattern := coredata.CommonTrackerPattern{
ID: gid.New(gid.NilTenant, coredata.CommonTrackerPatternEntityType),
CommonThirdPartyID: thirdPartyID,
TrackerType: trackerType,
Pattern: p.Pattern,
MatchType: matchType,
Description: p.Description,
MaxAgeSeconds: p.MaxAgeSeconds,
Confidence: p.Confidence,
CreatedAt: now,
UpdatedAt: now,
}
_, wasInserted, err := pattern.Upsert(ctx, tx)
if err != nil {
return fmt.Errorf("cannot upsert common tracker pattern %q: %w", p.Pattern, err)
}
if wasInserted {
inserted++
} else {
updated++
}
}
return nil
},
); err != nil {
return err
}
_, _ = fmt.Fprintf(
out,
"seeded %d patterns (%d inserted, %d updated, %d skipped, %d third parties auto-created)\n",
len(patterns)-skipped,
inserted,
updated,
skipped,
partiesCreated,
)
return nil return nil
}, },
); err != nil {
return err
} }
fmt.Printf(
"imported %d patterns (%d inserted, %d updated, %d skipped, %d third parties auto-created)\n",
len(patterns)-skipped,
inserted,
updated,
skipped,
partiesCreated,
)
return nil
} }
func cloneRepo() (string, func(), error) { func cloneRepo() (string, func(), error) {
@@ -360,7 +344,6 @@ var domainValidRe = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9.-]*[a-zA-Z]$`)
func normalizeDomain(s string) string { func normalizeDomain(s string) string {
s = strings.TrimSpace(s) s = strings.TrimSpace(s)
// Strip zero-width spaces and other Unicode control characters.
s = strings.Map(func(r rune) rune { s = strings.Map(func(r rune) rune {
if r == '\u200b' || r == '\ufeff' { if r == '\u200b' || r == '\ufeff' {
return -1 return -1
@@ -368,19 +351,15 @@ func normalizeDomain(s string) string {
return r return r
}, s) }, s)
// Take only the first domain when "or" separates multiple values,
// e.g. ".bing.com (3rd party) or .microsoft.com (3rd party)".
if idx := strings.Index(s, " or "); idx != -1 { if idx := strings.Index(s, " or "); idx != -1 {
s = s[:idx] s = s[:idx]
} }
// Handle values starting with "or ", e.g. "or demdex.net (3rd party)".
s = strings.TrimPrefix(s, "or ") s = strings.TrimPrefix(s, "or ")
if idx := strings.IndexByte(s, '('); idx != -1 { if idx := strings.IndexByte(s, '('); idx != -1 {
s = strings.TrimSpace(s[:idx]) s = strings.TrimSpace(s[:idx])
} }
// Strip placeholders like "[account]".
if strings.ContainsAny(s, "[]") { if strings.ContainsAny(s, "[]") {
return "" return ""
} }
@@ -487,43 +466,3 @@ func parseMatchType(s string) (coredata.TrackerPatternMatchType, error) {
return "", fmt.Errorf("unknown match type %q", s) return "", fmt.Errorf("unknown match type %q", s)
} }
} }
func newPgClientFromDSN(dsn string) (*pg.Client, error) {
u, err := url.Parse(dsn)
if err != nil {
return nil, fmt.Errorf("cannot parse DSN (check URL format)")
}
var opts []pg.Option
switch u.Query().Get("sslmode") {
case "", "disable":
case "require":
opts = append(opts, pg.WithUnsecureTLS())
case "prefer":
return nil, fmt.Errorf("unsupported sslmode %q (prefer fallback semantics are not supported)", u.Query().Get("sslmode"))
default:
return nil, fmt.Errorf("unsupported sslmode %q", u.Query().Get("sslmode"))
}
if u.Host != "" {
host := u.Host
if u.Port() == "" {
host = net.JoinHostPort(u.Hostname(), "5432")
}
opts = append(opts, pg.WithAddr(host))
}
if u.User != nil {
opts = append(opts, pg.WithUser(u.User.Username()))
if password, ok := u.User.Password(); ok {
opts = append(opts, pg.WithPassword(password))
}
}
if len(u.Path) > 1 {
opts = append(opts, pg.WithDatabase(u.Path[1:]))
}
return pg.NewClient(opts...)
}

34
pkg/proboctl/seed/seed.go Normal file
View File

@@ -0,0 +1,34 @@
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package seed
import (
"github.com/spf13/cobra"
"go.probo.inc/probo/pkg/proboctl/cmdutil"
commonthirdparties "go.probo.inc/probo/pkg/proboctl/seed/common-third-parties"
commontrackerpatterns "go.probo.inc/probo/pkg/proboctl/seed/common-tracker-patterns"
)
func NewCmdSeed(f *cmdutil.Factory) *cobra.Command {
cmd := &cobra.Command{
Use: "seed <command>",
Short: "Seed common reference data",
}
cmd.AddCommand(commonthirdparties.NewCmdCommonThirdParties(f))
cmd.AddCommand(commontrackerpatterns.NewCmdCommonTrackerPatterns(f))
return cmd
}

View File

@@ -0,0 +1,49 @@
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package version
import (
"fmt"
"github.com/spf13/cobra"
"go.probo.inc/probo/pkg/proboctl/cmdutil"
"go.probo.inc/probo/pkg/version"
)
func NewCmdVersion(f *cmdutil.Factory) *cobra.Command {
return &cobra.Command{
Use: "version",
Short: "Print the version of proboctl",
RunE: func(cmd *cobra.Command, args []string) error {
info := version.GetBuildInfo()
v := f.Version
if v == "" || v == "unknown" {
v = info.Version
}
_, _ = fmt.Fprintf(
f.IOStreams.Out,
"proboctl version %s (commit: %s, built: %s, go: %s)\n",
v,
info.Commit,
info.BuildDate,
info.GoVersion,
)
return nil
},
}
}