rohit/probo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add proboctl CLI and move seed commands into it

Browse Source 
Introduce a new proboctl Cobra CLI for Probo instance management
that connects directly to PostgreSQL. Move the standalone
common-third-parties-import and common-tracker-patterns-import
commands into proboctl as `proboctl seed common-third-parties`
and `proboctl seed common-tracker-patterns`, replacing flag-based
PG connection with a shared factory pattern.

Signed-off-by: Émile Ré <emile@probo.com>
This commit is contained in:
Émile Ré
2026-05-14 18:37:42 +04:00
parent 844817543d
commit 48a494461e
11 changed files with 520 additions and 386 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
GNUmakefile
View File

@@ -25,10 +25,12 @@ DOCKER_COMPOSE= $(DOCKER) compose -f compose.yaml $(DOCKER_COMPOSE_FLAGS)
PRB_VERSION= $(shell cat cmd/prb/VERSION)
PROBOD_VERSION= $(shell cat cmd/probod/VERSION)
PROBOD_BOOTSTRAP_VERSION=$(shell cat cmd/probod-bootstrap/VERSION)
PROBOCTL_VERSION= $(shell cat cmd/proboctl/VERSION)
PRB_LDFLAGS= -ldflags "-X 'main.version=$(PRB_VERSION)'"
PROBOD_LDFLAGS= -ldflags "-X 'main.version=$(PROBOD_VERSION)' -X 'main.env=prod'"
PROBOD_BOOTSTRAP_LDFLAGS=-ldflags "-X 'main.version=$(PROBOD_BOOTSTRAP_VERSION)'"
PROBOCTL_LDFLAGS= -ldflags "-X 'main.version=$(PROBOCTL_VERSION)'"
GCFLAGS= -gcflags="-e"
@@ -73,6 +75,9 @@ PRB_SRC= cmd/prb/main.go
PROBOD_BOOTSTRAP_BIN= bin/probod-bootstrap
PROBOD_BOOTSTRAP_SRC= cmd/probod-bootstrap/main.go
PROBOCTL_BIN= bin/proboctl
PROBOCTL_SRC= cmd/proboctl/main.go
ifdef WITH_APPS
GENERATED += relay
EMBEDDED += \
@@ -172,7 +177,7 @@ coverage-combined: coverage-report test-e2e-coverage ## Generate combined covera
$(GO) tool cover -html=coverage-combined.out -o=coverage-combined.html
.PHONY: build
build: $(PROBOD_BIN) bin/prb bin/probod-bootstrap
build: $(PROBOD_BIN) bin/prb bin/probod-bootstrap bin/proboctl
CFG_DEV_OAUTH2_KEY = cfg/.dev-oauth2-signing-key.pem
DEV_ENV = .env
@@ -252,6 +257,10 @@ bin/prb:
$(PROBOD_BOOTSTRAP_BIN):
$(GO_BUILD) $(PROBOD_BOOTSTRAP_LDFLAGS) -o $(PROBOD_BOOTSTRAP_BIN) $(PROBOD_BOOTSTRAP_SRC)
.PHONY: bin/proboctl
bin/proboctl:
$(GO_BUILD) $(PROBOCTL_LDFLAGS) -o $(PROBOCTL_BIN) $(PROBOCTL_SRC)
.PHONY: @probo/emails
@probo/emails:
$(NPM) --workspace $@ run build

1
cmd/proboctl/CHANGELOG.md Normal file
View File

@@ -0,0 +1 @@
# Changelog

1
cmd/proboctl/VERSION Normal file
View File

@@ -0,0 +1 @@
0.0.0

42
cmd/proboctl/main.go Normal file
View File

@@ -0,0 +1,42 @@
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package main
import (
"fmt"
"os"
"go.probo.inc/probo/pkg/cmd/iostreams"
"go.probo.inc/probo/pkg/proboctl/cmdutil"
"go.probo.inc/probo/pkg/proboctl/root"
)
var version string = "unknown"
func main() {
ios := iostreams.System()
f := &cmdutil.Factory{
IOStreams: ios,
Version: version,
}
cmd := root.NewCmdRoot(f)
if err := cmd.Execute(); err != nil {
fmt.Fprintf(os.Stderr, "Error: %s\n", err)
os.Exit(1)
}
}

36
pkg/proboctl/cmdutil/cmdutil.go Normal file
View File

@@ -0,0 +1,36 @@
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package cmdutil
import (
"fmt"
"go.gearno.de/kit/pg"
"go.probo.inc/probo/pkg/cmd/iostreams"
"go.probo.inc/probo/pkg/proboctl/pgconn"
)
type Factory struct {
IOStreams *iostreams.IOStreams
Version string
PgDSN string
}
func (f *Factory) PgClient() (*pg.Client, error) {
if f.PgDSN == "" {
return nil, fmt.Errorf("set --pg-dsn or DATABASE_URL")
}
return pgconn.NewPgClientFromDSN(f.PgDSN)
}

66
pkg/proboctl/pgconn/pgconn.go Normal file
View File

@@ -0,0 +1,66 @@
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package pgconn
import (
"fmt"
"net"
"net/url"
"go.gearno.de/kit/pg"
)
func NewPgClientFromDSN(dsn string) (*pg.Client, error) {
u, err := url.Parse(dsn)
if err != nil {
return nil, fmt.Errorf("cannot parse DSN (check URL format)")
}
var opts []pg.Option
switch u.Query().Get("sslmode") {
case "", "disable":
case "require":
opts = append(opts, pg.WithUnsecureTLS())
case "prefer":
return nil, fmt.Errorf(
"unsupported sslmode %q (prefer fallback semantics are not supported)",
u.Query().Get("sslmode"),
)
default:
return nil, fmt.Errorf("unsupported sslmode %q", u.Query().Get("sslmode"))
}
if u.Host != "" {
host := u.Host
if u.Port() == "" {
host = net.JoinHostPort(u.Hostname(), "5432")
}
opts = append(opts, pg.WithAddr(host))
}
if u.User != nil {
opts = append(opts, pg.WithUser(u.User.Username()))
if password, ok := u.User.Password(); ok {
opts = append(opts, pg.WithPassword(password))
}
}
if len(u.Path) > 1 {
opts = append(opts, pg.WithDatabase(u.Path[1:]))
}
return pg.NewClient(opts...)
}

45
pkg/proboctl/root/root.go Normal file
View File

@@ -0,0 +1,45 @@
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package root
import (
"os"
"github.com/spf13/cobra"
"go.probo.inc/probo/pkg/proboctl/cmdutil"
"go.probo.inc/probo/pkg/proboctl/seed"
"go.probo.inc/probo/pkg/proboctl/version"
)
func NewCmdRoot(f *cmdutil.Factory) *cobra.Command {
cmd := &cobra.Command{
Use: "proboctl <command> [flags]",
Short: "Probo instance management CLI",
SilenceUsage: true,
SilenceErrors: true,
}
cmd.PersistentFlags().StringVar(
&f.PgDSN,
"pg-dsn",
os.Getenv("DATABASE_URL"),
"PostgreSQL connection URL (default: DATABASE_URL env)",
)
cmd.AddCommand(seed.NewCmdSeed(f))
cmd.AddCommand(version.NewCmdVersion(f))
return cmd
}

352
cmd/common-third-parties-import/main.go → pkg/proboctl/seed/common-third-parties/common_third_parties.go
View File

@@ -12,24 +12,14 @@
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
// Command common-third-parties-import seeds the common_third_parties table from
// packages/thirdParties/data.json. It is idempotent: re-running upserts on conflict
// (slug) so existing rows keep their id and created_at.
//
// When -fetch-logos is set, the tool inspects each third party's website to
// find the best available logo (SVG icon, apple-touch-icon, etc.) and stores
// it in S3 as a public file, linking it to each common third party via
// logo_file_id.
package main
package commonthirdparties
import (
"bytes"
"context"
"encoding/json"
"flag"
"fmt"
"io"
"net"
"net/http"
"net/url"
"os"
@@ -38,12 +28,14 @@ import (
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/spf13/cobra"
"go.gearno.de/crypto/uuid"
"go.gearno.de/kit/httpclient"
"go.gearno.de/kit/pg"
"go.probo.inc/probo/pkg/coredata"
"go.probo.inc/probo/pkg/filemanager"
"go.probo.inc/probo/pkg/gid"
"go.probo.inc/probo/pkg/proboctl/cmdutil"
"go.probo.inc/probo/pkg/slug"
"go.probo.inc/probo/pkg/version"
"go.probo.inc/probo/pkg/webinspect"
@@ -69,191 +61,151 @@ type thirdPartyData struct {
Domains []string `json:"domains,omitempty"`
}
func main() {
if err := run(); err != nil {
fmt.Fprintf(os.Stderr, "error: %v\n", err)
os.Exit(1)
}
}
func run() error {
func NewCmdCommonThirdParties(f *cmdutil.Factory) *cobra.Command {
var (
pgDSN string
dataPath string
fetchLogos bool
s3Bucket string
s3Endpoint string
s3Region string
s3AccessKey string
s3SecretKey string
s3UsePathStyle bool
flagData string
flagFetchLogos bool
flagS3Bucket string
flagS3Endpoint string
flagS3Region string
flagS3AccessKey string
flagS3SecretKey string
flagS3UsePathStyle bool
)
flag.StringVar(
&pgDSN,
"pg-dsn",
os.Getenv("DATABASE_URL"),
"PostgreSQL connection URL (default: DATABASE_URL env)",
)
flag.StringVar(
&dataPath,
"data",
"",
"Path to the third-party data.json file",
)
flag.BoolVar(
&fetchLogos,
"fetch-logos",
false,
"Fetch favicons from Google and store them in S3",
)
flag.StringVar(
&s3Bucket,
"s3-bucket",
os.Getenv("AWS_S3_BUCKET"),
"S3 bucket name (default: AWS_S3_BUCKET env)",
)
flag.StringVar(
&s3Endpoint,
"s3-endpoint",
os.Getenv("AWS_ENDPOINT_URL"),
"S3 endpoint URL (default: AWS_ENDPOINT_URL env)",
)
flag.StringVar(
&s3Region,
"s3-region",
os.Getenv("AWS_REGION"),
"S3 region (default: AWS_REGION env)",
)
flag.StringVar(
&s3AccessKey,
"s3-access-key",
os.Getenv("AWS_ACCESS_KEY_ID"),
"S3 access key ID (default: AWS_ACCESS_KEY_ID env)",
)
flag.StringVar(
&s3SecretKey,
"s3-secret-key",
os.Getenv("AWS_SECRET_ACCESS_KEY"),
"S3 secret access key (default: AWS_SECRET_ACCESS_KEY env)",
)
flag.BoolVar(
&s3UsePathStyle,
"s3-path-style",
false,
"Use S3 path-style addressing",
)
flag.Parse()
cmd := &cobra.Command{
Use: "common-third-parties",
Short: "Seed common third parties from a data.json file",
Long: "Seed the common_third_parties table from a JSON file. " +
"Re-running is safe: existing rows are upserted on conflict (slug) " +
"so ids and created_at are preserved.",
RunE: func(cmd *cobra.Command, args []string) error {
out := f.IOStreams.Out
errOut := f.IOStreams.ErrOut
if pgDSN == "" {
return fmt.Errorf("set -pg-dsn or DATABASE_URL")
}
if flagFetchLogos && flagS3Bucket == "" {
return fmt.Errorf("set --s3-bucket or AWS_S3_BUCKET when using --fetch-logos")
}
if fetchLogos && s3Bucket == "" {
return fmt.Errorf("set -s3-bucket or AWS_S3_BUCKET when using -fetch-logos")
}
ctx := cmd.Context()
ctx := context.Background()
thirdParties, err := loadThirdParties(flagData)
if err != nil {
return fmt.Errorf("cannot load third-party data: %w", err)
}
thirdParties, err := loadThirdParties(dataPath)
if err != nil {
return fmt.Errorf("cannot load third-party data: %w", err)
}
pgClient, err := f.PgClient()
if err != nil {
return fmt.Errorf("cannot create pg client: %w", err)
}
pgClient, err := newPgClientFromDSN(pgDSN)
if err != nil {
return fmt.Errorf("cannot create pg client: %w", err)
}
_, _ = fmt.Fprintf(out, "seeding %d common third parties from %s\n", len(thirdParties), flagData)
fmt.Printf("importing %d common third parties from %s\n", len(thirdParties), dataPath)
var inserted, updated, domainsInserted, domainsUpdated int
var inserted, updated, domainsInserted, domainsUpdated int
if err := pgClient.WithTx(
ctx,
func(ctx context.Context, tx pg.Tx) error {
now := time.Now()
if err := pgClient.WithTx(
ctx,
func(ctx context.Context, tx pg.Tx) error {
now := time.Now()
for _, tp := range thirdParties {
party := coredata.CommonThirdParty{
ID: gid.New(gid.NilTenant, coredata.CommonThirdPartyEntityType),
Name: tp.Name,
Slug: slug.Make(tp.Name),
Category: parseCategory(errOut, tp),
HeadquarterAddress: tp.HeadquarterAddress,
LegalName: tp.LegalName,
WebsiteURL: tp.WebsiteURL,
PrivacyPolicyURL: tp.PrivacyPolicyURL,
ServiceLevelAgreementURL: tp.ServiceLevelAgreementURL,
ServiceSoftwareAgreementURL: tp.ServiceSoftwareAgreementURL,
DataProcessingAgreementURL: tp.DataProcessingAgreementURL,
BusinessAssociateAgreementURL: tp.BusinessAssociateAgreementURL,
SubprocessorsListURL: tp.SubprocessorsListURL,
Certifications: tp.Certifications,
StatusPageURL: tp.StatusPageURL,
TermsOfServiceURL: tp.TermsOfServiceURL,
SecurityPageURL: tp.SecurityPageURL,
TrustPageURL: tp.TrustPageURL,
CreatedAt: now,
UpdatedAt: now,
}
for _, tp := range thirdParties {
party := coredata.CommonThirdParty{
ID: gid.New(gid.NilTenant, coredata.CommonThirdPartyEntityType),
Name: tp.Name,
Slug: slug.Make(tp.Name),
Category: parseCategory(tp),
HeadquarterAddress: tp.HeadquarterAddress,
LegalName: tp.LegalName,
WebsiteURL: tp.WebsiteURL,
PrivacyPolicyURL: tp.PrivacyPolicyURL,
ServiceLevelAgreementURL: tp.ServiceLevelAgreementURL,
ServiceSoftwareAgreementURL: tp.ServiceSoftwareAgreementURL,
DataProcessingAgreementURL: tp.DataProcessingAgreementURL,
BusinessAssociateAgreementURL: tp.BusinessAssociateAgreementURL,
SubprocessorsListURL: tp.SubprocessorsListURL,
Certifications: tp.Certifications,
StatusPageURL: tp.StatusPageURL,
TermsOfServiceURL: tp.TermsOfServiceURL,
SecurityPageURL: tp.SecurityPageURL,
TrustPageURL: tp.TrustPageURL,
CreatedAt: now,
UpdatedAt: now,
}
wasInserted, err := party.Upsert(ctx, tx)
if err != nil {
return fmt.Errorf("cannot upsert common third party %q: %w", tp.Name, err)
}
wasInserted, err := party.Upsert(ctx, tx)
if err != nil {
return fmt.Errorf("cannot upsert common third party %q: %w", tp.Name, err)
}
if wasInserted {
inserted++
} else {
updated++
if err := party.LoadByName(ctx, tx, tp.Name); err != nil {
return fmt.Errorf("cannot reload common third party %q: %w", tp.Name, err)
}
}
if wasInserted {
inserted++
} else {
updated++
if err := party.LoadByName(ctx, tx, tp.Name); err != nil {
return fmt.Errorf("cannot reload common third party %q: %w", tp.Name, err)
}
}
for _, domain := range tp.Domains {
d := coredata.CommonThirdPartyDomain{
ID: gid.New(gid.NilTenant, coredata.CommonThirdPartyDomainEntityType),
CommonThirdPartyID: party.ID,
Domain: domain,
CreatedAt: now,
UpdatedAt: now,
}
for _, domain := range tp.Domains {
d := coredata.CommonThirdPartyDomain{
ID: gid.New(gid.NilTenant, coredata.CommonThirdPartyDomainEntityType),
CommonThirdPartyID: party.ID,
Domain: domain,
CreatedAt: now,
UpdatedAt: now,
domainInserted, err := d.Upsert(ctx, tx)
if err != nil {
return fmt.Errorf("cannot upsert domain %q for %q: %w", domain, tp.Name, err)
}
if domainInserted {
domainsInserted++
} else {
domainsUpdated++
}
}
}
domainInserted, err := d.Upsert(ctx, tx)
if err != nil {
return fmt.Errorf("cannot upsert domain %q for %q: %w", domain, tp.Name, err)
}
return nil
},
); err != nil {
return err
}
if domainInserted {
domainsInserted++
} else {
domainsUpdated++
}
_, _ = fmt.Fprintf(out, "seeded %d third parties (%d inserted, %d updated)\n", len(thirdParties), inserted, updated)
_, _ = fmt.Fprintf(out, "seeded %d domains (%d inserted, %d updated)\n", domainsInserted+domainsUpdated, domainsInserted, domainsUpdated)
if flagFetchLogos {
if err := fetchAndStoreLogos(
ctx, out, errOut, pgClient, thirdParties,
flagS3Bucket, flagS3Endpoint, flagS3Region, flagS3AccessKey, flagS3SecretKey, flagS3UsePathStyle,
); err != nil {
return fmt.Errorf("cannot fetch logos: %w", err)
}
}
return nil
},
); err != nil {
return err
}
fmt.Printf("imported %d third parties (%d inserted, %d updated)\n", len(thirdParties), inserted, updated)
fmt.Printf("imported %d domains (%d inserted, %d updated)\n", domainsInserted+domainsUpdated, domainsInserted, domainsUpdated)
cmd.Flags().StringVar(&flagData, "data", "", "Path to the third-party data.json file")
_ = cmd.MarkFlagRequired("data")
cmd.Flags().BoolVar(&flagFetchLogos, "fetch-logos", false, "Fetch favicons and store them in S3")
cmd.Flags().StringVar(&flagS3Bucket, "s3-bucket", os.Getenv("AWS_S3_BUCKET"), "S3 bucket name (default: AWS_S3_BUCKET env)")
cmd.Flags().StringVar(&flagS3Endpoint, "s3-endpoint", os.Getenv("AWS_ENDPOINT_URL"), "S3 endpoint URL (default: AWS_ENDPOINT_URL env)")
cmd.Flags().StringVar(&flagS3Region, "s3-region", os.Getenv("AWS_REGION"), "S3 region (default: AWS_REGION env)")
cmd.Flags().StringVar(&flagS3AccessKey, "s3-access-key", os.Getenv("AWS_ACCESS_KEY_ID"), "S3 access key ID (default: AWS_ACCESS_KEY_ID env)")
cmd.Flags().StringVar(&flagS3SecretKey, "s3-secret-key", os.Getenv("AWS_SECRET_ACCESS_KEY"), "S3 secret access key (default: AWS_SECRET_ACCESS_KEY env)")
cmd.Flags().BoolVar(&flagS3UsePathStyle, "s3-path-style", false, "Use S3 path-style addressing")
if fetchLogos {
if err := fetchAndStoreLogos(ctx, pgClient, thirdParties, s3Bucket, s3Endpoint, s3Region, s3AccessKey, s3SecretKey, s3UsePathStyle); err != nil {
return fmt.Errorf("cannot fetch logos: %w", err)
}
}
return nil
return cmd
}
func fetchAndStoreLogos(
ctx context.Context,
out, errOut io.Writer,
pgClient *pg.Client,
thirdParties []thirdPartyData,
bucket, endpoint, region, accessKey, secretKey string,
@@ -264,7 +216,7 @@ func fetchAndStoreLogos(
httpClient := httpclient.DefaultPooledClient(httpclient.WithSSRFProtection())
httpClient.Transport = &userAgentTransport{
next: httpClient.Transport,
ua: version.UserAgent("common-third-parties-import"),
ua: version.UserAgent("proboctl"),
}
scope := coredata.NewScope(gid.NilTenant)
@@ -280,7 +232,7 @@ func fetchAndStoreLogos(
if err := pgClient.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error {
return party.LoadByName(ctx, conn, tp.Name)
}); err != nil {
fmt.Fprintf(os.Stderr, "warning: cannot load %q, skipping logo: %v\n", tp.Name, err)
_, _ = fmt.Fprintf(errOut, "warning: cannot load %q, skipping logo: %v\n", tp.Name, err)
failed++
continue
}
@@ -293,17 +245,17 @@ func fetchAndStoreLogos(
var logoURL string
pageInfo, err := webinspect.Parse(ctx, httpClient, *tp.WebsiteURL)
if err != nil {
fmt.Fprintf(os.Stderr, "warning: cannot inspect page for %q, trying default apple-touch-icon: %v\n", tp.Name, err)
_, _ = fmt.Fprintf(errOut, "warning: cannot inspect page for %q, trying default apple-touch-icon: %v\n", tp.Name, err)
} else {
logoURL, err = webinspect.FindLogoURL(pageInfo)
if err != nil {
fmt.Fprintf(os.Stderr, "warning: cannot find logo for %q, trying default apple-touch-icon: %v\n", tp.Name, err)
_, _ = fmt.Fprintf(errOut, "warning: cannot find logo for %q, trying default apple-touch-icon: %v\n", tp.Name, err)
}
}
parsed, err := url.Parse(*tp.WebsiteURL)
if err != nil {
fmt.Fprintf(os.Stderr, "warning: cannot parse URL for %q, skipping logo: %v\n", tp.Name, err)
_, _ = fmt.Fprintf(errOut, "warning: cannot parse URL for %q, skipping logo: %v\n", tp.Name, err)
failed++
continue
}
@@ -313,7 +265,8 @@ func fetchAndStoreLogos(
candidateURLs = append(candidateURLs, logoURL)
}
base := fmt.Sprintf("%s://%s", parsed.Scheme, parsed.Host)
candidateURLs = append(candidateURLs,
candidateURLs = append(
candidateURLs,
base+"/apple-touch-icon.png",
base+"/apple-touch-icon-precomposed.png",
"https://logo.debounce.com/"+parsed.Host,
@@ -342,7 +295,7 @@ func fetchAndStoreLogos(
}
if len(body) == 0 {
fmt.Fprintf(os.Stderr, "warning: cannot fetch logo for %q from any candidate URL\n", tp.Name)
_, _ = fmt.Fprintf(errOut, "warning: cannot fetch logo for %q from any candidate URL\n", tp.Name)
failed++
continue
}
@@ -376,7 +329,7 @@ func fetchAndStoreLogos(
"type": "common-third-party-logo",
"common-third-party-id": party.ID.String(),
}); err != nil {
fmt.Fprintf(os.Stderr, "warning: cannot upload logo for %q to S3: %v\n", tp.Name, err)
_, _ = fmt.Fprintf(errOut, "warning: cannot upload logo for %q to S3: %v\n", tp.Name, err)
failed++
continue
}
@@ -394,16 +347,16 @@ func fetchAndStoreLogos(
return nil
}); err != nil {
fmt.Fprintf(os.Stderr, "warning: cannot store logo for %q: %v\n", tp.Name, err)
_, _ = fmt.Fprintf(errOut, "warning: cannot store logo for %q: %v\n", tp.Name, err)
failed++
continue
}
fetched++
fmt.Printf(" fetched logo for %q\n", tp.Name)
_, _ = fmt.Fprintf(out, " fetched logo for %q\n", tp.Name)
}
fmt.Printf("logos: %d fetched, %d skipped, %d failed\n", fetched, skipped, failed)
_, _ = fmt.Fprintf(out, "logos: %d fetched, %d skipped, %d failed\n", fetched, skipped, failed)
return nil
}
@@ -447,61 +400,20 @@ func loadThirdParties(path string) ([]thirdPartyData, error) {
return thirdParties, nil
}
func parseCategory(tp thirdPartyData) coredata.ThirdPartyCategory {
func parseCategory(errOut io.Writer, tp thirdPartyData) coredata.ThirdPartyCategory {
if tp.Category == nil || *tp.Category == "" {
return coredata.ThirdPartyCategoryOther
}
var c coredata.ThirdPartyCategory
if err := c.Scan(*tp.Category); err != nil {
fmt.Fprintf(os.Stderr, "warning: third party %q has unknown category %q, falling back to OTHER\n", tp.Name, *tp.Category)
_, _ = fmt.Fprintf(errOut, "warning: third party %q has unknown category %q, falling back to OTHER\n", tp.Name, *tp.Category)
return coredata.ThirdPartyCategoryOther
}
return c
}
func newPgClientFromDSN(dsn string) (*pg.Client, error) {
u, err := url.Parse(dsn)
if err != nil {
return nil, fmt.Errorf("cannot parse DSN (check URL format)")
}
var opts []pg.Option
switch u.Query().Get("sslmode") {
case "", "disable":
// plain connection, no TLS
case "require":
opts = append(opts, pg.WithUnsecureTLS())
case "prefer":
return nil, fmt.Errorf("unsupported sslmode %q (prefer fallback semantics are not supported)", u.Query().Get("sslmode"))
default:
return nil, fmt.Errorf("unsupported sslmode %q", u.Query().Get("sslmode"))
}
if u.Host != "" {
host := u.Host
if u.Port() == "" {
host = net.JoinHostPort(u.Hostname(), "5432")
}
opts = append(opts, pg.WithAddr(host))
}
if u.User != nil {
opts = append(opts, pg.WithUser(u.User.Username()))
if password, ok := u.User.Password(); ok {
opts = append(opts, pg.WithPassword(password))
}
}
if len(u.Path) > 1 {
opts = append(opts, pg.WithDatabase(u.Path[1:]))
}
return pg.NewClient(opts...)
}
type userAgentTransport struct {
next http.RoundTripper
ua string

269
cmd/common-tracker-patterns-import/main.go → pkg/proboctl/seed/common-tracker-patterns/common_tracker_patterns.go
View File

@@ -12,26 +12,14 @@
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
// Command common-tracker-patterns-import seeds the common_tracker_patterns
// table from the Open Cookie Database (https://github.com/jkwakman/Open-Cookie-Database).
// It clones the repository into a temporary directory, reads
// open-cookie-database.json, and upserts each entry. Re-running is safe:
// existing rows are updated on the unique constraint so ids and created_at
// are preserved.
//
// Entries are linked to the matching common_third_parties row via a
// three-step cascade: slug lookup, domain lookup, then auto-create.
package main
package commontrackerpatterns
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"math"
"net"
"net/url"
"os"
"path/filepath"
"regexp"
@@ -41,9 +29,11 @@ import (
"time"
git "github.com/go-git/go-git/v5"
"github.com/spf13/cobra"
"go.gearno.de/kit/pg"
"go.probo.inc/probo/pkg/coredata"
"go.probo.inc/probo/pkg/gid"
"go.probo.inc/probo/pkg/proboctl/cmdutil"
"go.probo.inc/probo/pkg/slug"
)
@@ -78,120 +68,114 @@ type (
}
)
func main() {
if err := run(); err != nil {
fmt.Fprintf(os.Stderr, "error: %v\n", err)
os.Exit(1)
}
}
func NewCmdCommonTrackerPatterns(f *cmdutil.Factory) *cobra.Command {
return &cobra.Command{
Use: "common-tracker-patterns",
Short: "Seed common tracker patterns from the Open Cookie Database",
Long: "Seed the common_tracker_patterns table from the Open Cookie Database " +
"(https://github.com/jkwakman/Open-Cookie-Database). " +
"The repository is cloned into a temporary directory. " +
"Re-running is safe: existing rows are upserted so ids and created_at are preserved. " +
"Entries are linked to matching common_third_parties rows via slug lookup, " +
"domain lookup, then auto-create.",
RunE: func(cmd *cobra.Command, args []string) error {
out := f.IOStreams.Out
errOut := f.IOStreams.ErrOut
ctx := cmd.Context()
func run() error {
var pgDSN string
_, _ = fmt.Fprintf(out, "cloning %s\n", ocdRepoURL)
flag.StringVar(
&pgDSN,
"pg-dsn",
os.Getenv("DATABASE_URL"),
"PostgreSQL connection URL (default: DATABASE_URL env)",
)
flag.Parse()
if pgDSN == "" {
return fmt.Errorf("set -pg-dsn or DATABASE_URL")
}
ctx := context.Background()
fmt.Printf("cloning %s\n", ocdRepoURL)
tmpDir, cleanup, err := cloneRepo()
if err != nil {
return fmt.Errorf("cannot clone repository: %w", err)
}
defer cleanup()
patterns, err := loadPatternsFromOCD(tmpDir)
if err != nil {
return fmt.Errorf("cannot load tracker pattern data: %w", err)
}
pgClient, err := newPgClientFromDSN(pgDSN)
if err != nil {
return fmt.Errorf("cannot create pg client: %w", err)
}
fmt.Printf("importing %d common tracker patterns from Open Cookie Database\n", len(patterns))
var inserted, updated, skipped int
var partiesCreated int
if err := pgClient.WithTx(
ctx,
func(ctx context.Context, tx pg.Tx) error {
now := time.Now()
thirdPartyCache := make(map[string]*gid.GID)
for _, p := range patterns {
thirdPartyID, err := resolveThirdParty(ctx, tx, p, thirdPartyCache, now, &partiesCreated)
if err != nil {
return err
}
trackerType, err := parseTrackerType(p.TrackerType)
if err != nil {
fmt.Fprintf(os.Stderr, "warning: %v, skipping pattern %q\n", err, p.Pattern)
skipped++
continue
}
matchType, err := parseMatchType(p.MatchType)
if err != nil {
fmt.Fprintf(os.Stderr, "warning: %v, skipping pattern %q\n", err, p.Pattern)
skipped++
continue
}
pattern := coredata.CommonTrackerPattern{
ID: gid.New(gid.NilTenant, coredata.CommonTrackerPatternEntityType),
CommonThirdPartyID: thirdPartyID,
TrackerType: trackerType,
Pattern: p.Pattern,
MatchType: matchType,
Description: p.Description,
MaxAgeSeconds: p.MaxAgeSeconds,
Confidence: p.Confidence,
CreatedAt: now,
UpdatedAt: now,
}
_, wasInserted, err := pattern.Upsert(ctx, tx)
if err != nil {
return fmt.Errorf("cannot upsert common tracker pattern %q: %w", p.Pattern, err)
}
if wasInserted {
inserted++
} else {
updated++
}
tmpDir, cleanup, err := cloneRepo()
if err != nil {
return fmt.Errorf("cannot clone repository: %w", err)
}
defer cleanup()
patterns, err := loadPatternsFromOCD(tmpDir)
if err != nil {
return fmt.Errorf("cannot load tracker pattern data: %w", err)
}
pgClient, err := f.PgClient()
if err != nil {
return fmt.Errorf("cannot create pg client: %w", err)
}
_, _ = fmt.Fprintf(out, "seeding %d common tracker patterns from Open Cookie Database\n", len(patterns))
var inserted, updated, skipped int
var partiesCreated int
if err := pgClient.WithTx(
ctx,
func(ctx context.Context, tx pg.Tx) error {
now := time.Now()
thirdPartyCache := make(map[string]*gid.GID)
for _, p := range patterns {
thirdPartyID, err := resolveThirdParty(ctx, tx, p, thirdPartyCache, now, &partiesCreated)
if err != nil {
return err
}
trackerType, err := parseTrackerType(p.TrackerType)
if err != nil {
_, _ = fmt.Fprintf(errOut, "warning: %v, skipping pattern %q\n", err, p.Pattern)
skipped++
continue
}
matchType, err := parseMatchType(p.MatchType)
if err != nil {
_, _ = fmt.Fprintf(errOut, "warning: %v, skipping pattern %q\n", err, p.Pattern)
skipped++
continue
}
pattern := coredata.CommonTrackerPattern{
ID: gid.New(gid.NilTenant, coredata.CommonTrackerPatternEntityType),
CommonThirdPartyID: thirdPartyID,
TrackerType: trackerType,
Pattern: p.Pattern,
MatchType: matchType,
Description: p.Description,
MaxAgeSeconds: p.MaxAgeSeconds,
Confidence: p.Confidence,
CreatedAt: now,
UpdatedAt: now,
}
_, wasInserted, err := pattern.Upsert(ctx, tx)
if err != nil {
return fmt.Errorf("cannot upsert common tracker pattern %q: %w", p.Pattern, err)
}
if wasInserted {
inserted++
} else {
updated++
}
}
return nil
},
); err != nil {
return err
}
_, _ = fmt.Fprintf(
out,
"seeded %d patterns (%d inserted, %d updated, %d skipped, %d third parties auto-created)\n",
len(patterns)-skipped,
inserted,
updated,
skipped,
partiesCreated,
)
return nil
},
); err != nil {
return err
}
fmt.Printf(
"imported %d patterns (%d inserted, %d updated, %d skipped, %d third parties auto-created)\n",
len(patterns)-skipped,
inserted,
updated,
skipped,
partiesCreated,
)
return nil
}
func cloneRepo() (string, func(), error) {
@@ -360,7 +344,6 @@ var domainValidRe = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9.-]*[a-zA-Z]$`)
func normalizeDomain(s string) string {
s = strings.TrimSpace(s)
// Strip zero-width spaces and other Unicode control characters.
s = strings.Map(func(r rune) rune {
if r == '\u200b' || r == '\ufeff' {
return -1
@@ -368,19 +351,15 @@ func normalizeDomain(s string) string {
return r
}, s)
// Take only the first domain when "or" separates multiple values,
// e.g. ".bing.com (3rd party) or .microsoft.com (3rd party)".
if idx := strings.Index(s, " or "); idx != -1 {
s = s[:idx]
}
// Handle values starting with "or ", e.g. "or demdex.net (3rd party)".
s = strings.TrimPrefix(s, "or ")
if idx := strings.IndexByte(s, '('); idx != -1 {
s = strings.TrimSpace(s[:idx])
}
// Strip placeholders like "[account]".
if strings.ContainsAny(s, "[]") {
return ""
}
@@ -487,43 +466,3 @@ func parseMatchType(s string) (coredata.TrackerPatternMatchType, error) {
return "", fmt.Errorf("unknown match type %q", s)
}
}
func newPgClientFromDSN(dsn string) (*pg.Client, error) {
u, err := url.Parse(dsn)
if err != nil {
return nil, fmt.Errorf("cannot parse DSN (check URL format)")
}
var opts []pg.Option
switch u.Query().Get("sslmode") {
case "", "disable":
case "require":
opts = append(opts, pg.WithUnsecureTLS())
case "prefer":
return nil, fmt.Errorf("unsupported sslmode %q (prefer fallback semantics are not supported)", u.Query().Get("sslmode"))
default:
return nil, fmt.Errorf("unsupported sslmode %q", u.Query().Get("sslmode"))
}
if u.Host != "" {
host := u.Host
if u.Port() == "" {
host = net.JoinHostPort(u.Hostname(), "5432")
}
opts = append(opts, pg.WithAddr(host))
}
if u.User != nil {
opts = append(opts, pg.WithUser(u.User.Username()))
if password, ok := u.User.Password(); ok {
opts = append(opts, pg.WithPassword(password))
}
}
if len(u.Path) > 1 {
opts = append(opts, pg.WithDatabase(u.Path[1:]))
}
return pg.NewClient(opts...)
}

34
pkg/proboctl/seed/seed.go Normal file
View File

@@ -0,0 +1,34 @@
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package seed
import (
"github.com/spf13/cobra"
"go.probo.inc/probo/pkg/proboctl/cmdutil"
commonthirdparties "go.probo.inc/probo/pkg/proboctl/seed/common-third-parties"
commontrackerpatterns "go.probo.inc/probo/pkg/proboctl/seed/common-tracker-patterns"
)
func NewCmdSeed(f *cmdutil.Factory) *cobra.Command {
cmd := &cobra.Command{
Use: "seed <command>",
Short: "Seed common reference data",
}
cmd.AddCommand(commonthirdparties.NewCmdCommonThirdParties(f))
cmd.AddCommand(commontrackerpatterns.NewCmdCommonTrackerPatterns(f))
return cmd
}

49
pkg/proboctl/version/version.go Normal file
View File

@@ -0,0 +1,49 @@
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package version
import (
"fmt"
"github.com/spf13/cobra"
"go.probo.inc/probo/pkg/proboctl/cmdutil"
"go.probo.inc/probo/pkg/version"
)
func NewCmdVersion(f *cmdutil.Factory) *cobra.Command {
return &cobra.Command{
Use: "version",
Short: "Print the version of proboctl",
RunE: func(cmd *cobra.Command, args []string) error {
info := version.GetBuildInfo()
v := f.Version
if v == "" || v == "unknown" {
v = info.Version
}
_, _ = fmt.Fprintf(
f.IOStreams.Out,
"proboctl version %s (commit: %s, built: %s, go: %s)\n",
v,
info.Commit,
info.BuildDate,
info.GoVersion,
)
return nil
},
}
}