diff --git a/GNUmakefile b/GNUmakefile index aae952963..1931efb1f 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -25,10 +25,12 @@ DOCKER_COMPOSE= $(DOCKER) compose -f compose.yaml $(DOCKER_COMPOSE_FLAGS) PRB_VERSION= $(shell cat cmd/prb/VERSION) PROBOD_VERSION= $(shell cat cmd/probod/VERSION) PROBOD_BOOTSTRAP_VERSION=$(shell cat cmd/probod-bootstrap/VERSION) +PROBOCTL_VERSION= $(shell cat cmd/proboctl/VERSION) PRB_LDFLAGS= -ldflags "-X 'main.version=$(PRB_VERSION)'" PROBOD_LDFLAGS= -ldflags "-X 'main.version=$(PROBOD_VERSION)' -X 'main.env=prod'" PROBOD_BOOTSTRAP_LDFLAGS=-ldflags "-X 'main.version=$(PROBOD_BOOTSTRAP_VERSION)'" +PROBOCTL_LDFLAGS= -ldflags "-X 'main.version=$(PROBOCTL_VERSION)'" GCFLAGS= -gcflags="-e" @@ -73,6 +75,9 @@ PRB_SRC= cmd/prb/main.go PROBOD_BOOTSTRAP_BIN= bin/probod-bootstrap PROBOD_BOOTSTRAP_SRC= cmd/probod-bootstrap/main.go +PROBOCTL_BIN= bin/proboctl +PROBOCTL_SRC= cmd/proboctl/main.go + ifdef WITH_APPS GENERATED += relay EMBEDDED += \ @@ -172,7 +177,7 @@ coverage-combined: coverage-report test-e2e-coverage ## Generate combined covera $(GO) tool cover -html=coverage-combined.out -o=coverage-combined.html .PHONY: build -build: $(PROBOD_BIN) bin/prb bin/probod-bootstrap +build: $(PROBOD_BIN) bin/prb bin/probod-bootstrap bin/proboctl CFG_DEV_OAUTH2_KEY = cfg/.dev-oauth2-signing-key.pem DEV_ENV = .env @@ -252,6 +257,10 @@ bin/prb: $(PROBOD_BOOTSTRAP_BIN): $(GO_BUILD) $(PROBOD_BOOTSTRAP_LDFLAGS) -o $(PROBOD_BOOTSTRAP_BIN) $(PROBOD_BOOTSTRAP_SRC) +.PHONY: bin/proboctl +bin/proboctl: + $(GO_BUILD) $(PROBOCTL_LDFLAGS) -o $(PROBOCTL_BIN) $(PROBOCTL_SRC) + .PHONY: @probo/emails @probo/emails: $(NPM) --workspace $@ run build diff --git a/cmd/proboctl/CHANGELOG.md b/cmd/proboctl/CHANGELOG.md new file mode 100644 index 000000000..825c32f0d --- /dev/null +++ b/cmd/proboctl/CHANGELOG.md @@ -0,0 +1 @@ +# Changelog diff --git a/cmd/proboctl/VERSION b/cmd/proboctl/VERSION new file mode 100644 index 000000000..77d6f4ca2 --- /dev/null +++ b/cmd/proboctl/VERSION @@ -0,0 +1 @@ +0.0.0 diff --git a/cmd/proboctl/main.go b/cmd/proboctl/main.go new file mode 100644 index 000000000..29f9f3818 --- /dev/null +++ b/cmd/proboctl/main.go @@ -0,0 +1,42 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package main + +import ( + "fmt" + "os" + + "go.probo.inc/probo/pkg/cmd/iostreams" + "go.probo.inc/probo/pkg/proboctl/cmdutil" + "go.probo.inc/probo/pkg/proboctl/root" +) + +var version string = "unknown" + +func main() { + ios := iostreams.System() + + f := &cmdutil.Factory{ + IOStreams: ios, + Version: version, + } + + cmd := root.NewCmdRoot(f) + + if err := cmd.Execute(); err != nil { + fmt.Fprintf(os.Stderr, "Error: %s\n", err) + os.Exit(1) + } +} diff --git a/pkg/proboctl/cmdutil/cmdutil.go b/pkg/proboctl/cmdutil/cmdutil.go new file mode 100644 index 000000000..2086695fa --- /dev/null +++ b/pkg/proboctl/cmdutil/cmdutil.go @@ -0,0 +1,36 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package cmdutil + +import ( + "fmt" + + "go.gearno.de/kit/pg" + "go.probo.inc/probo/pkg/cmd/iostreams" + "go.probo.inc/probo/pkg/proboctl/pgconn" +) + +type Factory struct { + IOStreams *iostreams.IOStreams + Version string + PgDSN string +} + +func (f *Factory) PgClient() (*pg.Client, error) { + if f.PgDSN == "" { + return nil, fmt.Errorf("set --pg-dsn or DATABASE_URL") + } + return pgconn.NewPgClientFromDSN(f.PgDSN) +} diff --git a/pkg/proboctl/pgconn/pgconn.go b/pkg/proboctl/pgconn/pgconn.go new file mode 100644 index 000000000..ca094e079 --- /dev/null +++ b/pkg/proboctl/pgconn/pgconn.go @@ -0,0 +1,66 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package pgconn + +import ( + "fmt" + "net" + "net/url" + + "go.gearno.de/kit/pg" +) + +func NewPgClientFromDSN(dsn string) (*pg.Client, error) { + u, err := url.Parse(dsn) + if err != nil { + return nil, fmt.Errorf("cannot parse DSN (check URL format)") + } + + var opts []pg.Option + + switch u.Query().Get("sslmode") { + case "", "disable": + case "require": + opts = append(opts, pg.WithUnsecureTLS()) + case "prefer": + return nil, fmt.Errorf( + "unsupported sslmode %q (prefer fallback semantics are not supported)", + u.Query().Get("sslmode"), + ) + default: + return nil, fmt.Errorf("unsupported sslmode %q", u.Query().Get("sslmode")) + } + + if u.Host != "" { + host := u.Host + if u.Port() == "" { + host = net.JoinHostPort(u.Hostname(), "5432") + } + opts = append(opts, pg.WithAddr(host)) + } + + if u.User != nil { + opts = append(opts, pg.WithUser(u.User.Username())) + if password, ok := u.User.Password(); ok { + opts = append(opts, pg.WithPassword(password)) + } + } + + if len(u.Path) > 1 { + opts = append(opts, pg.WithDatabase(u.Path[1:])) + } + + return pg.NewClient(opts...) +} diff --git a/pkg/proboctl/root/root.go b/pkg/proboctl/root/root.go new file mode 100644 index 000000000..f843c6d76 --- /dev/null +++ b/pkg/proboctl/root/root.go @@ -0,0 +1,45 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package root + +import ( + "os" + + "github.com/spf13/cobra" + "go.probo.inc/probo/pkg/proboctl/cmdutil" + "go.probo.inc/probo/pkg/proboctl/seed" + "go.probo.inc/probo/pkg/proboctl/version" +) + +func NewCmdRoot(f *cmdutil.Factory) *cobra.Command { + cmd := &cobra.Command{ + Use: "proboctl [flags]", + Short: "Probo instance management CLI", + SilenceUsage: true, + SilenceErrors: true, + } + + cmd.PersistentFlags().StringVar( + &f.PgDSN, + "pg-dsn", + os.Getenv("DATABASE_URL"), + "PostgreSQL connection URL (default: DATABASE_URL env)", + ) + + cmd.AddCommand(seed.NewCmdSeed(f)) + cmd.AddCommand(version.NewCmdVersion(f)) + + return cmd +} diff --git a/cmd/common-third-parties-import/main.go b/pkg/proboctl/seed/common-third-parties/common_third_parties.go similarity index 50% rename from cmd/common-third-parties-import/main.go rename to pkg/proboctl/seed/common-third-parties/common_third_parties.go index bf65e039e..f3ef88d49 100644 --- a/cmd/common-third-parties-import/main.go +++ b/pkg/proboctl/seed/common-third-parties/common_third_parties.go @@ -12,24 +12,14 @@ // OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR // PERFORMANCE OF THIS SOFTWARE. -// Command common-third-parties-import seeds the common_third_parties table from -// packages/thirdParties/data.json. It is idempotent: re-running upserts on conflict -// (slug) so existing rows keep their id and created_at. -// -// When -fetch-logos is set, the tool inspects each third party's website to -// find the best available logo (SVG icon, apple-touch-icon, etc.) and stores -// it in S3 as a public file, linking it to each common third party via -// logo_file_id. -package main +package commonthirdparties import ( "bytes" "context" "encoding/json" - "flag" "fmt" "io" - "net" "net/http" "net/url" "os" @@ -38,12 +28,14 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/spf13/cobra" "go.gearno.de/crypto/uuid" "go.gearno.de/kit/httpclient" "go.gearno.de/kit/pg" "go.probo.inc/probo/pkg/coredata" "go.probo.inc/probo/pkg/filemanager" "go.probo.inc/probo/pkg/gid" + "go.probo.inc/probo/pkg/proboctl/cmdutil" "go.probo.inc/probo/pkg/slug" "go.probo.inc/probo/pkg/version" "go.probo.inc/probo/pkg/webinspect" @@ -69,191 +61,151 @@ type thirdPartyData struct { Domains []string `json:"domains,omitempty"` } -func main() { - if err := run(); err != nil { - fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } -} - -func run() error { +func NewCmdCommonThirdParties(f *cmdutil.Factory) *cobra.Command { var ( - pgDSN string - dataPath string - fetchLogos bool - s3Bucket string - s3Endpoint string - s3Region string - s3AccessKey string - s3SecretKey string - s3UsePathStyle bool + flagData string + flagFetchLogos bool + flagS3Bucket string + flagS3Endpoint string + flagS3Region string + flagS3AccessKey string + flagS3SecretKey string + flagS3UsePathStyle bool ) - flag.StringVar( - &pgDSN, - "pg-dsn", - os.Getenv("DATABASE_URL"), - "PostgreSQL connection URL (default: DATABASE_URL env)", - ) - flag.StringVar( - &dataPath, - "data", - "", - "Path to the third-party data.json file", - ) - flag.BoolVar( - &fetchLogos, - "fetch-logos", - false, - "Fetch favicons from Google and store them in S3", - ) - flag.StringVar( - &s3Bucket, - "s3-bucket", - os.Getenv("AWS_S3_BUCKET"), - "S3 bucket name (default: AWS_S3_BUCKET env)", - ) - flag.StringVar( - &s3Endpoint, - "s3-endpoint", - os.Getenv("AWS_ENDPOINT_URL"), - "S3 endpoint URL (default: AWS_ENDPOINT_URL env)", - ) - flag.StringVar( - &s3Region, - "s3-region", - os.Getenv("AWS_REGION"), - "S3 region (default: AWS_REGION env)", - ) - flag.StringVar( - &s3AccessKey, - "s3-access-key", - os.Getenv("AWS_ACCESS_KEY_ID"), - "S3 access key ID (default: AWS_ACCESS_KEY_ID env)", - ) - flag.StringVar( - &s3SecretKey, - "s3-secret-key", - os.Getenv("AWS_SECRET_ACCESS_KEY"), - "S3 secret access key (default: AWS_SECRET_ACCESS_KEY env)", - ) - flag.BoolVar( - &s3UsePathStyle, - "s3-path-style", - false, - "Use S3 path-style addressing", - ) - flag.Parse() + cmd := &cobra.Command{ + Use: "common-third-parties", + Short: "Seed common third parties from a data.json file", + Long: "Seed the common_third_parties table from a JSON file. " + + "Re-running is safe: existing rows are upserted on conflict (slug) " + + "so ids and created_at are preserved.", + RunE: func(cmd *cobra.Command, args []string) error { + out := f.IOStreams.Out + errOut := f.IOStreams.ErrOut - if pgDSN == "" { - return fmt.Errorf("set -pg-dsn or DATABASE_URL") - } + if flagFetchLogos && flagS3Bucket == "" { + return fmt.Errorf("set --s3-bucket or AWS_S3_BUCKET when using --fetch-logos") + } - if fetchLogos && s3Bucket == "" { - return fmt.Errorf("set -s3-bucket or AWS_S3_BUCKET when using -fetch-logos") - } + ctx := cmd.Context() - ctx := context.Background() + thirdParties, err := loadThirdParties(flagData) + if err != nil { + return fmt.Errorf("cannot load third-party data: %w", err) + } - thirdParties, err := loadThirdParties(dataPath) - if err != nil { - return fmt.Errorf("cannot load third-party data: %w", err) - } + pgClient, err := f.PgClient() + if err != nil { + return fmt.Errorf("cannot create pg client: %w", err) + } - pgClient, err := newPgClientFromDSN(pgDSN) - if err != nil { - return fmt.Errorf("cannot create pg client: %w", err) - } + _, _ = fmt.Fprintf(out, "seeding %d common third parties from %s\n", len(thirdParties), flagData) - fmt.Printf("importing %d common third parties from %s\n", len(thirdParties), dataPath) + var inserted, updated, domainsInserted, domainsUpdated int - var inserted, updated, domainsInserted, domainsUpdated int + if err := pgClient.WithTx( + ctx, + func(ctx context.Context, tx pg.Tx) error { + now := time.Now() - if err := pgClient.WithTx( - ctx, - func(ctx context.Context, tx pg.Tx) error { - now := time.Now() + for _, tp := range thirdParties { + party := coredata.CommonThirdParty{ + ID: gid.New(gid.NilTenant, coredata.CommonThirdPartyEntityType), + Name: tp.Name, + Slug: slug.Make(tp.Name), + Category: parseCategory(errOut, tp), + HeadquarterAddress: tp.HeadquarterAddress, + LegalName: tp.LegalName, + WebsiteURL: tp.WebsiteURL, + PrivacyPolicyURL: tp.PrivacyPolicyURL, + ServiceLevelAgreementURL: tp.ServiceLevelAgreementURL, + ServiceSoftwareAgreementURL: tp.ServiceSoftwareAgreementURL, + DataProcessingAgreementURL: tp.DataProcessingAgreementURL, + BusinessAssociateAgreementURL: tp.BusinessAssociateAgreementURL, + SubprocessorsListURL: tp.SubprocessorsListURL, + Certifications: tp.Certifications, + StatusPageURL: tp.StatusPageURL, + TermsOfServiceURL: tp.TermsOfServiceURL, + SecurityPageURL: tp.SecurityPageURL, + TrustPageURL: tp.TrustPageURL, + CreatedAt: now, + UpdatedAt: now, + } - for _, tp := range thirdParties { - party := coredata.CommonThirdParty{ - ID: gid.New(gid.NilTenant, coredata.CommonThirdPartyEntityType), - Name: tp.Name, - Slug: slug.Make(tp.Name), - Category: parseCategory(tp), - HeadquarterAddress: tp.HeadquarterAddress, - LegalName: tp.LegalName, - WebsiteURL: tp.WebsiteURL, - PrivacyPolicyURL: tp.PrivacyPolicyURL, - ServiceLevelAgreementURL: tp.ServiceLevelAgreementURL, - ServiceSoftwareAgreementURL: tp.ServiceSoftwareAgreementURL, - DataProcessingAgreementURL: tp.DataProcessingAgreementURL, - BusinessAssociateAgreementURL: tp.BusinessAssociateAgreementURL, - SubprocessorsListURL: tp.SubprocessorsListURL, - Certifications: tp.Certifications, - StatusPageURL: tp.StatusPageURL, - TermsOfServiceURL: tp.TermsOfServiceURL, - SecurityPageURL: tp.SecurityPageURL, - TrustPageURL: tp.TrustPageURL, - CreatedAt: now, - UpdatedAt: now, - } + wasInserted, err := party.Upsert(ctx, tx) + if err != nil { + return fmt.Errorf("cannot upsert common third party %q: %w", tp.Name, err) + } - wasInserted, err := party.Upsert(ctx, tx) - if err != nil { - return fmt.Errorf("cannot upsert common third party %q: %w", tp.Name, err) - } + if wasInserted { + inserted++ + } else { + updated++ + if err := party.LoadByName(ctx, tx, tp.Name); err != nil { + return fmt.Errorf("cannot reload common third party %q: %w", tp.Name, err) + } + } - if wasInserted { - inserted++ - } else { - updated++ - if err := party.LoadByName(ctx, tx, tp.Name); err != nil { - return fmt.Errorf("cannot reload common third party %q: %w", tp.Name, err) - } - } + for _, domain := range tp.Domains { + d := coredata.CommonThirdPartyDomain{ + ID: gid.New(gid.NilTenant, coredata.CommonThirdPartyDomainEntityType), + CommonThirdPartyID: party.ID, + Domain: domain, + CreatedAt: now, + UpdatedAt: now, + } - for _, domain := range tp.Domains { - d := coredata.CommonThirdPartyDomain{ - ID: gid.New(gid.NilTenant, coredata.CommonThirdPartyDomainEntityType), - CommonThirdPartyID: party.ID, - Domain: domain, - CreatedAt: now, - UpdatedAt: now, + domainInserted, err := d.Upsert(ctx, tx) + if err != nil { + return fmt.Errorf("cannot upsert domain %q for %q: %w", domain, tp.Name, err) + } + + if domainInserted { + domainsInserted++ + } else { + domainsUpdated++ + } + } } - domainInserted, err := d.Upsert(ctx, tx) - if err != nil { - return fmt.Errorf("cannot upsert domain %q for %q: %w", domain, tp.Name, err) - } + return nil + }, + ); err != nil { + return err + } - if domainInserted { - domainsInserted++ - } else { - domainsUpdated++ - } + _, _ = fmt.Fprintf(out, "seeded %d third parties (%d inserted, %d updated)\n", len(thirdParties), inserted, updated) + _, _ = fmt.Fprintf(out, "seeded %d domains (%d inserted, %d updated)\n", domainsInserted+domainsUpdated, domainsInserted, domainsUpdated) + + if flagFetchLogos { + if err := fetchAndStoreLogos( + ctx, out, errOut, pgClient, thirdParties, + flagS3Bucket, flagS3Endpoint, flagS3Region, flagS3AccessKey, flagS3SecretKey, flagS3UsePathStyle, + ); err != nil { + return fmt.Errorf("cannot fetch logos: %w", err) } } return nil }, - ); err != nil { - return err } - fmt.Printf("imported %d third parties (%d inserted, %d updated)\n", len(thirdParties), inserted, updated) - fmt.Printf("imported %d domains (%d inserted, %d updated)\n", domainsInserted+domainsUpdated, domainsInserted, domainsUpdated) + cmd.Flags().StringVar(&flagData, "data", "", "Path to the third-party data.json file") + _ = cmd.MarkFlagRequired("data") + cmd.Flags().BoolVar(&flagFetchLogos, "fetch-logos", false, "Fetch favicons and store them in S3") + cmd.Flags().StringVar(&flagS3Bucket, "s3-bucket", os.Getenv("AWS_S3_BUCKET"), "S3 bucket name (default: AWS_S3_BUCKET env)") + cmd.Flags().StringVar(&flagS3Endpoint, "s3-endpoint", os.Getenv("AWS_ENDPOINT_URL"), "S3 endpoint URL (default: AWS_ENDPOINT_URL env)") + cmd.Flags().StringVar(&flagS3Region, "s3-region", os.Getenv("AWS_REGION"), "S3 region (default: AWS_REGION env)") + cmd.Flags().StringVar(&flagS3AccessKey, "s3-access-key", os.Getenv("AWS_ACCESS_KEY_ID"), "S3 access key ID (default: AWS_ACCESS_KEY_ID env)") + cmd.Flags().StringVar(&flagS3SecretKey, "s3-secret-key", os.Getenv("AWS_SECRET_ACCESS_KEY"), "S3 secret access key (default: AWS_SECRET_ACCESS_KEY env)") + cmd.Flags().BoolVar(&flagS3UsePathStyle, "s3-path-style", false, "Use S3 path-style addressing") - if fetchLogos { - if err := fetchAndStoreLogos(ctx, pgClient, thirdParties, s3Bucket, s3Endpoint, s3Region, s3AccessKey, s3SecretKey, s3UsePathStyle); err != nil { - return fmt.Errorf("cannot fetch logos: %w", err) - } - } - - return nil + return cmd } func fetchAndStoreLogos( ctx context.Context, + out, errOut io.Writer, pgClient *pg.Client, thirdParties []thirdPartyData, bucket, endpoint, region, accessKey, secretKey string, @@ -264,7 +216,7 @@ func fetchAndStoreLogos( httpClient := httpclient.DefaultPooledClient(httpclient.WithSSRFProtection()) httpClient.Transport = &userAgentTransport{ next: httpClient.Transport, - ua: version.UserAgent("common-third-parties-import"), + ua: version.UserAgent("proboctl"), } scope := coredata.NewScope(gid.NilTenant) @@ -280,7 +232,7 @@ func fetchAndStoreLogos( if err := pgClient.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error { return party.LoadByName(ctx, conn, tp.Name) }); err != nil { - fmt.Fprintf(os.Stderr, "warning: cannot load %q, skipping logo: %v\n", tp.Name, err) + _, _ = fmt.Fprintf(errOut, "warning: cannot load %q, skipping logo: %v\n", tp.Name, err) failed++ continue } @@ -293,17 +245,17 @@ func fetchAndStoreLogos( var logoURL string pageInfo, err := webinspect.Parse(ctx, httpClient, *tp.WebsiteURL) if err != nil { - fmt.Fprintf(os.Stderr, "warning: cannot inspect page for %q, trying default apple-touch-icon: %v\n", tp.Name, err) + _, _ = fmt.Fprintf(errOut, "warning: cannot inspect page for %q, trying default apple-touch-icon: %v\n", tp.Name, err) } else { logoURL, err = webinspect.FindLogoURL(pageInfo) if err != nil { - fmt.Fprintf(os.Stderr, "warning: cannot find logo for %q, trying default apple-touch-icon: %v\n", tp.Name, err) + _, _ = fmt.Fprintf(errOut, "warning: cannot find logo for %q, trying default apple-touch-icon: %v\n", tp.Name, err) } } parsed, err := url.Parse(*tp.WebsiteURL) if err != nil { - fmt.Fprintf(os.Stderr, "warning: cannot parse URL for %q, skipping logo: %v\n", tp.Name, err) + _, _ = fmt.Fprintf(errOut, "warning: cannot parse URL for %q, skipping logo: %v\n", tp.Name, err) failed++ continue } @@ -313,7 +265,8 @@ func fetchAndStoreLogos( candidateURLs = append(candidateURLs, logoURL) } base := fmt.Sprintf("%s://%s", parsed.Scheme, parsed.Host) - candidateURLs = append(candidateURLs, + candidateURLs = append( + candidateURLs, base+"/apple-touch-icon.png", base+"/apple-touch-icon-precomposed.png", "https://logo.debounce.com/"+parsed.Host, @@ -342,7 +295,7 @@ func fetchAndStoreLogos( } if len(body) == 0 { - fmt.Fprintf(os.Stderr, "warning: cannot fetch logo for %q from any candidate URL\n", tp.Name) + _, _ = fmt.Fprintf(errOut, "warning: cannot fetch logo for %q from any candidate URL\n", tp.Name) failed++ continue } @@ -376,7 +329,7 @@ func fetchAndStoreLogos( "type": "common-third-party-logo", "common-third-party-id": party.ID.String(), }); err != nil { - fmt.Fprintf(os.Stderr, "warning: cannot upload logo for %q to S3: %v\n", tp.Name, err) + _, _ = fmt.Fprintf(errOut, "warning: cannot upload logo for %q to S3: %v\n", tp.Name, err) failed++ continue } @@ -394,16 +347,16 @@ func fetchAndStoreLogos( return nil }); err != nil { - fmt.Fprintf(os.Stderr, "warning: cannot store logo for %q: %v\n", tp.Name, err) + _, _ = fmt.Fprintf(errOut, "warning: cannot store logo for %q: %v\n", tp.Name, err) failed++ continue } fetched++ - fmt.Printf(" fetched logo for %q\n", tp.Name) + _, _ = fmt.Fprintf(out, " fetched logo for %q\n", tp.Name) } - fmt.Printf("logos: %d fetched, %d skipped, %d failed\n", fetched, skipped, failed) + _, _ = fmt.Fprintf(out, "logos: %d fetched, %d skipped, %d failed\n", fetched, skipped, failed) return nil } @@ -447,61 +400,20 @@ func loadThirdParties(path string) ([]thirdPartyData, error) { return thirdParties, nil } -func parseCategory(tp thirdPartyData) coredata.ThirdPartyCategory { +func parseCategory(errOut io.Writer, tp thirdPartyData) coredata.ThirdPartyCategory { if tp.Category == nil || *tp.Category == "" { return coredata.ThirdPartyCategoryOther } var c coredata.ThirdPartyCategory if err := c.Scan(*tp.Category); err != nil { - fmt.Fprintf(os.Stderr, "warning: third party %q has unknown category %q, falling back to OTHER\n", tp.Name, *tp.Category) + _, _ = fmt.Fprintf(errOut, "warning: third party %q has unknown category %q, falling back to OTHER\n", tp.Name, *tp.Category) return coredata.ThirdPartyCategoryOther } return c } -func newPgClientFromDSN(dsn string) (*pg.Client, error) { - u, err := url.Parse(dsn) - if err != nil { - return nil, fmt.Errorf("cannot parse DSN (check URL format)") - } - - var opts []pg.Option - - switch u.Query().Get("sslmode") { - case "", "disable": - // plain connection, no TLS - case "require": - opts = append(opts, pg.WithUnsecureTLS()) - case "prefer": - return nil, fmt.Errorf("unsupported sslmode %q (prefer fallback semantics are not supported)", u.Query().Get("sslmode")) - default: - return nil, fmt.Errorf("unsupported sslmode %q", u.Query().Get("sslmode")) - } - - if u.Host != "" { - host := u.Host - if u.Port() == "" { - host = net.JoinHostPort(u.Hostname(), "5432") - } - opts = append(opts, pg.WithAddr(host)) - } - - if u.User != nil { - opts = append(opts, pg.WithUser(u.User.Username())) - if password, ok := u.User.Password(); ok { - opts = append(opts, pg.WithPassword(password)) - } - } - - if len(u.Path) > 1 { - opts = append(opts, pg.WithDatabase(u.Path[1:])) - } - - return pg.NewClient(opts...) -} - type userAgentTransport struct { next http.RoundTripper ua string diff --git a/cmd/common-tracker-patterns-import/main.go b/pkg/proboctl/seed/common-tracker-patterns/common_tracker_patterns.go similarity index 67% rename from cmd/common-tracker-patterns-import/main.go rename to pkg/proboctl/seed/common-tracker-patterns/common_tracker_patterns.go index adba5f148..43f3ab187 100644 --- a/cmd/common-tracker-patterns-import/main.go +++ b/pkg/proboctl/seed/common-tracker-patterns/common_tracker_patterns.go @@ -12,26 +12,14 @@ // OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR // PERFORMANCE OF THIS SOFTWARE. -// Command common-tracker-patterns-import seeds the common_tracker_patterns -// table from the Open Cookie Database (https://github.com/jkwakman/Open-Cookie-Database). -// It clones the repository into a temporary directory, reads -// open-cookie-database.json, and upserts each entry. Re-running is safe: -// existing rows are updated on the unique constraint so ids and created_at -// are preserved. -// -// Entries are linked to the matching common_third_parties row via a -// three-step cascade: slug lookup, domain lookup, then auto-create. -package main +package commontrackerpatterns import ( "context" "encoding/json" "errors" - "flag" "fmt" "math" - "net" - "net/url" "os" "path/filepath" "regexp" @@ -41,9 +29,11 @@ import ( "time" git "github.com/go-git/go-git/v5" + "github.com/spf13/cobra" "go.gearno.de/kit/pg" "go.probo.inc/probo/pkg/coredata" "go.probo.inc/probo/pkg/gid" + "go.probo.inc/probo/pkg/proboctl/cmdutil" "go.probo.inc/probo/pkg/slug" ) @@ -78,120 +68,114 @@ type ( } ) -func main() { - if err := run(); err != nil { - fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } -} +func NewCmdCommonTrackerPatterns(f *cmdutil.Factory) *cobra.Command { + return &cobra.Command{ + Use: "common-tracker-patterns", + Short: "Seed common tracker patterns from the Open Cookie Database", + Long: "Seed the common_tracker_patterns table from the Open Cookie Database " + + "(https://github.com/jkwakman/Open-Cookie-Database). " + + "The repository is cloned into a temporary directory. " + + "Re-running is safe: existing rows are upserted so ids and created_at are preserved. " + + "Entries are linked to matching common_third_parties rows via slug lookup, " + + "domain lookup, then auto-create.", + RunE: func(cmd *cobra.Command, args []string) error { + out := f.IOStreams.Out + errOut := f.IOStreams.ErrOut + ctx := cmd.Context() -func run() error { - var pgDSN string + _, _ = fmt.Fprintf(out, "cloning %s\n", ocdRepoURL) - flag.StringVar( - &pgDSN, - "pg-dsn", - os.Getenv("DATABASE_URL"), - "PostgreSQL connection URL (default: DATABASE_URL env)", - ) - flag.Parse() - - if pgDSN == "" { - return fmt.Errorf("set -pg-dsn or DATABASE_URL") - } - - ctx := context.Background() - - fmt.Printf("cloning %s\n", ocdRepoURL) - - tmpDir, cleanup, err := cloneRepo() - if err != nil { - return fmt.Errorf("cannot clone repository: %w", err) - } - defer cleanup() - - patterns, err := loadPatternsFromOCD(tmpDir) - if err != nil { - return fmt.Errorf("cannot load tracker pattern data: %w", err) - } - - pgClient, err := newPgClientFromDSN(pgDSN) - if err != nil { - return fmt.Errorf("cannot create pg client: %w", err) - } - - fmt.Printf("importing %d common tracker patterns from Open Cookie Database\n", len(patterns)) - - var inserted, updated, skipped int - var partiesCreated int - - if err := pgClient.WithTx( - ctx, - func(ctx context.Context, tx pg.Tx) error { - now := time.Now() - thirdPartyCache := make(map[string]*gid.GID) - - for _, p := range patterns { - thirdPartyID, err := resolveThirdParty(ctx, tx, p, thirdPartyCache, now, &partiesCreated) - if err != nil { - return err - } - - trackerType, err := parseTrackerType(p.TrackerType) - if err != nil { - fmt.Fprintf(os.Stderr, "warning: %v, skipping pattern %q\n", err, p.Pattern) - skipped++ - continue - } - - matchType, err := parseMatchType(p.MatchType) - if err != nil { - fmt.Fprintf(os.Stderr, "warning: %v, skipping pattern %q\n", err, p.Pattern) - skipped++ - continue - } - - pattern := coredata.CommonTrackerPattern{ - ID: gid.New(gid.NilTenant, coredata.CommonTrackerPatternEntityType), - CommonThirdPartyID: thirdPartyID, - TrackerType: trackerType, - Pattern: p.Pattern, - MatchType: matchType, - Description: p.Description, - MaxAgeSeconds: p.MaxAgeSeconds, - Confidence: p.Confidence, - CreatedAt: now, - UpdatedAt: now, - } - - _, wasInserted, err := pattern.Upsert(ctx, tx) - if err != nil { - return fmt.Errorf("cannot upsert common tracker pattern %q: %w", p.Pattern, err) - } - - if wasInserted { - inserted++ - } else { - updated++ - } + tmpDir, cleanup, err := cloneRepo() + if err != nil { + return fmt.Errorf("cannot clone repository: %w", err) } + defer cleanup() + + patterns, err := loadPatternsFromOCD(tmpDir) + if err != nil { + return fmt.Errorf("cannot load tracker pattern data: %w", err) + } + + pgClient, err := f.PgClient() + if err != nil { + return fmt.Errorf("cannot create pg client: %w", err) + } + + _, _ = fmt.Fprintf(out, "seeding %d common tracker patterns from Open Cookie Database\n", len(patterns)) + + var inserted, updated, skipped int + var partiesCreated int + + if err := pgClient.WithTx( + ctx, + func(ctx context.Context, tx pg.Tx) error { + now := time.Now() + thirdPartyCache := make(map[string]*gid.GID) + + for _, p := range patterns { + thirdPartyID, err := resolveThirdParty(ctx, tx, p, thirdPartyCache, now, &partiesCreated) + if err != nil { + return err + } + + trackerType, err := parseTrackerType(p.TrackerType) + if err != nil { + _, _ = fmt.Fprintf(errOut, "warning: %v, skipping pattern %q\n", err, p.Pattern) + skipped++ + continue + } + + matchType, err := parseMatchType(p.MatchType) + if err != nil { + _, _ = fmt.Fprintf(errOut, "warning: %v, skipping pattern %q\n", err, p.Pattern) + skipped++ + continue + } + + pattern := coredata.CommonTrackerPattern{ + ID: gid.New(gid.NilTenant, coredata.CommonTrackerPatternEntityType), + CommonThirdPartyID: thirdPartyID, + TrackerType: trackerType, + Pattern: p.Pattern, + MatchType: matchType, + Description: p.Description, + MaxAgeSeconds: p.MaxAgeSeconds, + Confidence: p.Confidence, + CreatedAt: now, + UpdatedAt: now, + } + + _, wasInserted, err := pattern.Upsert(ctx, tx) + if err != nil { + return fmt.Errorf("cannot upsert common tracker pattern %q: %w", p.Pattern, err) + } + + if wasInserted { + inserted++ + } else { + updated++ + } + } + + return nil + }, + ); err != nil { + return err + } + + _, _ = fmt.Fprintf( + out, + "seeded %d patterns (%d inserted, %d updated, %d skipped, %d third parties auto-created)\n", + len(patterns)-skipped, + inserted, + updated, + skipped, + partiesCreated, + ) return nil }, - ); err != nil { - return err } - - fmt.Printf( - "imported %d patterns (%d inserted, %d updated, %d skipped, %d third parties auto-created)\n", - len(patterns)-skipped, - inserted, - updated, - skipped, - partiesCreated, - ) - - return nil } func cloneRepo() (string, func(), error) { @@ -360,7 +344,6 @@ var domainValidRe = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9.-]*[a-zA-Z]$`) func normalizeDomain(s string) string { s = strings.TrimSpace(s) - // Strip zero-width spaces and other Unicode control characters. s = strings.Map(func(r rune) rune { if r == '\u200b' || r == '\ufeff' { return -1 @@ -368,19 +351,15 @@ func normalizeDomain(s string) string { return r }, s) - // Take only the first domain when "or" separates multiple values, - // e.g. ".bing.com (3rd party) or .microsoft.com (3rd party)". if idx := strings.Index(s, " or "); idx != -1 { s = s[:idx] } - // Handle values starting with "or ", e.g. "or demdex.net (3rd party)". s = strings.TrimPrefix(s, "or ") if idx := strings.IndexByte(s, '('); idx != -1 { s = strings.TrimSpace(s[:idx]) } - // Strip placeholders like "[account]". if strings.ContainsAny(s, "[]") { return "" } @@ -487,43 +466,3 @@ func parseMatchType(s string) (coredata.TrackerPatternMatchType, error) { return "", fmt.Errorf("unknown match type %q", s) } } - -func newPgClientFromDSN(dsn string) (*pg.Client, error) { - u, err := url.Parse(dsn) - if err != nil { - return nil, fmt.Errorf("cannot parse DSN (check URL format)") - } - - var opts []pg.Option - - switch u.Query().Get("sslmode") { - case "", "disable": - case "require": - opts = append(opts, pg.WithUnsecureTLS()) - case "prefer": - return nil, fmt.Errorf("unsupported sslmode %q (prefer fallback semantics are not supported)", u.Query().Get("sslmode")) - default: - return nil, fmt.Errorf("unsupported sslmode %q", u.Query().Get("sslmode")) - } - - if u.Host != "" { - host := u.Host - if u.Port() == "" { - host = net.JoinHostPort(u.Hostname(), "5432") - } - opts = append(opts, pg.WithAddr(host)) - } - - if u.User != nil { - opts = append(opts, pg.WithUser(u.User.Username())) - if password, ok := u.User.Password(); ok { - opts = append(opts, pg.WithPassword(password)) - } - } - - if len(u.Path) > 1 { - opts = append(opts, pg.WithDatabase(u.Path[1:])) - } - - return pg.NewClient(opts...) -} diff --git a/pkg/proboctl/seed/seed.go b/pkg/proboctl/seed/seed.go new file mode 100644 index 000000000..5c0f30c64 --- /dev/null +++ b/pkg/proboctl/seed/seed.go @@ -0,0 +1,34 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package seed + +import ( + "github.com/spf13/cobra" + "go.probo.inc/probo/pkg/proboctl/cmdutil" + commonthirdparties "go.probo.inc/probo/pkg/proboctl/seed/common-third-parties" + commontrackerpatterns "go.probo.inc/probo/pkg/proboctl/seed/common-tracker-patterns" +) + +func NewCmdSeed(f *cmdutil.Factory) *cobra.Command { + cmd := &cobra.Command{ + Use: "seed ", + Short: "Seed common reference data", + } + + cmd.AddCommand(commonthirdparties.NewCmdCommonThirdParties(f)) + cmd.AddCommand(commontrackerpatterns.NewCmdCommonTrackerPatterns(f)) + + return cmd +} diff --git a/pkg/proboctl/version/version.go b/pkg/proboctl/version/version.go new file mode 100644 index 000000000..0a8176ade --- /dev/null +++ b/pkg/proboctl/version/version.go @@ -0,0 +1,49 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package version + +import ( + "fmt" + + "github.com/spf13/cobra" + "go.probo.inc/probo/pkg/proboctl/cmdutil" + "go.probo.inc/probo/pkg/version" +) + +func NewCmdVersion(f *cmdutil.Factory) *cobra.Command { + return &cobra.Command{ + Use: "version", + Short: "Print the version of proboctl", + RunE: func(cmd *cobra.Command, args []string) error { + info := version.GetBuildInfo() + + v := f.Version + if v == "" || v == "unknown" { + v = info.Version + } + + _, _ = fmt.Fprintf( + f.IOStreams.Out, + "proboctl version %s (commit: %s, built: %s, go: %s)\n", + v, + info.Commit, + info.BuildDate, + info.GoVersion, + ) + + return nil + }, + } +}