Add proboctl CLI and move seed commands into it
Introduce a new proboctl Cobra CLI for Probo instance management that connects directly to PostgreSQL. Move the standalone common-third-parties-import and common-tracker-patterns-import commands into proboctl as `proboctl seed common-third-parties` and `proboctl seed common-tracker-patterns`, replacing flag-based PG connection with a shared factory pattern. Signed-off-by: Émile Ré <emile@probo.com>
This commit is contained in:
11
GNUmakefile
11
GNUmakefile
@@ -25,10 +25,12 @@ DOCKER_COMPOSE= $(DOCKER) compose -f compose.yaml $(DOCKER_COMPOSE_FLAGS)
|
|||||||
PRB_VERSION= $(shell cat cmd/prb/VERSION)
|
PRB_VERSION= $(shell cat cmd/prb/VERSION)
|
||||||
PROBOD_VERSION= $(shell cat cmd/probod/VERSION)
|
PROBOD_VERSION= $(shell cat cmd/probod/VERSION)
|
||||||
PROBOD_BOOTSTRAP_VERSION=$(shell cat cmd/probod-bootstrap/VERSION)
|
PROBOD_BOOTSTRAP_VERSION=$(shell cat cmd/probod-bootstrap/VERSION)
|
||||||
|
PROBOCTL_VERSION= $(shell cat cmd/proboctl/VERSION)
|
||||||
|
|
||||||
PRB_LDFLAGS= -ldflags "-X 'main.version=$(PRB_VERSION)'"
|
PRB_LDFLAGS= -ldflags "-X 'main.version=$(PRB_VERSION)'"
|
||||||
PROBOD_LDFLAGS= -ldflags "-X 'main.version=$(PROBOD_VERSION)' -X 'main.env=prod'"
|
PROBOD_LDFLAGS= -ldflags "-X 'main.version=$(PROBOD_VERSION)' -X 'main.env=prod'"
|
||||||
PROBOD_BOOTSTRAP_LDFLAGS=-ldflags "-X 'main.version=$(PROBOD_BOOTSTRAP_VERSION)'"
|
PROBOD_BOOTSTRAP_LDFLAGS=-ldflags "-X 'main.version=$(PROBOD_BOOTSTRAP_VERSION)'"
|
||||||
|
PROBOCTL_LDFLAGS= -ldflags "-X 'main.version=$(PROBOCTL_VERSION)'"
|
||||||
|
|
||||||
GCFLAGS= -gcflags="-e"
|
GCFLAGS= -gcflags="-e"
|
||||||
|
|
||||||
@@ -73,6 +75,9 @@ PRB_SRC= cmd/prb/main.go
|
|||||||
PROBOD_BOOTSTRAP_BIN= bin/probod-bootstrap
|
PROBOD_BOOTSTRAP_BIN= bin/probod-bootstrap
|
||||||
PROBOD_BOOTSTRAP_SRC= cmd/probod-bootstrap/main.go
|
PROBOD_BOOTSTRAP_SRC= cmd/probod-bootstrap/main.go
|
||||||
|
|
||||||
|
PROBOCTL_BIN= bin/proboctl
|
||||||
|
PROBOCTL_SRC= cmd/proboctl/main.go
|
||||||
|
|
||||||
ifdef WITH_APPS
|
ifdef WITH_APPS
|
||||||
GENERATED += relay
|
GENERATED += relay
|
||||||
EMBEDDED += \
|
EMBEDDED += \
|
||||||
@@ -172,7 +177,7 @@ coverage-combined: coverage-report test-e2e-coverage ## Generate combined covera
|
|||||||
$(GO) tool cover -html=coverage-combined.out -o=coverage-combined.html
|
$(GO) tool cover -html=coverage-combined.out -o=coverage-combined.html
|
||||||
|
|
||||||
.PHONY: build
|
.PHONY: build
|
||||||
build: $(PROBOD_BIN) bin/prb bin/probod-bootstrap
|
build: $(PROBOD_BIN) bin/prb bin/probod-bootstrap bin/proboctl
|
||||||
|
|
||||||
CFG_DEV_OAUTH2_KEY = cfg/.dev-oauth2-signing-key.pem
|
CFG_DEV_OAUTH2_KEY = cfg/.dev-oauth2-signing-key.pem
|
||||||
DEV_ENV = .env
|
DEV_ENV = .env
|
||||||
@@ -252,6 +257,10 @@ bin/prb:
|
|||||||
$(PROBOD_BOOTSTRAP_BIN):
|
$(PROBOD_BOOTSTRAP_BIN):
|
||||||
$(GO_BUILD) $(PROBOD_BOOTSTRAP_LDFLAGS) -o $(PROBOD_BOOTSTRAP_BIN) $(PROBOD_BOOTSTRAP_SRC)
|
$(GO_BUILD) $(PROBOD_BOOTSTRAP_LDFLAGS) -o $(PROBOD_BOOTSTRAP_BIN) $(PROBOD_BOOTSTRAP_SRC)
|
||||||
|
|
||||||
|
.PHONY: bin/proboctl
|
||||||
|
bin/proboctl:
|
||||||
|
$(GO_BUILD) $(PROBOCTL_LDFLAGS) -o $(PROBOCTL_BIN) $(PROBOCTL_SRC)
|
||||||
|
|
||||||
.PHONY: @probo/emails
|
.PHONY: @probo/emails
|
||||||
@probo/emails:
|
@probo/emails:
|
||||||
$(NPM) --workspace $@ run build
|
$(NPM) --workspace $@ run build
|
||||||
|
|||||||
1
cmd/proboctl/CHANGELOG.md
Normal file
1
cmd/proboctl/CHANGELOG.md
Normal file
@@ -0,0 +1 @@
|
|||||||
|
# Changelog
|
||||||
1
cmd/proboctl/VERSION
Normal file
1
cmd/proboctl/VERSION
Normal file
@@ -0,0 +1 @@
|
|||||||
|
0.0.0
|
||||||
42
cmd/proboctl/main.go
Normal file
42
cmd/proboctl/main.go
Normal file
@@ -0,0 +1,42 @@
|
|||||||
|
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||||
|
//
|
||||||
|
// Permission to use, copy, modify, and/or distribute this software for any
|
||||||
|
// purpose with or without fee is hereby granted, provided that the above
|
||||||
|
// copyright notice and this permission notice appear in all copies.
|
||||||
|
//
|
||||||
|
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||||
|
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||||
|
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||||
|
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||||
|
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||||
|
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||||
|
// PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"go.probo.inc/probo/pkg/cmd/iostreams"
|
||||||
|
"go.probo.inc/probo/pkg/proboctl/cmdutil"
|
||||||
|
"go.probo.inc/probo/pkg/proboctl/root"
|
||||||
|
)
|
||||||
|
|
||||||
|
var version string = "unknown"
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
ios := iostreams.System()
|
||||||
|
|
||||||
|
f := &cmdutil.Factory{
|
||||||
|
IOStreams: ios,
|
||||||
|
Version: version,
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd := root.NewCmdRoot(f)
|
||||||
|
|
||||||
|
if err := cmd.Execute(); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "Error: %s\n", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
36
pkg/proboctl/cmdutil/cmdutil.go
Normal file
36
pkg/proboctl/cmdutil/cmdutil.go
Normal file
@@ -0,0 +1,36 @@
|
|||||||
|
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||||
|
//
|
||||||
|
// Permission to use, copy, modify, and/or distribute this software for any
|
||||||
|
// purpose with or without fee is hereby granted, provided that the above
|
||||||
|
// copyright notice and this permission notice appear in all copies.
|
||||||
|
//
|
||||||
|
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||||
|
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||||
|
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||||
|
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||||
|
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||||
|
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||||
|
// PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
|
||||||
|
package cmdutil
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"go.gearno.de/kit/pg"
|
||||||
|
"go.probo.inc/probo/pkg/cmd/iostreams"
|
||||||
|
"go.probo.inc/probo/pkg/proboctl/pgconn"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Factory struct {
|
||||||
|
IOStreams *iostreams.IOStreams
|
||||||
|
Version string
|
||||||
|
PgDSN string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *Factory) PgClient() (*pg.Client, error) {
|
||||||
|
if f.PgDSN == "" {
|
||||||
|
return nil, fmt.Errorf("set --pg-dsn or DATABASE_URL")
|
||||||
|
}
|
||||||
|
return pgconn.NewPgClientFromDSN(f.PgDSN)
|
||||||
|
}
|
||||||
66
pkg/proboctl/pgconn/pgconn.go
Normal file
66
pkg/proboctl/pgconn/pgconn.go
Normal file
@@ -0,0 +1,66 @@
|
|||||||
|
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||||
|
//
|
||||||
|
// Permission to use, copy, modify, and/or distribute this software for any
|
||||||
|
// purpose with or without fee is hereby granted, provided that the above
|
||||||
|
// copyright notice and this permission notice appear in all copies.
|
||||||
|
//
|
||||||
|
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||||
|
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||||
|
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||||
|
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||||
|
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||||
|
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||||
|
// PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
|
||||||
|
package pgconn
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
|
"net/url"
|
||||||
|
|
||||||
|
"go.gearno.de/kit/pg"
|
||||||
|
)
|
||||||
|
|
||||||
|
func NewPgClientFromDSN(dsn string) (*pg.Client, error) {
|
||||||
|
u, err := url.Parse(dsn)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("cannot parse DSN (check URL format)")
|
||||||
|
}
|
||||||
|
|
||||||
|
var opts []pg.Option
|
||||||
|
|
||||||
|
switch u.Query().Get("sslmode") {
|
||||||
|
case "", "disable":
|
||||||
|
case "require":
|
||||||
|
opts = append(opts, pg.WithUnsecureTLS())
|
||||||
|
case "prefer":
|
||||||
|
return nil, fmt.Errorf(
|
||||||
|
"unsupported sslmode %q (prefer fallback semantics are not supported)",
|
||||||
|
u.Query().Get("sslmode"),
|
||||||
|
)
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("unsupported sslmode %q", u.Query().Get("sslmode"))
|
||||||
|
}
|
||||||
|
|
||||||
|
if u.Host != "" {
|
||||||
|
host := u.Host
|
||||||
|
if u.Port() == "" {
|
||||||
|
host = net.JoinHostPort(u.Hostname(), "5432")
|
||||||
|
}
|
||||||
|
opts = append(opts, pg.WithAddr(host))
|
||||||
|
}
|
||||||
|
|
||||||
|
if u.User != nil {
|
||||||
|
opts = append(opts, pg.WithUser(u.User.Username()))
|
||||||
|
if password, ok := u.User.Password(); ok {
|
||||||
|
opts = append(opts, pg.WithPassword(password))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(u.Path) > 1 {
|
||||||
|
opts = append(opts, pg.WithDatabase(u.Path[1:]))
|
||||||
|
}
|
||||||
|
|
||||||
|
return pg.NewClient(opts...)
|
||||||
|
}
|
||||||
45
pkg/proboctl/root/root.go
Normal file
45
pkg/proboctl/root/root.go
Normal file
@@ -0,0 +1,45 @@
|
|||||||
|
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||||
|
//
|
||||||
|
// Permission to use, copy, modify, and/or distribute this software for any
|
||||||
|
// purpose with or without fee is hereby granted, provided that the above
|
||||||
|
// copyright notice and this permission notice appear in all copies.
|
||||||
|
//
|
||||||
|
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||||
|
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||||
|
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||||
|
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||||
|
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||||
|
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||||
|
// PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
|
||||||
|
package root
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
"go.probo.inc/probo/pkg/proboctl/cmdutil"
|
||||||
|
"go.probo.inc/probo/pkg/proboctl/seed"
|
||||||
|
"go.probo.inc/probo/pkg/proboctl/version"
|
||||||
|
)
|
||||||
|
|
||||||
|
func NewCmdRoot(f *cmdutil.Factory) *cobra.Command {
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: "proboctl <command> [flags]",
|
||||||
|
Short: "Probo instance management CLI",
|
||||||
|
SilenceUsage: true,
|
||||||
|
SilenceErrors: true,
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd.PersistentFlags().StringVar(
|
||||||
|
&f.PgDSN,
|
||||||
|
"pg-dsn",
|
||||||
|
os.Getenv("DATABASE_URL"),
|
||||||
|
"PostgreSQL connection URL (default: DATABASE_URL env)",
|
||||||
|
)
|
||||||
|
|
||||||
|
cmd.AddCommand(seed.NewCmdSeed(f))
|
||||||
|
cmd.AddCommand(version.NewCmdVersion(f))
|
||||||
|
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
@@ -12,24 +12,14 @@
|
|||||||
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||||
// PERFORMANCE OF THIS SOFTWARE.
|
// PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
|
||||||
// Command common-third-parties-import seeds the common_third_parties table from
|
package commonthirdparties
|
||||||
// packages/thirdParties/data.json. It is idempotent: re-running upserts on conflict
|
|
||||||
// (slug) so existing rows keep their id and created_at.
|
|
||||||
//
|
|
||||||
// When -fetch-logos is set, the tool inspects each third party's website to
|
|
||||||
// find the best available logo (SVG icon, apple-touch-icon, etc.) and stores
|
|
||||||
// it in S3 as a public file, linking it to each common third party via
|
|
||||||
// logo_file_id.
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"flag"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
@@ -38,12 +28,14 @@ import (
|
|||||||
"github.com/aws/aws-sdk-go-v2/aws"
|
"github.com/aws/aws-sdk-go-v2/aws"
|
||||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||||
|
"github.com/spf13/cobra"
|
||||||
"go.gearno.de/crypto/uuid"
|
"go.gearno.de/crypto/uuid"
|
||||||
"go.gearno.de/kit/httpclient"
|
"go.gearno.de/kit/httpclient"
|
||||||
"go.gearno.de/kit/pg"
|
"go.gearno.de/kit/pg"
|
||||||
"go.probo.inc/probo/pkg/coredata"
|
"go.probo.inc/probo/pkg/coredata"
|
||||||
"go.probo.inc/probo/pkg/filemanager"
|
"go.probo.inc/probo/pkg/filemanager"
|
||||||
"go.probo.inc/probo/pkg/gid"
|
"go.probo.inc/probo/pkg/gid"
|
||||||
|
"go.probo.inc/probo/pkg/proboctl/cmdutil"
|
||||||
"go.probo.inc/probo/pkg/slug"
|
"go.probo.inc/probo/pkg/slug"
|
||||||
"go.probo.inc/probo/pkg/version"
|
"go.probo.inc/probo/pkg/version"
|
||||||
"go.probo.inc/probo/pkg/webinspect"
|
"go.probo.inc/probo/pkg/webinspect"
|
||||||
@@ -69,103 +61,45 @@ type thirdPartyData struct {
|
|||||||
Domains []string `json:"domains,omitempty"`
|
Domains []string `json:"domains,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func main() {
|
func NewCmdCommonThirdParties(f *cmdutil.Factory) *cobra.Command {
|
||||||
if err := run(); err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "error: %v\n", err)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func run() error {
|
|
||||||
var (
|
var (
|
||||||
pgDSN string
|
flagData string
|
||||||
dataPath string
|
flagFetchLogos bool
|
||||||
fetchLogos bool
|
flagS3Bucket string
|
||||||
s3Bucket string
|
flagS3Endpoint string
|
||||||
s3Endpoint string
|
flagS3Region string
|
||||||
s3Region string
|
flagS3AccessKey string
|
||||||
s3AccessKey string
|
flagS3SecretKey string
|
||||||
s3SecretKey string
|
flagS3UsePathStyle bool
|
||||||
s3UsePathStyle bool
|
|
||||||
)
|
)
|
||||||
|
|
||||||
flag.StringVar(
|
cmd := &cobra.Command{
|
||||||
&pgDSN,
|
Use: "common-third-parties",
|
||||||
"pg-dsn",
|
Short: "Seed common third parties from a data.json file",
|
||||||
os.Getenv("DATABASE_URL"),
|
Long: "Seed the common_third_parties table from a JSON file. " +
|
||||||
"PostgreSQL connection URL (default: DATABASE_URL env)",
|
"Re-running is safe: existing rows are upserted on conflict (slug) " +
|
||||||
)
|
"so ids and created_at are preserved.",
|
||||||
flag.StringVar(
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
&dataPath,
|
out := f.IOStreams.Out
|
||||||
"data",
|
errOut := f.IOStreams.ErrOut
|
||||||
"",
|
|
||||||
"Path to the third-party data.json file",
|
|
||||||
)
|
|
||||||
flag.BoolVar(
|
|
||||||
&fetchLogos,
|
|
||||||
"fetch-logos",
|
|
||||||
false,
|
|
||||||
"Fetch favicons from Google and store them in S3",
|
|
||||||
)
|
|
||||||
flag.StringVar(
|
|
||||||
&s3Bucket,
|
|
||||||
"s3-bucket",
|
|
||||||
os.Getenv("AWS_S3_BUCKET"),
|
|
||||||
"S3 bucket name (default: AWS_S3_BUCKET env)",
|
|
||||||
)
|
|
||||||
flag.StringVar(
|
|
||||||
&s3Endpoint,
|
|
||||||
"s3-endpoint",
|
|
||||||
os.Getenv("AWS_ENDPOINT_URL"),
|
|
||||||
"S3 endpoint URL (default: AWS_ENDPOINT_URL env)",
|
|
||||||
)
|
|
||||||
flag.StringVar(
|
|
||||||
&s3Region,
|
|
||||||
"s3-region",
|
|
||||||
os.Getenv("AWS_REGION"),
|
|
||||||
"S3 region (default: AWS_REGION env)",
|
|
||||||
)
|
|
||||||
flag.StringVar(
|
|
||||||
&s3AccessKey,
|
|
||||||
"s3-access-key",
|
|
||||||
os.Getenv("AWS_ACCESS_KEY_ID"),
|
|
||||||
"S3 access key ID (default: AWS_ACCESS_KEY_ID env)",
|
|
||||||
)
|
|
||||||
flag.StringVar(
|
|
||||||
&s3SecretKey,
|
|
||||||
"s3-secret-key",
|
|
||||||
os.Getenv("AWS_SECRET_ACCESS_KEY"),
|
|
||||||
"S3 secret access key (default: AWS_SECRET_ACCESS_KEY env)",
|
|
||||||
)
|
|
||||||
flag.BoolVar(
|
|
||||||
&s3UsePathStyle,
|
|
||||||
"s3-path-style",
|
|
||||||
false,
|
|
||||||
"Use S3 path-style addressing",
|
|
||||||
)
|
|
||||||
flag.Parse()
|
|
||||||
|
|
||||||
if pgDSN == "" {
|
if flagFetchLogos && flagS3Bucket == "" {
|
||||||
return fmt.Errorf("set -pg-dsn or DATABASE_URL")
|
return fmt.Errorf("set --s3-bucket or AWS_S3_BUCKET when using --fetch-logos")
|
||||||
}
|
}
|
||||||
|
|
||||||
if fetchLogos && s3Bucket == "" {
|
ctx := cmd.Context()
|
||||||
return fmt.Errorf("set -s3-bucket or AWS_S3_BUCKET when using -fetch-logos")
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx := context.Background()
|
thirdParties, err := loadThirdParties(flagData)
|
||||||
|
|
||||||
thirdParties, err := loadThirdParties(dataPath)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("cannot load third-party data: %w", err)
|
return fmt.Errorf("cannot load third-party data: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
pgClient, err := newPgClientFromDSN(pgDSN)
|
pgClient, err := f.PgClient()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("cannot create pg client: %w", err)
|
return fmt.Errorf("cannot create pg client: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("importing %d common third parties from %s\n", len(thirdParties), dataPath)
|
_, _ = fmt.Fprintf(out, "seeding %d common third parties from %s\n", len(thirdParties), flagData)
|
||||||
|
|
||||||
var inserted, updated, domainsInserted, domainsUpdated int
|
var inserted, updated, domainsInserted, domainsUpdated int
|
||||||
|
|
||||||
@@ -179,7 +113,7 @@ func run() error {
|
|||||||
ID: gid.New(gid.NilTenant, coredata.CommonThirdPartyEntityType),
|
ID: gid.New(gid.NilTenant, coredata.CommonThirdPartyEntityType),
|
||||||
Name: tp.Name,
|
Name: tp.Name,
|
||||||
Slug: slug.Make(tp.Name),
|
Slug: slug.Make(tp.Name),
|
||||||
Category: parseCategory(tp),
|
Category: parseCategory(errOut, tp),
|
||||||
HeadquarterAddress: tp.HeadquarterAddress,
|
HeadquarterAddress: tp.HeadquarterAddress,
|
||||||
LegalName: tp.LegalName,
|
LegalName: tp.LegalName,
|
||||||
WebsiteURL: tp.WebsiteURL,
|
WebsiteURL: tp.WebsiteURL,
|
||||||
@@ -240,20 +174,38 @@ func run() error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("imported %d third parties (%d inserted, %d updated)\n", len(thirdParties), inserted, updated)
|
_, _ = fmt.Fprintf(out, "seeded %d third parties (%d inserted, %d updated)\n", len(thirdParties), inserted, updated)
|
||||||
fmt.Printf("imported %d domains (%d inserted, %d updated)\n", domainsInserted+domainsUpdated, domainsInserted, domainsUpdated)
|
_, _ = fmt.Fprintf(out, "seeded %d domains (%d inserted, %d updated)\n", domainsInserted+domainsUpdated, domainsInserted, domainsUpdated)
|
||||||
|
|
||||||
if fetchLogos {
|
if flagFetchLogos {
|
||||||
if err := fetchAndStoreLogos(ctx, pgClient, thirdParties, s3Bucket, s3Endpoint, s3Region, s3AccessKey, s3SecretKey, s3UsePathStyle); err != nil {
|
if err := fetchAndStoreLogos(
|
||||||
|
ctx, out, errOut, pgClient, thirdParties,
|
||||||
|
flagS3Bucket, flagS3Endpoint, flagS3Region, flagS3AccessKey, flagS3SecretKey, flagS3UsePathStyle,
|
||||||
|
); err != nil {
|
||||||
return fmt.Errorf("cannot fetch logos: %w", err)
|
return fmt.Errorf("cannot fetch logos: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd.Flags().StringVar(&flagData, "data", "", "Path to the third-party data.json file")
|
||||||
|
_ = cmd.MarkFlagRequired("data")
|
||||||
|
cmd.Flags().BoolVar(&flagFetchLogos, "fetch-logos", false, "Fetch favicons and store them in S3")
|
||||||
|
cmd.Flags().StringVar(&flagS3Bucket, "s3-bucket", os.Getenv("AWS_S3_BUCKET"), "S3 bucket name (default: AWS_S3_BUCKET env)")
|
||||||
|
cmd.Flags().StringVar(&flagS3Endpoint, "s3-endpoint", os.Getenv("AWS_ENDPOINT_URL"), "S3 endpoint URL (default: AWS_ENDPOINT_URL env)")
|
||||||
|
cmd.Flags().StringVar(&flagS3Region, "s3-region", os.Getenv("AWS_REGION"), "S3 region (default: AWS_REGION env)")
|
||||||
|
cmd.Flags().StringVar(&flagS3AccessKey, "s3-access-key", os.Getenv("AWS_ACCESS_KEY_ID"), "S3 access key ID (default: AWS_ACCESS_KEY_ID env)")
|
||||||
|
cmd.Flags().StringVar(&flagS3SecretKey, "s3-secret-key", os.Getenv("AWS_SECRET_ACCESS_KEY"), "S3 secret access key (default: AWS_SECRET_ACCESS_KEY env)")
|
||||||
|
cmd.Flags().BoolVar(&flagS3UsePathStyle, "s3-path-style", false, "Use S3 path-style addressing")
|
||||||
|
|
||||||
|
return cmd
|
||||||
}
|
}
|
||||||
|
|
||||||
func fetchAndStoreLogos(
|
func fetchAndStoreLogos(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
|
out, errOut io.Writer,
|
||||||
pgClient *pg.Client,
|
pgClient *pg.Client,
|
||||||
thirdParties []thirdPartyData,
|
thirdParties []thirdPartyData,
|
||||||
bucket, endpoint, region, accessKey, secretKey string,
|
bucket, endpoint, region, accessKey, secretKey string,
|
||||||
@@ -264,7 +216,7 @@ func fetchAndStoreLogos(
|
|||||||
httpClient := httpclient.DefaultPooledClient(httpclient.WithSSRFProtection())
|
httpClient := httpclient.DefaultPooledClient(httpclient.WithSSRFProtection())
|
||||||
httpClient.Transport = &userAgentTransport{
|
httpClient.Transport = &userAgentTransport{
|
||||||
next: httpClient.Transport,
|
next: httpClient.Transport,
|
||||||
ua: version.UserAgent("common-third-parties-import"),
|
ua: version.UserAgent("proboctl"),
|
||||||
}
|
}
|
||||||
scope := coredata.NewScope(gid.NilTenant)
|
scope := coredata.NewScope(gid.NilTenant)
|
||||||
|
|
||||||
@@ -280,7 +232,7 @@ func fetchAndStoreLogos(
|
|||||||
if err := pgClient.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error {
|
if err := pgClient.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error {
|
||||||
return party.LoadByName(ctx, conn, tp.Name)
|
return party.LoadByName(ctx, conn, tp.Name)
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
fmt.Fprintf(os.Stderr, "warning: cannot load %q, skipping logo: %v\n", tp.Name, err)
|
_, _ = fmt.Fprintf(errOut, "warning: cannot load %q, skipping logo: %v\n", tp.Name, err)
|
||||||
failed++
|
failed++
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -293,17 +245,17 @@ func fetchAndStoreLogos(
|
|||||||
var logoURL string
|
var logoURL string
|
||||||
pageInfo, err := webinspect.Parse(ctx, httpClient, *tp.WebsiteURL)
|
pageInfo, err := webinspect.Parse(ctx, httpClient, *tp.WebsiteURL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintf(os.Stderr, "warning: cannot inspect page for %q, trying default apple-touch-icon: %v\n", tp.Name, err)
|
_, _ = fmt.Fprintf(errOut, "warning: cannot inspect page for %q, trying default apple-touch-icon: %v\n", tp.Name, err)
|
||||||
} else {
|
} else {
|
||||||
logoURL, err = webinspect.FindLogoURL(pageInfo)
|
logoURL, err = webinspect.FindLogoURL(pageInfo)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintf(os.Stderr, "warning: cannot find logo for %q, trying default apple-touch-icon: %v\n", tp.Name, err)
|
_, _ = fmt.Fprintf(errOut, "warning: cannot find logo for %q, trying default apple-touch-icon: %v\n", tp.Name, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
parsed, err := url.Parse(*tp.WebsiteURL)
|
parsed, err := url.Parse(*tp.WebsiteURL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintf(os.Stderr, "warning: cannot parse URL for %q, skipping logo: %v\n", tp.Name, err)
|
_, _ = fmt.Fprintf(errOut, "warning: cannot parse URL for %q, skipping logo: %v\n", tp.Name, err)
|
||||||
failed++
|
failed++
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -313,7 +265,8 @@ func fetchAndStoreLogos(
|
|||||||
candidateURLs = append(candidateURLs, logoURL)
|
candidateURLs = append(candidateURLs, logoURL)
|
||||||
}
|
}
|
||||||
base := fmt.Sprintf("%s://%s", parsed.Scheme, parsed.Host)
|
base := fmt.Sprintf("%s://%s", parsed.Scheme, parsed.Host)
|
||||||
candidateURLs = append(candidateURLs,
|
candidateURLs = append(
|
||||||
|
candidateURLs,
|
||||||
base+"/apple-touch-icon.png",
|
base+"/apple-touch-icon.png",
|
||||||
base+"/apple-touch-icon-precomposed.png",
|
base+"/apple-touch-icon-precomposed.png",
|
||||||
"https://logo.debounce.com/"+parsed.Host,
|
"https://logo.debounce.com/"+parsed.Host,
|
||||||
@@ -342,7 +295,7 @@ func fetchAndStoreLogos(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if len(body) == 0 {
|
if len(body) == 0 {
|
||||||
fmt.Fprintf(os.Stderr, "warning: cannot fetch logo for %q from any candidate URL\n", tp.Name)
|
_, _ = fmt.Fprintf(errOut, "warning: cannot fetch logo for %q from any candidate URL\n", tp.Name)
|
||||||
failed++
|
failed++
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -376,7 +329,7 @@ func fetchAndStoreLogos(
|
|||||||
"type": "common-third-party-logo",
|
"type": "common-third-party-logo",
|
||||||
"common-third-party-id": party.ID.String(),
|
"common-third-party-id": party.ID.String(),
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
fmt.Fprintf(os.Stderr, "warning: cannot upload logo for %q to S3: %v\n", tp.Name, err)
|
_, _ = fmt.Fprintf(errOut, "warning: cannot upload logo for %q to S3: %v\n", tp.Name, err)
|
||||||
failed++
|
failed++
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -394,16 +347,16 @@ func fetchAndStoreLogos(
|
|||||||
|
|
||||||
return nil
|
return nil
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
fmt.Fprintf(os.Stderr, "warning: cannot store logo for %q: %v\n", tp.Name, err)
|
_, _ = fmt.Fprintf(errOut, "warning: cannot store logo for %q: %v\n", tp.Name, err)
|
||||||
failed++
|
failed++
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
fetched++
|
fetched++
|
||||||
fmt.Printf(" fetched logo for %q\n", tp.Name)
|
_, _ = fmt.Fprintf(out, " fetched logo for %q\n", tp.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("logos: %d fetched, %d skipped, %d failed\n", fetched, skipped, failed)
|
_, _ = fmt.Fprintf(out, "logos: %d fetched, %d skipped, %d failed\n", fetched, skipped, failed)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -447,61 +400,20 @@ func loadThirdParties(path string) ([]thirdPartyData, error) {
|
|||||||
return thirdParties, nil
|
return thirdParties, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func parseCategory(tp thirdPartyData) coredata.ThirdPartyCategory {
|
func parseCategory(errOut io.Writer, tp thirdPartyData) coredata.ThirdPartyCategory {
|
||||||
if tp.Category == nil || *tp.Category == "" {
|
if tp.Category == nil || *tp.Category == "" {
|
||||||
return coredata.ThirdPartyCategoryOther
|
return coredata.ThirdPartyCategoryOther
|
||||||
}
|
}
|
||||||
|
|
||||||
var c coredata.ThirdPartyCategory
|
var c coredata.ThirdPartyCategory
|
||||||
if err := c.Scan(*tp.Category); err != nil {
|
if err := c.Scan(*tp.Category); err != nil {
|
||||||
fmt.Fprintf(os.Stderr, "warning: third party %q has unknown category %q, falling back to OTHER\n", tp.Name, *tp.Category)
|
_, _ = fmt.Fprintf(errOut, "warning: third party %q has unknown category %q, falling back to OTHER\n", tp.Name, *tp.Category)
|
||||||
return coredata.ThirdPartyCategoryOther
|
return coredata.ThirdPartyCategoryOther
|
||||||
}
|
}
|
||||||
|
|
||||||
return c
|
return c
|
||||||
}
|
}
|
||||||
|
|
||||||
func newPgClientFromDSN(dsn string) (*pg.Client, error) {
|
|
||||||
u, err := url.Parse(dsn)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("cannot parse DSN (check URL format)")
|
|
||||||
}
|
|
||||||
|
|
||||||
var opts []pg.Option
|
|
||||||
|
|
||||||
switch u.Query().Get("sslmode") {
|
|
||||||
case "", "disable":
|
|
||||||
// plain connection, no TLS
|
|
||||||
case "require":
|
|
||||||
opts = append(opts, pg.WithUnsecureTLS())
|
|
||||||
case "prefer":
|
|
||||||
return nil, fmt.Errorf("unsupported sslmode %q (prefer fallback semantics are not supported)", u.Query().Get("sslmode"))
|
|
||||||
default:
|
|
||||||
return nil, fmt.Errorf("unsupported sslmode %q", u.Query().Get("sslmode"))
|
|
||||||
}
|
|
||||||
|
|
||||||
if u.Host != "" {
|
|
||||||
host := u.Host
|
|
||||||
if u.Port() == "" {
|
|
||||||
host = net.JoinHostPort(u.Hostname(), "5432")
|
|
||||||
}
|
|
||||||
opts = append(opts, pg.WithAddr(host))
|
|
||||||
}
|
|
||||||
|
|
||||||
if u.User != nil {
|
|
||||||
opts = append(opts, pg.WithUser(u.User.Username()))
|
|
||||||
if password, ok := u.User.Password(); ok {
|
|
||||||
opts = append(opts, pg.WithPassword(password))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(u.Path) > 1 {
|
|
||||||
opts = append(opts, pg.WithDatabase(u.Path[1:]))
|
|
||||||
}
|
|
||||||
|
|
||||||
return pg.NewClient(opts...)
|
|
||||||
}
|
|
||||||
|
|
||||||
type userAgentTransport struct {
|
type userAgentTransport struct {
|
||||||
next http.RoundTripper
|
next http.RoundTripper
|
||||||
ua string
|
ua string
|
||||||
@@ -12,26 +12,14 @@
|
|||||||
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||||
// PERFORMANCE OF THIS SOFTWARE.
|
// PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
|
||||||
// Command common-tracker-patterns-import seeds the common_tracker_patterns
|
package commontrackerpatterns
|
||||||
// table from the Open Cookie Database (https://github.com/jkwakman/Open-Cookie-Database).
|
|
||||||
// It clones the repository into a temporary directory, reads
|
|
||||||
// open-cookie-database.json, and upserts each entry. Re-running is safe:
|
|
||||||
// existing rows are updated on the unique constraint so ids and created_at
|
|
||||||
// are preserved.
|
|
||||||
//
|
|
||||||
// Entries are linked to the matching common_third_parties row via a
|
|
||||||
// three-step cascade: slug lookup, domain lookup, then auto-create.
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"flag"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"math"
|
"math"
|
||||||
"net"
|
|
||||||
"net/url"
|
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"regexp"
|
"regexp"
|
||||||
@@ -41,9 +29,11 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
git "github.com/go-git/go-git/v5"
|
git "github.com/go-git/go-git/v5"
|
||||||
|
"github.com/spf13/cobra"
|
||||||
"go.gearno.de/kit/pg"
|
"go.gearno.de/kit/pg"
|
||||||
"go.probo.inc/probo/pkg/coredata"
|
"go.probo.inc/probo/pkg/coredata"
|
||||||
"go.probo.inc/probo/pkg/gid"
|
"go.probo.inc/probo/pkg/gid"
|
||||||
|
"go.probo.inc/probo/pkg/proboctl/cmdutil"
|
||||||
"go.probo.inc/probo/pkg/slug"
|
"go.probo.inc/probo/pkg/slug"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -78,31 +68,22 @@ type (
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
func main() {
|
func NewCmdCommonTrackerPatterns(f *cmdutil.Factory) *cobra.Command {
|
||||||
if err := run(); err != nil {
|
return &cobra.Command{
|
||||||
fmt.Fprintf(os.Stderr, "error: %v\n", err)
|
Use: "common-tracker-patterns",
|
||||||
os.Exit(1)
|
Short: "Seed common tracker patterns from the Open Cookie Database",
|
||||||
}
|
Long: "Seed the common_tracker_patterns table from the Open Cookie Database " +
|
||||||
}
|
"(https://github.com/jkwakman/Open-Cookie-Database). " +
|
||||||
|
"The repository is cloned into a temporary directory. " +
|
||||||
|
"Re-running is safe: existing rows are upserted so ids and created_at are preserved. " +
|
||||||
|
"Entries are linked to matching common_third_parties rows via slug lookup, " +
|
||||||
|
"domain lookup, then auto-create.",
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
out := f.IOStreams.Out
|
||||||
|
errOut := f.IOStreams.ErrOut
|
||||||
|
ctx := cmd.Context()
|
||||||
|
|
||||||
func run() error {
|
_, _ = fmt.Fprintf(out, "cloning %s\n", ocdRepoURL)
|
||||||
var pgDSN string
|
|
||||||
|
|
||||||
flag.StringVar(
|
|
||||||
&pgDSN,
|
|
||||||
"pg-dsn",
|
|
||||||
os.Getenv("DATABASE_URL"),
|
|
||||||
"PostgreSQL connection URL (default: DATABASE_URL env)",
|
|
||||||
)
|
|
||||||
flag.Parse()
|
|
||||||
|
|
||||||
if pgDSN == "" {
|
|
||||||
return fmt.Errorf("set -pg-dsn or DATABASE_URL")
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
fmt.Printf("cloning %s\n", ocdRepoURL)
|
|
||||||
|
|
||||||
tmpDir, cleanup, err := cloneRepo()
|
tmpDir, cleanup, err := cloneRepo()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -115,12 +96,12 @@ func run() error {
|
|||||||
return fmt.Errorf("cannot load tracker pattern data: %w", err)
|
return fmt.Errorf("cannot load tracker pattern data: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
pgClient, err := newPgClientFromDSN(pgDSN)
|
pgClient, err := f.PgClient()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("cannot create pg client: %w", err)
|
return fmt.Errorf("cannot create pg client: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("importing %d common tracker patterns from Open Cookie Database\n", len(patterns))
|
_, _ = fmt.Fprintf(out, "seeding %d common tracker patterns from Open Cookie Database\n", len(patterns))
|
||||||
|
|
||||||
var inserted, updated, skipped int
|
var inserted, updated, skipped int
|
||||||
var partiesCreated int
|
var partiesCreated int
|
||||||
@@ -139,14 +120,14 @@ func run() error {
|
|||||||
|
|
||||||
trackerType, err := parseTrackerType(p.TrackerType)
|
trackerType, err := parseTrackerType(p.TrackerType)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintf(os.Stderr, "warning: %v, skipping pattern %q\n", err, p.Pattern)
|
_, _ = fmt.Fprintf(errOut, "warning: %v, skipping pattern %q\n", err, p.Pattern)
|
||||||
skipped++
|
skipped++
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
matchType, err := parseMatchType(p.MatchType)
|
matchType, err := parseMatchType(p.MatchType)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintf(os.Stderr, "warning: %v, skipping pattern %q\n", err, p.Pattern)
|
_, _ = fmt.Fprintf(errOut, "warning: %v, skipping pattern %q\n", err, p.Pattern)
|
||||||
skipped++
|
skipped++
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -182,8 +163,9 @@ func run() error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf(
|
_, _ = fmt.Fprintf(
|
||||||
"imported %d patterns (%d inserted, %d updated, %d skipped, %d third parties auto-created)\n",
|
out,
|
||||||
|
"seeded %d patterns (%d inserted, %d updated, %d skipped, %d third parties auto-created)\n",
|
||||||
len(patterns)-skipped,
|
len(patterns)-skipped,
|
||||||
inserted,
|
inserted,
|
||||||
updated,
|
updated,
|
||||||
@@ -192,6 +174,8 @@ func run() error {
|
|||||||
)
|
)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func cloneRepo() (string, func(), error) {
|
func cloneRepo() (string, func(), error) {
|
||||||
@@ -360,7 +344,6 @@ var domainValidRe = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9.-]*[a-zA-Z]$`)
|
|||||||
func normalizeDomain(s string) string {
|
func normalizeDomain(s string) string {
|
||||||
s = strings.TrimSpace(s)
|
s = strings.TrimSpace(s)
|
||||||
|
|
||||||
// Strip zero-width spaces and other Unicode control characters.
|
|
||||||
s = strings.Map(func(r rune) rune {
|
s = strings.Map(func(r rune) rune {
|
||||||
if r == '\u200b' || r == '\ufeff' {
|
if r == '\u200b' || r == '\ufeff' {
|
||||||
return -1
|
return -1
|
||||||
@@ -368,19 +351,15 @@ func normalizeDomain(s string) string {
|
|||||||
return r
|
return r
|
||||||
}, s)
|
}, s)
|
||||||
|
|
||||||
// Take only the first domain when "or" separates multiple values,
|
|
||||||
// e.g. ".bing.com (3rd party) or .microsoft.com (3rd party)".
|
|
||||||
if idx := strings.Index(s, " or "); idx != -1 {
|
if idx := strings.Index(s, " or "); idx != -1 {
|
||||||
s = s[:idx]
|
s = s[:idx]
|
||||||
}
|
}
|
||||||
// Handle values starting with "or ", e.g. "or demdex.net (3rd party)".
|
|
||||||
s = strings.TrimPrefix(s, "or ")
|
s = strings.TrimPrefix(s, "or ")
|
||||||
|
|
||||||
if idx := strings.IndexByte(s, '('); idx != -1 {
|
if idx := strings.IndexByte(s, '('); idx != -1 {
|
||||||
s = strings.TrimSpace(s[:idx])
|
s = strings.TrimSpace(s[:idx])
|
||||||
}
|
}
|
||||||
|
|
||||||
// Strip placeholders like "[account]".
|
|
||||||
if strings.ContainsAny(s, "[]") {
|
if strings.ContainsAny(s, "[]") {
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
@@ -487,43 +466,3 @@ func parseMatchType(s string) (coredata.TrackerPatternMatchType, error) {
|
|||||||
return "", fmt.Errorf("unknown match type %q", s)
|
return "", fmt.Errorf("unknown match type %q", s)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func newPgClientFromDSN(dsn string) (*pg.Client, error) {
|
|
||||||
u, err := url.Parse(dsn)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("cannot parse DSN (check URL format)")
|
|
||||||
}
|
|
||||||
|
|
||||||
var opts []pg.Option
|
|
||||||
|
|
||||||
switch u.Query().Get("sslmode") {
|
|
||||||
case "", "disable":
|
|
||||||
case "require":
|
|
||||||
opts = append(opts, pg.WithUnsecureTLS())
|
|
||||||
case "prefer":
|
|
||||||
return nil, fmt.Errorf("unsupported sslmode %q (prefer fallback semantics are not supported)", u.Query().Get("sslmode"))
|
|
||||||
default:
|
|
||||||
return nil, fmt.Errorf("unsupported sslmode %q", u.Query().Get("sslmode"))
|
|
||||||
}
|
|
||||||
|
|
||||||
if u.Host != "" {
|
|
||||||
host := u.Host
|
|
||||||
if u.Port() == "" {
|
|
||||||
host = net.JoinHostPort(u.Hostname(), "5432")
|
|
||||||
}
|
|
||||||
opts = append(opts, pg.WithAddr(host))
|
|
||||||
}
|
|
||||||
|
|
||||||
if u.User != nil {
|
|
||||||
opts = append(opts, pg.WithUser(u.User.Username()))
|
|
||||||
if password, ok := u.User.Password(); ok {
|
|
||||||
opts = append(opts, pg.WithPassword(password))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(u.Path) > 1 {
|
|
||||||
opts = append(opts, pg.WithDatabase(u.Path[1:]))
|
|
||||||
}
|
|
||||||
|
|
||||||
return pg.NewClient(opts...)
|
|
||||||
}
|
|
||||||
34
pkg/proboctl/seed/seed.go
Normal file
34
pkg/proboctl/seed/seed.go
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||||
|
//
|
||||||
|
// Permission to use, copy, modify, and/or distribute this software for any
|
||||||
|
// purpose with or without fee is hereby granted, provided that the above
|
||||||
|
// copyright notice and this permission notice appear in all copies.
|
||||||
|
//
|
||||||
|
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||||
|
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||||
|
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||||
|
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||||
|
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||||
|
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||||
|
// PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
|
||||||
|
package seed
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
"go.probo.inc/probo/pkg/proboctl/cmdutil"
|
||||||
|
commonthirdparties "go.probo.inc/probo/pkg/proboctl/seed/common-third-parties"
|
||||||
|
commontrackerpatterns "go.probo.inc/probo/pkg/proboctl/seed/common-tracker-patterns"
|
||||||
|
)
|
||||||
|
|
||||||
|
func NewCmdSeed(f *cmdutil.Factory) *cobra.Command {
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: "seed <command>",
|
||||||
|
Short: "Seed common reference data",
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd.AddCommand(commonthirdparties.NewCmdCommonThirdParties(f))
|
||||||
|
cmd.AddCommand(commontrackerpatterns.NewCmdCommonTrackerPatterns(f))
|
||||||
|
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
49
pkg/proboctl/version/version.go
Normal file
49
pkg/proboctl/version/version.go
Normal file
@@ -0,0 +1,49 @@
|
|||||||
|
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||||
|
//
|
||||||
|
// Permission to use, copy, modify, and/or distribute this software for any
|
||||||
|
// purpose with or without fee is hereby granted, provided that the above
|
||||||
|
// copyright notice and this permission notice appear in all copies.
|
||||||
|
//
|
||||||
|
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||||
|
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||||
|
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||||
|
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||||
|
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||||
|
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||||
|
// PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
|
||||||
|
package version
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
"go.probo.inc/probo/pkg/proboctl/cmdutil"
|
||||||
|
"go.probo.inc/probo/pkg/version"
|
||||||
|
)
|
||||||
|
|
||||||
|
func NewCmdVersion(f *cmdutil.Factory) *cobra.Command {
|
||||||
|
return &cobra.Command{
|
||||||
|
Use: "version",
|
||||||
|
Short: "Print the version of proboctl",
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
info := version.GetBuildInfo()
|
||||||
|
|
||||||
|
v := f.Version
|
||||||
|
if v == "" || v == "unknown" {
|
||||||
|
v = info.Version
|
||||||
|
}
|
||||||
|
|
||||||
|
_, _ = fmt.Fprintf(
|
||||||
|
f.IOStreams.Out,
|
||||||
|
"proboctl version %s (commit: %s, built: %s, go: %s)\n",
|
||||||
|
v,
|
||||||
|
info.Commit,
|
||||||
|
info.BuildDate,
|
||||||
|
info.GoVersion,
|
||||||
|
)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user