rohit/probo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add proboctl CLI and move seed commands into it

Browse Source 
Introduce a new proboctl Cobra CLI for Probo instance management
that connects directly to PostgreSQL. Move the standalone
common-third-parties-import and common-tracker-patterns-import
commands into proboctl as `proboctl seed common-third-parties`
and `proboctl seed common-tracker-patterns`, replacing flag-based
PG connection with a shared factory pattern.

Signed-off-by: Émile Ré <emile@probo.com>
This commit is contained in:
Émile Ré
2026-05-14 18:37:42 +04:00
parent 844817543d
commit 48a494461e
11 changed files with 520 additions and 386 deletions
Download Patch File Download Diff File Expand all files Collapse all files

516
cmd/common-third-parties-import/main.go
View File

@@ -1,516 +0,0 @@
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
// Command common-third-parties-import seeds the common_third_parties table from
// packages/thirdParties/data.json. It is idempotent: re-running upserts on conflict
// (slug) so existing rows keep their id and created_at.
//
// When -fetch-logos is set, the tool inspects each third party's website to
// find the best available logo (SVG icon, apple-touch-icon, etc.) and stores
// it in S3 as a public file, linking it to each common third party via
// logo_file_id.
package main
import (
"bytes"
"context"
"encoding/json"
"flag"
"fmt"
"io"
"net"
"net/http"
"net/url"
"os"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
"go.gearno.de/crypto/uuid"
"go.gearno.de/kit/httpclient"
"go.gearno.de/kit/pg"
"go.probo.inc/probo/pkg/coredata"
"go.probo.inc/probo/pkg/filemanager"
"go.probo.inc/probo/pkg/gid"
"go.probo.inc/probo/pkg/slug"
"go.probo.inc/probo/pkg/version"
"go.probo.inc/probo/pkg/webinspect"
)
type thirdPartyData struct {
Name string `json:"name"`
Category *string `json:"category,omitempty"`
HeadquarterAddress *string `json:"headquarterAddress,omitempty"`
LegalName *string `json:"legalName,omitempty"`
WebsiteURL *string `json:"websiteUrl,omitempty"`
PrivacyPolicyURL *string `json:"privacyPolicyUrl,omitempty"`
ServiceLevelAgreementURL *string `json:"serviceLevelAgreementUrl,omitempty"`
ServiceSoftwareAgreementURL *string `json:"serviceSoftwareAgreementUrl,omitempty"`
DataProcessingAgreementURL *string `json:"dataProcessingAgreementUrl,omitempty"`
BusinessAssociateAgreementURL *string `json:"businessAssociateAgreementUrl,omitempty"`
SubprocessorsListURL *string `json:"subprocessorsListUrl,omitempty"`
Certifications []string `json:"certifications,omitempty"`
StatusPageURL *string `json:"statusPageUrl,omitempty"`
TermsOfServiceURL *string `json:"termsOfServiceUrl,omitempty"`
SecurityPageURL *string `json:"securityPageUrl,omitempty"`
TrustPageURL *string `json:"trustPageUrl,omitempty"`
Domains []string `json:"domains,omitempty"`
}
func main() {
if err := run(); err != nil {
fmt.Fprintf(os.Stderr, "error: %v\n", err)
os.Exit(1)
}
}
func run() error {
var (
pgDSN string
dataPath string
fetchLogos bool
s3Bucket string
s3Endpoint string
s3Region string
s3AccessKey string
s3SecretKey string
s3UsePathStyle bool
)
flag.StringVar(
&pgDSN,
"pg-dsn",
os.Getenv("DATABASE_URL"),
"PostgreSQL connection URL (default: DATABASE_URL env)",
)
flag.StringVar(
&dataPath,
"data",
"",
"Path to the third-party data.json file",
)
flag.BoolVar(
&fetchLogos,
"fetch-logos",
false,
"Fetch favicons from Google and store them in S3",
)
flag.StringVar(
&s3Bucket,
"s3-bucket",
os.Getenv("AWS_S3_BUCKET"),
"S3 bucket name (default: AWS_S3_BUCKET env)",
)
flag.StringVar(
&s3Endpoint,
"s3-endpoint",
os.Getenv("AWS_ENDPOINT_URL"),
"S3 endpoint URL (default: AWS_ENDPOINT_URL env)",
)
flag.StringVar(
&s3Region,
"s3-region",
os.Getenv("AWS_REGION"),
"S3 region (default: AWS_REGION env)",
)
flag.StringVar(
&s3AccessKey,
"s3-access-key",
os.Getenv("AWS_ACCESS_KEY_ID"),
"S3 access key ID (default: AWS_ACCESS_KEY_ID env)",
)
flag.StringVar(
&s3SecretKey,
"s3-secret-key",
os.Getenv("AWS_SECRET_ACCESS_KEY"),
"S3 secret access key (default: AWS_SECRET_ACCESS_KEY env)",
)
flag.BoolVar(
&s3UsePathStyle,
"s3-path-style",
false,
"Use S3 path-style addressing",
)
flag.Parse()
if pgDSN == "" {
return fmt.Errorf("set -pg-dsn or DATABASE_URL")
}
if fetchLogos && s3Bucket == "" {
return fmt.Errorf("set -s3-bucket or AWS_S3_BUCKET when using -fetch-logos")
}
ctx := context.Background()
thirdParties, err := loadThirdParties(dataPath)
if err != nil {
return fmt.Errorf("cannot load third-party data: %w", err)
}
pgClient, err := newPgClientFromDSN(pgDSN)
if err != nil {
return fmt.Errorf("cannot create pg client: %w", err)
}
fmt.Printf("importing %d common third parties from %s\n", len(thirdParties), dataPath)
var inserted, updated, domainsInserted, domainsUpdated int
if err := pgClient.WithTx(
ctx,
func(ctx context.Context, tx pg.Tx) error {
now := time.Now()
for _, tp := range thirdParties {
party := coredata.CommonThirdParty{
ID: gid.New(gid.NilTenant, coredata.CommonThirdPartyEntityType),
Name: tp.Name,
Slug: slug.Make(tp.Name),
Category: parseCategory(tp),
HeadquarterAddress: tp.HeadquarterAddress,
LegalName: tp.LegalName,
WebsiteURL: tp.WebsiteURL,
PrivacyPolicyURL: tp.PrivacyPolicyURL,
ServiceLevelAgreementURL: tp.ServiceLevelAgreementURL,
ServiceSoftwareAgreementURL: tp.ServiceSoftwareAgreementURL,
DataProcessingAgreementURL: tp.DataProcessingAgreementURL,
BusinessAssociateAgreementURL: tp.BusinessAssociateAgreementURL,
SubprocessorsListURL: tp.SubprocessorsListURL,
Certifications: tp.Certifications,
StatusPageURL: tp.StatusPageURL,
TermsOfServiceURL: tp.TermsOfServiceURL,
SecurityPageURL: tp.SecurityPageURL,
TrustPageURL: tp.TrustPageURL,
CreatedAt: now,
UpdatedAt: now,
}
wasInserted, err := party.Upsert(ctx, tx)
if err != nil {
return fmt.Errorf("cannot upsert common third party %q: %w", tp.Name, err)
}
if wasInserted {
inserted++
} else {
updated++
if err := party.LoadByName(ctx, tx, tp.Name); err != nil {
return fmt.Errorf("cannot reload common third party %q: %w", tp.Name, err)
}
}
for _, domain := range tp.Domains {
d := coredata.CommonThirdPartyDomain{
ID: gid.New(gid.NilTenant, coredata.CommonThirdPartyDomainEntityType),
CommonThirdPartyID: party.ID,
Domain: domain,
CreatedAt: now,
UpdatedAt: now,
}
domainInserted, err := d.Upsert(ctx, tx)
if err != nil {
return fmt.Errorf("cannot upsert domain %q for %q: %w", domain, tp.Name, err)
}
if domainInserted {
domainsInserted++
} else {
domainsUpdated++
}
}
}
return nil
},
); err != nil {
return err
}
fmt.Printf("imported %d third parties (%d inserted, %d updated)\n", len(thirdParties), inserted, updated)
fmt.Printf("imported %d domains (%d inserted, %d updated)\n", domainsInserted+domainsUpdated, domainsInserted, domainsUpdated)
if fetchLogos {
if err := fetchAndStoreLogos(ctx, pgClient, thirdParties, s3Bucket, s3Endpoint, s3Region, s3AccessKey, s3SecretKey, s3UsePathStyle); err != nil {
return fmt.Errorf("cannot fetch logos: %w", err)
}
}
return nil
}
func fetchAndStoreLogos(
ctx context.Context,
pgClient *pg.Client,
thirdParties []thirdPartyData,
bucket, endpoint, region, accessKey, secretKey string,
usePathStyle bool,
) error {
s3Client := newS3Client(endpoint, region, accessKey, secretKey, usePathStyle)
fileMgr := filemanager.NewService(s3Client)
httpClient := httpclient.DefaultPooledClient(httpclient.WithSSRFProtection())
httpClient.Transport = &userAgentTransport{
next: httpClient.Transport,
ua: version.UserAgent("common-third-parties-import"),
}
scope := coredata.NewScope(gid.NilTenant)
var fetched, skipped, failed int
for _, tp := range thirdParties {
if tp.WebsiteURL == nil || *tp.WebsiteURL == "" {
skipped++
continue
}
var party coredata.CommonThirdParty
if err := pgClient.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error {
return party.LoadByName(ctx, conn, tp.Name)
}); err != nil {
fmt.Fprintf(os.Stderr, "warning: cannot load %q, skipping logo: %v\n", tp.Name, err)
failed++
continue
}
if party.LogoFileID != nil {
skipped++
continue
}
var logoURL string
pageInfo, err := webinspect.Parse(ctx, httpClient, *tp.WebsiteURL)
if err != nil {
fmt.Fprintf(os.Stderr, "warning: cannot inspect page for %q, trying default apple-touch-icon: %v\n", tp.Name, err)
} else {
logoURL, err = webinspect.FindLogoURL(pageInfo)
if err != nil {
fmt.Fprintf(os.Stderr, "warning: cannot find logo for %q, trying default apple-touch-icon: %v\n", tp.Name, err)
}
}
parsed, err := url.Parse(*tp.WebsiteURL)
if err != nil {
fmt.Fprintf(os.Stderr, "warning: cannot parse URL for %q, skipping logo: %v\n", tp.Name, err)
failed++
continue
}
var candidateURLs []string
if logoURL != "" {
candidateURLs = append(candidateURLs, logoURL)
}
base := fmt.Sprintf("%s://%s", parsed.Scheme, parsed.Host)
candidateURLs = append(candidateURLs,
base+"/apple-touch-icon.png",
base+"/apple-touch-icon-precomposed.png",
"https://logo.debounce.com/"+parsed.Host,
)
var (
body []byte
contentType string
)
for _, candidate := range candidateURLs {
resp, err := httpClient.Get(candidate)
if err != nil {
continue
}
b, err := io.ReadAll(resp.Body)
_ = resp.Body.Close()
if err != nil || resp.StatusCode != http.StatusOK || len(b) == 0 {
continue
}
body = b
contentType = resp.Header.Get("Content-Type")
break
}
if len(body) == 0 {
fmt.Fprintf(os.Stderr, "warning: cannot fetch logo for %q from any candidate URL\n", tp.Name)
failed++
continue
}
if contentType == "" {
contentType = "image/png"
}
objectKey, err := uuid.NewV7()
if err != nil {
return fmt.Errorf("cannot generate object key: %w", err)
}
now := time.Now()
fileID := gid.New(gid.NilTenant, coredata.FileEntityType)
fileRecord := &coredata.File{
ID: fileID,
OrganizationID: gid.Nil,
BucketName: bucket,
MimeType: contentType,
FileName: tp.Name + "-logo" + webinspect.ExtensionForMIME(contentType),
FileKey: objectKey.String(),
FileSize: int64(len(body)),
Visibility: coredata.FileVisibilityPublic,
CreatedAt: now,
UpdatedAt: now,
}
if _, err := fileMgr.PutFile(ctx, fileRecord, bytes.NewReader(body), map[string]string{
"type": "common-third-party-logo",
"common-third-party-id": party.ID.String(),
}); err != nil {
fmt.Fprintf(os.Stderr, "warning: cannot upload logo for %q to S3: %v\n", tp.Name, err)
failed++
continue
}
if err := pgClient.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error {
if err := fileRecord.Insert(ctx, tx, scope); err != nil {
return fmt.Errorf("cannot insert file record: %w", err)
}
party.LogoFileID = &fileID
party.UpdatedAt = now
if err := party.UpdateLogoFileID(ctx, tx); err != nil {
return fmt.Errorf("cannot update logo_file_id: %w", err)
}
return nil
}); err != nil {
fmt.Fprintf(os.Stderr, "warning: cannot store logo for %q: %v\n", tp.Name, err)
failed++
continue
}
fetched++
fmt.Printf(" fetched logo for %q\n", tp.Name)
}
fmt.Printf("logos: %d fetched, %d skipped, %d failed\n", fetched, skipped, failed)
return nil
}
func newS3Client(endpoint, region, accessKey, secretKey string, usePathStyle bool) *s3.Client {
if region == "" {
region = "us-east-2"
}
cfg := aws.Config{
Region: region,
}
if accessKey != "" && secretKey != "" {
cfg.Credentials = credentials.NewStaticCredentialsProvider(accessKey, secretKey, "")
}
if endpoint != "" {
cfg.BaseEndpoint = &endpoint
}
return s3.NewFromConfig(cfg, func(o *s3.Options) {
o.UsePathStyle = usePathStyle
})
}
func loadThirdParties(path string) ([]thirdPartyData, error) {
f, err := os.Open(path)
if err != nil {
return nil, fmt.Errorf("cannot open %s: %w", path, err)
}
defer func() { _ = f.Close() }()
var thirdParties []thirdPartyData
dec := json.NewDecoder(f)
dec.DisallowUnknownFields()
if err := dec.Decode(&thirdParties); err != nil {
return nil, fmt.Errorf("cannot decode %s: %w", path, err)
}
return thirdParties, nil
}
func parseCategory(tp thirdPartyData) coredata.ThirdPartyCategory {
if tp.Category == nil || *tp.Category == "" {
return coredata.ThirdPartyCategoryOther
}
var c coredata.ThirdPartyCategory
if err := c.Scan(*tp.Category); err != nil {
fmt.Fprintf(os.Stderr, "warning: third party %q has unknown category %q, falling back to OTHER\n", tp.Name, *tp.Category)
return coredata.ThirdPartyCategoryOther
}
return c
}
func newPgClientFromDSN(dsn string) (*pg.Client, error) {
u, err := url.Parse(dsn)
if err != nil {
return nil, fmt.Errorf("cannot parse DSN (check URL format)")
}
var opts []pg.Option
switch u.Query().Get("sslmode") {
case "", "disable":
// plain connection, no TLS
case "require":
opts = append(opts, pg.WithUnsecureTLS())
case "prefer":
return nil, fmt.Errorf("unsupported sslmode %q (prefer fallback semantics are not supported)", u.Query().Get("sslmode"))
default:
return nil, fmt.Errorf("unsupported sslmode %q", u.Query().Get("sslmode"))
}
if u.Host != "" {
host := u.Host
if u.Port() == "" {
host = net.JoinHostPort(u.Hostname(), "5432")
}
opts = append(opts, pg.WithAddr(host))
}
if u.User != nil {
opts = append(opts, pg.WithUser(u.User.Username()))
if password, ok := u.User.Password(); ok {
opts = append(opts, pg.WithPassword(password))
}
}
if len(u.Path) > 1 {
opts = append(opts, pg.WithDatabase(u.Path[1:]))
}
return pg.NewClient(opts...)
}
type userAgentTransport struct {
next http.RoundTripper
ua string
}
func (t *userAgentTransport) RoundTrip(req *http.Request) (*http.Response, error) {
req = req.Clone(req.Context())
req.Header.Set("User-Agent", t.ua)
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8")
req.Header.Set("Accept-Language", "en-US,en;q=0.5")
return t.next.RoundTrip(req)
}

529
cmd/common-tracker-patterns-import/main.go
View File

@@ -1,529 +0,0 @@
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
// Command common-tracker-patterns-import seeds the common_tracker_patterns
// table from the Open Cookie Database (https://github.com/jkwakman/Open-Cookie-Database).
// It clones the repository into a temporary directory, reads
// open-cookie-database.json, and upserts each entry. Re-running is safe:
// existing rows are updated on the unique constraint so ids and created_at
// are preserved.
//
// Entries are linked to the matching common_third_parties row via a
// three-step cascade: slug lookup, domain lookup, then auto-create.
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"math"
"net"
"net/url"
"os"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"time"
git "github.com/go-git/go-git/v5"
"go.gearno.de/kit/pg"
"go.probo.inc/probo/pkg/coredata"
"go.probo.inc/probo/pkg/gid"
"go.probo.inc/probo/pkg/slug"
)
const (
ocdRepoURL = "https://github.com/jkwakman/Open-Cookie-Database.git"
ocdJSONFile = "open-cookie-database.json"
)
type (
ocdEntry struct {
ID string `json:"id"`
Category string `json:"category"`
Cookie string `json:"cookie"`
Domain string `json:"domain"`
Description string `json:"description"`
RetentionPeriod string `json:"retentionPeriod"`
DataController string `json:"dataController"`
PrivacyLink string `json:"privacyLink"`
WildcardMatch string `json:"wildcardMatch"`
}
trackerPatternData struct {
Pattern string
TrackerType string
MatchType string
ThirdPartyName *string
Domain string
Category string
Description string
MaxAgeSeconds *int
Confidence float32
}
)
func main() {
if err := run(); err != nil {
fmt.Fprintf(os.Stderr, "error: %v\n", err)
os.Exit(1)
}
}
func run() error {
var pgDSN string
flag.StringVar(
&pgDSN,
"pg-dsn",
os.Getenv("DATABASE_URL"),
"PostgreSQL connection URL (default: DATABASE_URL env)",
)
flag.Parse()
if pgDSN == "" {
return fmt.Errorf("set -pg-dsn or DATABASE_URL")
}
ctx := context.Background()
fmt.Printf("cloning %s\n", ocdRepoURL)
tmpDir, cleanup, err := cloneRepo()
if err != nil {
return fmt.Errorf("cannot clone repository: %w", err)
}
defer cleanup()
patterns, err := loadPatternsFromOCD(tmpDir)
if err != nil {
return fmt.Errorf("cannot load tracker pattern data: %w", err)
}
pgClient, err := newPgClientFromDSN(pgDSN)
if err != nil {
return fmt.Errorf("cannot create pg client: %w", err)
}
fmt.Printf("importing %d common tracker patterns from Open Cookie Database\n", len(patterns))
var inserted, updated, skipped int
var partiesCreated int
if err := pgClient.WithTx(
ctx,
func(ctx context.Context, tx pg.Tx) error {
now := time.Now()
thirdPartyCache := make(map[string]*gid.GID)
for _, p := range patterns {
thirdPartyID, err := resolveThirdParty(ctx, tx, p, thirdPartyCache, now, &partiesCreated)
if err != nil {
return err
}
trackerType, err := parseTrackerType(p.TrackerType)
if err != nil {
fmt.Fprintf(os.Stderr, "warning: %v, skipping pattern %q\n", err, p.Pattern)
skipped++
continue
}
matchType, err := parseMatchType(p.MatchType)
if err != nil {
fmt.Fprintf(os.Stderr, "warning: %v, skipping pattern %q\n", err, p.Pattern)
skipped++
continue
}
pattern := coredata.CommonTrackerPattern{
ID: gid.New(gid.NilTenant, coredata.CommonTrackerPatternEntityType),
CommonThirdPartyID: thirdPartyID,
TrackerType: trackerType,
Pattern: p.Pattern,
MatchType: matchType,
Description: p.Description,
MaxAgeSeconds: p.MaxAgeSeconds,
Confidence: p.Confidence,
CreatedAt: now,
UpdatedAt: now,
}
_, wasInserted, err := pattern.Upsert(ctx, tx)
if err != nil {
return fmt.Errorf("cannot upsert common tracker pattern %q: %w", p.Pattern, err)
}
if wasInserted {
inserted++
} else {
updated++
}
}
return nil
},
); err != nil {
return err
}
fmt.Printf(
"imported %d patterns (%d inserted, %d updated, %d skipped, %d third parties auto-created)\n",
len(patterns)-skipped,
inserted,
updated,
skipped,
partiesCreated,
)
return nil
}
func cloneRepo() (string, func(), error) {
tmpDir, err := os.MkdirTemp("", "ocd-*")
if err != nil {
return "", nil, fmt.Errorf("cannot create temp dir: %w", err)
}
cleanup := func() { _ = os.RemoveAll(tmpDir) }
_, err = git.PlainClone(
tmpDir,
false,
&git.CloneOptions{
URL: ocdRepoURL,
Depth: 1,
},
)
if err != nil {
cleanup()
return "", nil, fmt.Errorf("cannot clone %s: %w", ocdRepoURL, err)
}
return tmpDir, cleanup, nil
}
func loadPatternsFromOCD(dir string) ([]trackerPatternData, error) {
f, err := os.Open(filepath.Join(dir, ocdJSONFile))
if err != nil {
return nil, fmt.Errorf("cannot open %s: %w", ocdJSONFile, err)
}
defer func() { _ = f.Close() }()
var db map[string][]ocdEntry
if err := json.NewDecoder(f).Decode(&db); err != nil {
return nil, fmt.Errorf("cannot decode %s: %w", ocdJSONFile, err)
}
platforms := make([]string, 0, len(db))
for k := range db {
platforms = append(platforms, k)
}
sort.Strings(platforms)
var patterns []trackerPatternData
for _, platform := range platforms {
for _, e := range db[platform] {
if e.Cookie == "" {
continue
}
matchType := "EXACT"
if e.WildcardMatch == "1" {
matchType = "GLOB"
}
cookiePattern := e.Cookie
if matchType == "GLOB" && !strings.ContainsAny(cookiePattern, "*?") {
cookiePattern += "*"
}
patterns = append(
patterns,
trackerPatternData{
Pattern: cookiePattern,
TrackerType: "COOKIE",
MatchType: matchType,
ThirdPartyName: new(platform),
Domain: e.Domain,
Category: e.Category,
Description: e.Description,
MaxAgeSeconds: parseRetentionPeriod(e.RetentionPeriod),
Confidence: 1.0,
},
)
}
}
return patterns, nil
}
func resolveThirdParty(
ctx context.Context,
tx pg.Tx,
p trackerPatternData,
cache map[string]*gid.GID,
now time.Time,
created *int,
) (*gid.GID, error) {
if p.ThirdPartyName == nil || *p.ThirdPartyName == "" {
return nil, nil
}
platformSlug := slug.Make(*p.ThirdPartyName)
if platformSlug == "" {
return nil, nil
}
if cached, ok := cache[platformSlug]; ok {
return cached, nil
}
var party coredata.CommonThirdParty
if err := party.LoadBySlug(ctx, tx, platformSlug); err != nil {
if !errors.Is(err, coredata.ErrResourceNotFound) {
return nil, fmt.Errorf("cannot look up common third party by slug %q: %w", platformSlug, err)
}
} else {
cache[platformSlug] = &party.ID
return &party.ID, nil
}
domain := normalizeDomain(p.Domain)
if domain != "" {
var domainRow coredata.CommonThirdPartyDomain
if err := domainRow.LoadByDomain(ctx, tx, domain); err != nil {
if !errors.Is(err, coredata.ErrResourceNotFound) {
return nil, fmt.Errorf("cannot look up common third party by domain %q: %w", domain, err)
}
} else {
if err := party.LoadByID(ctx, tx, domainRow.CommonThirdPartyID); err != nil {
return nil, fmt.Errorf("cannot load common third party by ID %s: %w", domainRow.CommonThirdPartyID, err)
}
cache[platformSlug] = &party.ID
return &party.ID, nil
}
}
party = coredata.CommonThirdParty{
ID: gid.New(gid.NilTenant, coredata.CommonThirdPartyEntityType),
Name: *p.ThirdPartyName,
Slug: platformSlug,
Category: mapOCDCategory(p.Category),
Certifications: []string{},
CreatedAt: now,
UpdatedAt: now,
}
if _, err := party.Upsert(ctx, tx); err != nil {
return nil, fmt.Errorf("cannot auto-create common third party %q: %w", *p.ThirdPartyName, err)
}
if err := party.LoadBySlug(ctx, tx, platformSlug); err != nil {
return nil, fmt.Errorf("cannot reload auto-created common third party %q: %w", *p.ThirdPartyName, err)
}
if domain != "" {
d := coredata.CommonThirdPartyDomain{
ID: gid.New(gid.NilTenant, coredata.CommonThirdPartyDomainEntityType),
CommonThirdPartyID: party.ID,
Domain: domain,
CreatedAt: now,
UpdatedAt: now,
}
if _, err := d.Upsert(ctx, tx); err != nil {
return nil, fmt.Errorf("cannot upsert domain %q for %q: %w", domain, *p.ThirdPartyName, err)
}
}
*created++
cache[platformSlug] = &party.ID
return &party.ID, nil
}
var domainValidRe = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9.-]*[a-zA-Z]$`)
func normalizeDomain(s string) string {
s = strings.TrimSpace(s)
// Strip zero-width spaces and other Unicode control characters.
s = strings.Map(func(r rune) rune {
if r == '\u200b' || r == '\ufeff' {
return -1
}
return r
}, s)
// Take only the first domain when "or" separates multiple values,
// e.g. ".bing.com (3rd party) or .microsoft.com (3rd party)".
if idx := strings.Index(s, " or "); idx != -1 {
s = s[:idx]
}
// Handle values starting with "or ", e.g. "or demdex.net (3rd party)".
s = strings.TrimPrefix(s, "or ")
if idx := strings.IndexByte(s, '('); idx != -1 {
s = strings.TrimSpace(s[:idx])
}
// Strip placeholders like "[account]".
if strings.ContainsAny(s, "[]") {
return ""
}
s = strings.TrimPrefix(s, ".")
s = strings.TrimSuffix(s, ".")
s = strings.TrimSpace(s)
if s == "" || !domainValidRe.MatchString(s) {
return ""
}
return s
}
func mapOCDCategory(s string) coredata.ThirdPartyCategory {
switch strings.ToLower(strings.TrimSpace(s)) {
case "analytics":
return coredata.ThirdPartyCategoryAnalytics
case "marketing":
return coredata.ThirdPartyCategoryMarketing
default:
return coredata.ThirdPartyCategoryOther
}
}
var retentionRe = regexp.MustCompile(`(?i)^(\d+)\s+(second|seconds|sec|secs|minute|minutes|mins|min|hour|hours|day|days|week|weeks|month|months|year|years)`)
func parseRetentionPeriod(s string) *int {
s = strings.TrimSpace(s)
if s == "" {
return nil
}
lower := strings.ToLower(s)
switch {
case lower == "session" || lower == "sessions" || lower == "seesion" ||
lower == "session cookie" || strings.HasPrefix(lower, "end of session"):
return nil
case lower == "varies" || lower == "various" || lower == "unknown" ||
lower == "undefined" || lower == "persistent" || lower == "permanent" ||
lower == "forever" || lower == "unlimited" || lower == "no expiration" ||
lower == "local storage":
return nil
}
m := retentionRe.FindStringSubmatch(s)
if m == nil {
return nil
}
n, err := strconv.Atoi(m[1])
if err != nil {
return nil
}
var multiplier int
switch strings.ToLower(m[2]) {
case "second", "seconds", "sec", "secs":
multiplier = 1
case "minute", "minutes", "mins", "min":
multiplier = 60
case "hour", "hours":
multiplier = 3600
case "day", "days":
multiplier = 86400
case "week", "weeks":
multiplier = 604800
case "month", "months":
multiplier = 2592000
case "year", "years":
multiplier = 31536000
default:
return nil
}
result := min(n*multiplier, math.MaxInt32)
return &result
}
func parseTrackerType(s string) (coredata.TrackerType, error) {
switch s {
case "COOKIE":
return coredata.TrackerTypeCookie, nil
case "LOCAL_STORAGE":
return coredata.TrackerTypeLocalStorage, nil
case "SESSION_STORAGE":
return coredata.TrackerTypeSessionStorage, nil
case "INDEXED_DB":
return coredata.TrackerTypeIndexedDB, nil
default:
return "", fmt.Errorf("unknown tracker type %q", s)
}
}
func parseMatchType(s string) (coredata.TrackerPatternMatchType, error) {
switch s {
case "EXACT":
return coredata.TrackerPatternMatchTypeExact, nil
case "GLOB":
return coredata.TrackerPatternMatchTypeGlob, nil
case "PREFIX":
return coredata.TrackerPatternMatchTypePrefix, nil
default:
return "", fmt.Errorf("unknown match type %q", s)
}
}
func newPgClientFromDSN(dsn string) (*pg.Client, error) {
u, err := url.Parse(dsn)
if err != nil {
return nil, fmt.Errorf("cannot parse DSN (check URL format)")
}
var opts []pg.Option
switch u.Query().Get("sslmode") {
case "", "disable":
case "require":
opts = append(opts, pg.WithUnsecureTLS())
case "prefer":
return nil, fmt.Errorf("unsupported sslmode %q (prefer fallback semantics are not supported)", u.Query().Get("sslmode"))
default:
return nil, fmt.Errorf("unsupported sslmode %q", u.Query().Get("sslmode"))
}
if u.Host != "" {
host := u.Host
if u.Port() == "" {
host = net.JoinHostPort(u.Hostname(), "5432")
}
opts = append(opts, pg.WithAddr(host))
}
if u.User != nil {
opts = append(opts, pg.WithUser(u.User.Username()))
if password, ok := u.User.Password(); ok {
opts = append(opts, pg.WithPassword(password))
}
}
if len(u.Path) > 1 {
opts = append(opts, pg.WithDatabase(u.Path[1:]))
}
return pg.NewClient(opts...)
}

1
cmd/proboctl/CHANGELOG.md Normal file
View File

@@ -0,0 +1 @@
# Changelog

1
cmd/proboctl/VERSION Normal file
View File

@@ -0,0 +1 @@
0.0.0

42
cmd/proboctl/main.go Normal file
View File

@@ -0,0 +1,42 @@
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package main
import (
"fmt"
"os"
"go.probo.inc/probo/pkg/cmd/iostreams"
"go.probo.inc/probo/pkg/proboctl/cmdutil"
"go.probo.inc/probo/pkg/proboctl/root"
)
var version string = "unknown"
func main() {
ios := iostreams.System()
f := &cmdutil.Factory{
IOStreams: ios,
Version: version,
}
cmd := root.NewCmdRoot(f)
if err := cmd.Execute(); err != nil {
fmt.Fprintf(os.Stderr, "Error: %s\n", err)
os.Exit(1)
}
}