Move common third-party resolver to thirdparty pkg
resolveOrCreateCommonThirdParty lived as a package-level helper in the tracker mapping worker, but the common pattern enrichment worker now reuses it. Homing shared catalog logic in a mapping-named file made the enrichment worker quietly depend on the mapping worker's file, and it is not a mapping concern. Move it to pkg/thirdparty as exported ResolveOrCreateCommonThirdParty, decoupled from cookiebanner's TrackerMappingAgentResult (it now takes a name and category) to avoid an import cycle. It stays a transaction- scoped free function so both workers compose it into their own tx for atomicity rather than receiving a service that owns its own connection. Relocate the catalog dedup DB test alongside it. Signed-off-by: Émile Ré <emile@probo.com>
This commit is contained in:
Émile Ré
94
pkg/thirdparty/resolver.go
vendored
Normal file
94
pkg/thirdparty/resolver.go
vendored
Normal file
@@ -0,0 +1,94 @@
|
||||
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||
//
|
||||
// Permission to use, copy, modify, and/or distribute this software for any
|
||||
// purpose with or without fee is hereby granted, provided that the above
|
||||
// copyright notice and this permission notice appear in all copies.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||
// PERFORMANCE OF THIS SOFTWARE.
|
||||
|
||||
package thirdparty
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"go.gearno.de/kit/log"
|
||||
"go.gearno.de/kit/pg"
|
||||
"go.probo.inc/probo/pkg/coredata"
|
||||
"go.probo.inc/probo/pkg/gid"
|
||||
"go.probo.inc/probo/pkg/slug"
|
||||
)
|
||||
|
||||
// ResolveOrCreateCommonThirdParty links a named vendor to the global
|
||||
// catalog, creating a row when none matches. Dedup is deterministic:
|
||||
// exact name, then slug, before insert. Callers run inside their own
|
||||
// transaction and pass the logger explicitly, so it is shared by the
|
||||
// tracker mapping worker (which supplies observed domains) and the
|
||||
// common pattern enrichment worker (which has none).
|
||||
func ResolveOrCreateCommonThirdParty(
|
||||
ctx context.Context,
|
||||
tx pg.Tx,
|
||||
logger *log.Logger,
|
||||
name string,
|
||||
category coredata.ThirdPartyCategory,
|
||||
domains []string,
|
||||
) (*gid.GID, error) {
|
||||
var party coredata.CommonThirdParty
|
||||
if err := party.LoadByName(ctx, tx, name); err == nil {
|
||||
return &party.ID, nil
|
||||
}
|
||||
|
||||
partySlug := slug.Make(name)
|
||||
if partySlug == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if err := party.LoadBySlug(ctx, tx, partySlug); err == nil {
|
||||
return &party.ID, nil
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
party = coredata.CommonThirdParty{
|
||||
ID: gid.New(gid.NilTenant, coredata.CommonThirdPartyEntityType),
|
||||
Name: name,
|
||||
Slug: partySlug,
|
||||
Category: category,
|
||||
Certifications: []string{},
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
if err := party.Insert(ctx, tx); err != nil {
|
||||
return nil, fmt.Errorf("cannot create common third party: %w", err)
|
||||
}
|
||||
|
||||
for _, domain := range domains {
|
||||
domainRecord := coredata.CommonThirdPartyDomain{
|
||||
ID: gid.New(gid.NilTenant, coredata.CommonThirdPartyDomainEntityType),
|
||||
CommonThirdPartyID: party.ID,
|
||||
Domain: domain,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
if _, err := domainRecord.Upsert(ctx, tx); err != nil {
|
||||
return nil, fmt.Errorf("cannot create common third party domain: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
logger.InfoCtx(
|
||||
ctx,
|
||||
"created common third party from agent identification",
|
||||
log.String("name", name),
|
||||
log.String("category", category.String()),
|
||||
)
|
||||
|
||||
return &party.ID, nil
|
||||
}
|
||||
224
pkg/thirdparty/resolver_test.go
vendored
Normal file
224
pkg/thirdparty/resolver_test.go
vendored
Normal file
@@ -0,0 +1,224 @@
|
||||
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||
//
|
||||
// Permission to use, copy, modify, and/or distribute this software for any
|
||||
// purpose with or without fee is hereby granted, provided that the above
|
||||
// copyright notice and this permission notice appear in all copies.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||
// PERFORMANCE OF THIS SOFTWARE.
|
||||
|
||||
package thirdparty
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net"
|
||||
"net/url"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.gearno.de/kit/log"
|
||||
"go.gearno.de/kit/pg"
|
||||
"go.probo.inc/probo/pkg/coredata"
|
||||
"go.probo.inc/probo/pkg/gid"
|
||||
"go.probo.inc/probo/pkg/slug"
|
||||
)
|
||||
|
||||
const testPgDSNEnvVar = "PROBO_TEST_PG_URL"
|
||||
|
||||
func newTestPgClient(t *testing.T) *pg.Client {
|
||||
t.Helper()
|
||||
|
||||
dsn := os.Getenv(testPgDSNEnvVar)
|
||||
if dsn == "" {
|
||||
t.Skipf("skipping: %s not set (requires a migrated test database)", testPgDSNEnvVar)
|
||||
}
|
||||
|
||||
u, err := url.Parse(dsn)
|
||||
require.NoError(t, err, "invalid %s value", testPgDSNEnvVar)
|
||||
|
||||
opts := []pg.Option{pg.WithRegisterer(prometheus.NewRegistry())}
|
||||
|
||||
if u.Host != "" {
|
||||
host := u.Host
|
||||
if u.Port() == "" {
|
||||
host = net.JoinHostPort(u.Hostname(), "5432")
|
||||
}
|
||||
|
||||
opts = append(opts, pg.WithAddr(host))
|
||||
}
|
||||
|
||||
if u.User != nil {
|
||||
opts = append(opts, pg.WithUser(u.User.Username()))
|
||||
if password, ok := u.User.Password(); ok {
|
||||
opts = append(opts, pg.WithPassword(password))
|
||||
}
|
||||
}
|
||||
|
||||
if len(u.Path) > 1 {
|
||||
opts = append(opts, pg.WithDatabase(u.Path[1:]))
|
||||
}
|
||||
|
||||
client, err := pg.NewClient(opts...)
|
||||
require.NoError(t, err)
|
||||
|
||||
t.Cleanup(func() {
|
||||
client.Close()
|
||||
})
|
||||
|
||||
return client
|
||||
}
|
||||
|
||||
func discardLogger() *log.Logger {
|
||||
return log.NewLogger(log.WithOutput(io.Discard))
|
||||
}
|
||||
|
||||
// seedCatalogThirdParty inserts a catalog third party with the given
|
||||
// name and slug and registers its cleanup.
|
||||
func seedCatalogThirdParty(
|
||||
t *testing.T,
|
||||
ctx context.Context,
|
||||
client *pg.Client,
|
||||
name string,
|
||||
slugValue string,
|
||||
) coredata.CommonThirdParty {
|
||||
t.Helper()
|
||||
|
||||
now := time.Now().UTC().Truncate(time.Microsecond)
|
||||
party := coredata.CommonThirdParty{
|
||||
ID: gid.New(gid.NilTenant, coredata.CommonThirdPartyEntityType),
|
||||
Name: name,
|
||||
Slug: slugValue,
|
||||
Category: coredata.ThirdPartyCategoryAnalytics,
|
||||
Certifications: []string{},
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error {
|
||||
return party.Insert(ctx, tx)
|
||||
}))
|
||||
|
||||
t.Cleanup(func() {
|
||||
_ = client.WithTx(context.Background(), func(ctx context.Context, tx pg.Tx) error {
|
||||
_, err := tx.Exec(ctx, `DELETE FROM common_third_parties WHERE id = $1`, party.ID)
|
||||
return err
|
||||
})
|
||||
})
|
||||
|
||||
return party
|
||||
}
|
||||
|
||||
// TestResolveOrCreateCommonThirdParty pins the catalog dedup that the
|
||||
// mapping and enrichment workers reuse to link a vendor: an exact name
|
||||
// match and a slug match both return the existing row, and a name absent
|
||||
// from the catalog creates a fresh row (name, slug, category) rather than
|
||||
// duplicating one.
|
||||
func TestResolveOrCreateCommonThirdParty(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
client := newTestPgClient(t)
|
||||
ctx := context.Background()
|
||||
logger := discardLogger()
|
||||
|
||||
token := slug.Make(gid.New(gid.NilTenant, coredata.CommonThirdPartyEntityType).String())
|
||||
|
||||
t.Run("returns existing row on name match", func(t *testing.T) {
|
||||
name := "Hotjar " + token
|
||||
party := seedCatalogThirdParty(t, ctx, client, name, slug.Make(name))
|
||||
|
||||
var got *gid.GID
|
||||
require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error {
|
||||
id, err := ResolveOrCreateCommonThirdParty(
|
||||
ctx,
|
||||
tx,
|
||||
logger,
|
||||
name,
|
||||
coredata.ThirdPartyCategoryAnalytics,
|
||||
nil,
|
||||
)
|
||||
got = id
|
||||
|
||||
return err
|
||||
}))
|
||||
|
||||
require.NotNil(t, got)
|
||||
assert.Equal(t, party.ID, *got)
|
||||
})
|
||||
|
||||
t.Run("returns existing row on slug match", func(t *testing.T) {
|
||||
name := "Matomo " + token
|
||||
party := seedCatalogThirdParty(t, ctx, client, name, slug.Make(name))
|
||||
|
||||
// A differently-spelled name that normalizes to the same slug
|
||||
// must resolve to the existing row, not create a duplicate.
|
||||
variant := "Matomo " + token + "!!!"
|
||||
require.NotEqual(t, name, variant)
|
||||
require.Equal(t, slug.Make(name), slug.Make(variant))
|
||||
|
||||
var got *gid.GID
|
||||
require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error {
|
||||
id, err := ResolveOrCreateCommonThirdParty(
|
||||
ctx,
|
||||
tx,
|
||||
logger,
|
||||
variant,
|
||||
coredata.ThirdPartyCategoryAnalytics,
|
||||
nil,
|
||||
)
|
||||
got = id
|
||||
|
||||
return err
|
||||
}))
|
||||
|
||||
require.NotNil(t, got)
|
||||
assert.Equal(t, party.ID, *got)
|
||||
})
|
||||
|
||||
t.Run("creates a new row when absent", func(t *testing.T) {
|
||||
name := "Freshvendor " + token
|
||||
expectedSlug := slug.Make(name)
|
||||
|
||||
var got *gid.GID
|
||||
require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error {
|
||||
id, err := ResolveOrCreateCommonThirdParty(
|
||||
ctx,
|
||||
tx,
|
||||
logger,
|
||||
name,
|
||||
coredata.ThirdPartyCategoryMarketing,
|
||||
nil,
|
||||
)
|
||||
got = id
|
||||
|
||||
return err
|
||||
}))
|
||||
|
||||
require.NotNil(t, got)
|
||||
|
||||
t.Cleanup(func() {
|
||||
_ = client.WithTx(context.Background(), func(ctx context.Context, tx pg.Tx) error {
|
||||
_, err := tx.Exec(ctx, `DELETE FROM common_third_parties WHERE id = $1`, *got)
|
||||
return err
|
||||
})
|
||||
})
|
||||
|
||||
var created coredata.CommonThirdParty
|
||||
require.NoError(t, client.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error {
|
||||
return created.LoadByID(ctx, conn, *got)
|
||||
}))
|
||||
|
||||
assert.Equal(t, name, created.Name)
|
||||
assert.Equal(t, expectedSlug, created.Slug)
|
||||
assert.Equal(t, coredata.ThirdPartyCategoryMarketing, created.Category)
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user