Add common_third_parties shared reference table

Introduce a globally-shared, non-tenant-scoped common_third_parties
table that mirrors the public subset of vendor metadata, plus a
one-shot cmd/common-third-parties-import CLI that seeds it from
packages/vendors/data.json. The catalog will back future flows (e.g.
vendor autocomplete) so each tenant no longer needs to duplicate the
same baseline data.

The importer is idempotent via ON CONFLICT (lower(name)) DO UPDATE
and prints inserted/updated counts. GIDs use gid.NilTenant since the
table is not tenant-scoped; uniqueness still comes from the entity
type plus 14 bytes of timestamp/random suffix.

Signed-off-by: Émile Ré <emile@getprobo.com>
This commit is contained in:
Émile Ré
2026-05-08 14:31:26 +04:00
parent ee22e9f964
commit 361fe1d713
5 changed files with 680 additions and 2 deletions

View File

@@ -0,0 +1,214 @@
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
// Command common-third-parties-import seeds the common_third_parties table from
// packages/vendors/data.json. It is idempotent: re-running upserts on conflict
// (lower(name)) so existing rows keep their id and created_at.
package main
import (
"context"
"encoding/json"
"flag"
"fmt"
"net"
"net/url"
"os"
"time"
"go.gearno.de/kit/pg"
"go.probo.inc/probo/pkg/coredata"
"go.probo.inc/probo/pkg/gid"
)
type thirdPartyData struct {
Name string `json:"name"`
Description *string `json:"description,omitempty"`
Category *string `json:"category,omitempty"`
HeadquarterAddress *string `json:"headquarterAddress,omitempty"`
LegalName *string `json:"legalName,omitempty"`
WebsiteURL *string `json:"websiteUrl,omitempty"`
PrivacyPolicyURL *string `json:"privacyPolicyUrl,omitempty"`
ServiceLevelAgreementURL *string `json:"serviceLevelAgreementUrl,omitempty"`
ServiceSoftwareAgreementURL *string `json:"serviceSoftwareAgreementUrl,omitempty"`
DataProcessingAgreementURL *string `json:"dataProcessingAgreementUrl,omitempty"`
BusinessAssociateAgreementURL *string `json:"businessAssociateAgreementUrl,omitempty"`
SubprocessorsListURL *string `json:"subprocessorsListUrl,omitempty"`
Certifications []string `json:"certifications,omitempty"`
StatusPageURL *string `json:"statusPageUrl,omitempty"`
TermsOfServiceURL *string `json:"termsOfServiceUrl,omitempty"`
SecurityPageURL *string `json:"securityPageUrl,omitempty"`
TrustPageURL *string `json:"trustPageUrl,omitempty"`
}
func main() {
if err := run(); err != nil {
fmt.Fprintf(os.Stderr, "error: %v\n", err)
os.Exit(1)
}
}
func run() error {
var (
pgDSN string
dataPath string
)
flag.StringVar(
&pgDSN,
"pg-dsn",
os.Getenv("DATABASE_URL"),
"PostgreSQL connection URL (default: DATABASE_URL env)",
)
flag.StringVar(
&dataPath,
"data",
"",
"Path to the third-party data.json file",
)
flag.Parse()
if pgDSN == "" {
return fmt.Errorf("set -pg-dsn or DATABASE_URL")
}
ctx := context.Background()
thirdParties, err := loadThirdParties(dataPath)
if err != nil {
return fmt.Errorf("cannot load third-party data: %w", err)
}
pgClient, err := newPgClientFromDSN(pgDSN)
if err != nil {
return fmt.Errorf("cannot create pg client: %w", err)
}
fmt.Printf("importing %d common third parties from %s\n", len(thirdParties), dataPath)
var inserted, updated int
if err := pgClient.WithTx(
ctx,
func(ctx context.Context, tx pg.Tx) error {
now := time.Now()
for _, tp := range thirdParties {
party := coredata.CommonThirdParty{
ID: gid.New(gid.NilTenant, coredata.CommonThirdPartyEntityType),
Name: tp.Name,
Description: tp.Description,
Category: parseCategory(tp),
HeadquarterAddress: tp.HeadquarterAddress,
LegalName: tp.LegalName,
WebsiteURL: tp.WebsiteURL,
PrivacyPolicyURL: tp.PrivacyPolicyURL,
ServiceLevelAgreementURL: tp.ServiceLevelAgreementURL,
ServiceSoftwareAgreementURL: tp.ServiceSoftwareAgreementURL,
DataProcessingAgreementURL: tp.DataProcessingAgreementURL,
BusinessAssociateAgreementURL: tp.BusinessAssociateAgreementURL,
SubprocessorsListURL: tp.SubprocessorsListURL,
Certifications: tp.Certifications,
StatusPageURL: tp.StatusPageURL,
TermsOfServiceURL: tp.TermsOfServiceURL,
SecurityPageURL: tp.SecurityPageURL,
TrustPageURL: tp.TrustPageURL,
CreatedAt: now,
UpdatedAt: now,
}
wasInserted, err := party.Upsert(ctx, tx)
if err != nil {
return fmt.Errorf("cannot upsert common third party %q: %w", tp.Name, err)
}
if wasInserted {
inserted++
} else {
updated++
}
}
return nil
},
); err != nil {
return err
}
fmt.Printf("imported %d rows (%d inserted, %d updated)\n", len(thirdParties), inserted, updated)
return nil
}
func loadThirdParties(path string) ([]thirdPartyData, error) {
f, err := os.Open(path)
if err != nil {
return nil, fmt.Errorf("cannot open %s: %w", path, err)
}
defer func() { _ = f.Close() }()
var thirdParties []thirdPartyData
dec := json.NewDecoder(f)
dec.DisallowUnknownFields()
if err := dec.Decode(&thirdParties); err != nil {
return nil, fmt.Errorf("cannot decode %s: %w", path, err)
}
return thirdParties, nil
}
func parseCategory(tp thirdPartyData) coredata.VendorCategory {
if tp.Category == nil || *tp.Category == "" {
return coredata.VendorCategoryOther
}
var c coredata.VendorCategory
if err := c.Scan(*tp.Category); err != nil {
fmt.Fprintf(os.Stderr, "warning: third party %q has unknown category %q, falling back to OTHER\n", tp.Name, *tp.Category)
return coredata.VendorCategoryOther
}
return c
}
func newPgClientFromDSN(dsn string) (*pg.Client, error) {
u, err := url.Parse(dsn)
if err != nil {
return nil, fmt.Errorf("cannot parse DSN: %w", err)
}
var opts []pg.Option
if u.Host != "" {
host := u.Host
if u.Port() == "" {
host = net.JoinHostPort(u.Hostname(), "5432")
}
opts = append(opts, pg.WithAddr(host))
}
if u.User != nil {
opts = append(opts, pg.WithUser(u.User.Username()))
if password, ok := u.User.Password(); ok {
opts = append(opts, pg.WithPassword(password))
}
}
if len(u.Path) > 1 {
opts = append(opts, pg.WithDatabase(u.Path[1:]))
}
return pg.NewClient(opts...)
}

View File

@@ -2269,7 +2269,7 @@
"headquarterAddress": "5314 56 St, Camrose, Alberta T4V 2E5, Canada",
"websiteUrl": "https://www.danami.com",
"description": "Developer of security, firewall and anti-spam extensions for the Plesk hosting control panel.",
"category": "DEVELOPER_TOOLS",
"category": "ENGINEERING",
"privacyPolicyUrl": "https://www.danami.com/legal/privacy-policy",
"termsOfServiceUrl": "https://www.danami.com/legal/terms-of-service"
},
@@ -2432,4 +2432,4 @@
"securityPageUrl": "https://www.scaleway.com/en/security-and-resilience/",
"statusPageUrl": "https://status.scaleway.com/"
}
]
]

View File

@@ -0,0 +1,422 @@
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package coredata
import (
"context"
"errors"
"fmt"
"time"
"github.com/jackc/pgx/v5"
"go.gearno.de/kit/pg"
"go.probo.inc/probo/pkg/gid"
)
type (
CommonThirdParty struct {
ID gid.GID `db:"id"`
Name string `db:"name"`
Description *string `db:"description"`
Category VendorCategory `db:"category"`
HeadquarterAddress *string `db:"headquarter_address"`
LegalName *string `db:"legal_name"`
WebsiteURL *string `db:"website_url"`
PrivacyPolicyURL *string `db:"privacy_policy_url"`
ServiceLevelAgreementURL *string `db:"service_level_agreement_url"`
ServiceSoftwareAgreementURL *string `db:"service_software_agreement_url"`
DataProcessingAgreementURL *string `db:"data_processing_agreement_url"`
BusinessAssociateAgreementURL *string `db:"business_associate_agreement_url"`
SubprocessorsListURL *string `db:"subprocessors_list_url"`
Certifications []string `db:"certifications"`
StatusPageURL *string `db:"status_page_url"`
TermsOfServiceURL *string `db:"terms_of_service_url"`
SecurityPageURL *string `db:"security_page_url"`
TrustPageURL *string `db:"trust_page_url"`
CreatedAt time.Time `db:"created_at"`
UpdatedAt time.Time `db:"updated_at"`
}
CommonThirdParties []*CommonThirdParty
)
func (t *CommonThirdParty) LoadByID(
ctx context.Context,
conn pg.Querier,
id gid.GID,
) error {
q := `
SELECT
id,
name,
description,
category,
headquarter_address,
legal_name,
website_url,
privacy_policy_url,
service_level_agreement_url,
service_software_agreement_url,
data_processing_agreement_url,
business_associate_agreement_url,
subprocessors_list_url,
certifications,
status_page_url,
terms_of_service_url,
security_page_url,
trust_page_url,
created_at,
updated_at
FROM
common_third_parties
WHERE
id = @id
LIMIT 1;
`
args := pgx.StrictNamedArgs{"id": id}
rows, err := conn.Query(ctx, q, args)
if err != nil {
return fmt.Errorf("cannot query common third party: %w", err)
}
defer rows.Close()
row, err := pgx.CollectExactlyOneRow(rows, pgx.RowToStructByName[CommonThirdParty])
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return ErrResourceNotFound
}
return fmt.Errorf("cannot collect common third party: %w", err)
}
*t = row
return nil
}
func (t *CommonThirdParty) LoadByName(
ctx context.Context,
conn pg.Querier,
name string,
) error {
q := `
SELECT
id,
name,
description,
category,
headquarter_address,
legal_name,
website_url,
privacy_policy_url,
service_level_agreement_url,
service_software_agreement_url,
data_processing_agreement_url,
business_associate_agreement_url,
subprocessors_list_url,
certifications,
status_page_url,
terms_of_service_url,
security_page_url,
trust_page_url,
created_at,
updated_at
FROM
common_third_parties
WHERE
lower(name) = lower(@name)
LIMIT 1;
`
args := pgx.StrictNamedArgs{"name": name}
rows, err := conn.Query(ctx, q, args)
if err != nil {
return fmt.Errorf("cannot query common third party by name: %w", err)
}
defer rows.Close()
row, err := pgx.CollectExactlyOneRow(rows, pgx.RowToStructByName[CommonThirdParty])
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return ErrResourceNotFound
}
return fmt.Errorf("cannot collect common third party by name: %w", err)
}
*t = row
return nil
}
func (t CommonThirdParty) Insert(
ctx context.Context,
conn pg.Tx,
) error {
q := `
INSERT INTO common_third_parties (
id,
name,
description,
category,
headquarter_address,
legal_name,
website_url,
privacy_policy_url,
service_level_agreement_url,
service_software_agreement_url,
data_processing_agreement_url,
business_associate_agreement_url,
subprocessors_list_url,
certifications,
status_page_url,
terms_of_service_url,
security_page_url,
trust_page_url,
created_at,
updated_at
) VALUES (
@id,
@name,
@description,
@category,
@headquarter_address,
@legal_name,
@website_url,
@privacy_policy_url,
@service_level_agreement_url,
@service_software_agreement_url,
@data_processing_agreement_url,
@business_associate_agreement_url,
@subprocessors_list_url,
@certifications,
@status_page_url,
@terms_of_service_url,
@security_page_url,
@trust_page_url,
@created_at,
@updated_at
)
`
args := pgx.StrictNamedArgs{
"id": t.ID,
"name": t.Name,
"description": t.Description,
"category": t.Category,
"headquarter_address": t.HeadquarterAddress,
"legal_name": t.LegalName,
"website_url": t.WebsiteURL,
"privacy_policy_url": t.PrivacyPolicyURL,
"service_level_agreement_url": t.ServiceLevelAgreementURL,
"service_software_agreement_url": t.ServiceSoftwareAgreementURL,
"data_processing_agreement_url": t.DataProcessingAgreementURL,
"business_associate_agreement_url": t.BusinessAssociateAgreementURL,
"subprocessors_list_url": t.SubprocessorsListURL,
"certifications": t.Certifications,
"status_page_url": t.StatusPageURL,
"terms_of_service_url": t.TermsOfServiceURL,
"security_page_url": t.SecurityPageURL,
"trust_page_url": t.TrustPageURL,
"created_at": t.CreatedAt,
"updated_at": t.UpdatedAt,
}
_, err := conn.Exec(ctx, q, args)
if err != nil {
return fmt.Errorf("cannot insert common third party: %w", err)
}
return nil
}
// Upsert inserts a row, or on lower(name) conflict updates every column except
// id and created_at. Returns true if a new row was inserted, false if an
// existing row was updated.
func (t CommonThirdParty) Upsert(
ctx context.Context,
conn pg.Tx,
) (inserted bool, err error) {
q := `
INSERT INTO common_third_parties (
id,
name,
description,
category,
headquarter_address,
legal_name,
website_url,
privacy_policy_url,
service_level_agreement_url,
service_software_agreement_url,
data_processing_agreement_url,
business_associate_agreement_url,
subprocessors_list_url,
certifications,
status_page_url,
terms_of_service_url,
security_page_url,
trust_page_url,
created_at,
updated_at
) VALUES (
@id,
@name,
@description,
@category,
@headquarter_address,
@legal_name,
@website_url,
@privacy_policy_url,
@service_level_agreement_url,
@service_software_agreement_url,
@data_processing_agreement_url,
@business_associate_agreement_url,
@subprocessors_list_url,
@certifications,
@status_page_url,
@terms_of_service_url,
@security_page_url,
@trust_page_url,
@created_at,
@updated_at
)
ON CONFLICT (lower(name)) DO UPDATE
SET
name = EXCLUDED.name,
description = EXCLUDED.description,
category = EXCLUDED.category,
headquarter_address = EXCLUDED.headquarter_address,
legal_name = EXCLUDED.legal_name,
website_url = EXCLUDED.website_url,
privacy_policy_url = EXCLUDED.privacy_policy_url,
service_level_agreement_url = EXCLUDED.service_level_agreement_url,
service_software_agreement_url = EXCLUDED.service_software_agreement_url,
data_processing_agreement_url = EXCLUDED.data_processing_agreement_url,
business_associate_agreement_url = EXCLUDED.business_associate_agreement_url,
subprocessors_list_url = EXCLUDED.subprocessors_list_url,
certifications = EXCLUDED.certifications,
status_page_url = EXCLUDED.status_page_url,
terms_of_service_url = EXCLUDED.terms_of_service_url,
security_page_url = EXCLUDED.security_page_url,
trust_page_url = EXCLUDED.trust_page_url,
updated_at = EXCLUDED.updated_at
RETURNING (xmax = 0) AS inserted
`
args := pgx.StrictNamedArgs{
"id": t.ID,
"name": t.Name,
"description": t.Description,
"category": t.Category,
"headquarter_address": t.HeadquarterAddress,
"legal_name": t.LegalName,
"website_url": t.WebsiteURL,
"privacy_policy_url": t.PrivacyPolicyURL,
"service_level_agreement_url": t.ServiceLevelAgreementURL,
"service_software_agreement_url": t.ServiceSoftwareAgreementURL,
"data_processing_agreement_url": t.DataProcessingAgreementURL,
"business_associate_agreement_url": t.BusinessAssociateAgreementURL,
"subprocessors_list_url": t.SubprocessorsListURL,
"certifications": t.Certifications,
"status_page_url": t.StatusPageURL,
"terms_of_service_url": t.TermsOfServiceURL,
"security_page_url": t.SecurityPageURL,
"trust_page_url": t.TrustPageURL,
"created_at": t.CreatedAt,
"updated_at": t.UpdatedAt,
}
rows, err := conn.Query(ctx, q, args)
if err != nil {
return false, fmt.Errorf("cannot upsert common third party: %w", err)
}
defer rows.Close()
inserted, err = pgx.CollectExactlyOneRow(rows, pgx.RowTo[bool])
if err != nil {
return false, fmt.Errorf("cannot collect upsert result: %w", err)
}
return inserted, nil
}
func (t CommonThirdParty) Delete(
ctx context.Context,
conn pg.Tx,
id gid.GID,
) error {
q := `DELETE FROM common_third_parties WHERE id = @id`
args := pgx.StrictNamedArgs{"id": id}
result, err := conn.Exec(ctx, q, args)
if err != nil {
return fmt.Errorf("cannot delete common third party: %w", err)
}
if result.RowsAffected() == 0 {
return ErrResourceNotFound
}
return nil
}
func (t *CommonThirdParties) LoadAll(
ctx context.Context,
conn pg.Querier,
) error {
q := `
SELECT
id,
name,
description,
category,
headquarter_address,
legal_name,
website_url,
privacy_policy_url,
service_level_agreement_url,
service_software_agreement_url,
data_processing_agreement_url,
business_associate_agreement_url,
subprocessors_list_url,
certifications,
status_page_url,
terms_of_service_url,
security_page_url,
trust_page_url,
created_at,
updated_at
FROM
common_third_parties
ORDER BY name ASC
`
rows, err := conn.Query(ctx, q)
if err != nil {
return fmt.Errorf("cannot query common third parties: %w", err)
}
parties, err := pgx.CollectRows(rows, pgx.RowToAddrOfStructByName[CommonThirdParty])
if err != nil {
return fmt.Errorf("cannot collect common third parties: %w", err)
}
*t = parties
return nil
}

View File

@@ -115,6 +115,7 @@ const (
TrackerPatternEntityType uint16 = 89
DetectedTrackerEntityType uint16 = 90
TrackerResourceEntityType uint16 = 91
CommonThirdPartyEntityType uint16 = 92
)
func NewEntityFromID(id gid.GID) (any, bool) {
@@ -287,6 +288,8 @@ func NewEntityFromID(id gid.GID) (any, bool) {
return &DetectedTracker{ID: id}, true
case TrackerResourceEntityType:
return &TrackerResource{ID: id}, true
case CommonThirdPartyEntityType:
return &CommonThirdParty{ID: id}, true
default:
return nil, false
}

View File

@@ -0,0 +1,39 @@
-- Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
--
-- Permission to use, copy, modify, and/or distribute this software for any
-- purpose with or without fee is hereby granted, provided that the above
-- copyright notice and this permission notice appear in all copies.
--
-- THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
-- REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
-- AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
-- INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
-- LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
-- OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
-- PERFORMANCE OF THIS SOFTWARE.
CREATE TABLE common_third_parties (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
description TEXT,
category vendor_category NOT NULL,
headquarter_address TEXT,
legal_name TEXT,
website_url TEXT,
privacy_policy_url TEXT,
service_level_agreement_url TEXT,
service_software_agreement_url TEXT,
data_processing_agreement_url TEXT,
business_associate_agreement_url TEXT,
subprocessors_list_url TEXT,
certifications TEXT[],
status_page_url TEXT,
terms_of_service_url TEXT,
security_page_url TEXT,
trust_page_url TEXT,
created_at TIMESTAMP WITH TIME ZONE NOT NULL,
updated_at TIMESTAMP WITH TIME ZONE NOT NULL
);
CREATE UNIQUE INDEX common_third_parties_name_key
ON common_third_parties (lower(name));