diff --git a/cmd/common-third-parties-import/main.go b/cmd/common-third-parties-import/main.go new file mode 100644 index 000000000..fbe8e0e9c --- /dev/null +++ b/cmd/common-third-parties-import/main.go @@ -0,0 +1,214 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +// Command common-third-parties-import seeds the common_third_parties table from +// packages/vendors/data.json. It is idempotent: re-running upserts on conflict +// (lower(name)) so existing rows keep their id and created_at. +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "net" + "net/url" + "os" + "time" + + "go.gearno.de/kit/pg" + "go.probo.inc/probo/pkg/coredata" + "go.probo.inc/probo/pkg/gid" +) + +type thirdPartyData struct { + Name string `json:"name"` + Description *string `json:"description,omitempty"` + Category *string `json:"category,omitempty"` + HeadquarterAddress *string `json:"headquarterAddress,omitempty"` + LegalName *string `json:"legalName,omitempty"` + WebsiteURL *string `json:"websiteUrl,omitempty"` + PrivacyPolicyURL *string `json:"privacyPolicyUrl,omitempty"` + ServiceLevelAgreementURL *string `json:"serviceLevelAgreementUrl,omitempty"` + ServiceSoftwareAgreementURL *string `json:"serviceSoftwareAgreementUrl,omitempty"` + DataProcessingAgreementURL *string `json:"dataProcessingAgreementUrl,omitempty"` + BusinessAssociateAgreementURL *string `json:"businessAssociateAgreementUrl,omitempty"` + SubprocessorsListURL *string `json:"subprocessorsListUrl,omitempty"` + Certifications []string `json:"certifications,omitempty"` + StatusPageURL *string `json:"statusPageUrl,omitempty"` + TermsOfServiceURL *string `json:"termsOfServiceUrl,omitempty"` + SecurityPageURL *string `json:"securityPageUrl,omitempty"` + TrustPageURL *string `json:"trustPageUrl,omitempty"` +} + +func main() { + if err := run(); err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } +} + +func run() error { + var ( + pgDSN string + dataPath string + ) + + flag.StringVar( + &pgDSN, + "pg-dsn", + os.Getenv("DATABASE_URL"), + "PostgreSQL connection URL (default: DATABASE_URL env)", + ) + flag.StringVar( + &dataPath, + "data", + "", + "Path to the third-party data.json file", + ) + flag.Parse() + + if pgDSN == "" { + return fmt.Errorf("set -pg-dsn or DATABASE_URL") + } + + ctx := context.Background() + + thirdParties, err := loadThirdParties(dataPath) + if err != nil { + return fmt.Errorf("cannot load third-party data: %w", err) + } + + pgClient, err := newPgClientFromDSN(pgDSN) + if err != nil { + return fmt.Errorf("cannot create pg client: %w", err) + } + + fmt.Printf("importing %d common third parties from %s\n", len(thirdParties), dataPath) + + var inserted, updated int + + if err := pgClient.WithTx( + ctx, + func(ctx context.Context, tx pg.Tx) error { + now := time.Now() + + for _, tp := range thirdParties { + party := coredata.CommonThirdParty{ + ID: gid.New(gid.NilTenant, coredata.CommonThirdPartyEntityType), + Name: tp.Name, + Description: tp.Description, + Category: parseCategory(tp), + HeadquarterAddress: tp.HeadquarterAddress, + LegalName: tp.LegalName, + WebsiteURL: tp.WebsiteURL, + PrivacyPolicyURL: tp.PrivacyPolicyURL, + ServiceLevelAgreementURL: tp.ServiceLevelAgreementURL, + ServiceSoftwareAgreementURL: tp.ServiceSoftwareAgreementURL, + DataProcessingAgreementURL: tp.DataProcessingAgreementURL, + BusinessAssociateAgreementURL: tp.BusinessAssociateAgreementURL, + SubprocessorsListURL: tp.SubprocessorsListURL, + Certifications: tp.Certifications, + StatusPageURL: tp.StatusPageURL, + TermsOfServiceURL: tp.TermsOfServiceURL, + SecurityPageURL: tp.SecurityPageURL, + TrustPageURL: tp.TrustPageURL, + CreatedAt: now, + UpdatedAt: now, + } + + wasInserted, err := party.Upsert(ctx, tx) + if err != nil { + return fmt.Errorf("cannot upsert common third party %q: %w", tp.Name, err) + } + + if wasInserted { + inserted++ + } else { + updated++ + } + } + + return nil + }, + ); err != nil { + return err + } + + fmt.Printf("imported %d rows (%d inserted, %d updated)\n", len(thirdParties), inserted, updated) + + return nil +} + +func loadThirdParties(path string) ([]thirdPartyData, error) { + f, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("cannot open %s: %w", path, err) + } + defer func() { _ = f.Close() }() + + var thirdParties []thirdPartyData + dec := json.NewDecoder(f) + dec.DisallowUnknownFields() + + if err := dec.Decode(&thirdParties); err != nil { + return nil, fmt.Errorf("cannot decode %s: %w", path, err) + } + + return thirdParties, nil +} + +func parseCategory(tp thirdPartyData) coredata.VendorCategory { + if tp.Category == nil || *tp.Category == "" { + return coredata.VendorCategoryOther + } + + var c coredata.VendorCategory + if err := c.Scan(*tp.Category); err != nil { + fmt.Fprintf(os.Stderr, "warning: third party %q has unknown category %q, falling back to OTHER\n", tp.Name, *tp.Category) + return coredata.VendorCategoryOther + } + + return c +} + +func newPgClientFromDSN(dsn string) (*pg.Client, error) { + u, err := url.Parse(dsn) + if err != nil { + return nil, fmt.Errorf("cannot parse DSN: %w", err) + } + + var opts []pg.Option + + if u.Host != "" { + host := u.Host + if u.Port() == "" { + host = net.JoinHostPort(u.Hostname(), "5432") + } + opts = append(opts, pg.WithAddr(host)) + } + + if u.User != nil { + opts = append(opts, pg.WithUser(u.User.Username())) + if password, ok := u.User.Password(); ok { + opts = append(opts, pg.WithPassword(password)) + } + } + + if len(u.Path) > 1 { + opts = append(opts, pg.WithDatabase(u.Path[1:])) + } + + return pg.NewClient(opts...) +} diff --git a/packages/vendors/data.json b/packages/vendors/data.json index 8d68a4271..8d62ca31a 100644 --- a/packages/vendors/data.json +++ b/packages/vendors/data.json @@ -2269,7 +2269,7 @@ "headquarterAddress": "5314 56 St, Camrose, Alberta T4V 2E5, Canada", "websiteUrl": "https://www.danami.com", "description": "Developer of security, firewall and anti-spam extensions for the Plesk hosting control panel.", - "category": "DEVELOPER_TOOLS", + "category": "ENGINEERING", "privacyPolicyUrl": "https://www.danami.com/legal/privacy-policy", "termsOfServiceUrl": "https://www.danami.com/legal/terms-of-service" }, @@ -2432,4 +2432,4 @@ "securityPageUrl": "https://www.scaleway.com/en/security-and-resilience/", "statusPageUrl": "https://status.scaleway.com/" } -] +] \ No newline at end of file diff --git a/pkg/coredata/common_third_party.go b/pkg/coredata/common_third_party.go new file mode 100644 index 000000000..3dd460fbc --- /dev/null +++ b/pkg/coredata/common_third_party.go @@ -0,0 +1,422 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package coredata + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/jackc/pgx/v5" + "go.gearno.de/kit/pg" + "go.probo.inc/probo/pkg/gid" +) + +type ( + CommonThirdParty struct { + ID gid.GID `db:"id"` + Name string `db:"name"` + Description *string `db:"description"` + Category VendorCategory `db:"category"` + HeadquarterAddress *string `db:"headquarter_address"` + LegalName *string `db:"legal_name"` + WebsiteURL *string `db:"website_url"` + PrivacyPolicyURL *string `db:"privacy_policy_url"` + ServiceLevelAgreementURL *string `db:"service_level_agreement_url"` + ServiceSoftwareAgreementURL *string `db:"service_software_agreement_url"` + DataProcessingAgreementURL *string `db:"data_processing_agreement_url"` + BusinessAssociateAgreementURL *string `db:"business_associate_agreement_url"` + SubprocessorsListURL *string `db:"subprocessors_list_url"` + Certifications []string `db:"certifications"` + StatusPageURL *string `db:"status_page_url"` + TermsOfServiceURL *string `db:"terms_of_service_url"` + SecurityPageURL *string `db:"security_page_url"` + TrustPageURL *string `db:"trust_page_url"` + CreatedAt time.Time `db:"created_at"` + UpdatedAt time.Time `db:"updated_at"` + } + + CommonThirdParties []*CommonThirdParty +) + +func (t *CommonThirdParty) LoadByID( + ctx context.Context, + conn pg.Querier, + id gid.GID, +) error { + q := ` +SELECT + id, + name, + description, + category, + headquarter_address, + legal_name, + website_url, + privacy_policy_url, + service_level_agreement_url, + service_software_agreement_url, + data_processing_agreement_url, + business_associate_agreement_url, + subprocessors_list_url, + certifications, + status_page_url, + terms_of_service_url, + security_page_url, + trust_page_url, + created_at, + updated_at +FROM + common_third_parties +WHERE + id = @id +LIMIT 1; +` + + args := pgx.StrictNamedArgs{"id": id} + + rows, err := conn.Query(ctx, q, args) + if err != nil { + return fmt.Errorf("cannot query common third party: %w", err) + } + defer rows.Close() + + row, err := pgx.CollectExactlyOneRow(rows, pgx.RowToStructByName[CommonThirdParty]) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return ErrResourceNotFound + } + return fmt.Errorf("cannot collect common third party: %w", err) + } + + *t = row + + return nil +} + +func (t *CommonThirdParty) LoadByName( + ctx context.Context, + conn pg.Querier, + name string, +) error { + q := ` +SELECT + id, + name, + description, + category, + headquarter_address, + legal_name, + website_url, + privacy_policy_url, + service_level_agreement_url, + service_software_agreement_url, + data_processing_agreement_url, + business_associate_agreement_url, + subprocessors_list_url, + certifications, + status_page_url, + terms_of_service_url, + security_page_url, + trust_page_url, + created_at, + updated_at +FROM + common_third_parties +WHERE + lower(name) = lower(@name) +LIMIT 1; +` + + args := pgx.StrictNamedArgs{"name": name} + + rows, err := conn.Query(ctx, q, args) + if err != nil { + return fmt.Errorf("cannot query common third party by name: %w", err) + } + defer rows.Close() + + row, err := pgx.CollectExactlyOneRow(rows, pgx.RowToStructByName[CommonThirdParty]) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return ErrResourceNotFound + } + return fmt.Errorf("cannot collect common third party by name: %w", err) + } + + *t = row + + return nil +} + +func (t CommonThirdParty) Insert( + ctx context.Context, + conn pg.Tx, +) error { + q := ` +INSERT INTO common_third_parties ( + id, + name, + description, + category, + headquarter_address, + legal_name, + website_url, + privacy_policy_url, + service_level_agreement_url, + service_software_agreement_url, + data_processing_agreement_url, + business_associate_agreement_url, + subprocessors_list_url, + certifications, + status_page_url, + terms_of_service_url, + security_page_url, + trust_page_url, + created_at, + updated_at +) VALUES ( + @id, + @name, + @description, + @category, + @headquarter_address, + @legal_name, + @website_url, + @privacy_policy_url, + @service_level_agreement_url, + @service_software_agreement_url, + @data_processing_agreement_url, + @business_associate_agreement_url, + @subprocessors_list_url, + @certifications, + @status_page_url, + @terms_of_service_url, + @security_page_url, + @trust_page_url, + @created_at, + @updated_at +) +` + + args := pgx.StrictNamedArgs{ + "id": t.ID, + "name": t.Name, + "description": t.Description, + "category": t.Category, + "headquarter_address": t.HeadquarterAddress, + "legal_name": t.LegalName, + "website_url": t.WebsiteURL, + "privacy_policy_url": t.PrivacyPolicyURL, + "service_level_agreement_url": t.ServiceLevelAgreementURL, + "service_software_agreement_url": t.ServiceSoftwareAgreementURL, + "data_processing_agreement_url": t.DataProcessingAgreementURL, + "business_associate_agreement_url": t.BusinessAssociateAgreementURL, + "subprocessors_list_url": t.SubprocessorsListURL, + "certifications": t.Certifications, + "status_page_url": t.StatusPageURL, + "terms_of_service_url": t.TermsOfServiceURL, + "security_page_url": t.SecurityPageURL, + "trust_page_url": t.TrustPageURL, + "created_at": t.CreatedAt, + "updated_at": t.UpdatedAt, + } + + _, err := conn.Exec(ctx, q, args) + if err != nil { + return fmt.Errorf("cannot insert common third party: %w", err) + } + + return nil +} + +// Upsert inserts a row, or on lower(name) conflict updates every column except +// id and created_at. Returns true if a new row was inserted, false if an +// existing row was updated. +func (t CommonThirdParty) Upsert( + ctx context.Context, + conn pg.Tx, +) (inserted bool, err error) { + q := ` +INSERT INTO common_third_parties ( + id, + name, + description, + category, + headquarter_address, + legal_name, + website_url, + privacy_policy_url, + service_level_agreement_url, + service_software_agreement_url, + data_processing_agreement_url, + business_associate_agreement_url, + subprocessors_list_url, + certifications, + status_page_url, + terms_of_service_url, + security_page_url, + trust_page_url, + created_at, + updated_at +) VALUES ( + @id, + @name, + @description, + @category, + @headquarter_address, + @legal_name, + @website_url, + @privacy_policy_url, + @service_level_agreement_url, + @service_software_agreement_url, + @data_processing_agreement_url, + @business_associate_agreement_url, + @subprocessors_list_url, + @certifications, + @status_page_url, + @terms_of_service_url, + @security_page_url, + @trust_page_url, + @created_at, + @updated_at +) +ON CONFLICT (lower(name)) DO UPDATE +SET + name = EXCLUDED.name, + description = EXCLUDED.description, + category = EXCLUDED.category, + headquarter_address = EXCLUDED.headquarter_address, + legal_name = EXCLUDED.legal_name, + website_url = EXCLUDED.website_url, + privacy_policy_url = EXCLUDED.privacy_policy_url, + service_level_agreement_url = EXCLUDED.service_level_agreement_url, + service_software_agreement_url = EXCLUDED.service_software_agreement_url, + data_processing_agreement_url = EXCLUDED.data_processing_agreement_url, + business_associate_agreement_url = EXCLUDED.business_associate_agreement_url, + subprocessors_list_url = EXCLUDED.subprocessors_list_url, + certifications = EXCLUDED.certifications, + status_page_url = EXCLUDED.status_page_url, + terms_of_service_url = EXCLUDED.terms_of_service_url, + security_page_url = EXCLUDED.security_page_url, + trust_page_url = EXCLUDED.trust_page_url, + updated_at = EXCLUDED.updated_at +RETURNING (xmax = 0) AS inserted +` + + args := pgx.StrictNamedArgs{ + "id": t.ID, + "name": t.Name, + "description": t.Description, + "category": t.Category, + "headquarter_address": t.HeadquarterAddress, + "legal_name": t.LegalName, + "website_url": t.WebsiteURL, + "privacy_policy_url": t.PrivacyPolicyURL, + "service_level_agreement_url": t.ServiceLevelAgreementURL, + "service_software_agreement_url": t.ServiceSoftwareAgreementURL, + "data_processing_agreement_url": t.DataProcessingAgreementURL, + "business_associate_agreement_url": t.BusinessAssociateAgreementURL, + "subprocessors_list_url": t.SubprocessorsListURL, + "certifications": t.Certifications, + "status_page_url": t.StatusPageURL, + "terms_of_service_url": t.TermsOfServiceURL, + "security_page_url": t.SecurityPageURL, + "trust_page_url": t.TrustPageURL, + "created_at": t.CreatedAt, + "updated_at": t.UpdatedAt, + } + + rows, err := conn.Query(ctx, q, args) + if err != nil { + return false, fmt.Errorf("cannot upsert common third party: %w", err) + } + defer rows.Close() + + inserted, err = pgx.CollectExactlyOneRow(rows, pgx.RowTo[bool]) + if err != nil { + return false, fmt.Errorf("cannot collect upsert result: %w", err) + } + + return inserted, nil +} + +func (t CommonThirdParty) Delete( + ctx context.Context, + conn pg.Tx, + id gid.GID, +) error { + q := `DELETE FROM common_third_parties WHERE id = @id` + + args := pgx.StrictNamedArgs{"id": id} + + result, err := conn.Exec(ctx, q, args) + if err != nil { + return fmt.Errorf("cannot delete common third party: %w", err) + } + + if result.RowsAffected() == 0 { + return ErrResourceNotFound + } + + return nil +} + +func (t *CommonThirdParties) LoadAll( + ctx context.Context, + conn pg.Querier, +) error { + q := ` +SELECT + id, + name, + description, + category, + headquarter_address, + legal_name, + website_url, + privacy_policy_url, + service_level_agreement_url, + service_software_agreement_url, + data_processing_agreement_url, + business_associate_agreement_url, + subprocessors_list_url, + certifications, + status_page_url, + terms_of_service_url, + security_page_url, + trust_page_url, + created_at, + updated_at +FROM + common_third_parties +ORDER BY name ASC +` + + rows, err := conn.Query(ctx, q) + if err != nil { + return fmt.Errorf("cannot query common third parties: %w", err) + } + + parties, err := pgx.CollectRows(rows, pgx.RowToAddrOfStructByName[CommonThirdParty]) + if err != nil { + return fmt.Errorf("cannot collect common third parties: %w", err) + } + + *t = parties + + return nil +} diff --git a/pkg/coredata/entity_type_reg.go b/pkg/coredata/entity_type_reg.go index 1de0c38b1..caafad862 100644 --- a/pkg/coredata/entity_type_reg.go +++ b/pkg/coredata/entity_type_reg.go @@ -115,6 +115,7 @@ const ( TrackerPatternEntityType uint16 = 89 DetectedTrackerEntityType uint16 = 90 TrackerResourceEntityType uint16 = 91 + CommonThirdPartyEntityType uint16 = 92 ) func NewEntityFromID(id gid.GID) (any, bool) { @@ -287,6 +288,8 @@ func NewEntityFromID(id gid.GID) (any, bool) { return &DetectedTracker{ID: id}, true case TrackerResourceEntityType: return &TrackerResource{ID: id}, true + case CommonThirdPartyEntityType: + return &CommonThirdParty{ID: id}, true default: return nil, false } diff --git a/pkg/coredata/migrations/20260508T083200Z.sql b/pkg/coredata/migrations/20260508T083200Z.sql new file mode 100644 index 000000000..4aab4baf9 --- /dev/null +++ b/pkg/coredata/migrations/20260508T083200Z.sql @@ -0,0 +1,39 @@ +-- Copyright (c) 2026 Probo Inc . +-- +-- Permission to use, copy, modify, and/or distribute this software for any +-- purpose with or without fee is hereby granted, provided that the above +-- copyright notice and this permission notice appear in all copies. +-- +-- THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +-- REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +-- AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +-- INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +-- LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +-- OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +-- PERFORMANCE OF THIS SOFTWARE. + +CREATE TABLE common_third_parties ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + description TEXT, + category vendor_category NOT NULL, + headquarter_address TEXT, + legal_name TEXT, + website_url TEXT, + privacy_policy_url TEXT, + service_level_agreement_url TEXT, + service_software_agreement_url TEXT, + data_processing_agreement_url TEXT, + business_associate_agreement_url TEXT, + subprocessors_list_url TEXT, + certifications TEXT[], + status_page_url TEXT, + terms_of_service_url TEXT, + security_page_url TEXT, + trust_page_url TEXT, + created_at TIMESTAMP WITH TIME ZONE NOT NULL, + updated_at TIMESTAMP WITH TIME ZONE NOT NULL +); + +CREATE UNIQUE INDEX common_third_parties_name_key + ON common_third_parties (lower(name));