rohit/probo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Dedupe org third parties and add unique index

Browse Source 
Add the partial unique index that enforces one org ThirdParty per catalog
vendor per organization, on (organization_id, common_third_party_id)
where common_third_party_id is not null. Manually created vendors (null
common id) stay unconstrained.

Existing data may already hold duplicates left by the old auto-create
race, which would make the index build fail, so a preceding DO block
merges them first. It maps each duplicate onto the earliest-created
survivor and, driven by pg_constraint / pg_index introspection, repoints
every foreign key that references third_parties(id) onto the survivor,
removing link rows that would collide on a referencing table's unique
key before repointing, then deletes the extras.

The dedupe is destructive and introspection-driven; validate it against a
production dump before deploying.

Signed-off-by: Émile Ré <emile@probo.com>
This commit is contained in:
Émile Ré
2026-06-10 14:53:55 +02:00
parent 27ff38df7d
commit 11c74617d0
1 changed files with 134 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

134
pkg/coredata/migrations/20260611T120000Z.sql Normal file
View File

@@ -0,0 +1,134 @@
-- Copyright (c) 2026 Probo Inc <hello@probo.com>.
--
-- Permission to use, copy, modify, and/or distribute this software for any
-- purpose with or without fee is hereby granted, provided that the above
-- copyright notice and this permission notice appear in all copies.
--
-- THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
-- REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
-- AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
-- INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
-- LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
-- OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
-- PERFORMANCE OF THIS SOFTWARE.
-- Enforce a single org ThirdParty per catalog vendor per organization.
--
-- The tracker-mapping worker used to materialize org third parties from
-- the catalog and could race the load-then-create check, leaving several
-- rows that share the same (organization_id, common_third_party_id).
-- Org third parties are now created only through the explicit import
-- action; this migration both cleans up the historical duplicates and
-- adds the partial unique index that makes the invariant enforceable.
--
-- DESTRUCTIVE: the DO block below merges duplicate org third parties onto
-- the earliest-created survivor by repointing every foreign key that
-- references third_parties(id) and then deleting the extras. It is driven
-- by catalog introspection (pg_constraint / pg_index) so it covers every
-- referencing table without hard-coding the list, and it removes link
-- rows that would collide on a referencing table's unique key before
-- repointing. Validate it against a production dump before deploying.
DO $$
DECLARE
fk RECORD;
uq RECORD;
has_dupes boolean;
BEGIN
-- Map each duplicate row to the survivor it should be merged into:
-- the earliest-created row for the (organization_id,
-- common_third_party_id) pair, ties broken by id.
CREATE TEMP TABLE _tp_dedupe ON COMMIT DROP AS
SELECT t.id AS dup_id, k.survivor_id
FROM third_parties t
JOIN (
SELECT
organization_id,
common_third_party_id,
(array_agg(id ORDER BY created_at ASC, id ASC))[1] AS survivor_id
FROM third_parties
WHERE common_third_party_id IS NOT NULL
GROUP BY organization_id, common_third_party_id
HAVING count(*) > 1
) k
ON t.organization_id = k.organization_id
AND t.common_third_party_id = k.common_third_party_id
WHERE t.id <> k.survivor_id;
SELECT EXISTS (SELECT 1 FROM _tp_dedupe) INTO has_dupes;
IF has_dupes THEN
-- For every single-column foreign key that references
-- third_parties(id) ...
FOR fk IN
SELECT c.conrelid::regclass::text AS tbl,
a.attname::text AS col
FROM pg_constraint c
JOIN pg_attribute a
ON a.attrelid = c.conrelid
AND a.attnum = c.conkey[1]
WHERE c.contype = 'f'
AND c.confrelid = 'third_parties'::regclass
AND array_length(c.conkey, 1) = 1
LOOP
-- ... drop duplicate-side rows that would collide with a
-- survivor-side row on any unique key that includes the FK
-- column (the survivor's row wins; this realizes the union of
-- the two vendors' links).
FOR uq IN
SELECT (
SELECT string_agg(
format('s.%I IS NOT DISTINCT FROM d.%I',
att.attname, att.attname),
' AND ')
FROM pg_attribute att
WHERE att.attrelid = i.indrelid
AND att.attnum = ANY (i.indkey)
AND att.attname <> fk.col
) AS match_pred
FROM pg_index i
WHERE i.indrelid = fk.tbl::regclass
AND (i.indisunique OR i.indisprimary)
AND EXISTS (
SELECT 1
FROM pg_attribute att
WHERE att.attrelid = i.indrelid
AND att.attnum = ANY (i.indkey)
AND att.attname = fk.col
)
LOOP
IF uq.match_pred IS NOT NULL AND length(uq.match_pred) > 0 THEN
EXECUTE format(
'DELETE FROM %1$s d
USING _tp_dedupe m
WHERE d.%2$I = m.dup_id
AND EXISTS (
SELECT 1 FROM %1$s s
WHERE s.%2$I = m.survivor_id
AND %3$s
)',
fk.tbl, fk.col, uq.match_pred
);
END IF;
END LOOP;
-- Repoint the remaining references onto the survivor.
EXECUTE format(
'UPDATE %1$s d
SET %2$I = m.survivor_id
FROM _tp_dedupe m
WHERE d.%2$I = m.dup_id',
fk.tbl, fk.col
);
END LOOP;
-- Drop the now-unreferenced duplicate rows.
DELETE FROM third_parties t
USING _tp_dedupe m
WHERE t.id = m.dup_id;
END IF;
END $$;
CREATE UNIQUE INDEX third_parties_org_common_key
ON third_parties (organization_id, common_third_party_id)
WHERE common_third_party_id IS NOT NULL;