Refacto load all functions

Unbounded LoadAll* loaders materialised an entire result set in one
query with no ceiling. A table that is small in development can grow
without bound in production, so these loaders were a latent memory
and query-time hazard.

Remove the LoadAll* methods from pkg/coredata and walk the cursor-
paginated LoadBy* siblings instead through a shared page.LoadAll
helper. The helper advances a MaxCursorSize forward cursor until the
result set is exhausted and concatenates the pages. It caps a single
call at MaxLoadAllPages (20) batches of 500 rows and errors past that
rather than materialising an unbounded set, so a runaway caller fails
loudly instead of exhausting memory.

Callers that genuinely need every row now express that explicitly,
and the coredata load-naming rule and docs are updated to discourage
new unbounded loaders.

Signed-off-by: Sacha Al Himdani <sacha@probo.com>
This commit is contained in:
Sacha Al Himdani
2026-06-09 19:33:36 +02:00
committed by Sacha Al Himdani
parent 853f2404a6
commit 9ab8ea2085
46 changed files with 1218 additions and 1674 deletions

View File

@@ -295,59 +295,6 @@ LIMIT @limit;
return ids, nil
}
// LoadAllByTrackerPatternID returns every detected tracker linked to the
// pattern, with no pagination. It backs the banner-reset rebuild, which
// recreates exact patterns from a glob's detections.
func (dts *DetectedTrackers) LoadAllByTrackerPatternID(
ctx context.Context,
conn pg.Querier,
scope Scoper,
trackerPatternID gid.GID,
) error {
q := `
SELECT
id,
cookie_banner_id,
tracker_pattern_id,
tracker_type,
identifier,
max_age_seconds,
source,
value_size,
initiator_url,
initiator_domain,
last_detected_at,
created_at,
updated_at
FROM
detected_trackers
WHERE
%s
AND tracker_pattern_id = @tracker_pattern_id
ORDER BY
identifier ASC, id ASC
`
q = fmt.Sprintf(q, scope.SQLFragment())
args := pgx.StrictNamedArgs{"tracker_pattern_id": trackerPatternID}
maps.Copy(args, scope.SQLArguments())
rows, err := conn.Query(ctx, q, args)
if err != nil {
return fmt.Errorf("cannot query detected trackers: %w", err)
}
trackers, err := pgx.CollectRows(rows, pgx.RowToAddrOfStructByName[DetectedTracker])
if err != nil {
return fmt.Errorf("cannot collect detected trackers: %w", err)
}
*dts = trackers
return nil
}
// UpdateTrackerPatternID repoints a single detected tracker at another
// pattern. It is the per-row counterpart of RelinkByTrackerPatternID,
// used by the banner-reset rebuild where each detection of a glob moves