Refacto load all functions
Unbounded LoadAll* loaders materialised an entire result set in one query with no ceiling. A table that is small in development can grow without bound in production, so these loaders were a latent memory and query-time hazard. Remove the LoadAll* methods from pkg/coredata and walk the cursor- paginated LoadBy* siblings instead through a shared page.LoadAll helper. The helper advances a MaxCursorSize forward cursor until the result set is exhausted and concatenates the pages. It caps a single call at MaxLoadAllPages (20) batches of 500 rows and errors past that rather than materialising an unbounded set, so a runaway caller fails loudly instead of exhausting memory. Callers that genuinely need every row now express that explicitly, and the coredata load-naming rule and docs are updated to discourage new unbounded loaders. Signed-off-by: Sacha Al Himdani <sacha@probo.com>
This commit is contained in:
committed by
Sacha Al Himdani
parent
853f2404a6
commit
9ab8ea2085
@@ -377,64 +377,6 @@ SELECT * FROM base WHERE %s
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Documents) LoadAllByOrganizationID(
|
||||
ctx context.Context,
|
||||
conn pg.Querier,
|
||||
scope Scoper,
|
||||
organizationID gid.GID,
|
||||
filter *DocumentFilter,
|
||||
) error {
|
||||
q := `
|
||||
WITH latest_versions AS (
|
||||
SELECT DISTINCT ON (document_id) document_id, title, document_type
|
||||
FROM document_versions
|
||||
ORDER BY document_id, major DESC, minor DESC
|
||||
)
|
||||
SELECT
|
||||
documents.id,
|
||||
documents.organization_id,
|
||||
documents.current_published_major,
|
||||
documents.current_published_minor,
|
||||
documents.write_mode,
|
||||
documents.trust_center_visibility,
|
||||
documents.status,
|
||||
documents.archived_at,
|
||||
documents.created_at,
|
||||
documents.updated_at,
|
||||
COALESCE(lv.title, '') AS title,
|
||||
COALESCE(lv.document_type, 'OTHER') AS document_type
|
||||
FROM
|
||||
documents
|
||||
LEFT JOIN latest_versions lv ON lv.document_id = documents.id
|
||||
WHERE
|
||||
%s
|
||||
AND documents.deleted_at IS NULL
|
||||
AND documents.organization_id = @organization_id
|
||||
AND %s
|
||||
ORDER BY title ASC
|
||||
`
|
||||
|
||||
q = fmt.Sprintf(q, scope.SQLFragment(), filter.SQLFragment())
|
||||
|
||||
args := pgx.NamedArgs{"organization_id": organizationID}
|
||||
maps.Copy(args, scope.SQLArguments())
|
||||
maps.Copy(args, filter.SQLArguments())
|
||||
|
||||
rows, err := conn.Query(ctx, q, args)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot query documents: %w", err)
|
||||
}
|
||||
|
||||
documents, err := pgx.CollectRows(rows, pgx.RowToAddrOfStructByName[Document])
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot collect documents: %w", err)
|
||||
}
|
||||
|
||||
*p = documents
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Documents) LoadPublishedByOrganizationID(
|
||||
ctx context.Context,
|
||||
conn pg.Querier,
|
||||
|
||||
Reference in New Issue
Block a user