Refacto load all functions
Unbounded LoadAll* loaders materialised an entire result set in one query with no ceiling. A table that is small in development can grow without bound in production, so these loaders were a latent memory and query-time hazard. Remove the LoadAll* methods from pkg/coredata and walk the cursor- paginated LoadBy* siblings instead through a shared page.LoadAll helper. The helper advances a MaxCursorSize forward cursor until the result set is exhausted and concatenates the pages. It caps a single call at MaxLoadAllPages (20) batches of 500 rows and errors past that rather than materialising an unbounded set, so a runaway caller fails loudly instead of exhausting memory. Callers that genuinely need every row now express that explicitly, and the coredata load-naming rule and docs are updated to discourage new unbounded loaders. Signed-off-by: Sacha Al Himdani <sacha@probo.com>
This commit is contained in:
Sacha Al Himdani
committed by
Sacha Al Himdani
Sacha Al Himdani
parent
853f2404a6
commit
9ab8ea2085
@@ -776,76 +776,6 @@ WHERE
|
||||
return count, nil
|
||||
}
|
||||
|
||||
func (v *ThirdParties) LoadAllByOrganizationID(
|
||||
ctx context.Context,
|
||||
conn pg.Querier,
|
||||
scope Scoper,
|
||||
organizationID gid.GID,
|
||||
filter *ThirdPartyFilter,
|
||||
) error {
|
||||
q := `
|
||||
SELECT
|
||||
id,
|
||||
organization_id,
|
||||
parent_third_party_id,
|
||||
common_third_party_id,
|
||||
name,
|
||||
description,
|
||||
category,
|
||||
headquarter_address,
|
||||
legal_name,
|
||||
website_url,
|
||||
privacy_policy_url,
|
||||
service_level_agreement_url,
|
||||
data_processing_agreement_url,
|
||||
business_associate_agreement_url,
|
||||
subprocessors_list_url,
|
||||
certifications,
|
||||
countries,
|
||||
business_owner_profile_id,
|
||||
security_owner_profile_id,
|
||||
status_page_url,
|
||||
terms_of_service_url,
|
||||
security_page_url,
|
||||
trust_page_url,
|
||||
show_on_trust_center,
|
||||
level,
|
||||
vetting_status,
|
||||
vetting_website_url,
|
||||
vetting_procedure,
|
||||
vetting_processing_started_at,
|
||||
vetting_error_message,
|
||||
created_at,
|
||||
updated_at
|
||||
FROM
|
||||
third_parties
|
||||
WHERE
|
||||
%s
|
||||
AND organization_id = @organization_id
|
||||
AND %s
|
||||
ORDER BY name ASC
|
||||
`
|
||||
q = fmt.Sprintf(q, scope.SQLFragment(), filter.SQLFragment())
|
||||
|
||||
args := pgx.StrictNamedArgs{"organization_id": organizationID}
|
||||
maps.Copy(args, scope.SQLArguments())
|
||||
maps.Copy(args, filter.SQLArguments())
|
||||
|
||||
rows, err := conn.Query(ctx, q, args)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot query thirdParties: %w", err)
|
||||
}
|
||||
|
||||
thirdParties, err := pgx.CollectRows(rows, pgx.RowToAddrOfStructByName[ThirdParty])
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot collect thirdParties: %w", err)
|
||||
}
|
||||
|
||||
*v = thirdParties
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (v *ThirdParties) LoadByOrganizationID(
|
||||
ctx context.Context,
|
||||
conn pg.Querier,
|
||||
@@ -1230,113 +1160,6 @@ WHERE %s
|
||||
return count, nil
|
||||
}
|
||||
|
||||
func (vs *ThirdParties) LoadAllByDatumID(
|
||||
ctx context.Context,
|
||||
conn pg.Querier,
|
||||
scope Scoper,
|
||||
datumID gid.GID,
|
||||
) error {
|
||||
q := `
|
||||
WITH vend AS (
|
||||
SELECT
|
||||
v.id,
|
||||
v.tenant_id,
|
||||
v.organization_id,
|
||||
v.parent_third_party_id,
|
||||
v.common_third_party_id,
|
||||
v.name,
|
||||
v.description,
|
||||
v.category,
|
||||
v.headquarter_address,
|
||||
v.legal_name,
|
||||
v.website_url,
|
||||
v.privacy_policy_url,
|
||||
v.service_level_agreement_url,
|
||||
v.data_processing_agreement_url,
|
||||
v.business_associate_agreement_url,
|
||||
v.subprocessors_list_url,
|
||||
v.certifications,
|
||||
v.countries,
|
||||
v.business_owner_profile_id,
|
||||
v.security_owner_profile_id,
|
||||
v.status_page_url,
|
||||
v.terms_of_service_url,
|
||||
v.security_page_url,
|
||||
v.trust_page_url,
|
||||
v.show_on_trust_center,
|
||||
v.level,
|
||||
v.vetting_status,
|
||||
v.vetting_website_url,
|
||||
v.vetting_procedure,
|
||||
v.vetting_processing_started_at,
|
||||
v.vetting_error_message,
|
||||
v.created_at,
|
||||
v.updated_at
|
||||
FROM
|
||||
third_parties v
|
||||
INNER JOIN
|
||||
data_third_parties dv ON v.id = dv.third_party_id
|
||||
WHERE
|
||||
dv.datum_id = @datum_id
|
||||
)
|
||||
SELECT
|
||||
id,
|
||||
organization_id,
|
||||
parent_third_party_id,
|
||||
common_third_party_id,
|
||||
name,
|
||||
description,
|
||||
category,
|
||||
headquarter_address,
|
||||
legal_name,
|
||||
website_url,
|
||||
privacy_policy_url,
|
||||
service_level_agreement_url,
|
||||
data_processing_agreement_url,
|
||||
business_associate_agreement_url,
|
||||
subprocessors_list_url,
|
||||
certifications,
|
||||
countries,
|
||||
business_owner_profile_id,
|
||||
security_owner_profile_id,
|
||||
status_page_url,
|
||||
terms_of_service_url,
|
||||
security_page_url,
|
||||
trust_page_url,
|
||||
show_on_trust_center,
|
||||
level,
|
||||
vetting_status,
|
||||
vetting_website_url,
|
||||
vetting_procedure,
|
||||
vetting_processing_started_at,
|
||||
vetting_error_message,
|
||||
created_at,
|
||||
updated_at
|
||||
FROM
|
||||
vend
|
||||
WHERE %s
|
||||
ORDER BY name ASC
|
||||
`
|
||||
q = fmt.Sprintf(q, scope.SQLFragment())
|
||||
|
||||
args := pgx.StrictNamedArgs{"datum_id": datumID}
|
||||
maps.Copy(args, scope.SQLArguments())
|
||||
|
||||
rows, err := conn.Query(ctx, q, args)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot query thirdParties: %w", err)
|
||||
}
|
||||
|
||||
thirdParties, err := pgx.CollectRows(rows, pgx.RowToAddrOfStructByName[ThirdParty])
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot collect thirdParties: %w", err)
|
||||
}
|
||||
|
||||
*vs = thirdParties
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (vs *ThirdParties) LoadByDatumID(
|
||||
ctx context.Context,
|
||||
conn pg.Querier,
|
||||
@@ -1624,113 +1447,6 @@ ORDER BY
|
||||
return thirdPartyMap, nil
|
||||
}
|
||||
|
||||
func (vs *ThirdParties) LoadAllByAssetID(
|
||||
ctx context.Context,
|
||||
conn pg.Querier,
|
||||
scope Scoper,
|
||||
assetID gid.GID,
|
||||
) error {
|
||||
q := `
|
||||
WITH vend AS (
|
||||
SELECT
|
||||
v.id,
|
||||
v.tenant_id,
|
||||
v.organization_id,
|
||||
v.parent_third_party_id,
|
||||
v.common_third_party_id,
|
||||
v.name,
|
||||
v.description,
|
||||
v.category,
|
||||
v.headquarter_address,
|
||||
v.legal_name,
|
||||
v.website_url,
|
||||
v.privacy_policy_url,
|
||||
v.service_level_agreement_url,
|
||||
v.data_processing_agreement_url,
|
||||
v.business_associate_agreement_url,
|
||||
v.subprocessors_list_url,
|
||||
v.certifications,
|
||||
v.countries,
|
||||
v.business_owner_profile_id,
|
||||
v.security_owner_profile_id,
|
||||
v.status_page_url,
|
||||
v.terms_of_service_url,
|
||||
v.security_page_url,
|
||||
v.trust_page_url,
|
||||
v.show_on_trust_center,
|
||||
v.level,
|
||||
v.vetting_status,
|
||||
v.vetting_website_url,
|
||||
v.vetting_procedure,
|
||||
v.vetting_processing_started_at,
|
||||
v.vetting_error_message,
|
||||
v.created_at,
|
||||
v.updated_at
|
||||
FROM
|
||||
third_parties v
|
||||
INNER JOIN
|
||||
asset_third_parties av ON v.id = av.third_party_id
|
||||
WHERE
|
||||
av.asset_id = @asset_id
|
||||
)
|
||||
SELECT
|
||||
id,
|
||||
organization_id,
|
||||
parent_third_party_id,
|
||||
common_third_party_id,
|
||||
name,
|
||||
description,
|
||||
category,
|
||||
headquarter_address,
|
||||
legal_name,
|
||||
website_url,
|
||||
privacy_policy_url,
|
||||
service_level_agreement_url,
|
||||
data_processing_agreement_url,
|
||||
business_associate_agreement_url,
|
||||
subprocessors_list_url,
|
||||
certifications,
|
||||
countries,
|
||||
business_owner_profile_id,
|
||||
security_owner_profile_id,
|
||||
status_page_url,
|
||||
terms_of_service_url,
|
||||
security_page_url,
|
||||
trust_page_url,
|
||||
show_on_trust_center,
|
||||
level,
|
||||
vetting_status,
|
||||
vetting_website_url,
|
||||
vetting_procedure,
|
||||
vetting_processing_started_at,
|
||||
vetting_error_message,
|
||||
created_at,
|
||||
updated_at
|
||||
FROM
|
||||
vend
|
||||
WHERE %s
|
||||
ORDER BY name ASC
|
||||
`
|
||||
q = fmt.Sprintf(q, scope.SQLFragment())
|
||||
|
||||
args := pgx.StrictNamedArgs{"asset_id": assetID}
|
||||
maps.Copy(args, scope.SQLArguments())
|
||||
|
||||
rows, err := conn.Query(ctx, q, args)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot query thirdParties: %w", err)
|
||||
}
|
||||
|
||||
thirdParties, err := pgx.CollectRows(rows, pgx.RowToAddrOfStructByName[ThirdParty])
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot collect thirdParties: %w", err)
|
||||
}
|
||||
|
||||
*vs = thirdParties
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (v *ThirdParty) LoadByOrganizationIDAndCommonThirdPartyID(
|
||||
ctx context.Context,
|
||||
conn pg.Querier,
|
||||
|
||||
Reference in New Issue
Block a user