Files
probo/pkg/coredata/processing_activities.go
Sacha Al Himdani 9ab8ea2085 Refacto load all functions
Unbounded LoadAll* loaders materialised an entire result set in one
query with no ceiling. A table that is small in development can grow
without bound in production, so these loaders were a latent memory
and query-time hazard.

Remove the LoadAll* methods from pkg/coredata and walk the cursor-
paginated LoadBy* siblings instead through a shared page.LoadAll
helper. The helper advances a MaxCursorSize forward cursor until the
result set is exhausted and concatenates the pages. It caps a single
call at MaxLoadAllPages (20) batches of 500 rows and errors past that
rather than materialising an unbounded set, so a runaway caller fails
loudly instead of exhausting memory.

Callers that genuinely need every row now express that explicitly,
and the coredata load-naming rule and docs are updated to discourage
new unbounded loaders.

Signed-off-by: Sacha Al Himdani <sacha@probo.com>
2026-06-16 14:35:16 +02:00

626 lines
16 KiB
Go

// Copyright (c) 2025-2026 Probo Inc <hello@probo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package coredata
import (
"context"
"errors"
"fmt"
"maps"
"time"
"github.com/jackc/pgx/v5"
"go.gearno.de/kit/pg"
"go.probo.inc/probo/pkg/gid"
"go.probo.inc/probo/pkg/iam/policy"
"go.probo.inc/probo/pkg/page"
)
func (p ProcessingActivity) GetGeneratedDocumentID(
ctx context.Context,
conn pg.Querier,
organizationID gid.GID,
) (*gid.GID, error) {
var documentID *gid.GID
err := conn.QueryRow(
ctx,
`
SELECT
processing_activities_document_id
FROM
generated_documents
WHERE
organization_id = @organization_id
`,
pgx.NamedArgs{"organization_id": organizationID},
).Scan(&documentID)
if errors.Is(err, pgx.ErrNoRows) {
return nil, nil
}
if err != nil {
return nil, fmt.Errorf("cannot get processing activity list document ID: %w", err)
}
return documentID, nil
}
func (p ProcessingActivity) UpsertGeneratedDocumentID(
ctx context.Context,
conn pg.Tx,
organizationID gid.GID,
tenantID gid.TenantID,
documentID gid.GID,
) error {
now := time.Now()
_, err := conn.Exec(
ctx,
`
INSERT INTO generated_documents (
organization_id,
tenant_id,
processing_activities_document_id,
created_at,
updated_at
) VALUES (
@organization_id,
@tenant_id,
@processing_activities_document_id,
@created_at,
@updated_at
)
ON CONFLICT (organization_id) DO UPDATE
SET
processing_activities_document_id = @processing_activities_document_id,
updated_at = @updated_at
`,
pgx.NamedArgs{
"organization_id": organizationID,
"tenant_id": tenantID,
"processing_activities_document_id": documentID,
"created_at": now,
"updated_at": now,
},
)
if err != nil {
return fmt.Errorf("cannot upsert processing activity list document ID: %w", err)
}
return nil
}
func (p ProcessingActivity) ClearGeneratedDocumentID(
ctx context.Context,
conn pg.Tx,
documentIDs []gid.GID,
) error {
ids := make([]string, len(documentIDs))
for i, id := range documentIDs {
ids[i] = id.String()
}
_, err := conn.Exec(
ctx,
`
UPDATE
generated_documents
SET
processing_activities_document_id = NULL,
updated_at = @now
WHERE
processing_activities_document_id = ANY(@ids)
`,
pgx.NamedArgs{
"ids": ids,
"now": time.Now(),
},
)
if err != nil {
return fmt.Errorf("cannot clear processing activity list document references: %w", err)
}
return nil
}
type (
ProcessingActivity struct {
ID gid.GID `db:"id"`
OrganizationID gid.GID `db:"organization_id"`
Name string `db:"name"`
Purpose *string `db:"purpose"`
DataSubjectCategory *string `db:"data_subject_category"`
PersonalDataCategory *string `db:"personal_data_category"`
SpecialOrCriminalData ProcessingActivitySpecialOrCriminalDatum `db:"special_or_criminal_data"`
ConsentEvidenceLink *string `db:"consent_evidence_link"`
LawfulBasis ProcessingActivityLawfulBasis `db:"lawful_basis"`
Recipients *string `db:"recipients"`
Location *string `db:"location"`
InternationalTransfers bool `db:"international_transfers"`
TransferSafeguard *ProcessingActivityTransferSafeguard `db:"transfer_safeguards"`
RetentionPeriod *string `db:"retention_period"`
SecurityMeasures *string `db:"security_measures"`
DataProtectionImpactAssessmentNeeded ProcessingActivityDataProtectionImpactAssessment `db:"data_protection_impact_assessment_needed"`
TransferImpactAssessmentNeeded ProcessingActivityTransferImpactAssessment `db:"transfer_impact_assessment_needed"`
LastReviewDate *time.Time `db:"last_review_date"`
NextReviewDate *time.Time `db:"next_review_date"`
Role ProcessingActivityRole `db:"role"`
DataProtectionOfficerID *gid.GID `db:"dpo_profile_id"`
CreatedAt time.Time `db:"created_at"`
UpdatedAt time.Time `db:"updated_at"`
}
ProcessingActivities []*ProcessingActivity
)
func (p *ProcessingActivity) CursorKey(field ProcessingActivityOrderField) page.CursorKey {
switch field {
case ProcessingActivityOrderFieldCreatedAt:
return page.NewCursorKey(p.ID, p.CreatedAt)
case ProcessingActivityOrderFieldName:
return page.NewCursorKey(p.ID, p.Name)
}
panic(fmt.Sprintf("unsupported order by: %s", field))
}
func (p *ProcessingActivity) AuthorizationAttributes(
ctx context.Context,
conn pg.Querier,
resourceIDs []gid.GID,
) (policy.AttributesByID, error) {
q := `SELECT id, organization_id FROM processing_activities WHERE id = ANY(@resource_ids::text[])`
args := pgx.StrictNamedArgs{
"resource_ids": resourceIDs,
}
rows, err := conn.Query(ctx, q, args)
if err != nil {
return nil, fmt.Errorf("cannot query authorization attributes: %w", err)
}
defer rows.Close()
attrsByID := make(policy.AttributesByID)
for rows.Next() {
var id, organizationID gid.GID
if err := rows.Scan(&id, &organizationID); err != nil {
return nil, fmt.Errorf("cannot scan authorization attributes: %w", err)
}
attrsByID[id] = policy.Attributes{
"organization_id": organizationID.String(),
}
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("cannot iterate authorization attributes: %w", err)
}
return attrsByID, nil
}
func (p *ProcessingActivity) LoadByID(
ctx context.Context,
conn pg.Querier,
scope Scoper,
processingActivityID gid.GID,
) error {
q := `
SELECT
id,
organization_id,
name,
purpose,
data_subject_category,
personal_data_category,
special_or_criminal_data,
consent_evidence_link,
lawful_basis,
recipients,
location,
international_transfers,
transfer_safeguards,
retention_period,
security_measures,
data_protection_impact_assessment_needed,
transfer_impact_assessment_needed,
last_review_date,
next_review_date,
role,
dpo_profile_id,
created_at,
updated_at
FROM
processing_activities
WHERE
%s
AND id = @processing_activity_id
LIMIT 1;
`
q = fmt.Sprintf(q, scope.SQLFragment())
args := pgx.StrictNamedArgs{"processing_activity_id": processingActivityID}
maps.Copy(args, scope.SQLArguments())
rows, err := conn.Query(ctx, q, args)
if err != nil {
return fmt.Errorf("cannot query processing activity: %w", err)
}
processingActivity, err := pgx.CollectExactlyOneRow(rows, pgx.RowToStructByName[ProcessingActivity])
if err != nil {
return fmt.Errorf("cannot collect processing activity: %w", err)
}
*p = processingActivity
return nil
}
func (p *ProcessingActivities) CountByOrganizationID(
ctx context.Context,
conn pg.Querier,
scope Scoper,
organizationID gid.GID,
) (int, error) {
q := `
SELECT
COUNT(id)
FROM
processing_activities
WHERE
%s
AND organization_id = @organization_id
`
q = fmt.Sprintf(q, scope.SQLFragment())
args := pgx.StrictNamedArgs{"organization_id": organizationID}
maps.Copy(args, scope.SQLArguments())
row := conn.QueryRow(ctx, q, args)
var count int
err := row.Scan(&count)
if err != nil {
return 0, fmt.Errorf("cannot count processing activities: %w", err)
}
return count, nil
}
func (p *ProcessingActivities) LoadByIDs(
ctx context.Context,
conn pg.Querier,
scope Scoper,
processingActivityIDs []gid.GID,
) error {
if len(processingActivityIDs) == 0 {
*p = ProcessingActivities{}
return nil
}
q := `
SELECT
id,
organization_id,
name,
purpose,
data_subject_category,
personal_data_category,
special_or_criminal_data,
consent_evidence_link,
lawful_basis,
recipients,
location,
international_transfers,
transfer_safeguards,
retention_period,
security_measures,
data_protection_impact_assessment_needed,
transfer_impact_assessment_needed,
last_review_date,
next_review_date,
role,
dpo_profile_id,
created_at,
updated_at
FROM
processing_activities
WHERE
%s
AND id = ANY(@ids)
`
q = fmt.Sprintf(q, scope.SQLFragment())
ids := make([]string, len(processingActivityIDs))
for i, id := range processingActivityIDs {
ids[i] = id.String()
}
args := pgx.StrictNamedArgs{"ids": ids}
maps.Copy(args, scope.SQLArguments())
rows, err := conn.Query(ctx, q, args)
if err != nil {
return fmt.Errorf("cannot query processing activities: %w", err)
}
processingActivities, err := pgx.CollectRows(rows, pgx.RowToAddrOfStructByName[ProcessingActivity])
if err != nil {
return fmt.Errorf("cannot collect processing activities: %w", err)
}
*p = processingActivities
return nil
}
func (p *ProcessingActivities) LoadByOrganizationID(
ctx context.Context,
conn pg.Querier,
scope Scoper,
organizationID gid.GID,
cursor *page.Cursor[ProcessingActivityOrderField],
) error {
q := `
SELECT
id,
organization_id,
name,
purpose,
data_subject_category,
personal_data_category,
special_or_criminal_data,
consent_evidence_link,
lawful_basis,
recipients,
location,
international_transfers,
transfer_safeguards,
retention_period,
security_measures,
data_protection_impact_assessment_needed,
transfer_impact_assessment_needed,
last_review_date,
next_review_date,
role,
dpo_profile_id,
created_at,
updated_at
FROM
processing_activities
WHERE
%s
AND organization_id = @organization_id
AND %s
`
q = fmt.Sprintf(q, scope.SQLFragment(), cursor.SQLFragment())
args := pgx.StrictNamedArgs{"organization_id": organizationID}
maps.Copy(args, scope.SQLArguments())
maps.Copy(args, cursor.SQLArguments())
rows, err := conn.Query(ctx, q, args)
if err != nil {
return fmt.Errorf("cannot query processing activities: %w", err)
}
processingActivities, err := pgx.CollectRows(rows, pgx.RowToAddrOfStructByName[ProcessingActivity])
if err != nil {
return fmt.Errorf("cannot collect processing activities: %w", err)
}
*p = processingActivities
return nil
}
func (p *ProcessingActivity) Insert(
ctx context.Context,
conn pg.Tx,
scope Scoper,
) error {
q := `
INSERT INTO processing_activities (
id,
tenant_id,
organization_id,
name,
purpose,
data_subject_category,
personal_data_category,
special_or_criminal_data,
consent_evidence_link,
lawful_basis,
recipients,
location,
international_transfers,
transfer_safeguards,
retention_period,
security_measures,
data_protection_impact_assessment_needed,
transfer_impact_assessment_needed,
last_review_date,
next_review_date,
role,
dpo_profile_id,
created_at,
updated_at
) VALUES (
@id,
@tenant_id,
@organization_id,
@name,
@purpose,
@data_subject_category,
@personal_data_category,
@special_or_criminal_data,
@consent_evidence_link,
@lawful_basis,
@recipients,
@location,
@international_transfers,
@transfer_safeguards,
@retention_period,
@security_measures,
@data_protection_impact_assessment_needed,
@transfer_impact_assessment_needed,
@last_review_date,
@next_review_date,
@role,
@dpo_profile_id,
@created_at,
@updated_at
)
`
args := pgx.StrictNamedArgs{
"id": p.ID,
"tenant_id": scope.GetTenantID(),
"organization_id": p.OrganizationID,
"name": p.Name,
"purpose": p.Purpose,
"data_subject_category": p.DataSubjectCategory,
"personal_data_category": p.PersonalDataCategory,
"special_or_criminal_data": p.SpecialOrCriminalData,
"consent_evidence_link": p.ConsentEvidenceLink,
"lawful_basis": p.LawfulBasis,
"recipients": p.Recipients,
"location": p.Location,
"international_transfers": p.InternationalTransfers,
"transfer_safeguards": p.TransferSafeguard,
"retention_period": p.RetentionPeriod,
"security_measures": p.SecurityMeasures,
"data_protection_impact_assessment_needed": p.DataProtectionImpactAssessmentNeeded,
"transfer_impact_assessment_needed": p.TransferImpactAssessmentNeeded,
"last_review_date": p.LastReviewDate,
"next_review_date": p.NextReviewDate,
"role": p.Role,
"dpo_profile_id": p.DataProtectionOfficerID,
"created_at": p.CreatedAt,
"updated_at": p.UpdatedAt,
}
_, err := conn.Exec(ctx, q, args)
if err != nil {
return fmt.Errorf("cannot insert processing activity: %w", err)
}
return nil
}
func (p *ProcessingActivity) Update(
ctx context.Context,
conn pg.Tx,
scope Scoper,
) error {
q := `
UPDATE processing_activities
SET
name = @name,
purpose = @purpose,
data_subject_category = @data_subject_category,
personal_data_category = @personal_data_category,
special_or_criminal_data = @special_or_criminal_data,
consent_evidence_link = @consent_evidence_link,
lawful_basis = @lawful_basis,
recipients = @recipients,
location = @location,
international_transfers = @international_transfers,
transfer_safeguards = @transfer_safeguards,
retention_period = @retention_period,
security_measures = @security_measures,
data_protection_impact_assessment_needed = @data_protection_impact_assessment_needed,
transfer_impact_assessment_needed = @transfer_impact_assessment_needed,
last_review_date = @last_review_date,
next_review_date = @next_review_date,
role = @role,
dpo_profile_id = @dpo_profile_id,
updated_at = @updated_at
WHERE
%s
AND id = @id
`
q = fmt.Sprintf(q, scope.SQLFragment())
args := pgx.StrictNamedArgs{
"id": p.ID,
"name": p.Name,
"purpose": p.Purpose,
"data_subject_category": p.DataSubjectCategory,
"personal_data_category": p.PersonalDataCategory,
"special_or_criminal_data": p.SpecialOrCriminalData,
"consent_evidence_link": p.ConsentEvidenceLink,
"lawful_basis": p.LawfulBasis,
"recipients": p.Recipients,
"location": p.Location,
"international_transfers": p.InternationalTransfers,
"transfer_safeguards": p.TransferSafeguard,
"retention_period": p.RetentionPeriod,
"security_measures": p.SecurityMeasures,
"data_protection_impact_assessment_needed": p.DataProtectionImpactAssessmentNeeded,
"transfer_impact_assessment_needed": p.TransferImpactAssessmentNeeded,
"last_review_date": p.LastReviewDate,
"next_review_date": p.NextReviewDate,
"role": p.Role,
"dpo_profile_id": p.DataProtectionOfficerID,
"updated_at": p.UpdatedAt,
}
maps.Copy(args, scope.SQLArguments())
_, err := conn.Exec(ctx, q, args)
if err != nil {
return fmt.Errorf("cannot update processing activity: %w", err)
}
return nil
}
func (p *ProcessingActivity) Delete(
ctx context.Context,
conn pg.Tx,
scope Scoper,
) error {
q := `
DELETE FROM processing_activities
WHERE
%s
AND id = @id
`
q = fmt.Sprintf(q, scope.SQLFragment())
args := pgx.StrictNamedArgs{"id": p.ID}
maps.Copy(args, scope.SQLArguments())
_, err := conn.Exec(ctx, q, args)
if err != nil {
return fmt.Errorf("cannot delete processing activity: %w", err)
}
return nil
}