rohit/probo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add log export for audit logs and SCIM events

Browse Source 
Route audit-log and SCIM-event exports through export_jobs with typed
arguments, an iam BuildAndUploadExport/SendExportEmail implementation,
and a concurrent export-job worker with stale recovery. Stream JSONL via
page.WalkAll into S3, and expose the request flow on console, connect,
MCP, and CLI.

Co-authored-by: Bryan Frimin <bryan@getprobo.com>
Signed-off-by: Sacha Al Himdani <sacha@probo.com>
This commit is contained in:
Sacha Al Himdani
2026-07-29 16:12:23 +02:00
parent b2e2d15582
commit cd6c46212a
45 changed files with 2340 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pkg/coredata/audit_log_entry.go
View File

@@ -255,7 +255,7 @@ LIMIT 1;
return nil
}
func (es *AuditLogEntries) LoadAllByOrganizationID(
func (es *AuditLogEntries) LoadByOrganizationID(
ctx context.Context,
conn pg.Querier,
scope Scoper,

44
pkg/coredata/audit_log_entry_filter.go
View File

@@ -21,6 +21,8 @@
package coredata
import (
"time"
"github.com/jackc/pgx/v5"
"go.probo.inc/probo/pkg/gid"
)
@@ -30,6 +32,8 @@ type AuditLogEntryFilter struct {
actorID *gid.GID
resourceType *string
resourceID *gid.GID
createdAtGte *time.Time
createdAtLt *time.Time
}
func NewAuditLogEntryFilter() *AuditLogEntryFilter {
@@ -56,6 +60,16 @@ func (f *AuditLogEntryFilter) WithResourceID(resourceID gid.GID) *AuditLogEntryF
return f
}
func (f *AuditLogEntryFilter) WithCreatedAtGte(t time.Time) *AuditLogEntryFilter {
f.createdAtGte = &t
return f
}
func (f *AuditLogEntryFilter) WithCreatedAtLt(t time.Time) *AuditLogEntryFilter {
f.createdAtLt = &t
return f
}
func (f *AuditLogEntryFilter) SQLFragment() string {
return `
(
@@ -82,15 +96,29 @@ func (f *AuditLogEntryFilter) SQLFragment() string {
resource_id = @filter_resource_id::text
ELSE TRUE
END
AND
CASE
WHEN @filter_created_at_gte::timestamptz IS NOT NULL THEN
created_at >= @filter_created_at_gte::timestamptz
ELSE TRUE
END
AND
CASE
WHEN @filter_created_at_lt::timestamptz IS NOT NULL THEN
created_at < @filter_created_at_lt::timestamptz
ELSE TRUE
END
)`
}
func (f *AuditLogEntryFilter) SQLArguments() pgx.StrictNamedArgs {
args := pgx.StrictNamedArgs{
"filter_action": nil,
"filter_actor_id": nil,
"filter_resource_type": nil,
"filter_resource_id": nil,
"filter_action": nil,
"filter_actor_id": nil,
"filter_resource_type": nil,
"filter_resource_id": nil,
"filter_created_at_gte": nil,
"filter_created_at_lt": nil,
}
if f.action != nil {
@@ -109,5 +137,13 @@ func (f *AuditLogEntryFilter) SQLArguments() pgx.StrictNamedArgs {
args["filter_resource_id"] = *f.resourceID
}
if f.createdAtGte != nil {
args["filter_created_at_gte"] = *f.createdAtGte
}
if f.createdAtLt != nil {
args["filter_created_at_lt"] = *f.createdAtLt
}
return args
}

190
pkg/coredata/export_job.go
View File

@@ -63,6 +63,11 @@ type (
FrameworkExportArguments struct {
FrameworkID gid.GID `json:"framework_id"`
}
LogExportArguments struct {
FromTime time.Time `json:"from_time"`
ToTime time.Time `json:"to_time"`
}
)
var (
@@ -164,7 +169,6 @@ SET
status = @status,
error = @error,
file_id = @file_id,
started_at = @started_at,
completed_at = @completed_at
WHERE
%s
@@ -175,14 +179,110 @@ WHERE
"status": ej.Status,
"error": ej.Error,
"file_id": ej.FileID,
"started_at": ej.StartedAt,
"completed_at": ej.CompletedAt,
"id": ej.ID,
}
maps.Copy(args, scope.SQLArguments())
_, err := conn.Exec(ctx, q, args)
return err
result, err := conn.Exec(ctx, q, args)
if err != nil {
return fmt.Errorf("cannot update export job: %w", err)
}
if result.RowsAffected() == 0 {
return ErrResourceNotFound
}
return nil
}
// UpdateIfStatus updates the export job only when its current status matches
// expected. Used to finalize a claim without clobbering a job that was
// reclaimed and reassigned to another worker.
func (ej *ExportJob) UpdateIfStatus(
ctx context.Context,
conn pg.Tx,
scope Scoper,
expected ExportJobStatus,
) error {
q := `
UPDATE
export_jobs
SET
status = @status,
error = @error,
file_id = @file_id,
completed_at = @completed_at
WHERE
%s
AND id = @id
AND status = @expected_status
`
q = fmt.Sprintf(q, scope.SQLFragment())
args := pgx.StrictNamedArgs{
"status": ej.Status,
"error": ej.Error,
"file_id": ej.FileID,
"completed_at": ej.CompletedAt,
"id": ej.ID,
"expected_status": expected,
}
maps.Copy(args, scope.SQLArguments())
result, err := conn.Exec(ctx, q, args)
if err != nil {
return fmt.Errorf("cannot update export job: %w", err)
}
if result.RowsAffected() == 0 {
return ErrResourceNotFound
}
return nil
}
// MarkProcessing claims the job for processing and starts its lease clock.
func (ej *ExportJob) MarkProcessing(
ctx context.Context,
conn pg.Tx,
scope Scoper,
) error {
now := time.Now()
ej.Status = ExportJobStatusProcessing
ej.StartedAt = &now
q := `
UPDATE
export_jobs
SET
status = @status,
started_at = @started_at,
error = NULL,
completed_at = NULL
WHERE
%s
AND id = @id
AND status = @pending_status
`
q = fmt.Sprintf(q, scope.SQLFragment())
args := pgx.StrictNamedArgs{
"status": ej.Status,
"started_at": ej.StartedAt,
"id": ej.ID,
"pending_status": ExportJobStatusPending,
}
maps.Copy(args, scope.SQLArguments())
result, err := conn.Exec(ctx, q, args)
if err != nil {
return fmt.Errorf("cannot mark export job as processing: %w", err)
}
if result.RowsAffected() == 0 {
return ErrResourceNotFound
}
return nil
}
func (ej *ExportJob) LoadByID(
@@ -323,3 +423,85 @@ func (ej *ExportJob) GetFrameworkID() (gid.GID, error) {
return args.FrameworkID, nil
}
func (ej *ExportJob) GetLogExportArguments() (*LogExportArguments, error) {
switch ej.Type {
case ExportJobTypeAuditLog, ExportJobTypeSCIMEvent:
default:
return nil, fmt.Errorf("export job is not a log export")
}
var args LogExportArguments
if err := json.Unmarshal(ej.Arguments, &args); err != nil {
return nil, fmt.Errorf("cannot unmarshal log export arguments: %w", err)
}
return &args, nil
}
func ResetStaleExportJobs(
ctx context.Context,
conn pg.Querier,
staleAfter time.Duration,
) error {
q := `
UPDATE export_jobs
SET
status = @pending_status,
started_at = NULL
WHERE
status = @processing_status
AND started_at < @stale_threshold
`
args := pgx.StrictNamedArgs{
"pending_status": ExportJobStatusPending,
"processing_status": ExportJobStatusProcessing,
"stale_threshold": time.Now().Add(-staleAfter),
}
_, err := conn.Exec(ctx, q, args)
if err != nil {
return fmt.Errorf("cannot reset stale export jobs: %w", err)
}
return nil
}
// TouchExportJobLease renews the processing lease so long-running exports
// are not requeued by ResetStaleExportJobs while still actively working.
func TouchExportJobLease(
ctx context.Context,
conn pg.Querier,
scope Scoper,
id gid.GID,
) error {
q := `
UPDATE export_jobs
SET
started_at = @started_at
WHERE
%s
AND id = @id
AND status = @processing_status
`
q = fmt.Sprintf(q, scope.SQLFragment())
args := pgx.StrictNamedArgs{
"id": id,
"started_at": time.Now(),
"processing_status": ExportJobStatusProcessing,
}
maps.Copy(args, scope.SQLArguments())
result, err := conn.Exec(ctx, q, args)
if err != nil {
return fmt.Errorf("cannot touch export job lease: %w", err)
}
if result.RowsAffected() == 0 {
return ErrResourceNotFound
}
return nil
}

8
pkg/coredata/export_job_type.go
View File

@@ -32,6 +32,8 @@ type (
const (
ExportJobTypeFramework ExportJobType = "FRAMEWORK"
ExportJobTypeDocument ExportJobType = "DOCUMENT"
ExportJobTypeAuditLog ExportJobType = "AUDIT_LOG"
ExportJobTypeSCIMEvent ExportJobType = "SCIM_EVENT"
)
var (
@@ -44,6 +46,8 @@ func ExportJobTypes() []ExportJobType {
return []ExportJobType{
ExportJobTypeFramework,
ExportJobTypeDocument,
ExportJobTypeAuditLog,
ExportJobTypeSCIMEvent,
}
}
@@ -51,7 +55,9 @@ func (v ExportJobType) IsValid() bool {
switch v {
case
ExportJobTypeFramework,
ExportJobTypeDocument:
ExportJobTypeDocument,
ExportJobTypeAuditLog,
ExportJobTypeSCIMEvent:
return true
}

22
pkg/coredata/migrations/20260729T135958Z.sql Normal file
View File

@@ -0,0 +1,22 @@
-- Copyright (c) 2026 Probo Inc <hello@probo.com>.
--
-- Permission is hereby granted, free of charge, to any person obtaining a copy
-- of this software and associated documentation files (the "Software"), to deal
-- in the Software without restriction, including without limitation the rights
-- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-- copies of the Software, and to permit persons to whom the Software is
-- furnished to do so, subject to the following conditions:
--
-- The above copyright notice and this permission notice shall be included in
-- all copies or substantial portions of the Software.
--
-- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-- SOFTWARE.
ALTER TYPE export_jobs_type ADD VALUE IF NOT EXISTS 'AUDIT_LOG';
ALTER TYPE export_jobs_type ADD VALUE IF NOT EXISTS 'SCIM_EVENT';

5
pkg/coredata/scim_event.go
View File

@@ -221,6 +221,7 @@ func (s *SCIMEvents) LoadByOrganizationID(
scope Scoper,
organizationID gid.GID,
cursor *page.Cursor[SCIMEventOrderField],
filter *SCIMEventFilter,
) error {
q := `
SELECT
@@ -242,12 +243,14 @@ WHERE
%s
AND organization_id = @organization_id
AND %s
AND %s
`
q = fmt.Sprintf(q, scope.SQLFragment(), cursor.SQLFragment())
q = fmt.Sprintf(q, scope.SQLFragment(), filter.SQLFragment(), cursor.SQLFragment())
args := pgx.StrictNamedArgs{"organization_id": organizationID}
maps.Copy(args, scope.SQLArguments())
maps.Copy(args, filter.SQLArguments())
maps.Copy(args, cursor.SQLArguments())
rows, err := conn.Query(ctx, q, args)

80
pkg/coredata/scim_event_filter.go Normal file
View File

@@ -0,0 +1,80 @@
// Copyright (c) 2026 Probo Inc <hello@probo.com>.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
package coredata
import (
"time"
"github.com/jackc/pgx/v5"
)
type SCIMEventFilter struct {
createdAtGte *time.Time
createdAtLt *time.Time
}
func NewSCIMEventFilter() *SCIMEventFilter {
return &SCIMEventFilter{}
}
func (f *SCIMEventFilter) WithCreatedAtGte(t time.Time) *SCIMEventFilter {
f.createdAtGte = &t
return f
}
func (f *SCIMEventFilter) WithCreatedAtLt(t time.Time) *SCIMEventFilter {
f.createdAtLt = &t
return f
}
func (f *SCIMEventFilter) SQLFragment() string {
return `
(
CASE
WHEN @filter_created_at_gte::timestamptz IS NOT NULL THEN
created_at >= @filter_created_at_gte::timestamptz
ELSE TRUE
END
AND
CASE
WHEN @filter_created_at_lt::timestamptz IS NOT NULL THEN
created_at < @filter_created_at_lt::timestamptz
ELSE TRUE
END
)`
}
func (f *SCIMEventFilter) SQLArguments() pgx.StrictNamedArgs {
args := pgx.StrictNamedArgs{
"filter_created_at_gte": nil,
"filter_created_at_lt": nil,
}
if f.createdAtGte != nil {
args["filter_created_at_gte"] = *f.createdAtGte
}
if f.createdAtLt != nil {
args["filter_created_at_lt"] = *f.createdAtLt
}
return args
}