rohit/probo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add log export for audit logs and SCIM events

Browse Source 
Route audit-log and SCIM-event exports through export_jobs with typed
arguments, an iam BuildAndUploadExport/SendExportEmail implementation,
and a concurrent export-job worker with stale recovery. Stream JSONL via
page.WalkAll into S3, and expose the request flow on console, connect,
MCP, and CLI.

Co-authored-by: Bryan Frimin <bryan@getprobo.com>
Signed-off-by: Sacha Al Himdani <sacha@probo.com>
This commit is contained in:
Sacha Al Himdani
2026-07-29 16:12:23 +02:00
parent b2e2d15582
commit cd6c46212a
45 changed files with 2340 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pkg/cmd/auditlog/audit_log.go
View File

@@ -22,6 +22,7 @@ package auditlog
import (
"github.com/spf13/cobra"
"go.probo.inc/probo/pkg/cmd/auditlog/export"
"go.probo.inc/probo/pkg/cmd/auditlog/list"
"go.probo.inc/probo/pkg/cmd/auditlog/view"
"go.probo.inc/probo/pkg/cmd/cmdutil"
@@ -33,6 +34,7 @@ func NewCmdAuditLog(f *cmdutil.Factory) *cobra.Command {
Short: "Manage audit log entries",
}
cmd.AddCommand(export.NewCmdExport(f))
cmd.AddCommand(list.NewCmdList(f))
cmd.AddCommand(view.NewCmdView(f))

126
pkg/cmd/auditlog/export/export.go Normal file
View File

@@ -0,0 +1,126 @@
// Copyright (c) 2026 Probo Inc <hello@probo.com>.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
package export
import (
"encoding/json"
"fmt"
"github.com/spf13/cobra"
"go.probo.inc/probo/pkg/cli/api"
"go.probo.inc/probo/pkg/cmd/cmdutil"
)
const exportMutation = `
mutation($input: RequestAuditLogExportInput!) {
requestAuditLogExport(input: $input) {
exportJobId
}
}
`
func NewCmdExport(f *cmdutil.Factory) *cobra.Command {
var (
flagOrg string
flagFrom string
flagTo string
)
cmd := &cobra.Command{
Use: "export",
Short: "Export audit log entries",
Example: ` prb audit-log export --org <id> --from 2026-01-01T00:00:00Z --to 2026-02-01T00:00:00Z
prb audit-log export --from 2026-03-01T00:00:00Z --to 2026-03-24T00:00:00Z`,
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, args []string) error {
cfg, err := f.Config()
if err != nil {
return err
}
host, hc, err := cfg.DefaultHost()
if err != nil {
return err
}
client := api.NewClient(
host,
hc.Token,
"/api/console/v1/graphql",
cfg.HTTPTimeoutDuration(),
cmdutil.TokenRefreshOption(cfg, host, hc),
)
if flagOrg == "" {
flagOrg = hc.Organization
}
if flagOrg == "" {
return fmt.Errorf("organization is required; pass --org or set a default with 'prb auth login'")
}
if flagFrom == "" {
return fmt.Errorf("--from is required (RFC3339 timestamp)")
}
if flagTo == "" {
return fmt.Errorf("--to is required (RFC3339 timestamp)")
}
variables := map[string]any{
"input": map[string]any{
"organizationId": flagOrg,
"fromTime": flagFrom,
"toTime": flagTo,
},
}
data, err := client.Do(exportMutation, variables)
if err != nil {
return err
}
var resp struct {
RequestAuditLogExport struct {
ExportJobID string `json:"exportJobId"`
} `json:"requestAuditLogExport"`
}
if err := json.Unmarshal(data, &resp); err != nil {
return err
}
_, _ = fmt.Fprintf(
f.IOStreams.Out,
"Audit log export requested %s\nYou will receive an email with a download link when the export is ready.\n",
resp.RequestAuditLogExport.ExportJobID,
)
return nil
},
}
cmd.Flags().StringVar(&flagOrg, "org", "", "Organization ID")
cmd.Flags().StringVar(&flagFrom, "from", "", "Start time in RFC3339 format (e.g. 2026-01-01T00:00:00Z)")
cmd.Flags().StringVar(&flagTo, "to", "", "End time in RFC3339 format (e.g. 2026-02-01T00:00:00Z)")
return cmd
}

2
pkg/cmd/scim/event/event.go
View File

@@ -23,6 +23,7 @@ package event
import (
"github.com/spf13/cobra"
"go.probo.inc/probo/pkg/cmd/cmdutil"
"go.probo.inc/probo/pkg/cmd/scim/event/export"
"go.probo.inc/probo/pkg/cmd/scim/event/list"
)
@@ -33,6 +34,7 @@ func NewCmdEvent(f *cmdutil.Factory) *cobra.Command {
}
cmd.AddCommand(list.NewCmdList(f))
cmd.AddCommand(export.NewCmdExport(f))
return cmd
}

126
pkg/cmd/scim/event/export/export.go Normal file
View File

@@ -0,0 +1,126 @@
// Copyright (c) 2026 Probo Inc <hello@probo.com>.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
package export
import (
"encoding/json"
"fmt"
"github.com/spf13/cobra"
"go.probo.inc/probo/pkg/cli/api"
"go.probo.inc/probo/pkg/cmd/cmdutil"
)
const exportMutation = `
mutation($input: RequestSCIMEventExportInput!) {
requestSCIMEventExport(input: $input) {
exportJobId
}
}
`
func NewCmdExport(f *cmdutil.Factory) *cobra.Command {
var (
flagOrg string
flagFrom string
flagTo string
)
cmd := &cobra.Command{
Use: "export",
Short: "Export SCIM events",
Example: ` prb scim event export --org <id> --from 2026-01-01T00:00:00Z --to 2026-02-01T00:00:00Z
prb scim event export --from 2026-03-01T00:00:00Z --to 2026-03-24T00:00:00Z`,
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, args []string) error {
cfg, err := f.Config()
if err != nil {
return err
}
host, hc, err := cfg.DefaultHost()
if err != nil {
return err
}
client := api.NewClient(
host,
hc.Token,
"/api/connect/v1/graphql",
cfg.HTTPTimeoutDuration(),
cmdutil.TokenRefreshOption(cfg, host, hc),
)
if flagOrg == "" {
flagOrg = hc.Organization
}
if flagOrg == "" {
return fmt.Errorf("organization is required; pass --org or set a default with 'prb auth login'")
}
if flagFrom == "" {
return fmt.Errorf("--from is required (RFC3339 timestamp)")
}
if flagTo == "" {
return fmt.Errorf("--to is required (RFC3339 timestamp)")
}
variables := map[string]any{
"input": map[string]any{
"organizationId": flagOrg,
"fromTime": flagFrom,
"toTime": flagTo,
},
}
data, err := client.Do(exportMutation, variables)
if err != nil {
return err
}
var resp struct {
RequestSCIMEventExport struct {
ExportJobID string `json:"exportJobId"`
} `json:"requestSCIMEventExport"`
}
if err := json.Unmarshal(data, &resp); err != nil {
return err
}
_, _ = fmt.Fprintf(
f.IOStreams.Out,
"SCIM event export requested %s\nYou will receive an email with a download link when the export is ready.\n",
resp.RequestSCIMEventExport.ExportJobID,
)
return nil
},
}
cmd.Flags().StringVar(&flagOrg, "org", "", "Organization ID")
cmd.Flags().StringVar(&flagFrom, "from", "", "Start time in RFC3339 format (e.g. 2026-01-01T00:00:00Z)")
cmd.Flags().StringVar(&flagTo, "to", "", "End time in RFC3339 format (e.g. 2026-02-01T00:00:00Z)")
return cmd
}

2
pkg/coredata/audit_log_entry.go
View File

@@ -255,7 +255,7 @@ LIMIT 1;
return nil
}
func (es *AuditLogEntries) LoadAllByOrganizationID(
func (es *AuditLogEntries) LoadByOrganizationID(
ctx context.Context,
conn pg.Querier,
scope Scoper,

44
pkg/coredata/audit_log_entry_filter.go
View File

@@ -21,6 +21,8 @@
package coredata
import (
"time"
"github.com/jackc/pgx/v5"
"go.probo.inc/probo/pkg/gid"
)
@@ -30,6 +32,8 @@ type AuditLogEntryFilter struct {
actorID *gid.GID
resourceType *string
resourceID *gid.GID
createdAtGte *time.Time
createdAtLt *time.Time
}
func NewAuditLogEntryFilter() *AuditLogEntryFilter {
@@ -56,6 +60,16 @@ func (f *AuditLogEntryFilter) WithResourceID(resourceID gid.GID) *AuditLogEntryF
return f
}
func (f *AuditLogEntryFilter) WithCreatedAtGte(t time.Time) *AuditLogEntryFilter {
f.createdAtGte = &t
return f
}
func (f *AuditLogEntryFilter) WithCreatedAtLt(t time.Time) *AuditLogEntryFilter {
f.createdAtLt = &t
return f
}
func (f *AuditLogEntryFilter) SQLFragment() string {
return `
(
@@ -82,15 +96,29 @@ func (f *AuditLogEntryFilter) SQLFragment() string {
resource_id = @filter_resource_id::text
ELSE TRUE
END
AND
CASE
WHEN @filter_created_at_gte::timestamptz IS NOT NULL THEN
created_at >= @filter_created_at_gte::timestamptz
ELSE TRUE
END
AND
CASE
WHEN @filter_created_at_lt::timestamptz IS NOT NULL THEN
created_at < @filter_created_at_lt::timestamptz
ELSE TRUE
END
)`
}
func (f *AuditLogEntryFilter) SQLArguments() pgx.StrictNamedArgs {
args := pgx.StrictNamedArgs{
"filter_action": nil,
"filter_actor_id": nil,
"filter_resource_type": nil,
"filter_resource_id": nil,
"filter_action": nil,
"filter_actor_id": nil,
"filter_resource_type": nil,
"filter_resource_id": nil,
"filter_created_at_gte": nil,
"filter_created_at_lt": nil,
}
if f.action != nil {
@@ -109,5 +137,13 @@ func (f *AuditLogEntryFilter) SQLArguments() pgx.StrictNamedArgs {
args["filter_resource_id"] = *f.resourceID
}
if f.createdAtGte != nil {
args["filter_created_at_gte"] = *f.createdAtGte
}
if f.createdAtLt != nil {
args["filter_created_at_lt"] = *f.createdAtLt
}
return args
}

190
pkg/coredata/export_job.go
View File

@@ -63,6 +63,11 @@ type (
FrameworkExportArguments struct {
FrameworkID gid.GID `json:"framework_id"`
}
LogExportArguments struct {
FromTime time.Time `json:"from_time"`
ToTime time.Time `json:"to_time"`
}
)
var (
@@ -164,7 +169,6 @@ SET
status = @status,
error = @error,
file_id = @file_id,
started_at = @started_at,
completed_at = @completed_at
WHERE
%s
@@ -175,14 +179,110 @@ WHERE
"status": ej.Status,
"error": ej.Error,
"file_id": ej.FileID,
"started_at": ej.StartedAt,
"completed_at": ej.CompletedAt,
"id": ej.ID,
}
maps.Copy(args, scope.SQLArguments())
_, err := conn.Exec(ctx, q, args)
return err
result, err := conn.Exec(ctx, q, args)
if err != nil {
return fmt.Errorf("cannot update export job: %w", err)
}
if result.RowsAffected() == 0 {
return ErrResourceNotFound
}
return nil
}
// UpdateIfStatus updates the export job only when its current status matches
// expected. Used to finalize a claim without clobbering a job that was
// reclaimed and reassigned to another worker.
func (ej *ExportJob) UpdateIfStatus(
ctx context.Context,
conn pg.Tx,
scope Scoper,
expected ExportJobStatus,
) error {
q := `
UPDATE
export_jobs
SET
status = @status,
error = @error,
file_id = @file_id,
completed_at = @completed_at
WHERE
%s
AND id = @id
AND status = @expected_status
`
q = fmt.Sprintf(q, scope.SQLFragment())
args := pgx.StrictNamedArgs{
"status": ej.Status,
"error": ej.Error,
"file_id": ej.FileID,
"completed_at": ej.CompletedAt,
"id": ej.ID,
"expected_status": expected,
}
maps.Copy(args, scope.SQLArguments())
result, err := conn.Exec(ctx, q, args)
if err != nil {
return fmt.Errorf("cannot update export job: %w", err)
}
if result.RowsAffected() == 0 {
return ErrResourceNotFound
}
return nil
}
// MarkProcessing claims the job for processing and starts its lease clock.
func (ej *ExportJob) MarkProcessing(
ctx context.Context,
conn pg.Tx,
scope Scoper,
) error {
now := time.Now()
ej.Status = ExportJobStatusProcessing
ej.StartedAt = &now
q := `
UPDATE
export_jobs
SET
status = @status,
started_at = @started_at,
error = NULL,
completed_at = NULL
WHERE
%s
AND id = @id
AND status = @pending_status
`
q = fmt.Sprintf(q, scope.SQLFragment())
args := pgx.StrictNamedArgs{
"status": ej.Status,
"started_at": ej.StartedAt,
"id": ej.ID,
"pending_status": ExportJobStatusPending,
}
maps.Copy(args, scope.SQLArguments())
result, err := conn.Exec(ctx, q, args)
if err != nil {
return fmt.Errorf("cannot mark export job as processing: %w", err)
}
if result.RowsAffected() == 0 {
return ErrResourceNotFound
}
return nil
}
func (ej *ExportJob) LoadByID(
@@ -323,3 +423,85 @@ func (ej *ExportJob) GetFrameworkID() (gid.GID, error) {
return args.FrameworkID, nil
}
func (ej *ExportJob) GetLogExportArguments() (*LogExportArguments, error) {
switch ej.Type {
case ExportJobTypeAuditLog, ExportJobTypeSCIMEvent:
default:
return nil, fmt.Errorf("export job is not a log export")
}
var args LogExportArguments
if err := json.Unmarshal(ej.Arguments, &args); err != nil {
return nil, fmt.Errorf("cannot unmarshal log export arguments: %w", err)
}
return &args, nil
}
func ResetStaleExportJobs(
ctx context.Context,
conn pg.Querier,
staleAfter time.Duration,
) error {
q := `
UPDATE export_jobs
SET
status = @pending_status,
started_at = NULL
WHERE
status = @processing_status
AND started_at < @stale_threshold
`
args := pgx.StrictNamedArgs{
"pending_status": ExportJobStatusPending,
"processing_status": ExportJobStatusProcessing,
"stale_threshold": time.Now().Add(-staleAfter),
}
_, err := conn.Exec(ctx, q, args)
if err != nil {
return fmt.Errorf("cannot reset stale export jobs: %w", err)
}
return nil
}
// TouchExportJobLease renews the processing lease so long-running exports
// are not requeued by ResetStaleExportJobs while still actively working.
func TouchExportJobLease(
ctx context.Context,
conn pg.Querier,
scope Scoper,
id gid.GID,
) error {
q := `
UPDATE export_jobs
SET
started_at = @started_at
WHERE
%s
AND id = @id
AND status = @processing_status
`
q = fmt.Sprintf(q, scope.SQLFragment())
args := pgx.StrictNamedArgs{
"id": id,
"started_at": time.Now(),
"processing_status": ExportJobStatusProcessing,
}
maps.Copy(args, scope.SQLArguments())
result, err := conn.Exec(ctx, q, args)
if err != nil {
return fmt.Errorf("cannot touch export job lease: %w", err)
}
if result.RowsAffected() == 0 {
return ErrResourceNotFound
}
return nil
}

8
pkg/coredata/export_job_type.go
View File

@@ -32,6 +32,8 @@ type (
const (
ExportJobTypeFramework ExportJobType = "FRAMEWORK"
ExportJobTypeDocument ExportJobType = "DOCUMENT"
ExportJobTypeAuditLog ExportJobType = "AUDIT_LOG"
ExportJobTypeSCIMEvent ExportJobType = "SCIM_EVENT"
)
var (
@@ -44,6 +46,8 @@ func ExportJobTypes() []ExportJobType {
return []ExportJobType{
ExportJobTypeFramework,
ExportJobTypeDocument,
ExportJobTypeAuditLog,
ExportJobTypeSCIMEvent,
}
}
@@ -51,7 +55,9 @@ func (v ExportJobType) IsValid() bool {
switch v {
case
ExportJobTypeFramework,
ExportJobTypeDocument:
ExportJobTypeDocument,
ExportJobTypeAuditLog,
ExportJobTypeSCIMEvent:
return true
}

22
pkg/coredata/migrations/20260729T135958Z.sql Normal file
View File

@@ -0,0 +1,22 @@
-- Copyright (c) 2026 Probo Inc <hello@probo.com>.
--
-- Permission is hereby granted, free of charge, to any person obtaining a copy
-- of this software and associated documentation files (the "Software"), to deal
-- in the Software without restriction, including without limitation the rights
-- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-- copies of the Software, and to permit persons to whom the Software is
-- furnished to do so, subject to the following conditions:
--
-- The above copyright notice and this permission notice shall be included in
-- all copies or substantial portions of the Software.
--
-- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-- SOFTWARE.
ALTER TYPE export_jobs_type ADD VALUE IF NOT EXISTS 'AUDIT_LOG';
ALTER TYPE export_jobs_type ADD VALUE IF NOT EXISTS 'SCIM_EVENT';

5
pkg/coredata/scim_event.go
View File

@@ -221,6 +221,7 @@ func (s *SCIMEvents) LoadByOrganizationID(
scope Scoper,
organizationID gid.GID,
cursor *page.Cursor[SCIMEventOrderField],
filter *SCIMEventFilter,
) error {
q := `
SELECT
@@ -242,12 +243,14 @@ WHERE
%s
AND organization_id = @organization_id
AND %s
AND %s
`
q = fmt.Sprintf(q, scope.SQLFragment(), cursor.SQLFragment())
q = fmt.Sprintf(q, scope.SQLFragment(), filter.SQLFragment(), cursor.SQLFragment())
args := pgx.StrictNamedArgs{"organization_id": organizationID}
maps.Copy(args, scope.SQLArguments())
maps.Copy(args, filter.SQLArguments())
maps.Copy(args, cursor.SQLArguments())
rows, err := conn.Query(ctx, q, args)

80
pkg/coredata/scim_event_filter.go Normal file
View File

@@ -0,0 +1,80 @@
// Copyright (c) 2026 Probo Inc <hello@probo.com>.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
package coredata
import (
"time"
"github.com/jackc/pgx/v5"
)
type SCIMEventFilter struct {
createdAtGte *time.Time
createdAtLt *time.Time
}
func NewSCIMEventFilter() *SCIMEventFilter {
return &SCIMEventFilter{}
}
func (f *SCIMEventFilter) WithCreatedAtGte(t time.Time) *SCIMEventFilter {
f.createdAtGte = &t
return f
}
func (f *SCIMEventFilter) WithCreatedAtLt(t time.Time) *SCIMEventFilter {
f.createdAtLt = &t
return f
}
func (f *SCIMEventFilter) SQLFragment() string {
return `
(
CASE
WHEN @filter_created_at_gte::timestamptz IS NOT NULL THEN
created_at >= @filter_created_at_gte::timestamptz
ELSE TRUE
END
AND
CASE
WHEN @filter_created_at_lt::timestamptz IS NOT NULL THEN
created_at < @filter_created_at_lt::timestamptz
ELSE TRUE
END
)`
}
func (f *SCIMEventFilter) SQLArguments() pgx.StrictNamedArgs {
args := pgx.StrictNamedArgs{
"filter_created_at_gte": nil,
"filter_created_at_lt": nil,
}
if f.createdAtGte != nil {
args["filter_created_at_gte"] = *f.createdAtGte
}
if f.createdAtLt != nil {
args["filter_created_at_lt"] = *f.createdAtLt
}
return args
}

65
pkg/filemanager/s3.go
View File

@@ -31,6 +31,7 @@ import (
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager"
"github.com/aws/aws-sdk-go-v2/service/s3"
smithyhttp "github.com/aws/smithy-go/transport/http"
"go.probo.inc/probo/pkg/coredata"
@@ -203,23 +204,51 @@ func (s *Service) OpenFile(
return obj, nil
}
type putFileConfig struct {
attachmentDisposition bool
}
// PutFileOption configures optional PutFile behavior.
type PutFileOption func(*putFileConfig)
// WithAttachmentContentDisposition stores Content-Disposition: attachment on the
// uploaded object so browsers download it when served from object storage.
func WithAttachmentContentDisposition() PutFileOption {
return func(cfg *putFileConfig) {
cfg.attachmentDisposition = true
}
}
func (s *Service) PutFile(
ctx context.Context,
file *coredata.File,
content io.Reader,
metadata map[string]string,
opts ...PutFileOption,
) (int64, error) {
_, err := s.s3Client.PutObject(
ctx,
&s3.PutObjectInput{
Bucket: new(file.BucketName),
Key: new(file.FileKey),
Body: content,
ContentType: new(file.MimeType),
CacheControl: new("private, max-age=3600"),
Metadata: metadata,
},
)
cfg := putFileConfig{}
for _, opt := range opts {
opt(&cfg)
}
// Transfer manager accepts unseekable readers (e.g. io.Pipe) by buffering
// parts in memory, which works against plain-HTTP S3-compatible endpoints
// where PutObject checksums require a seekable body.
uploader := transfermanager.New(s.s3Client)
input := &transfermanager.UploadObjectInput{
Bucket: new(file.BucketName),
Key: new(file.FileKey),
Body: content,
ContentType: new(file.MimeType),
CacheControl: new("private, max-age=3600"),
Metadata: metadata,
}
if cfg.attachmentDisposition && file.FileName != "" {
input.ContentDisposition = new(attachmentContentDisposition(file.FileName))
}
_, err := uploader.UploadObject(ctx, input)
if err != nil {
return 0, fmt.Errorf("cannot upload file to S3: %w", err)
}
@@ -245,11 +274,7 @@ func (s *Service) GeneratePresignedURL(
) (string, error) {
presignClient := s3.NewPresignClient(s.s3Client)
contentDisposition := fmt.Sprintf(
"attachment; filename=%q; filename*=UTF-8''%s",
asciiFilename(file.FileName),
url.PathEscape(file.FileName),
)
contentDisposition := attachmentContentDisposition(file.FileName)
presignedReq, err := presignClient.PresignGetObject(
ctx,
@@ -300,6 +325,14 @@ func ifRangeMatches(ifRange, etag string, lastModified time.Time) bool {
return !lastModified.IsZero() && lastModified.Truncate(time.Second).Equal(t)
}
func attachmentContentDisposition(filename string) string {
return fmt.Sprintf(
"attachment; filename=%q; filename*=UTF-8''%s",
asciiFilename(filename),
url.PathEscape(filename),
)
}
func asciiFilename(filename string) string {
var b strings.Builder
b.Grow(len(filename))

21
pkg/iam/errors.go
View File

@@ -499,3 +499,24 @@ func NewConnectorNotFoundError(connectorID gid.GID) error {
func (e ErrConnectorNotFound) Error() string {
return fmt.Sprintf("connector %q not found", e.ConnectorID)
}
type ErrInvalidLogExportTimeRange struct{ message string }
const maxLogExportTimeRangeYears = 1
func NewInvalidLogExportTimeRangeError() error {
return &ErrInvalidLogExportTimeRange{message: "from_time must be before to_time"}
}
func NewLogExportTimeRangeTooLargeError() error {
return &ErrInvalidLogExportTimeRange{
message: fmt.Sprintf(
"export time range must not exceed %d year",
maxLogExportTimeRangeYears,
),
}
}
func (e ErrInvalidLogExportTimeRange) Error() string {
return e.message
}

4
pkg/iam/iam_actions.go
View File

@@ -106,4 +106,8 @@ const (
// Audit log entry actions
ActionAuditLogEntryGet = "iam:audit-log-entry:get"
ActionAuditLogEntryList = "iam:audit-log-entry:list"
// Log export actions
ActionAuditLogExport = "iam:audit-log:export"
ActionSCIMEventExport = "iam:scim-event:export"
)

9
pkg/iam/iam_policies.go
View File

@@ -242,6 +242,7 @@ var IAMOwnerPolicy = policy.NewPolicy(
policy.Allow(
ActionAuditLogEntryGet,
ActionAuditLogEntryList,
ActionAuditLogExport,
).
WithSID("audit-log-entry-access").
When(policy.Equals("principal.organization_id", "resource.organization_id")),
@@ -360,15 +361,21 @@ var IAMAdminPolicy = policy.NewPolicy(
).
WithSID("deny-scim-management"),
// Can view audit log entries (scoped to own organization)
// Can view and export audit log entries (scoped to own organization)
policy.Allow(
ActionAuditLogEntryGet,
ActionAuditLogEntryList,
ActionAuditLogExport,
).
WithSID("audit-log-entry-admin-access").
When(
policy.Equals("principal.organization_id", "resource.organization_id"),
),
// Can export SCIM events (scoped to own organization)
policy.Allow(ActionSCIMEventExport).
WithSID("scim-event-export-admin-access").
When(policy.Equals("principal.organization_id", "resource.organization_id")),
).
WithDescription("IAM admin access - can manage members but cannot delete organization or manage SAML/SCIM")

329
pkg/iam/log_export_service.go Normal file
View File

@@ -0,0 +1,329 @@
// Copyright (c) 2026 Probo Inc <hello@probo.com>.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
package iam
import (
"context"
"encoding/json"
"fmt"
"io"
"strings"
"time"
"go.gearno.de/crypto/uuid"
"go.gearno.de/kit/pg"
"go.probo.inc/probo/packages/emails"
"go.probo.inc/probo/pkg/coredata"
"go.probo.inc/probo/pkg/filemanager"
"go.probo.inc/probo/pkg/gid"
"go.probo.inc/probo/pkg/mail"
"go.probo.inc/probo/pkg/page"
)
type LogExportService struct {
pg *pg.Client
fm *filemanager.Service
bucket string
baseURL string
}
func NewLogExportService(
pgClient *pg.Client,
fm *filemanager.Service,
bucket string,
baseURL string,
) *LogExportService {
return &LogExportService{
pg: pgClient,
fm: fm,
bucket: bucket,
baseURL: baseURL,
}
}
func (s *LogExportService) BuildAndUploadExport(
ctx context.Context,
scope coredata.Scoper,
exportJobID gid.GID,
) (*coredata.ExportJob, error) {
exportJob := &coredata.ExportJob{}
if err := s.pg.WithConn(
ctx,
func(ctx context.Context, conn pg.Querier) error {
return exportJob.LoadByID(ctx, conn, scope, exportJobID)
},
); err != nil {
return nil, fmt.Errorf("cannot load export job: %w", err)
}
args, err := exportJob.GetLogExportArguments()
if err != nil {
return nil, err
}
typeName := "audit-log"
if exportJob.Type == coredata.ExportJobTypeSCIMEvent {
typeName = "scim-event"
}
now := time.Now()
fileKey := uuid.MustNewV4().String()
fileName := fmt.Sprintf(
"%s-export-%s-to-%s.jsonl",
typeName,
args.FromTime.Format("2006-01-02"),
args.ToTime.Format("2006-01-02"),
)
file := coredata.File{
ID: gid.New(exportJob.ID.TenantID(), coredata.FileEntityType),
OrganizationID: exportJob.OrganizationID,
BucketName: s.bucket,
MimeType: "application/octet-stream",
FileName: fileName,
FileKey: fileKey,
Visibility: coredata.FileVisibilityPrivate,
CreatedAt: now,
UpdatedAt: now,
}
pr, pw := io.Pipe()
var uploadErr error
var fileSize int64
uploadDone := make(chan struct{})
go func() {
defer close(uploadDone)
fileSize, uploadErr = s.fm.PutFile(
ctx,
&file,
pr,
map[string]string{
"type": typeName + "-export",
"export-job-id": exportJob.ID.String(),
"organization-id": exportJob.OrganizationID.String(),
},
filemanager.WithAttachmentContentDisposition(),
)
_ = pr.CloseWithError(uploadErr)
}()
writeErr := s.streamJSONL(ctx, exportJob, args, scope, pw)
if writeErr != nil {
_ = pw.CloseWithError(writeErr)
} else {
_ = pw.Close()
}
<-uploadDone
if writeErr != nil {
return nil, fmt.Errorf("cannot write JSONL: %w", writeErr)
}
if uploadErr != nil {
return nil, fmt.Errorf("cannot upload file to S3: %w", uploadErr)
}
file.FileSize = fileSize
if err := s.pg.WithTx(
ctx,
func(ctx context.Context, tx pg.Tx) error {
if err := file.Insert(ctx, tx, scope); err != nil {
return fmt.Errorf("cannot insert file: %w", err)
}
exportJob.FileID = &file.ID
if err := exportJob.Update(ctx, tx, scope); err != nil {
return fmt.Errorf("cannot update export job: %w", err)
}
return nil
},
); err != nil {
return nil, err
}
return exportJob, nil
}
func (s *LogExportService) SendExportEmail(
ctx context.Context,
scope coredata.Scoper,
fileID gid.GID,
recipientName string,
recipientEmail mail.Addr,
) error {
return s.pg.WithTx(
ctx,
func(ctx context.Context, tx pg.Tx) error {
file := &coredata.File{}
if err := file.LoadByID(ctx, tx, scope, fileID); err != nil {
return fmt.Errorf("cannot load file: %w", err)
}
downloadURL := s.fm.GenerateFileURL(file)
emailPresenter := emails.NewPresenter(s.baseURL, recipientName)
isSCIMEvent := strings.HasPrefix(file.FileName, "scim-event-export")
subject, textBody, htmlBody, err := emailPresenter.RenderLogExport(ctx, downloadURL, isSCIMEvent)
if err != nil {
return fmt.Errorf("cannot render log export email: %w", err)
}
email := coredata.NewEmail(
recipientName,
recipientEmail,
subject,
textBody,
htmlBody,
nil,
)
if err := email.Insert(ctx, tx); err != nil {
return fmt.Errorf("cannot insert email: %w", err)
}
return nil
},
)
}
func (s *LogExportService) streamJSONL(
ctx context.Context,
exportJob *coredata.ExportJob,
args *coredata.LogExportArguments,
scope coredata.Scoper,
pw io.Writer,
) error {
return s.pg.WithConn(
ctx,
func(ctx context.Context, conn pg.Querier) error {
enc := json.NewEncoder(pw)
switch exportJob.Type {
case coredata.ExportJobTypeAuditLog:
return s.streamAuditLogEntries(ctx, conn, scope, exportJob.OrganizationID, args, enc)
case coredata.ExportJobTypeSCIMEvent:
return s.streamSCIMEvents(ctx, conn, scope, exportJob.OrganizationID, args, enc)
default:
return fmt.Errorf("unsupported log export type: %q", exportJob.Type)
}
},
)
}
func (s *LogExportService) streamAuditLogEntries(
ctx context.Context,
conn pg.Querier,
scope coredata.Scoper,
organizationID gid.GID,
args *coredata.LogExportArguments,
enc *json.Encoder,
) error {
filter := coredata.NewAuditLogEntryFilter().
WithCreatedAtGte(args.FromTime).
WithCreatedAtLt(args.ToTime)
return page.WalkAll(
ctx,
page.OrderBy[coredata.AuditLogEntryOrderField]{
Field: coredata.AuditLogEntryOrderFieldCreatedAt,
Direction: page.OrderDirectionAsc,
},
func(ctx context.Context, cursor *page.Cursor[coredata.AuditLogEntryOrderField]) ([]*coredata.AuditLogEntry, error) {
var batch coredata.AuditLogEntries
if err := batch.LoadByOrganizationID(
ctx,
conn,
scope,
organizationID,
cursor,
filter,
); err != nil {
return nil, err
}
return batch, nil
},
func(entries []*coredata.AuditLogEntry) error {
for _, entry := range entries {
if err := enc.Encode(entry); err != nil {
return fmt.Errorf("cannot encode audit log entry: %w", err)
}
}
return nil
},
)
}
func (s *LogExportService) streamSCIMEvents(
ctx context.Context,
conn pg.Querier,
scope coredata.Scoper,
organizationID gid.GID,
args *coredata.LogExportArguments,
enc *json.Encoder,
) error {
filter := coredata.NewSCIMEventFilter().
WithCreatedAtGte(args.FromTime).
WithCreatedAtLt(args.ToTime)
return page.WalkAll(
ctx,
page.OrderBy[coredata.SCIMEventOrderField]{
Field: coredata.SCIMEventOrderFieldCreatedAt,
Direction: page.OrderDirectionAsc,
},
func(ctx context.Context, cursor *page.Cursor[coredata.SCIMEventOrderField]) ([]*coredata.SCIMEvent, error) {
var batch coredata.SCIMEvents
if err := batch.LoadByOrganizationID(
ctx,
conn,
scope,
organizationID,
cursor,
filter,
); err != nil {
return nil, err
}
return batch, nil
},
func(events []*coredata.SCIMEvent) error {
for _, event := range events {
if err := enc.Encode(event); err != nil {
return fmt.Errorf("cannot encode SCIM event: %w", err)
}
}
return nil
},
)
}

2
pkg/iam/oauth2_scopes.go
View File

@@ -109,5 +109,7 @@ var IAMOAuth2ScopeMappings = map[coredata.OAuth2Scope][]string{
ActionSCIMBridgeUpdate,
ActionSCIMBridgeDelete,
ActionOAuth2ConsentApprove,
ActionAuditLogExport,
ActionSCIMEventExport,
},
}

80
pkg/iam/organization_service.go
View File

@@ -22,6 +22,7 @@ package iam
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
@@ -1565,7 +1566,14 @@ func (s OrganizationService) ListSCIMEvents(
err := s.pg.WithConn(
ctx,
func(ctx context.Context, conn pg.Querier) error {
err := scimEvents.LoadByOrganizationID(ctx, conn, scope, organizationID, cursor)
err := scimEvents.LoadByOrganizationID(
ctx,
conn,
scope,
organizationID,
cursor,
coredata.NewSCIMEventFilter(),
)
if err != nil {
return fmt.Errorf("cannot load scim events: %w", err)
}
@@ -2353,7 +2361,7 @@ func (s *OrganizationService) ListAuditLogEntries(
err := s.pg.WithConn(
ctx,
func(ctx context.Context, conn pg.Querier) error {
if err := entries.LoadAllByOrganizationID(ctx, conn, scope, organizationID, cursor, filter); err != nil {
if err := entries.LoadByOrganizationID(ctx, conn, scope, organizationID, cursor, filter); err != nil {
return fmt.Errorf("cannot load audit log entries: %w", err)
}
@@ -2393,3 +2401,71 @@ func (s *OrganizationService) CountAuditLogEntries(
return count, err
}
type RequestLogExportRequest struct {
OrganizationID gid.GID
Type coredata.ExportJobType
FromTime time.Time
ToTime time.Time
RecipientEmail mail.Addr
RecipientName string
}
func (s *OrganizationService) RequestLogExport(
ctx context.Context,
scope coredata.Scoper,
req RequestLogExportRequest,
) (*coredata.ExportJob, error) {
if !req.FromTime.Before(req.ToTime) {
return nil, NewInvalidLogExportTimeRangeError()
}
if req.ToTime.After(req.FromTime.AddDate(maxLogExportTimeRangeYears, 0, 0)) {
return nil, NewLogExportTimeRangeTooLargeError()
}
switch req.Type {
case coredata.ExportJobTypeAuditLog, coredata.ExportJobTypeSCIMEvent:
default:
return nil, fmt.Errorf("unsupported log export type: %q", req.Type)
}
arguments, err := json.Marshal(coredata.LogExportArguments{
FromTime: req.FromTime,
ToTime: req.ToTime,
})
if err != nil {
return nil, fmt.Errorf("cannot marshal log export arguments: %w", err)
}
exportJob := &coredata.ExportJob{}
err = s.pg.WithTx(
ctx,
func(ctx context.Context, tx pg.Tx) error {
now := time.Now()
exportJob = &coredata.ExportJob{
ID: gid.New(scope.GetTenantID(), coredata.ExportJobEntityType),
OrganizationID: req.OrganizationID,
Type: req.Type,
Arguments: arguments,
Status: coredata.ExportJobStatusPending,
RecipientEmail: req.RecipientEmail,
RecipientName: req.RecipientName,
CreatedAt: now,
}
if err := exportJob.Insert(ctx, tx, scope); err != nil {
return fmt.Errorf("cannot insert export job: %w", err)
}
return nil
},
)
if err != nil {
return nil, fmt.Errorf("cannot request log export: %w", err)
}
return exportJob, nil
}

8
pkg/iam/service.go
View File

@@ -82,6 +82,7 @@ type (
OAuth2ScopeRegistry *oauth2scope.Registry
samlDomainVerifier *SAMLDomainVerifier
LogExports *LogExportService
}
Config struct {
@@ -241,6 +242,13 @@ func NewService(
cfg.DomainVerificationResolverAddr,
)
svc.LogExports = NewLogExportService(
pgClient,
fm,
cfg.Bucket,
cfg.BaseURL.String(),
)
return svc, nil
}

87
pkg/page/load_all.go
View File

@@ -27,7 +27,9 @@ import (
// MaxLoadAllPages caps how many pages LoadAll walks, bounding a single
// call to MaxLoadAllPages*MaxCursorSize rows. Past that, LoadAll errors
// rather than materialising an unbounded set.
// rather than materialising an unbounded set. WalkAll is uncapped: it
// streams page-by-page and is meant for callers that can process rows
// without holding the full set in memory.
const MaxLoadAllPages = 20
// Loader runs one paginated query for the given cursor and returns the
@@ -36,6 +38,39 @@ const MaxLoadAllPages = 20
// LoadBy* on a fresh receiver).
type Loader[T Paginable[U], U OrderField] func(ctx context.Context, cursor *Cursor[U]) ([]T, error)
// WalkAll walks every matching row via keyset pagination, advancing a
// MaxCursorSize forward cursor until no rows remain, and invokes walk with
// every page of rows. Unlike LoadAll, it does not apply MaxLoadAllPages.
func WalkAll[T Paginable[U], U OrderField](
ctx context.Context,
orderBy OrderBy[U],
fetch Loader[T, U],
walk func(rows []T) error,
) error {
var key *CursorKey
for {
cursor := NewCursor(MaxCursorSize, key, Head, orderBy)
rows, err := fetch(ctx, cursor)
if err != nil {
return fmt.Errorf("cannot load all rows: %w", err)
}
p := NewPage(rows, cursor)
if err := walk(p.Data); err != nil {
return err
}
if !p.Info.HasNext {
return nil
}
k := p.Last().CursorKey(orderBy.Field)
key = &k
}
}
// LoadAll walks every matching row via keyset pagination, advancing a
// MaxCursorSize forward cursor until no rows remain, and returns them
// concatenated. fetch runs the paginated query for the cursor. It errors
@@ -46,36 +81,34 @@ func LoadAll[T Paginable[U], U OrderField](
fetch Loader[T, U],
) ([]T, error) {
var (
all []T
key *CursorKey
all []T
pages int
)
for page := 0; ; page++ {
if page >= MaxLoadAllPages {
return nil, fmt.Errorf(
"cannot load all rows: result set exceeds %d rows (%d pages of %d)",
MaxLoadAllPages*MaxCursorSize,
MaxLoadAllPages,
MaxCursorSize,
)
}
err := WalkAll(
ctx,
orderBy,
func(ctx context.Context, cursor *Cursor[U]) ([]T, error) {
if pages >= MaxLoadAllPages {
return nil, fmt.Errorf(
"cannot load all rows: result set exceeds %d rows (%d pages of %d)",
MaxLoadAllPages*MaxCursorSize,
MaxLoadAllPages,
MaxCursorSize,
)
}
cursor := NewCursor(MaxCursorSize, key, Head, orderBy)
pages++
rows, err := fetch(ctx, cursor)
if err != nil {
return nil, fmt.Errorf("cannot load all rows: %w", err)
}
p := NewPage(rows, cursor)
all = append(all, p.Data...)
if !p.Info.HasNext {
break
}
k := p.Last().CursorKey(orderBy.Field)
key = &k
return fetch(ctx, cursor)
},
func(rows []T) error {
all = append(all, rows...)
return nil
},
)
if err != nil {
return nil, err
}
return all, nil

29
pkg/page/load_all_test.go
View File

@@ -183,3 +183,32 @@ func TestLoadAllRefusesUnboundedResultSet(t *testing.T) {
assert.Contains(t, err.Error(), "result set exceeds")
assert.Equal(t, MaxLoadAllPages, fetchs, "stops after walking the max number of pages")
}
func TestWalkAllHasNoPageCap(t *testing.T) {
t.Parallel()
// More pages than MaxLoadAllPages: WalkAll must keep going.
store := newLoadAllStore(MaxLoadAllPages*MaxCursorSize + 1)
fetchs := 0
var got []*loadAllItem
err := WalkAll(
context.Background(),
ascOrderBy(),
func(_ context.Context, cursor *Cursor[testOrderField]) ([]*loadAllItem, error) {
fetchs++
return keysetPage(store, cursor), nil
},
func(rows []*loadAllItem) error {
got = append(got, rows...)
return nil
},
)
require.NoError(t, err)
require.Len(t, got, len(store))
assert.Equal(t, loadAllValues(store), loadAllValues(got))
assert.Greater(t, fetchs, MaxLoadAllPages)
}

164
pkg/probo/export_job_worker.go Normal file
View File

@@ -0,0 +1,164 @@
// Copyright (c) 2026 Probo Inc <hello@probo.com>.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
package probo
import (
"context"
"errors"
"fmt"
"time"
"go.gearno.de/kit/log"
"go.gearno.de/kit/pg"
"go.gearno.de/kit/worker"
"go.probo.inc/probo/pkg/coredata"
"go.probo.inc/probo/pkg/gid"
)
const defaultExportJobStaleAfter = 25 * time.Minute
type (
exportJobHandler struct {
service *Service
logger *log.Logger
staleAfter time.Duration
}
ExportJobWorkerConfig struct {
StaleAfter time.Duration
}
)
var (
_ worker.Handler[coredata.ExportJob] = (*exportJobHandler)(nil)
_ worker.StaleRecoverer = (*exportJobHandler)(nil)
)
func NewExportJobWorker(
service *Service,
logger *log.Logger,
cfg ExportJobWorkerConfig,
opts ...worker.Option,
) *worker.Worker[coredata.ExportJob] {
staleAfter := cfg.StaleAfter
if staleAfter <= 0 {
staleAfter = defaultExportJobStaleAfter
}
h := &exportJobHandler{
service: service,
logger: logger,
staleAfter: staleAfter,
}
return worker.New(
"export-job-worker",
h,
logger,
opts...,
)
}
func (h *exportJobHandler) Claim(ctx context.Context) (coredata.ExportJob, error) {
exportJob, err := h.service.lockExportJob(ctx)
if err != nil {
if errors.Is(err, coredata.ErrNoExportJobAvailable) {
return coredata.ExportJob{}, worker.ErrNoTask
}
return coredata.ExportJob{}, err
}
return *exportJob, nil
}
func (h *exportJobHandler) Process(ctx context.Context, exportJob coredata.ExportJob) error {
stopHeartbeat := h.startHeartbeat(ctx, exportJob.ID)
defer stopHeartbeat()
if err := h.service.processExportJob(ctx, &exportJob); err != nil {
h.logger.ErrorCtx(
ctx,
"export job worker failure",
log.Error(err),
log.String("export_job_id", exportJob.ID.String()),
log.String("export_job_type", exportJob.Type.String()),
)
return err
}
return nil
}
func (h *exportJobHandler) RecoverStale(ctx context.Context) error {
return h.service.pg.WithConn(
ctx,
func(ctx context.Context, conn pg.Querier) error {
if err := coredata.ResetStaleExportJobs(ctx, conn, h.staleAfter); err != nil {
return fmt.Errorf("cannot reset stale export jobs: %w", err)
}
return nil
},
)
}
func (h *exportJobHandler) startHeartbeat(ctx context.Context, exportJobID gid.GID) func() {
done := make(chan struct{})
interval := max(h.staleAfter/2, time.Second)
go func() {
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-done:
return
case <-ctx.Done():
return
case <-ticker.C:
if err := h.service.pg.WithConn(
ctx,
func(ctx context.Context, conn pg.Querier) error {
return coredata.TouchExportJobLease(
ctx,
conn,
coredata.NewScope(exportJobID.TenantID()),
exportJobID,
)
},
); err != nil {
h.logger.ErrorCtx(
ctx,
"cannot renew export job lease",
log.Error(err),
log.String("export_job_id", exportJobID.String()),
)
}
}
}
}()
return func() { close(done) }
}

20
pkg/probo/service.go
View File

@@ -118,6 +118,7 @@ type (
GeneratedDocuments *GeneratedDocumentService
Files *FileService
SlackMessages *slack.Service
LogExports ExportService
}
)
@@ -231,16 +232,12 @@ func NewService(
svc.GeneratedDocuments = &GeneratedDocumentService{svc: svc}
svc.Files = &FileService{svc: svc}
svc.SlackMessages = slackService
svc.LogExports = iamService.LogExports
return svc, nil
}
func (s *Service) ExportJob(ctx context.Context) error {
exportJob, err := s.lockExportJob(ctx)
if err != nil {
return fmt.Errorf("cannot lock export job: %w", err)
}
func (s *Service) processExportJob(ctx context.Context, exportJob *coredata.ExportJob) error {
scope := coredata.NewScope(exportJob.ID.TenantID())
var exportService ExportService
@@ -250,6 +247,8 @@ func (s *Service) ExportJob(ctx context.Context) error {
exportService = s.Frameworks
case coredata.ExportJobTypeDocument:
exportService = s.Documents
case coredata.ExportJobTypeAuditLog, coredata.ExportJobTypeSCIMEvent:
exportService = s.LogExports
default:
unknownTypeErr := fmt.Errorf("unknown export job type: %q", exportJob.Type)
if err := s.commitFailedExport(ctx, exportJob, unknownTypeErr); err != nil {
@@ -314,10 +313,7 @@ func (s *Service) lockExportJob(ctx context.Context) (*coredata.ExportJob, error
scope = coredata.NewScope(exportJob.ID.TenantID())
exportJob.Status = coredata.ExportJobStatusProcessing
exportJob.StartedAt = new(time.Now())
if err := exportJob.Update(ctx, tx, scope); err != nil {
if err := exportJob.MarkProcessing(ctx, tx, scope); err != nil {
return fmt.Errorf("cannot update %s export job: %w", exportJob.Type, err)
}
@@ -341,7 +337,7 @@ func (s *Service) commitFailedExport(ctx context.Context, exportJob *coredata.Ex
ctx,
func(ctx context.Context, tx pg.Tx) error {
scope := coredata.NewScope(exportJob.ID.TenantID())
if err := exportJob.Update(ctx, tx, scope); err != nil {
if err := exportJob.UpdateIfStatus(ctx, tx, scope, coredata.ExportJobStatusProcessing); err != nil {
return fmt.Errorf("cannot update %s export job: %w", exportJob.Type, err)
}
@@ -358,7 +354,7 @@ func (s *Service) commitSuccessfulExport(ctx context.Context, exportJob *coredat
ctx,
func(ctx context.Context, tx pg.Tx) error {
scope := coredata.NewScope(exportJob.ID.TenantID())
if err := exportJob.Update(ctx, tx, scope); err != nil {
if err := exportJob.UpdateIfStatus(ctx, tx, scope, coredata.ExportJobStatusProcessing); err != nil {
return fmt.Errorf("cannot update %s export job: %w", exportJob.Type, err)
}

19
pkg/probod/probod.go
View File

@@ -1218,19 +1218,12 @@ func (impl *Implm) runExportJob(
proboService *probo.Service,
l *log.Logger,
) error {
LOOP:
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(30 * time.Second):
if err := proboService.ExportJob(ctx); err != nil {
if !errors.Is(err, coredata.ErrNoExportJobAvailable) {
l.ErrorCtx(ctx, "cannot process export job", log.Error(err))
}
}
goto LOOP
}
return probo.NewExportJobWorker(
proboService,
l,
probo.ExportJobWorkerConfig{},
worker.WithMaxConcurrency(3),
).Run(ctx)
}
func (impl *Implm) runApiServer(

39
pkg/server/api/connect/v1/audit_log_resolvers.go
View File

@@ -7,9 +7,12 @@ package connect_v1
import (
"context"
"errors"
"go.gearno.de/kit/log"
"go.probo.inc/probo/pkg/coredata"
"go.probo.inc/probo/pkg/iam"
"go.probo.inc/probo/pkg/server/api/authn"
"go.probo.inc/probo/pkg/server/api/connect/v1/schema"
"go.probo.inc/probo/pkg/server/api/connect/v1/types"
"go.probo.inc/probo/pkg/server/gqlutils"
@@ -41,6 +44,42 @@ func (r *auditLogEntryConnectionResolver) TotalCount(ctx context.Context, obj *t
return count, nil
}
// RequestAuditLogExport is the resolver for the requestAuditLogExport field.
func (r *mutationResolver) RequestAuditLogExport(ctx context.Context, input types.RequestAuditLogExportInput) (*types.RequestAuditLogExportPayload, error) {
scope, err := r.authorize(ctx, input.OrganizationID, iam.ActionAuditLogExport)
if err != nil {
return nil, err
}
identity := authn.IdentityFromContext(ctx)
logExport, err := r.iam.OrganizationService.RequestLogExport(
ctx,
scope,
iam.RequestLogExportRequest{
OrganizationID: input.OrganizationID,
Type: coredata.ExportJobTypeAuditLog,
FromTime: input.FromTime,
ToTime: input.ToTime,
RecipientEmail: identity.EmailAddress,
RecipientName: identity.FullName,
},
)
if err != nil {
if _, ok := errors.AsType[*iam.ErrInvalidLogExportTimeRange](err); ok {
return nil, gqlutils.Invalid(ctx, err)
}
r.logger.ErrorCtx(ctx, "cannot request audit log export", log.Error(err))
return nil, gqlutils.Internal(ctx)
}
return &types.RequestAuditLogExportPayload{
ExportJobID: logExport.ID,
}, nil
}
// AuditLogEntry returns schema.AuditLogEntryResolver implementation.
func (r *Resolver) AuditLogEntry() schema.AuditLogEntryResolver { return &auditLogEntryResolver{r} }

16
pkg/server/api/connect/v1/graphql/audit_log.graphql
View File

@@ -70,3 +70,19 @@ type AuditLogEntryEdge {
cursor: CursorKey!
node: AuditLogEntry!
}
extend type Mutation {
requestAuditLogExport(
input: RequestAuditLogExportInput!
): RequestAuditLogExportPayload @authentication(required: PRESENT) @sessionOnly
}
input RequestAuditLogExportInput {
organizationId: ID!
fromTime: Datetime!
toTime: Datetime!
}
type RequestAuditLogExportPayload {
exportJobId: ID!
}

13
pkg/server/api/connect/v1/graphql/scim.graphql
View File

@@ -140,6 +140,9 @@ extend type Mutation {
updateSCIMBridge(
input: UpdateSCIMBridgeInput!
): UpdateSCIMBridgePayload @authentication(required: PRESENT)
requestSCIMEventExport(
input: RequestSCIMEventExportInput!
): RequestSCIMEventExportPayload @authentication(required: PRESENT) @sessionOnly
}
input CreateSCIMConfigurationInput {
@@ -181,3 +184,13 @@ type RegenerateSCIMTokenPayload {
type UpdateSCIMBridgePayload {
scimBridge: SCIMBridge!
}
input RequestSCIMEventExportInput {
organizationId: ID!
fromTime: Datetime!
toTime: Datetime!
}
type RequestSCIMEventExportPayload {
exportJobId: ID!
}

37
pkg/server/api/connect/v1/scim_resolvers.go
View File

@@ -13,6 +13,7 @@ import (
"go.probo.inc/probo/pkg/coredata"
"go.probo.inc/probo/pkg/iam"
"go.probo.inc/probo/pkg/page"
"go.probo.inc/probo/pkg/server/api/authn"
"go.probo.inc/probo/pkg/server/api/connect/v1/schema"
"go.probo.inc/probo/pkg/server/api/connect/v1/types"
"go.probo.inc/probo/pkg/server/gqlutils"
@@ -107,6 +108,42 @@ func (r *mutationResolver) UpdateSCIMBridge(ctx context.Context, input types.Upd
}, nil
}
// RequestSCIMEventExport is the resolver for the requestSCIMEventExport field.
func (r *mutationResolver) RequestSCIMEventExport(ctx context.Context, input types.RequestSCIMEventExportInput) (*types.RequestSCIMEventExportPayload, error) {
scope, err := r.authorize(ctx, input.OrganizationID, iam.ActionSCIMEventExport)
if err != nil {
return nil, err
}
identity := authn.IdentityFromContext(ctx)
logExport, err := r.iam.OrganizationService.RequestLogExport(
ctx,
scope,
iam.RequestLogExportRequest{
OrganizationID: input.OrganizationID,
Type: coredata.ExportJobTypeSCIMEvent,
FromTime: input.FromTime,
ToTime: input.ToTime,
RecipientEmail: identity.EmailAddress,
RecipientName: identity.FullName,
},
)
if err != nil {
if _, ok := errors.AsType[*iam.ErrInvalidLogExportTimeRange](err); ok {
return nil, gqlutils.Invalid(ctx, err)
}
r.logger.ErrorCtx(ctx, "cannot request SCIM event export", log.Error(err))
return nil, gqlutils.Internal(ctx)
}
return &types.RequestSCIMEventExportPayload{
ExportJobID: logExport.ID,
}, nil
}
// ScimConfiguration is the resolver for the scimConfiguration field.
func (r *sCIMBridgeResolver) ScimConfiguration(ctx context.Context, obj *types.SCIMBridge) (*types.SCIMConfiguration, error) {
if _, err := r.authorize(ctx, obj.ScimConfiguration.ID, iam.ActionSCIMConfigurationGet); err != nil {

37
pkg/server/api/console/v1/audit_log_resolvers.go
View File

@@ -14,6 +14,7 @@ import (
"go.probo.inc/probo/pkg/coredata"
"go.probo.inc/probo/pkg/iam"
"go.probo.inc/probo/pkg/probo"
"go.probo.inc/probo/pkg/server/api/authn"
"go.probo.inc/probo/pkg/server/api/console/v1/dataloader"
"go.probo.inc/probo/pkg/server/api/console/v1/schema"
"go.probo.inc/probo/pkg/server/api/console/v1/types"
@@ -67,6 +68,42 @@ func (r *auditLogEntryConnectionResolver) TotalCount(ctx context.Context, obj *t
return count, nil
}
// RequestAuditLogExport is the resolver for the requestAuditLogExport field.
func (r *mutationResolver) RequestAuditLogExport(ctx context.Context, input types.RequestAuditLogExportInput) (*types.RequestAuditLogExportPayload, error) {
scope, err := r.authorize(ctx, input.OrganizationID, iam.ActionAuditLogExport)
if err != nil {
return nil, err
}
identity := authn.IdentityFromContext(ctx)
logExport, err := r.iam.OrganizationService.RequestLogExport(
ctx,
scope,
iam.RequestLogExportRequest{
OrganizationID: input.OrganizationID,
Type: coredata.ExportJobTypeAuditLog,
FromTime: input.FromTime,
ToTime: input.ToTime,
RecipientEmail: identity.EmailAddress,
RecipientName: identity.FullName,
},
)
if err != nil {
if _, ok := errors.AsType[*iam.ErrInvalidLogExportTimeRange](err); ok {
return nil, gqlutils.Invalid(ctx, err)
}
r.logger.ErrorCtx(ctx, "cannot request audit log export", log.Error(err))
return nil, gqlutils.Internal(ctx)
}
return &types.RequestAuditLogExportPayload{
ExportJobID: logExport.ID,
}, nil
}
// AuditLogEntry returns schema.AuditLogEntryResolver implementation.
func (r *Resolver) AuditLogEntry() schema.AuditLogEntryResolver { return &auditLogEntryResolver{r} }

16
pkg/server/api/console/v1/graphql/audit_log.graphql
View File

@@ -68,3 +68,19 @@ type AuditLogEntryEdge {
cursor: CursorKey!
node: AuditLogEntry!
}
extend type Mutation {
requestAuditLogExport(
input: RequestAuditLogExportInput!
): RequestAuditLogExportPayload!
}
input RequestAuditLogExportInput {
organizationId: ID!
fromTime: Datetime!
toTime: Datetime!
}
type RequestAuditLogExportPayload {
exportJobId: ID!
}

66
pkg/server/api/mcp/v1/schema.resolvers.go
View File

@@ -7351,3 +7351,69 @@ func (r *Resolver) DeleteCommitmentTool(ctx context.Context, req *mcp.CallToolRe
return nil, types.DeleteCommitmentOutput{DeletedCommitmentID: input.ID}, nil
}
func (r *Resolver) RequestAuditLogExportTool(ctx context.Context, req *mcp.CallToolRequest, input *types.RequestAuditLogExportInput) (*mcp.CallToolResult, types.RequestAuditLogExportOutput, error) {
scope, err := r.Authorize(ctx, input.OrganizationID, iam.ActionAuditLogExport)
if err != nil {
return nil, types.RequestAuditLogExportOutput{}, err
}
identity := authn.IdentityFromContext(ctx)
logExport, err := r.iamSvc.OrganizationService.RequestLogExport(
ctx,
scope,
iam.RequestLogExportRequest{
OrganizationID: input.OrganizationID,
Type: coredata.ExportJobTypeAuditLog,
FromTime: input.FromTime,
ToTime: input.ToTime,
RecipientEmail: identity.EmailAddress,
RecipientName: identity.FullName,
},
)
if err != nil {
if _, ok := errors.AsType[*iam.ErrInvalidLogExportTimeRange](err); ok {
return nil, types.RequestAuditLogExportOutput{}, err
}
return nil, types.RequestAuditLogExportOutput{}, fmt.Errorf("cannot request audit log export: %w", err)
}
return nil, types.RequestAuditLogExportOutput{
ExportJobID: logExport.ID,
}, nil
}
func (r *Resolver) RequestSCIMEventExportTool(ctx context.Context, req *mcp.CallToolRequest, input *types.RequestSCIMEventExportInput) (*mcp.CallToolResult, types.RequestSCIMEventExportOutput, error) {
scope, err := r.Authorize(ctx, input.OrganizationID, iam.ActionSCIMEventExport)
if err != nil {
return nil, types.RequestSCIMEventExportOutput{}, err
}
identity := authn.IdentityFromContext(ctx)
logExport, err := r.iamSvc.OrganizationService.RequestLogExport(
ctx,
scope,
iam.RequestLogExportRequest{
OrganizationID: input.OrganizationID,
Type: coredata.ExportJobTypeSCIMEvent,
FromTime: input.FromTime,
ToTime: input.ToTime,
RecipientEmail: identity.EmailAddress,
RecipientName: identity.FullName,
},
)
if err != nil {
if _, ok := errors.AsType[*iam.ErrInvalidLogExportTimeRange](err); ok {
return nil, types.RequestSCIMEventExportOutput{}, err
}
return nil, types.RequestSCIMEventExportOutput{}, fmt.Errorf("cannot request SCIM event export: %w", err)
}
return nil, types.RequestSCIMEventExportOutput{
ExportJobID: logExport.ID,
}, nil
}

78
pkg/server/api/mcp/v1/specification.yaml
View File

@@ -8487,6 +8487,66 @@ components:
items:
$ref: "#/components/schemas/AuditLogEntry"
RequestAuditLogExportInput:
type: object
required:
- organization_id
- from_time
- to_time
properties:
organization_id:
$ref: "#/components/schemas/GID"
description: Organization ID
from_time:
type: string
format: date-time
go.probo.inc/mcpgen/type: time.Time
description: Start of the time range (inclusive). The range must not exceed 1 year.
to_time:
type: string
format: date-time
go.probo.inc/mcpgen/type: time.Time
description: End of the time range (exclusive). The range must not exceed 1 year.
RequestAuditLogExportOutput:
type: object
required:
- export_job_id
properties:
export_job_id:
$ref: "#/components/schemas/GID"
description: ID of the created log export
RequestSCIMEventExportInput:
type: object
required:
- organization_id
- from_time
- to_time
properties:
organization_id:
$ref: "#/components/schemas/GID"
description: Organization ID
from_time:
type: string
format: date-time
go.probo.inc/mcpgen/type: time.Time
description: Start of the time range (inclusive). The range must not exceed 1 year.
to_time:
type: string
format: date-time
go.probo.inc/mcpgen/type: time.Time
description: End of the time range (exclusive). The range must not exceed 1 year.
RequestSCIMEventExportOutput:
type: object
required:
- export_job_id
properties:
export_job_id:
$ref: "#/components/schemas/GID"
description: ID of the created log export
AuditLogEntry:
type: object
required:
@@ -13955,6 +14015,24 @@ tools:
$ref: "#/components/schemas/ListAuditLogEntriesInput"
outputSchema:
$ref: "#/components/schemas/ListAuditLogEntriesOutput"
- name: requestAuditLogExport
description: Request an export of audit log entries for the organization within a time range. The export will be emailed as a JSONL download link.
hints:
readonly: false
idempotent: false
inputSchema:
$ref: "#/components/schemas/RequestAuditLogExportInput"
outputSchema:
$ref: "#/components/schemas/RequestAuditLogExportOutput"
- name: requestSCIMEventExport
description: Request an export of SCIM events for the organization within a time range. The export will be emailed as a JSONL download link.
hints:
readonly: false
idempotent: false
inputSchema:
$ref: "#/components/schemas/RequestSCIMEventExportInput"
outputSchema:
$ref: "#/components/schemas/RequestSCIMEventExportOutput"
- name: listWebhookSubscriptions
description: List all webhook subscriptions for the organization
hints: