Add log export for audit logs and SCIM events
Route audit-log and SCIM-event exports through export_jobs with typed arguments, an iam BuildAndUploadExport/SendExportEmail implementation, and a concurrent export-job worker with stale recovery. Stream JSONL via page.WalkAll into S3, and expose the request flow on console, connect, MCP, and CLI. Co-authored-by: Bryan Frimin <bryan@getprobo.com> Signed-off-by: Sacha Al Himdani <sacha@probo.com>
This commit is contained in:
Sacha Al Himdani
164
pkg/probo/export_job_worker.go
Normal file
164
pkg/probo/export_job_worker.go
Normal file
@@ -0,0 +1,164 @@
|
||||
// Copyright (c) 2026 Probo Inc <hello@probo.com>.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
// SOFTWARE.
|
||||
|
||||
package probo
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"go.gearno.de/kit/log"
|
||||
"go.gearno.de/kit/pg"
|
||||
"go.gearno.de/kit/worker"
|
||||
"go.probo.inc/probo/pkg/coredata"
|
||||
"go.probo.inc/probo/pkg/gid"
|
||||
)
|
||||
|
||||
const defaultExportJobStaleAfter = 25 * time.Minute
|
||||
|
||||
type (
|
||||
exportJobHandler struct {
|
||||
service *Service
|
||||
logger *log.Logger
|
||||
staleAfter time.Duration
|
||||
}
|
||||
|
||||
ExportJobWorkerConfig struct {
|
||||
StaleAfter time.Duration
|
||||
}
|
||||
)
|
||||
|
||||
var (
|
||||
_ worker.Handler[coredata.ExportJob] = (*exportJobHandler)(nil)
|
||||
_ worker.StaleRecoverer = (*exportJobHandler)(nil)
|
||||
)
|
||||
|
||||
func NewExportJobWorker(
|
||||
service *Service,
|
||||
logger *log.Logger,
|
||||
cfg ExportJobWorkerConfig,
|
||||
opts ...worker.Option,
|
||||
) *worker.Worker[coredata.ExportJob] {
|
||||
staleAfter := cfg.StaleAfter
|
||||
if staleAfter <= 0 {
|
||||
staleAfter = defaultExportJobStaleAfter
|
||||
}
|
||||
|
||||
h := &exportJobHandler{
|
||||
service: service,
|
||||
logger: logger,
|
||||
staleAfter: staleAfter,
|
||||
}
|
||||
|
||||
return worker.New(
|
||||
"export-job-worker",
|
||||
h,
|
||||
logger,
|
||||
opts...,
|
||||
)
|
||||
}
|
||||
|
||||
func (h *exportJobHandler) Claim(ctx context.Context) (coredata.ExportJob, error) {
|
||||
exportJob, err := h.service.lockExportJob(ctx)
|
||||
if err != nil {
|
||||
if errors.Is(err, coredata.ErrNoExportJobAvailable) {
|
||||
return coredata.ExportJob{}, worker.ErrNoTask
|
||||
}
|
||||
|
||||
return coredata.ExportJob{}, err
|
||||
}
|
||||
|
||||
return *exportJob, nil
|
||||
}
|
||||
|
||||
func (h *exportJobHandler) Process(ctx context.Context, exportJob coredata.ExportJob) error {
|
||||
stopHeartbeat := h.startHeartbeat(ctx, exportJob.ID)
|
||||
defer stopHeartbeat()
|
||||
|
||||
if err := h.service.processExportJob(ctx, &exportJob); err != nil {
|
||||
h.logger.ErrorCtx(
|
||||
ctx,
|
||||
"export job worker failure",
|
||||
log.Error(err),
|
||||
log.String("export_job_id", exportJob.ID.String()),
|
||||
log.String("export_job_type", exportJob.Type.String()),
|
||||
)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *exportJobHandler) RecoverStale(ctx context.Context) error {
|
||||
return h.service.pg.WithConn(
|
||||
ctx,
|
||||
func(ctx context.Context, conn pg.Querier) error {
|
||||
if err := coredata.ResetStaleExportJobs(ctx, conn, h.staleAfter); err != nil {
|
||||
return fmt.Errorf("cannot reset stale export jobs: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func (h *exportJobHandler) startHeartbeat(ctx context.Context, exportJobID gid.GID) func() {
|
||||
done := make(chan struct{})
|
||||
|
||||
interval := max(h.staleAfter/2, time.Second)
|
||||
|
||||
go func() {
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-done:
|
||||
return
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if err := h.service.pg.WithConn(
|
||||
ctx,
|
||||
func(ctx context.Context, conn pg.Querier) error {
|
||||
return coredata.TouchExportJobLease(
|
||||
ctx,
|
||||
conn,
|
||||
coredata.NewScope(exportJobID.TenantID()),
|
||||
exportJobID,
|
||||
)
|
||||
},
|
||||
); err != nil {
|
||||
h.logger.ErrorCtx(
|
||||
ctx,
|
||||
"cannot renew export job lease",
|
||||
log.Error(err),
|
||||
log.String("export_job_id", exportJobID.String()),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return func() { close(done) }
|
||||
}
|
||||
@@ -118,6 +118,7 @@ type (
|
||||
GeneratedDocuments *GeneratedDocumentService
|
||||
Files *FileService
|
||||
SlackMessages *slack.Service
|
||||
LogExports ExportService
|
||||
}
|
||||
)
|
||||
|
||||
@@ -231,16 +232,12 @@ func NewService(
|
||||
svc.GeneratedDocuments = &GeneratedDocumentService{svc: svc}
|
||||
svc.Files = &FileService{svc: svc}
|
||||
svc.SlackMessages = slackService
|
||||
svc.LogExports = iamService.LogExports
|
||||
|
||||
return svc, nil
|
||||
}
|
||||
|
||||
func (s *Service) ExportJob(ctx context.Context) error {
|
||||
exportJob, err := s.lockExportJob(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot lock export job: %w", err)
|
||||
}
|
||||
|
||||
func (s *Service) processExportJob(ctx context.Context, exportJob *coredata.ExportJob) error {
|
||||
scope := coredata.NewScope(exportJob.ID.TenantID())
|
||||
|
||||
var exportService ExportService
|
||||
@@ -250,6 +247,8 @@ func (s *Service) ExportJob(ctx context.Context) error {
|
||||
exportService = s.Frameworks
|
||||
case coredata.ExportJobTypeDocument:
|
||||
exportService = s.Documents
|
||||
case coredata.ExportJobTypeAuditLog, coredata.ExportJobTypeSCIMEvent:
|
||||
exportService = s.LogExports
|
||||
default:
|
||||
unknownTypeErr := fmt.Errorf("unknown export job type: %q", exportJob.Type)
|
||||
if err := s.commitFailedExport(ctx, exportJob, unknownTypeErr); err != nil {
|
||||
@@ -314,10 +313,7 @@ func (s *Service) lockExportJob(ctx context.Context) (*coredata.ExportJob, error
|
||||
|
||||
scope = coredata.NewScope(exportJob.ID.TenantID())
|
||||
|
||||
exportJob.Status = coredata.ExportJobStatusProcessing
|
||||
|
||||
exportJob.StartedAt = new(time.Now())
|
||||
if err := exportJob.Update(ctx, tx, scope); err != nil {
|
||||
if err := exportJob.MarkProcessing(ctx, tx, scope); err != nil {
|
||||
return fmt.Errorf("cannot update %s export job: %w", exportJob.Type, err)
|
||||
}
|
||||
|
||||
@@ -341,7 +337,7 @@ func (s *Service) commitFailedExport(ctx context.Context, exportJob *coredata.Ex
|
||||
ctx,
|
||||
func(ctx context.Context, tx pg.Tx) error {
|
||||
scope := coredata.NewScope(exportJob.ID.TenantID())
|
||||
if err := exportJob.Update(ctx, tx, scope); err != nil {
|
||||
if err := exportJob.UpdateIfStatus(ctx, tx, scope, coredata.ExportJobStatusProcessing); err != nil {
|
||||
return fmt.Errorf("cannot update %s export job: %w", exportJob.Type, err)
|
||||
}
|
||||
|
||||
@@ -358,7 +354,7 @@ func (s *Service) commitSuccessfulExport(ctx context.Context, exportJob *coredat
|
||||
ctx,
|
||||
func(ctx context.Context, tx pg.Tx) error {
|
||||
scope := coredata.NewScope(exportJob.ID.TenantID())
|
||||
if err := exportJob.Update(ctx, tx, scope); err != nil {
|
||||
if err := exportJob.UpdateIfStatus(ctx, tx, scope, coredata.ExportJobStatusProcessing); err != nil {
|
||||
return fmt.Errorf("cannot update %s export job: %w", exportJob.Type, err)
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user