rohit/probo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Migrate workers to kit/worker

Browse Source 
Replace hand-rolled polling loops, semaphores, and WaitGroups
in all 7 background workers with go.gearno.de/kit/worker. Each
worker now implements Handler[T] (Claim/Process) and optionally
StaleRecoverer, gaining automatic Prometheus metrics and
OpenTelemetry tracing. Bumps kit from v0.3.0 to v0.5.0.

Signed-off-by: Bryan Frimin <bryan@getprobo.com>
This commit is contained in:
Bryan Frimin
2026-04-09 19:42:41 +02:00
parent 3ca0fac939
commit 43336078f4
13 changed files with 589 additions and 962 deletions
Download Patch File Download Diff File Expand all files Collapse all files

191
pkg/esign/completion_certificate_worker.go
View File

@@ -18,12 +18,12 @@ import (
"context"
"errors"
"fmt"
"sync"
"time"
"go.gearno.de/crypto/uuid"
"go.gearno.de/kit/log"
"go.gearno.de/kit/pg"
"go.gearno.de/kit/worker"
"go.gearno.de/x/ref"
"go.probo.inc/probo/pkg/coredata"
"go.probo.inc/probo/pkg/filemanager"
@@ -37,42 +37,20 @@ import (
// organization that owns the given trust center.
type EmailPresenterConfigFunc func(ctx context.Context, organizationID gid.GID) (emails.PresenterConfig, error)
type (
CompletionCertificateWorker struct {
pg *pg.Client
fileManager *filemanager.Service
certificateGen *CertificateGenerator
presenterConfigFunc EmailPresenterConfigFunc
bucket string
logger *log.Logger
interval time.Duration
staleAfter time.Duration
maxConcurrency int
}
CompletionCertificateWorkerOption func(*CompletionCertificateWorker)
)
type completionCertificateHandler struct {
pg *pg.Client
fileManager *filemanager.Service
certificateGen *CertificateGenerator
presenterConfigFunc EmailPresenterConfigFunc
bucket string
logger *log.Logger
staleAfter time.Duration
}
const (
certificateFilename = "certificate-of-completion.pdf"
)
func WithCompletionCertificateWorkerInterval(d time.Duration) CompletionCertificateWorkerOption {
return func(w *CompletionCertificateWorker) { w.interval = d }
}
func WithCompletionCertificateWorkerStaleAfter(d time.Duration) CompletionCertificateWorkerOption {
return func(w *CompletionCertificateWorker) { w.staleAfter = d }
}
func WithCompletionCertificateWorkerMaxConcurrency(n int) CompletionCertificateWorkerOption {
return func(w *CompletionCertificateWorker) {
if n > 0 {
w.maxConcurrency = n
}
}
}
func NewCompletionCertificateWorker(
pgClient *pg.Client,
fileManager *filemanager.Service,
@@ -80,123 +58,93 @@ func NewCompletionCertificateWorker(
presenterConfigFunc EmailPresenterConfigFunc,
bucket string,
logger *log.Logger,
opts ...CompletionCertificateWorkerOption,
) *CompletionCertificateWorker {
w := &CompletionCertificateWorker{
opts ...worker.Option,
) *worker.Worker[coredata.ElectronicSignature] {
h := &completionCertificateHandler{
pg: pgClient,
fileManager: fileManager,
certificateGen: certificateGen,
presenterConfigFunc: presenterConfigFunc,
bucket: bucket,
logger: logger,
interval: 10 * time.Second,
staleAfter: 10 * time.Minute,
maxConcurrency: 5,
}
for _, opt := range opts {
opt(w)
}
return w
return worker.New(
"completion-certificate-worker",
h,
logger,
opts...,
)
}
func (w *CompletionCertificateWorker) Run(ctx context.Context) error {
var (
wg sync.WaitGroup
sem = make(chan struct{}, w.maxConcurrency)
)
defer wg.Wait()
func (h *completionCertificateHandler) Claim(ctx context.Context) (coredata.ElectronicSignature, error) {
var signature coredata.ElectronicSignature
LOOP:
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(w.interval):
// From there we should not accept cancelations anymore.
nonCancelableCtx := context.WithoutCancel(ctx)
w.recoverStaleCertificateRows(nonCancelableCtx)
for {
if err := w.processNext(ctx, sem, &wg); err != nil {
if !errors.Is(err, coredata.ErrResourceNotFound) {
w.logger.ErrorCtx(ctx, "cannot process certificate", log.Error(err))
}
break
}
}
goto LOOP
}
}
func (w *CompletionCertificateWorker) processNext(ctx context.Context, sem chan struct{}, wg *sync.WaitGroup) error {
select {
case sem <- struct{}{}:
case <-ctx.Done(): // FIXME: this will never be fired
return ctx.Err()
}
var (
signature coredata.ElectronicSignature
now = time.Now()
// From there we should not accept cancelations anymore.
nonCancelableCtx = context.WithoutCancel(ctx)
)
if err := w.pg.WithTx(
nonCancelableCtx,
if err := h.pg.WithTx(
ctx,
func(ctx context.Context, tx pg.Tx) error {
if err := signature.LoadNextCompletedWithoutCertificateForUpdate(nonCancelableCtx, tx); err != nil {
if err := signature.LoadNextCompletedWithoutCertificateForUpdate(ctx, tx); err != nil {
return err
}
now := time.Now()
scope := coredata.NewScopeFromObjectID(signature.ID)
signature.CertificateProcessingStartedAt = &now
signature.AttemptCount++
signature.LastAttemptedAt = &now
signature.UpdatedAt = now
if err := signature.Update(nonCancelableCtx, tx, scope); err != nil {
if err := signature.Update(ctx, tx, scope); err != nil {
return fmt.Errorf("cannot update signature: %w", err)
}
return nil
},
); err != nil {
<-sem
return err
if errors.Is(err, coredata.ErrResourceNotFound) {
return coredata.ElectronicSignature{}, worker.ErrNoTask
}
return coredata.ElectronicSignature{}, err
}
wg.Add(1)
go func(signature coredata.ElectronicSignature) {
defer wg.Done()
defer func() { <-sem }()
return signature, nil
}
scope := coredata.NewScopeFromObjectID(signature.ID)
func (h *completionCertificateHandler) Process(ctx context.Context, signature coredata.ElectronicSignature) error {
scope := coredata.NewScopeFromObjectID(signature.ID)
if err := w.generateAndCommit(nonCancelableCtx, &signature); err != nil {
if err := w.handleCertFailure(nonCancelableCtx, &signature, scope, err); err != nil {
w.logger.ErrorCtx(nonCancelableCtx, "cannot handle certificate failure", log.Error(err))
}
if err := h.generateAndCommit(ctx, &signature); err != nil {
if err := h.handleCertFailure(ctx, &signature, scope, err); err != nil {
h.logger.ErrorCtx(ctx, "cannot handle certificate failure", log.Error(err))
}
}(signature)
return err
}
return nil
}
func (w *CompletionCertificateWorker) generateAndCommit(
func (h *completionCertificateHandler) RecoverStale(ctx context.Context) error {
return h.pg.WithConn(
ctx,
func(ctx context.Context, conn pg.Querier) error {
return coredata.ResetStaleCertificateProcessing(ctx, conn, h.staleAfter)
},
)
}
func (h *completionCertificateHandler) generateAndCommit(
ctx context.Context,
signature *coredata.ElectronicSignature,
) error {
var (
scope = coredata.NewScopeFromObjectID(signature.ID)
)
scope := coredata.NewScopeFromObjectID(signature.ID)
email, attachments, err := w.generateCertificate(ctx, signature, scope)
email, attachments, err := h.generateCertificate(ctx, signature, scope)
if err != nil {
return err
}
if err := w.pg.WithTx(
if err := h.pg.WithTx(
ctx,
func(ctx context.Context, tx pg.Tx) error {
signature.CertificateFileID = &attachments[1].FileID
@@ -232,7 +180,7 @@ func (w *CompletionCertificateWorker) generateAndCommit(
return nil
}
func (w *CompletionCertificateWorker) generateCertificate(
func (h *completionCertificateHandler) generateCertificate(
ctx context.Context,
signature *coredata.ElectronicSignature,
scope coredata.Scoper,
@@ -243,7 +191,7 @@ func (w *CompletionCertificateWorker) generateCertificate(
organization = coredata.Organization{}
)
if err := w.pg.WithConn(
if err := h.pg.WithConn(
ctx,
func(ctx context.Context, conn pg.Querier) error {
if err := events.LoadBySignatureID(ctx, conn, scope, signature.ID); err != nil {
@@ -264,7 +212,7 @@ func (w *CompletionCertificateWorker) generateCertificate(
return nil, nil, err
}
certificatePDFReader, err := w.certificateGen.Generate(ctx, signature, events)
certificatePDFReader, err := h.certificateGen.Generate(ctx, signature, events)
if err != nil {
return nil, nil, fmt.Errorf("cannot generate certificate: %w", err)
}
@@ -272,7 +220,7 @@ func (w *CompletionCertificateWorker) generateCertificate(
certificateOfCompletionFile := coredata.File{
ID: gid.New(scope.GetTenantID(), coredata.FileEntityType),
OrganizationID: signature.OrganizationID,
BucketName: w.bucket,
BucketName: h.bucket,
MimeType: "application/pdf",
FileName: certificateFilename,
FileKey: uuid.MustNewV4().String(),
@@ -281,7 +229,7 @@ func (w *CompletionCertificateWorker) generateCertificate(
UpdatedAt: time.Now(),
}
certificateOfCompletionFileSize, err := w.fileManager.PutFile(
certificateOfCompletionFileSize, err := h.fileManager.PutFile(
ctx,
&certificateOfCompletionFile,
certificatePDFReader,
@@ -296,7 +244,7 @@ func (w *CompletionCertificateWorker) generateCertificate(
certificateOfCompletionFile.FileSize = certificateOfCompletionFileSize
if err := w.pg.WithTx(
if err := h.pg.WithTx(
ctx,
func(ctx context.Context, tx pg.Tx) error {
if err := certificateOfCompletionFile.Insert(ctx, tx, scope); err != nil {
@@ -309,11 +257,11 @@ func (w *CompletionCertificateWorker) generateCertificate(
return nil, nil, err
}
presenterCfg, err := w.presenterConfigFunc(ctx, signature.OrganizationID)
presenterCfg, err := h.presenterConfigFunc(ctx, signature.OrganizationID)
if err != nil {
return nil, nil, fmt.Errorf("cannot resolve presenter config: %w", err)
}
emailPresenter := emails.NewPresenterFromConfig(w.fileManager, presenterCfg, ref.UnrefOrZero(signature.SignerFullName))
emailPresenter := emails.NewPresenterFromConfig(h.fileManager, presenterCfg, ref.UnrefOrZero(signature.SignerFullName))
docName := ref.UnrefOrZero(signature.DocumentName)
if docName == "" {
@@ -351,20 +299,20 @@ func (w *CompletionCertificateWorker) generateCertificate(
return email, attachments, nil
}
func (w *CompletionCertificateWorker) handleCertFailure(
func (h *completionCertificateHandler) handleCertFailure(
ctx context.Context,
signature *coredata.ElectronicSignature,
scope coredata.Scoper,
processingError error,
) error {
w.logger.ErrorCtx(
h.logger.ErrorCtx(
ctx,
"certificate worker failure",
log.Error(processingError),
log.String("signature_id", signature.ID.String()),
)
return w.pg.WithTx(
return h.pg.WithTx(
ctx,
func(ctx context.Context, tx pg.Tx) error {
errStr := processingError.Error()
@@ -384,14 +332,3 @@ func (w *CompletionCertificateWorker) handleCertFailure(
},
)
}
func (w *CompletionCertificateWorker) recoverStaleCertificateRows(ctx context.Context) {
if err := w.pg.WithConn(
ctx,
func(ctx context.Context, conn pg.Querier) error {
return coredata.ResetStaleCertificateProcessing(ctx, conn, w.staleAfter)
},
); err != nil {
w.logger.ErrorCtx(ctx, "cannot recover stale certificates", log.Error(err))
}
}

185
pkg/esign/sealing_worker.go
View File

@@ -18,11 +18,11 @@ import (
"context"
"errors"
"fmt"
"sync"
"time"
"go.gearno.de/kit/log"
"go.gearno.de/kit/pg"
"go.gearno.de/kit/worker"
"go.probo.inc/probo/pkg/coredata"
"go.probo.inc/probo/pkg/crypto/hash"
"go.probo.inc/probo/pkg/filemanager"
@@ -40,38 +40,24 @@ const (
)
type (
SealingWorker struct {
pg *pg.Client
fileManager *filemanager.Service
tsaClient *TSAClient
logger *log.Logger
interval time.Duration
tsaTimeout time.Duration
staleAfter time.Duration
maxConcurrency int
sealingHandler struct {
pg *pg.Client
fileManager *filemanager.Service
tsaClient *TSAClient
logger *log.Logger
tsaTimeout time.Duration
staleAfter time.Duration
}
SealingWorkerOption func(*SealingWorker)
SealingWorkerOption func(*sealingHandler)
)
func WithSealingWorkerInterval(d time.Duration) SealingWorkerOption {
return func(w *SealingWorker) { w.interval = d }
}
func WithSealingWorkerTSATimeout(d time.Duration) SealingWorkerOption {
return func(w *SealingWorker) { w.tsaTimeout = d }
return func(h *sealingHandler) { h.tsaTimeout = d }
}
func WithSealingWorkerStaleAfter(d time.Duration) SealingWorkerOption {
return func(w *SealingWorker) { w.staleAfter = d }
}
func WithSealingWorkerMaxConcurrency(n int) SealingWorkerOption {
return func(w *SealingWorker) {
if n > 0 {
w.maxConcurrency = n
}
}
return func(h *sealingHandler) { h.staleAfter = d }
}
func NewSealingWorker(
@@ -79,110 +65,82 @@ func NewSealingWorker(
fileManager *filemanager.Service,
tsaClient *TSAClient,
logger *log.Logger,
opts ...SealingWorkerOption,
) *SealingWorker {
w := &SealingWorker{
pg: pgClient,
fileManager: fileManager,
tsaClient: tsaClient,
logger: logger,
interval: 10 * time.Second,
tsaTimeout: 10 * time.Second,
staleAfter: 5 * time.Minute,
maxConcurrency: 5,
handlerOpts []SealingWorkerOption,
workerOpts ...worker.Option,
) *worker.Worker[coredata.ElectronicSignature] {
h := &sealingHandler{
pg: pgClient,
fileManager: fileManager,
tsaClient: tsaClient,
logger: logger,
tsaTimeout: 10 * time.Second,
staleAfter: 5 * time.Minute,
}
for _, opt := range opts {
opt(w)
for _, opt := range handlerOpts {
opt(h)
}
return w
return worker.New(
"sealing-worker",
h,
logger,
workerOpts...,
)
}
func (w *SealingWorker) Run(ctx context.Context) error {
var (
wg sync.WaitGroup
sem = make(chan struct{}, w.maxConcurrency)
)
func (h *sealingHandler) Claim(ctx context.Context) (coredata.ElectronicSignature, error) {
var signature coredata.ElectronicSignature
defer wg.Wait()
LOOP:
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(w.interval):
// From there we should not accept cancelations anymore.
nonCancelableCtx := context.WithoutCancel(ctx)
w.recoverStaleRows(nonCancelableCtx)
for {
if err := w.processNext(ctx, sem, &wg); err != nil {
if !errors.Is(err, coredata.ErrResourceNotFound) {
w.logger.ErrorCtx(nonCancelableCtx, "cannot claim signature", log.Error(err))
}
break
}
}
goto LOOP
}
}
func (w *SealingWorker) processNext(ctx context.Context, sem chan struct{}, wg *sync.WaitGroup) error {
select {
case sem <- struct{}{}:
case <-ctx.Done():
return ctx.Err()
}
var (
signature = coredata.ElectronicSignature{}
now = time.Now()
// From there we should not accept cancelations anymore.
nonCancelableCtx = context.WithoutCancel(ctx)
)
if err := w.pg.WithTx(
nonCancelableCtx,
if err := h.pg.WithTx(
ctx,
func(ctx context.Context, tx pg.Tx) error {
if err := signature.LoadNextAcceptedForUpdateSkipLocked(nonCancelableCtx, tx); err != nil {
if err := signature.LoadNextAcceptedForUpdateSkipLocked(ctx, tx); err != nil {
return err
}
now := time.Now()
signature.Status = coredata.ElectronicSignatureStatusProcessing
signature.ProcessingStartedAt = &now
signature.AttemptCount++
signature.LastAttemptedAt = &now
signature.UpdatedAt = now
if err := signature.Update(nonCancelableCtx, tx, coredata.NewNoScope()); err != nil {
if err := signature.Update(ctx, tx, coredata.NewNoScope()); err != nil {
return fmt.Errorf("cannot update signature: %w", err)
}
return nil
},
); err != nil {
<-sem
return err
if errors.Is(err, coredata.ErrResourceNotFound) {
return coredata.ElectronicSignature{}, worker.ErrNoTask
}
return coredata.ElectronicSignature{}, err
}
wg.Add(1)
go func(signature coredata.ElectronicSignature) {
defer wg.Done()
defer func() { <-sem }()
return signature, nil
}
if err := w.sealAndCommit(nonCancelableCtx, &signature); err != nil {
if err := w.failSignature(nonCancelableCtx, &signature, err); err != nil {
w.logger.ErrorCtx(nonCancelableCtx, "cannot fail signature", log.Error(err))
}
func (h *sealingHandler) Process(ctx context.Context, signature coredata.ElectronicSignature) error {
if err := h.sealAndCommit(ctx, &signature); err != nil {
if err := h.failSignature(ctx, &signature, err); err != nil {
h.logger.ErrorCtx(ctx, "cannot fail signature", log.Error(err))
}
}(signature)
return err
}
return nil
}
func (w *SealingWorker) sealAndCommit(
func (h *sealingHandler) RecoverStale(ctx context.Context) error {
return h.pg.WithConn(
ctx,
func(ctx context.Context, conn pg.Querier) error {
return coredata.ResetStaleProcessingSignatures(ctx, conn, h.staleAfter)
},
)
}
func (h *sealingHandler) sealAndCommit(
ctx context.Context,
signature *coredata.ElectronicSignature,
) error {
@@ -192,7 +150,7 @@ func (w *SealingWorker) sealAndCommit(
events []coredata.ElectronicSignatureEvent
)
if err := w.pg.WithConn(
if err := h.pg.WithConn(
ctx,
func(ctx context.Context, conn pg.Querier) error {
if err := file.LoadByID(ctx, conn, scope, signature.FileID); err != nil {
@@ -205,7 +163,7 @@ func (w *SealingWorker) sealAndCommit(
return fmt.Errorf("%w: %w", ErrLoadFile, err)
}
pdfBytes, err := w.fileManager.GetFileBytes(ctx, &file)
pdfBytes, err := h.fileManager.GetFileBytes(ctx, &file)
if err != nil {
return fmt.Errorf("%w: %w", ErrDownloadPDF, err)
}
@@ -227,9 +185,9 @@ func (w *SealingWorker) sealAndCommit(
),
)
tsaCtx, cancel := context.WithTimeout(ctx, w.tsaTimeout)
tsaCtx, cancel := context.WithTimeout(ctx, h.tsaTimeout)
defer cancel()
tsaToken, err := w.tsaClient.Timestamp(tsaCtx, []byte(seal))
tsaToken, err := h.tsaClient.Timestamp(tsaCtx, []byte(seal))
if err != nil {
return fmt.Errorf("%w: %w", ErrTSATimestamp, err)
}
@@ -242,7 +200,7 @@ func (w *SealingWorker) sealAndCommit(
),
)
if err := w.pg.WithTx(
if err := h.pg.WithTx(
ctx,
func(ctx context.Context, tx pg.Tx) error {
var current coredata.ElectronicSignature
@@ -282,21 +240,21 @@ func (w *SealingWorker) sealAndCommit(
return nil
}
func (w *SealingWorker) failSignature(
func (h *sealingHandler) failSignature(
ctx context.Context,
signature *coredata.ElectronicSignature,
processingError error,
) error {
scope := coredata.NewScopeFromObjectID(signature.ID)
w.logger.ErrorCtx(
h.logger.ErrorCtx(
ctx,
"sealing worker failure",
log.Error(processingError),
log.String("signature_id", signature.ID.String()),
)
return w.pg.WithTx(
return h.pg.WithTx(
ctx,
func(ctx context.Context, tx pg.Tx) error {
errStr := userFacingError(processingError)
@@ -336,14 +294,3 @@ func userFacingError(err error) string {
return "An unexpected error occurred while processing your signature."
}
}
func (w *SealingWorker) recoverStaleRows(ctx context.Context) {
if err := w.pg.WithConn(
ctx,
func(ctx context.Context, conn pg.Querier) error {
return coredata.ResetStaleProcessingSignatures(ctx, conn, w.staleAfter)
},
); err != nil {
w.logger.ErrorCtx(ctx, "cannot recover stale signatures", log.Error(err))
}
}

7
pkg/esign/service.go
View File

@@ -111,18 +111,17 @@ func NewService(
func (s *Service) Run(ctx context.Context, presenterConfigFunc EmailPresenterConfigFunc) error {
g, gctx := errgroup.WithContext(ctx)
nonCancelableCtx := context.WithoutCancel(ctx)
sealingWorkerCtx, stopSealingWorker := context.WithCancel(nonCancelableCtx)
sealingWorkerCtx, stopSealingWorker := context.WithCancel(ctx)
sealingWorker := NewSealingWorker(
s.pg,
s.fileManager,
s.tsaClient,
s.logger.Named("sealing-worker"),
nil,
)
g.Go(func() error { return sealingWorker.Run(sealingWorkerCtx) })
certWorkerCtx, stopCertWorker := context.WithCancel(nonCancelableCtx)
certWorkerCtx, stopCertWorker := context.WithCancel(ctx)
certWorker := NewCompletionCertificateWorker(
s.pg,
s.fileManager,