Migrate workers to kit/worker
Replace hand-rolled polling loops, semaphores, and WaitGroups in all 7 background workers with go.gearno.de/kit/worker. Each worker now implements Handler[T] (Claim/Process) and optionally StaleRecoverer, gaining automatic Prometheus metrics and OpenTelemetry tracing. Bumps kit from v0.3.0 to v0.5.0. Signed-off-by: Bryan Frimin <bryan@getprobo.com>
This commit is contained in:
Bryan Frimin
@@ -18,163 +18,114 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go.gearno.de/kit/log"
|
||||
"go.gearno.de/kit/pg"
|
||||
"go.gearno.de/kit/worker"
|
||||
"go.probo.inc/probo/pkg/coredata"
|
||||
)
|
||||
|
||||
type (
|
||||
MailingListWorker struct {
|
||||
service *Service
|
||||
pg *pg.Client
|
||||
logger *log.Logger
|
||||
interval time.Duration
|
||||
staleAfter time.Duration
|
||||
maxConcurrency int
|
||||
}
|
||||
|
||||
MailingListWorkerOption func(*MailingListWorker)
|
||||
)
|
||||
|
||||
func WithMailingListWorkerInterval(d time.Duration) MailingListWorkerOption {
|
||||
return func(w *MailingListWorker) { w.interval = d }
|
||||
}
|
||||
|
||||
func WithMailingListWorkerStaleAfter(d time.Duration) MailingListWorkerOption {
|
||||
return func(w *MailingListWorker) { w.staleAfter = d }
|
||||
}
|
||||
|
||||
func WithMailingListWorkerMaxConcurrency(n int) MailingListWorkerOption {
|
||||
return func(w *MailingListWorker) {
|
||||
if n > 0 {
|
||||
w.maxConcurrency = n
|
||||
}
|
||||
}
|
||||
type mailingListHandler struct {
|
||||
service *Service
|
||||
pg *pg.Client
|
||||
logger *log.Logger
|
||||
staleAfter time.Duration
|
||||
}
|
||||
|
||||
func NewMailingListWorker(
|
||||
service *Service,
|
||||
pgClient *pg.Client,
|
||||
logger *log.Logger,
|
||||
opts ...MailingListWorkerOption,
|
||||
) *MailingListWorker {
|
||||
w := &MailingListWorker{
|
||||
service: service,
|
||||
pg: pgClient,
|
||||
logger: logger,
|
||||
interval: 10 * time.Second,
|
||||
staleAfter: 5 * time.Minute,
|
||||
maxConcurrency: 5,
|
||||
opts ...worker.Option,
|
||||
) *worker.Worker[coredata.MailingListUpdate] {
|
||||
h := &mailingListHandler{
|
||||
service: service,
|
||||
pg: pgClient,
|
||||
logger: logger,
|
||||
staleAfter: 5 * time.Minute,
|
||||
}
|
||||
|
||||
for _, opt := range opts {
|
||||
opt(w)
|
||||
}
|
||||
|
||||
return w
|
||||
return worker.New(
|
||||
"mailing-list-worker",
|
||||
h,
|
||||
logger,
|
||||
opts...,
|
||||
)
|
||||
}
|
||||
|
||||
func (w *MailingListWorker) Run(ctx context.Context) error {
|
||||
var (
|
||||
wg sync.WaitGroup
|
||||
sem = make(chan struct{}, w.maxConcurrency)
|
||||
)
|
||||
func (h *mailingListHandler) Claim(ctx context.Context) (coredata.MailingListUpdate, error) {
|
||||
var mlu coredata.MailingListUpdate
|
||||
|
||||
defer wg.Wait()
|
||||
|
||||
LOOP:
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(w.interval):
|
||||
// From there we should not accept cancellations anymore.
|
||||
nonCancelableCtx := context.WithoutCancel(ctx)
|
||||
w.recoverStaleRows(nonCancelableCtx)
|
||||
|
||||
for {
|
||||
if err := w.processNext(ctx, sem, &wg); err != nil {
|
||||
if !errors.Is(err, coredata.ErrResourceNotFound) {
|
||||
w.logger.ErrorCtx(nonCancelableCtx, "cannot claim mailing list update", log.Error(err))
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
goto LOOP
|
||||
}
|
||||
}
|
||||
|
||||
func (w *MailingListWorker) processNext(ctx context.Context, sem chan struct{}, wg *sync.WaitGroup) error {
|
||||
select {
|
||||
case sem <- struct{}{}:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
var (
|
||||
mlu coredata.MailingListUpdate
|
||||
now = time.Now()
|
||||
nonCancelableCtx = context.WithoutCancel(ctx)
|
||||
)
|
||||
|
||||
if err := w.pg.WithTx(
|
||||
nonCancelableCtx,
|
||||
if err := h.pg.WithTx(
|
||||
ctx,
|
||||
func(ctx context.Context, tx pg.Tx) error {
|
||||
if err := mlu.LoadNextEnqueuedForUpdateSkipLocked(nonCancelableCtx, tx); err != nil {
|
||||
if err := mlu.LoadNextEnqueuedForUpdateSkipLocked(ctx, tx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
scope := coredata.NewScopeFromObjectID(mlu.ID)
|
||||
mlu.Status = coredata.MailingListUpdateStatusProcessing
|
||||
mlu.UpdatedAt = now
|
||||
mlu.UpdatedAt = time.Now()
|
||||
|
||||
if err := mlu.Update(nonCancelableCtx, tx, scope); err != nil {
|
||||
if err := mlu.Update(ctx, tx, scope); err != nil {
|
||||
return fmt.Errorf("cannot claim mailing list update: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
); err != nil {
|
||||
<-sem
|
||||
return err
|
||||
if errors.Is(err, coredata.ErrResourceNotFound) {
|
||||
return coredata.MailingListUpdate{}, worker.ErrNoTask
|
||||
}
|
||||
return coredata.MailingListUpdate{}, err
|
||||
}
|
||||
|
||||
wg.Add(1)
|
||||
go func(mlu coredata.MailingListUpdate) {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
return mlu, nil
|
||||
}
|
||||
|
||||
if err := w.sendAndCommit(nonCancelableCtx, &mlu); err != nil {
|
||||
w.logger.ErrorCtx(
|
||||
nonCancelableCtx,
|
||||
"cannot send mailing list update",
|
||||
func (h *mailingListHandler) Process(ctx context.Context, mlu coredata.MailingListUpdate) error {
|
||||
if err := h.sendAndCommit(ctx, &mlu); err != nil {
|
||||
h.logger.ErrorCtx(
|
||||
ctx,
|
||||
"cannot send mailing list update",
|
||||
log.Error(err),
|
||||
log.String("mailing_list_update_id", mlu.ID.String()),
|
||||
)
|
||||
|
||||
if err := h.resetEnqueued(ctx, &mlu); err != nil {
|
||||
h.logger.ErrorCtx(
|
||||
ctx,
|
||||
"cannot reset mailing list update to enqueued",
|
||||
log.Error(err),
|
||||
log.String("mailing_list_update_id", mlu.ID.String()),
|
||||
)
|
||||
|
||||
if err := w.resetEnqueued(nonCancelableCtx, &mlu); err != nil {
|
||||
w.logger.ErrorCtx(
|
||||
nonCancelableCtx,
|
||||
"cannot reset mailing list update to enqueued",
|
||||
log.Error(err),
|
||||
log.String("mailing_list_update_id", mlu.ID.String()),
|
||||
)
|
||||
}
|
||||
}
|
||||
}(mlu)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *MailingListWorker) sendAndCommit(ctx context.Context, mlu *coredata.MailingListUpdate) error {
|
||||
if err := w.service.CreateUpdateEmails(ctx, mlu.MailingListID, mlu.ID, mlu.Title, mlu.Body); err != nil {
|
||||
func (h *mailingListHandler) RecoverStale(ctx context.Context) error {
|
||||
return h.pg.WithTx(
|
||||
ctx,
|
||||
func(ctx context.Context, tx pg.Tx) error {
|
||||
if err := coredata.ResetStaleProcessingMailingListUpdates(ctx, tx, h.staleAfter); err != nil {
|
||||
return fmt.Errorf("cannot reset stale processing mailing list updates: %w", err)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func (h *mailingListHandler) sendAndCommit(ctx context.Context, mlu *coredata.MailingListUpdate) error {
|
||||
if err := h.service.CreateUpdateEmails(ctx, mlu.MailingListID, mlu.ID, mlu.Title, mlu.Body); err != nil {
|
||||
return fmt.Errorf("cannot create update emails: %w", err)
|
||||
}
|
||||
|
||||
return w.pg.WithTx(
|
||||
return h.pg.WithTx(
|
||||
ctx,
|
||||
func(ctx context.Context, tx pg.Tx) error {
|
||||
scope := coredata.NewScopeFromObjectID(mlu.ID)
|
||||
@@ -200,8 +151,8 @@ func (w *MailingListWorker) sendAndCommit(ctx context.Context, mlu *coredata.Mai
|
||||
)
|
||||
}
|
||||
|
||||
func (w *MailingListWorker) resetEnqueued(ctx context.Context, mlu *coredata.MailingListUpdate) error {
|
||||
return w.pg.WithTx(
|
||||
func (h *mailingListHandler) resetEnqueued(ctx context.Context, mlu *coredata.MailingListUpdate) error {
|
||||
return h.pg.WithTx(
|
||||
ctx,
|
||||
func(ctx context.Context, tx pg.Tx) error {
|
||||
scope := coredata.NewScopeFromObjectID(mlu.ID)
|
||||
@@ -216,19 +167,3 @@ func (w *MailingListWorker) resetEnqueued(ctx context.Context, mlu *coredata.Mai
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func (w *MailingListWorker) recoverStaleRows(ctx context.Context) {
|
||||
err := w.pg.WithTx(
|
||||
ctx,
|
||||
func(ctx context.Context, tx pg.Tx) error {
|
||||
if err := coredata.ResetStaleProcessingMailingListUpdates(ctx, tx, w.staleAfter); err != nil {
|
||||
return fmt.Errorf("cannot reset stale processing mailing list updates: %w", err)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
w.logger.ErrorCtx(ctx, "cannot recover stale processing mailing list updates", log.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user