Migrate workers to kit/worker
Replace hand-rolled polling loops, semaphores, and WaitGroups in all 7 background workers with go.gearno.de/kit/worker. Each worker now implements Handler[T] (Claim/Process) and optionally StaleRecoverer, gaining automatic Prometheus metrics and OpenTelemetry tracing. Bumps kit from v0.3.0 to v0.5.0. Signed-off-by: Bryan Frimin <bryan@getprobo.com>
This commit is contained in:
@@ -21,6 +21,7 @@ import (
|
||||
|
||||
"go.gearno.de/kit/log"
|
||||
"go.gearno.de/kit/pg"
|
||||
"go.gearno.de/kit/worker"
|
||||
"go.probo.inc/probo/pkg/connector"
|
||||
"go.probo.inc/probo/pkg/coredata"
|
||||
"go.probo.inc/probo/pkg/crypto/cipher"
|
||||
@@ -35,16 +36,20 @@ type (
|
||||
connectorRegistry *connector.ConnectorRegistry
|
||||
logger *log.Logger
|
||||
|
||||
worker *SourceFetchWorker
|
||||
sourceNameWorker *SourceNameWorker
|
||||
fetchWorker *worker.Worker[coredata.AccessReviewCampaignSourceFetch]
|
||||
sourceNameWorker *worker.Worker[coredata.AccessSource]
|
||||
}
|
||||
|
||||
Option func(*Service)
|
||||
Option func(*options)
|
||||
|
||||
options struct {
|
||||
fetchInterval time.Duration
|
||||
}
|
||||
)
|
||||
|
||||
func WithFetchInterval(interval time.Duration) Option {
|
||||
return func(s *Service) {
|
||||
s.worker.interval = interval
|
||||
return func(o *options) {
|
||||
o.fetchInterval = interval
|
||||
}
|
||||
}
|
||||
|
||||
@@ -55,6 +60,11 @@ func NewService(
|
||||
logger *log.Logger,
|
||||
opts ...Option,
|
||||
) *Service {
|
||||
var o options
|
||||
for _, opt := range opts {
|
||||
opt(&o)
|
||||
}
|
||||
|
||||
s := &Service{
|
||||
pg: pgClient,
|
||||
encryptionKey: encryptionKey,
|
||||
@@ -62,7 +72,20 @@ func NewService(
|
||||
logger: logger,
|
||||
}
|
||||
|
||||
s.worker = NewSourceFetchWorker(s, pgClient, logger)
|
||||
var fetchWorkerOpts []worker.Option
|
||||
if o.fetchInterval > 0 {
|
||||
fetchWorkerOpts = append(fetchWorkerOpts, worker.WithInterval(o.fetchInterval))
|
||||
} else {
|
||||
fetchWorkerOpts = append(fetchWorkerOpts, worker.WithInterval(30*time.Second))
|
||||
}
|
||||
fetchWorkerOpts = append(fetchWorkerOpts, worker.WithMaxConcurrency(20))
|
||||
|
||||
s.fetchWorker = NewSourceFetchWorker(
|
||||
s,
|
||||
pgClient,
|
||||
logger,
|
||||
fetchWorkerOpts...,
|
||||
)
|
||||
s.sourceNameWorker = NewSourceNameWorker(
|
||||
pgClient,
|
||||
encryptionKey,
|
||||
@@ -70,10 +93,6 @@ func NewService(
|
||||
logger.Named("source-name"),
|
||||
)
|
||||
|
||||
for _, opt := range opts {
|
||||
opt(s)
|
||||
}
|
||||
|
||||
return s
|
||||
}
|
||||
|
||||
@@ -134,14 +153,10 @@ func (s *Service) ResolveEntryOrganizationID(ctx context.Context, entryID gid.GI
|
||||
}
|
||||
|
||||
func (s *Service) Run(ctx context.Context) error {
|
||||
gCtx, cancel := context.WithCancel(context.WithoutCancel(ctx))
|
||||
g, gCtx := errgroup.WithContext(gCtx)
|
||||
g, gCtx := errgroup.WithContext(ctx)
|
||||
|
||||
g.Go(func() error { return s.worker.Run(gCtx) })
|
||||
g.Go(func() error { return s.fetchWorker.Run(gCtx) })
|
||||
g.Go(func() error { return s.sourceNameWorker.Run(gCtx) })
|
||||
|
||||
<-ctx.Done()
|
||||
cancel()
|
||||
|
||||
return g.Wait()
|
||||
}
|
||||
|
||||
@@ -23,20 +23,20 @@ import (
|
||||
|
||||
"go.gearno.de/kit/log"
|
||||
"go.gearno.de/kit/pg"
|
||||
"go.gearno.de/kit/worker"
|
||||
"go.probo.inc/probo/pkg/accessreview/drivers"
|
||||
"go.probo.inc/probo/pkg/connector"
|
||||
"go.probo.inc/probo/pkg/coredata"
|
||||
"go.probo.inc/probo/pkg/crypto/cipher"
|
||||
)
|
||||
|
||||
// SourceNameWorker polls for access sources that have a connector but no
|
||||
// sourceNameHandler polls for access sources that have a connector but no
|
||||
// synced name, resolves the provider instance name, and updates the source.
|
||||
type SourceNameWorker struct {
|
||||
type sourceNameHandler struct {
|
||||
pg *pg.Client
|
||||
encryptionKey cipher.EncryptionKey
|
||||
connectorRegistry *connector.ConnectorRegistry
|
||||
logger *log.Logger
|
||||
interval time.Duration
|
||||
}
|
||||
|
||||
func NewSourceNameWorker(
|
||||
@@ -44,54 +44,49 @@ func NewSourceNameWorker(
|
||||
encryptionKey cipher.EncryptionKey,
|
||||
connectorRegistry *connector.ConnectorRegistry,
|
||||
logger *log.Logger,
|
||||
) *SourceNameWorker {
|
||||
return &SourceNameWorker{
|
||||
opts ...worker.Option,
|
||||
) *worker.Worker[coredata.AccessSource] {
|
||||
h := &sourceNameHandler{
|
||||
pg: pgClient,
|
||||
encryptionKey: encryptionKey,
|
||||
connectorRegistry: connectorRegistry,
|
||||
logger: logger,
|
||||
interval: 10 * time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
func (w *SourceNameWorker) Run(ctx context.Context) error {
|
||||
w.logger.InfoCtx(ctx, "source name worker started",
|
||||
log.String("interval", w.interval.String()),
|
||||
defaultOpts := []worker.Option{
|
||||
worker.WithInterval(10 * time.Second),
|
||||
worker.WithMaxConcurrency(1),
|
||||
}
|
||||
|
||||
return worker.New(
|
||||
"source-name-worker",
|
||||
h,
|
||||
logger,
|
||||
append(defaultOpts, opts...)...,
|
||||
)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
w.logger.InfoCtx(context.WithoutCancel(ctx), "source name worker stopping")
|
||||
return ctx.Err()
|
||||
case <-time.After(w.interval):
|
||||
nonCancelableCtx := context.WithoutCancel(ctx)
|
||||
for {
|
||||
if err := w.processNext(nonCancelableCtx); err != nil {
|
||||
if !errors.Is(err, coredata.ErrNoAccessSourceNameSyncAvailable) {
|
||||
w.logger.ErrorCtx(nonCancelableCtx, "cannot sync source name", log.Error(err))
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *SourceNameWorker) processNext(ctx context.Context) error {
|
||||
func (h *sourceNameHandler) Claim(ctx context.Context) (coredata.AccessSource, error) {
|
||||
var source coredata.AccessSource
|
||||
|
||||
err := w.pg.WithTx(
|
||||
err := h.pg.WithTx(
|
||||
ctx,
|
||||
func(ctx context.Context, tx pg.Tx) error {
|
||||
return source.LoadNextUnsyncedNameForUpdateSkipLocked(ctx, tx)
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
if errors.Is(err, coredata.ErrNoAccessSourceNameSyncAvailable) {
|
||||
return coredata.AccessSource{}, worker.ErrNoTask
|
||||
}
|
||||
return coredata.AccessSource{}, err
|
||||
}
|
||||
|
||||
w.logger.InfoCtx(ctx, "syncing source name",
|
||||
return source, nil
|
||||
}
|
||||
|
||||
func (h *sourceNameHandler) Process(ctx context.Context, source coredata.AccessSource) error {
|
||||
h.logger.InfoCtx(ctx, "syncing source name",
|
||||
log.String("source_id", source.ID.String()),
|
||||
log.String("current_name", source.Name),
|
||||
)
|
||||
@@ -101,7 +96,7 @@ func (w *SourceNameWorker) processNext(ctx context.Context) error {
|
||||
resolver drivers.NameResolver
|
||||
)
|
||||
|
||||
err = w.pg.WithTx(
|
||||
err := h.pg.WithTx(
|
||||
ctx,
|
||||
func(ctx context.Context, tx pg.Tx) error {
|
||||
scope := coredata.NewScopeFromObjectID(source.ID)
|
||||
@@ -109,7 +104,7 @@ func (w *SourceNameWorker) processNext(ctx context.Context) error {
|
||||
return fmt.Errorf("source %s has no connector", source.ID)
|
||||
}
|
||||
|
||||
if err := dbConnector.LoadByID(ctx, tx, scope, *source.ConnectorID, w.encryptionKey); err != nil {
|
||||
if err := dbConnector.LoadByID(ctx, tx, scope, *source.ConnectorID, h.encryptionKey); err != nil {
|
||||
return fmt.Errorf("cannot load connector %s: %w", *source.ConnectorID, err)
|
||||
}
|
||||
|
||||
@@ -118,7 +113,7 @@ func (w *SourceNameWorker) processNext(ctx context.Context) error {
|
||||
tokenBefore = oauth2Conn.AccessToken
|
||||
}
|
||||
|
||||
httpClient, err := w.connectorHTTPClient(ctx, &dbConnector)
|
||||
httpClient, err := h.connectorHTTPClient(ctx, &dbConnector)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot create HTTP client for connector: %w", err)
|
||||
}
|
||||
@@ -126,18 +121,18 @@ func (w *SourceNameWorker) processNext(ctx context.Context) error {
|
||||
if oauth2Conn, ok := dbConnector.Connection.(*connector.OAuth2Connection); ok {
|
||||
if oauth2Conn.AccessToken != tokenBefore {
|
||||
dbConnector.UpdatedAt = time.Now()
|
||||
if err := dbConnector.Update(ctx, tx, scope, w.encryptionKey); err != nil {
|
||||
if err := dbConnector.Update(ctx, tx, scope, h.encryptionKey); err != nil {
|
||||
return fmt.Errorf("cannot persist refreshed token for connector %s: %w", *source.ConnectorID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resolver = w.buildResolver(&dbConnector, httpClient)
|
||||
resolver = h.buildResolver(&dbConnector, httpClient)
|
||||
return nil
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
w.logger.ErrorCtx(ctx, "cannot load connector for source name sync",
|
||||
h.logger.ErrorCtx(ctx, "cannot load connector for source name sync",
|
||||
log.String("source_id", source.ID.String()),
|
||||
log.Error(err),
|
||||
)
|
||||
@@ -145,11 +140,11 @@ func (w *SourceNameWorker) processNext(ctx context.Context) error {
|
||||
}
|
||||
|
||||
if resolver == nil {
|
||||
w.logger.InfoCtx(ctx, "no name resolver for provider, keeping generic name",
|
||||
h.logger.InfoCtx(ctx, "no name resolver for provider, keeping generic name",
|
||||
log.String("source_id", source.ID.String()),
|
||||
log.String("provider", dbConnector.Provider.String()),
|
||||
)
|
||||
return w.markNameSynced(ctx, &source)
|
||||
return h.markNameSynced(ctx, &source)
|
||||
}
|
||||
|
||||
resolveCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
@@ -157,7 +152,7 @@ func (w *SourceNameWorker) processNext(ctx context.Context) error {
|
||||
|
||||
instanceName, err := resolver.ResolveInstanceName(resolveCtx)
|
||||
if err != nil {
|
||||
w.logger.ErrorCtx(ctx, "cannot resolve instance name",
|
||||
h.logger.ErrorCtx(ctx, "cannot resolve instance name",
|
||||
log.String("source_id", source.ID.String()),
|
||||
log.String("provider", dbConnector.Provider.String()),
|
||||
log.Error(err),
|
||||
@@ -166,31 +161,31 @@ func (w *SourceNameWorker) processNext(ctx context.Context) error {
|
||||
}
|
||||
|
||||
if instanceName == "" {
|
||||
w.logger.InfoCtx(ctx, "instance name is empty, keeping generic name",
|
||||
h.logger.InfoCtx(ctx, "instance name is empty, keeping generic name",
|
||||
log.String("source_id", source.ID.String()),
|
||||
log.String("provider", dbConnector.Provider.String()),
|
||||
)
|
||||
return w.markNameSynced(ctx, &source)
|
||||
return h.markNameSynced(ctx, &source)
|
||||
}
|
||||
|
||||
displayName := drivers.ProviderDisplayName(dbConnector.Provider)
|
||||
newName := displayName + " " + instanceName
|
||||
|
||||
w.logger.InfoCtx(ctx, "resolved source name",
|
||||
h.logger.InfoCtx(ctx, "resolved source name",
|
||||
log.String("source_id", source.ID.String()),
|
||||
log.String("old_name", source.Name),
|
||||
log.String("new_name", newName),
|
||||
)
|
||||
|
||||
source.Name = newName
|
||||
return w.markNameSynced(ctx, &source)
|
||||
return h.markNameSynced(ctx, &source)
|
||||
}
|
||||
|
||||
func (w *SourceNameWorker) markNameSynced(
|
||||
func (h *sourceNameHandler) markNameSynced(
|
||||
ctx context.Context,
|
||||
source *coredata.AccessSource,
|
||||
) error {
|
||||
return w.pg.WithTx(
|
||||
return h.pg.WithTx(
|
||||
ctx,
|
||||
func(ctx context.Context, tx pg.Tx) error {
|
||||
scope := coredata.NewScopeFromObjectID(source.ID)
|
||||
@@ -212,7 +207,7 @@ func (w *SourceNameWorker) markNameSynced(
|
||||
// For OAuth2 connections it uses RefreshableClient when a refresh config
|
||||
// is registered for the provider, so that short-lived tokens are
|
||||
// transparently refreshed.
|
||||
func (w *SourceNameWorker) connectorHTTPClient(
|
||||
func (h *sourceNameHandler) connectorHTTPClient(
|
||||
ctx context.Context,
|
||||
dbConnector *coredata.Connector,
|
||||
) (*http.Client, error) {
|
||||
@@ -221,8 +216,8 @@ func (w *SourceNameWorker) connectorHTTPClient(
|
||||
return dbConnector.Connection.Client(ctx)
|
||||
}
|
||||
|
||||
if w.connectorRegistry != nil {
|
||||
refreshCfg := w.connectorRegistry.GetOAuth2RefreshConfig(string(dbConnector.Provider))
|
||||
if h.connectorRegistry != nil {
|
||||
refreshCfg := h.connectorRegistry.GetOAuth2RefreshConfig(string(dbConnector.Provider))
|
||||
if refreshCfg != nil {
|
||||
return oauth2Conn.RefreshableClient(ctx, *refreshCfg)
|
||||
}
|
||||
@@ -231,7 +226,7 @@ func (w *SourceNameWorker) connectorHTTPClient(
|
||||
return oauth2Conn.Client(ctx)
|
||||
}
|
||||
|
||||
func (w *SourceNameWorker) buildResolver(
|
||||
func (h *sourceNameHandler) buildResolver(
|
||||
dbConnector *coredata.Connector,
|
||||
httpClient *http.Client,
|
||||
) drivers.NameResolver {
|
||||
@@ -249,7 +244,7 @@ func (w *SourceNameWorker) buildResolver(
|
||||
case coredata.ConnectorProviderTally:
|
||||
tallySettings, err := dbConnector.TallySettings()
|
||||
if err != nil {
|
||||
w.logger.Error("cannot read tally connector settings", log.Error(err))
|
||||
h.logger.Error("cannot read tally connector settings", log.Error(err))
|
||||
return nil
|
||||
}
|
||||
return drivers.NewTallyNameResolver(httpClient, tallySettings.OrganizationID)
|
||||
@@ -262,21 +257,21 @@ func (w *SourceNameWorker) buildResolver(
|
||||
case coredata.ConnectorProviderSentry:
|
||||
sentrySettings, err := dbConnector.SentrySettings()
|
||||
if err != nil {
|
||||
w.logger.Error("cannot read sentry connector settings", log.Error(err))
|
||||
h.logger.Error("cannot read sentry connector settings", log.Error(err))
|
||||
return nil
|
||||
}
|
||||
return drivers.NewSentryNameResolver(httpClient, sentrySettings.OrganizationSlug)
|
||||
case coredata.ConnectorProviderGitHub:
|
||||
githubSettings, err := dbConnector.GitHubSettings()
|
||||
if err != nil {
|
||||
w.logger.Error("cannot read github connector settings", log.Error(err))
|
||||
h.logger.Error("cannot read github connector settings", log.Error(err))
|
||||
return nil
|
||||
}
|
||||
return drivers.NewGitHubNameResolver(httpClient, githubSettings.Organization)
|
||||
case coredata.ConnectorProviderSupabase:
|
||||
supabaseSettings, err := dbConnector.SupabaseSettings()
|
||||
if err != nil {
|
||||
w.logger.Error("cannot read supabase connector settings", log.Error(err))
|
||||
h.logger.Error("cannot read supabase connector settings", log.Error(err))
|
||||
return nil
|
||||
}
|
||||
return drivers.NewSupabaseNameResolver(supabaseSettings.OrganizationSlug)
|
||||
|
||||
@@ -18,120 +18,54 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go.gearno.de/kit/log"
|
||||
"go.gearno.de/kit/pg"
|
||||
"go.gearno.de/kit/worker"
|
||||
"go.probo.inc/probo/pkg/coredata"
|
||||
"go.probo.inc/probo/pkg/gid"
|
||||
)
|
||||
|
||||
type (
|
||||
SourceFetchWorker struct {
|
||||
svc *Service
|
||||
pg *pg.Client
|
||||
logger *log.Logger
|
||||
interval time.Duration
|
||||
staleAfter time.Duration
|
||||
maxConcurrency int
|
||||
}
|
||||
|
||||
SourceFetchWorkerOption func(*SourceFetchWorker)
|
||||
)
|
||||
|
||||
func WithSourceFetchWorkerIntervalDuration(interval time.Duration) SourceFetchWorkerOption {
|
||||
return func(w *SourceFetchWorker) {
|
||||
w.interval = interval
|
||||
}
|
||||
}
|
||||
|
||||
func WithSourceFetchWorkerStaleAfter(staleAfter time.Duration) SourceFetchWorkerOption {
|
||||
return func(w *SourceFetchWorker) {
|
||||
w.staleAfter = staleAfter
|
||||
}
|
||||
}
|
||||
|
||||
func WithSourceFetchWorkerMaxConcurrency(maxConcurrency int) SourceFetchWorkerOption {
|
||||
return func(w *SourceFetchWorker) {
|
||||
w.maxConcurrency = maxConcurrency
|
||||
}
|
||||
type sourceFetchHandler struct {
|
||||
svc *Service
|
||||
pg *pg.Client
|
||||
logger *log.Logger
|
||||
staleAfter time.Duration
|
||||
}
|
||||
|
||||
func NewSourceFetchWorker(
|
||||
svc *Service,
|
||||
pgClient *pg.Client,
|
||||
logger *log.Logger,
|
||||
opts ...SourceFetchWorkerOption,
|
||||
) *SourceFetchWorker {
|
||||
w := &SourceFetchWorker{
|
||||
svc: svc,
|
||||
pg: pgClient,
|
||||
logger: logger,
|
||||
interval: 30 * time.Second,
|
||||
staleAfter: 5 * time.Minute,
|
||||
maxConcurrency: 20,
|
||||
opts ...worker.Option,
|
||||
) *worker.Worker[coredata.AccessReviewCampaignSourceFetch] {
|
||||
h := &sourceFetchHandler{
|
||||
svc: svc,
|
||||
pg: pgClient,
|
||||
logger: logger,
|
||||
staleAfter: 5 * time.Minute,
|
||||
}
|
||||
|
||||
for _, opt := range opts {
|
||||
opt(w)
|
||||
}
|
||||
|
||||
return w
|
||||
return worker.New(
|
||||
"source-fetch-worker",
|
||||
h,
|
||||
logger,
|
||||
opts...,
|
||||
)
|
||||
}
|
||||
|
||||
func (w *SourceFetchWorker) Run(ctx context.Context) error {
|
||||
var (
|
||||
wg sync.WaitGroup
|
||||
sem = make(chan struct{}, w.maxConcurrency)
|
||||
ticker = time.NewTicker(w.interval)
|
||||
)
|
||||
defer ticker.Stop()
|
||||
defer wg.Wait()
|
||||
func (h *sourceFetchHandler) Claim(ctx context.Context) (coredata.AccessReviewCampaignSourceFetch, error) {
|
||||
var sourceFetch coredata.AccessReviewCampaignSourceFetch
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-ticker.C:
|
||||
nonCancelableCtx := context.WithoutCancel(ctx)
|
||||
w.recoverStaleRows(nonCancelableCtx)
|
||||
for {
|
||||
if err := w.processNext(ctx, sem, &wg); err != nil {
|
||||
if !errors.Is(err, coredata.ErrNoAccessReviewCampaignSourceFetchAvailable) {
|
||||
w.logger.ErrorCtx(nonCancelableCtx, "cannot claim item", log.Error(err))
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *SourceFetchWorker) processNext(
|
||||
ctx context.Context,
|
||||
sem chan struct{},
|
||||
wg *sync.WaitGroup,
|
||||
) error {
|
||||
select {
|
||||
case sem <- struct{}{}:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
var (
|
||||
sourceFetch coredata.AccessReviewCampaignSourceFetch
|
||||
now = time.Now()
|
||||
nonCancelableCtx = context.WithoutCancel(ctx)
|
||||
)
|
||||
|
||||
if err := w.pg.WithTx(
|
||||
nonCancelableCtx,
|
||||
if err := h.pg.WithTx(
|
||||
ctx,
|
||||
func(ctx context.Context, tx pg.Tx) error {
|
||||
if err := sourceFetch.LoadNextQueuedForUpdateSkipLocked(nonCancelableCtx, tx); err != nil {
|
||||
return err // sentinel errors checked by caller
|
||||
if err := sourceFetch.LoadNextQueuedForUpdateSkipLocked(ctx, tx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
sourceFetch.Status = coredata.AccessReviewCampaignSourceFetchStatusFetching
|
||||
sourceFetch.AttemptCount++
|
||||
sourceFetch.LastError = nil
|
||||
@@ -140,38 +74,60 @@ func (w *SourceFetchWorker) processNext(
|
||||
sourceFetch.UpdatedAt = now
|
||||
|
||||
scope := coredata.NewScope(sourceFetch.TenantID)
|
||||
if err := sourceFetch.Update(nonCancelableCtx, tx, scope); err != nil {
|
||||
if err := sourceFetch.Update(ctx, tx, scope); err != nil {
|
||||
return fmt.Errorf("cannot update source fetch status: %w", err)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
); err != nil {
|
||||
<-sem
|
||||
return fmt.Errorf("cannot claim source fetch: %w", err)
|
||||
if errors.Is(err, coredata.ErrNoAccessReviewCampaignSourceFetchAvailable) {
|
||||
return coredata.AccessReviewCampaignSourceFetch{}, worker.ErrNoTask
|
||||
}
|
||||
return coredata.AccessReviewCampaignSourceFetch{}, fmt.Errorf("cannot claim source fetch: %w", err)
|
||||
}
|
||||
|
||||
wg.Add(1)
|
||||
go func(sourceFetch coredata.AccessReviewCampaignSourceFetch) {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
|
||||
if err := w.handle(nonCancelableCtx, &sourceFetch); err != nil {
|
||||
w.logger.ErrorCtx(nonCancelableCtx, "cannot process source fetch", log.Error(err))
|
||||
}
|
||||
}(sourceFetch)
|
||||
|
||||
return nil
|
||||
return sourceFetch, nil
|
||||
}
|
||||
|
||||
func (w *SourceFetchWorker) handle(
|
||||
func (h *sourceFetchHandler) Process(ctx context.Context, sourceFetch coredata.AccessReviewCampaignSourceFetch) error {
|
||||
return h.handle(ctx, &sourceFetch)
|
||||
}
|
||||
|
||||
func (h *sourceFetchHandler) RecoverStale(ctx context.Context) error {
|
||||
now := time.Now()
|
||||
staleThreshold := now.Add(-h.staleAfter)
|
||||
|
||||
return h.pg.WithTx(
|
||||
ctx,
|
||||
func(ctx context.Context, tx pg.Tx) error {
|
||||
var fetches coredata.AccessReviewCampaignSourceFetches
|
||||
count, err := fetches.RecoverStale(ctx, tx, staleThreshold, now)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot recover stale source fetches: %w", err)
|
||||
}
|
||||
|
||||
if count > 0 {
|
||||
h.logger.InfoCtx(
|
||||
ctx,
|
||||
"recovered stale source fetches",
|
||||
log.Int64("count", count),
|
||||
)
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func (h *sourceFetchHandler) handle(
|
||||
ctx context.Context,
|
||||
sourceFetch *coredata.AccessReviewCampaignSourceFetch,
|
||||
) error {
|
||||
scope := coredata.NewScope(sourceFetch.TenantID)
|
||||
|
||||
campaign, err := w.svc.Campaigns(scope).Get(ctx, sourceFetch.AccessReviewCampaignID)
|
||||
campaign, err := h.svc.Campaigns(scope).Get(ctx, sourceFetch.AccessReviewCampaignID)
|
||||
if err != nil {
|
||||
commitErr := w.commitFailedSourceFetch(
|
||||
commitErr := h.commitFailedSourceFetch(
|
||||
ctx,
|
||||
sourceFetch,
|
||||
fmt.Errorf("cannot load campaign: %w", err),
|
||||
@@ -182,60 +138,31 @@ func (w *SourceFetchWorker) handle(
|
||||
return fmt.Errorf("cannot load campaign: %w", err)
|
||||
}
|
||||
|
||||
count, err := w.svc.Engine(scope).FetchSource(ctx, campaign, sourceFetch.AccessSourceID)
|
||||
count, err := h.svc.Engine(scope).FetchSource(ctx, campaign, sourceFetch.AccessSourceID)
|
||||
if err != nil {
|
||||
commitErr := w.commitFailedSourceFetch(ctx, sourceFetch, err)
|
||||
commitErr := h.commitFailedSourceFetch(ctx, sourceFetch, err)
|
||||
if commitErr != nil {
|
||||
return fmt.Errorf("cannot fetch source: %w, and cannot commit failed source fetch: %w", err, commitErr)
|
||||
}
|
||||
|
||||
if finalizeErr := w.finalizeCampaignFetchLifecycle(ctx, sourceFetch.TenantID, sourceFetch.AccessReviewCampaignID); finalizeErr != nil {
|
||||
if finalizeErr := h.finalizeCampaignFetchLifecycle(ctx, sourceFetch.TenantID, sourceFetch.AccessReviewCampaignID); finalizeErr != nil {
|
||||
return fmt.Errorf("cannot finalize campaign after failed source fetch: %w", finalizeErr)
|
||||
}
|
||||
return fmt.Errorf("cannot fetch source: %w", err)
|
||||
}
|
||||
|
||||
if err := w.commitSuccessfulSourceFetch(ctx, sourceFetch, count); err != nil {
|
||||
if err := h.commitSuccessfulSourceFetch(ctx, sourceFetch, count); err != nil {
|
||||
return fmt.Errorf("cannot commit successful source fetch: %w", err)
|
||||
}
|
||||
|
||||
if err := w.finalizeCampaignFetchLifecycle(ctx, sourceFetch.TenantID, sourceFetch.AccessReviewCampaignID); err != nil {
|
||||
if err := h.finalizeCampaignFetchLifecycle(ctx, sourceFetch.TenantID, sourceFetch.AccessReviewCampaignID); err != nil {
|
||||
return fmt.Errorf("cannot finalize campaign fetch lifecycle: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *SourceFetchWorker) recoverStaleRows(ctx context.Context) {
|
||||
now := time.Now()
|
||||
staleThreshold := now.Add(-w.staleAfter)
|
||||
|
||||
err := w.pg.WithTx(
|
||||
ctx,
|
||||
func(ctx context.Context, tx pg.Tx) error {
|
||||
var fetches coredata.AccessReviewCampaignSourceFetches
|
||||
count, err := fetches.RecoverStale(ctx, tx, staleThreshold, now)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot recover stale source fetches: %w", err)
|
||||
}
|
||||
|
||||
if count > 0 {
|
||||
w.logger.InfoCtx(
|
||||
ctx,
|
||||
"recovered stale source fetches",
|
||||
log.Int64("count", count),
|
||||
)
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
w.logger.ErrorCtx(ctx, "cannot recover stale rows", log.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
func (w *SourceFetchWorker) commitFailedSourceFetch(
|
||||
func (h *sourceFetchHandler) commitFailedSourceFetch(
|
||||
ctx context.Context,
|
||||
sourceFetch *coredata.AccessReviewCampaignSourceFetch,
|
||||
failureErr error,
|
||||
@@ -251,7 +178,7 @@ func (w *SourceFetchWorker) commitFailedSourceFetch(
|
||||
sourceFetch.CompletedAt = new(now)
|
||||
sourceFetch.UpdatedAt = now
|
||||
|
||||
return w.pg.WithTx(
|
||||
return h.pg.WithTx(
|
||||
ctx,
|
||||
func(ctx context.Context, tx pg.Tx) error {
|
||||
return sourceFetch.Update(ctx, tx, scope)
|
||||
@@ -259,7 +186,7 @@ func (w *SourceFetchWorker) commitFailedSourceFetch(
|
||||
)
|
||||
}
|
||||
|
||||
func (w *SourceFetchWorker) commitSuccessfulSourceFetch(
|
||||
func (h *sourceFetchHandler) commitSuccessfulSourceFetch(
|
||||
ctx context.Context,
|
||||
sourceFetch *coredata.AccessReviewCampaignSourceFetch,
|
||||
fetchedAccountsCount int,
|
||||
@@ -275,7 +202,7 @@ func (w *SourceFetchWorker) commitSuccessfulSourceFetch(
|
||||
sourceFetch.CompletedAt = new(now)
|
||||
sourceFetch.UpdatedAt = now
|
||||
|
||||
return w.pg.WithTx(
|
||||
return h.pg.WithTx(
|
||||
ctx,
|
||||
func(ctx context.Context, tx pg.Tx) error {
|
||||
return sourceFetch.Update(ctx, tx, scope)
|
||||
@@ -283,14 +210,14 @@ func (w *SourceFetchWorker) commitSuccessfulSourceFetch(
|
||||
)
|
||||
}
|
||||
|
||||
func (w *SourceFetchWorker) finalizeCampaignFetchLifecycle(
|
||||
func (h *sourceFetchHandler) finalizeCampaignFetchLifecycle(
|
||||
ctx context.Context,
|
||||
tenantID gid.TenantID,
|
||||
campaignID gid.GID,
|
||||
) error {
|
||||
scope := coredata.NewScope(tenantID)
|
||||
|
||||
return w.pg.WithTx(
|
||||
return h.pg.WithTx(
|
||||
ctx,
|
||||
func(ctx context.Context, tx pg.Tx) error {
|
||||
if err := lockCampaignForUpdate(ctx, tx, scope, campaignID); err != nil {
|
||||
|
||||
Reference in New Issue
Block a user