Fix failed ACME block the queue
Signed-off-by: Bryan Frimin <bryan@getprobo.com>
This commit is contained in:
Bryan Frimin
@@ -17,7 +17,6 @@ package certmanager
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/getprobo/probo/pkg/coredata"
|
||||
@@ -79,6 +78,10 @@ func (p *Provisioner) checkPendingDomains(ctx context.Context) error {
|
||||
return p.pg.WithConn(
|
||||
ctx,
|
||||
func(conn pg.Conn) error {
|
||||
if err := p.handleStaleProvisioningAttempts(ctx, conn); err != nil {
|
||||
p.logger.ErrorCtx(ctx, "cannot handle stale provisioning attempts", log.Error(err))
|
||||
}
|
||||
|
||||
var domains coredata.CustomDomains
|
||||
if err := domains.ListDomainsWithPendingHTTPChallenges(ctx, conn, coredata.NewNoScope()); err != nil {
|
||||
return fmt.Errorf("cannot load domains with pending challenges: %w", err)
|
||||
@@ -112,39 +115,72 @@ func (p *Provisioner) checkPendingDomains(ctx context.Context) error {
|
||||
)
|
||||
}
|
||||
|
||||
func isChallengeFailedError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
func (p *Provisioner) handleStaleProvisioningAttempts(ctx context.Context, conn pg.Conn) error {
|
||||
var domains coredata.CustomDomains
|
||||
if err := domains.ListStaleProvisioningDomains(ctx, conn, coredata.NewNoScope()); err != nil {
|
||||
return fmt.Errorf("cannot load stale provisioning domains: %w", err)
|
||||
}
|
||||
|
||||
errStr := strings.ToLower(err.Error())
|
||||
if len(domains) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// These errors indicate the challenge/order is no longer valid
|
||||
return strings.Contains(errStr, "authorization must be pending") ||
|
||||
strings.Contains(errStr, "order") && strings.Contains(errStr, "invalid") ||
|
||||
strings.Contains(errStr, "authorization") && strings.Contains(errStr, "invalid") ||
|
||||
strings.Contains(errStr, "challenge") && strings.Contains(errStr, "invalid")
|
||||
p.logger.InfoCtx(ctx, "found stale provisioning attempts to reset", log.Int("count", len(domains)))
|
||||
|
||||
for _, domain := range domains {
|
||||
if err := p.resetStaleDomain(ctx, conn, domain); err != nil {
|
||||
p.logger.ErrorCtx(
|
||||
ctx,
|
||||
"cannot reset stale domain",
|
||||
log.String("domain", domain.Domain),
|
||||
log.Error(err),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Provisioner) resetDomainToRetry(
|
||||
func (p *Provisioner) resetStaleDomain(
|
||||
ctx context.Context,
|
||||
conn pg.Conn,
|
||||
domain *coredata.CustomDomain,
|
||||
) error {
|
||||
fullDomain := &coredata.CustomDomain{}
|
||||
if err := fullDomain.LoadByIDForUpdate(ctx, conn, coredata.NewNoScope(), p.encryptionKey, domain.ID); err != nil {
|
||||
return fmt.Errorf("cannot load domain for update: %w", err)
|
||||
return fmt.Errorf("cannot load stale domain for update: %w", err)
|
||||
}
|
||||
|
||||
staleDuration := time.Since(fullDomain.UpdatedAt)
|
||||
|
||||
p.logger.InfoCtx(
|
||||
ctx,
|
||||
"resetting stale domain",
|
||||
log.String("domain", fullDomain.Domain),
|
||||
log.String("status", string(fullDomain.SSLStatus)),
|
||||
log.Duration("stale_duration", staleDuration),
|
||||
log.Int("retry_count", fullDomain.SSLRetryCount),
|
||||
)
|
||||
|
||||
fullDomain.HTTPChallengeToken = nil
|
||||
fullDomain.HTTPChallengeKeyAuth = nil
|
||||
fullDomain.HTTPChallengeURL = nil
|
||||
fullDomain.HTTPOrderURL = nil
|
||||
|
||||
fullDomain.SSLStatus = coredata.CustomDomainSSLStatusPending
|
||||
|
||||
if fullDomain.SSLLastAttemptAt != nil && time.Since(*fullDomain.SSLLastAttemptAt) > 24*time.Hour {
|
||||
p.logger.InfoCtx(
|
||||
ctx,
|
||||
"resetting retry count due to old last attempt",
|
||||
log.String("domain", fullDomain.Domain),
|
||||
log.Time("last_attempt", *fullDomain.SSLLastAttemptAt),
|
||||
)
|
||||
fullDomain.SSLRetryCount = 0
|
||||
fullDomain.SSLLastAttemptAt = nil
|
||||
}
|
||||
|
||||
if err := fullDomain.Update(ctx, conn, coredata.NewNoScope(), p.encryptionKey); err != nil {
|
||||
return fmt.Errorf("cannot update domain: %w", err)
|
||||
return fmt.Errorf("cannot update stale domain: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -208,35 +244,82 @@ func (p *Provisioner) provisionDomainCertificate(
|
||||
ctx,
|
||||
"cannot complete HTTP challenge",
|
||||
log.String("domain", domain.Domain),
|
||||
log.Int("retry_count", domain.SSLRetryCount),
|
||||
log.Error(err),
|
||||
)
|
||||
|
||||
// Check if the error indicates the challenge/order has failed
|
||||
// and needs to be reset for a fresh attempt
|
||||
if isChallengeFailedError(err) {
|
||||
p.logger.InfoCtx(
|
||||
fullDomain := &coredata.CustomDomain{}
|
||||
if loadErr := fullDomain.LoadByIDForUpdate(ctx, conn, coredata.NewNoScope(), p.encryptionKey, domain.ID); loadErr != nil {
|
||||
p.logger.ErrorCtx(
|
||||
ctx,
|
||||
"challenge or order is no longer valid, resetting domain to retry with fresh challenge",
|
||||
"cannot load domain for retry tracking",
|
||||
log.String("domain", domain.Domain),
|
||||
log.Error(loadErr),
|
||||
)
|
||||
return loadErr
|
||||
}
|
||||
|
||||
fullDomain.SSLRetryCount++
|
||||
now := time.Now()
|
||||
fullDomain.SSLLastAttemptAt = &now
|
||||
|
||||
const maxRetries = 3
|
||||
if fullDomain.SSLRetryCount >= maxRetries {
|
||||
p.logger.ErrorCtx(
|
||||
ctx,
|
||||
"domain has exceeded max retry attempts, marking as failed",
|
||||
log.String("domain", domain.Domain),
|
||||
log.Int("retry_count", fullDomain.SSLRetryCount),
|
||||
)
|
||||
|
||||
if resetErr := p.resetDomainToRetry(ctx, conn, domain); resetErr != nil {
|
||||
fullDomain.SSLStatus = coredata.CustomDomainSSLStatusFailed
|
||||
fullDomain.HTTPChallengeToken = nil
|
||||
fullDomain.HTTPChallengeKeyAuth = nil
|
||||
fullDomain.HTTPChallengeURL = nil
|
||||
fullDomain.HTTPOrderURL = nil
|
||||
|
||||
if updateErr := fullDomain.Update(ctx, conn, coredata.NewNoScope(), p.encryptionKey); updateErr != nil {
|
||||
p.logger.ErrorCtx(
|
||||
ctx,
|
||||
"cannot reset domain for retry",
|
||||
"cannot mark domain as failed",
|
||||
log.String("domain", domain.Domain),
|
||||
log.Error(resetErr),
|
||||
log.Error(updateErr),
|
||||
)
|
||||
return resetErr
|
||||
return updateErr
|
||||
}
|
||||
|
||||
p.logger.InfoCtx(
|
||||
ctx,
|
||||
"domain reset to pending, will retry with new challenge on next cycle",
|
||||
log.String("domain", domain.Domain),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
p.logger.InfoCtx(
|
||||
ctx,
|
||||
"resetting domain to retry with fresh challenge",
|
||||
log.String("domain", domain.Domain),
|
||||
log.Int("retry_count", fullDomain.SSLRetryCount),
|
||||
)
|
||||
|
||||
fullDomain.HTTPChallengeToken = nil
|
||||
fullDomain.HTTPChallengeKeyAuth = nil
|
||||
fullDomain.HTTPChallengeURL = nil
|
||||
fullDomain.HTTPOrderURL = nil
|
||||
fullDomain.SSLStatus = coredata.CustomDomainSSLStatusPending
|
||||
|
||||
if updateErr := fullDomain.Update(ctx, conn, coredata.NewNoScope(), p.encryptionKey); updateErr != nil {
|
||||
p.logger.ErrorCtx(
|
||||
ctx,
|
||||
"cannot reset domain for retry",
|
||||
log.String("domain", domain.Domain),
|
||||
log.Error(updateErr),
|
||||
)
|
||||
return updateErr
|
||||
}
|
||||
|
||||
p.logger.InfoCtx(
|
||||
ctx,
|
||||
"domain reset to pending, will retry with new challenge on next cycle",
|
||||
log.String("domain", domain.Domain),
|
||||
)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -261,6 +344,9 @@ func (p *Provisioner) provisionDomainCertificate(
|
||||
fullDomain.SSLExpiresAt = &cert.ExpiresAt
|
||||
fullDomain.SSLStatus = coredata.CustomDomainSSLStatusActive
|
||||
|
||||
fullDomain.SSLRetryCount = 0
|
||||
fullDomain.SSLLastAttemptAt = nil
|
||||
|
||||
fullDomain.HTTPChallengeToken = nil
|
||||
fullDomain.HTTPChallengeKeyAuth = nil
|
||||
fullDomain.HTTPChallengeURL = nil
|
||||
|
||||
@@ -171,7 +171,65 @@ func (r *Renewer) renewDomain(ctx context.Context, conn pg.Conn, domain *coredat
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("cannot renew certificate: %w", err)
|
||||
r.logger.WarnCtx(
|
||||
ctx,
|
||||
"cannot renew certificate",
|
||||
log.String("domain", lockedDomain.Domain),
|
||||
log.Int("retry_count", lockedDomain.SSLRetryCount),
|
||||
log.Error(err),
|
||||
)
|
||||
|
||||
lockedDomain.SSLRetryCount++
|
||||
now := time.Now()
|
||||
lockedDomain.SSLLastAttemptAt = &now
|
||||
|
||||
const maxRetries = 3
|
||||
if lockedDomain.SSLRetryCount >= maxRetries {
|
||||
r.logger.ErrorCtx(
|
||||
ctx,
|
||||
"domain has exceeded max renewal retry attempts, marking as failed",
|
||||
log.String("domain", lockedDomain.Domain),
|
||||
log.Int("retry_count", lockedDomain.SSLRetryCount),
|
||||
)
|
||||
|
||||
lockedDomain.SSLStatus = coredata.CustomDomainSSLStatusFailed
|
||||
lockedDomain.HTTPChallengeToken = nil
|
||||
lockedDomain.HTTPChallengeKeyAuth = nil
|
||||
lockedDomain.HTTPChallengeURL = nil
|
||||
lockedDomain.HTTPOrderURL = nil
|
||||
|
||||
if updateErr := lockedDomain.Update(ctx, conn, coredata.NewNoScope(), r.encryptionKey); updateErr != nil {
|
||||
r.logger.ErrorCtx(
|
||||
ctx,
|
||||
"cannot mark domain as failed",
|
||||
log.String("domain", lockedDomain.Domain),
|
||||
log.Error(updateErr),
|
||||
)
|
||||
return updateErr
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Update retry tracking but keep domain ACTIVE for next renewal cycle
|
||||
if updateErr := lockedDomain.Update(ctx, conn, coredata.NewNoScope(), r.encryptionKey); updateErr != nil {
|
||||
r.logger.ErrorCtx(
|
||||
ctx,
|
||||
"cannot update domain retry tracking",
|
||||
log.String("domain", lockedDomain.Domain),
|
||||
log.Error(updateErr),
|
||||
)
|
||||
return updateErr
|
||||
}
|
||||
|
||||
r.logger.InfoCtx(
|
||||
ctx,
|
||||
"domain will retry renewal on next cycle",
|
||||
log.String("domain", lockedDomain.Domain),
|
||||
log.Int("retry_count", lockedDomain.SSLRetryCount),
|
||||
)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
r.logger.InfoCtx(
|
||||
@@ -190,6 +248,9 @@ func (r *Renewer) renewDomain(ctx context.Context, conn pg.Conn, domain *coredat
|
||||
lockedDomain.SSLExpiresAt = &cert.ExpiresAt
|
||||
lockedDomain.SSLStatus = coredata.CustomDomainSSLStatusActive
|
||||
|
||||
lockedDomain.SSLRetryCount = 0
|
||||
lockedDomain.SSLLastAttemptAt = nil
|
||||
|
||||
lockedDomain.HTTPChallengeToken = nil
|
||||
lockedDomain.HTTPChallengeKeyAuth = nil
|
||||
lockedDomain.HTTPChallengeURL = nil
|
||||
|
||||
Reference in New Issue
Block a user