Fix ACME renew infinit loop

Signed-off-by: Bryan Frimin <bryan@getprobo.com>

pkg/certmanager/acme.go

@@ -21,7 +21,6 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"crypto/x509/pkix"
-	"errors"
 	"fmt"
 	"time"
 
@@ -57,10 +56,6 @@ type (
 	}
 )
 
-// ErrHTTPChallengeRequired indicates that an HTTP-01 challenge needs to be
-// completed before the certificate can be issued or renewed.
-var ErrHTTPChallengeRequired = errors.New("HTTP challenge required")
-
 func NewACMEService(
 	email string,
 	keyType keys.Type,
@@ -230,90 +225,6 @@ func (s *ACMEService) CompleteHTTPChallenge(
 	}, nil
 }
 
-func (s *ACMEService) ObtainCertificate(
-	ctx context.Context,
-	domain string,
-) (*Certificate, error) {
-	challenge, err := s.GetHTTPChallenge(ctx, domain)
-	if err != nil {
-		return nil, fmt.Errorf("cannot get HTTP challenge: %w", err)
-	}
-
-	// The challenge token and key auth will be stored and served via HTTP
-	// The caller is responsible for ensuring the HTTP endpoint is ready
-	// before calling CompleteHTTPChallenge
-	return nil, fmt.Errorf("%w: token=%s", ErrHTTPChallengeRequired, challenge.Token)
-}
-
-func (s *ACMEService) RenewCertificate(
-	ctx context.Context,
-	domain string,
-) (*Certificate, error) {
-	cert, err := s.renewWithExistingAuth(ctx, domain)
-	if err == nil {
-		return cert, nil
-	}
-
-	s.logger.WarnCtx(ctx, "renewal with existing authorization failed, need new HTTP challenge",
-		log.String("domain", domain),
-		log.Error(err))
-
-	return s.ObtainCertificate(ctx, domain)
-}
-
-func (s *ACMEService) renewWithExistingAuth(ctx context.Context, domain string) (*Certificate, error) {
-	order, err := s.client.AuthorizeOrder(ctx, acme.DomainIDs(domain))
-	if err != nil {
-		return nil, fmt.Errorf("cannot create renewal order: %w", err)
-	}
-
-	if order.Status != acme.StatusReady {
-		order, err = s.client.WaitOrder(ctx, order.URI)
-		if err != nil {
-			return nil, fmt.Errorf("authorization not valid or expired: %w", err)
-		}
-	}
-
-	certKey, err := keys.Generate(s.keyType)
-	if err != nil {
-		return nil, fmt.Errorf("cannot generate certificate key: %w", err)
-	}
-
-	csr, err := createCSR(domain, certKey)
-	if err != nil {
-		return nil, fmt.Errorf("cannot create CSR: %w", err)
-	}
-
-	der, _, err := s.client.CreateOrderCert(ctx, order.FinalizeURL, csr, true)
-	if err != nil {
-		return nil, fmt.Errorf("cannot create certificate: %w", err)
-	}
-
-	cert, err := x509.ParseCertificate(der[0])
-	if err != nil {
-		return nil, fmt.Errorf("cannot parse certificate: %w", err)
-	}
-
-	certPEM := pem.EncodeCertificate(der[0])
-	keyPEM, err := pem.EncodePrivateKey(certKey)
-	if err != nil {
-		return nil, fmt.Errorf("cannot encode key: %w", err)
-	}
-
-	var chainDER [][]byte
-	if len(der) > 1 {
-		chainDER = der[1:]
-	}
-	chainPEM := pem.EncodeCertificateChain(chainDER)
-
-	return &Certificate{
-		CertPEM:   certPEM,
-		KeyPEM:    keyPEM,
-		ChainPEM:  chainPEM,
-		ExpiresAt: cert.NotAfter,
-	}, nil
-}
-
 func (s *ACMEService) CheckRenewalNeeded(expiresAt time.Time, threshold time.Duration) bool {
 	return time.Until(expiresAt) <= threshold
 }

pkg/certmanager/provisioner.go

@@ -16,15 +16,19 @@ package certmanager
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net"
 	"strings"
 	"time"
 
+	"github.com/jackc/pgx/v5"
 	"go.gearno.de/kit/log"
 	"go.gearno.de/kit/pg"
+	"go.gearno.de/x/ref"
 	"go.probo.inc/probo/pkg/coredata"
 	"go.probo.inc/probo/pkg/crypto/cipher"
+	"go.probo.inc/probo/pkg/gid"
 )
 
 type (
@@ -38,6 +42,10 @@ type (
 	}
 )
 
+const (
+	maxRetries = 3
+)
+
 func NewProvisioner(
 	pg *pg.Client,
 	acmeService *ACMEService,
@@ -100,15 +108,25 @@ func (p *Provisioner) checkDNSConfiguration(domain string) error {
 }
 
 func (p *Provisioner) checkPendingDomains(ctx context.Context) error {
-	return p.pg.WithConn(
+	err := p.pg.WithTx(
 		ctx,
-		func(conn pg.Conn) error {
+		func(tx pg.Conn) error {
-			if err := p.handleStaleProvisioningAttempts(ctx, conn); err != nil {
+			if err := p.handleStaleProvisioningAttempts(ctx, tx); err != nil {
-				p.logger.ErrorCtx(ctx, "cannot handle stale provisioning attempts", log.Error(err))
+				return fmt.Errorf("cannot handle stale provisioning attempts: %w", err)
 			}
 
+			return nil
+		},
+	)
+	if err != nil {
+		return fmt.Errorf("cannot handle stale provisioning attempts: %w", err)
+	}
+
+	err = p.pg.WithTx(
+		ctx,
+		func(tx pg.Conn) error {
 			var domains coredata.CustomDomains
-			if err := domains.ListDomainsWithPendingHTTPChallenges(ctx, conn, coredata.NewNoScope()); err != nil {
+			if err := domains.ListDomainsWithPendingHTTPChallenges(ctx, tx, coredata.NewNoScope()); err != nil {
 				return fmt.Errorf("cannot load domains with pending challenges: %w", err)
 			}
 
@@ -125,39 +143,25 @@ func (p *Provisioner) checkPendingDomains(ctx context.Context) error {
 				default:
 				}
 
-				if err := p.provisionDomainCertificate(ctx, conn, domain); err != nil {
+				if err := p.provisionDomainCertificate(ctx, tx, domain.ID); err != nil {
-					p.logger.ErrorCtx(
+					return fmt.Errorf("cannot provision certificate for domain %q: %w", domain.Domain, err)
-						ctx,
-						"cannot provision certificate for domain",
-						log.String("domain", domain.Domain),
-						log.Error(err),
-					)
 				}
 			}
 
 			return nil
 		},
 	)
-}
 
-func isFatalChallengeError(err error) bool {
+	if err != nil {
-	if err == nil {
+		return fmt.Errorf("cannot provision domains: %w", err)
-		return false
 	}
 
-	errStr := strings.ToLower(err.Error())
+	return nil
-
-	return (strings.Contains(errStr, "invalid") &&
-		(strings.Contains(errStr, "challenge") ||
-			strings.Contains(errStr, "authorization") ||
-			strings.Contains(errStr, "order"))) ||
-		strings.Contains(errStr, "authorization must be pending") ||
-		strings.Contains(errStr, "expired")
 }
 
-func (p *Provisioner) handleStaleProvisioningAttempts(ctx context.Context, conn pg.Conn) error {
+func (p *Provisioner) handleStaleProvisioningAttempts(ctx context.Context, tx pg.Conn) error {
 	var domains coredata.CustomDomains
-	if err := domains.ListStaleProvisioningDomains(ctx, conn, coredata.NewNoScope()); err != nil {
+	if err := domains.ListStaleProvisioningDomains(ctx, tx, coredata.NewNoScope()); err != nil {
 		return fmt.Errorf("cannot load stale provisioning domains: %w", err)
 	}
 
@@ -168,7 +172,7 @@ func (p *Provisioner) handleStaleProvisioningAttempts(ctx context.Context, conn
 	p.logger.InfoCtx(ctx, "found stale provisioning attempts to reset", log.Int("count", len(domains)))
 
 	for _, domain := range domains {
-		if err := p.resetStaleDomain(ctx, conn, domain); err != nil {
+		if err := p.resetStaleDomain(ctx, tx, domain); err != nil {
 			p.logger.ErrorCtx(
 				ctx,
 				"cannot reset stale domain",
@@ -183,11 +187,15 @@ func (p *Provisioner) handleStaleProvisioningAttempts(ctx context.Context, conn
 
 func (p *Provisioner) resetStaleDomain(
 	ctx context.Context,
-	conn pg.Conn,
+	tx pg.Conn,
 	domain *coredata.CustomDomain,
 ) error {
 	fullDomain := &coredata.CustomDomain{}
-	if err := fullDomain.LoadByIDForUpdate(ctx, conn, coredata.NewNoScope(), p.encryptionKey, domain.ID); err != nil {
+	if err := fullDomain.LoadByIDForUpdateSkipLocked(ctx, tx, coredata.NewNoScope(), p.encryptionKey, domain.ID); err != nil {
+		if errors.Is(err, pgx.ErrNoRows) {
+			return nil
+		}
+
 		return fmt.Errorf("cannot load stale domain for update: %w", err)
 	}
 
@@ -219,7 +227,7 @@ func (p *Provisioner) resetStaleDomain(
 		fullDomain.SSLLastAttemptAt = nil
 	}
 
-	if err := fullDomain.Update(ctx, conn, coredata.NewNoScope(), p.encryptionKey); err != nil {
+	if err := fullDomain.Update(ctx, tx, coredata.NewNoScope(), p.encryptionKey); err != nil {
 		return fmt.Errorf("cannot update stale domain: %w", err)
 	}
 
@@ -228,10 +236,19 @@ func (p *Provisioner) resetStaleDomain(
 
 func (p *Provisioner) provisionDomainCertificate(
 	ctx context.Context,
-	conn pg.Conn,
+	tx pg.Conn,
-	domain *coredata.CustomDomain,
+	domainID gid.GID,
 ) error {
-	if domain.SSLStatus == coredata.CustomDomainSSLStatusPending {
+	domain := &coredata.CustomDomain{}
+	if err := domain.LoadByIDForUpdateSkipLocked(ctx, tx, coredata.NewNoScope(), p.encryptionKey, domainID); err != nil {
+		if errors.Is(err, pgx.ErrNoRows) {
+			return nil
+		}
+
+		return fmt.Errorf("cannot load by id for update %q custom domain: %w", domainID, err)
+	}
+
+	if domain.SSLStatus == coredata.CustomDomainSSLStatusPending || domain.SSLStatus == coredata.CustomDomainSSLStatusRenewing {
 		if err := p.checkDNSConfiguration(domain.Domain); err != nil {
 			p.logger.WarnCtx(
 				ctx,
@@ -256,18 +273,13 @@ func (p *Provisioner) provisionDomainCertificate(
 			return err
 		}
 
-		fullDomain := &coredata.CustomDomain{}
+		domain.HTTPChallengeToken = &challenge.Token
-		if err := fullDomain.LoadByIDForUpdate(ctx, conn, coredata.NewNoScope(), p.encryptionKey, domain.ID); err != nil {
+		domain.HTTPChallengeKeyAuth = &challenge.KeyAuth
-			return fmt.Errorf("cannot load domain for update: %w", err)
+		domain.HTTPChallengeURL = &challenge.URL
-		}
+		domain.HTTPOrderURL = &challenge.OrderURL
+		domain.SSLStatus = coredata.CustomDomainSSLStatusProvisioning
 
-		fullDomain.HTTPChallengeToken = &challenge.Token
+		if err := domain.Update(ctx, tx, coredata.NewNoScope(), p.encryptionKey); err != nil {
-		fullDomain.HTTPChallengeKeyAuth = &challenge.KeyAuth
-		fullDomain.HTTPChallengeURL = &challenge.URL
-		fullDomain.HTTPOrderURL = &challenge.OrderURL
-		fullDomain.SSLStatus = coredata.CustomDomainSSLStatusProvisioning
-
-		if err := fullDomain.Update(ctx, conn, coredata.NewNoScope(), p.encryptionKey); err != nil {
 			return fmt.Errorf("cannot update domain with challenge: %w", err)
 		}
 
@@ -299,91 +311,26 @@ func (p *Provisioner) provisionDomainCertificate(
 			log.Error(err),
 		)
 
-		fullDomain := &coredata.CustomDomain{}
+		domain.SSLRetryCount = domain.SSLRetryCount + 1
-		if loadErr := fullDomain.LoadByIDForUpdate(ctx, conn, coredata.NewNoScope(), p.encryptionKey, domain.ID); loadErr != nil {
+		domain.SSLLastAttemptAt = ref.Ref(time.Now())
-			p.logger.ErrorCtx(
-				ctx,
-				"cannot load domain for retry tracking",
-				log.String("domain", domain.Domain),
-				log.Error(loadErr),
-			)
-			return loadErr
-		}
 
-		fullDomain.SSLRetryCount++
+		if domain.SSLRetryCount >= maxRetries {
-		now := time.Now()
-		fullDomain.SSLLastAttemptAt = &now
-
-		const maxRetries = 3
-		if fullDomain.SSLRetryCount >= maxRetries {
 			p.logger.ErrorCtx(
 				ctx,
 				"domain has exceeded max retry attempts, marking as failed",
 				log.String("domain", domain.Domain),
-				log.Int("retry_count", fullDomain.SSLRetryCount),
+				log.Int("retry_count", domain.SSLRetryCount),
 			)
 
-			fullDomain.SSLStatus = coredata.CustomDomainSSLStatusFailed
+			domain.SSLStatus = coredata.CustomDomainSSLStatusFailed
-			fullDomain.HTTPChallengeToken = nil
+			domain.HTTPChallengeToken = nil
-			fullDomain.HTTPChallengeKeyAuth = nil
+			domain.HTTPChallengeKeyAuth = nil
-			fullDomain.HTTPChallengeURL = nil
+			domain.HTTPChallengeURL = nil
-			fullDomain.HTTPOrderURL = nil
+			domain.HTTPOrderURL = nil
-
-			if updateErr := fullDomain.Update(ctx, conn, coredata.NewNoScope(), p.encryptionKey); updateErr != nil {
-				p.logger.ErrorCtx(
-					ctx,
-					"cannot mark domain as failed",
-					log.String("domain", domain.Domain),
-					log.Error(updateErr),
-				)
-				return updateErr
-			}
-
-			return nil
 		}
 
-		if isFatalChallengeError(err) {
+		if err := domain.Update(ctx, tx, coredata.NewNoScope(), p.encryptionKey); err != nil {
-			p.logger.InfoCtx(
+			return fmt.Errorf("cannot update domain: %w", err)
-				ctx,
-				"fatal challenge error, resetting domain to retry with fresh challenge",
-				log.String("domain", domain.Domain),
-				log.Int("retry_count", fullDomain.SSLRetryCount),
-			)
-
-			fullDomain.HTTPChallengeToken = nil
-			fullDomain.HTTPChallengeKeyAuth = nil
-			fullDomain.HTTPChallengeURL = nil
-			fullDomain.HTTPOrderURL = nil
-			fullDomain.SSLStatus = coredata.CustomDomainSSLStatusPending
-
-			if updateErr := fullDomain.Update(ctx, conn, coredata.NewNoScope(), p.encryptionKey); updateErr != nil {
-				p.logger.ErrorCtx(
-					ctx,
-					"cannot reset domain for retry",
-					log.String("domain", domain.Domain),
-					log.Error(updateErr),
-				)
-				return updateErr
-			}
-
-			return nil
-		}
-
-		p.logger.InfoCtx(
-			ctx,
-			"transient error, keeping existing challenge for retry",
-			log.String("domain", domain.Domain),
-			log.Int("retry_count", fullDomain.SSLRetryCount),
-		)
-
-		if updateErr := fullDomain.Update(ctx, conn, coredata.NewNoScope(), p.encryptionKey); updateErr != nil {
-			p.logger.ErrorCtx(
-				ctx,
-				"cannot update domain retry tracking",
-				log.String("domain", domain.Domain),
-				log.Error(updateErr),
-			)
-			return updateErr
 		}
 
 		return nil
@@ -396,47 +343,42 @@ func (p *Provisioner) provisionDomainCertificate(
 		log.Time("expires_at", cert.ExpiresAt),
 	)
 
-	fullDomain := &coredata.CustomDomain{}
+	domain.SSLCertificatePEM = cert.CertPEM
-	if err := fullDomain.LoadByID(ctx, conn, coredata.NewNoScope(), p.encryptionKey, domain.ID); err != nil {
+	if err := domain.EncryptPrivateKey(cert.KeyPEM, p.encryptionKey); err != nil {
-		return fmt.Errorf("cannot load domain: %w", err)
-	}
-
-	fullDomain.SSLCertificatePEM = cert.CertPEM
-	if err := fullDomain.EncryptPrivateKey(cert.KeyPEM, p.encryptionKey); err != nil {
 		return fmt.Errorf("cannot encrypt private key: %w", err)
 	}
 	chainStr := string(cert.ChainPEM)
-	fullDomain.SSLCertificateChain = &chainStr
+	domain.SSLCertificateChain = &chainStr
-	fullDomain.SSLExpiresAt = &cert.ExpiresAt
+	domain.SSLExpiresAt = &cert.ExpiresAt
-	fullDomain.SSLStatus = coredata.CustomDomainSSLStatusActive
+	domain.SSLStatus = coredata.CustomDomainSSLStatusActive
 
-	fullDomain.SSLRetryCount = 0
+	domain.SSLRetryCount = 0
-	fullDomain.SSLLastAttemptAt = nil
+	domain.SSLLastAttemptAt = nil
 
-	fullDomain.HTTPChallengeToken = nil
+	domain.HTTPChallengeToken = nil
-	fullDomain.HTTPChallengeKeyAuth = nil
+	domain.HTTPChallengeKeyAuth = nil
-	fullDomain.HTTPChallengeURL = nil
+	domain.HTTPChallengeURL = nil
-	fullDomain.HTTPOrderURL = nil
+	domain.HTTPOrderURL = nil
 
-	if err := fullDomain.Update(ctx, conn, coredata.NewNoScope(), p.encryptionKey); err != nil {
+	if err := domain.Update(ctx, tx, coredata.NewNoScope(), p.encryptionKey); err != nil {
 		return fmt.Errorf("cannot update domain: %w", err)
 	}
 
 	cache := &coredata.CachedCertificate{
-		Domain:           fullDomain.Domain,
+		Domain:           domain.Domain,
 		CertificatePEM:   string(cert.CertPEM),
 		PrivateKeyPEM:    string(cert.KeyPEM),
 		CertificateChain: &chainStr,
 		ExpiresAt:        cert.ExpiresAt,
 		CachedAt:         time.Now(),
-		CustomDomainID:   fullDomain.ID,
+		CustomDomainID:   domain.ID,
 	}
 
-	if err := cache.Upsert(ctx, conn); err != nil {
+	if err := cache.Upsert(ctx, tx); err != nil {
 		p.logger.ErrorCtx(
 			ctx,
 			"cannot update certificate cache",
-			log.String("domain", fullDomain.Domain),
+			log.String("domain", domain.Domain),
 			log.Error(err),
 		)
 	}

pkg/certmanager/renewer.go

@@ -20,10 +20,12 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/jackc/pgx/v5"
 	"go.gearno.de/kit/log"
 	"go.gearno.de/kit/pg"
 	"go.probo.inc/probo/pkg/coredata"
 	"go.probo.inc/probo/pkg/crypto/cipher"
+	"go.probo.inc/probo/pkg/gid"
 )
 
 type (
@@ -73,11 +75,11 @@ func (r *Renewer) Run(ctx context.Context) error {
 }
 
 func (r *Renewer) checkAndRenew(ctx context.Context) error {
-	return r.pg.WithConn(
+	return r.pg.WithTx(
 		ctx,
-		func(conn pg.Conn) error {
+		func(tx pg.Conn) error {
 			var caches coredata.CachedCertificates
-			cacheCount, err := caches.CountAll(ctx, conn)
+			cacheCount, err := caches.CountAll(ctx, tx)
 			if err != nil {
 				r.logger.ErrorCtx(ctx, "cannot count certificate cache", log.Error(err))
 			} else if cacheCount == 0 {
@@ -91,13 +93,13 @@ func (r *Renewer) checkAndRenew(ctx context.Context) error {
 				}
 			}
 
-			if err := caches.CleanExpired(ctx, conn); err != nil {
+			if err := caches.CleanExpired(ctx, tx); err != nil {
 				r.logger.ErrorCtx(ctx, "cannot clean certificate cache", log.Error(err))
 			}
 
 			domains := coredata.CustomDomains{}
 			scope := coredata.NewNoScope()
-			if err := domains.ListDomainsForRenewal(ctx, conn, scope); err != nil {
+			if err := domains.ListDomainsForRenewal(ctx, tx, scope); err != nil {
 				return fmt.Errorf("cannot list domains for renewal: %w", err)
 			}
 
@@ -115,7 +117,7 @@ func (r *Renewer) checkAndRenew(ctx context.Context) error {
 				}
 
 				r.logger.InfoCtx(ctx, "renewing certificate for domain", log.String("domain", domain.Domain))
-				if err := r.renewDomain(ctx, conn, domain); err != nil {
+				if err := r.renewDomain(ctx, tx, domain.ID); err != nil {
 					r.logger.ErrorCtx(ctx, "cannot renew certificate", log.String("domain", domain.Domain), log.Error(err))
 				} else {
 					r.logger.InfoCtx(ctx, "successfully renewed certificate", log.String("domain", domain.Domain))
@@ -127,13 +129,17 @@ func (r *Renewer) checkAndRenew(ctx context.Context) error {
 	)
 }
 
-func (r *Renewer) renewDomain(ctx context.Context, conn pg.Conn, domain *coredata.CustomDomain) error {
+func (r *Renewer) renewDomain(ctx context.Context, tx pg.Conn, domainID gid.GID) error {
-	lockedDomain := &coredata.CustomDomain{}
+	domain := &coredata.CustomDomain{}
-	if err := lockedDomain.LoadByIDForUpdate(ctx, conn, coredata.NewNoScope(), r.encryptionKey, domain.ID); err != nil {
+	if err := domain.LoadByIDForUpdateSkipLocked(ctx, tx, coredata.NewNoScope(), r.encryptionKey, domainID); err != nil {
+		if errors.Is(err, pgx.ErrNoRows) {
+			return nil
+		}
+
 		return fmt.Errorf("cannot lock domain for renewal: %w", err)
 	}
 
-	if lockedDomain.SSLStatus != coredata.CustomDomainSSLStatusActive {
+	if domain.SSLStatus != coredata.CustomDomainSSLStatusActive {
 		r.logger.InfoCtx(
 			ctx,
 			"domain status changed, skipping renewal",
@@ -143,141 +149,9 @@ func (r *Renewer) renewDomain(ctx context.Context, conn pg.Conn, domain *coredat
 		return nil
 	}
 
-	cert, err := r.acmeService.RenewCertificate(ctx, lockedDomain.Domain)
+	domain.SSLStatus = coredata.CustomDomainSSLStatusRenewing
-	if err != nil {
+	if err := domain.Update(ctx, tx, coredata.NewNoScope(), r.encryptionKey); err != nil {
-		if errors.Is(err, ErrHTTPChallengeRequired) {
+		return fmt.Errorf("cannot update domain status: %w", err)
-			challenge, err := r.acmeService.GetHTTPChallenge(ctx, lockedDomain.Domain)
-			if err != nil {
-				return fmt.Errorf("cannot get HTTP challenge for renewal: %w", err)
-			}
-
-			r.logger.WarnCtx(
-				ctx,
-				"HTTP challenge required for renewal",
-				log.String("domain", lockedDomain.Domain),
-				log.String("token", challenge.Token),
-			)
-
-			lockedDomain.HTTPChallengeToken = &challenge.Token
-			lockedDomain.HTTPChallengeKeyAuth = &challenge.KeyAuth
-			lockedDomain.HTTPChallengeURL = &challenge.URL
-			lockedDomain.HTTPOrderURL = &challenge.OrderURL
-			lockedDomain.SSLStatus = coredata.CustomDomainSSLStatusRenewing
-
-			if err := lockedDomain.Update(ctx, conn, coredata.NewNoScope(), r.encryptionKey); err != nil {
-				return fmt.Errorf("cannot update domain with renewal challenge: %w", err)
-			}
-
-			return nil
-		}
-
-		r.logger.WarnCtx(
-			ctx,
-			"cannot renew certificate",
-			log.String("domain", lockedDomain.Domain),
-			log.Int("retry_count", lockedDomain.SSLRetryCount),
-			log.Error(err),
-		)
-
-		lockedDomain.SSLRetryCount++
-		now := time.Now()
-		lockedDomain.SSLLastAttemptAt = &now
-
-		const maxRetries = 3
-		if lockedDomain.SSLRetryCount >= maxRetries {
-			r.logger.ErrorCtx(
-				ctx,
-				"domain has exceeded max renewal retry attempts, marking as failed",
-				log.String("domain", lockedDomain.Domain),
-				log.Int("retry_count", lockedDomain.SSLRetryCount),
-			)
-
-			lockedDomain.SSLStatus = coredata.CustomDomainSSLStatusFailed
-			lockedDomain.HTTPChallengeToken = nil
-			lockedDomain.HTTPChallengeKeyAuth = nil
-			lockedDomain.HTTPChallengeURL = nil
-			lockedDomain.HTTPOrderURL = nil
-
-			if updateErr := lockedDomain.Update(ctx, conn, coredata.NewNoScope(), r.encryptionKey); updateErr != nil {
-				r.logger.ErrorCtx(
-					ctx,
-					"cannot mark domain as failed",
-					log.String("domain", lockedDomain.Domain),
-					log.Error(updateErr),
-				)
-				return updateErr
-			}
-
-			return fmt.Errorf("domain marked as failed after %d retry attempts: %w", maxRetries, err)
-		}
-
-		// Update retry tracking but keep domain ACTIVE for next renewal cycle
-		if updateErr := lockedDomain.Update(ctx, conn, coredata.NewNoScope(), r.encryptionKey); updateErr != nil {
-			r.logger.ErrorCtx(
-				ctx,
-				"cannot update domain retry tracking",
-				log.String("domain", lockedDomain.Domain),
-				log.Error(updateErr),
-			)
-			return updateErr
-		}
-
-		r.logger.InfoCtx(
-			ctx,
-			"domain will retry renewal on next cycle",
-			log.String("domain", lockedDomain.Domain),
-			log.Int("retry_count", lockedDomain.SSLRetryCount),
-		)
-
-		// Return the original error so caller knows renewal failed
-		return fmt.Errorf("renewal failed, will retry: %w", err)
-	}
-
-	r.logger.InfoCtx(
-		ctx,
-		"certificate renewed successfully",
-		log.String("domain", lockedDomain.Domain),
-		log.Time("expires_at", cert.ExpiresAt),
-	)
-
-	lockedDomain.SSLCertificatePEM = cert.CertPEM
-	if err := lockedDomain.EncryptPrivateKey(cert.KeyPEM, r.encryptionKey); err != nil {
-		return fmt.Errorf("cannot encrypt private key: %w", err)
-	}
-	chainStr := string(cert.ChainPEM)
-	lockedDomain.SSLCertificateChain = &chainStr
-	lockedDomain.SSLExpiresAt = &cert.ExpiresAt
-	lockedDomain.SSLStatus = coredata.CustomDomainSSLStatusActive
-
-	lockedDomain.SSLRetryCount = 0
-	lockedDomain.SSLLastAttemptAt = nil
-
-	lockedDomain.HTTPChallengeToken = nil
-	lockedDomain.HTTPChallengeKeyAuth = nil
-	lockedDomain.HTTPChallengeURL = nil
-	lockedDomain.HTTPOrderURL = nil
-
-	if err := lockedDomain.Update(ctx, conn, coredata.NewNoScope(), r.encryptionKey); err != nil {
-		return fmt.Errorf("cannot update domain with renewed certificate: %w", err)
-	}
-
-	cache := &coredata.CachedCertificate{
-		Domain:           lockedDomain.Domain,
-		CertificatePEM:   string(cert.CertPEM),
-		PrivateKeyPEM:    string(cert.KeyPEM),
-		CertificateChain: &chainStr,
-		ExpiresAt:        cert.ExpiresAt,
-		CachedAt:         time.Now(),
-		CustomDomainID:   lockedDomain.ID,
-	}
-
-	if err := cache.Upsert(ctx, conn); err != nil {
-		r.logger.ErrorCtx(
-			ctx,
-			"cannot update certificate cache",
-			log.String("domain", domain.Domain),
-			log.Error(err),
-		)
 	}
 
 	return nil

pkg/coredata/custom_domain.go

@@ -203,7 +203,7 @@ LIMIT 1
 	return nil
 }
 
-func (cd *CustomDomain) LoadByIDForUpdate(
+func (cd *CustomDomain) LoadByIDForUpdateSkipLocked(
 	ctx context.Context,
 	conn pg.Conn,
 	scope Scoper,
@@ -234,7 +234,7 @@ WHERE
 	%s
 	AND id = @id
 LIMIT 1
-FOR UPDATE
+FOR UPDATE SKIP LOCKED
 `
 	q = fmt.Sprintf(q, scope.SQLFragment())
 
@@ -565,7 +565,6 @@ FROM
 WHERE
 	%s
 	AND ssl_status = @status
-	AND ssl_expires_at IS NOT NULL
 	AND ssl_expires_at <= CURRENT_TIMESTAMP + INTERVAL '30 days'
 ORDER BY
 	ssl_expires_at ASC