Fix certmanager provisioning retry and metrics

Address several provisioning defects that either stalled the retry
budget or crashed the process:

- Classify CAA resolver/transport failures apart from a real CAA policy
  denial. Both shared the "caa records" wording, so a transient resolver
  error was persisted as customer misconfiguration and retried forever
  without consuming the retry budget. A new ErrCAANotPermitted sentinel
  now marks the genuine misconfiguration; other CAA errors are treated
  as ordinary transient failures.

- Honor an explicit Retry-After: 0 (or a past date) as permission for an
  immediate retry instead of promoting it to the one-hour default
  cooldown. acme.RateLimit collapses zero, invalid, and absent headers
  to a zero duration, so the header is now parsed directly to tell an
  explicit zero apart from a missing one.

- Reuse already-registered Prometheus collectors when a second
  ACMEService shares a registerer. The fixed-name collectors were
  MustRegistered, so a duplicate registration panicked the process.

- Persist provisioning failures on a context detached from the process
  tick deadline. A timed-out attempt reached persistFailure with an
  expired context, so the write-back failed and the retry budget never
  advanced, leaving the certificate indefinitely retriable.

- Use pgx.StrictNamedArgs in the certificate FOR UPDATE loaders to match
  the coredata SQL contract.

Signed-off-by: Bryan Frimin <bryan@probo.com>
This commit is contained in:
Bryan Frimin
2026-07-22 12:43:22 +02:00
parent 9724a2ce50
commit 121d4dcf93
7 changed files with 200 additions and 52 deletions

View File

@@ -26,11 +26,34 @@ import (
"testing"
"time"
"github.com/prometheus/client_golang/prometheus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/crypto/acme"
)
func TestNewMetrics_SharedRegistererDoesNotPanic(t *testing.T) {
t.Parallel()
registerer := prometheus.NewRegistry()
first := newMetrics(registerer)
require.NotNil(t, first)
// A second ACMEService sharing the registerer re-registers fixed-name
// collectors; this must reuse the existing ones instead of panicking.
var second *metrics
require.NotPanics(t, func() {
second = newMetrics(registerer)
})
require.NotNil(t, second)
assert.Same(t, first.provisionSteps, second.provisionSteps)
assert.Same(t, first.acmeErrors, second.acmeErrors)
assert.Same(t, first.stepDuration, second.stepDuration)
assert.Equal(t, first.acmeCooldown, second.acmeCooldown)
}
func TestNewACMEError_RateLimited(t *testing.T) {
t.Parallel()
@@ -63,6 +86,41 @@ func TestNewACMEError_RateLimitedDefaultCooldown(t *testing.T) {
assert.Equal(t, defaultCooldown, err.RetryAfter())
}
func TestNewACMEError_RateLimitedRetryAfterZero(t *testing.T) {
t.Parallel()
err := newACMEError(
"cannot create order",
&acme.Error{
ProblemType: "urn:ietf:params:acme:error:rateLimited",
Header: http.Header{"Retry-After": []string{"0"}},
},
)
require.NotNil(t, err)
assert.ErrorIs(t, err, ErrACMERateLimited)
// An explicit Retry-After: 0 permits an immediate retry and must not be
// promoted to the one-hour default cooldown.
assert.Equal(t, time.Duration(0), err.RetryAfter())
}
func TestNewACMEError_RateLimitedRetryAfterInvalid(t *testing.T) {
t.Parallel()
err := newACMEError(
"cannot create order",
&acme.Error{
ProblemType: "urn:ietf:params:acme:error:rateLimited",
Header: http.Header{"Retry-After": []string{"not-a-date"}},
},
)
require.NotNil(t, err)
assert.ErrorIs(t, err, ErrACMERateLimited)
// An unparseable header falls back to the default cooldown.
assert.Equal(t, defaultCooldown, err.RetryAfter())
}
func TestNewACMEError_NonRateLimited(t *testing.T) {
t.Parallel()