Fix certmanager provisioning retry and metrics
Address several provisioning defects that either stalled the retry budget or crashed the process: - Classify CAA resolver/transport failures apart from a real CAA policy denial. Both shared the "caa records" wording, so a transient resolver error was persisted as customer misconfiguration and retried forever without consuming the retry budget. A new ErrCAANotPermitted sentinel now marks the genuine misconfiguration; other CAA errors are treated as ordinary transient failures. - Honor an explicit Retry-After: 0 (or a past date) as permission for an immediate retry instead of promoting it to the one-hour default cooldown. acme.RateLimit collapses zero, invalid, and absent headers to a zero duration, so the header is now parsed directly to tell an explicit zero apart from a missing one. - Reuse already-registered Prometheus collectors when a second ACMEService shares a registerer. The fixed-name collectors were MustRegistered, so a duplicate registration panicked the process. - Persist provisioning failures on a context detached from the process tick deadline. A timed-out attempt reached persistFailure with an expired context, so the write-back failed and the retry budget never advanced, leaving the certificate indefinitely retriable. - Use pgx.StrictNamedArgs in the certificate FOR UPDATE loaders to match the coredata SQL contract. Signed-off-by: Bryan Frimin <bryan@probo.com>
This commit is contained in:
@@ -26,11 +26,34 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/crypto/acme"
|
||||
)
|
||||
|
||||
func TestNewMetrics_SharedRegistererDoesNotPanic(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
registerer := prometheus.NewRegistry()
|
||||
|
||||
first := newMetrics(registerer)
|
||||
require.NotNil(t, first)
|
||||
|
||||
// A second ACMEService sharing the registerer re-registers fixed-name
|
||||
// collectors; this must reuse the existing ones instead of panicking.
|
||||
var second *metrics
|
||||
require.NotPanics(t, func() {
|
||||
second = newMetrics(registerer)
|
||||
})
|
||||
require.NotNil(t, second)
|
||||
|
||||
assert.Same(t, first.provisionSteps, second.provisionSteps)
|
||||
assert.Same(t, first.acmeErrors, second.acmeErrors)
|
||||
assert.Same(t, first.stepDuration, second.stepDuration)
|
||||
assert.Equal(t, first.acmeCooldown, second.acmeCooldown)
|
||||
}
|
||||
|
||||
func TestNewACMEError_RateLimited(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -63,6 +86,41 @@ func TestNewACMEError_RateLimitedDefaultCooldown(t *testing.T) {
|
||||
assert.Equal(t, defaultCooldown, err.RetryAfter())
|
||||
}
|
||||
|
||||
func TestNewACMEError_RateLimitedRetryAfterZero(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
err := newACMEError(
|
||||
"cannot create order",
|
||||
&acme.Error{
|
||||
ProblemType: "urn:ietf:params:acme:error:rateLimited",
|
||||
Header: http.Header{"Retry-After": []string{"0"}},
|
||||
},
|
||||
)
|
||||
|
||||
require.NotNil(t, err)
|
||||
assert.ErrorIs(t, err, ErrACMERateLimited)
|
||||
// An explicit Retry-After: 0 permits an immediate retry and must not be
|
||||
// promoted to the one-hour default cooldown.
|
||||
assert.Equal(t, time.Duration(0), err.RetryAfter())
|
||||
}
|
||||
|
||||
func TestNewACMEError_RateLimitedRetryAfterInvalid(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
err := newACMEError(
|
||||
"cannot create order",
|
||||
&acme.Error{
|
||||
ProblemType: "urn:ietf:params:acme:error:rateLimited",
|
||||
Header: http.Header{"Retry-After": []string{"not-a-date"}},
|
||||
},
|
||||
)
|
||||
|
||||
require.NotNil(t, err)
|
||||
assert.ErrorIs(t, err, ErrACMERateLimited)
|
||||
// An unparseable header falls back to the default cooldown.
|
||||
assert.Equal(t, defaultCooldown, err.RetryAfter())
|
||||
}
|
||||
|
||||
func TestNewACMEError_NonRateLimited(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user