Several race and validity gaps could leave certificate provisioning stuck, unusable, or noisy: - Accept the HTTP-01 challenge only after the key authorization is committed, so the CA cannot hit the token before this instance can serve it and invalidate the order. - Persist challenge metadata under a blocking write-back lock; a row merely locked by a competing transaction no longer silently drops the accepted order. - Abandon a recovered VALID order and restart instead of issuing it with a freshly generated key that cannot match the existing cert. - Exclude rate-limited rows from the ten-minute stale reset so the resumable order survives the ACME cooldown. - Size the provisioning poll lease to exceed the max processing window so a released claim lock cannot let another worker process the same row concurrently. - Parse Retry-After as unsigned seconds and clamp overflow so malformed values fall back to the default cooldown instead of disabling it. - Normalize the acme_errors problem_type label to the RFC 8555 set to bound Prometheus cardinality. Signed-off-by: Bryan Frimin <bryan@probo.com>
136 lines
4.1 KiB
Go
136 lines
4.1 KiB
Go
// Copyright (c) 2026 Probo Inc <hello@probo.com>.
|
|
//
|
|
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
// of this software and associated documentation files (the "Software"), to deal
|
|
// in the Software without restriction, including without limitation the rights
|
|
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
// copies of the Software, and to permit persons to whom the Software is
|
|
// furnished to do so, subject to the following conditions:
|
|
//
|
|
// The above copyright notice and this permission notice shall be included in
|
|
// all copies or substantial portions of the Software.
|
|
//
|
|
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
// SOFTWARE.
|
|
|
|
package certmanager
|
|
|
|
import (
|
|
"errors"
|
|
"net/http"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"golang.org/x/crypto/acme"
|
|
)
|
|
|
|
func TestNewMetrics_SharedRegistererDoesNotPanic(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
registerer := prometheus.NewRegistry()
|
|
|
|
first := newMetrics(registerer)
|
|
require.NotNil(t, first)
|
|
|
|
// A second ACMEService sharing the registerer re-registers fixed-name
|
|
// collectors; this must reuse the existing ones instead of panicking.
|
|
var second *metrics
|
|
|
|
require.NotPanics(t, func() {
|
|
second = newMetrics(registerer)
|
|
})
|
|
require.NotNil(t, second)
|
|
|
|
assert.Same(t, first.provisionSteps, second.provisionSteps)
|
|
assert.Same(t, first.acmeErrors, second.acmeErrors)
|
|
assert.Same(t, first.stepDuration, second.stepDuration)
|
|
assert.Same(t, first.acmeCooldown, second.acmeCooldown)
|
|
}
|
|
|
|
func TestNewACMEError_RateLimited(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
err := newACMEError(
|
|
"cannot create order",
|
|
&acme.Error{
|
|
ProblemType: "urn:ietf:params:acme:error:rateLimited",
|
|
Detail: "too many requests",
|
|
Header: http.Header{"Retry-After": []string{"120"}},
|
|
},
|
|
)
|
|
|
|
require.NotNil(t, err)
|
|
assert.ErrorIs(t, err, ErrACMERateLimited)
|
|
assert.Equal(t, 2*time.Minute, err.RetryAfter())
|
|
assert.Equal(t, "urn:ietf:params:acme:error:rateLimited", err.problemType)
|
|
assert.Equal(t, "too many requests", err.detail)
|
|
}
|
|
|
|
func TestNewACMEError_RateLimitedDefaultCooldown(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
err := newACMEError(
|
|
"cannot create order",
|
|
&acme.Error{ProblemType: "URN:IETF:PARAMS:ACME:ERROR:RATELIMITED"},
|
|
)
|
|
|
|
require.NotNil(t, err)
|
|
assert.ErrorIs(t, err, ErrACMERateLimited)
|
|
assert.Equal(t, defaultCooldown, err.RetryAfter())
|
|
}
|
|
|
|
func TestNewACMEError_RateLimitedRetryAfterZero(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
err := newACMEError(
|
|
"cannot create order",
|
|
&acme.Error{
|
|
ProblemType: "urn:ietf:params:acme:error:rateLimited",
|
|
Header: http.Header{"Retry-After": []string{"0"}},
|
|
},
|
|
)
|
|
|
|
require.NotNil(t, err)
|
|
assert.ErrorIs(t, err, ErrACMERateLimited)
|
|
// An explicit Retry-After: 0 permits an immediate retry and must not be
|
|
// promoted to the one-hour default cooldown.
|
|
assert.Equal(t, time.Duration(0), err.RetryAfter())
|
|
}
|
|
|
|
func TestNewACMEError_RateLimitedRetryAfterInvalid(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
err := newACMEError(
|
|
"cannot create order",
|
|
&acme.Error{
|
|
ProblemType: "urn:ietf:params:acme:error:rateLimited",
|
|
Header: http.Header{"Retry-After": []string{"not-a-date"}},
|
|
},
|
|
)
|
|
|
|
require.NotNil(t, err)
|
|
assert.ErrorIs(t, err, ErrACMERateLimited)
|
|
// An unparseable header falls back to the default cooldown.
|
|
assert.Equal(t, defaultCooldown, err.RetryAfter())
|
|
}
|
|
|
|
func TestNewACMEError_NonRateLimited(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
cause := errors.New("network timeout")
|
|
err := newACMEError("cannot get order", cause)
|
|
|
|
require.NotNil(t, err)
|
|
assert.False(t, errors.Is(err, ErrACMERateLimited))
|
|
assert.ErrorIs(t, err, cause)
|
|
assert.Equal(t, time.Duration(0), err.RetryAfter())
|
|
}
|