Address several provisioning defects that either stalled the retry budget or crashed the process: - Classify CAA resolver/transport failures apart from a real CAA policy denial. Both shared the "caa records" wording, so a transient resolver error was persisted as customer misconfiguration and retried forever without consuming the retry budget. A new ErrCAANotPermitted sentinel now marks the genuine misconfiguration; other CAA errors are treated as ordinary transient failures. - Honor an explicit Retry-After: 0 (or a past date) as permission for an immediate retry instead of promoting it to the one-hour default cooldown. acme.RateLimit collapses zero, invalid, and absent headers to a zero duration, so the header is now parsed directly to tell an explicit zero apart from a missing one. - Reuse already-registered Prometheus collectors when a second ACMEService shares a registerer. The fixed-name collectors were MustRegistered, so a duplicate registration panicked the process. - Persist provisioning failures on a context detached from the process tick deadline. A timed-out attempt reached persistFailure with an expired context, so the write-back failed and the retry budget never advanced, leaving the certificate indefinitely retriable. - Use pgx.StrictNamedArgs in the certificate FOR UPDATE loaders to match the coredata SQL contract. Signed-off-by: Bryan Frimin <bryan@probo.com>
170 lines
6.3 KiB
Go
170 lines
6.3 KiB
Go
// Copyright (c) 2026 Probo Inc <hello@probo.com>.
|
|
//
|
|
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
// of this software and associated documentation files (the "Software"), to deal
|
|
// in the Software without restriction, including without limitation the rights
|
|
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
// copies of the Software, and to permit persons to whom the Software is
|
|
// furnished to do so, subject to the following conditions:
|
|
//
|
|
// The above copyright notice and this permission notice shall be included in
|
|
// all copies or substantial portions of the Software.
|
|
//
|
|
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
// SOFTWARE.
|
|
|
|
package certmanager
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"go.probo.inc/probo/pkg/coredata"
|
|
)
|
|
|
|
func TestClassifyProvisioningError(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
assert.Equal(t, ProvisioningErrorACMERateLimited, classifyProvisioningError(ErrACMERateLimited))
|
|
assert.Equal(t, ProvisioningErrorACMEInvalidOrder, classifyProvisioningError(ErrOrderInvalid))
|
|
assert.Equal(t, ProvisioningErrorDNSCNAME, classifyProvisioningError(errors.New("cname target mismatch")))
|
|
assert.Equal(t, ProvisioningErrorDNSCAA, classifyProvisioningError(fmt.Errorf("%w: domain %q", ErrCAANotPermitted, "example.com")))
|
|
assert.Equal(t, ProvisioningErrorACMETemporary, classifyProvisioningError(errors.New("network timeout")))
|
|
// A CAA resolver/transport failure shares the "caa records" wording with a
|
|
// real CAA misconfiguration but must consume the normal retry budget.
|
|
assert.Equal(t, ProvisioningErrorACMETemporary, classifyProvisioningError(errors.New("cannot exchange dns message for caa records: i/o timeout")))
|
|
}
|
|
|
|
func TestDecideProvisioningOutcome_RateLimitKeepsRetryCountAndOrder(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
orderURL := "https://acme.example/order/1"
|
|
certificate := &coredata.Certificate{
|
|
Status: coredata.CertificateStatusProvisioning,
|
|
SSLRetryCount: 2,
|
|
HTTPOrderURL: &orderURL,
|
|
}
|
|
|
|
outcome := decideProvisioningOutcome(certificate, ProvisioningErrorACMERateLimited)
|
|
|
|
assert.Equal(t, coredata.CertificateStatusProvisioning, outcome.status)
|
|
assert.Equal(t, 2, outcome.retryCount)
|
|
assert.False(t, outcome.clearACMEState)
|
|
assert.Equal(t, ProvisioningErrorACMERateLimited, outcome.errorCode)
|
|
}
|
|
|
|
func TestDecideProvisioningOutcome_RateLimitThenTransientDoesNotFail(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
orderURL := "https://acme.example/order/1"
|
|
certificate := &coredata.Certificate{
|
|
Status: coredata.CertificateStatusProvisioning,
|
|
SSLRetryCount: 0,
|
|
HTTPOrderURL: &orderURL,
|
|
}
|
|
|
|
rateLimited := decideProvisioningOutcome(certificate, ProvisioningErrorACMERateLimited)
|
|
certificate.SSLRetryCount = rateLimited.retryCount
|
|
certificate.Status = rateLimited.status
|
|
|
|
// Previously rate-limit floored ssl_retry_count to 5, so the next transient
|
|
// immediately crossed maxProvisioningRetries and marked FAILED.
|
|
transient := decideProvisioningOutcome(certificate, ProvisioningErrorACMETemporary)
|
|
|
|
assert.Equal(t, coredata.CertificateStatusProvisioning, transient.status)
|
|
assert.Equal(t, 1, transient.retryCount)
|
|
assert.False(t, transient.clearACMEState)
|
|
assert.NotEqual(t, coredata.CertificateStatusFailed, transient.status)
|
|
}
|
|
|
|
func TestDecideProvisioningOutcome_MarksFailedAfterMaxRetries(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
certificate := &coredata.Certificate{
|
|
Status: coredata.CertificateStatusProvisioning,
|
|
SSLRetryCount: maxProvisioningRetries - 1,
|
|
}
|
|
|
|
outcome := decideProvisioningOutcome(certificate, ProvisioningErrorACMEInvalidOrder)
|
|
|
|
assert.Equal(t, coredata.CertificateStatusFailed, outcome.status)
|
|
assert.Equal(t, maxProvisioningRetries, outcome.retryCount)
|
|
assert.True(t, outcome.clearACMEState)
|
|
assert.Equal(t, ProvisioningErrorACMEFailed, outcome.errorCode)
|
|
}
|
|
|
|
func TestDecideProvisioningOutcome_TransientPreservesOrder(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
orderURL := "https://acme.example/order/1"
|
|
certificate := &coredata.Certificate{
|
|
Status: coredata.CertificateStatusProvisioning,
|
|
SSLRetryCount: 0,
|
|
HTTPOrderURL: &orderURL,
|
|
}
|
|
|
|
outcome := decideProvisioningOutcome(certificate, ProvisioningErrorACMETemporary)
|
|
|
|
assert.Equal(t, coredata.CertificateStatusProvisioning, outcome.status)
|
|
assert.Equal(t, 1, outcome.retryCount)
|
|
assert.False(t, outcome.clearACMEState)
|
|
assert.Equal(t, ProvisioningErrorACMETemporary, outcome.errorCode)
|
|
}
|
|
|
|
func TestDecideProvisioningOutcome_InvalidOrderClearsState(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
orderURL := "https://acme.example/order/1"
|
|
certificate := &coredata.Certificate{
|
|
Status: coredata.CertificateStatusProvisioning,
|
|
SSLRetryCount: 0,
|
|
HTTPOrderURL: &orderURL,
|
|
}
|
|
|
|
outcome := decideProvisioningOutcome(certificate, ProvisioningErrorACMEInvalidOrder)
|
|
|
|
assert.Equal(t, coredata.CertificateStatusPending, outcome.status)
|
|
assert.Equal(t, 1, outcome.retryCount)
|
|
assert.True(t, outcome.clearACMEState)
|
|
assert.Equal(t, ProvisioningErrorACMEInvalidOrder, outcome.errorCode)
|
|
}
|
|
|
|
func TestDecideProvisioningOutcome_DNSIsNonTerminal(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
certificate := &coredata.Certificate{
|
|
Status: coredata.CertificateStatusPending,
|
|
SSLRetryCount: 0,
|
|
}
|
|
|
|
outcome := decideProvisioningOutcome(certificate, ProvisioningErrorDNSCNAME)
|
|
|
|
assert.Equal(t, coredata.CertificateStatusPending, outcome.status)
|
|
assert.Equal(t, 0, outcome.retryCount)
|
|
assert.True(t, outcome.clearACMEState)
|
|
assert.Equal(t, ProvisioningErrorDNSCNAME, outcome.errorCode)
|
|
}
|
|
|
|
func TestDecideProvisioningOutcome_RateLimitWithoutOrderStaysPending(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
certificate := &coredata.Certificate{
|
|
Status: coredata.CertificateStatusPending,
|
|
SSLRetryCount: 1,
|
|
}
|
|
|
|
outcome := decideProvisioningOutcome(certificate, ProvisioningErrorACMERateLimited)
|
|
|
|
require.Equal(t, coredata.CertificateStatusPending, outcome.status)
|
|
assert.Equal(t, 1, outcome.retryCount)
|
|
assert.False(t, outcome.clearACMEState)
|
|
}
|