Fix certmanager provisioning retry and metrics

Address several provisioning defects that either stalled the retry
budget or crashed the process:

- Classify CAA resolver/transport failures apart from a real CAA policy
  denial. Both shared the "caa records" wording, so a transient resolver
  error was persisted as customer misconfiguration and retried forever
  without consuming the retry budget. A new ErrCAANotPermitted sentinel
  now marks the genuine misconfiguration; other CAA errors are treated
  as ordinary transient failures.

- Honor an explicit Retry-After: 0 (or a past date) as permission for an
  immediate retry instead of promoting it to the one-hour default
  cooldown. acme.RateLimit collapses zero, invalid, and absent headers
  to a zero duration, so the header is now parsed directly to tell an
  explicit zero apart from a missing one.

- Reuse already-registered Prometheus collectors when a second
  ACMEService shares a registerer. The fixed-name collectors were
  MustRegistered, so a duplicate registration panicked the process.

- Persist provisioning failures on a context detached from the process
  tick deadline. A timed-out attempt reached persistFailure with an
  expired context, so the write-back failed and the retry budget never
  advanced, leaving the certificate indefinitely retriable.

- Use pgx.StrictNamedArgs in the certificate FOR UPDATE loaders to match
  the coredata SQL contract.

Signed-off-by: Bryan Frimin <bryan@probo.com>

pkg/certmanager/metrics.go

@@ -21,6 +21,7 @@
 package certmanager
 
 import (
+	"errors"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
@@ -56,48 +57,68 @@ func newMetrics(registerer prometheus.Registerer) *metrics {
 		registerer = prometheus.DefaultRegisterer
 	}
 
-	m := &metrics{
-		provisionSteps: prometheus.NewCounterVec(
-			prometheus.CounterOpts{
-				Subsystem: "certmanager",
-				Name:      "certificate_provision_steps_total",
-				Help:      "Certificate provisioning steps by phase and result.",
-			},
-			[]string{"phase", "result"},
+	return &metrics{
+		provisionSteps: registerCollector(
+			registerer,
+			prometheus.NewCounterVec(
+				prometheus.CounterOpts{
+					Subsystem: "certmanager",
+					Name:      "certificate_provision_steps_total",
+					Help:      "Certificate provisioning steps by phase and result.",
+				},
+				[]string{"phase", "result"},
+			),
 		),
-		acmeErrors: prometheus.NewCounterVec(
-			prometheus.CounterOpts{
-				Subsystem: "certmanager",
-				Name:      "certificate_acme_errors_total",
-				Help:      "ACME errors by problem type.",
-			},
-			[]string{"problem_type"},
+		acmeErrors: registerCollector(
+			registerer,
+			prometheus.NewCounterVec(
+				prometheus.CounterOpts{
+					Subsystem: "certmanager",
+					Name:      "certificate_acme_errors_total",
+					Help:      "ACME errors by problem type.",
+				},
+				[]string{"problem_type"},
+			),
 		),
-		acmeCooldown: prometheus.NewGauge(
-			prometheus.GaugeOpts{
-				Subsystem: "certmanager",
-				Name:      "certificate_acme_cooldown",
-				Help:      "1 while the ACME client is in a global rate-limit cooldown.",
-			},
+		acmeCooldown: registerCollector(
+			registerer,
+			prometheus.NewGauge(
+				prometheus.GaugeOpts{
+					Subsystem: "certmanager",
+					Name:      "certificate_acme_cooldown",
+					Help:      "1 while the ACME client is in a global rate-limit cooldown.",
+				},
+			),
 		),
-		stepDuration: prometheus.NewHistogramVec(
-			prometheus.HistogramOpts{
-				Subsystem: "certmanager",
-				Name:      "certificate_provision_step_duration_seconds",
-				Help:      "Duration of certificate provisioning steps in seconds.",
-			},
-			[]string{"phase"},
+		stepDuration: registerCollector(
+			registerer,
+			prometheus.NewHistogramVec(
+				prometheus.HistogramOpts{
+					Subsystem: "certmanager",
+					Name:      "certificate_provision_step_duration_seconds",
+					Help:      "Duration of certificate provisioning steps in seconds.",
+				},
+				[]string{"phase"},
+			),
 		),
 	}
+}
 
-	registerer.MustRegister(
-		m.provisionSteps,
-		m.acmeErrors,
-		m.acmeCooldown,
-		m.stepDuration,
-	)
+func registerCollector[T prometheus.Collector](
+	registerer prometheus.Registerer,
+	collector T,
+) T {
+	if err := registerer.Register(collector); err != nil {
+		if already, ok := errors.AsType[prometheus.AlreadyRegisteredError](err); ok {
+			if existing, ok := already.ExistingCollector.(T); ok {
+				return existing
+			}
+		}
 
-	return m
+		panic(err)
+	}
+
+	return collector
 }
 
 func (m *metrics) observeStep(phase provisionPhase, result provisionResult, started time.Time) {