Close cert provisioning correctness gaps

Several race and validity gaps could leave certificate provisioning
stuck, unusable, or noisy:

- Accept the HTTP-01 challenge only after the key authorization is
  committed, so the CA cannot hit the token before this instance can
  serve it and invalidate the order.
- Persist challenge metadata under a blocking write-back lock; a row
  merely locked by a competing transaction no longer silently drops the
  accepted order.
- Abandon a recovered VALID order and restart instead of issuing it
  with a freshly generated key that cannot match the existing cert.
- Exclude rate-limited rows from the ten-minute stale reset so the
  resumable order survives the ACME cooldown.
- Size the provisioning poll lease to exceed the max processing window
  so a released claim lock cannot let another worker process the same
  row concurrently.
- Parse Retry-After as unsigned seconds and clamp overflow so malformed
  values fall back to the default cooldown instead of disabling it.
- Normalize the acme_errors problem_type label to the RFC 8555 set to
  bound Prometheus cardinality.

Signed-off-by: Bryan Frimin <bryan@probo.com>

pkg/certmanager/metrics.go

@@ -22,6 +22,7 @@ package certmanager
 
 import (
 	"errors"
+	"strings"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
@@ -127,11 +128,49 @@ func (m *metrics) observeStep(phase provisionPhase, result provisionResult, star
 }
 
 func (m *metrics) recordACMEError(problemType string) {
-	if problemType == "" {
-		problemType = "unknown"
+	m.acmeErrors.WithLabelValues(normalizeProblemType(problemType)).Inc()
+}
+
+const acmeProblemTypePrefix = "urn:ietf:params:acme:error:"
+
+var knownACMEProblemTypes = map[string]struct{}{
+	"accountdoesnotexist":     {},
+	"alreadyrevoked":          {},
+	"badcsr":                  {},
+	"badnonce":                {},
+	"badpublickey":            {},
+	"badrevocationreason":     {},
+	"badsignaturealgorithm":   {},
+	"caa":                     {},
+	"compound":                {},
+	"connection":              {},
+	"dns":                     {},
+	"externalaccountrequired": {},
+	"incorrectresponse":       {},
+	"invalidcontact":          {},
+	"malformed":               {},
+	"ordernotready":           {},
+	"ratelimited":             {},
+	"rejectedidentifier":      {},
+	"serverinternal":          {},
+	"tls":                     {},
+	"unauthorized":            {},
+	"unsupportedcontact":      {},
+	"unsupportedidentifier":   {},
+	"useractionrequired":      {},
+}
+
+func normalizeProblemType(problemType string) string {
+	suffix, ok := strings.CutPrefix(strings.ToLower(problemType), acmeProblemTypePrefix)
+	if !ok {
+		return "unknown"
 	}
 
-	m.acmeErrors.WithLabelValues(problemType).Inc()
+	if _, ok := knownACMEProblemTypes[suffix]; !ok {
+		return "unknown"
+	}
+
+	return suffix
 }
 
 func (m *metrics) setCooldown(active bool) {