Close cert provisioning correctness gaps

Several race and validity gaps could leave certificate provisioning
stuck, unusable, or noisy:

- Accept the HTTP-01 challenge only after the key authorization is
  committed, so the CA cannot hit the token before this instance can
  serve it and invalidate the order.
- Persist challenge metadata under a blocking write-back lock; a row
  merely locked by a competing transaction no longer silently drops the
  accepted order.
- Abandon a recovered VALID order and restart instead of issuing it
  with a freshly generated key that cannot match the existing cert.
- Exclude rate-limited rows from the ten-minute stale reset so the
  resumable order survives the ACME cooldown.
- Size the provisioning poll lease to exceed the max processing window
  so a released claim lock cannot let another worker process the same
  row concurrently.
- Parse Retry-After as unsigned seconds and clamp overflow so malformed
  values fall back to the default cooldown instead of disabling it.
- Normalize the acme_errors problem_type label to the RFC 8555 set to
  bound Prometheus cardinality.

Signed-off-by: Bryan Frimin <bryan@probo.com>

pkg/certmanager/acme.go

@@ -182,8 +182,6 @@ func (s *ACMEService) registerAccount(ctx context.Context) error {
 	return nil
 }
 
-// StartHTTPChallenge creates an ACME order, accepts the HTTP-01 challenge, and
-// returns the persisted challenge metadata. It never waits for order completion.
 func (s *ACMEService) StartHTTPChallenge(ctx context.Context, domain string) (*HTTPChallenge, error) {
 	started := time.Now()
 
@@ -223,15 +221,6 @@ func (s *ACMEService) StartHTTPChallenge(ctx context.Context, domain string) (*H
 		return nil, fmt.Errorf("cannot get challenge response: %w", err)
 	}
 
-	challenge1 := &acme.Challenge{
-		URI:   challenge.URI,
-		Token: challenge.Token,
-	}
-
-	if _, err := s.client.Accept(ctx, challenge1); err != nil && !isChallengeAlreadyValid(err) {
-		return nil, s.handleError(provisionPhaseCreateOrder, started, "cannot accept challenge", err)
-	}
-
 	s.metrics.observeStep(provisionPhaseCreateOrder, provisionResultOK, started)
 
 	return &HTTPChallenge{
@@ -243,7 +232,23 @@ func (s *ACMEService) StartHTTPChallenge(ctx context.Context, domain string) (*H
 	}, nil
 }
 
-// PollOrder performs a single GetOrder call and classifies the result.
+func (s *ACMEService) AcceptHTTPChallenge(ctx context.Context, challenge *HTTPChallenge) error {
+	started := time.Now()
+
+	acceptChallenge := &acme.Challenge{
+		URI:   challenge.URL,
+		Token: challenge.Token,
+	}
+
+	if _, err := s.client.Accept(ctx, acceptChallenge); err != nil && !isChallengeAlreadyValid(err) {
+		return s.handleError(provisionPhaseCreateOrder, started, "cannot accept challenge", err)
+	}
+
+	s.metrics.observeStep(provisionPhaseCreateOrder, provisionResultOK, started)
+
+	return nil
+}
+
 func (s *ACMEService) PollOrder(ctx context.Context, orderURL string) (*OrderPollResult, error) {
 	started := time.Now()
 
@@ -279,7 +284,6 @@ func (s *ACMEService) PollOrder(ctx context.Context, orderURL string) (*OrderPol
 	return result, nil
 }
 
-// IssueCertificate finalizes a ready order or fetches a valid certificate.
 func (s *ACMEService) IssueCertificate(
 	ctx context.Context,
 	challenge *HTTPChallenge,