From b524e9b4978d761218818fef56e7a98cbb9a1ee1 Mon Sep 17 00:00:00 2001 From: Bryan Frimin Date: Fri, 10 Jul 2026 15:10:19 +0200 Subject: [PATCH] Refactor certmanager to worker service Replace the Provisioner and Renewer with poll-based provision and renew workers orchestrated by a certmanager Service. Certificate operations are now hostname-centric and driven by the certificates table, decoupled from custom-domain business logic. Signed-off-by: Bryan Frimin --- pkg/certmanager/acme.go | 134 ++++- pkg/certmanager/acme_challenge_handler.go | 8 +- pkg/certmanager/cache_store.go | 50 +- pkg/certmanager/provision_worker.go | 571 ++++++++++++++++++++++ pkg/certmanager/provisioner.go | 520 -------------------- pkg/certmanager/renew_worker.go | 127 +++++ pkg/certmanager/renewer.go | 166 ------- pkg/certmanager/selector.go | 34 +- pkg/certmanager/service.go | 239 +++++++++ 9 files changed, 1115 insertions(+), 734 deletions(-) create mode 100644 pkg/certmanager/provision_worker.go delete mode 100644 pkg/certmanager/provisioner.go create mode 100644 pkg/certmanager/renew_worker.go delete mode 100644 pkg/certmanager/renewer.go create mode 100644 pkg/certmanager/service.go diff --git a/pkg/certmanager/acme.go b/pkg/certmanager/acme.go index 3a9e00da4..34d704a49 100644 --- a/pkg/certmanager/acme.go +++ b/pkg/certmanager/acme.go @@ -27,7 +27,9 @@ import ( "crypto/tls" "crypto/x509" "crypto/x509/pkix" + "errors" "fmt" + "strings" "time" "go.gearno.de/kit/httpclient" @@ -183,7 +185,7 @@ func (s *ACMEService) CompleteHTTPChallenge( Token: challenge0.Token, } - if _, err := s.client.Accept(ctx, challenge1); err != nil { + if _, err := s.client.Accept(ctx, challenge1); err != nil && !isChallengeAlreadyValid(err) { return nil, fmt.Errorf("cannot accept challenge: %w", err) } @@ -202,7 +204,7 @@ func (s *ACMEService) CompleteHTTPChallenge( return nil, fmt.Errorf("cannot create CSR: %w", err) } - der, _, err := s.client.CreateOrderCert(ctx, order.FinalizeURL, csr, true) + der, err := s.issueOrderCertificate(ctx, order, challenge0.OrderURL, csr) if err != nil { return nil, fmt.Errorf("cannot create certificate: %w", err) } @@ -238,6 +240,134 @@ func (s *ACMEService) CheckRenewalNeeded(expiresAt time.Time, threshold time.Dur return time.Until(expiresAt) <= threshold } +func isChallengeAlreadyValid(err error) bool { + acmeErr, ok := errors.AsType[*acme.Error](err) + if !ok { + return false + } + + return acmeErr.ProblemType == "urn:ietf:params:acme:error:malformed" && + strings.Contains(acmeErr.Detail, "status valid") +} + +func (s *ACMEService) issueOrderCertificate( + ctx context.Context, + order *acme.Order, + orderURL string, + csr []byte, +) ([][]byte, error) { + pollURL := acmeOrderURL(order, orderURL) + + switch order.Status { + case acme.StatusValid: + return s.fetchOrderCertificate(ctx, pollURL, order) + case acme.StatusReady: + if order.FinalizeURL == "" { + return nil, fmt.Errorf("order is ready but finalize URL is missing") + } + + der, _, err := s.client.CreateOrderCert(ctx, order.FinalizeURL, csr, true) + if err == nil { + return der, nil + } + + // CreateOrderCert finalizes the order but may fail to download the + // certificate when the CA marks the order valid before the certificate + // URL is populated. Poll the order using the known order URL because + // some CAs (including Pebble) omit the Location header on poll + // responses, leaving order.URI empty. + return s.fetchOrderCertificateAfterFinalize(ctx, pollURL, err) + default: + return nil, fmt.Errorf("order is in unexpected status %q", order.Status) + } +} + +func acmeOrderURL(order *acme.Order, orderURL string) string { + if order.URI != "" { + return order.URI + } + + return orderURL +} + +func (s *ACMEService) fetchOrderCertificateAfterFinalize( + ctx context.Context, + orderURL string, + finalizeErr error, +) ([][]byte, error) { + refreshed, err := s.client.GetOrder(ctx, orderURL) + if err != nil { + return nil, fmt.Errorf("cannot refresh order after finalize: %w", err) + } + + if refreshed.Status != acme.StatusValid { + refreshed, err = s.client.WaitOrder(ctx, orderURL) + if err != nil { + return nil, fmt.Errorf("cannot wait for order after finalize: %w", err) + } + } + + der, err := s.fetchOrderCertificate(ctx, orderURL, refreshed) + if err != nil { + return nil, fmt.Errorf("cannot fetch certificate after finalize: %w: %w", finalizeErr, err) + } + + return der, nil +} + +func (s *ACMEService) fetchOrderCertificate( + ctx context.Context, + orderURL string, + order *acme.Order, +) ([][]byte, error) { + orderWithCertURL, err := s.waitForCertificateURL(ctx, orderURL, order) + if err != nil { + return nil, err + } + + der, err := s.client.FetchCert(ctx, orderWithCertURL.CertURL, true) + if err != nil { + return nil, fmt.Errorf("cannot fetch certificate: %w", err) + } + + return der, nil +} + +func (s *ACMEService) waitForCertificateURL( + ctx context.Context, + orderURL string, + order *acme.Order, +) (*acme.Order, error) { + if order.CertURL != "" { + return order, nil + } + + deadline := time.Now().Add(30 * time.Second) + + for time.Now().Before(deadline) { + refreshed, err := s.client.GetOrder(ctx, orderURL) + if err != nil { + return nil, fmt.Errorf("cannot refresh order: %w", err) + } + + if refreshed.CertURL != "" { + return refreshed, nil + } + + if refreshed.Status != acme.StatusValid { + return nil, fmt.Errorf("order left valid state while waiting for certificate URL") + } + + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(500 * time.Millisecond): + } + } + + return nil, fmt.Errorf("timed out waiting for certificate URL") +} + func createCSR(domain string, key crypto.Signer) ([]byte, error) { template := &x509.CertificateRequest{ Subject: pkix.Name{ diff --git a/pkg/certmanager/acme_challenge_handler.go b/pkg/certmanager/acme_challenge_handler.go index 4df418736..362ec60c5 100644 --- a/pkg/certmanager/acme_challenge_handler.go +++ b/pkg/certmanager/acme_challenge_handler.go @@ -84,16 +84,16 @@ func (h *ACMEChallengeHandler) getKeyAuthForToken(ctx context.Context, token str err := h.pg.WithConn( ctx, func(ctx context.Context, conn pg.Querier) error { - domain := &coredata.CustomDomain{} - if err := domain.LoadByHTTPChallengeToken(ctx, conn, coredata.NewNoScope(), token); err != nil { + certificate := &coredata.Certificate{} + if err := certificate.LoadByHTTPChallengeToken(ctx, conn, coredata.NewNoScope(), token); err != nil { return err } - if domain.HTTPChallengeKeyAuth == nil { + if certificate.HTTPChallengeKeyAuth == nil { return http.ErrNotSupported } - keyAuth = *domain.HTTPChallengeKeyAuth + keyAuth = *certificate.HTTPChallengeKeyAuth return nil }, diff --git a/pkg/certmanager/cache_store.go b/pkg/certmanager/cache_store.go index d6b4cc7cb..a515d6d81 100644 --- a/pkg/certmanager/cache_store.go +++ b/pkg/certmanager/cache_store.go @@ -59,35 +59,35 @@ func (w *CacheStore) WarmCache(ctx context.Context) error { err := w.pg.WithConn( ctx, func(ctx context.Context, conn pg.Querier) error { - domains := coredata.CustomDomains{} - if err := domains.LoadActiveCertificates(ctx, conn, coredata.NewNoScope()); err != nil { + certificates := coredata.Certificates{} + if err := certificates.LoadActive(ctx, conn, coredata.NewNoScope()); err != nil { return fmt.Errorf("cannot load active certificates: %w", err) } - if len(domains) == 0 { + if len(certificates) == 0 { w.logger.InfoCtx(ctx, "no active certificates to warm") return nil } - w.logger.InfoCtx(ctx, "found active certificates to cache", log.Int("count", len(domains))) + w.logger.InfoCtx(ctx, "found active certificates to cache", log.Int("count", len(certificates))) successCount := 0 - for _, domain := range domains { + for _, certificate := range certificates { select { case <-ctx.Done(): return ctx.Err() default: } - if err := w.warmDomain(ctx, conn, domain); err != nil { - w.logger.ErrorCtx(ctx, "cannot warm certificate cache for domain", log.String("domain", domain.Domain), log.Error(err)) + if err := w.warmCertificate(ctx, conn, certificate); err != nil { + w.logger.ErrorCtx(ctx, "cannot warm certificate cache for hostname", log.String("hostname", certificate.Hostname), log.Error(err)) } else { successCount++ } } - w.logger.InfoCtx(ctx, "successfully warmed cache", log.Int("success_count", successCount), log.Int("total_count", len(domains))) + w.logger.InfoCtx(ctx, "successfully warmed cache", log.Int("success_count", successCount), log.Int("total_count", len(certificates))) return nil }, @@ -101,45 +101,45 @@ func (w *CacheStore) WarmCache(ctx context.Context) error { return nil } -func (w *CacheStore) warmDomain(ctx context.Context, conn pg.Querier, domain *coredata.CustomDomain) error { - var loadedDomain coredata.CustomDomain - if err := loadedDomain.LoadByID(ctx, conn, coredata.NewNoScope(), domain.ID); err != nil { - return fmt.Errorf("cannot load domain with decrypted values: %w", err) +func (w *CacheStore) warmCertificate(ctx context.Context, conn pg.Querier, certificate *coredata.Certificate) error { + var loadedCertificate coredata.Certificate + if err := loadedCertificate.LoadByID(ctx, conn, coredata.NewNoScope(), certificate.ID); err != nil { + return fmt.Errorf("cannot load certificate with decrypted values: %w", err) } - if err := loadedDomain.ParseCertificate(w.encryptionKey); err != nil { + if err := loadedCertificate.ParseCertificate(w.encryptionKey); err != nil { return fmt.Errorf("cannot parse certificate: %w", err) } - if len(loadedDomain.SSLCertificatePEM) == 0 { - return fmt.Errorf("domain has no certificate PEM") + if len(loadedCertificate.SSLCertificatePEM) == 0 { + return fmt.Errorf("certificate has no certificate PEM") } - privateKeyPEM, err := loadedDomain.DecryptPrivateKey(w.encryptionKey) + privateKeyPEM, err := loadedCertificate.DecryptPrivateKey(w.encryptionKey) if err != nil { return fmt.Errorf("cannot decrypt private key: %w", err) } if len(privateKeyPEM) == 0 { - return fmt.Errorf("domain has no private key PEM") + return fmt.Errorf("certificate has no private key PEM") } - if loadedDomain.SSLExpiresAt == nil { - return fmt.Errorf("domain certificate has no expiry date") + if loadedCertificate.SSLExpiresAt == nil { + return fmt.Errorf("certificate has no expiry date") } - if time.Now().After(*loadedDomain.SSLExpiresAt) { + if time.Now().After(*loadedCertificate.SSLExpiresAt) { return fmt.Errorf("certificate has expired") } cache := &coredata.CachedCertificate{ - Domain: loadedDomain.Domain, - CertificatePEM: string(loadedDomain.SSLCertificatePEM), + Domain: loadedCertificate.Hostname, + CertificatePEM: string(loadedCertificate.SSLCertificatePEM), PrivateKeyPEM: string(privateKeyPEM), - CertificateChain: loadedDomain.SSLCertificateChain, - ExpiresAt: *loadedDomain.SSLExpiresAt, + CertificateChain: loadedCertificate.SSLCertificateChain, + ExpiresAt: *loadedCertificate.SSLExpiresAt, CachedAt: time.Now(), - CustomDomainID: loadedDomain.ID, + CertificateID: loadedCertificate.ID, } if err := cache.Upsert(ctx, conn); err != nil { diff --git a/pkg/certmanager/provision_worker.go b/pkg/certmanager/provision_worker.go new file mode 100644 index 000000000..6da73b30c --- /dev/null +++ b/pkg/certmanager/provision_worker.go @@ -0,0 +1,571 @@ +// Copyright (c) 2025-2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package certmanager + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "codeberg.org/miekg/dns" + "go.gearno.de/kit/log" + "go.gearno.de/kit/pg" + "go.gearno.de/kit/worker" + "go.probo.inc/probo/pkg/coredata" + "go.probo.inc/probo/pkg/crypto/cipher" + "go.probo.inc/probo/pkg/gid" +) + +const maxProvisioningRetries = 3 + +type provisionHandler struct { + pg *pg.Client + acmeService *ACMEService + encryptionKey cipher.EncryptionKey + cnameTarget string + caaIssuerDomain string + resolverAddr string + managedBaseDomain string + logger *log.Logger +} + +var ( + _ worker.Handler[coredata.Certificate] = (*provisionHandler)(nil) + _ worker.StaleRecoverer = (*provisionHandler)(nil) +) + +func NewProvisionWorker( + pgClient *pg.Client, + acmeService *ACMEService, + encryptionKey cipher.EncryptionKey, + cnameTarget string, + caaIssuerDomain string, + resolverAddr string, + managedBaseDomain string, + logger *log.Logger, + opts ...worker.Option, +) *worker.Worker[coredata.Certificate] { + h := &provisionHandler{ + pg: pgClient, + acmeService: acmeService, + encryptionKey: encryptionKey, + cnameTarget: cnameTarget, + caaIssuerDomain: caaIssuerDomain, + resolverAddr: resolverAddr, + managedBaseDomain: managedBaseDomain, + logger: logger, + } + + return worker.New( + "certificate-provision-worker", + h, + logger, + opts..., + ) +} + +func (h *provisionHandler) Claim(ctx context.Context) (coredata.Certificate, error) { + var certificate coredata.Certificate + + if err := h.pg.WithTx( + ctx, + func(ctx context.Context, tx pg.Tx) error { + if err := certificate.LoadNextForProvisioningForUpdateSkipLocked(ctx, tx); err != nil { + return err + } + + return nil + }, + ); err != nil { + if errors.Is(err, coredata.ErrResourceNotFound) { + return coredata.Certificate{}, worker.ErrNoTask + } + + return coredata.Certificate{}, err + } + + return certificate, nil +} + +func (h *provisionHandler) Process(ctx context.Context, certificate coredata.Certificate) error { + challengeInitiated, err := h.runProvisionCertificate(ctx, certificate.ID) + if err != nil { + h.logger.ErrorCtx( + ctx, + "cannot provision certificate", + log.String("hostname", certificate.Hostname), + log.Error(err), + ) + + return err + } + + if !challengeInitiated { + return nil + } + + if _, err := h.runProvisionCertificate(ctx, certificate.ID); err != nil { + h.logger.ErrorCtx( + ctx, + "cannot complete certificate challenge", + log.String("hostname", certificate.Hostname), + log.Error(err), + ) + + return err + } + + return nil +} + +func (h *provisionHandler) runProvisionCertificate( + ctx context.Context, + certificateID gid.GID, +) (bool, error) { + var challengeInitiated bool + + err := h.pg.WithTx( + ctx, + func(ctx context.Context, tx pg.Tx) error { + var err error + + challengeInitiated, err = h.provisionCertificate(ctx, tx, certificateID) + + return err + }, + ) + if err != nil { + return false, err + } + + return challengeInitiated, nil +} + +func (h *provisionHandler) RecoverStale(ctx context.Context) error { + return h.pg.WithTx( + ctx, + func(ctx context.Context, tx pg.Tx) error { + var certificates coredata.Certificates + if err := certificates.ListStaleProvisioning(ctx, tx, coredata.NewNoScope()); err != nil { + return fmt.Errorf("cannot load stale provisioning certificates: %w", err) + } + + if len(certificates) == 0 { + return nil + } + + h.logger.InfoCtx(ctx, "found stale provisioning attempts to reset", log.Int("count", len(certificates))) + + for _, certificate := range certificates { + if err := h.resetStaleCertificate(ctx, tx, certificate); err != nil { + h.logger.ErrorCtx( + ctx, + "cannot reset stale certificate", + log.String("hostname", certificate.Hostname), + log.Error(err), + ) + } + } + + return nil + }, + ) +} + +func (h *provisionHandler) checkDNSConfiguration(hostname string) error { + customerFQDN := hostname + if !strings.HasSuffix(customerFQDN, ".") { + customerFQDN = customerFQDN + "." + } + + expectedFQDN := h.cnameTarget + if !strings.HasSuffix(expectedFQDN, ".") { + expectedFQDN = expectedFQDN + "." + } + + msg := &dns.Msg{MsgHeader: dns.MsgHeader{ID: dns.ID(), RecursionDesired: true}} + msg.Question = []dns.RR{&dns.CNAME{Hdr: dns.Header{Name: customerFQDN, Class: dns.ClassINET}}} + + client := dns.NewClient() + + resp, _, err := client.Exchange(context.Background(), msg, "udp", h.resolverAddr) + if err != nil { + return fmt.Errorf("cannot exchange dns message: %w", err) + } + + if len(resp.Answer) == 0 { + return fmt.Errorf("no cname records found for domain %q", hostname) + } + + if len(resp.Answer) > 1 { + return fmt.Errorf("multiple cname records found for domain %q", hostname) + } + + resolvedRecord, ok := resp.Answer[0].(*dns.CNAME) + if !ok { + return fmt.Errorf("first answer is not a cname record for domain %q", hostname) + } + + if !strings.EqualFold(expectedFQDN, resolvedRecord.Target) { + return fmt.Errorf( + "cname target mismatch: domain %q resolves to %q, expected %q", + hostname, + resolvedRecord.Target, + expectedFQDN, + ) + } + + return nil +} + +func (h *provisionHandler) checkCAARecords(hostname string) error { + fqdn := hostname + if !strings.HasSuffix(fqdn, ".") { + fqdn = fqdn + "." + } + + msg := &dns.Msg{MsgHeader: dns.MsgHeader{ID: dns.ID(), RecursionDesired: true}} + msg.Question = []dns.RR{&dns.CAA{Hdr: dns.Header{Name: fqdn, Class: dns.ClassINET}}} + + client := dns.NewClient() + + resp, _, err := client.Exchange( + context.Background(), + msg, + "udp", + h.resolverAddr, + ) + if err != nil { + return fmt.Errorf("cannot exchange dns message for caa records: %w", err) + } + + var caaRecords []*dns.CAA + + for _, rr := range resp.Answer { + if caa, ok := rr.(*dns.CAA); ok { + caaRecords = append(caaRecords, caa) + } + } + + if len(caaRecords) == 0 { + return nil + } + + for _, caa := range caaRecords { + if caa.Tag == "issue" { + issuer, _, _ := strings.Cut(caa.Value, ";") + if strings.EqualFold(strings.TrimSpace(issuer), h.caaIssuerDomain) { + return nil + } + } + } + + return fmt.Errorf( + "caa records for domain %q do not permit issuance by %q", + hostname, + h.caaIssuerDomain, + ) +} + +func (h *provisionHandler) skipsDNSChecks( + ctx context.Context, + conn pg.Querier, + hostname string, +) (bool, error) { + if h.managedBaseDomain == "" { + return false, nil + } + + suffix := "." + h.managedBaseDomain + if hostname != h.managedBaseDomain && !strings.HasSuffix(hostname, suffix) { + return false, nil + } + + domain := &coredata.CustomDomain{} + err := domain.LoadByDomain(ctx, conn, coredata.NewNoScope(), hostname) + if errors.Is(err, coredata.ErrResourceNotFound) { + return true, nil + } + + if err != nil { + return false, fmt.Errorf("cannot load custom domain: %w", err) + } + + return domain.Managed, nil +} + +func (h *provisionHandler) resetStaleCertificate( + ctx context.Context, + tx pg.Tx, + certificate *coredata.Certificate, +) error { + fullCertificate := &coredata.Certificate{} + if err := fullCertificate.LoadByIDForUpdateSkipLocked(ctx, tx, coredata.NewNoScope(), certificate.ID); err != nil { + if errors.Is(err, coredata.ErrResourceNotFound) { + return nil + } + + return fmt.Errorf("cannot load stale certificate for update: %w", err) + } + + staleDuration := time.Since(fullCertificate.UpdatedAt) + + h.logger.InfoCtx( + ctx, + "resetting stale certificate", + log.String("hostname", fullCertificate.Hostname), + log.String("status", string(fullCertificate.Status)), + log.Duration("stale_duration", staleDuration), + log.Int("retry_count", fullCertificate.SSLRetryCount), + ) + + fullCertificate.HTTPChallengeToken = nil + fullCertificate.HTTPChallengeKeyAuth = nil + fullCertificate.HTTPChallengeURL = nil + fullCertificate.HTTPOrderURL = nil + fullCertificate.ProvisioningError = nil + fullCertificate.Status = coredata.CertificateStatusPending + + if fullCertificate.SSLLastAttemptAt != nil && time.Since(*fullCertificate.SSLLastAttemptAt) > 24*time.Hour { + h.logger.InfoCtx( + ctx, + "resetting retry count due to old last attempt", + log.String("hostname", fullCertificate.Hostname), + log.Time("last_attempt", *fullCertificate.SSLLastAttemptAt), + ) + fullCertificate.SSLRetryCount = 0 + fullCertificate.SSLLastAttemptAt = nil + } + + if err := fullCertificate.Update(ctx, tx, coredata.NewNoScope()); err != nil { + return fmt.Errorf("cannot update stale certificate: %w", err) + } + + return nil +} + +func (h *provisionHandler) provisionCertificate( + ctx context.Context, + tx pg.Tx, + certificateID gid.GID, +) (bool, error) { + certificate := &coredata.Certificate{} + if err := certificate.LoadByIDForUpdateSkipLocked(ctx, tx, coredata.NewNoScope(), certificateID); err != nil { + if errors.Is(err, coredata.ErrResourceNotFound) { + return false, nil + } + + return false, fmt.Errorf("cannot load by id for update %q certificate: %w", certificateID, err) + } + + if certificate.Status == coredata.CertificateStatusPending || certificate.Status == coredata.CertificateStatusRenewing { + skipDNSChecks, err := h.skipsDNSChecks(ctx, tx, certificate.Hostname) + if err != nil { + return false, fmt.Errorf("cannot check managed domain: %w", err) + } + + if !skipDNSChecks { + if err := h.checkDNSConfiguration(certificate.Hostname); err != nil { + h.logger.WarnCtx( + ctx, + "dns configuration check failed", + log.String("hostname", certificate.Hostname), + log.Error(err), + ) + + errMsg := err.Error() + + certificate.ProvisioningError = &errMsg + if err := certificate.Update(ctx, tx, coredata.NewNoScope()); err != nil { + return false, fmt.Errorf("cannot update certificate with provisioning error: %w", err) + } + + return false, nil + } + + if err := h.checkCAARecords(certificate.Hostname); err != nil { + h.logger.WarnCtx( + ctx, + "caa record check failed", + log.String("hostname", certificate.Hostname), + log.Error(err), + ) + + errMsg := err.Error() + + certificate.ProvisioningError = &errMsg + if err := certificate.Update(ctx, tx, coredata.NewNoScope()); err != nil { + return false, fmt.Errorf("cannot update certificate with provisioning error: %w", err) + } + + return false, nil + } + } + + certificate.ProvisioningError = nil + if err := certificate.Update(ctx, tx, coredata.NewNoScope()); err != nil { + return false, fmt.Errorf("cannot clear provisioning error: %w", err) + } + + h.logger.InfoCtx(ctx, "DNS configuration verified, initiating HTTP challenge for hostname", log.String("hostname", certificate.Hostname)) + + challenge, err := h.acmeService.GetHTTPChallenge(ctx, certificate.Hostname) + if err != nil { + h.logger.ErrorCtx( + ctx, + "cannot get HTTP challenge", + log.String("hostname", certificate.Hostname), + log.Error(err), + ) + + return false, err + } + + certificate.HTTPChallengeToken = &challenge.Token + certificate.HTTPChallengeKeyAuth = &challenge.KeyAuth + certificate.HTTPChallengeURL = &challenge.URL + certificate.HTTPOrderURL = &challenge.OrderURL + certificate.Status = coredata.CertificateStatusProvisioning + + if err := certificate.Update(ctx, tx, coredata.NewNoScope()); err != nil { + return false, fmt.Errorf("cannot update certificate with challenge: %w", err) + } + + h.logger.InfoCtx( + ctx, + "HTTP challenge initiated, completing in same cycle", + log.String("hostname", certificate.Hostname), + log.String("token", challenge.Token), + ) + + return true, nil + } + + if certificate.HTTPChallengeToken == nil || + certificate.HTTPChallengeKeyAuth == nil || + certificate.HTTPChallengeURL == nil || + certificate.HTTPOrderURL == nil { + certificate.Status = coredata.CertificateStatusPending + + if err := certificate.Update(ctx, tx, coredata.NewNoScope()); err != nil { + return false, fmt.Errorf("cannot reset certificate without challenge data: %w", err) + } + + return false, nil + } + + challenge := &HTTPChallenge{ + Domain: certificate.Hostname, + Token: *certificate.HTTPChallengeToken, + KeyAuth: *certificate.HTTPChallengeKeyAuth, + URL: *certificate.HTTPChallengeURL, + OrderURL: *certificate.HTTPOrderURL, + } + + cert, err := h.acmeService.CompleteHTTPChallenge(ctx, challenge) + if err != nil { + h.logger.WarnCtx( + ctx, + "cannot complete HTTP challenge", + log.String("hostname", certificate.Hostname), + log.Int("retry_count", certificate.SSLRetryCount), + log.Error(err), + ) + + errMsg := err.Error() + certificate.ProvisioningError = &errMsg + certificate.SSLRetryCount = certificate.SSLRetryCount + 1 + now := time.Now() + certificate.SSLLastAttemptAt = &now + + certificate.HTTPChallengeToken = nil + certificate.HTTPChallengeKeyAuth = nil + certificate.HTTPChallengeURL = nil + certificate.HTTPOrderURL = nil + + if certificate.SSLRetryCount >= maxProvisioningRetries { + h.logger.ErrorCtx( + ctx, + "certificate has exceeded max retry attempts, marking as failed", + log.String("hostname", certificate.Hostname), + log.Int("retry_count", certificate.SSLRetryCount), + ) + + certificate.Status = coredata.CertificateStatusFailed + } else { + certificate.Status = coredata.CertificateStatusPending + } + + if err := certificate.Update(ctx, tx, coredata.NewNoScope()); err != nil { + return false, fmt.Errorf("cannot update certificate: %w", err) + } + + return false, nil + } + + h.logger.InfoCtx( + ctx, + "certificate obtained successfully", + log.String("hostname", certificate.Hostname), + log.Time("expires_at", cert.ExpiresAt), + ) + + certificate.ProvisioningError = nil + + certificate.SSLCertificatePEM = cert.CertPEM + if err := certificate.EncryptPrivateKey(cert.KeyPEM, h.encryptionKey); err != nil { + return false, fmt.Errorf("cannot encrypt private key: %w", err) + } + + chainStr := string(cert.ChainPEM) + certificate.SSLCertificateChain = &chainStr + certificate.SSLExpiresAt = &cert.ExpiresAt + certificate.Status = coredata.CertificateStatusActive + + certificate.SSLRetryCount = 0 + certificate.SSLLastAttemptAt = nil + + certificate.HTTPChallengeToken = nil + certificate.HTTPChallengeKeyAuth = nil + certificate.HTTPChallengeURL = nil + certificate.HTTPOrderURL = nil + + if err := certificate.Update(ctx, tx, coredata.NewNoScope()); err != nil { + return false, fmt.Errorf("cannot update certificate: %w", err) + } + + cache := &coredata.CachedCertificate{ + Domain: certificate.Hostname, + CertificatePEM: string(cert.CertPEM), + PrivateKeyPEM: string(cert.KeyPEM), + CertificateChain: &chainStr, + ExpiresAt: cert.ExpiresAt, + CachedAt: time.Now(), + CertificateID: certificate.ID, + } + + if err := cache.Upsert(ctx, tx); err != nil { + h.logger.ErrorCtx( + ctx, + "cannot update certificate cache", + log.String("hostname", certificate.Hostname), + log.Error(err), + ) + } + + return false, nil +} diff --git a/pkg/certmanager/provisioner.go b/pkg/certmanager/provisioner.go deleted file mode 100644 index 65447ebe8..000000000 --- a/pkg/certmanager/provisioner.go +++ /dev/null @@ -1,520 +0,0 @@ -// Copyright (c) 2025-2026 Probo Inc . -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -package certmanager - -import ( - "context" - "errors" - "fmt" - "strings" - "time" - - "codeberg.org/miekg/dns" - "go.gearno.de/kit/log" - "go.gearno.de/kit/pg" - "go.probo.inc/probo/pkg/coredata" - "go.probo.inc/probo/pkg/crypto/cipher" - "go.probo.inc/probo/pkg/gid" -) - -type ( - Provisioner struct { - pg *pg.Client - acmeService *ACMEService - encryptionKey cipher.EncryptionKey - cnameTarget string - caaIssuerDomain string - interval time.Duration - resolverAddr string - logger *log.Logger - } -) - -const ( - maxRetries = 3 -) - -func NewProvisioner( - pg *pg.Client, - acmeService *ACMEService, - encryptionKey cipher.EncryptionKey, - cnameTarget string, - caaIssuerDomain string, - interval time.Duration, - resolverAddr string, - logger *log.Logger, -) *Provisioner { - return &Provisioner{ - pg: pg, - acmeService: acmeService, - encryptionKey: encryptionKey, - cnameTarget: cnameTarget, - caaIssuerDomain: caaIssuerDomain, - interval: interval, - resolverAddr: resolverAddr, - logger: logger.Named("certmanager.provisioner"), - } -} - -func (p *Provisioner) Run(ctx context.Context) error { - p.logger.InfoCtx(ctx, "certificate provisioner starting", log.Duration("interval", p.interval)) - - if err := p.checkPendingDomains(ctx); err != nil { - p.logger.ErrorCtx(ctx, "initial check failed", log.Error(err)) - } - - ticker := time.NewTicker(p.interval) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - p.logger.InfoCtx(ctx, "certificate provisioner shutting down") - return ctx.Err() - case <-ticker.C: - if err := p.checkPendingDomains(ctx); err != nil { - p.logger.ErrorCtx(ctx, "periodic check failed", log.Error(err)) - } - } - } -} - -func (p *Provisioner) checkDNSConfiguration(domain string) error { - customerFQDN := domain - if !strings.HasSuffix(customerFQDN, ".") { - customerFQDN = customerFQDN + "." - } - - expectedFQDN := p.cnameTarget - if !strings.HasSuffix(expectedFQDN, ".") { - expectedFQDN = expectedFQDN + "." - } - - msg := &dns.Msg{MsgHeader: dns.MsgHeader{ID: dns.ID(), RecursionDesired: true}} - msg.Question = []dns.RR{&dns.CNAME{Hdr: dns.Header{Name: customerFQDN, Class: dns.ClassINET}}} - - client := dns.NewClient() - - resp, _, err := client.Exchange(context.Background(), msg, "udp", p.resolverAddr) - if err != nil { - return fmt.Errorf("cannot exchange dns message: %w", err) - } - - if len(resp.Answer) == 0 { - return fmt.Errorf("no cname records found for domain %q", domain) - } - - if len(resp.Answer) > 1 { - return fmt.Errorf("multiple cname records found for domain %q", domain) - } - - resolvedRecord, ok := resp.Answer[0].(*dns.CNAME) - if !ok { - return fmt.Errorf("first answer is not a cname record for domain %q", domain) - } - - if !strings.EqualFold(expectedFQDN, resolvedRecord.Target) { - return fmt.Errorf( - "cname target mismatch: domain %q resolves to %q, expected %q", - domain, - resolvedRecord.Target, - expectedFQDN, - ) - } - - return nil -} - -func (p *Provisioner) checkCAARecords(domain string) error { - fqdn := domain - if !strings.HasSuffix(fqdn, ".") { - fqdn = fqdn + "." - } - - msg := &dns.Msg{MsgHeader: dns.MsgHeader{ID: dns.ID(), RecursionDesired: true}} - msg.Question = []dns.RR{&dns.CAA{Hdr: dns.Header{Name: fqdn, Class: dns.ClassINET}}} - - client := dns.NewClient() - - resp, _, err := client.Exchange( - context.Background(), - msg, - "udp", - p.resolverAddr, - ) - if err != nil { - return fmt.Errorf("cannot exchange dns message for caa records: %w", err) - } - - var caaRecords []*dns.CAA - - for _, rr := range resp.Answer { - if caa, ok := rr.(*dns.CAA); ok { - caaRecords = append(caaRecords, caa) - } - } - - if len(caaRecords) == 0 { - return nil - } - - for _, caa := range caaRecords { - if caa.Tag == "issue" { - issuer, _, _ := strings.Cut(caa.Value, ";") - if strings.EqualFold(strings.TrimSpace(issuer), p.caaIssuerDomain) { - return nil - } - } - } - - return fmt.Errorf( - "caa records for domain %q do not permit issuance by %q", - domain, - p.caaIssuerDomain, - ) -} - -func (p *Provisioner) checkPendingDomains(ctx context.Context) error { - err := p.pg.WithTx( - ctx, - func(ctx context.Context, tx pg.Tx) error { - if err := p.handleStaleProvisioningAttempts(ctx, tx); err != nil { - return fmt.Errorf("cannot handle stale provisioning attempts: %w", err) - } - - return nil - }, - ) - if err != nil { - return fmt.Errorf("cannot handle stale provisioning attempts: %w", err) - } - - err = p.pg.WithTx( - ctx, - func(ctx context.Context, tx pg.Tx) error { - var domains coredata.CustomDomains - if err := domains.ListDomainsWithPendingHTTPChallenges(ctx, tx, coredata.NewNoScope()); err != nil { - return fmt.Errorf("cannot load domains with pending challenges: %w", err) - } - - if len(domains) == 0 { - return nil - } - - p.logger.InfoCtx(ctx, "found domains needing SSL provisioning", log.Int("count", len(domains))) - - for _, domain := range domains { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - - if err := p.provisionDomainCertificate(ctx, tx, domain.ID); err != nil { - p.logger.ErrorCtx( - ctx, - "cannot provision certificate", - log.String("domain", domain.Domain), - log.Error(err), - ) - } - } - - return nil - }, - ) - if err != nil { - return fmt.Errorf("cannot provision domains: %w", err) - } - - return nil -} - -func (p *Provisioner) handleStaleProvisioningAttempts(ctx context.Context, tx pg.Tx) error { - var domains coredata.CustomDomains - if err := domains.ListStaleProvisioningDomains(ctx, tx, coredata.NewNoScope()); err != nil { - return fmt.Errorf("cannot load stale provisioning domains: %w", err) - } - - if len(domains) == 0 { - return nil - } - - p.logger.InfoCtx(ctx, "found stale provisioning attempts to reset", log.Int("count", len(domains))) - - for _, domain := range domains { - if err := p.resetStaleDomain(ctx, tx, domain); err != nil { - p.logger.ErrorCtx( - ctx, - "cannot reset stale domain", - log.String("domain", domain.Domain), - log.Error(err), - ) - } - } - - return nil -} - -func (p *Provisioner) resetStaleDomain( - ctx context.Context, - tx pg.Tx, - domain *coredata.CustomDomain, -) error { - fullDomain := &coredata.CustomDomain{} - if err := fullDomain.LoadByIDForUpdateSkipLocked(ctx, tx, coredata.NewNoScope(), domain.ID); err != nil { - if errors.Is(err, coredata.ErrResourceNotFound) { - return nil - } - - return fmt.Errorf("cannot load stale domain for update: %w", err) - } - - staleDuration := time.Since(fullDomain.UpdatedAt) - - p.logger.InfoCtx( - ctx, - "resetting stale domain", - log.String("domain", fullDomain.Domain), - log.String("status", string(fullDomain.SSLStatus)), - log.Duration("stale_duration", staleDuration), - log.Int("retry_count", fullDomain.SSLRetryCount), - ) - - fullDomain.HTTPChallengeToken = nil - fullDomain.HTTPChallengeKeyAuth = nil - fullDomain.HTTPChallengeURL = nil - fullDomain.HTTPOrderURL = nil - fullDomain.ProvisioningError = nil - fullDomain.SSLStatus = coredata.CustomDomainSSLStatusPending - - if fullDomain.SSLLastAttemptAt != nil && time.Since(*fullDomain.SSLLastAttemptAt) > 24*time.Hour { - p.logger.InfoCtx( - ctx, - "resetting retry count due to old last attempt", - log.String("domain", fullDomain.Domain), - log.Time("last_attempt", *fullDomain.SSLLastAttemptAt), - ) - fullDomain.SSLRetryCount = 0 - fullDomain.SSLLastAttemptAt = nil - } - - if err := fullDomain.Update(ctx, tx, coredata.NewNoScope()); err != nil { - return fmt.Errorf("cannot update stale domain: %w", err) - } - - return nil -} - -func (p *Provisioner) provisionDomainCertificate( - ctx context.Context, - tx pg.Tx, - domainID gid.GID, -) error { - domain := &coredata.CustomDomain{} - if err := domain.LoadByIDForUpdateSkipLocked(ctx, tx, coredata.NewNoScope(), domainID); err != nil { - if errors.Is(err, coredata.ErrResourceNotFound) { - return nil - } - - return fmt.Errorf("cannot load by id for update %q custom domain: %w", domainID, err) - } - - if domain.SSLStatus == coredata.CustomDomainSSLStatusPending || domain.SSLStatus == coredata.CustomDomainSSLStatusRenewing { - if err := p.checkDNSConfiguration(domain.Domain); err != nil { - p.logger.WarnCtx( - ctx, - "dns configuration check failed", - log.String("domain", domain.Domain), - log.Error(err), - ) - - errMsg := err.Error() - - domain.ProvisioningError = &errMsg - if err := domain.Update(ctx, tx, coredata.NewNoScope()); err != nil { - return fmt.Errorf("cannot update domain with provisioning error: %w", err) - } - - return nil - } - - if err := p.checkCAARecords(domain.Domain); err != nil { - p.logger.WarnCtx( - ctx, - "caa record check failed", - log.String("domain", domain.Domain), - log.Error(err), - ) - - errMsg := err.Error() - - domain.ProvisioningError = &errMsg - if err := domain.Update(ctx, tx, coredata.NewNoScope()); err != nil { - return fmt.Errorf("cannot update domain with provisioning error: %w", err) - } - - return nil - } - - domain.ProvisioningError = nil - if err := domain.Update(ctx, tx, coredata.NewNoScope()); err != nil { - return fmt.Errorf("cannot clear provisioning error: %w", err) - } - - p.logger.InfoCtx(ctx, "DNS configuration verified, initiating HTTP challenge for domain", log.String("domain", domain.Domain)) - - challenge, err := p.acmeService.GetHTTPChallenge(ctx, domain.Domain) - if err != nil { - p.logger.ErrorCtx( - ctx, - "cannot get HTTP challenge", - log.String("domain", domain.Domain), - log.Error(err), - ) - - return err - } - - domain.HTTPChallengeToken = &challenge.Token - domain.HTTPChallengeKeyAuth = &challenge.KeyAuth - domain.HTTPChallengeURL = &challenge.URL - domain.HTTPOrderURL = &challenge.OrderURL - domain.SSLStatus = coredata.CustomDomainSSLStatusProvisioning - - if err := domain.Update(ctx, tx, coredata.NewNoScope()); err != nil { - return fmt.Errorf("cannot update domain with challenge: %w", err) - } - - p.logger.InfoCtx( - ctx, - "HTTP challenge initiated, will complete in next cycle", - log.String("domain", domain.Domain), - log.String("token", challenge.Token), - ) - - return nil - } - - challenge := &HTTPChallenge{ - Domain: domain.Domain, - Token: *domain.HTTPChallengeToken, - KeyAuth: *domain.HTTPChallengeKeyAuth, - URL: *domain.HTTPChallengeURL, - OrderURL: *domain.HTTPOrderURL, - } - - cert, err := p.acmeService.CompleteHTTPChallenge(ctx, challenge) - if err != nil { - p.logger.WarnCtx( - ctx, - "cannot complete HTTP challenge", - log.String("domain", domain.Domain), - log.Int("retry_count", domain.SSLRetryCount), - log.Error(err), - ) - - errMsg := err.Error() - domain.ProvisioningError = &errMsg - domain.SSLRetryCount = domain.SSLRetryCount + 1 - domain.SSLLastAttemptAt = new(time.Now()) - - // Clear challenge data and reset to pending so the next attempt - // creates a fresh ACME order. Once a challenge fails validation, - // Let's Encrypt marks it as invalid and retrying the same - // challenge always fails with "authorization must be pending". - domain.HTTPChallengeToken = nil - domain.HTTPChallengeKeyAuth = nil - domain.HTTPChallengeURL = nil - domain.HTTPOrderURL = nil - - if domain.SSLRetryCount >= maxRetries { - p.logger.ErrorCtx( - ctx, - "domain has exceeded max retry attempts, marking as failed", - log.String("domain", domain.Domain), - log.Int("retry_count", domain.SSLRetryCount), - ) - - domain.SSLStatus = coredata.CustomDomainSSLStatusFailed - } else { - domain.SSLStatus = coredata.CustomDomainSSLStatusPending - } - - if err := domain.Update(ctx, tx, coredata.NewNoScope()); err != nil { - return fmt.Errorf("cannot update domain: %w", err) - } - - return nil - } - - p.logger.InfoCtx( - ctx, - "certificate obtained successfully", - log.String("domain", domain.Domain), - log.Time("expires_at", cert.ExpiresAt), - ) - - domain.ProvisioningError = nil - - domain.SSLCertificatePEM = cert.CertPEM - if err := domain.EncryptPrivateKey(cert.KeyPEM, p.encryptionKey); err != nil { - return fmt.Errorf("cannot encrypt private key: %w", err) - } - - chainStr := string(cert.ChainPEM) - domain.SSLCertificateChain = &chainStr - domain.SSLExpiresAt = &cert.ExpiresAt - domain.SSLStatus = coredata.CustomDomainSSLStatusActive - - domain.SSLRetryCount = 0 - domain.SSLLastAttemptAt = nil - - domain.HTTPChallengeToken = nil - domain.HTTPChallengeKeyAuth = nil - domain.HTTPChallengeURL = nil - domain.HTTPOrderURL = nil - - if err := domain.Update(ctx, tx, coredata.NewNoScope()); err != nil { - return fmt.Errorf("cannot update domain: %w", err) - } - - cache := &coredata.CachedCertificate{ - Domain: domain.Domain, - CertificatePEM: string(cert.CertPEM), - PrivateKeyPEM: string(cert.KeyPEM), - CertificateChain: &chainStr, - ExpiresAt: cert.ExpiresAt, - CachedAt: time.Now(), - CustomDomainID: domain.ID, - } - - if err := cache.Upsert(ctx, tx); err != nil { - p.logger.ErrorCtx( - ctx, - "cannot update certificate cache", - log.String("domain", domain.Domain), - log.Error(err), - ) - } - - return nil -} diff --git a/pkg/certmanager/renew_worker.go b/pkg/certmanager/renew_worker.go new file mode 100644 index 000000000..fe625af27 --- /dev/null +++ b/pkg/certmanager/renew_worker.go @@ -0,0 +1,127 @@ +// Copyright (c) 2025-2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package certmanager + +import ( + "context" + "errors" + "fmt" + + "go.gearno.de/kit/log" + "go.gearno.de/kit/pg" + "go.gearno.de/kit/worker" + "go.probo.inc/probo/pkg/coredata" + "go.probo.inc/probo/pkg/crypto/cipher" +) + +type renewHandler struct { + pg *pg.Client + encryptionKey cipher.EncryptionKey + logger *log.Logger +} + +var ( + _ worker.Handler[coredata.Certificate] = (*renewHandler)(nil) + _ worker.StaleRecoverer = (*renewHandler)(nil) +) + +func NewRenewWorker( + pgClient *pg.Client, + encryptionKey cipher.EncryptionKey, + logger *log.Logger, + opts ...worker.Option, +) *worker.Worker[coredata.Certificate] { + h := &renewHandler{ + pg: pgClient, + encryptionKey: encryptionKey, + logger: logger, + } + + return worker.New( + "certificate-renew-worker", + h, + logger, + opts..., + ) +} + +func (h *renewHandler) Claim(ctx context.Context) (coredata.Certificate, error) { + var certificate coredata.Certificate + + if err := h.pg.WithTx( + ctx, + func(ctx context.Context, tx pg.Tx) error { + if err := certificate.LoadNextForRenewalForUpdateSkipLocked(ctx, tx); err != nil { + return err + } + + certificate.Status = coredata.CertificateStatusRenewing + if err := certificate.Update(ctx, tx, coredata.NewNoScope()); err != nil { + return fmt.Errorf("cannot update certificate status: %w", err) + } + + h.logger.InfoCtx( + ctx, + "queued certificate for renewal", + log.String("hostname", certificate.Hostname), + ) + + return nil + }, + ); err != nil { + if errors.Is(err, coredata.ErrResourceNotFound) { + return coredata.Certificate{}, worker.ErrNoTask + } + + return coredata.Certificate{}, err + } + + return certificate, nil +} + +func (h *renewHandler) Process(_ context.Context, _ coredata.Certificate) error { + return nil +} + +func (h *renewHandler) RecoverStale(ctx context.Context) error { + return h.pg.WithConn( + ctx, + func(ctx context.Context, conn pg.Querier) error { + var caches coredata.CachedCertificates + + cacheCount, err := caches.CountAll(ctx, conn) + if err != nil { + return fmt.Errorf("cannot count certificate cache: %w", err) + } + + if cacheCount == 0 { + h.logger.InfoCtx(ctx, "certificate cache is empty, rebuilding from certificates") + + warmer := NewCacheStore(h.pg, h.encryptionKey, h.logger) + if err := warmer.WarmCache(ctx); err != nil { + return fmt.Errorf("cannot rebuild certificate cache: %w", err) + } + + h.logger.InfoCtx(ctx, "certificate cache rebuilt successfully") + } + + if err := caches.CleanExpired(ctx, conn); err != nil { + return fmt.Errorf("cannot clean certificate cache: %w", err) + } + + return nil + }, + ) +} diff --git a/pkg/certmanager/renewer.go b/pkg/certmanager/renewer.go deleted file mode 100644 index d1ddeba05..000000000 --- a/pkg/certmanager/renewer.go +++ /dev/null @@ -1,166 +0,0 @@ -// Copyright (c) 2025-2026 Probo Inc . -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -package certmanager - -import ( - "context" - "errors" - "fmt" - "time" - - "go.gearno.de/kit/log" - "go.gearno.de/kit/pg" - "go.probo.inc/probo/pkg/coredata" - "go.probo.inc/probo/pkg/crypto/cipher" - "go.probo.inc/probo/pkg/gid" -) - -type ( - Renewer struct { - pg *pg.Client - acmeService *ACMEService - encryptionKey cipher.EncryptionKey - interval time.Duration - logger *log.Logger - } -) - -func NewRenewer( - pg *pg.Client, - acmeService *ACMEService, - encryptionKey cipher.EncryptionKey, - interval time.Duration, - logger *log.Logger, -) *Renewer { - return &Renewer{ - pg: pg, - acmeService: acmeService, - encryptionKey: encryptionKey, - interval: interval, - logger: logger.Named("certmanager.renewer"), - } -} - -func (r *Renewer) Run(ctx context.Context) error { - r.logger.InfoCtx(ctx, "certificate renewer starting") - - if err := r.checkAndRenew(ctx); err != nil { - r.logger.ErrorCtx(ctx, "cannot perform initial renewal check", log.Error(err)) - } - - for { - select { - case <-ctx.Done(): - r.logger.InfoCtx(ctx, "certificate renewer shutting down") - return ctx.Err() - case <-time.After(r.interval): - if err := r.checkAndRenew(ctx); err != nil { - r.logger.ErrorCtx(ctx, "cannot perform renewal check", log.Error(err)) - } - } - } -} - -func (r *Renewer) checkAndRenew(ctx context.Context) error { - return r.pg.WithTx( - ctx, - func(ctx context.Context, tx pg.Tx) error { - var caches coredata.CachedCertificates - - cacheCount, err := caches.CountAll(ctx, tx) - if err != nil { - r.logger.ErrorCtx(ctx, "cannot count certificate cache", log.Error(err)) - } else if cacheCount == 0 { - r.logger.InfoCtx(ctx, "certificate cache is empty, rebuilding from custom_domains") - - warmer := NewCacheStore(r.pg, r.encryptionKey, r.logger) - if err := warmer.WarmCache(ctx); err != nil { - r.logger.ErrorCtx(ctx, "cannot rebuild certificate cache", log.Error(err)) - } else { - r.logger.InfoCtx(ctx, "certificate cache rebuilt successfully") - } - } - - if err := caches.CleanExpired(ctx, tx); err != nil { - r.logger.ErrorCtx(ctx, "cannot clean certificate cache", log.Error(err)) - } - - domains := coredata.CustomDomains{} - - scope := coredata.NewNoScope() - if err := domains.ListDomainsForRenewal(ctx, tx, scope); err != nil { - return fmt.Errorf("cannot list domains for renewal: %w", err) - } - - if len(domains) == 0 { - return nil - } - - r.logger.InfoCtx(ctx, "found domains needing renewal", log.Int("count", len(domains))) - - for _, domain := range domains { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - - r.logger.InfoCtx(ctx, "renewing certificate for domain", log.String("domain", domain.Domain)) - - if err := r.renewDomain(ctx, tx, domain.ID); err != nil { - r.logger.ErrorCtx(ctx, "cannot renew certificate", log.String("domain", domain.Domain), log.Error(err)) - } else { - r.logger.InfoCtx(ctx, "successfully renewed certificate", log.String("domain", domain.Domain)) - } - } - - return nil - }, - ) -} - -func (r *Renewer) renewDomain(ctx context.Context, tx pg.Tx, domainID gid.GID) error { - domain := &coredata.CustomDomain{} - if err := domain.LoadByIDForUpdateSkipLocked(ctx, tx, coredata.NewNoScope(), domainID); err != nil { - if errors.Is(err, coredata.ErrResourceNotFound) { - return nil - } - - return fmt.Errorf("cannot lock domain for renewal: %w", err) - } - - if domain.SSLStatus != coredata.CustomDomainSSLStatusActive { - r.logger.InfoCtx( - ctx, - "domain status changed, skipping renewal", - log.String("domain", domain.Domain), - ) - - return nil - } - - domain.SSLStatus = coredata.CustomDomainSSLStatusRenewing - if err := domain.Update(ctx, tx, coredata.NewNoScope()); err != nil { - return fmt.Errorf("cannot update domain status: %w", err) - } - - return nil -} diff --git a/pkg/certmanager/selector.go b/pkg/certmanager/selector.go index 3f2b4491c..0deaf15df 100644 --- a/pkg/certmanager/selector.go +++ b/pkg/certmanager/selector.go @@ -123,42 +123,42 @@ func (s *Selector) loadFromDatabase(domain string) (*tls.Certificate, error) { } func (s *Selector) rebuildCacheEntry(ctx context.Context, conn pg.Querier, domain string) error { - var customDomain coredata.CustomDomain - if err := customDomain.LoadByDomain(ctx, conn, coredata.NewNoScope(), domain); err != nil { - return fmt.Errorf("cannot load domain: %w", err) + var certificate coredata.Certificate + if err := certificate.LoadByHostname(ctx, conn, coredata.NewNoScope(), domain); err != nil { + return fmt.Errorf("cannot load certificate: %w", err) } - if customDomain.SSLStatus != coredata.CustomDomainSSLStatusActive { - return fmt.Errorf("domain does not have active SSL certificate") + if certificate.Status != coredata.CertificateStatusActive { + return fmt.Errorf("hostname does not have active SSL certificate") } - if err := customDomain.ParseCertificate(s.encryptionKey); err != nil { + if err := certificate.ParseCertificate(s.encryptionKey); err != nil { return fmt.Errorf("cannot parse certificate: %w", err) } - if len(customDomain.SSLCertificatePEM) == 0 { - return fmt.Errorf("domain has no certificate PEM data") + if len(certificate.SSLCertificatePEM) == 0 { + return fmt.Errorf("certificate has no certificate PEM data") } - if len(customDomain.EncryptedSSLPrivateKey) == 0 { - return fmt.Errorf("domain has no encrypted private key data") + if len(certificate.EncryptedSSLPrivateKey) == 0 { + return fmt.Errorf("certificate has no encrypted private key data") } - privateKeyPEM, err := customDomain.DecryptPrivateKey(s.encryptionKey) + privateKeyPEM, err := certificate.DecryptPrivateKey(s.encryptionKey) if err != nil { return fmt.Errorf("cannot decrypt private key: %w", err) } - s.cache.Store(domain, customDomain.SSLCertificate) + s.cache.Store(domain, certificate.SSLCertificate) cache := &coredata.CachedCertificate{ - Domain: customDomain.Domain, - CertificatePEM: string(customDomain.SSLCertificatePEM), + Domain: certificate.Hostname, + CertificatePEM: string(certificate.SSLCertificatePEM), PrivateKeyPEM: string(privateKeyPEM), - CertificateChain: customDomain.SSLCertificateChain, - ExpiresAt: *customDomain.SSLExpiresAt, + CertificateChain: certificate.SSLCertificateChain, + ExpiresAt: *certificate.SSLExpiresAt, CachedAt: time.Now(), - CustomDomainID: customDomain.ID, + CertificateID: certificate.ID, } if err := cache.Upsert(ctx, conn); err != nil { diff --git a/pkg/certmanager/service.go b/pkg/certmanager/service.go new file mode 100644 index 000000000..15a2eeeef --- /dev/null +++ b/pkg/certmanager/service.go @@ -0,0 +1,239 @@ +// Copyright (c) 2025-2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package certmanager + +import ( + "context" + "errors" + "fmt" + "time" + + "go.gearno.de/kit/log" + "go.gearno.de/kit/pg" + "go.gearno.de/kit/worker" + "go.probo.inc/probo/pkg/coredata" + "go.probo.inc/probo/pkg/crypto/cipher" + "go.probo.inc/probo/pkg/gid" + "golang.org/x/sync/errgroup" +) + +type ( + // Service owns the TLS certificate lifecycle for arbitrary hostnames. It is + // a generic core service and knows nothing about the resources a + // certificate protects; callers reference a certificate by its ID. + Service struct { + pg *pg.Client + acmeService *ACMEService + encryptionKey cipher.EncryptionKey + logger *log.Logger + provisionWorker *worker.Worker[coredata.Certificate] + renewWorker *worker.Worker[coredata.Certificate] + } + + // Config holds the SSL provisioning parameters for the service workers. + Config struct { + CnameTarget string + CAAIssuerDomain string + ResolverAddr string + ManagedBaseDomain string + RenewalInterval time.Duration + ProvisionInterval time.Duration + } +) + +func NewService( + pgClient *pg.Client, + acmeService *ACMEService, + encryptionKey cipher.EncryptionKey, + cfg Config, + logger *log.Logger, +) *Service { + provisionInterval := cfg.ProvisionInterval + if provisionInterval <= 0 { + provisionInterval = 30 * time.Second + } + + renewalInterval := cfg.RenewalInterval + if renewalInterval <= 0 { + renewalInterval = time.Hour + } + + return &Service{ + pg: pgClient, + acmeService: acmeService, + encryptionKey: encryptionKey, + logger: logger, + provisionWorker: NewProvisionWorker( + pgClient, + acmeService, + encryptionKey, + cfg.CnameTarget, + cfg.CAAIssuerDomain, + cfg.ResolverAddr, + cfg.ManagedBaseDomain, + logger.Named("provision-worker"), + worker.WithInterval(provisionInterval), + ), + renewWorker: NewRenewWorker( + pgClient, + encryptionKey, + logger.Named("renew-worker"), + worker.WithInterval(renewalInterval), + ), + } +} + +func (s *Service) Run(ctx context.Context) error { + g, gctx := errgroup.WithContext(ctx) + + g.Go( + func() error { + return s.provisionWorker.Run(gctx) + }, + ) + + g.Go( + func() error { + return s.renewWorker.Run(gctx) + }, + ) + + return g.Wait() +} + +// EnsureCertificate returns the certificate for the given hostname, creating a +// pending one within the given transaction when it does not exist yet. The +// certificate lifecycle is then driven asynchronously by the provision worker. +func (s *Service) EnsureCertificate( + ctx context.Context, + tx pg.Tx, + scope coredata.Scoper, + hostname string, +) (*coredata.Certificate, error) { + certificate := &coredata.Certificate{} + + err := certificate.LoadByHostname(ctx, tx, scope, hostname) + if err == nil { + return certificate, nil + } + + if !errors.Is(err, coredata.ErrResourceNotFound) { + return nil, fmt.Errorf("cannot load certificate: %w", err) + } + + certificate = coredata.NewCertificate(scope.GetTenantID(), hostname) + if err := certificate.Insert(ctx, tx, scope); err != nil { + return nil, fmt.Errorf("cannot insert certificate: %w", err) + } + + return certificate, nil +} + +// UpdateHostname renames a certificate and resets its lifecycle so a fresh +// certificate is provisioned for the new hostname. It returns the certificate +// unchanged when the hostname already matches. +func (s *Service) UpdateHostname( + ctx context.Context, + tx pg.Tx, + scope coredata.Scoper, + certificateID gid.GID, + newHostname string, +) (*coredata.Certificate, error) { + certificate := &coredata.Certificate{} + if err := certificate.LoadByID(ctx, tx, scope, certificateID); err != nil { + return nil, fmt.Errorf("cannot load certificate: %w", err) + } + + if certificate.Hostname == newHostname { + return certificate, nil + } + + certificate.Hostname = newHostname + certificate.Status = coredata.CertificateStatusPending + certificate.SSLRetryCount = 0 + certificate.SSLLastAttemptAt = nil + certificate.ProvisioningError = nil + certificate.HTTPChallengeToken = nil + certificate.HTTPChallengeKeyAuth = nil + certificate.HTTPChallengeURL = nil + certificate.HTTPOrderURL = nil + + if err := certificate.Update(ctx, tx, scope); err != nil { + return nil, fmt.Errorf("cannot update certificate: %w", err) + } + + return certificate, nil +} + +// Get returns a certificate by ID. +func (s *Service) Get( + ctx context.Context, + scope coredata.Scoper, + certificateID gid.GID, +) (*coredata.Certificate, error) { + certificate := &coredata.Certificate{} + + err := s.pg.WithConn( + ctx, + func(ctx context.Context, conn pg.Querier) error { + return certificate.LoadByID(ctx, conn, scope, certificateID) + }, + ) + if err != nil { + return nil, err + } + + return certificate, nil +} + +// GetByHostname returns the certificate matching the given hostname. It is +// unscoped because it powers public host resolution across all tenants. +func (s *Service) GetByHostname( + ctx context.Context, + hostname string, +) (*coredata.Certificate, error) { + certificate := &coredata.Certificate{} + + err := s.pg.WithConn( + ctx, + func(ctx context.Context, conn pg.Querier) error { + return certificate.LoadByHostname(ctx, conn, coredata.NewNoScope(), hostname) + }, + ) + if err != nil { + return nil, err + } + + return certificate, nil +} + +// Delete removes a certificate within the given transaction. +func (s *Service) Delete( + ctx context.Context, + tx pg.Tx, + scope coredata.Scoper, + certificateID gid.GID, +) error { + certificate := &coredata.Certificate{} + if err := certificate.LoadByID(ctx, tx, scope, certificateID); err != nil { + return fmt.Errorf("cannot load certificate: %w", err) + } + + if err := certificate.Delete(ctx, tx, scope); err != nil { + return fmt.Errorf("cannot delete certificate: %w", err) + } + + return nil +}