Replace Pebble with step-ca for local ACME

Pebble skipped real HTTP-01 validation, which hid integration gaps for compliance-page custom domains. step-ca with a Caddy port-80 proxy exercises the same path production uses while keeping issued certs persistent across compose restarts. Signed-off-by: Bryan Frimin <bryan@probo.com>
2026-07-15 12:14:45 +02:00
parent e83f9e3a2e
commit 14604a3176
8 changed files with 79 additions and 51 deletions
--- a/.env.example
+++ b/.env.example
@@ -20,7 +20,8 @@
 # PROBOD_API_ADDR=:8080
 # PROBOD_API_CORS_ALLOWED_ORIGINS=http://localhost:8080,http://localhost:5173,http://localhost:5174,http://localhost:5175
 # PROBOD_TRUST_CENTER_HTTP_ADDR=:10080
-# PROBOD_TRUST_CENTER_HTTPS_ADDR=:10443
+# PROBOD_TRUST_CENTER_HTTPS_ADDR=:443
+# PROBOD_TRUST_CENTER_BASE_DOMAIN=probopage.localhost

 # ── Observability ─────────────────────────────────────────────────────
 # PROBOD_METRICS_ADDR=localhost:8081
@@ -154,7 +155,9 @@
 # OAuth clients (e.g. MCP connectors). Leave unset to disable CIMD.
 # PROBOD_OAUTH2_SERVER_CIMD_ALLOWED_CLIENT_IDS=https://chatgpt.com/oauth/client.json,https://claude.ai/oauth/client.json

-# ── Custom domains (Pebble ACME via compose) ──────────────────────────
+# ── Custom domains (step-ca ACME via compose) ─────────────────────────
 # PROBOD_CUSTOM_DOMAINS_CNAME_TARGET=custom.getprobo.com
-# PROBOD_ACME_DIRECTORY=https://localhost:14000/dir
+# PROBOD_ACME_DIRECTORY=https://localhost:9000/acme/acme/directory
 # PROBOD_ACME_EMAIL=admin@probo.com
+# Install the local root once for browser/CIMD trust:
+#   step certificate install compose/step-ca/certs/root_ca.crt
--- a/27
+++ b/27
@@ -199,7 +199,7 @@ $(CFG_DEV_OAUTH2_KEY):
 	@$(MKDIR) $(@D)
 	$(OPENSSL) genrsa -out $@ 2048

-cfg/dev.yaml: bin/probod-bootstrap $(CFG_DEV_OAUTH2_KEY) compose/pebble/certs/rootCA.pem $(wildcard $(DEV_ENV))
+cfg/dev.yaml: bin/probod-bootstrap $(CFG_DEV_OAUTH2_KEY) compose/step-ca/certs/root_ca.crt $(wildcard $(DEV_ENV))
 	@$(MKDIR) $(@D)
 	set -a; \
 	PROBOD_BASE_URL=http://localhost:8080; \
@@ -217,7 +217,8 @@ cfg/dev.yaml: bin/probod-bootstrap $(CFG_DEV_OAUTH2_KEY) compose/pebble/certs/ro
 	PROBOD_PG_PASSWORD=postgres; \
 	PROBOD_PG_DATABASE=probod; \
 	PROBOD_TRUST_CENTER_HTTP_ADDR=:10080; \
-	PROBOD_TRUST_CENTER_HTTPS_ADDR=:10443; \
+	PROBOD_TRUST_CENTER_HTTPS_ADDR=:443; \
+	PROBOD_TRUST_CENTER_BASE_DOMAIN=probopage.localhost; \
 	PROBOD_AWS_REGION=us-east-1; \
 	PROBOD_AWS_BUCKET=probod; \
 	PROBOD_AWS_ACCESS_KEY_ID=probod; \
@@ -230,9 +231,9 @@ cfg/dev.yaml: bin/probod-bootstrap $(CFG_DEV_OAUTH2_KEY) compose/pebble/certs/ro
 	PROBOD_AGENT_THIRD_PARTY_VETTER_PROVIDER=openai; \
 	PROBOD_AGENT_THIRD_PARTY_VETTER_MODEL_NAME=gpt-4o; \
 	PROBOD_CHROME_DP_ADDR=localhost:9222; \
-	PROBOD_ACME_DIRECTORY=https://localhost:14000/dir; \
+	PROBOD_ACME_DIRECTORY=https://localhost:9000/acme/acme/directory; \
 	PROBOD_ACME_EMAIL=admin@probo.com; \
-	PROBOD_ACME_ROOT_CA="$$($(CAT) compose/pebble/certs/rootCA.pem)"; \
+	PROBOD_ACME_ROOT_CA="$$($(CAT) compose/step-ca/certs/root_ca.crt)"; \
 	if [ -f $(DEV_ENV) ]; then . $(DEV_ENV); fi; \
 	set +a; \
 	./bin/probod-bootstrap -output $@
@@ -407,7 +408,7 @@ clean: ## Clean the project (node_modules and build artifacts)
 	find apps -type d -name __generated__ -exec $(RM) -rf {} +

 .PHONY: stack-up
-stack-up: compose/pebble/certs/rootCA.pem compose/keycloak/probo-realm.json ## Start the docker stack as a deamon
+stack-up: compose/step-ca/certs/root_ca.crt compose/keycloak/probo-realm.json ## Start the docker stack as a deamon
 	$(DOCKER_COMPOSE) up -d

 .PHONY: stack-down
@@ -422,13 +423,15 @@ stack-ps: ## List the docker stack containers
 psql: ## Open a psql shell to the postgres container
 	$(DOCKER_COMPOSE) exec postgres psql -U probod -d probod

-compose/pebble/certs/rootCA.pem:
-	@$(MKDIR) compose/pebble/certs
-	$(MKCERT) -cert-file compose/pebble/certs/pebble.crt \
-		-key-file compose/pebble/certs/pebble.key \
-		localhost 127.0.0.1 ::1 pebble
-	$(CP) "$$($(MKCERT) -CAROOT)/rootCA.pem" compose/pebble/certs/rootCA.pem
-	$(CP) "$$($(MKCERT) -CAROOT)/rootCA-key.pem" compose/pebble/certs/rootCA-key.pem
+compose/step-ca/certs/root_ca.crt:
+	@$(MKDIR) compose/step-ca/secrets
+	$(DOCKER_COMPOSE) up -d acme-http-01-proxy step-ca
+	@i=0; \
+	while [ ! -f $@ ] && [ $$i -lt 60 ]; do \
+		sleep 1; \
+		i=$$((i + 1)); \
+	done
+	@test -f $@ || (echo "step-ca root CA not ready; check: docker compose logs step-ca" >&2; exit 1)

 compose/keycloak/certs/cert.pem:
 	$(MKDIR) ./compose/keycloak/certs
--- a/compose.yaml
+++ b/compose.yaml
@@ -122,28 +122,34 @@ services:
      - "--disable-background-timer-throttling"
      - "--disable-extensions"

-  pebble:
-    image: "ghcr.io/letsencrypt/pebble:2.10.0@sha256:68cf1ec8a8db96f64244d5f559c448bc8e54f2934e0dd53a414eabffda7a6f22"
+  # step-ca validates HTTP-01 on port 80; probod serves challenges on :10080.
+  # step-ca shares this service's network so loopback :80 hits Caddy, not an
+  # isolated container namespace (where *.localhost would miss the proxy).
+  acme-http-01-proxy:
+    image: "caddy:2.10.2@sha256:c3d7ee5d2b11f9dc54f947f68a734c84e9c9666c92c88a7f30b9cba5da182adb"
    pull_policy: missing
    ports:
-      - "14000:14000" # ACME server
-      - "15000:15000" # Management interface
-    environment:
-      PEBBLE_VA_NOSLEEP: "1" # Don't sleep during validation for faster testing
-      PEBBLE_WFE_NONCEREJECT: "0" # Don't reject reused nonces
-      PEBBLE_VA_ALWAYS_VALID: "1" # Skip actual HTTP/DNS validation for local dev
-    command: -config /test/config/pebble-config.json -dnsserver 127.0.0.1:8053
+      - "80:80"
+      - "9000:9000" # step-ca ACME API (network_mode: service:… below)
    volumes:
-      - "./compose/pebble:/test/config:ro"
+      - "./compose/caddy/Caddyfile:/etc/caddy/Caddyfile:ro"
+    extra_hosts:
+      - "host.docker.internal:host-gateway"

-  pebble-challtestsrv:
-    image: "ghcr.io/letsencrypt/pebble-challtestsrv:2.10.0@sha256:df85447c39114cd2b3da971e751682ad2db19a996034b47d5392cd4a0f43406a"
+  step-ca:
+    image: "smallstep/step-ca:0.28.4@sha256:0f88382ac5af5c6b7bbba0c6e8fcefef52aee6f22ea364df8e02a09ffd0d22f3"
    pull_policy: missing
-    ports:
-      - "8055:8055" # HTTP-01 challenge test server
-      - "8053:8053" # DNS server
-      - "8056:8056" # Management API
-    command: -dnsserver ":8053" -http01 ":8055" -management ":8056"
+    network_mode: "service:acme-http-01-proxy"
+    depends_on:
+      - acme-http-01-proxy
+    environment:
+      DOCKER_STEPCA_INIT_NAME: "Probo Local CA"
+      DOCKER_STEPCA_INIT_DNS_NAMES: "localhost,127.0.0.1,step-ca"
+      DOCKER_STEPCA_INIT_REMOTE_MANAGEMENT: "true"
+      DOCKER_STEPCA_INIT_ACME: "true"
+      DOCKER_STEPCA_INIT_PASSWORD: "probod-local-dev-ca"
+    volumes:
+      - "./compose/step-ca:/home/step:rw"

  keycloak:
    image: "quay.io/keycloak/keycloak:26.6.1@sha256:26ae26445475f7fac5f90ee138b1bdb64324f5815fb16133ffdbdb122d97c4d8"
--- a/compose/caddy/Caddyfile
+++ b/compose/caddy/Caddyfile
@@ -0,0 +1,5 @@
+# Proxies ACME HTTP-01 validation (port 80) to probod's trust-center HTTP
+# listener on the host (:10080 by default).
+:80 {
+	reverse_proxy host.docker.internal:10080
+}
--- a/compose/pebble/pebble-config.json
+++ b/compose/pebble/pebble-config.json
@@ -1,14 +0,0 @@
-{
-  "pebble": {
-    "listenAddress": "0.0.0.0:14000",
-    "managementListenAddress": "0.0.0.0:15000",
-    "certificate": "/test/config/certs/pebble.crt",
-    "privateKey": "/test/config/certs/pebble.key",
-    "httpPort": 5002,
-    "tlsPort": 5001,
-    "ocspResponderURL": "",
-    "externalAccountBindingRequired": false,
-    "domainBlocklist": [],
-    "certificateValidityPeriod": 157766400
-  }
-}
--- a/compose/step-ca/README.md
+++ b/compose/step-ca/README.md
@@ -0,0 +1,26 @@
+# step-ca local CA
+
+This directory holds the persistent [step-ca](https://github.com/smallstep/certificates)
+state for local custom-domain TLS. It is initialized on first `make stack-up`.
+
+After the first run, install the root CA once so browsers and server-side TLS
+clients (e.g. CIMD OAuth) trust issued certificates across restarts:
+
+```bash
+step certificate install compose/step-ca/certs/root_ca.crt
+```
+
+The ACME directory URL is `https://localhost:9000/acme/acme/directory`.
+
+step-ca shares the `acme-http-01-proxy` container network so HTTP-01 validation
+to `http://<hostname>/.well-known/acme-challenge/...` reaches Caddy on port 80,
+which forwards to probod's trust-center HTTP listener on the host.
+
+## Custom domain DNS (optional)
+
+Managed compliance-page domains (`*.probopage.localhost`) resolve via the
+`.localhost` TLD and skip DNS checks.
+
+For customer custom domains in local dev, point DNS at the host via
+`/etc/hosts` and ensure HTTP-01 reaches probod through the
+`acme-http-01-proxy` service on port 80.
--- a/contrib/lima/provision.sh
+++ b/contrib/lima/provision.sh
@@ -96,7 +96,7 @@ VM_IP=$(ip -4 -j addr show dev lima0 | jq -r '.[0].addr_info[0].local')

 su - "${LIMA_USER}" -c "export PATH=/usr/local/go/bin:\$HOME/go/bin:\$PATH && cd /workspace && make bin/probod-bootstrap"

-make -C /workspace compose/pebble/certs/rootCA.pem compose/keycloak/probo-realm.json
+make -C /workspace compose/step-ca/certs/root_ca.crt compose/keycloak/probo-realm.json

 mkdir -p /etc/probod

@@ -125,10 +125,10 @@ PROBOD_AWS_ENDPOINT="http://127.0.0.1:8333" \
 PROBOD_AWS_ACCESS_KEY_ID="probod" \
 PROBOD_AWS_SECRET_ACCESS_KEY="thisisnotasecret" \
 PROBOD_AWS_USE_PATH_STYLE=true \
-PROBOD_ACME_DIRECTORY="https://127.0.0.1:14000/dir" \
+PROBOD_ACME_DIRECTORY="https://127.0.0.1:9000/acme/acme/directory" \
 PROBOD_ACME_EMAIL="admin@probo.com" \
 PROBOD_ACME_KEY_TYPE="EC256" \
-PROBOD_ACME_ROOT_CA="$(cat /workspace/compose/pebble/certs/rootCA.pem)" \
+PROBOD_ACME_ROOT_CA="$(cat /workspace/compose/step-ca/certs/root_ca.crt)" \
    /workspace/bin/probod-bootstrap -output /etc/probod/config.yml

 # probod runs as ${LIMA_USER} but bootstrap writes config.yml as root with 0600
--- a/pkg/certmanager/acme.go
+++ b/pkg/certmanager/acme.go
@@ -274,8 +274,7 @@ func (s *ACMEService) issueOrderCertificate(
 		// CreateOrderCert finalizes the order but may fail to download the
 		// certificate when the CA marks the order valid before the certificate
 		// URL is populated. Poll the order using the known order URL because
-		// some CAs (including Pebble) omit the Location header on poll
-		// responses, leaving order.URI empty.
+		// some CAs omit the Location header on poll responses, leaving order.URI empty.
 		return s.fetchOrderCertificateAfterFinalize(ctx, pollURL, err)
 	default:
 		return nil, fmt.Errorf("order is in unexpected status %q", order.Status)