Serialize enrollment install with enrolling.lock

Concurrent enroll-url launches could both pass the enrollment
marker check and run overlapping elevated installs, racing on
LoadOrExchangeAPIKey and overwriting agent.key.

Add an exclusive flock on {configDir}/enrolling.lock for the
full install path and re-check IsEnrolled under that lock so
only one install exchanges a token and configures the device.

Signed-off-by: Ludovic Vielle <ludovic@probo.com>

pkg/deviceagent/config_paths_other.go

@@ -29,7 +29,7 @@ func DefaultConfigDir() string {
 }
 
 // DefaultEnrollmentRunDir returns the runtime directory for the public
-// enrollment marker on non-Windows hosts.
+// enrollment marker and enrolling.lock on non-Windows hosts.
 func DefaultEnrollmentRunDir() string {
 	return "/var/run/probo-agent"
 }

pkg/deviceagent/config_paths_windows.go

@@ -37,7 +37,7 @@ func DefaultConfigDir() string {
 }
 
 // DefaultEnrollmentRunDir returns the runtime directory for the public
-// enrollment marker on Windows.
+// enrollment marker and enrolling.lock on Windows.
 func DefaultEnrollmentRunDir() string {
 	programData := os.Getenv("ProgramData")
 	if programData == "" {

pkg/deviceagent/enroll.go

@@ -36,6 +36,9 @@ var ErrServerURLMismatch = errors.New("server URL does not match persisted confi
 // only when serverURL matches the server_url persisted in config.json.
 // Otherwise the enrollment token is exchanged, the key and server binding
 // are saved to disk immediately, and then the key is returned.
+//
+// Concurrent callers are serialized with an exclusive lock so only one
+// process exchanges a one-shot enrollment token for a given state dir.
 func LoadOrExchangeAPIKey(
 	ctx context.Context,
 	dir string,
@@ -48,6 +51,12 @@ func LoadOrExchangeAPIKey(
 		return "", fmt.Errorf("invalid server URL: %w", err)
 	}
 
+	release, err := AcquireEnrollmentLock(dir)
+	if err != nil {
+		return "", fmt.Errorf("cannot acquire enrollment lock: %w", err)
+	}
+	defer release()
+
 	apiKey, err := LoadAPIKey(dir)
 	if err == nil {
 		if err := validatePersistedServerURL(dir, normalized); err != nil {

pkg/deviceagent/enrollment_lock.go

@@ -0,0 +1,84 @@
+// Copyright (c) 2026 Probo Inc <hello@probo.com>.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+package deviceagent
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+const (
+	// EnrollmentLockFileName serializes concurrent LoadOrExchangeAPIKey
+	// calls that exchange enrollment tokens and write agent.key. It lives
+	// under EnrollmentRunDir so a reboot clears it with the run tree.
+	EnrollmentLockFileName = "enrolling.lock"
+
+	// enrollmentLockMode is owner-only; only the elevated install path
+	// creates and holds this lock.
+	enrollmentLockMode = 0o600
+)
+
+// EnrollmentLockPath returns the absolute path of the enrollment lock file
+// under EnrollmentRunDir for the given state directory.
+func EnrollmentLockPath(configDir string) string {
+	return filepath.Join(EnrollmentRunDir(configDir), EnrollmentLockFileName)
+}
+
+// AcquireEnrollmentLock takes an exclusive lock on enrolling.lock under
+// EnrollmentRunDir(configDir) for the duration of credential exchange. The
+// returned release function unlocks and closes the lock file. The path is
+// left in place so waiters that already opened the inode stay serialized
+// with later acquirers; the kernel releases the lock on close (including
+// after crashes). Callers must not unlink this path while a holder may
+// exist.
+func AcquireEnrollmentLock(configDir string) (release func(), err error) {
+	runDir := EnrollmentRunDir(configDir)
+
+	if err := os.MkdirAll(runDir, EnrollmentRunDirMode); err != nil {
+		return nil, fmt.Errorf("cannot create enrollment run dir: %w", err)
+	}
+
+	if err := os.Chmod(runDir, EnrollmentRunDirMode); err != nil {
+		return nil, fmt.Errorf("cannot set enrollment run dir permissions: %w", err)
+	}
+
+	path := EnrollmentLockPath(configDir)
+
+	file, err := os.OpenFile(
+		path,
+		os.O_RDWR|os.O_CREATE,
+		enrollmentLockMode,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("cannot open enrollment lock: %w", err)
+	}
+
+	if err := lockFileExclusive(file); err != nil {
+		_ = file.Close()
+		return nil, fmt.Errorf("cannot acquire enrollment lock: %w", err)
+	}
+
+	return func() {
+		_ = unlockFile(file)
+		_ = file.Close()
+	}, nil
+}

pkg/deviceagent/enrollment_lock_test.go

@@ -0,0 +1,86 @@
+// Copyright (c) 2026 Probo Inc <hello@probo.com>.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+package deviceagent
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestEnrollmentLockPath(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	assert.Equal(
+		t,
+		filepath.Join(EnrollmentRunDir(dir), EnrollmentLockFileName),
+		EnrollmentLockPath(dir),
+	)
+}
+
+func TestAcquireEnrollmentLockConcurrent(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+
+	releaseFirst, err := AcquireEnrollmentLock(dir)
+	require.NoError(t, err)
+
+	acquired := make(chan error, 1)
+
+	go func() {
+		releaseSecond, err := AcquireEnrollmentLock(dir)
+		if err != nil {
+			acquired <- err
+			return
+		}
+
+		releaseSecond()
+
+		acquired <- nil
+	}()
+
+	time.Sleep(100 * time.Millisecond)
+
+	select {
+	case err := <-acquired:
+		require.NoError(t, err)
+		t.Fatal("second install acquired enrollment lock while first still holds it")
+	default:
+	}
+
+	releaseFirst()
+
+	select {
+	case err := <-acquired:
+		require.NoError(t, err)
+	case <-time.After(2 * time.Second):
+		t.Fatal("second install did not acquire enrollment lock after first released it")
+	}
+
+	_, err = os.Stat(EnrollmentLockPath(dir))
+	require.NoError(t, err)
+}

pkg/deviceagent/enrollment_lock_unix.go

@@ -0,0 +1,37 @@
+// Copyright (c) 2026 Probo Inc <hello@probo.com>.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+//go:build !windows
+
+package deviceagent
+
+import (
+	"os"
+
+	"golang.org/x/sys/unix"
+)
+
+func lockFileExclusive(file *os.File) error {
+	return unix.Flock(int(file.Fd()), unix.LOCK_EX)
+}
+
+func unlockFile(file *os.File) error {
+	return unix.Flock(int(file.Fd()), unix.LOCK_UN)
+}

pkg/deviceagent/enrollment_lock_windows.go

@@ -0,0 +1,50 @@
+// Copyright (c) 2026 Probo Inc <hello@probo.com>.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+//go:build windows
+
+package deviceagent
+
+import (
+	"os"
+
+	"golang.org/x/sys/windows"
+)
+
+func lockFileExclusive(file *os.File) error {
+	return windows.LockFileEx(
+		windows.Handle(file.Fd()),
+		windows.LOCKFILE_EXCLUSIVE_LOCK,
+		0,
+		1,
+		0,
+		&windows.Overlapped{},
+	)
+}
+
+func unlockFile(file *os.File) error {
+	return windows.UnlockFileEx(
+		windows.Handle(file.Fd()),
+		0,
+		1,
+		0,
+		&windows.Overlapped{},
+	)
+}

pkg/deviceagent/enrollment_state.go

@@ -40,8 +40,9 @@ const (
 )
 
 // EnrollmentRunDir returns the runtime directory for the public enrollment
-// marker. Production installs use DefaultEnrollmentRunDir(); custom --dir
-// values get an isolated sibling run tree for dev and tests.
+// marker and the short-lived enrolling.lock. Production installs use
+// DefaultEnrollmentRunDir(); custom --dir values get an isolated sibling
+// run tree for dev and tests.
 func EnrollmentRunDir(configDir string) string {
 	if configDir == "" {
 		configDir = DefaultConfigDir()

pkg/deviceagent/service/service_windows.go

@@ -39,6 +39,10 @@ func Install(cfg Config) error {
 
 	name := DefaultWindowsName
 
+	// Remove any previous registration so install is idempotent
+	// (matches Darwin's bootout-before-bootstrap).
+	_ = Uninstall(cfg)
+
 	bin := fmt.Sprintf(`"%s" run --dir "%s"`, cfg.ExePath, cfg.Dir)
 	if out, err := exec.Command(
 		"sc.exe",

pkg/deviceagent/state_cleanup.go

@@ -23,88 +23,132 @@ package deviceagent
 import (
 	"errors"
 	"fmt"
+	"io/fs"
 	"os"
 	"path/filepath"
 )
 
 const (
-	// SigstoreCacheDirName is the TUF metadata cache under the agent
-	// state directory.
 	SigstoreCacheDirName = "sigstore-cache"
-
-	// configFileTmpName is the atomic-write leftover from SaveConfig.
-	configFileTmpName = ConfigFileName + ".tmp"
+	configFileTmpName    = ConfigFileName + ".tmp"
 )
 
-// RemoveLocalState deletes known agent local state for an install: allowlisted
-// files under the state dir and EnrollmentRunDir, plus the sigstore-cache
-// subdirectory. It never recursively deletes either directory itself.
-// Missing paths are ignored. Filesystem roots and empty paths are rejected.
-//
-// Every path is attempted even if an earlier removal fails.
-//
-// enrolling.lock is intentionally retained: flock binds to the inode,
-// so unlinking the path while LoadOrExchangeAPIKey holds the lock would
-// let a concurrent caller create a new inode and bypass serialization.
+var stateCleanupNames = []string{
+	ConfigFileName,
+	configFileTmpName,
+	KeyFileName,
+	KeyFileName + ".tmp",
+	KeyFileName + ".old",
+	pendingPosturesFileName,
+	pendingPosturesFileName + ".tmp",
+}
+
+// RemoveLocalState removes allowlisted state/run files and sigstore-cache.
+// Missing paths are ignored; filesystem roots are refused. Does not remove
+// enrolling.lock (flock is inode-based).
 func RemoveLocalState(dir string) error {
 	if dir == "" {
 		return errors.New("state directory is empty")
 	}
 
+	stateDir, err := resolveCleanupDir(dir)
+	if err != nil {
+		return err
+	}
+
+	runDir, err := resolveCleanupDir(EnrollmentRunDir(dir))
+	if err != nil {
+		return err
+	}
+
+	return errors.Join(
+		removeAllUnder(stateDir, SigstoreCacheDirName),
+		cleanDir(stateDir, stateCleanupNames),
+		cleanDir(runDir, []string{EnrollmentMarkerName}),
+	)
+}
+
+// resolveCleanupDir resolves dir and rejects filesystem roots.
+// Missing paths return ("", nil).
+func resolveCleanupDir(dir string) (string, error) {
 	abs, err := filepath.Abs(dir)
 	if err != nil {
-		return fmt.Errorf("cannot resolve state directory: %w", err)
+		return "", fmt.Errorf("cannot resolve path: %w", err)
 	}
 
 	cleaned := filepath.Clean(abs)
 	if isFilesystemRoot(cleaned) {
-		return fmt.Errorf("refusing to remove filesystem root %q", cleaned)
+		return "", fmt.Errorf("refusing to remove filesystem root %q", cleaned)
 	}
 
-	// Use the original dir (not Abs/Clean) so custom relative --dir values
-	// resolve the same sibling run tree as MarkEnrolled / IsEnrolled.
-	runDir := EnrollmentRunDir(dir)
-	runDirAbs, err := filepath.Abs(runDir)
+	real, err := filepath.EvalSymlinks(cleaned)
 	if err != nil {
-		return fmt.Errorf("cannot resolve enrollment run dir: %w", err)
+		if errors.Is(err, fs.ErrNotExist) {
+			return "", nil
+		}
+
+		return "", fmt.Errorf("cannot resolve path: %w", err)
 	}
 
-	runDir = filepath.Clean(runDirAbs)
-	if isFilesystemRoot(runDir) {
-		return fmt.Errorf("refusing to remove enrollment run dir %q", runDir)
+	if isFilesystemRoot(real) {
+		return "", fmt.Errorf("refusing to remove filesystem root %q", real)
 	}
 
-	knownPaths := []string{
-		filepath.Join(cleaned, ConfigFileName),
-		filepath.Join(cleaned, configFileTmpName),
-		filepath.Join(cleaned, KeyFileName),
-		filepath.Join(cleaned, KeyFileName+".tmp"),
-		filepath.Join(cleaned, KeyFileName+".old"),
-		filepath.Join(cleaned, pendingPosturesFileName),
-		filepath.Join(cleaned, pendingPosturesFileName+".tmp"),
-		filepath.Join(runDir, EnrollmentMarkerName),
+	return real, nil
+}
+
+func cleanDir(dir string, names []string) error {
+	if dir == "" {
+		return nil
 	}
 
+	root, err := os.OpenRoot(dir)
+	if err != nil {
+		if errors.Is(err, fs.ErrNotExist) {
+			return nil
+		}
+
+		return fmt.Errorf("cannot open %s: %w", dir, err)
+	}
+
+	defer func() { _ = root.Close() }()
+
 	var errs error
-	for _, path := range knownPaths {
-		if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
-			errs = errors.Join(errs, fmt.Errorf("cannot remove %s: %w", path, err))
+
+	for _, name := range names {
+		if err := root.Remove(name); err != nil && !errors.Is(err, fs.ErrNotExist) {
+			errs = errors.Join(errs, fmt.Errorf("cannot remove %s: %w", name, err))
 		}
 	}
 
-	cacheDir := filepath.Join(cleaned, SigstoreCacheDirName)
-	if err := os.RemoveAll(cacheDir); err != nil {
-		errs = errors.Join(errs, fmt.Errorf("cannot remove %s: %w", SigstoreCacheDirName, err))
-	}
-
-	// Best-effort: remove the dirs only when empty. Leave them alone when
-	// foreign files remain or removal is otherwise refused.
-	_ = os.Remove(cleaned)
-	_ = os.Remove(runDir)
+	_ = os.Remove(dir)
 
 	return errs
 }
 
+func removeAllUnder(dir, name string) error {
+	if dir == "" {
+		return nil
+	}
+
+	root, err := os.OpenRoot(dir)
+	if err != nil {
+		if errors.Is(err, fs.ErrNotExist) {
+			return nil
+		}
+
+		return fmt.Errorf("cannot open %s: %w", dir, err)
+	}
+
+	defer func() { _ = root.Close() }()
+
+	if err := root.RemoveAll(name); err != nil {
+		return fmt.Errorf("cannot remove %s: %w", name, err)
+	}
+
+	return nil
+}
+
 func isFilesystemRoot(path string) bool {
 	cleaned := filepath.Clean(path)
 	return cleaned != "" && cleaned == filepath.Dir(cleaned)