Drop agent-run lease and add approval resume

The worker leaned on a lease plus a heartbeat goroutine and a stale
recovery sweep to reclaim runs from crashed workers. That machinery
raced with long LLM and tool calls and conflated graceful stops with
failures. Remove the lease columns, heartbeat, and stale recovery, and
rely on FOR UPDATE SKIP LOCKED for single-claim plus explicit state
transitions: a graceful suspend returns the run to PENDING and a crash
now leaves it RUNNING for manual recovery.

Treat an approval interruption as a known stop that parks the run in
AWAITING_APPROVAL, and add SubmitApproval to merge human decisions into
the checkpoint and requeue the run to PENDING. The decisions must cover
exactly the pending approvals, since a missing one would resume as an
implicit denial. Expose this through the submitAgentRunApproval
mutation.

Signed-off-by: Bryan Frimin <bryan@probo.com>
This commit is contained in:
Bryan Frimin
2026-06-08 14:51:12 +02:00
parent 98a8d90391
commit c14bacb157
9 changed files with 398 additions and 363 deletions

View File

@@ -16,10 +16,19 @@ package agent
import (
"context"
"encoding/json"
"errors"
"fmt"
"go.probo.inc/probo/pkg/llm"
)
// ErrApprovalDecisionsMismatch is returned by MergeApprovalDecisions when
// the supplied decisions do not cover exactly the checkpoint's pending
// approvals. A missing decision would resume as an implicit denial, so
// partial submissions are rejected.
var ErrApprovalDecisionsMismatch = errors.New("approval decisions do not match the checkpoint's pending approvals")
type (
ApprovalConfig struct {
ToolNames []string
@@ -38,6 +47,50 @@ type (
}
)
// MergeApprovalDecisions decodes an awaiting-approval checkpoint, records
// the human decisions into its ApprovalInput, and returns the re-encoded
// checkpoint ready to be persisted. decisions is keyed by pending
// tool-call ID and must cover exactly the checkpoint's pending approvals;
// otherwise ErrApprovalDecisionsMismatch is returned.
func MergeApprovalDecisions(
raw json.RawMessage,
decisions map[string]ApprovalResult,
) (json.RawMessage, error) {
var cp Checkpoint
if err := json.Unmarshal(raw, &cp); err != nil {
return nil, fmt.Errorf("cannot unmarshal checkpoint: %w", err)
}
pending := make(map[string]struct{}, len(cp.PendingApprovals))
for _, toolCall := range cp.PendingApprovals {
pending[toolCall.ID] = struct{}{}
}
if len(decisions) != len(pending) {
return nil, ErrApprovalDecisionsMismatch
}
for id := range decisions {
if _, ok := pending[id]; !ok {
return nil, ErrApprovalDecisionsMismatch
}
}
if cp.ApprovalInput == nil {
cp.ApprovalInput = make(map[string]ApprovalResult, len(decisions))
}
for id, decision := range decisions {
cp.ApprovalInput[id] = decision
}
data, err := json.Marshal(&cp)
if err != nil {
return nil, fmt.Errorf("cannot marshal checkpoint: %w", err)
}
return data, nil
}
func buildToolNameSet(names []string) map[string]struct{} {
if len(names) == 0 {
return nil