Files
probo/pkg/agentrun/service.go
Bryan Frimin c14bacb157 Drop agent-run lease and add approval resume
The worker leaned on a lease plus a heartbeat goroutine and a stale
recovery sweep to reclaim runs from crashed workers. That machinery
raced with long LLM and tool calls and conflated graceful stops with
failures. Remove the lease columns, heartbeat, and stale recovery, and
rely on FOR UPDATE SKIP LOCKED for single-claim plus explicit state
transitions: a graceful suspend returns the run to PENDING and a crash
now leaves it RUNNING for manual recovery.

Treat an approval interruption as a known stop that parks the run in
AWAITING_APPROVAL, and add SubmitApproval to merge human decisions into
the checkpoint and requeue the run to PENDING. The decisions must cover
exactly the pending approvals, since a missing one would resume as an
implicit denial. Expose this through the submitAgentRunApproval
mutation.

Signed-off-by: Bryan Frimin <bryan@probo.com>
2026-06-08 15:27:56 +02:00

179 lines
4.5 KiB
Go

// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package agentrun
import (
"context"
"errors"
"fmt"
"time"
"go.gearno.de/kit/pg"
"go.probo.inc/probo/pkg/agent"
"go.probo.inc/probo/pkg/coredata"
"go.probo.inc/probo/pkg/gid"
"go.probo.inc/probo/pkg/page"
)
type Service struct {
pg *pg.Client
}
func NewService(pgClient *pg.Client) *Service {
return &Service{pg: pgClient}
}
func (s *Service) Get(
ctx context.Context,
scope coredata.Scoper,
agentRunID gid.GID,
) (*coredata.AgentRun, error) {
run := &coredata.AgentRun{}
err := s.pg.WithConn(
ctx,
func(ctx context.Context, conn pg.Querier) error {
if err := run.LoadByID(ctx, conn, scope, agentRunID); err != nil {
return fmt.Errorf("cannot load agent run: %w", err)
}
return nil
},
)
if err != nil {
return nil, err
}
return run, nil
}
func (s *Service) ListForOrganizationID(
ctx context.Context,
scope coredata.Scoper,
organizationID gid.GID,
cursor *page.Cursor[coredata.AgentRunOrderField],
) (*page.Page[*coredata.AgentRun, coredata.AgentRunOrderField], error) {
var runs coredata.AgentRuns
err := s.pg.WithConn(
ctx,
func(ctx context.Context, conn pg.Querier) error {
organization := &coredata.Organization{}
if err := organization.LoadByID(ctx, conn, scope, organizationID); err != nil {
return fmt.Errorf("cannot load organization: %w", err)
}
if err := runs.LoadByOrganizationID(ctx, conn, scope, organization.ID, cursor); err != nil {
return fmt.Errorf("cannot load agent runs: %w", err)
}
return nil
},
)
if err != nil {
return nil, err
}
return page.NewPage(runs, cursor), nil
}
// SubmitApproval records human approval decisions for a run parked in
// AWAITING_APPROVAL and requeues it to PENDING so a worker resumes it.
// decisions is keyed by pending tool-call ID and must cover exactly the
// run's pending approvals (a missing decision would be treated as an
// implicit denial on resume, so partial submissions are rejected). The
// refreshed run is returned.
func (s *Service) SubmitApproval(
ctx context.Context,
scope coredata.Scoper,
agentRunID gid.GID,
decisions map[string]agent.ApprovalResult,
) (*coredata.AgentRun, error) {
run := &coredata.AgentRun{}
err := s.pg.WithTx(
ctx,
func(ctx context.Context, tx pg.Tx) error {
if err := run.LoadByIDForUpdate(ctx, tx, scope, agentRunID); err != nil {
if errors.Is(err, coredata.ErrResourceNotFound) {
return ErrAgentRunNotFound
}
return fmt.Errorf("cannot load agent run: %w", err)
}
if run.Status != coredata.AgentRunStatusAwaitingApproval {
return ErrNotAwaitingApproval
}
if run.Checkpoint == nil {
return fmt.Errorf("agent run %s has no checkpoint", agentRunID)
}
checkpoint, err := agent.MergeApprovalDecisions(run.Checkpoint, decisions)
if err != nil {
if errors.Is(err, agent.ErrApprovalDecisionsMismatch) {
return ErrApprovalDecisionsMismatch
}
return fmt.Errorf("cannot merge approval decisions: %w", err)
}
run.Checkpoint = checkpoint
run.Status = coredata.AgentRunStatusPending
run.StartedAt = nil
run.UpdatedAt = time.Now()
if err := run.RequeueForApprovalResume(ctx, tx, scope); err != nil {
return fmt.Errorf("cannot requeue agent run for approval resume: %w", err)
}
return nil
},
)
if err != nil {
return nil, err
}
return run, nil
}
func (s *Service) CountForOrganizationID(
ctx context.Context,
scope coredata.Scoper,
organizationID gid.GID,
) (int, error) {
var count int
err := s.pg.WithConn(
ctx,
func(ctx context.Context, conn pg.Querier) (err error) {
runs := &coredata.AgentRuns{}
count, err = runs.CountByOrganizationID(ctx, conn, scope, organizationID)
if err != nil {
return fmt.Errorf("cannot count agent runs: %w", err)
}
return nil
},
)
if err != nil {
return 0, err
}
return count, nil
}