summaryrefslogtreecommitdiff
path: root/internal/executor/executor.go
diff options
context:
space:
mode:
authorClaude Sonnet 5 <noreply@anthropic.com>2026-07-03 23:22:42 +0000
committerClaude Sonnet 5 <noreply@anthropic.com>2026-07-03 23:22:42 +0000
commit997cd8b56bc086a02b9c7c006dd62b07b9fcd2f3 (patch)
tree94351bb0aebeb92cb4c0f63f2e0a2636e06c4f1e /internal/executor/executor.go
parent787b7fb1aed92c2b701724a7741576053b93cccb (diff)
feat(executor): add DAG auto-cascade-fail + role-typed subtask spawning (Phase 6)
Two prerequisites for safe parallel evaluator fan-out (later phase): 1. Auto-cascade-fail: previously, a failed task's dependents just sat PENDING/QUEUED forever (or until something eventually tried to dispatch them and discovered the dependency was dead). Pool.cascadeFail now fires right after a task lands in a terminal failure state (FAILED/TIMED_OUT/ CANCELLED/BUDGET_EXCEEDED, from handleRunResult, the budget-gate reject path, and the checkDepsReady dependency-failure path), recursively cancelling every not-yet-run dependent (PENDING/QUEUED only -- RUNNING and terminal states are left alone) with a message referencing the upstream failure. A visited-set guards recursion, which turned out to be load-bearing rather than defense-in-depth: task creation does not prevent dependency cycles anywhere in this codebase. Correction to an earlier assumption: internal/executor's waitForDependencies is dead code, never called. The live mechanism is checkDepsReady, invoked synchronously in execute() with a self-requeue via time.AfterFunc. Added a freshness re-check (GetTask, bail if no longer QUEUED) at the top of that block so a task cascade-cancelled while sitting in the requeue loop stops silently instead of hitting an invalid CANCELLED->CANCELLED transition or, worse, still getting dispatched. 2. storeChannel.SpawnSubtask hardcoded Agent.Type: "claude" on every spawned child regardless of what role it should play -- a hard blocker for a Planner/Builder task spawning role-typed evaluator subtasks. SubtaskSpec (internal/agentchannel) gains a Role field; when set, the child task gets Agent.Role instead of a hardcoded Type, so Phase 5's role-resolution picks provider/model from that role's escalation ladder. spec.Role == "" (every existing caller) preserves today's exact behavior byte-for-byte -- proven by an explicit regression test, not just new-feature coverage. Threaded the new `role` parameter through both spawn_subtask transports: the native tool-use loop (internal/agentloop/tools.go) and the MCP tool exposed to ContainerRunner-driven claude/gemini agents (internal/executor/agentmcp.go). go build/vet/test -race -count=1 all pass, full suite (20 packages). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01V1moSNCJRcP6kykA4tyUSs
Diffstat (limited to 'internal/executor/executor.go')
-rw-r--r--internal/executor/executor.go94
1 files changed, 94 insertions, 0 deletions
diff --git a/internal/executor/executor.go b/internal/executor/executor.go
index 981a3ad..ed38d7d 100644
--- a/internal/executor/executor.go
+++ b/internal/executor/executor.go
@@ -45,6 +45,9 @@ type Store interface {
// if none is active. Used by execute() to resolve tier 0 of a role-typed
// task's escalation ladder.
GetActiveRoleConfig(role string) (*storage.RoleConfigRow, error)
+ // ListDependents returns tasks that directly depend on taskID. Used by
+ // cascadeFail to proactively cancel a failed task's downstream subtree.
+ ListDependents(taskID string) ([]*task.Task, error)
}
// LogPather is an optional interface runners can implement to provide the log
@@ -524,6 +527,16 @@ func (p *Pool) handleRunResult(ctx context.Context, t *task.Task, exec *storage.
}
}
+ // Proactively cancel the downstream dependency subtree the moment this
+ // task lands in a terminal failure state, so tasks depending on it (e.g.
+ // evaluator subtasks fanned out from a Builder) don't sit around waiting
+ // on a dependency that will never complete. BLOCKED/READY/COMPLETED are
+ // intentionally excluded — those aren't terminal failures.
+ switch exec.Status {
+ case "FAILED", "TIMED_OUT", "CANCELLED", "BUDGET_EXCEEDED":
+ p.cascadeFail(t.ID, exec.ErrorMsg)
+ }
+
summary := exec.Summary
if summary == "" && exec.StdoutPath != "" {
summary = extractSummary(exec.StdoutPath)
@@ -550,6 +563,75 @@ func (p *Pool) handleRunResult(ctx context.Context, t *task.Task, exec *storage.
p.resultCh <- &Result{TaskID: t.ID, Execution: exec, Err: err}
}
+// cascadeCancellableStates are the task states a dependent task can safely be
+// preempted from: it hasn't started running yet. RUNNING, terminal states,
+// and BLOCKED (subtask delegation) are all left untouched — cascadeFail only
+// preempts work that hasn't started.
+var cascadeCancellableStates = map[task.State]bool{
+ task.StatePending: true,
+ task.StateQueued: true,
+}
+
+// cascadeFail proactively cancels the downstream dependency subtree rooted at
+// taskID. It is called right after a task is marked into a terminal failure
+// state (FAILED/TIMED_OUT/CANCELLED/BUDGET_EXCEEDED): every direct dependent
+// still waiting to run (PENDING or QUEUED) is recorded as CANCELLED with a
+// message referencing the upstream failure, and the same cancellation is
+// applied recursively to that dependent's own dependents — so a multi-level
+// chain (A fails, B depends on A, C depends on B) cascades all the way down,
+// not just one hop. Dependents that are already RUNNING or in a terminal
+// state are left alone.
+//
+// This exists so a fan-out pattern (e.g. a Builder task with several
+// Evaluator subtasks depending on it) doesn't leave those subtasks sitting
+// around forever — or worse, waiting on a dependency that will never
+// complete — when the upstream task fails. Without this, a dependent only
+// discovers its dependency failed the next time something tries to dispatch
+// it (see checkDepsReady), which may be never for a task nobody has
+// submitted to the pool yet.
+func (p *Pool) cascadeFail(taskID string, reason string) {
+ p.cascadeFailFrom(taskID, reason, make(map[string]bool))
+}
+
+// cascadeFailFrom does the actual work for cascadeFail, threading a visited
+// set through the recursion so a pathological dependency cycle can't cause
+// infinite recursion.
+func (p *Pool) cascadeFailFrom(taskID string, reason string, visited map[string]bool) {
+ if visited[taskID] {
+ return
+ }
+ visited[taskID] = true
+
+ dependents, err := p.store.ListDependents(taskID)
+ if err != nil {
+ p.logger.Error("cascadeFail: list dependents", "taskID", taskID, "error", err)
+ return
+ }
+ for _, dep := range dependents {
+ if !cascadeCancellableStates[dep.State] {
+ continue // already running, already terminal, or blocked — leave it alone
+ }
+ msg := fmt.Sprintf("upstream dependency %s failed: %s", taskID, reason)
+ now := time.Now().UTC()
+ exec := &storage.Execution{
+ ID: uuid.New().String(),
+ TaskID: dep.ID,
+ StartTime: now,
+ EndTime: now,
+ Status: "CANCELLED",
+ ErrorMsg: msg,
+ }
+ if createErr := p.store.CreateExecution(exec); createErr != nil {
+ p.logger.Error("cascadeFail: create execution", "taskID", dep.ID, "error", createErr)
+ }
+ if stateErr := p.store.UpdateTaskState(dep.ID, task.StateCancelled); stateErr != nil {
+ p.logger.Error("cascadeFail: update task state", "taskID", dep.ID, "error", stateErr)
+ continue // couldn't actually cancel it — don't cascade further from here
+ }
+ p.logger.Info("cascade-cancelled dependent task", "taskID", dep.ID, "upstream", taskID)
+ p.cascadeFailFrom(dep.ID, msg, visited)
+ }
+}
// ActiveCount returns the number of currently running tasks.
func (p *Pool) ActiveCount() int {
@@ -839,6 +921,7 @@ func (p *Pool) execute(ctx context.Context, t *task.Task) {
if err := p.store.UpdateTaskState(t.ID, task.StateBudgetExceeded); err != nil {
p.logger.Error("failed to update task state", "taskID", t.ID, "state", task.StateBudgetExceeded, "error", err)
}
+ p.cascadeFail(t.ID, exec.ErrorMsg)
p.resultCh <- &Result{TaskID: t.ID, Execution: exec, Err: fmt.Errorf("budget cap reached for %s", agentType)}
return
}
@@ -849,6 +932,16 @@ func (p *Pool) execute(ctx context.Context, t *task.Task) {
// if a dependent task holds the slot while waiting for its dependency to run,
// the dependency can never start (maxPerAgent=1).
if len(t.DependsOn) > 0 {
+ // cascadeFail (triggered when some ancestor of t failed) may already
+ // have cancelled t out from under it while it sat in the AfterFunc
+ // requeue loop below — re-check the fresh state so we don't try (and
+ // fail, since CANCELLED has no valid self-transition) to re-apply a
+ // terminal transition, and — more importantly — so a cascade-cancelled
+ // task never actually gets dispatched to a runner.
+ if fresh, ferr := p.store.GetTask(t.ID); ferr == nil && fresh.State != task.StateQueued {
+ p.logger.Info("skipping dispatch: task no longer queued (likely cascade-cancelled)", "taskID", t.ID, "state", fresh.State)
+ return
+ }
ready, depErr := p.checkDepsReady(t)
if depErr != nil {
// A dependency hit a terminal failure — cancel this task immediately.
@@ -867,6 +960,7 @@ func (p *Pool) execute(ctx context.Context, t *task.Task) {
if err := p.store.UpdateTaskState(t.ID, task.StateCancelled); err != nil {
p.logger.Error("failed to update task state", "taskID", t.ID, "state", task.StateCancelled, "error", err)
}
+ p.cascadeFail(t.ID, depErr.Error())
p.resultCh <- &Result{TaskID: t.ID, Execution: exec, Err: depErr}
return
}