diff options
Diffstat (limited to 'internal/executor/executor.go')
| -rw-r--r-- | internal/executor/executor.go | 94 |
1 files changed, 94 insertions, 0 deletions
diff --git a/internal/executor/executor.go b/internal/executor/executor.go index 981a3ad..ed38d7d 100644 --- a/internal/executor/executor.go +++ b/internal/executor/executor.go @@ -45,6 +45,9 @@ type Store interface { // if none is active. Used by execute() to resolve tier 0 of a role-typed // task's escalation ladder. GetActiveRoleConfig(role string) (*storage.RoleConfigRow, error) + // ListDependents returns tasks that directly depend on taskID. Used by + // cascadeFail to proactively cancel a failed task's downstream subtree. + ListDependents(taskID string) ([]*task.Task, error) } // LogPather is an optional interface runners can implement to provide the log @@ -524,6 +527,16 @@ func (p *Pool) handleRunResult(ctx context.Context, t *task.Task, exec *storage. } } + // Proactively cancel the downstream dependency subtree the moment this + // task lands in a terminal failure state, so tasks depending on it (e.g. + // evaluator subtasks fanned out from a Builder) don't sit around waiting + // on a dependency that will never complete. BLOCKED/READY/COMPLETED are + // intentionally excluded — those aren't terminal failures. + switch exec.Status { + case "FAILED", "TIMED_OUT", "CANCELLED", "BUDGET_EXCEEDED": + p.cascadeFail(t.ID, exec.ErrorMsg) + } + summary := exec.Summary if summary == "" && exec.StdoutPath != "" { summary = extractSummary(exec.StdoutPath) @@ -550,6 +563,75 @@ func (p *Pool) handleRunResult(ctx context.Context, t *task.Task, exec *storage. p.resultCh <- &Result{TaskID: t.ID, Execution: exec, Err: err} } +// cascadeCancellableStates are the task states a dependent task can safely be +// preempted from: it hasn't started running yet. RUNNING, terminal states, +// and BLOCKED (subtask delegation) are all left untouched — cascadeFail only +// preempts work that hasn't started. +var cascadeCancellableStates = map[task.State]bool{ + task.StatePending: true, + task.StateQueued: true, +} + +// cascadeFail proactively cancels the downstream dependency subtree rooted at +// taskID. It is called right after a task is marked into a terminal failure +// state (FAILED/TIMED_OUT/CANCELLED/BUDGET_EXCEEDED): every direct dependent +// still waiting to run (PENDING or QUEUED) is recorded as CANCELLED with a +// message referencing the upstream failure, and the same cancellation is +// applied recursively to that dependent's own dependents — so a multi-level +// chain (A fails, B depends on A, C depends on B) cascades all the way down, +// not just one hop. Dependents that are already RUNNING or in a terminal +// state are left alone. +// +// This exists so a fan-out pattern (e.g. a Builder task with several +// Evaluator subtasks depending on it) doesn't leave those subtasks sitting +// around forever — or worse, waiting on a dependency that will never +// complete — when the upstream task fails. Without this, a dependent only +// discovers its dependency failed the next time something tries to dispatch +// it (see checkDepsReady), which may be never for a task nobody has +// submitted to the pool yet. +func (p *Pool) cascadeFail(taskID string, reason string) { + p.cascadeFailFrom(taskID, reason, make(map[string]bool)) +} + +// cascadeFailFrom does the actual work for cascadeFail, threading a visited +// set through the recursion so a pathological dependency cycle can't cause +// infinite recursion. +func (p *Pool) cascadeFailFrom(taskID string, reason string, visited map[string]bool) { + if visited[taskID] { + return + } + visited[taskID] = true + + dependents, err := p.store.ListDependents(taskID) + if err != nil { + p.logger.Error("cascadeFail: list dependents", "taskID", taskID, "error", err) + return + } + for _, dep := range dependents { + if !cascadeCancellableStates[dep.State] { + continue // already running, already terminal, or blocked — leave it alone + } + msg := fmt.Sprintf("upstream dependency %s failed: %s", taskID, reason) + now := time.Now().UTC() + exec := &storage.Execution{ + ID: uuid.New().String(), + TaskID: dep.ID, + StartTime: now, + EndTime: now, + Status: "CANCELLED", + ErrorMsg: msg, + } + if createErr := p.store.CreateExecution(exec); createErr != nil { + p.logger.Error("cascadeFail: create execution", "taskID", dep.ID, "error", createErr) + } + if stateErr := p.store.UpdateTaskState(dep.ID, task.StateCancelled); stateErr != nil { + p.logger.Error("cascadeFail: update task state", "taskID", dep.ID, "error", stateErr) + continue // couldn't actually cancel it — don't cascade further from here + } + p.logger.Info("cascade-cancelled dependent task", "taskID", dep.ID, "upstream", taskID) + p.cascadeFailFrom(dep.ID, msg, visited) + } +} // ActiveCount returns the number of currently running tasks. func (p *Pool) ActiveCount() int { @@ -839,6 +921,7 @@ func (p *Pool) execute(ctx context.Context, t *task.Task) { if err := p.store.UpdateTaskState(t.ID, task.StateBudgetExceeded); err != nil { p.logger.Error("failed to update task state", "taskID", t.ID, "state", task.StateBudgetExceeded, "error", err) } + p.cascadeFail(t.ID, exec.ErrorMsg) p.resultCh <- &Result{TaskID: t.ID, Execution: exec, Err: fmt.Errorf("budget cap reached for %s", agentType)} return } @@ -849,6 +932,16 @@ func (p *Pool) execute(ctx context.Context, t *task.Task) { // if a dependent task holds the slot while waiting for its dependency to run, // the dependency can never start (maxPerAgent=1). if len(t.DependsOn) > 0 { + // cascadeFail (triggered when some ancestor of t failed) may already + // have cancelled t out from under it while it sat in the AfterFunc + // requeue loop below — re-check the fresh state so we don't try (and + // fail, since CANCELLED has no valid self-transition) to re-apply a + // terminal transition, and — more importantly — so a cascade-cancelled + // task never actually gets dispatched to a runner. + if fresh, ferr := p.store.GetTask(t.ID); ferr == nil && fresh.State != task.StateQueued { + p.logger.Info("skipping dispatch: task no longer queued (likely cascade-cancelled)", "taskID", t.ID, "state", fresh.State) + return + } ready, depErr := p.checkDepsReady(t) if depErr != nil { // A dependency hit a terminal failure — cancel this task immediately. @@ -867,6 +960,7 @@ func (p *Pool) execute(ctx context.Context, t *task.Task) { if err := p.store.UpdateTaskState(t.ID, task.StateCancelled); err != nil { p.logger.Error("failed to update task state", "taskID", t.ID, "state", task.StateCancelled, "error", err) } + p.cascadeFail(t.ID, depErr.Error()) p.resultCh <- &Result{TaskID: t.ID, Execution: exec, Err: depErr} return } |
