summaryrefslogtreecommitdiff
path: root/internal/executor/executor.go
diff options
context:
space:
mode:
Diffstat (limited to 'internal/executor/executor.go')
-rw-r--r--internal/executor/executor.go94
1 files changed, 94 insertions, 0 deletions
diff --git a/internal/executor/executor.go b/internal/executor/executor.go
index 981a3ad..ed38d7d 100644
--- a/internal/executor/executor.go
+++ b/internal/executor/executor.go
@@ -45,6 +45,9 @@ type Store interface {
// if none is active. Used by execute() to resolve tier 0 of a role-typed
// task's escalation ladder.
GetActiveRoleConfig(role string) (*storage.RoleConfigRow, error)
+ // ListDependents returns tasks that directly depend on taskID. Used by
+ // cascadeFail to proactively cancel a failed task's downstream subtree.
+ ListDependents(taskID string) ([]*task.Task, error)
}
// LogPather is an optional interface runners can implement to provide the log
@@ -524,6 +527,16 @@ func (p *Pool) handleRunResult(ctx context.Context, t *task.Task, exec *storage.
}
}
+ // Proactively cancel the downstream dependency subtree the moment this
+ // task lands in a terminal failure state, so tasks depending on it (e.g.
+ // evaluator subtasks fanned out from a Builder) don't sit around waiting
+ // on a dependency that will never complete. BLOCKED/READY/COMPLETED are
+ // intentionally excluded — those aren't terminal failures.
+ switch exec.Status {
+ case "FAILED", "TIMED_OUT", "CANCELLED", "BUDGET_EXCEEDED":
+ p.cascadeFail(t.ID, exec.ErrorMsg)
+ }
+
summary := exec.Summary
if summary == "" && exec.StdoutPath != "" {
summary = extractSummary(exec.StdoutPath)
@@ -550,6 +563,75 @@ func (p *Pool) handleRunResult(ctx context.Context, t *task.Task, exec *storage.
p.resultCh <- &Result{TaskID: t.ID, Execution: exec, Err: err}
}
+// cascadeCancellableStates are the task states a dependent task can safely be
+// preempted from: it hasn't started running yet. RUNNING, terminal states,
+// and BLOCKED (subtask delegation) are all left untouched — cascadeFail only
+// preempts work that hasn't started.
+var cascadeCancellableStates = map[task.State]bool{
+ task.StatePending: true,
+ task.StateQueued: true,
+}
+
+// cascadeFail proactively cancels the downstream dependency subtree rooted at
+// taskID. It is called right after a task is marked into a terminal failure
+// state (FAILED/TIMED_OUT/CANCELLED/BUDGET_EXCEEDED): every direct dependent
+// still waiting to run (PENDING or QUEUED) is recorded as CANCELLED with a
+// message referencing the upstream failure, and the same cancellation is
+// applied recursively to that dependent's own dependents — so a multi-level
+// chain (A fails, B depends on A, C depends on B) cascades all the way down,
+// not just one hop. Dependents that are already RUNNING or in a terminal
+// state are left alone.
+//
+// This exists so a fan-out pattern (e.g. a Builder task with several
+// Evaluator subtasks depending on it) doesn't leave those subtasks sitting
+// around forever — or worse, waiting on a dependency that will never
+// complete — when the upstream task fails. Without this, a dependent only
+// discovers its dependency failed the next time something tries to dispatch
+// it (see checkDepsReady), which may be never for a task nobody has
+// submitted to the pool yet.
+func (p *Pool) cascadeFail(taskID string, reason string) {
+ p.cascadeFailFrom(taskID, reason, make(map[string]bool))
+}
+
+// cascadeFailFrom does the actual work for cascadeFail, threading a visited
+// set through the recursion so a pathological dependency cycle can't cause
+// infinite recursion.
+func (p *Pool) cascadeFailFrom(taskID string, reason string, visited map[string]bool) {
+ if visited[taskID] {
+ return
+ }
+ visited[taskID] = true
+
+ dependents, err := p.store.ListDependents(taskID)
+ if err != nil {
+ p.logger.Error("cascadeFail: list dependents", "taskID", taskID, "error", err)
+ return
+ }
+ for _, dep := range dependents {
+ if !cascadeCancellableStates[dep.State] {
+ continue // already running, already terminal, or blocked — leave it alone
+ }
+ msg := fmt.Sprintf("upstream dependency %s failed: %s", taskID, reason)
+ now := time.Now().UTC()
+ exec := &storage.Execution{
+ ID: uuid.New().String(),
+ TaskID: dep.ID,
+ StartTime: now,
+ EndTime: now,
+ Status: "CANCELLED",
+ ErrorMsg: msg,
+ }
+ if createErr := p.store.CreateExecution(exec); createErr != nil {
+ p.logger.Error("cascadeFail: create execution", "taskID", dep.ID, "error", createErr)
+ }
+ if stateErr := p.store.UpdateTaskState(dep.ID, task.StateCancelled); stateErr != nil {
+ p.logger.Error("cascadeFail: update task state", "taskID", dep.ID, "error", stateErr)
+ continue // couldn't actually cancel it — don't cascade further from here
+ }
+ p.logger.Info("cascade-cancelled dependent task", "taskID", dep.ID, "upstream", taskID)
+ p.cascadeFailFrom(dep.ID, msg, visited)
+ }
+}
// ActiveCount returns the number of currently running tasks.
func (p *Pool) ActiveCount() int {
@@ -839,6 +921,7 @@ func (p *Pool) execute(ctx context.Context, t *task.Task) {
if err := p.store.UpdateTaskState(t.ID, task.StateBudgetExceeded); err != nil {
p.logger.Error("failed to update task state", "taskID", t.ID, "state", task.StateBudgetExceeded, "error", err)
}
+ p.cascadeFail(t.ID, exec.ErrorMsg)
p.resultCh <- &Result{TaskID: t.ID, Execution: exec, Err: fmt.Errorf("budget cap reached for %s", agentType)}
return
}
@@ -849,6 +932,16 @@ func (p *Pool) execute(ctx context.Context, t *task.Task) {
// if a dependent task holds the slot while waiting for its dependency to run,
// the dependency can never start (maxPerAgent=1).
if len(t.DependsOn) > 0 {
+ // cascadeFail (triggered when some ancestor of t failed) may already
+ // have cancelled t out from under it while it sat in the AfterFunc
+ // requeue loop below — re-check the fresh state so we don't try (and
+ // fail, since CANCELLED has no valid self-transition) to re-apply a
+ // terminal transition, and — more importantly — so a cascade-cancelled
+ // task never actually gets dispatched to a runner.
+ if fresh, ferr := p.store.GetTask(t.ID); ferr == nil && fresh.State != task.StateQueued {
+ p.logger.Info("skipping dispatch: task no longer queued (likely cascade-cancelled)", "taskID", t.ID, "state", fresh.State)
+ return
+ }
ready, depErr := p.checkDepsReady(t)
if depErr != nil {
// A dependency hit a terminal failure — cancel this task immediately.
@@ -867,6 +960,7 @@ func (p *Pool) execute(ctx context.Context, t *task.Task) {
if err := p.store.UpdateTaskState(t.ID, task.StateCancelled); err != nil {
p.logger.Error("failed to update task state", "taskID", t.ID, "state", task.StateCancelled, "error", err)
}
+ p.cascadeFail(t.ID, depErr.Error())
p.resultCh <- &Result{TaskID: t.ID, Execution: exec, Err: depErr}
return
}