diff options
Diffstat (limited to 'internal/scheduler/scheduler.go')
| -rw-r--r-- | internal/scheduler/scheduler.go | 48 |
1 files changed, 42 insertions, 6 deletions
diff --git a/internal/scheduler/scheduler.go b/internal/scheduler/scheduler.go index 6df854b..cc6fa3a 100644 --- a/internal/scheduler/scheduler.go +++ b/internal/scheduler/scheduler.go @@ -205,24 +205,26 @@ func (s *Scheduler) processTask(ctx context.Context, t *task.Task) { return // already decided for this execution — converged, nothing to do. } + currentRung := latest.EscalationRung + if currentRung < 0 { + currentRung = 0 + } + row, err := s.Store.GetActiveRoleConfig(t.Agent.Role) if err != nil { - s.logf("scheduler: no active role config for role", "role", t.Agent.Role, "taskID", t.ID, "error", err) + s.retryWithoutLadder(ctx, t, execs, currentRung, "no active role config for role "+t.Agent.Role) return } var rc role.RoleConfig if err := json.Unmarshal([]byte(row.ConfigJSON), &rc); err != nil { - s.logf("scheduler: decode role config", "role", t.Agent.Role, "taskID", t.ID, "error", err) + s.retryWithoutLadder(ctx, t, execs, currentRung, "failed to decode role config") return } if len(rc.EscalationLadder) == 0 { + s.retryWithoutLadder(ctx, t, execs, currentRung, "empty escalation ladder") return } - currentRung := latest.EscalationRung - if currentRung < 0 { - currentRung = 0 - } if currentRung >= len(rc.EscalationLadder) { // Ladder already exhausted (e.g. the ladder was shortened after this // task started climbing it) — nothing more to do. @@ -291,6 +293,40 @@ func (s *Scheduler) retrySameRung(ctx context.Context, t *task.Task, rung int) { } } +// maxRetriesNoRoleConfig bounds retryWithoutLadder's same-tier retries when +// a role-typed task has no usable escalation ladder to consult -- a simple, +// hardcoded safety net (mirrors maxFixAttempts in +// internal/scheduler/story_orchestrator.go), not real ladder-based tiering. +const maxRetriesNoRoleConfig = 2 + +// retryWithoutLadder handles a role-typed task whose role has no usable +// escalation ladder to consult -- no active role_configs row at all, a +// corrupt one, or one with an empty ladder -- by retrying at the task's +// existing Agent.Type/Model (unchanged, since there's no ladder to pick a +// tier from) up to maxRetriesNoRoleConfig times before giving up. Mirrors +// escalateAskUserTimeout's own "no active role config... still resuming, +// just without a provider/model change" fallback below, applied here to the +// FAILED-task retry path instead of the ask_user-timeout path. +// +// Added 2026-07-11: before this, a role with no seeded role_configs (every +// role except builder/planner, as of this fix -- see +// internal/storage.SeedRoleConfigs) got zero retry safety net at all: the +// very first transient infra failure (a generic "container execution failed: +// exit status 1", the same symptom repeatedly traced to disk-full or other +// non-deterministic causes this session) left the task FAILED forever with +// nothing ever looking at it again -- a real production incident (see +// internal/scheduler/story_orchestrator.go's finalizeArbitration doc +// comment for the related arbitration-side half of this same root cause). +func (s *Scheduler) retryWithoutLadder(ctx context.Context, t *task.Task, execs []*storage.Execution, rung int, reason string) { + attempts := attemptsAtRung(execs, rung) + if attempts < maxRetriesNoRoleConfig { + s.logf("scheduler: retrying without a usable escalation ladder", "role", t.Agent.Role, "taskID", t.ID, "reason", reason, "attempt", attempts+1, "maxAttempts", maxRetriesNoRoleConfig) + s.retrySameRung(ctx, t, rung) + return + } + s.decline(ctx, t, rung, "", reason+"; retries exhausted") +} + func (s *Scheduler) escalate(ctx context.Context, t *task.Task, fromRung, toRung int, target role.Rung) { fromProvider := t.Agent.Type newAgent := t.Agent |