feat(role): add versioned role configs + escalation ladder + scheduler (Phase 5)

Two parts:

Part A (fixes a gap from Phase 1): Groq/OpenRouter/OpenAI were documented in
docs/api-keys-setup.md as usable once configured, but nothing actually
constructed runners for them. internal/cli/cloudrunners.go consolidates
anthropic/google/groq/openrouter/openai NativeRunner construction into one
table-driven registerCloudRunners() helper, replacing the two hand-written
per-provider blocks in serve.go/run.go. Groq/OpenRouter/OpenAI reuse
openaicompat (no new adapter code) at SandboxKind: "docker".

Part B: the token-husbanding harness's core routing mechanism.

- internal/role: RoleConfig/Tier/Rung -- a role's system prompt and a
  multi-tier (provider, model) escalation ladder, versioned via config_json.
- storage: new role_configs table (draft/active/retired, UNIQUE(role,
  version)) with transactional activate-retires-prior-active semantics;
  new executions.escalation_rung column.
- task.AgentConfig.Role string -- purely additive; every existing task shape
  (Agent.Role == "") is unaffected, proven by TestPool_Execute_NonRoleTask_Unaffected
  plus the full pre-existing suite passing unchanged.
- executor.Pool.execute(): role-typed tasks with no Agent.Type yet resolve
  tier 0 of their active ladder (round-robin across multi-candidate tiers,
  skipping rate-limited providers, falling back to soonest-clearing) before
  the existing pickAgent/Classifier path runs; SystemPrompt applies to
  Agent.SystemPromptAppend. Already-resolved role tasks (scheduler resubmits)
  get their escalation_rung re-derived read-only via findTierIndex.
- internal/scheduler: polls role-typed FAILED tasks, retries at the same
  rung under MaxRetries or escalates to the next tier's first candidate when
  budget.Accountant.Allow() permits (emitting event.KindEscalated), else
  leaves the task FAILED with a final:true KindEscalated event. An
  in-memory per-execution-ID "handled" set keeps the poll loop convergent.
  Started by `serve` only, config knob [scheduler].poll_interval_seconds.
- internal/api: POST/GET /api/roles/{role}/versions, POST
  /api/roles/{role}/activate -- unauthenticated, matching the existing
  projects/tasks REST endpoints' auth posture (only chatbot MCP, agent MCP,
  and WebSocket are api_token-gated in this codebase today).

Documented as stored-but-not-yet-enforced (CLAUDE.md Design Debt, matching
how task.Priority/RetryConfig are already documented): RoleConfig.Tools/
SandboxKind don't affect dispatch yet; DefaultBudgetUSD is read narrowly as
the scheduler's escalation cost estimate, not enforced at initial dispatch;
scheduler escalation always targets Candidates[0] (no round-robin, unlike
initial-dispatch tier-0 resolution); the scheduler's dedupe is per-process
and resets on restart (idempotent, harmless).

go build/vet/test -race -count=1 all pass, 21 packages.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01V1moSNCJRcP6kykA4tyUSs

1 files changed, 277 insertions, 0 deletions

diff --git a/internal/scheduler/scheduler.go b/internal/scheduler/scheduler.go
new file mode 100644
index 0000000..b3756fc
--- /dev/null
+++ b/internal/scheduler/scheduler.go
@@ -0,0 +1,277 @@
+// Package scheduler implements a first-pass retry-then-escalate loop for
+// role-typed tasks (task.AgentConfig.Role != ""). On a poll interval it looks
+// for tasks whose most recent execution ended FAILED, and either resubmits
+// them at the same escalation-ladder tier (if under that tier's MaxRetries)
+// or escalates them to the next tier (if the budget allows) — recording an
+// event.KindEscalated event either way. If the ladder is exhausted or the
+// budget denies the escalation, the task is left FAILED for human attention.
+//
+// Explicit non-goals for this phase (see the Phase 5 task description):
+// no AskUser-timeout escalation, no DAG/cascade-fail logic. Handling for
+// TIMED_OUT/CANCELLED/BUDGET_EXCEEDED tasks follows the same shape as FAILED
+// but isn't implemented yet — only FAILED is polled.
+package scheduler
+
+import (
+	"context"
+	"encoding/json"
+	"log/slog"
+	"sync"
+	"time"
+
+	"github.com/thepeterstone/claudomator/internal/event"
+	"github.com/thepeterstone/claudomator/internal/role"
+	"github.com/thepeterstone/claudomator/internal/storage"
+	"github.com/thepeterstone/claudomator/internal/task"
+)
+
+// Store is the subset of storage.DB methods the Scheduler needs.
+type Store interface {
+	ListTasks(filter storage.TaskFilter) ([]*task.Task, error)
+	ListExecutions(taskID string) ([]*storage.Execution, error)
+	GetActiveRoleConfig(role string) (*storage.RoleConfigRow, error)
+	UpdateTaskAgent(id string, agent task.AgentConfig) error
+	UpdateTaskState(id string, newState task.State) error
+	CreateEvent(e *event.Event) error
+}
+
+// Pool is the subset of *executor.Pool the Scheduler needs. Satisfied by
+// *executor.Pool directly (see internal/cli/serve.go); declared as an
+// interface here purely so tests can supply a fake without dragging in the
+// executor package's runner/sandbox machinery.
+type Pool interface {
+	Submit(ctx context.Context, t *task.Task) error
+}
+
+// BudgetGate reports whether an escalation to provider estimated at estCost
+// is allowed. Satisfied by *budget.Accountant.
+type BudgetGate interface {
+	Allow(provider string, estCost float64) (bool, error)
+}
+
+// Scheduler polls for role-typed FAILED tasks and retries or escalates them
+// per their active role_configs escalation ladder.
+type Scheduler struct {
+	Store  Store
+	Pool   Pool
+	Budget BudgetGate // nil means "no budget gating" (always allow)
+	Logger *slog.Logger
+
+	// handled dedupes processing within a single running process: once a
+	// decision (retry/escalate/decline) has been made for a given
+	// execution ID, it is never reconsidered again by this Scheduler
+	// instance. This is what keeps Run's poll loop convergent — a task left
+	// FAILED after its ladder is exhausted (or an escalation is budget-
+	// denied) has the same "latest execution" on every subsequent tick, so
+	// without this it would emit a fresh "final" KindEscalated event, and
+	// re-run the same decision, every single poll forever. A restart clears
+	// this map, so a task can be reconsidered once more after a restart —
+	// intentional: it's an idempotent bookkeeping decision, not orchestration
+	// state, so re-deriving it once is harmless.
+	mu      sync.Mutex
+	handled map[string]bool
+}
+
+// DefaultPollInterval is used by Run when pollInterval <= 0.
+const DefaultPollInterval = 30 * time.Second
+
+// Run polls for role-typed FAILED tasks every pollInterval until ctx is
+// cancelled.
+func (s *Scheduler) Run(ctx context.Context, pollInterval time.Duration) {
+	if pollInterval <= 0 {
+		pollInterval = DefaultPollInterval
+	}
+	ticker := time.NewTicker(pollInterval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			s.Tick(ctx)
+		}
+	}
+}
+
+// Tick runs a single poll pass. Exported so tests can drive it directly
+// without waiting on a ticker.
+func (s *Scheduler) Tick(ctx context.Context) {
+	tasks, err := s.Store.ListTasks(storage.TaskFilter{State: task.StateFailed})
+	if err != nil {
+		s.logf("scheduler: list failed tasks", "error", err)
+		return
+	}
+	for _, t := range tasks {
+		if t.Agent.Role == "" {
+			continue
+		}
+		s.processTask(ctx, t)
+	}
+}
+
+func (s *Scheduler) logf(msg string, args ...any) {
+	if s.Logger != nil {
+		s.Logger.Warn(msg, args...)
+	}
+}
+
+func (s *Scheduler) markHandled(execID string) bool {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.handled == nil {
+		s.handled = make(map[string]bool)
+	}
+	if s.handled[execID] {
+		return false
+	}
+	s.handled[execID] = true
+	return true
+}
+
+func (s *Scheduler) processTask(ctx context.Context, t *task.Task) {
+	execs, err := s.Store.ListExecutions(t.ID)
+	if err != nil || len(execs) == 0 {
+		return
+	}
+	latest := execs[0] // ListExecutions orders DESC by start_time.
+	if latest.Status != "FAILED" {
+		return
+	}
+	if !s.markHandled(latest.ID) {
+		return // already decided for this execution — converged, nothing to do.
+	}
+
+	row, err := s.Store.GetActiveRoleConfig(t.Agent.Role)
+	if err != nil {
+		s.logf("scheduler: no active role config for role", "role", t.Agent.Role, "taskID", t.ID, "error", err)
+		return
+	}
+	var rc role.RoleConfig
+	if err := json.Unmarshal([]byte(row.ConfigJSON), &rc); err != nil {
+		s.logf("scheduler: decode role config", "role", t.Agent.Role, "taskID", t.ID, "error", err)
+		return
+	}
+	if len(rc.EscalationLadder) == 0 {
+		return
+	}
+
+	currentRung := latest.EscalationRung
+	if currentRung < 0 {
+		currentRung = 0
+	}
+	if currentRung >= len(rc.EscalationLadder) {
+		// Ladder already exhausted (e.g. the ladder was shortened after this
+		// task started climbing it) — nothing more to do.
+		return
+	}
+	tier := rc.EscalationLadder[currentRung]
+	attempts := attemptsAtRung(execs, currentRung)
+
+	if attempts < tier.MaxRetries {
+		s.retrySameRung(ctx, t, currentRung)
+		return
+	}
+
+	nextRung := currentRung + 1
+	if nextRung >= len(rc.EscalationLadder) || len(rc.EscalationLadder[nextRung].Candidates) == 0 {
+		s.decline(ctx, t, currentRung, "", "escalation ladder exhausted")
+		return
+	}
+	nextTier := rc.EscalationLadder[nextRung]
+	target := nextTier.Candidates[0]
+
+	estCost := rc.DefaultBudgetUSD
+	if estCost <= 0 {
+		estCost = t.Agent.MaxBudgetUSD
+	}
+	allowed := true
+	if s.Budget != nil {
+		var berr error
+		allowed, berr = s.Budget.Allow(target.Provider, estCost)
+		if berr != nil {
+			s.logf("scheduler: budget check failed; declining escalation", "taskID", t.ID, "error", berr)
+			allowed = false
+		}
+	}
+	if !allowed {
+		s.decline(ctx, t, currentRung, target.Provider, "budget denied")
+		return
+	}
+	s.escalate(ctx, t, currentRung, nextRung, target)
+}
+
+// attemptsAtRung counts how many executions, starting from the most recent
+// (execs[0]) and moving backward, ran at rung consecutively — i.e. how many
+// attempts have already been made at the task's current tier since it last
+// moved to (or started at) that tier.
+func attemptsAtRung(execs []*storage.Execution, rung int) int {
+	n := 0
+	for _, e := range execs {
+		if e.EscalationRung != rung {
+			break
+		}
+		n++
+	}
+	return n
+}
+
+func (s *Scheduler) retrySameRung(ctx context.Context, t *task.Task, rung int) {
+	if err := s.Store.UpdateTaskState(t.ID, task.StateQueued); err != nil {
+		s.logf("scheduler: retry: update task state", "taskID", t.ID, "error", err)
+		return
+	}
+	resubmit := *t
+	resubmit.State = task.StateQueued
+	if err := s.Pool.Submit(ctx, &resubmit); err != nil {
+		s.logf("scheduler: retry: submit", "taskID", t.ID, "error", err)
+	}
+}
+
+func (s *Scheduler) escalate(ctx context.Context, t *task.Task, fromRung, toRung int, target role.Rung) {
+	fromProvider := t.Agent.Type
+	newAgent := t.Agent
+	newAgent.Type = target.Provider
+	newAgent.Model = target.Model
+	if err := s.Store.UpdateTaskAgent(t.ID, newAgent); err != nil {
+		s.logf("scheduler: escalate: update task agent", "taskID", t.ID, "error", err)
+		return
+	}
+	if err := s.Store.UpdateTaskState(t.ID, task.StateQueued); err != nil {
+		s.logf("scheduler: escalate: update task state", "taskID", t.ID, "error", err)
+		return
+	}
+	s.emitEscalated(t.ID, fromRung, toRung, fromProvider, target.Provider, false, "")
+
+	resubmit := *t
+	resubmit.Agent = newAgent
+	resubmit.State = task.StateQueued
+	if err := s.Pool.Submit(ctx, &resubmit); err != nil {
+		s.logf("scheduler: escalate: submit", "taskID", t.ID, "error", err)
+	}
+}
+
+// decline records that no further escalation will happen for t right now
+// (ladder exhausted or budget denied) and leaves it FAILED for human
+// attention.
+func (s *Scheduler) decline(ctx context.Context, t *task.Task, atRung int, consideredProvider, reason string) {
+	s.emitEscalated(t.ID, atRung, atRung, t.Agent.Type, consideredProvider, true, reason)
+}
+
+func (s *Scheduler) emitEscalated(taskID string, fromRung, toRung int, fromProvider, toProvider string, final bool, reason string) {
+	payload, _ := json.Marshal(struct {
+		FromRung     int    `json:"from_rung"`
+		ToRung       int    `json:"to_rung"`
+		FromProvider string `json:"from_provider"`
+		ToProvider   string `json:"to_provider,omitempty"`
+		Final        bool   `json:"final"`
+		Reason       string `json:"reason,omitempty"`
+	}{FromRung: fromRung, ToRung: toRung, FromProvider: fromProvider, ToProvider: toProvider, Final: final, Reason: reason})
+	if err := s.Store.CreateEvent(&event.Event{
+		TaskID:  taskID,
+		Kind:    event.KindEscalated,
+		Actor:   event.ActorSystem,
+		Payload: payload,
+	}); err != nil {
+		s.logf("scheduler: emit escalated event", "taskID", taskID, "error", err)
+	}
+}