feat(role): add versioned role configs + escalation ladder + scheduler (Phase 5)

Two parts:

Part A (fixes a gap from Phase 1): Groq/OpenRouter/OpenAI were documented in
docs/api-keys-setup.md as usable once configured, but nothing actually
constructed runners for them. internal/cli/cloudrunners.go consolidates
anthropic/google/groq/openrouter/openai NativeRunner construction into one
table-driven registerCloudRunners() helper, replacing the two hand-written
per-provider blocks in serve.go/run.go. Groq/OpenRouter/OpenAI reuse
openaicompat (no new adapter code) at SandboxKind: "docker".

Part B: the token-husbanding harness's core routing mechanism.

- internal/role: RoleConfig/Tier/Rung -- a role's system prompt and a
  multi-tier (provider, model) escalation ladder, versioned via config_json.
- storage: new role_configs table (draft/active/retired, UNIQUE(role,
  version)) with transactional activate-retires-prior-active semantics;
  new executions.escalation_rung column.
- task.AgentConfig.Role string -- purely additive; every existing task shape
  (Agent.Role == "") is unaffected, proven by TestPool_Execute_NonRoleTask_Unaffected
  plus the full pre-existing suite passing unchanged.
- executor.Pool.execute(): role-typed tasks with no Agent.Type yet resolve
  tier 0 of their active ladder (round-robin across multi-candidate tiers,
  skipping rate-limited providers, falling back to soonest-clearing) before
  the existing pickAgent/Classifier path runs; SystemPrompt applies to
  Agent.SystemPromptAppend. Already-resolved role tasks (scheduler resubmits)
  get their escalation_rung re-derived read-only via findTierIndex.
- internal/scheduler: polls role-typed FAILED tasks, retries at the same
  rung under MaxRetries or escalates to the next tier's first candidate when
  budget.Accountant.Allow() permits (emitting event.KindEscalated), else
  leaves the task FAILED with a final:true KindEscalated event. An
  in-memory per-execution-ID "handled" set keeps the poll loop convergent.
  Started by `serve` only, config knob [scheduler].poll_interval_seconds.
- internal/api: POST/GET /api/roles/{role}/versions, POST
  /api/roles/{role}/activate -- unauthenticated, matching the existing
  projects/tasks REST endpoints' auth posture (only chatbot MCP, agent MCP,
  and WebSocket are api_token-gated in this codebase today).

Documented as stored-but-not-yet-enforced (CLAUDE.md Design Debt, matching
how task.Priority/RetryConfig are already documented): RoleConfig.Tools/
SandboxKind don't affect dispatch yet; DefaultBudgetUSD is read narrowly as
the scheduler's escalation cost estimate, not enforced at initial dispatch;
scheduler escalation always targets Candidates[0] (no round-robin, unlike
initial-dispatch tier-0 resolution); the scheduler's dedupe is per-process
and resets on restart (idempotent, harmless).

go build/vet/test -race -count=1 all pass, 21 packages.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01V1moSNCJRcP6kykA4tyUSs

3 files changed, 333 insertions, 10 deletions

diff --git a/internal/storage/db.go b/internal/storage/db.go
index 22a3d7b..a16ad4e 100644
--- a/internal/storage/db.go
+++ b/internal/storage/db.go
@@ -140,6 +140,20 @@ func (s *DB) migrate() error {
 		`CREATE INDEX IF NOT EXISTS idx_events_task_id_seq ON events(task_id, seq)`,
 		`CREATE INDEX IF NOT EXISTS idx_events_ts ON events(ts)`,
 		`CREATE INDEX IF NOT EXISTS idx_events_kind ON events(kind)`,
+		`ALTER TABLE executions ADD COLUMN escalation_rung INTEGER NOT NULL DEFAULT 0`,
+		`CREATE TABLE IF NOT EXISTS role_configs (
+			id TEXT PRIMARY KEY,
+			role TEXT NOT NULL,
+			version INTEGER NOT NULL,
+			status TEXT NOT NULL DEFAULT 'draft',
+			config_json TEXT NOT NULL,
+			created_at DATETIME NOT NULL,
+			activated_at DATETIME,
+			retired_at DATETIME,
+			proposed_by TEXT,
+			UNIQUE(role, version)
+		)`,
+		`CREATE INDEX IF NOT EXISTS idx_role_configs_role_status ON role_configs(role, status)`,
 	}
 	for _, m := range migrations {
 		if _, err := s.db.Exec(m); err != nil {
@@ -522,6 +536,15 @@ type Execution struct {
 	SessionID  string // claude --session-id; persisted for resume
 	SandboxDir string // preserved sandbox path when task is BLOCKED; resume must run here
 	Agent      string // provider that ran this execution (claude/gemini/local); for budget accounting
+	// EscalationRung is the 0-based index into the active role's
+	// EscalationLadder this execution ran at, for role-typed tasks
+	// (task.AgentConfig.Role != ""). 0 (the default) for non-role tasks and
+	// for role-typed executions where the resolved Agent.Type/Model didn't
+	// match any tier in the ladder (e.g. the budget gate rerouted to
+	// "local"). Set once at execution-creation time by
+	// internal/executor.Pool.execute(); internal/scheduler reads it back to
+	// know which tier to retry/escalate from.
+	EscalationRung int
 
 	Changestats *task.Changestats // stored as JSON; nil if not yet recorded
 	Commits     []task.GitCommit // stored as JSON; empty if no commits
@@ -570,10 +593,10 @@ func (s *DB) CreateExecutionAndSetRunning(e *Execution) error {
 		commitsJSON = string(b)
 	}
 	if _, err := tx.Exec(`
-		INSERT INTO executions (id, task_id, start_time, end_time, exit_code, status, stdout_path, stderr_path, artifact_dir, cost_usd, error_msg, session_id, sandbox_dir, changestats_json, commits_json, agent)
-		VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, ?)`,
+		INSERT INTO executions (id, task_id, start_time, end_time, exit_code, status, stdout_path, stderr_path, artifact_dir, cost_usd, error_msg, session_id, sandbox_dir, changestats_json, commits_json, agent, escalation_rung)
+		VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, ?, ?)`,
 		e.ID, e.TaskID, e.StartTime.UTC(), e.EndTime.UTC(), e.ExitCode, e.Status,
-		e.StdoutPath, e.StderrPath, e.ArtifactDir, e.CostUSD, e.ErrorMsg, e.SessionID, e.SandboxDir, commitsJSON, e.Agent,
+		e.StdoutPath, e.StderrPath, e.ArtifactDir, e.CostUSD, e.ErrorMsg, e.SessionID, e.SandboxDir, commitsJSON, e.Agent, e.EscalationRung,
 	); err != nil {
 		return err
 	}
@@ -633,23 +656,23 @@ func (s *DB) CreateExecution(e *Execution) error {
 		commitsJSON = string(b)
 	}
 	_, err := s.db.Exec(`
-		INSERT INTO executions (id, task_id, start_time, end_time, exit_code, status, stdout_path, stderr_path, artifact_dir, cost_usd, error_msg, session_id, sandbox_dir, changestats_json, commits_json, tokens_in, tokens_out, agent)
-		VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+		INSERT INTO executions (id, task_id, start_time, end_time, exit_code, status, stdout_path, stderr_path, artifact_dir, cost_usd, error_msg, session_id, sandbox_dir, changestats_json, commits_json, tokens_in, tokens_out, agent, escalation_rung)
+		VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
 		e.ID, e.TaskID, e.StartTime.UTC(), e.EndTime.UTC(), e.ExitCode, e.Status,
-		e.StdoutPath, e.StderrPath, e.ArtifactDir, e.CostUSD, e.ErrorMsg, e.SessionID, e.SandboxDir, changestatsJSON, commitsJSON, e.TokensIn, e.TokensOut, e.Agent,
+		e.StdoutPath, e.StderrPath, e.ArtifactDir, e.CostUSD, e.ErrorMsg, e.SessionID, e.SandboxDir, changestatsJSON, commitsJSON, e.TokensIn, e.TokensOut, e.Agent, e.EscalationRung,
 	)
 	return err
 }
 
 // GetExecution retrieves an execution by ID.
 func (s *DB) GetExecution(id string) (*Execution, error) {
-	row := s.db.QueryRow(`SELECT id, task_id, start_time, end_time, exit_code, status, stdout_path, stderr_path, artifact_dir, cost_usd, error_msg, session_id, sandbox_dir, changestats_json, commits_json, tokens_in, tokens_out, agent FROM executions WHERE id = ?`, id)
+	row := s.db.QueryRow(`SELECT id, task_id, start_time, end_time, exit_code, status, stdout_path, stderr_path, artifact_dir, cost_usd, error_msg, session_id, sandbox_dir, changestats_json, commits_json, tokens_in, tokens_out, agent, escalation_rung FROM executions WHERE id = ?`, id)
 	return scanExecution(row)
 }
 
 // ListExecutions returns executions for a task.
 func (s *DB) ListExecutions(taskID string) ([]*Execution, error) {
-	rows, err := s.db.Query(`SELECT id, task_id, start_time, end_time, exit_code, status, stdout_path, stderr_path, artifact_dir, cost_usd, error_msg, session_id, sandbox_dir, changestats_json, commits_json, tokens_in, tokens_out, agent FROM executions WHERE task_id = ? ORDER BY start_time DESC`, taskID)
+	rows, err := s.db.Query(`SELECT id, task_id, start_time, end_time, exit_code, status, stdout_path, stderr_path, artifact_dir, cost_usd, error_msg, session_id, sandbox_dir, changestats_json, commits_json, tokens_in, tokens_out, agent, escalation_rung FROM executions WHERE task_id = ? ORDER BY start_time DESC`, taskID)
 	if err != nil {
 		return nil, err
 	}
@@ -668,7 +691,7 @@ func (s *DB) ListExecutions(taskID string) ([]*Execution, error) {
 
 // GetLatestExecution returns the most recent execution for a task.
 func (s *DB) GetLatestExecution(taskID string) (*Execution, error) {
-	row := s.db.QueryRow(`SELECT id, task_id, start_time, end_time, exit_code, status, stdout_path, stderr_path, artifact_dir, cost_usd, error_msg, session_id, sandbox_dir, changestats_json, commits_json, tokens_in, tokens_out, agent FROM executions WHERE task_id = ? ORDER BY start_time DESC LIMIT 1`, taskID)
+	row := s.db.QueryRow(`SELECT id, task_id, start_time, end_time, exit_code, status, stdout_path, stderr_path, artifact_dir, cost_usd, error_msg, session_id, sandbox_dir, changestats_json, commits_json, tokens_in, tokens_out, agent, escalation_rung FROM executions WHERE task_id = ? ORDER BY start_time DESC LIMIT 1`, taskID)
 	return scanExecution(row)
 }
 
@@ -1123,11 +1146,13 @@ func scanExecution(row scanner) (*Execution, error) {
 	var tokensIn sql.NullInt64
 	var tokensOut sql.NullInt64
 	var agent sql.NullString
+	var escalationRung sql.NullInt64
 	err := row.Scan(&e.ID, &e.TaskID, &e.StartTime, &e.EndTime, &e.ExitCode, &e.Status,
-		&e.StdoutPath, &e.StderrPath, &e.ArtifactDir, &e.CostUSD, &e.ErrorMsg, &sessionID, &sandboxDir, &changestatsJSON, &commitsJSON, &tokensIn, &tokensOut, &agent)
+		&e.StdoutPath, &e.StderrPath, &e.ArtifactDir, &e.CostUSD, &e.ErrorMsg, &sessionID, &sandboxDir, &changestatsJSON, &commitsJSON, &tokensIn, &tokensOut, &agent, &escalationRung)
 	if err != nil {
 		return nil, err
 	}
+	e.EscalationRung = int(escalationRung.Int64)
 	e.SessionID = sessionID.String
 	e.SandboxDir = sandboxDir.String
 	e.TokensIn = tokensIn.Int64
diff --git a/internal/storage/roleconfig.go b/internal/storage/roleconfig.go
new file mode 100644
index 0000000..6ae043b
--- /dev/null
+++ b/internal/storage/roleconfig.go
@@ -0,0 +1,147 @@
+package storage
+
+import (
+	"database/sql"
+	"fmt"
+	"time"
+
+	"github.com/google/uuid"
+)
+
+// RoleConfigRow is one versioned row in the role_configs table. ConfigJSON
+// holds the raw JSON-encoded role.RoleConfig; storage intentionally does not
+// import internal/role (no need to — it never inspects the payload, only
+// stores/retrieves it), so callers (internal/executor, internal/scheduler,
+// internal/api) decode it themselves via encoding/json.
+type RoleConfigRow struct {
+	ID          string
+	Role        string
+	Version     int
+	Status      string // "draft" | "active" | "retired"
+	ConfigJSON  string
+	CreatedAt   time.Time
+	ActivatedAt *time.Time
+	RetiredAt   *time.Time
+	ProposedBy  string
+}
+
+// CreateRoleConfig inserts a new draft version for role, auto-assigning the
+// next version number for that role (1 if none exist yet).
+func (s *DB) CreateRoleConfig(roleName, configJSON, proposedBy string) (*RoleConfigRow, error) {
+	tx, err := s.db.Begin()
+	if err != nil {
+		return nil, err
+	}
+	defer tx.Rollback() //nolint:errcheck
+
+	var maxVersion sql.NullInt64
+	if err := tx.QueryRow(`SELECT MAX(version) FROM role_configs WHERE role = ?`, roleName).Scan(&maxVersion); err != nil {
+		return nil, err
+	}
+	version := int(maxVersion.Int64) + 1
+
+	id := uuid.NewString()
+	now := time.Now().UTC()
+	if _, err := tx.Exec(`
+		INSERT INTO role_configs (id, role, version, status, config_json, created_at, proposed_by)
+		VALUES (?, ?, ?, 'draft', ?, ?, ?)`,
+		id, roleName, version, configJSON, now, proposedBy,
+	); err != nil {
+		return nil, err
+	}
+	if err := tx.Commit(); err != nil {
+		return nil, err
+	}
+
+	return &RoleConfigRow{
+		ID:         id,
+		Role:       roleName,
+		Version:    version,
+		Status:     "draft",
+		ConfigJSON: configJSON,
+		CreatedAt:  now,
+		ProposedBy: proposedBy,
+	}, nil
+}
+
+// GetActiveRoleConfig returns the currently active role_configs row for
+// role. Returns sql.ErrNoRows (unwrapped, matching GetTask/GetProject
+// convention in this package) if no version is active.
+func (s *DB) GetActiveRoleConfig(roleName string) (*RoleConfigRow, error) {
+	row := s.db.QueryRow(`
+		SELECT id, role, version, status, config_json, created_at, activated_at, retired_at, proposed_by
+		FROM role_configs WHERE role = ? AND status = 'active' LIMIT 1`, roleName)
+	return scanRoleConfigRow(row)
+}
+
+// ListRoleConfigVersions returns all versions for role, oldest first.
+func (s *DB) ListRoleConfigVersions(roleName string) ([]*RoleConfigRow, error) {
+	rows, err := s.db.Query(`
+		SELECT id, role, version, status, config_json, created_at, activated_at, retired_at, proposed_by
+		FROM role_configs WHERE role = ? ORDER BY version ASC`, roleName)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var out []*RoleConfigRow
+	for rows.Next() {
+		r, err := scanRoleConfigRow(rows)
+		if err != nil {
+			return nil, err
+		}
+		out = append(out, r)
+	}
+	return out, rows.Err()
+}
+
+// ActivateRoleConfigVersion promotes the given version of role to active,
+// atomically retiring whatever version is currently active for that role (if
+// any) in the same transaction. This enforces "at most one active row per
+// role" without a DB-level constraint, the same way UpdateTaskState enforces
+// the task state machine in a transaction rather than in schema.
+func (s *DB) ActivateRoleConfigVersion(roleName string, version int) error {
+	tx, err := s.db.Begin()
+	if err != nil {
+		return err
+	}
+	defer tx.Rollback() //nolint:errcheck
+
+	var exists int
+	if err := tx.QueryRow(`SELECT COUNT(*) FROM role_configs WHERE role = ? AND version = ?`, roleName, version).Scan(&exists); err != nil {
+		return err
+	}
+	if exists == 0 {
+		return fmt.Errorf("role %q version %d not found", roleName, version)
+	}
+
+	now := time.Now().UTC()
+	if _, err := tx.Exec(`UPDATE role_configs SET status = 'retired', retired_at = ? WHERE role = ? AND status = 'active'`, now, roleName); err != nil {
+		return err
+	}
+	if _, err := tx.Exec(`UPDATE role_configs SET status = 'active', activated_at = ? WHERE role = ? AND version = ?`, now, roleName, version); err != nil {
+		return err
+	}
+	return tx.Commit()
+}
+
+func scanRoleConfigRow(row scanner) (*RoleConfigRow, error) {
+	var r RoleConfigRow
+	var createdAt time.Time
+	var activatedAt, retiredAt sql.NullTime
+	var proposedBy sql.NullString
+	if err := row.Scan(&r.ID, &r.Role, &r.Version, &r.Status, &r.ConfigJSON, &createdAt, &activatedAt, &retiredAt, &proposedBy); err != nil {
+		return nil, err
+	}
+	r.CreatedAt = createdAt
+	if activatedAt.Valid {
+		t := activatedAt.Time
+		r.ActivatedAt = &t
+	}
+	if retiredAt.Valid {
+		t := retiredAt.Time
+		r.RetiredAt = &t
+	}
+	r.ProposedBy = proposedBy.String
+	return &r, nil
+}
diff --git a/internal/storage/roleconfig_test.go b/internal/storage/roleconfig_test.go
new file mode 100644
index 0000000..b18eaef
--- /dev/null
+++ b/internal/storage/roleconfig_test.go
@@ -0,0 +1,151 @@
+package storage
+
+import (
+	"database/sql"
+	"errors"
+	"testing"
+)
+
+func TestCreateRoleConfig_AssignsIncrementingVersions(t *testing.T) {
+	db := testDB(t)
+
+	v1, err := db.CreateRoleConfig("coder", `{"role":"coder"}`, "human")
+	if err != nil {
+		t.Fatalf("CreateRoleConfig v1: %v", err)
+	}
+	if v1.Version != 1 {
+		t.Errorf("expected version 1, got %d", v1.Version)
+	}
+	if v1.Status != "draft" {
+		t.Errorf("expected status draft, got %q", v1.Status)
+	}
+
+	v2, err := db.CreateRoleConfig("coder", `{"role":"coder","system_prompt":"v2"}`, "human")
+	if err != nil {
+		t.Fatalf("CreateRoleConfig v2: %v", err)
+	}
+	if v2.Version != 2 {
+		t.Errorf("expected version 2, got %d", v2.Version)
+	}
+
+	// A different role starts its own version sequence at 1.
+	other, err := db.CreateRoleConfig("reviewer", `{"role":"reviewer"}`, "human")
+	if err != nil {
+		t.Fatalf("CreateRoleConfig other role: %v", err)
+	}
+	if other.Version != 1 {
+		t.Errorf("expected version 1 for a new role, got %d", other.Version)
+	}
+}
+
+func TestGetActiveRoleConfig_NoneActive_ReturnsErrNoRows(t *testing.T) {
+	db := testDB(t)
+	if _, err := db.CreateRoleConfig("coder", `{"role":"coder"}`, "human"); err != nil {
+		t.Fatalf("CreateRoleConfig: %v", err)
+	}
+	_, err := db.GetActiveRoleConfig("coder")
+	if !errors.Is(err, sql.ErrNoRows) {
+		t.Fatalf("expected sql.ErrNoRows, got %v", err)
+	}
+}
+
+func TestListRoleConfigVersions_OrderedByVersion(t *testing.T) {
+	db := testDB(t)
+	for i := 0; i < 3; i++ {
+		if _, err := db.CreateRoleConfig("coder", `{"role":"coder"}`, "human"); err != nil {
+			t.Fatalf("CreateRoleConfig: %v", err)
+		}
+	}
+	versions, err := db.ListRoleConfigVersions("coder")
+	if err != nil {
+		t.Fatalf("ListRoleConfigVersions: %v", err)
+	}
+	if len(versions) != 3 {
+		t.Fatalf("expected 3 versions, got %d", len(versions))
+	}
+	for i, v := range versions {
+		if v.Version != i+1 {
+			t.Errorf("versions[%d].Version = %d, want %d", i, v.Version, i+1)
+		}
+	}
+}
+
+// TestActivateRoleConfigVersion_ExactlyOneActive is the invariant test called
+// out in the phase spec: activating version 2 while version 1 is active must
+// atomically flip version 1 to retired.
+func TestActivateRoleConfigVersion_ExactlyOneActive(t *testing.T) {
+	db := testDB(t)
+
+	v1, err := db.CreateRoleConfig("coder", `{"role":"coder","system_prompt":"v1"}`, "human")
+	if err != nil {
+		t.Fatalf("CreateRoleConfig v1: %v", err)
+	}
+	v2, err := db.CreateRoleConfig("coder", `{"role":"coder","system_prompt":"v2"}`, "human")
+	if err != nil {
+		t.Fatalf("CreateRoleConfig v2: %v", err)
+	}
+
+	if err := db.ActivateRoleConfigVersion("coder", v1.Version); err != nil {
+		t.Fatalf("activate v1: %v", err)
+	}
+	active, err := db.GetActiveRoleConfig("coder")
+	if err != nil {
+		t.Fatalf("GetActiveRoleConfig: %v", err)
+	}
+	if active.Version != v1.Version {
+		t.Fatalf("expected active version %d, got %d", v1.Version, active.Version)
+	}
+	if active.ActivatedAt == nil {
+		t.Errorf("expected ActivatedAt to be set")
+	}
+
+	// Now activate v2 — v1 must atomically flip to retired, and v2 becomes
+	// the sole active row.
+	if err := db.ActivateRoleConfigVersion("coder", v2.Version); err != nil {
+		t.Fatalf("activate v2: %v", err)
+	}
+
+	active, err = db.GetActiveRoleConfig("coder")
+	if err != nil {
+		t.Fatalf("GetActiveRoleConfig after v2 activation: %v", err)
+	}
+	if active.Version != v2.Version {
+		t.Fatalf("expected active version %d, got %d", v2.Version, active.Version)
+	}
+
+	all, err := db.ListRoleConfigVersions("coder")
+	if err != nil {
+		t.Fatalf("ListRoleConfigVersions: %v", err)
+	}
+	activeCount := 0
+	for _, row := range all {
+		if row.Status == "active" {
+			activeCount++
+		}
+		if row.Version == v1.Version {
+			if row.Status != "retired" {
+				t.Errorf("expected v1 to be retired, got %q", row.Status)
+			}
+			if row.RetiredAt == nil {
+				t.Errorf("expected v1 RetiredAt to be set")
+			}
+		}
+	}
+	if activeCount != 1 {
+		t.Fatalf("expected exactly 1 active row, got %d", activeCount)
+	}
+}
+
+func TestActivateRoleConfigVersion_UnknownVersion_Errors(t *testing.T) {
+	db := testDB(t)
+	if _, err := db.CreateRoleConfig("coder", `{"role":"coder"}`, "human"); err != nil {
+		t.Fatalf("CreateRoleConfig: %v", err)
+	}
+	if err := db.ActivateRoleConfigVersion("coder", 99); err == nil {
+		t.Fatalf("expected error activating unknown version")
+	}
+	// No row should have been left active.
+	if _, err := db.GetActiveRoleConfig("coder"); !errors.Is(err, sql.ErrNoRows) {
+		t.Fatalf("expected sql.ErrNoRows, got %v", err)
+	}
+}