path: root/internal/scheduler/scheduler.go

// Package scheduler implements a first-pass retry-then-escalate loop for
// role-typed tasks (task.AgentConfig.Role != ""). On a poll interval it looks
// for tasks whose most recent execution ended FAILED, and either resubmits
// them at the same escalation-ladder tier (if under that tier's MaxRetries)
// or escalates them to the next tier (if the budget allows) — recording an
// event.KindEscalated event either way. If the ladder is exhausted or the
// budget denies the escalation, the task is left FAILED for human attention.
//
// Phase 7c extended this same "watch for stuck role-typed tasks and take
// escalation action" responsibility to a second trigger: a role-typed task
// BLOCKED on an ask_user question nobody has answered within
// SchedulerConfig.AskUserTimeoutSeconds (see tickAskUserTimeouts below). This
// is a natural extension of what Scheduler already does, not a new
// component — it owns retry/escalation for role-typed tasks generally, and a
// stuck question is just another way a role-typed task gets stuck.
//
// Explicit non-goal still remaining: no DAG/cascade-fail logic here (that's
// executor.Pool.cascadeFail's job). Handling for TIMED_OUT/CANCELLED/
// BUDGET_EXCEEDED tasks follows the same shape as FAILED but isn't
// implemented yet — only FAILED is polled for the failure-retry path.
package scheduler

import (
	"context"
	"encoding/json"
	"log/slog"
	"sync"
	"time"

	"github.com/google/uuid"
	"github.com/thepeterstone/claudomator/internal/event"
	"github.com/thepeterstone/claudomator/internal/role"
	"github.com/thepeterstone/claudomator/internal/storage"
	"github.com/thepeterstone/claudomator/internal/task"
)

// Store is the subset of storage.DB methods the Scheduler needs.
type Store interface {
	ListTasks(filter storage.TaskFilter) ([]*task.Task, error)
	ListExecutions(taskID string) ([]*storage.Execution, error)
	GetActiveRoleConfig(role string) (*storage.RoleConfigRow, error)
	UpdateTaskAgent(id string, agent task.AgentConfig) error
	UpdateTaskState(id string, newState task.State) error
	CreateEvent(e *event.Event) error
	// UpdateTaskQuestion, AppendTaskInteraction, and UpdateTaskNeedsReview
	// back tickAskUserTimeouts' resume-with-fallback-answer flow (Phase 7c):
	// clearing the stale question, recording the system-authored answer as an
	// interaction (mirroring api.answerTaskQuestion's audit trail for a real
	// human answer), and flagging the task for later human review.
	UpdateTaskQuestion(taskID, questionJSON string) error
	AppendTaskInteraction(taskID string, interaction task.Interaction) error
	UpdateTaskNeedsReview(id string, needsReview bool) error
}

// Pool is the subset of *executor.Pool the Scheduler needs. Satisfied by
// *executor.Pool directly (see internal/cli/serve.go); declared as an
// interface here purely so tests can supply a fake without dragging in the
// executor package's runner/sandbox machinery.
type Pool interface {
	Submit(ctx context.Context, t *task.Task) error
	// SubmitResume re-queues a BLOCKED (or otherwise interrupted) task using
	// a resume execution carrying ResumeSessionID/ResumeAnswer. Used by
	// tickAskUserTimeouts to resume a task with a system-authored fallback
	// answer the same way api.answerTaskQuestion resumes it with a real one.
	SubmitResume(ctx context.Context, t *task.Task, exec *storage.Execution) error
}

// BudgetGate reports whether an escalation to provider estimated at estCost
// is allowed. Satisfied by *budget.Accountant.
type BudgetGate interface {
	Allow(provider string, estCost float64) (bool, error)
}

// Scheduler polls for role-typed FAILED tasks and retries or escalates them
// per their active role_configs escalation ladder. It also polls for
// role-typed tasks BLOCKED on a stale ask_user question (see
// tickAskUserTimeouts).
type Scheduler struct {
	Store  Store
	Pool   Pool
	Budget BudgetGate // nil means "no budget gating" (always allow)
	Logger *slog.Logger

	// AskUserTimeout is how long a role-typed task may sit BLOCKED on an
	// unanswered ask_user question before tickAskUserTimeouts resumes it with
	// a system-authored fallback answer. <= 0 means DefaultAskUserTimeout
	// (see askUserTimeout()); set from config.SchedulerConfig.AskUserTimeout()
	// in production (internal/cli/serve.go).
	AskUserTimeout time.Duration

	// handled dedupes processing within a single running process: once a
	// decision (retry/escalate/decline) has been made for a given
	// execution ID, it is never reconsidered again by this Scheduler
	// instance. This is what keeps Run's poll loop convergent — a task left
	// FAILED after its ladder is exhausted (or an escalation is budget-
	// denied) has the same "latest execution" on every subsequent tick, so
	// without this it would emit a fresh "final" KindEscalated event, and
	// re-run the same decision, every single poll forever. A restart clears
	// this map, so a task can be reconsidered once more after a restart —
	// intentional: it's an idempotent bookkeeping decision, not orchestration
	// state, so re-deriving it once is harmless.
	//
	// tickAskUserTimeouts does NOT need an equivalent guard: successfully
	// resuming a task moves it out of BLOCKED (to QUEUED), which structurally
	// removes it from the next tick's BLOCKED query — the same
	// "idempotency via a state the next poll won't rediscover" pattern
	// StoryOrchestrator's ensureEvaluators/ensureArbitration use, rather than
	// an in-memory marker.
	mu      sync.Mutex
	handled map[string]bool
}

// DefaultPollInterval is used by Run when pollInterval <= 0.
const DefaultPollInterval = 30 * time.Second

// DefaultAskUserTimeout is used when Scheduler.AskUserTimeout <= 0. Mirrors
// config.SchedulerConfig.AskUserTimeout's default and reasoning (10 minutes:
// long enough for an actively-working human to notice and answer a
// clarification request, short enough that a role-typed task doesn't stall
// for hours on input that may never come — the common case for a
// self-hosted, typically single-operator deployment).
const DefaultAskUserTimeout = 10 * time.Minute

// askUserTimeout returns the effective ask-user timeout, defaulting to
// DefaultAskUserTimeout when unset.
func (s *Scheduler) askUserTimeout() time.Duration {
	if s.AskUserTimeout <= 0 {
		return DefaultAskUserTimeout
	}
	return s.AskUserTimeout
}

// fallbackAnswer is the system-authored answer injected into a role-typed
// task's ask_user question once it has gone unanswered for longer than
// askUserTimeout(). Clearly marked as a system fallback, not a real human
// answer, so anyone reading the task's interaction history or event stream
// later understands why the agent proceeded without a real decision.
const fallbackAnswer = "[auto-escalated: no human response within timeout; proceeding with best judgment]"

// Run polls for role-typed FAILED tasks every pollInterval until ctx is
// cancelled.
func (s *Scheduler) Run(ctx context.Context, pollInterval time.Duration) {
	if pollInterval <= 0 {
		pollInterval = DefaultPollInterval
	}
	ticker := time.NewTicker(pollInterval)
	defer ticker.Stop()
	for {
		select {
		case <-ctx.Done():
			return
		case <-ticker.C:
			s.Tick(ctx)
		}
	}
}

// Tick runs a single poll pass: FAILED-task retry/escalation, then
// ask_user-timeout escalation. Exported so tests can drive it directly
// without waiting on a ticker.
func (s *Scheduler) Tick(ctx context.Context) {
	tasks, err := s.Store.ListTasks(storage.TaskFilter{State: task.StateFailed})
	if err != nil {
		s.logf("scheduler: list failed tasks", "error", err)
	} else {
		for _, t := range tasks {
			if t.Agent.Role == "" {
				continue
			}
			s.processTask(ctx, t)
		}
	}
	s.tickAskUserTimeouts(ctx)
}

func (s *Scheduler) logf(msg string, args ...any) {
	if s.Logger != nil {
		s.Logger.Warn(msg, args...)
	}
}

func (s *Scheduler) markHandled(execID string) bool {
	s.mu.Lock()
	defer s.mu.Unlock()
	if s.handled == nil {
		s.handled = make(map[string]bool)
	}
	if s.handled[execID] {
		return false
	}
	s.handled[execID] = true
	return true
}

func (s *Scheduler) processTask(ctx context.Context, t *task.Task) {
	execs, err := s.Store.ListExecutions(t.ID)
	if err != nil || len(execs) == 0 {
		return
	}
	latest := execs[0] // ListExecutions orders DESC by start_time.
	if latest.Status != "FAILED" {
		return
	}
	if !s.markHandled(latest.ID) {
		return // already decided for this execution — converged, nothing to do.
	}

	currentRung := latest.EscalationRung
	if currentRung < 0 {
		currentRung = 0
	}

	row, err := s.Store.GetActiveRoleConfig(t.Agent.Role)
	if err != nil {
		s.retryWithoutLadder(ctx, t, execs, currentRung, "no active role config for role "+t.Agent.Role)
		return
	}
	var rc role.RoleConfig
	if err := json.Unmarshal([]byte(row.ConfigJSON), &rc); err != nil {
		s.retryWithoutLadder(ctx, t, execs, currentRung, "failed to decode role config")
		return
	}
	if len(rc.EscalationLadder) == 0 {
		s.retryWithoutLadder(ctx, t, execs, currentRung, "empty escalation ladder")
		return
	}

	if currentRung >= len(rc.EscalationLadder) {
		// Ladder already exhausted (e.g. the ladder was shortened after this
		// task started climbing it) — nothing more to do.
		return
	}
	tier := rc.EscalationLadder[currentRung]
	attempts := attemptsAtRung(execs, currentRung)

	if attempts < tier.MaxRetries {
		s.retrySameRung(ctx, t, currentRung)
		return
	}

	nextRung := currentRung + 1
	if nextRung >= len(rc.EscalationLadder) || len(rc.EscalationLadder[nextRung].Candidates) == 0 {
		s.decline(ctx, t, currentRung, "", "escalation ladder exhausted")
		return
	}
	nextTier := rc.EscalationLadder[nextRung]
	target := nextTier.Candidates[0]

	estCost := rc.DefaultBudgetUSD
	if estCost <= 0 {
		estCost = t.Agent.MaxBudgetUSD
	}
	allowed := true
	if s.Budget != nil {
		var berr error
		allowed, berr = s.Budget.Allow(target.Provider, estCost)
		if berr != nil {
			s.logf("scheduler: budget check failed; declining escalation", "taskID", t.ID, "error", berr)
			allowed = false
		}
	}
	if !allowed {
		s.decline(ctx, t, currentRung, target.Provider, "budget denied")
		return
	}
	s.escalate(ctx, t, currentRung, nextRung, target)
}

// attemptsAtRung counts how many executions, starting from the most recent
// (execs[0]) and moving backward, ran at rung consecutively — i.e. how many
// attempts have already been made at the task's current tier since it last
// moved to (or started at) that tier.
func attemptsAtRung(execs []*storage.Execution, rung int) int {
	n := 0
	for _, e := range execs {
		if e.EscalationRung != rung {
			break
		}
		n++
	}
	return n
}

func (s *Scheduler) retrySameRung(ctx context.Context, t *task.Task, rung int) {
	if err := s.Store.UpdateTaskState(t.ID, task.StateQueued); err != nil {
		s.logf("scheduler: retry: update task state", "taskID", t.ID, "error", err)
		return
	}
	resubmit := *t
	resubmit.State = task.StateQueued
	if err := s.Pool.Submit(ctx, &resubmit); err != nil {
		s.logf("scheduler: retry: submit", "taskID", t.ID, "error", err)
	}
}

// maxRetriesNoRoleConfig bounds retryWithoutLadder's same-tier retries when
// a role-typed task has no usable escalation ladder to consult -- a simple,
// hardcoded safety net (mirrors maxFixAttempts in
// internal/scheduler/story_orchestrator.go), not real ladder-based tiering.
const maxRetriesNoRoleConfig = 2

// retryWithoutLadder handles a role-typed task whose role has no usable
// escalation ladder to consult -- no active role_configs row at all, a
// corrupt one, or one with an empty ladder -- by retrying at the task's
// existing Agent.Type/Model (unchanged, since there's no ladder to pick a
// tier from) up to maxRetriesNoRoleConfig times before giving up. Mirrors
// escalateAskUserTimeout's own "no active role config... still resuming,
// just without a provider/model change" fallback below, applied here to the
// FAILED-task retry path instead of the ask_user-timeout path.
//
// Added 2026-07-11: before this, a role with no seeded role_configs (every
// role except builder/planner, as of this fix -- see
// internal/storage.SeedRoleConfigs) got zero retry safety net at all: the
// very first transient infra failure (a generic "container execution failed:
// exit status 1", the same symptom repeatedly traced to disk-full or other
// non-deterministic causes this session) left the task FAILED forever with
// nothing ever looking at it again -- a real production incident (see
// internal/scheduler/story_orchestrator.go's finalizeArbitration doc
// comment for the related arbitration-side half of this same root cause).
func (s *Scheduler) retryWithoutLadder(ctx context.Context, t *task.Task, execs []*storage.Execution, rung int, reason string) {
	attempts := attemptsAtRung(execs, rung)
	if attempts < maxRetriesNoRoleConfig {
		s.logf("scheduler: retrying without a usable escalation ladder", "role", t.Agent.Role, "taskID", t.ID, "reason", reason, "attempt", attempts+1, "maxAttempts", maxRetriesNoRoleConfig)
		s.retrySameRung(ctx, t, rung)
		return
	}
	s.decline(ctx, t, rung, "", reason+"; retries exhausted")
}

func (s *Scheduler) escalate(ctx context.Context, t *task.Task, fromRung, toRung int, target role.Rung) {
	fromProvider := t.Agent.Type
	newAgent := t.Agent
	newAgent.Type = target.Provider
	newAgent.Model = target.Model
	if err := s.Store.UpdateTaskAgent(t.ID, newAgent); err != nil {
		s.logf("scheduler: escalate: update task agent", "taskID", t.ID, "error", err)
		return
	}
	if err := s.Store.UpdateTaskState(t.ID, task.StateQueued); err != nil {
		s.logf("scheduler: escalate: update task state", "taskID", t.ID, "error", err)
		return
	}
	s.emitEscalated(t.ID, fromRung, toRung, fromProvider, target.Provider, false, "", "failure")

	resubmit := *t
	resubmit.Agent = newAgent
	resubmit.State = task.StateQueued
	if err := s.Pool.Submit(ctx, &resubmit); err != nil {
		s.logf("scheduler: escalate: submit", "taskID", t.ID, "error", err)
	}
}

// decline records that no further escalation will happen for t right now
// (ladder exhausted or budget denied) and leaves it FAILED for human
// attention.
func (s *Scheduler) decline(ctx context.Context, t *task.Task, atRung int, consideredProvider, reason string) {
	s.emitEscalated(t.ID, atRung, atRung, t.Agent.Type, consideredProvider, true, reason, "failure")
}

// emitEscalated records an event.KindEscalated event. trigger distinguishes
// what caused the scheduler to reconsider this task's tier: "failure" for
// the FAILED-task retry/escalation path above, "ask_user_timeout" for
// escalateAskUserTimeout below — so someone reading the event stream later
// can tell a stuck-question escalation from a stuck-failure one.
func (s *Scheduler) emitEscalated(taskID string, fromRung, toRung int, fromProvider, toProvider string, final bool, reason, trigger string) {
	payload, _ := json.Marshal(struct {
		FromRung     int    `json:"from_rung"`
		ToRung       int    `json:"to_rung"`
		FromProvider string `json:"from_provider"`
		ToProvider   string `json:"to_provider,omitempty"`
		Final        bool   `json:"final"`
		Reason       string `json:"reason,omitempty"`
		Trigger      string `json:"trigger,omitempty"`
	}{FromRung: fromRung, ToRung: toRung, FromProvider: fromProvider, ToProvider: toProvider, Final: final, Reason: reason, Trigger: trigger})
	if err := s.Store.CreateEvent(&event.Event{
		TaskID:  taskID,
		Kind:    event.KindEscalated,
		Actor:   event.ActorSystem,
		Payload: payload,
	}); err != nil {
		s.logf("scheduler: emit escalated event", "taskID", taskID, "error", err)
	}
}

// tickAskUserTimeouts finds role-typed tasks BLOCKED on an ask_user question
// that has been outstanding longer than askUserTimeout() and resumes each
// with a system-authored fallback answer, escalating to the next tier of the
// role's ladder where possible so a different (hopefully more capable)
// provider/model picks up where the original agent stalled.
//
// The "outstanding since" timestamp is task.UpdatedAt, not a new column:
// storage.DB.UpdateTaskQuestion (the last write made to a task's row on its
// way into BLOCKED — see executor.Pool.handleRunResult's BlockedError
// branch, which calls UpdateTaskState(BLOCKED) then
// UpdateTaskQuestion(questionJSON) in that order) stamps updated_at at the
// exact moment the question was recorded, and nothing else touches the row
// while it sits BLOCKED awaiting an answer. Reusing it avoids an entirely
// redundant "asked_at" column carrying the same information a second time.
func (s *Scheduler) tickAskUserTimeouts(ctx context.Context) {
	tasks, err := s.Store.ListTasks(storage.TaskFilter{State: task.StateBlocked})
	if err != nil {
		s.logf("scheduler: list blocked tasks", "error", err)
		return
	}
	timeout := s.askUserTimeout()
	for _, t := range tasks {
		if t.Agent.Role == "" || t.QuestionJSON == "" {
			// Not role-typed, or BLOCKED on pending subtasks rather than a
			// question (see task.go's state machine: BLOCKED covers both).
			continue
		}
		if time.Since(t.UpdatedAt) < timeout {
			continue // still within the grace period
		}
		s.escalateAskUserTimeout(ctx, t)
	}
}

// escalateAskUserTimeout resumes a single BLOCKED, role-typed task whose
// question has timed out. It resolves the role's escalation ladder from the
// tier the task was dispatched at (latest execution's EscalationRung) and
// picks the next tier up, mirroring processTask's escalate() above but
// applied to "stuck on a question" rather than "stuck on a failure" — if no
// higher tier exists (ladder exhausted, no active role config, etc.) it
// still resumes the task (unblocking it is the priority) at its current
// tier, just without a provider/model change, and marks the escalation
// event final:true so that distinction is visible in the event stream.
func (s *Scheduler) escalateAskUserTimeout(ctx context.Context, t *task.Task) {
	execs, err := s.Store.ListExecutions(t.ID)
	if err != nil || len(execs) == 0 {
		s.logf("scheduler: ask-user-timeout: no executions", "taskID", t.ID)
		return
	}
	latest := execs[0] // ListExecutions orders DESC by start_time.
	if latest.SessionID == "" {
		s.logf("scheduler: ask-user-timeout: no resumable session", "taskID", t.ID)
		return
	}

	currentRung := latest.EscalationRung
	if currentRung < 0 {
		currentRung = 0
	}

	// Captured before any Store mutation below: fakeStore-style test doubles
	// (and, in principle, a caching Store) may hand back the same *task.Task
	// pointer from ListTasks that UpdateTaskAgent then mutates in place, so
	// reading t.Agent.Type *after* that call would silently pick up the new
	// value instead of the original one — the same "read fromProvider before
	// mutating" care processTask's escalate() takes above.
	fromProvider := t.Agent.Type

	newAgent := t.Agent
	toRung := currentRung
	toProvider := fromProvider
	reason := ""
	final := false

	row, rcErr := s.Store.GetActiveRoleConfig(t.Agent.Role)
	if rcErr != nil {
		reason = "no active role config; resuming at same tier"
		final = true
	} else {
		var rc role.RoleConfig
		if jsonErr := json.Unmarshal([]byte(row.ConfigJSON), &rc); jsonErr != nil {
			reason = "failed to decode role config; resuming at same tier"
			final = true
		} else if len(rc.EscalationLadder) == 0 {
			reason = "empty escalation ladder; resuming at same tier"
			final = true
		} else {
			nextRung := currentRung + 1
			if nextRung >= len(rc.EscalationLadder) || len(rc.EscalationLadder[nextRung].Candidates) == 0 {
				reason = "escalation ladder exhausted; resuming at same tier"
				final = true
			} else {
				target := rc.EscalationLadder[nextRung].Candidates[0]
				newAgent.Type = target.Provider
				newAgent.Model = target.Model
				toRung = nextRung
				toProvider = target.Provider
			}
		}
	}

	// Record the system-authored fallback answer as an interaction before
	// clearing the question, mirroring the audit trail api.answerTaskQuestion
	// leaves for a real human answer.
	if t.QuestionJSON != "" {
		var qData struct {
			Text    string   `json:"text"`
			Options []string `json:"options"`
		}
		if json.Unmarshal([]byte(t.QuestionJSON), &qData) == nil {
			if err := s.Store.AppendTaskInteraction(t.ID, task.Interaction{
				QuestionText: qData.Text,
				Options:      qData.Options,
				Answer:       fallbackAnswer,
				AskedAt:      t.UpdatedAt,
			}); err != nil {
				s.logf("scheduler: ask-user-timeout: append interaction", "taskID", t.ID, "error", err)
			}
		}
	}
	if err := s.Store.UpdateTaskQuestion(t.ID, ""); err != nil {
		s.logf("scheduler: ask-user-timeout: clear question", "taskID", t.ID, "error", err)
		return
	}
	if newAgent.Type != fromProvider || newAgent.Model != t.Agent.Model {
		if err := s.Store.UpdateTaskAgent(t.ID, newAgent); err != nil {
			s.logf("scheduler: ask-user-timeout: update task agent", "taskID", t.ID, "error", err)
			return
		}
	}
	if err := s.Store.UpdateTaskState(t.ID, task.StateQueued); err != nil {
		s.logf("scheduler: ask-user-timeout: update task state", "taskID", t.ID, "error", err)
		return
	}
	if err := s.Store.UpdateTaskNeedsReview(t.ID, true); err != nil {
		s.logf("scheduler: ask-user-timeout: mark needs_review", "taskID", t.ID, "error", err)
	}

	s.emitEscalated(t.ID, currentRung, toRung, fromProvider, toProvider, final, reason, "ask_user_timeout")

	// SubmitResume requires the task passed in to carry a resumable State
	// (see executor.resumablePoolStates, which includes BLOCKED) — mirroring
	// api.answerTaskQuestion, which passes the pre-transition BLOCKED task
	// struct even though the DB row has already moved to QUEUED above.
	resume := *t
	resume.Agent = newAgent
	resume.State = task.StateBlocked
	resumeExec := &storage.Execution{
		ID:              uuid.NewString(),
		TaskID:          t.ID,
		ResumeSessionID: latest.SessionID,
		ResumeAnswer:    fallbackAnswer,
		SandboxDir:      latest.SandboxDir,
	}
	if err := s.Pool.SubmitResume(ctx, &resume, resumeExec); err != nil {
		s.logf("scheduler: ask-user-timeout: submit resume", "taskID", t.ID, "error", err)
	}
}