package scheduler import ( "context" "encoding/json" "errors" "sync" "testing" "time" "github.com/google/uuid" "github.com/thepeterstone/claudomator/internal/event" "github.com/thepeterstone/claudomator/internal/role" "github.com/thepeterstone/claudomator/internal/storage" "github.com/thepeterstone/claudomator/internal/task" ) // fakeStore is a minimal, in-memory implementation of Store for unit-testing // the Scheduler without a real SQLite database. type fakeStore struct { mu sync.Mutex tasks map[string]*task.Task execsByTask map[string][]*storage.Execution // index 0 = most recent (matches ListExecutions' DESC order) roleConfigs map[string]*storage.RoleConfigRow agentUpdates []task.AgentConfig stateUpdates []task.State events []*event.Event } func newFakeStore() *fakeStore { return &fakeStore{ tasks: make(map[string]*task.Task), execsByTask: make(map[string][]*storage.Execution), roleConfigs: make(map[string]*storage.RoleConfigRow), } } func (f *fakeStore) ListTasks(filter storage.TaskFilter) ([]*task.Task, error) { f.mu.Lock() defer f.mu.Unlock() var out []*task.Task for _, t := range f.tasks { if filter.State != "" && t.State != filter.State { continue } out = append(out, t) } return out, nil } func (f *fakeStore) ListExecutions(taskID string) ([]*storage.Execution, error) { f.mu.Lock() defer f.mu.Unlock() return f.execsByTask[taskID], nil } func (f *fakeStore) GetActiveRoleConfig(roleName string) (*storage.RoleConfigRow, error) { f.mu.Lock() defer f.mu.Unlock() row, ok := f.roleConfigs[roleName] if !ok { return nil, errors.New("not found") } return row, nil } func (f *fakeStore) UpdateTaskAgent(id string, agent task.AgentConfig) error { f.mu.Lock() defer f.mu.Unlock() f.agentUpdates = append(f.agentUpdates, agent) if t, ok := f.tasks[id]; ok { t.Agent = agent } return nil } func (f *fakeStore) UpdateTaskState(id string, newState task.State) error { f.mu.Lock() defer f.mu.Unlock() f.stateUpdates = append(f.stateUpdates, newState) if t, ok := f.tasks[id]; ok { t.State = newState } return nil } func (f *fakeStore) CreateEvent(e *event.Event) error { f.mu.Lock() defer f.mu.Unlock() e.ID = uuid.NewString() f.events = append(f.events, e) return nil } func (f *fakeStore) eventsOfKind(k event.Kind) []*event.Event { f.mu.Lock() defer f.mu.Unlock() var out []*event.Event for _, e := range f.events { if e.Kind == k { out = append(out, e) } } return out } // fakePool records every task submitted to it. type fakePool struct { mu sync.Mutex submitted []*task.Task err error } func (f *fakePool) Submit(_ context.Context, t *task.Task) error { f.mu.Lock() defer f.mu.Unlock() f.submitted = append(f.submitted, t) return f.err } func (f *fakePool) submitCount() int { f.mu.Lock() defer f.mu.Unlock() return len(f.submitted) } // fakeBudget is a configurable BudgetGate. type fakeBudget struct { allow bool err error } func (f *fakeBudget) Allow(_ string, _ float64) (bool, error) { return f.allow, f.err } func twoTierLadder() role.RoleConfig { return role.RoleConfig{ Role: "coder", EscalationLadder: []role.Tier{ { Candidates: []role.Rung{{Provider: "local", Model: "m0"}}, SelectionMode: "single", MaxRetries: 2, }, { Candidates: []role.Rung{{Provider: "anthropic", Model: "claude-sonnet-5"}}, MaxRetries: 1, }, }, } } func seedRoleConfig(t *testing.T, f *fakeStore, rc role.RoleConfig) { t.Helper() b, err := json.Marshal(rc) if err != nil { t.Fatalf("marshal role config: %v", err) } f.roleConfigs[rc.Role] = &storage.RoleConfigRow{ ID: uuid.NewString(), Role: rc.Role, Version: 1, Status: "active", ConfigJSON: string(b), } } func failedTask(id, roleName string, agentType string) *task.Task { return &task.Task{ ID: id, Name: "test", Agent: task.AgentConfig{Type: agentType, Role: roleName, MaxBudgetUSD: 0.5}, State: task.StateFailed, } } func failedExec(rung int) *storage.Execution { return &storage.Execution{ ID: uuid.NewString(), Status: "FAILED", EscalationRung: rung, StartTime: time.Now(), } } // TestScheduler_RetriesSameRung_WhileUnderMaxRetries proves that a task // whose current rung has fewer attempts than tier.MaxRetries is resubmitted // at the same rung, with no escalation event and no Agent.Type/Model change. func TestScheduler_RetriesSameRung_WhileUnderMaxRetries(t *testing.T) { store := newFakeStore() seedRoleConfig(t, store, twoTierLadder()) tk := failedTask("t1", "coder", "local") store.tasks[tk.ID] = tk // Only 1 attempt so far at rung 0; tier0.MaxRetries == 2, so 1 < 2 → retry. store.execsByTask[tk.ID] = []*storage.Execution{failedExec(0)} pool := &fakePool{} sch := &Scheduler{Store: store, Pool: pool} sch.Tick(context.Background()) if pool.submitCount() != 1 { t.Fatalf("expected 1 submission, got %d", pool.submitCount()) } resubmitted := pool.submitted[0] if resubmitted.Agent.Type != "local" { t.Errorf("retry should keep the same provider: got %q", resubmitted.Agent.Type) } if resubmitted.State != task.StateQueued { t.Errorf("resubmitted task should be QUEUED, got %v", resubmitted.State) } if len(store.eventsOfKind(event.KindEscalated)) != 0 { t.Errorf("a same-rung retry should not emit a KindEscalated event") } if len(store.agentUpdates) != 0 { t.Errorf("a same-rung retry should not change the task's Agent config, got %d UpdateTaskAgent calls", len(store.agentUpdates)) } } // TestScheduler_EscalatesToNextRung_WhenBudgetAllows proves that once a // tier's MaxRetries is exhausted, the scheduler escalates to the next tier // (when the budget allows it), updates the task's Agent to the new rung's // provider/model, resubmits, and emits a non-final KindEscalated event. func TestScheduler_EscalatesToNextRung_WhenBudgetAllows(t *testing.T) { store := newFakeStore() seedRoleConfig(t, store, twoTierLadder()) tk := failedTask("t1", "coder", "local") store.tasks[tk.ID] = tk // 2 consecutive failed attempts at rung 0; tier0.MaxRetries == 2, so // 2 < 2 is false → escalate. store.execsByTask[tk.ID] = []*storage.Execution{failedExec(0), failedExec(0)} pool := &fakePool{} sch := &Scheduler{Store: store, Pool: pool, Budget: &fakeBudget{allow: true}} sch.Tick(context.Background()) if pool.submitCount() != 1 { t.Fatalf("expected 1 submission, got %d", pool.submitCount()) } resubmitted := pool.submitted[0] if resubmitted.Agent.Type != "anthropic" || resubmitted.Agent.Model != "claude-sonnet-5" { t.Errorf("escalated task should carry tier 1's rung: got %q/%q", resubmitted.Agent.Type, resubmitted.Agent.Model) } if resubmitted.State != task.StateQueued { t.Errorf("resubmitted task should be QUEUED, got %v", resubmitted.State) } if len(store.agentUpdates) != 1 { t.Fatalf("expected 1 UpdateTaskAgent call, got %d", len(store.agentUpdates)) } evs := store.eventsOfKind(event.KindEscalated) if len(evs) != 1 { t.Fatalf("expected 1 KindEscalated event, got %d", len(evs)) } var payload struct { FromRung int `json:"from_rung"` ToRung int `json:"to_rung"` FromProvider string `json:"from_provider"` ToProvider string `json:"to_provider"` Final bool `json:"final"` } if err := json.Unmarshal(evs[0].Payload, &payload); err != nil { t.Fatalf("unmarshal event payload: %v", err) } if payload.Final { t.Errorf("escalation event should have final=false") } if payload.FromRung != 0 || payload.ToRung != 1 { t.Errorf("expected from_rung=0 to_rung=1, got %d -> %d", payload.FromRung, payload.ToRung) } if payload.ToProvider != "anthropic" { t.Errorf("expected to_provider=anthropic, got %q", payload.ToProvider) } } // TestScheduler_DeclinesFinal_WhenBudgetDenies proves that when the budget // denies an otherwise-due escalation, the task is left FAILED (no state // change, no resubmission) and a final:true KindEscalated event is recorded. func TestScheduler_DeclinesFinal_WhenBudgetDenies(t *testing.T) { store := newFakeStore() seedRoleConfig(t, store, twoTierLadder()) tk := failedTask("t1", "coder", "local") store.tasks[tk.ID] = tk store.execsByTask[tk.ID] = []*storage.Execution{failedExec(0), failedExec(0)} pool := &fakePool{} sch := &Scheduler{Store: store, Pool: pool, Budget: &fakeBudget{allow: false}} sch.Tick(context.Background()) if pool.submitCount() != 0 { t.Fatalf("budget-denied escalation should not resubmit, got %d submissions", pool.submitCount()) } if len(store.stateUpdates) != 0 { t.Errorf("budget-denied escalation should not change task state, got %d state updates", len(store.stateUpdates)) } evs := store.eventsOfKind(event.KindEscalated) if len(evs) != 1 { t.Fatalf("expected 1 KindEscalated event, got %d", len(evs)) } var payload struct { Final bool `json:"final"` } if err := json.Unmarshal(evs[0].Payload, &payload); err != nil { t.Fatalf("unmarshal event payload: %v", err) } if !payload.Final { t.Errorf("budget-denied escalation event should have final=true") } } // TestScheduler_DeclinesFinal_WhenLadderExhausted proves the same // final:true behavior when the task is already at the ladder's last tier. func TestScheduler_DeclinesFinal_WhenLadderExhausted(t *testing.T) { store := newFakeStore() seedRoleConfig(t, store, twoTierLadder()) tk := failedTask("t1", "coder", "anthropic") store.tasks[tk.ID] = tk // At rung 1 (the last tier), tier1.MaxRetries == 1, so 1 attempt already // made means 1 < 1 is false → would escalate, but there is no rung 2. store.execsByTask[tk.ID] = []*storage.Execution{failedExec(1)} pool := &fakePool{} sch := &Scheduler{Store: store, Pool: pool, Budget: &fakeBudget{allow: true}} sch.Tick(context.Background()) if pool.submitCount() != 0 { t.Fatalf("exhausted ladder should not resubmit, got %d submissions", pool.submitCount()) } evs := store.eventsOfKind(event.KindEscalated) if len(evs) != 1 { t.Fatalf("expected 1 KindEscalated event, got %d", len(evs)) } var payload struct { Final bool `json:"final"` } if err := json.Unmarshal(evs[0].Payload, &payload); err != nil { t.Fatalf("unmarshal event payload: %v", err) } if !payload.Final { t.Errorf("exhausted-ladder decline event should have final=true") } } // TestScheduler_Convergence_DoesNotReprocessSameExecution proves the // double-processing guard: ticking twice against the same unchanged FAILED // execution only acts once (no duplicate submissions or events), which is // what keeps a task stuck at the end of its ladder from generating a fresh // "final" event (or worse, a fresh escalation) on every single poll forever. func TestScheduler_Convergence_DoesNotReprocessSameExecution(t *testing.T) { store := newFakeStore() seedRoleConfig(t, store, twoTierLadder()) tk := failedTask("t1", "coder", "anthropic") store.tasks[tk.ID] = tk store.execsByTask[tk.ID] = []*storage.Execution{failedExec(1)} // last tier, exhausted pool := &fakePool{} sch := &Scheduler{Store: store, Pool: pool, Budget: &fakeBudget{allow: true}} sch.Tick(context.Background()) sch.Tick(context.Background()) sch.Tick(context.Background()) if got := len(store.eventsOfKind(event.KindEscalated)); got != 1 { t.Fatalf("expected exactly 1 KindEscalated event across 3 ticks, got %d", got) } if pool.submitCount() != 0 { t.Fatalf("expected 0 submissions, got %d", pool.submitCount()) } }