package scheduler import ( "context" "encoding/json" "fmt" "strings" "sync" "testing" "github.com/google/uuid" "github.com/thepeterstone/claudomator/internal/event" "github.com/thepeterstone/claudomator/internal/storage" "github.com/thepeterstone/claudomator/internal/story" "github.com/thepeterstone/claudomator/internal/task" ) // fakeStoryStore is a minimal, in-memory implementation of StoryStore for // unit-testing StoryOrchestrator without a real SQLite database. Mirrors the // fakeStore pattern already used by scheduler_test.go for the Phase 5 // Scheduler. type fakeStoryStore struct { mu sync.Mutex stories map[string]*story.Story tasks map[string]*task.Task events []*event.Event // executions and activeRoleConfigs back the Phase 8 retro stage's // context-gathering (ListExecutions/GetActiveRoleConfig) — keyed the // same way the real storage.DB is (by task ID / role name). executions map[string][]*storage.Execution activeRoleConfigs map[string]*storage.RoleConfigRow } func newFakeStoryStore() *fakeStoryStore { return &fakeStoryStore{ stories: make(map[string]*story.Story), tasks: make(map[string]*task.Task), } } func (f *fakeStoryStore) ListStories(_ storage.StoryFilter) ([]*story.Story, error) { f.mu.Lock() defer f.mu.Unlock() var out []*story.Story for _, s := range f.stories { cp := *s out = append(out, &cp) } return out, nil } func (f *fakeStoryStore) UpdateStory(st *story.Story) error { f.mu.Lock() defer f.mu.Unlock() if _, ok := f.stories[st.ID]; !ok { return fmt.Errorf("story %q not found", st.ID) } cp := *st f.stories[st.ID] = &cp return nil } func (f *fakeStoryStore) GetTask(id string) (*task.Task, error) { f.mu.Lock() defer f.mu.Unlock() t, ok := f.tasks[id] if !ok { return nil, fmt.Errorf("task %q not found", id) } cp := *t return &cp, nil } func (f *fakeStoryStore) ListDependents(taskID string) ([]*task.Task, error) { f.mu.Lock() defer f.mu.Unlock() var out []*task.Task for _, t := range f.tasks { for _, d := range t.DependsOn { if d == taskID { cp := *t out = append(out, &cp) break } } } return out, nil } func (f *fakeStoryStore) CreateTask(t *task.Task) error { f.mu.Lock() defer f.mu.Unlock() if _, ok := f.tasks[t.ID]; ok { return fmt.Errorf("task %q already exists", t.ID) } cp := *t f.tasks[t.ID] = &cp return nil } func (f *fakeStoryStore) UpdateTaskState(id string, newState task.State) error { f.mu.Lock() defer f.mu.Unlock() t, ok := f.tasks[id] if !ok { return fmt.Errorf("task %q not found", id) } t.State = newState return nil } func (f *fakeStoryStore) CreateEvent(e *event.Event) error { f.mu.Lock() defer f.mu.Unlock() e.ID = uuid.NewString() e.Seq = int64(len(f.events)) + 1 f.events = append(f.events, e) return nil } func (f *fakeStoryStore) ListSubtasks(parentID string) ([]*task.Task, error) { f.mu.Lock() defer f.mu.Unlock() var out []*task.Task for _, t := range f.tasks { if t.ParentTaskID == parentID { cp := *t out = append(out, &cp) } } return out, nil } func (f *fakeStoryStore) ListExecutions(taskID string) ([]*storage.Execution, error) { f.mu.Lock() defer f.mu.Unlock() return f.executions[taskID], nil } func (f *fakeStoryStore) GetActiveRoleConfig(roleName string) (*storage.RoleConfigRow, error) { f.mu.Lock() defer f.mu.Unlock() row, ok := f.activeRoleConfigs[roleName] if !ok { return nil, fmt.Errorf("no active role config for %q", roleName) } return row, nil } func (f *fakeStoryStore) ListEvents(taskID string, sinceSeq int64) ([]*event.Event, error) { f.mu.Lock() defer f.mu.Unlock() var out []*event.Event for _, e := range f.events { if e.TaskID == taskID && e.Seq > sinceSeq { out = append(out, e) } } return out, nil } func (f *fakeStoryStore) setTaskState(id string, s task.State) { f.mu.Lock() defer f.mu.Unlock() if t, ok := f.tasks[id]; ok { t.State = s } } func (f *fakeStoryStore) setTaskSummary(id, summary string) { f.mu.Lock() defer f.mu.Unlock() if t, ok := f.tasks[id]; ok { t.Summary = summary } } func (f *fakeStoryStore) eventsOfKind(k event.Kind) []*event.Event { f.mu.Lock() defer f.mu.Unlock() var out []*event.Event for _, e := range f.events { if e.Kind == k { out = append(out, e) } } return out } func (f *fakeStoryStore) dependentsWithRole(taskID, role string) []*task.Task { deps, _ := f.ListDependents(taskID) var out []*task.Task for _, d := range deps { if d.Agent.Role == role { out = append(out, d) } } return out } func builderTask(id string, state task.State) *task.Task { return &task.Task{ ID: id, Name: "Builder", Agent: task.AgentConfig{Type: "claude", Role: "builder", Instructions: "build it"}, RepositoryURL: "git@example.com:org/repo.git", State: state, } } func newStoryWithRoot(id, rootTaskID, status string) *story.Story { return &story.Story{ID: id, Name: "Test Story", Status: status, RootTaskID: rootTaskID} } // TestStoryOrchestrator_SpawnsEvaluators_WhenBuilderReady is verification // item (a): a builder task reaching READY for a story spawns exactly 4 // evaluator tasks with correct roles/depends_on, moves the story to // VALIDATING. The builder itself stays READY (not COMPLETED) — see // TestStoryOrchestrator_ReadyBuilder_SpawnsEvaluators_StaysReadyUntilApproved // for that specific assertion. func TestStoryOrchestrator_SpawnsEvaluators_WhenBuilderReady(t *testing.T) { store := newFakeStoryStore() root := builderTask("builder-1", task.StateReady) store.tasks[root.ID] = root st := newStoryWithRoot("story-1", root.ID, "IN_PROGRESS") store.stories[st.ID] = st pool := &fakePool{} orch := &StoryOrchestrator{Store: store, Pool: pool} orch.Tick(context.Background()) deps, err := store.ListDependents(root.ID) if err != nil { t.Fatal(err) } if len(deps) != 4 { t.Fatalf("expected 4 evaluator tasks, got %d: %+v", len(deps), deps) } gotRoles := map[string]bool{} for _, d := range deps { gotRoles[d.Agent.Role] = true if len(d.DependsOn) != 1 || d.DependsOn[0] != root.ID { t.Errorf("evaluator %s: DependsOn = %+v, want [%s]", d.ID, d.DependsOn, root.ID) } if d.ParentTaskID != "" { t.Errorf("evaluator %s: ParentTaskID = %q, want empty (DAG sibling, not subtask)", d.ID, d.ParentTaskID) } if d.State != task.StateQueued { t.Errorf("evaluator %s: State = %v, want QUEUED", d.ID, d.State) } } for _, r := range evaluatorRoles { if !gotRoles[r] { t.Errorf("missing evaluator with role %q", r) } } if pool.submitCount() != 4 { t.Fatalf("expected 4 pool submissions, got %d", pool.submitCount()) } got, err := func() (*story.Story, error) { stories, err := store.ListStories(storage.StoryFilter{}) if err != nil { return nil, err } for _, s := range stories { if s.ID == st.ID { return s, nil } } return nil, fmt.Errorf("not found") }() if err != nil { t.Fatal(err) } if got.Status != "VALIDATING" { t.Errorf("story status: want VALIDATING, got %q", got.Status) } } // TestStoryOrchestrator_DoesNotDuplicateEvaluators is verification item (b): // re-checking the same story after evaluators already exist does not spawn // duplicates. func TestStoryOrchestrator_DoesNotDuplicateEvaluators(t *testing.T) { store := newFakeStoryStore() root := builderTask("builder-1", task.StateReady) store.tasks[root.ID] = root st := newStoryWithRoot("story-1", root.ID, "IN_PROGRESS") store.stories[st.ID] = st pool := &fakePool{} orch := &StoryOrchestrator{Store: store, Pool: pool} orch.Tick(context.Background()) orch.Tick(context.Background()) orch.Tick(context.Background()) deps, _ := store.ListDependents(root.ID) if len(deps) != 4 { t.Fatalf("expected exactly 4 evaluator tasks after 3 ticks, got %d", len(deps)) } if pool.submitCount() != 4 { t.Fatalf("expected exactly 4 submissions after 3 ticks, got %d", pool.submitCount()) } } // evaluatorTask builds a completed (or not) evaluator task depending on // rootID with the given role. func evaluatorTask(id, rootID, role string, state task.State) *task.Task { return &task.Task{ ID: id, Name: role, Agent: task.AgentConfig{Role: role}, DependsOn: []string{rootID}, State: state, Summary: "looks good", } } // seedStoryWithEvaluators wires up a story whose builder is COMPLETED and // whose 4 evaluators already exist (in the given state), returning the // fakeStoryStore, story, and evaluator tasks (in evaluatorRoles order). func seedStoryWithEvaluators(t *testing.T, evalState task.State) (*fakeStoryStore, *story.Story, []*task.Task) { t.Helper() store := newFakeStoryStore() // root is READY, not COMPLETED: under the new rule, root only reaches // COMPLETED once its own arbitration approves it (see // finalizeArbitration), and every caller of this helper is simulating a // point in the pipeline before that approval has happened yet. // seedDoneStory (below) explicitly promotes root to COMPLETED itself for // its own already-approved narrative. root := builderTask("builder-1", task.StateReady) store.tasks[root.ID] = root st := newStoryWithRoot("story-1", root.ID, "VALIDATING") store.stories[st.ID] = st evaluators := make([]*task.Task, len(evaluatorRoles)) for i, r := range evaluatorRoles { ev := evaluatorTask(fmt.Sprintf("eval-%d", i), root.ID, r, evalState) store.tasks[ev.ID] = ev evaluators[i] = ev } return store, st, evaluators } // TestStoryOrchestrator_SpawnsArbitration_WhenAllEvaluatorsComplete is // verification item (c): all 4 evaluators reaching COMPLETED spawns exactly // 1 arbitration task depending on all 4. func TestStoryOrchestrator_SpawnsArbitration_WhenAllEvaluatorsComplete(t *testing.T) { store, _, evaluators := seedStoryWithEvaluators(t, task.StateCompleted) pool := &fakePool{} orch := &StoryOrchestrator{Store: store, Pool: pool} orch.Tick(context.Background()) arbitrations := store.dependentsWithRole(evaluators[0].ID, "planner") if len(arbitrations) != 1 { t.Fatalf("expected exactly 1 arbitration task, got %d: %+v", len(arbitrations), arbitrations) } arb := arbitrations[0] if len(arb.DependsOn) != len(evaluators) { t.Fatalf("arbitration DependsOn = %+v, want all %d evaluator IDs", arb.DependsOn, len(evaluators)) } for _, ev := range evaluators { if !dependsOnAll(arb, []string{ev.ID}) { t.Errorf("arbitration does not depend on evaluator %s", ev.ID) } } if arb.ParentTaskID != "" { t.Errorf("arbitration ParentTaskID = %q, want empty", arb.ParentTaskID) } if arb.State != task.StateQueued { t.Errorf("arbitration State = %v, want QUEUED", arb.State) } // Re-ticking must not spawn a second arbitration task. orch.Tick(context.Background()) orch.Tick(context.Background()) arbitrations = store.dependentsWithRole(evaluators[0].ID, "planner") if len(arbitrations) != 1 { t.Fatalf("expected exactly 1 arbitration task after repeated ticks, got %d", len(arbitrations)) } } // TestStoryOrchestrator_DoesNotSpawnArbitration_UntilAllEvaluatorsComplete // proves the fan-in gate: even with 3 of 4 evaluators COMPLETED, no // arbitration task is created yet. func TestStoryOrchestrator_DoesNotSpawnArbitration_UntilAllEvaluatorsComplete(t *testing.T) { store, _, evaluators := seedStoryWithEvaluators(t, task.StateCompleted) // Knock one evaluator back to RUNNING. store.setTaskState(evaluators[0].ID, task.StateRunning) pool := &fakePool{} orch := &StoryOrchestrator{Store: store, Pool: pool} orch.Tick(context.Background()) arbitrations := store.dependentsWithRole(evaluators[0].ID, "planner") if len(arbitrations) != 0 { t.Fatalf("expected no arbitration task while an evaluator is incomplete, got %d", len(arbitrations)) } } // TestStoryOrchestrator_EmitsEvalVerdict_OncePerEvaluator proves // maybeEmitVerdict fires exactly once per completed evaluator, attached to // the story's ID, even across repeated ticks. func TestStoryOrchestrator_EmitsEvalVerdict_OncePerEvaluator(t *testing.T) { store, st, evaluators := seedStoryWithEvaluators(t, task.StateCompleted) pool := &fakePool{} orch := &StoryOrchestrator{Store: store, Pool: pool} orch.Tick(context.Background()) orch.Tick(context.Background()) orch.Tick(context.Background()) verdicts := store.eventsOfKind(event.KindEvalVerdict) if len(verdicts) != len(evaluators) { t.Fatalf("expected exactly %d eval_verdict events, got %d", len(evaluators), len(verdicts)) } seenTaskIDs := map[string]bool{} for _, e := range verdicts { if e.TaskID != st.ID { t.Errorf("eval_verdict event attached to %q, want story ID %q", e.TaskID, st.ID) } var payload struct { TaskID string `json:"task_id"` Role string `json:"role"` Summary string `json:"summary"` } if err := json.Unmarshal(e.Payload, &payload); err != nil { t.Fatalf("unmarshal payload: %v", err) } seenTaskIDs[payload.TaskID] = true if payload.Role == "" { t.Errorf("payload missing role: %+v", payload) } if payload.Summary != "looks good" { t.Errorf("payload summary = %q, want %q", payload.Summary, "looks good") } } for _, ev := range evaluators { if !seenTaskIDs[ev.ID] { t.Errorf("no eval_verdict event found for evaluator %s", ev.ID) } } } // TestStoryOrchestrator_FinalizeArbitration_NestedNodeApproved_DoesNotTouchStoryStatus // proves finalizeArbitration, generalized to operate on any builder-role // node (not just the story root), promotes a NESTED node to COMPLETED on // approval without touching story.Status at all -- REVIEW_READY is reserved // for the actual root position (node.ParentTaskID == ""). func TestStoryOrchestrator_FinalizeArbitration_NestedNodeApproved_DoesNotTouchStoryStatus(t *testing.T) { store := newFakeStoryStore() root := builderTask("nested-approve-root", task.StateBlocked) store.tasks[root.ID] = root st := newStoryWithRoot("nested-approve-story", root.ID, "IN_PROGRESS") store.stories[st.ID] = st nested := builderTask("nested-approve-node", task.StateReady) nested.ParentTaskID = root.ID store.tasks[nested.ID] = nested arb := &task.Task{ ID: "nested-approve-arb", Name: "Arbitration", Agent: task.AgentConfig{Role: arbitrationRole}, State: task.StateCompleted, Summary: "ship it", } store.tasks[arb.ID] = arb payload, _ := json.Marshal(struct { Approved bool `json:"approved"` Reasoning string `json:"reasoning"` }{Approved: true, Reasoning: "meets criteria"}) if err := store.CreateEvent(&event.Event{TaskID: arb.ID, Kind: event.KindVerdictReported, Actor: event.ActorAgent, Payload: payload}); err != nil { t.Fatalf("seed verdict event: %v", err) } pool := &fakePool{} orch := &StoryOrchestrator{Store: store, Pool: pool} orch.finalizeArbitration(context.Background(), st, nested, arb) got, err := store.GetTask(nested.ID) if err != nil { t.Fatal(err) } if got.State != task.StateCompleted { t.Errorf("nested node State = %v, want COMPLETED", got.State) } stories, _ := store.ListStories(storage.StoryFilter{}) var gotStory *story.Story for _, s := range stories { if s.ID == st.ID { gotStory = s } } if gotStory.Status != "IN_PROGRESS" { t.Errorf("story Status = %q, want unchanged IN_PROGRESS (only root's own arbitration should touch story.Status)", gotStory.Status) } } // TestStoryOrchestrator_FinalizeArbitration_NestedNodeRejected_SpawnsFixAttemptDirectly // proves the nested-rejection path: since a nested position has no // story-level field to poll (unlike the root's NEEDS_FIX/ensureFixAttempt // flow), its fix-attempt is spawned immediately, right here -- same role, // same ParentTaskID as the rejected node (so task.CurrentAttempt resolves // it correctly), and story.Status is left untouched. func TestStoryOrchestrator_FinalizeArbitration_NestedNodeRejected_SpawnsFixAttemptDirectly(t *testing.T) { store := newFakeStoryStore() root := builderTask("nested-reject-root", task.StateBlocked) store.tasks[root.ID] = root st := newStoryWithRoot("nested-reject-story", root.ID, "IN_PROGRESS") store.stories[st.ID] = st nested := builderTask("nested-reject-node", task.StateReady) nested.ParentTaskID = root.ID store.tasks[nested.ID] = nested arb := &task.Task{ ID: "nested-reject-arb", Name: "Arbitration", Agent: task.AgentConfig{Role: arbitrationRole}, State: task.StateCompleted, Summary: "found a real problem", } store.tasks[arb.ID] = arb payload, _ := json.Marshal(struct { Approved bool `json:"approved"` Reasoning string `json:"reasoning"` }{Approved: false, Reasoning: "breaks the widget"}) if err := store.CreateEvent(&event.Event{TaskID: arb.ID, Kind: event.KindVerdictReported, Actor: event.ActorAgent, Payload: payload}); err != nil { t.Fatalf("seed verdict event: %v", err) } pool := &fakePool{} orch := &StoryOrchestrator{Store: store, Pool: pool} orch.finalizeArbitration(context.Background(), st, nested, arb) got, err := store.GetTask(nested.ID) if err != nil { t.Fatal(err) } if got.State != task.StateReady { t.Errorf("rejected nested node State = %v, want unchanged READY", got.State) } fixAttempts := store.dependentsWithRole(nested.ID, "builder") if len(fixAttempts) != 1 { t.Fatalf("expected exactly 1 fix-attempt task depending on the rejected nested node, got %d", len(fixAttempts)) } fix := fixAttempts[0] if fix.ParentTaskID != nested.ParentTaskID { t.Errorf("fix attempt ParentTaskID = %q, want %q (same position as the rejected node)", fix.ParentTaskID, nested.ParentTaskID) } if fix.State != task.StateQueued { t.Errorf("fix attempt State = %v, want QUEUED", fix.State) } stories, _ := store.ListStories(storage.StoryFilter{}) var gotStory *story.Story for _, s := range stories { if s.ID == st.ID { gotStory = s } } if gotStory.Status != "IN_PROGRESS" { t.Errorf("story Status = %q, want unchanged IN_PROGRESS", gotStory.Status) } } // TestStoryOrchestrator_FinalizeArbitration_NestedNodeRejected_CapsAtMaxFixAttempts // mirrors the root's own safety net (TestStoryOrchestrator_NeedsFix_CapsAtMaxFixAttempts) // for a nested position: once a chain of maxFixAttempts consecutive fix // attempts already exists at this position, no further fix attempt is // spawned. func TestStoryOrchestrator_FinalizeArbitration_NestedNodeRejected_CapsAtMaxFixAttempts(t *testing.T) { store := newFakeStoryStore() root := builderTask("nested-cap-root", task.StateBlocked) store.tasks[root.ID] = root st := newStoryWithRoot("nested-cap-story", root.ID, "IN_PROGRESS") store.stories[st.ID] = st nested := builderTask("nested-cap-node", task.StateReady) nested.ParentTaskID = root.ID store.tasks[nested.ID] = nested pool := &fakePool{} orch := &StoryOrchestrator{Store: store, Pool: pool} // Build a chain of maxFixAttempts prior fix-attempt tasks at this same // nested position, ending at prev -- simulating a position that has // already exhausted its automatic fix attempts. prev := nested for i := 0; i < maxFixAttempts; i++ { nt, err := orch.spawnRoleTask(context.Background(), fmt.Sprintf("Fix attempt %d", i), "builder", []string{prev.ID}, root.ID, root, "fix it") if err != nil { t.Fatalf("seed fix-attempt chain: %v", err) } prev = nt } arb := &task.Task{ ID: "nested-cap-arb", Name: "Arbitration", Agent: task.AgentConfig{Role: arbitrationRole}, State: task.StateCompleted, Summary: "still broken", } store.tasks[arb.ID] = arb payload, _ := json.Marshal(struct { Approved bool `json:"approved"` Reasoning string `json:"reasoning"` }{Approved: false, Reasoning: "still broken"}) if err := store.CreateEvent(&event.Event{TaskID: arb.ID, Kind: event.KindVerdictReported, Actor: event.ActorAgent, Payload: payload}); err != nil { t.Fatalf("seed verdict event: %v", err) } orch.finalizeArbitration(context.Background(), st, prev, arb) fixAttempts := store.dependentsWithRole(prev.ID, "builder") if len(fixAttempts) != 0 { t.Fatalf("expected no new fix-attempt task once maxFixAttempts is reached, got %d", len(fixAttempts)) } } // TestStoryOrchestrator_FinalizeArbitration_RootStillSetsStoryStatus proves // the generalized finalizeArbitration still drives story.Status exactly as // before when node IS the root position (node.ParentTaskID == "") -- the // one case where touching story.Status is still correct, now decided by // node.ParentTaskID rather than being the function's only mode. func TestStoryOrchestrator_FinalizeArbitration_RootStillSetsStoryStatus(t *testing.T) { store := newFakeStoryStore() root := builderTask("root-status-root", task.StateReady) store.tasks[root.ID] = root st := newStoryWithRoot("root-status-story", root.ID, "VALIDATING") store.stories[st.ID] = st arb := &task.Task{ ID: "root-status-arb", Name: "Arbitration", Agent: task.AgentConfig{Role: arbitrationRole}, State: task.StateCompleted, Summary: "ship it", } store.tasks[arb.ID] = arb payload, _ := json.Marshal(struct { Approved bool `json:"approved"` Reasoning string `json:"reasoning"` }{Approved: true, Reasoning: "meets criteria"}) if err := store.CreateEvent(&event.Event{TaskID: arb.ID, Kind: event.KindVerdictReported, Actor: event.ActorAgent, Payload: payload}); err != nil { t.Fatalf("seed verdict event: %v", err) } pool := &fakePool{} orch := &StoryOrchestrator{Store: store, Pool: pool} orch.finalizeArbitration(context.Background(), st, root, arb) gotRoot, err := store.GetTask(root.ID) if err != nil { t.Fatal(err) } if gotRoot.State != task.StateCompleted { t.Errorf("root State = %v, want COMPLETED", gotRoot.State) } stories, _ := store.ListStories(storage.StoryFilter{}) var gotStory *story.Story for _, s := range stories { if s.ID == st.ID { gotStory = s } } if gotStory.Status != "REVIEW_READY" { t.Errorf("story Status = %q, want REVIEW_READY (root's own approval must still drive it)", gotStory.Status) } } // TestStoryOrchestrator_FinalizeArbitration_IdempotentViaArbitrationDecidedEvent // proves the generalized idempotency check (a KindArbitrationDecided event // already referencing this arbitration.ID) correctly no-ops a repeated call // for the same already-decided arbitration -- the mechanism every node in a // story's tree now shares, replacing the root-only story.Status == // "VALIDATING" gate. func TestStoryOrchestrator_FinalizeArbitration_IdempotentViaArbitrationDecidedEvent(t *testing.T) { store := newFakeStoryStore() root := builderTask("idempotent-root", task.StateReady) store.tasks[root.ID] = root st := newStoryWithRoot("idempotent-story", root.ID, "VALIDATING") store.stories[st.ID] = st arb := &task.Task{ ID: "idempotent-arb", Name: "Arbitration", Agent: task.AgentConfig{Role: arbitrationRole}, State: task.StateCompleted, Summary: "ship it", } store.tasks[arb.ID] = arb payload, _ := json.Marshal(struct { Approved bool `json:"approved"` Reasoning string `json:"reasoning"` }{Approved: true, Reasoning: "meets criteria"}) if err := store.CreateEvent(&event.Event{TaskID: arb.ID, Kind: event.KindVerdictReported, Actor: event.ActorAgent, Payload: payload}); err != nil { t.Fatalf("seed verdict event: %v", err) } pool := &fakePool{} orch := &StoryOrchestrator{Store: store, Pool: pool} orch.finalizeArbitration(context.Background(), st, root, arb) orch.finalizeArbitration(context.Background(), st, root, arb) orch.finalizeArbitration(context.Background(), st, root, arb) decided := store.eventsOfKind(event.KindArbitrationDecided) if len(decided) != 1 { t.Fatalf("expected exactly 1 arbitration_decided event after 3 calls, got %d", len(decided)) } } // TestStoryOrchestrator_ArbitrationCompletes_EmitsDecisionAndReviewReady is // verification item (d): arbitration reaching COMPLETED emits // KindArbitrationDecided and moves the story to REVIEW_READY. func TestStoryOrchestrator_ArbitrationCompletes_EmitsDecisionAndReviewReady(t *testing.T) { store, st, evaluators := seedStoryWithEvaluators(t, task.StateCompleted) pool := &fakePool{} orch := &StoryOrchestrator{Store: store, Pool: pool} // Tick 1: spawns the arbitration task. orch.Tick(context.Background()) arbitrations := store.dependentsWithRole(evaluators[0].ID, "planner") if len(arbitrations) != 1 { t.Fatalf("expected 1 arbitration task, got %d", len(arbitrations)) } arb := arbitrations[0] store.setTaskState(arb.ID, task.StateCompleted) store.setTaskSummary(arb.ID, "ship it") // Tick 2: arbitration is now COMPLETED. orch.Tick(context.Background()) decided := store.eventsOfKind(event.KindArbitrationDecided) if len(decided) != 1 { t.Fatalf("expected exactly 1 arbitration_decided event, got %d", len(decided)) } if decided[0].TaskID != st.ID { t.Errorf("arbitration_decided attached to %q, want story ID %q", decided[0].TaskID, st.ID) } var payload struct { TaskID string `json:"task_id"` Summary string `json:"summary"` } if err := json.Unmarshal(decided[0].Payload, &payload); err != nil { t.Fatalf("unmarshal payload: %v", err) } if payload.TaskID != arb.ID || payload.Summary != "ship it" { t.Errorf("unexpected payload: %+v", payload) } stories, _ := store.ListStories(storage.StoryFilter{}) var got *story.Story for _, s := range stories { if s.ID == st.ID { got = s } } if got == nil { t.Fatal("story not found") } if got.Status != "REVIEW_READY" { t.Errorf("story status: want REVIEW_READY, got %q", got.Status) } // Tick 3+: must not re-emit or re-decide. orch.Tick(context.Background()) orch.Tick(context.Background()) decided = store.eventsOfKind(event.KindArbitrationDecided) if len(decided) != 1 { t.Fatalf("expected still exactly 1 arbitration_decided event after repeated ticks, got %d", len(decided)) } } // TestStoryOrchestrator_ArbitrationRejects_SetsStoryNeedsFix proves that // when the arbitration task reports a structured approved=false verdict // (via a KindVerdictReported event, the same one AgentChannel.ReportVerdict // records), finalizeArbitration routes the story to NEEDS_FIX instead of // unconditionally REVIEW_READY. func TestStoryOrchestrator_ArbitrationRejects_SetsStoryNeedsFix(t *testing.T) { store, st, evaluators := seedStoryWithEvaluators(t, task.StateCompleted) pool := &fakePool{} orch := &StoryOrchestrator{Store: store, Pool: pool} orch.Tick(context.Background()) // spawns arbitration arbitrations := store.dependentsWithRole(evaluators[0].ID, "planner") if len(arbitrations) != 1 { t.Fatalf("expected 1 arbitration task, got %d", len(arbitrations)) } arb := arbitrations[0] // Simulate the arbitration agent calling report_verdict with a rejection // before finishing, the same event storeChannel.ReportVerdict records. payload, _ := json.Marshal(struct { Approved bool `json:"approved"` Reasoning string `json:"reasoning"` }{Approved: false, Reasoning: "breaks the running-log panel styling"}) if err := store.CreateEvent(&event.Event{ TaskID: arb.ID, Kind: event.KindVerdictReported, Actor: event.ActorAgent, Payload: payload, }); err != nil { t.Fatalf("seed verdict event: %v", err) } store.setTaskState(arb.ID, task.StateCompleted) store.setTaskSummary(arb.ID, "found a real problem") orch.Tick(context.Background()) // finalizeArbitration runs stories, _ := store.ListStories(storage.StoryFilter{}) var got *story.Story for _, s := range stories { if s.ID == st.ID { got = s } } if got == nil { t.Fatal("story not found") } if got.Status != "NEEDS_FIX" { t.Errorf("story status: want NEEDS_FIX, got %q", got.Status) } } // TestStoryOrchestrator_ArbitrationApproves_SetsReviewReady proves the // mirror case: an explicit approved=true verdict still routes to // REVIEW_READY, same as today's unconditional behavior -- this isn't just // "absence of a verdict defaults to proceed", a real approval also proceeds. func TestStoryOrchestrator_ArbitrationApproves_SetsReviewReady(t *testing.T) { store, st, evaluators := seedStoryWithEvaluators(t, task.StateCompleted) pool := &fakePool{} orch := &StoryOrchestrator{Store: store, Pool: pool} orch.Tick(context.Background()) arbitrations := store.dependentsWithRole(evaluators[0].ID, "planner") arb := arbitrations[0] payload, _ := json.Marshal(struct { Approved bool `json:"approved"` Reasoning string `json:"reasoning"` }{Approved: true, Reasoning: "meets all acceptance criteria"}) if err := store.CreateEvent(&event.Event{ TaskID: arb.ID, Kind: event.KindVerdictReported, Actor: event.ActorAgent, Payload: payload, }); err != nil { t.Fatalf("seed verdict event: %v", err) } store.setTaskState(arb.ID, task.StateCompleted) store.setTaskSummary(arb.ID, "ship it") orch.Tick(context.Background()) stories, _ := store.ListStories(storage.StoryFilter{}) var got *story.Story for _, s := range stories { if s.ID == st.ID { got = s } } if got == nil { t.Fatal("story not found") } if got.Status != "REVIEW_READY" { t.Errorf("story status: want REVIEW_READY, got %q", got.Status) } } // seedNeedsFixStory drives a story through the full Builder -> Evaluators -> // Arbitration chain to a rejected verdict (mirrors // TestStoryOrchestrator_ArbitrationRejects_SetsStoryNeedsFix's own setup), // leaving it at status NEEDS_FIX with its original (rejected) root task // still in the store -- the starting point ensureFixAttempt operates on. // Returns the store, the story (as currently persisted), the rejected root // task, the evaluators, and the arbitration task. func seedNeedsFixStory(t *testing.T) (*fakeStoryStore, *story.Story, *task.Task, []*task.Task, *task.Task) { t.Helper() store, st, evaluators := seedStoryWithEvaluators(t, task.StateCompleted) oldRoot, err := store.GetTask(st.RootTaskID) if err != nil { t.Fatalf("get root: %v", err) } pool := &fakePool{} orch := &StoryOrchestrator{Store: store, Pool: pool} orch.Tick(context.Background()) // spawns arbitration arbitrations := store.dependentsWithRole(evaluators[0].ID, "planner") if len(arbitrations) != 1 { t.Fatalf("expected 1 arbitration task, got %d", len(arbitrations)) } arb := arbitrations[0] payload, _ := json.Marshal(struct { Approved bool `json:"approved"` Reasoning string `json:"reasoning"` }{Approved: false, Reasoning: "breaks the running-log panel styling"}) if err := store.CreateEvent(&event.Event{ TaskID: arb.ID, Kind: event.KindVerdictReported, Actor: event.ActorAgent, Payload: payload, }); err != nil { t.Fatalf("seed verdict event: %v", err) } store.setTaskState(arb.ID, task.StateCompleted) store.setTaskSummary(arb.ID, "found a real problem") orch.Tick(context.Background()) // finalizeArbitration runs, sets NEEDS_FIX stories, _ := store.ListStories(storage.StoryFilter{}) var got *story.Story for _, s := range stories { if s.ID == st.ID { got = s } } if got == nil || got.Status != "NEEDS_FIX" { t.Fatalf("setup failed: story = %+v, want status NEEDS_FIX", got) } return store, got, oldRoot, evaluators, arb } // TestStoryOrchestrator_ArbitrationRejects_LeavesRootReadyNotCompleted proves // the core "verified before complete" invariant this plan adds: a rejected // root is left at READY — never promoted to COMPLETED. Only // finalizeArbitration's approval branch ever writes root to COMPLETED; a // rejection must never look "done" to anything that trusts task.State // directly, the same gate a nested subtask's parent will rely on once // arbitrated review recurses into subtask trees (a future plan). func TestStoryOrchestrator_ArbitrationRejects_LeavesRootReadyNotCompleted(t *testing.T) { store, _, oldRoot, _, _ := seedNeedsFixStory(t) got, err := store.GetTask(oldRoot.ID) if err != nil { t.Fatal(err) } if got.State != task.StateReady { t.Errorf("rejected root State = %v, want READY (must never be promoted to COMPLETED on rejection)", got.State) } } // TestStoryOrchestrator_NeedsFix_SpawnsFixAttemptAndRepointsRoot proves the // core mechanism: a story at NEEDS_FIX gets a new builder-role fix-attempt // task spawned (depending on the rejected root), and the story is re-pointed // at it (RootTaskID updated, Status reset to IN_PROGRESS) so the very next // tick re-enters the normal Builder->Evaluators->Arbitration flow against // the new attempt. // TestStoryOrchestrator_NeedsFix_SpawnsFixAttempt proves the core mechanism: // a story at NEEDS_FIX gets a new builder-role fix-attempt task spawned // (depending on the rejected root), and the story's Status resets to // IN_PROGRESS so the very next tick re-enters the normal // Builder->Evaluators->Arbitration flow against the new attempt -- // discovered via task.CurrentAttempt resolving forward from st.RootTaskID, // which is never mutated (it's an immutable anchor set once at story // creation). func TestStoryOrchestrator_NeedsFix_SpawnsFixAttempt(t *testing.T) { store, st, oldRoot, _, _ := seedNeedsFixStory(t) pool := &fakePool{} orch := &StoryOrchestrator{Store: store, Pool: pool} orch.Tick(context.Background()) fixAttempts := store.dependentsWithRole(oldRoot.ID, "builder") if len(fixAttempts) != 1 { t.Fatalf("expected exactly 1 fix-attempt task depending on the rejected root, got %d", len(fixAttempts)) } fix := fixAttempts[0] if fix.State != task.StateQueued { t.Errorf("fix attempt State = %v, want QUEUED", fix.State) } if fix.ParentTaskID != "" { t.Errorf("fix attempt ParentTaskID = %q, want empty (top-level, DAG sibling not delegated subtask)", fix.ParentTaskID) } stories, _ := store.ListStories(storage.StoryFilter{}) var got *story.Story for _, s := range stories { if s.ID == st.ID { got = s } } if got == nil { t.Fatal("story not found") } if got.RootTaskID != oldRoot.ID { t.Errorf("story RootTaskID = %q, want unchanged (still %q -- it's an immutable anchor now, resolved via task.CurrentAttempt)", got.RootTaskID, oldRoot.ID) } if got.Status != "IN_PROGRESS" { t.Errorf("story Status = %q, want IN_PROGRESS", got.Status) } } // TestStoryOrchestrator_NeedsFix_FixAttemptInstructionsIncludeRejectionReasoning // proves the fix attempt's instructions carry the arbitration's structured // rejection reasoning (read via the KindVerdictReported event), not just a // generic "try again" -- the whole point of automating this loop is that the // next attempt has something concrete to act on. func TestStoryOrchestrator_NeedsFix_FixAttemptInstructionsIncludeRejectionReasoning(t *testing.T) { store, _, oldRoot, _, _ := seedNeedsFixStory(t) pool := &fakePool{} orch := &StoryOrchestrator{Store: store, Pool: pool} orch.Tick(context.Background()) fixAttempts := store.dependentsWithRole(oldRoot.ID, "builder") if len(fixAttempts) != 1 { t.Fatalf("expected exactly 1 fix-attempt task, got %d", len(fixAttempts)) } instructions := fixAttempts[0].Agent.Instructions if !strings.Contains(instructions, "breaks the running-log panel styling") { t.Errorf("fix-attempt instructions do not include the arbitration's rejection reasoning:\n%s", instructions) } if !strings.Contains(instructions, "Test Story") { t.Errorf("fix-attempt instructions do not include the story name:\n%s", instructions) } } // TestStoryOrchestrator_NeedsFix_IdempotentAcrossTicks proves repeated ticks // don't spawn more than one fix-attempt task -- after the first tick the // story's status flips to IN_PROGRESS, so the NEEDS_FIX branch simply isn't // re-entered on subsequent ticks (the normal Builder-not-complete-yet path // takes over instead, since the fix attempt is freshly QUEUED). func TestStoryOrchestrator_NeedsFix_IdempotentAcrossTicks(t *testing.T) { store, _, oldRoot, _, _ := seedNeedsFixStory(t) pool := &fakePool{} orch := &StoryOrchestrator{Store: store, Pool: pool} orch.Tick(context.Background()) orch.Tick(context.Background()) orch.Tick(context.Background()) fixAttempts := store.dependentsWithRole(oldRoot.ID, "builder") if len(fixAttempts) != 1 { t.Fatalf("expected exactly 1 fix-attempt task after repeated ticks, got %d", len(fixAttempts)) } } // TestStoryOrchestrator_EnsureFixAttempt_FindsExistingSpawnedAttempt proves // ensureFixAttempt's own structural idempotency: simulating a restart // between "fix task spawned" and "story Status reset" -- calling // ensureFixAttempt again must find the already-spawned builder-role // dependent rather than spawning a second one, and still reset Status. func TestStoryOrchestrator_EnsureFixAttempt_FindsExistingSpawnedAttempt(t *testing.T) { store, st, oldRoot, _, _ := seedNeedsFixStory(t) pool := &fakePool{} orch := &StoryOrchestrator{Store: store, Pool: pool} preexisting, err := orch.spawnRoleTask(context.Background(), "Fix attempt: pre-existing", "builder", []string{oldRoot.ID}, "", oldRoot, "fix it") if err != nil { t.Fatalf("seed pre-existing fix attempt: %v", err) } orch.ensureFixAttempt(context.Background(), st) fixAttempts := store.dependentsWithRole(oldRoot.ID, "builder") if len(fixAttempts) != 1 { t.Fatalf("expected exactly 1 fix-attempt task (the pre-existing one, not a new one), got %d", len(fixAttempts)) } if fixAttempts[0].ID != preexisting.ID { t.Errorf("expected ensureFixAttempt to reuse the pre-existing task %s, got a different one %s", preexisting.ID, fixAttempts[0].ID) } if st.RootTaskID != oldRoot.ID { t.Errorf("story RootTaskID = %q, want unchanged (still %q -- it's an immutable anchor now)", st.RootTaskID, oldRoot.ID) } if st.Status != "IN_PROGRESS" { t.Errorf("story Status = %q, want IN_PROGRESS", st.Status) } } // TestStoryOrchestrator_NeedsFix_CapsAtMaxFixAttempts proves the safety net: // once a chain of maxFixAttempts consecutive fix attempts already exists, // ensureFixAttempt stops spawning new ones and leaves the story at // NEEDS_FIX, exactly like today's fully-manual behavior. st.RootTaskID stays // at oldRoot.ID throughout (it's an immutable anchor); ensureFixAttempt // resolves the chain's tip itself via task.CurrentAttempt. func TestStoryOrchestrator_NeedsFix_CapsAtMaxFixAttempts(t *testing.T) { store, st, oldRoot, _, _ := seedNeedsFixStory(t) pool := &fakePool{} orch := &StoryOrchestrator{Store: store, Pool: pool} // Build a chain of maxFixAttempts prior fix-attempt tasks, each // depending on the one before it, ending at oldRoot -- simulating a // story that has already exhausted its automatic fix attempts. prev := oldRoot for i := 0; i < maxFixAttempts; i++ { nt, err := orch.spawnRoleTask(context.Background(), fmt.Sprintf("Fix attempt %d", i), "builder", []string{prev.ID}, "", oldRoot, "fix it") if err != nil { t.Fatalf("seed fix-attempt chain: %v", err) } prev = nt } orch.Tick(context.Background()) fixAttempts := store.dependentsWithRole(prev.ID, "builder") if len(fixAttempts) != 0 { t.Fatalf("expected no new fix-attempt task once maxFixAttempts is reached, got %d", len(fixAttempts)) } stories, _ := store.ListStories(storage.StoryFilter{}) var got *story.Story for _, s := range stories { if s.ID == st.ID { got = s } } if got.Status != "NEEDS_FIX" { t.Errorf("story Status = %q, want NEEDS_FIX (cap reached, no further automation)", got.Status) } } // TestStoryOrchestrator_DoesNothing_WhenBuilderNotComplete proves the // orchestrator is inert for a story whose builder task hasn't reached // COMPLETED yet and isn't auto-acceptable either (still RUNNING — not // READY, so autoAccept has nothing to do). func TestStoryOrchestrator_DoesNothing_WhenBuilderNotComplete(t *testing.T) { store := newFakeStoryStore() root := builderTask("builder-1", task.StateRunning) store.tasks[root.ID] = root st := newStoryWithRoot("story-1", root.ID, "IN_PROGRESS") store.stories[st.ID] = st pool := &fakePool{} orch := &StoryOrchestrator{Store: store, Pool: pool} orch.Tick(context.Background()) deps, _ := store.ListDependents(root.ID) if len(deps) != 0 { t.Fatalf("expected no evaluator tasks while builder is RUNNING, got %d", len(deps)) } if pool.submitCount() != 0 { t.Fatalf("expected no submissions, got %d", pool.submitCount()) } got, err := store.GetTask(root.ID) if err != nil { t.Fatal(err) } if got.State != task.StateRunning { t.Errorf("builder state must be untouched: want RUNNING, got %v", got.State) } } // TestStoryOrchestrator_ReadyBuilder_SpawnsEvaluators_StaysReadyUntilApproved // replaces the old "auto-accept the builder immediately" regression test: a // builder task sitting at READY (execution succeeded, awaiting what would // otherwise be a manual POST /api/tasks/{id}/accept) triggers the 4 // evaluators to spawn in the same tick — but, unlike evaluators/arbitration // tasks, the builder itself is NOT auto-accepted to COMPLETED here. It stays // READY (which already satisfies a dependent's DependsOn — see // executor.depDoneStates) until its own arbitration approves it (see // finalizeArbitration). This is what makes "COMPLETED" mean "verified", // uniformly — the same rule a nested builder subtask will need once // arbitrated review recurses into subtask trees. func TestStoryOrchestrator_ReadyBuilder_SpawnsEvaluators_StaysReadyUntilApproved(t *testing.T) { store := newFakeStoryStore() root := builderTask("builder-1", task.StateReady) store.tasks[root.ID] = root st := newStoryWithRoot("story-1", root.ID, "IN_PROGRESS") store.stories[st.ID] = st pool := &fakePool{} orch := &StoryOrchestrator{Store: store, Pool: pool} orch.Tick(context.Background()) got, err := store.GetTask(root.ID) if err != nil { t.Fatal(err) } if got.State != task.StateReady { t.Fatalf("builder must stay READY until arbitration approves it, got %v", got.State) } deps, _ := store.ListDependents(root.ID) if len(deps) != 4 { t.Fatalf("expected 4 evaluator tasks spawned in the same tick the builder reaches READY, got %d", len(deps)) } } // TestStoryOrchestrator_AutoAcceptsReadyEvaluators proves READY evaluator // tasks are auto-accepted to COMPLETED by the orchestrator, with no external // accept call, and that doing so unblocks arbitration spawning. func TestStoryOrchestrator_AutoAcceptsReadyEvaluators(t *testing.T) { store, _, evaluators := seedStoryWithEvaluators(t, task.StateReady) pool := &fakePool{} orch := &StoryOrchestrator{Store: store, Pool: pool} orch.Tick(context.Background()) for _, ev := range evaluators { got, err := store.GetTask(ev.ID) if err != nil { t.Fatal(err) } if got.State != task.StateCompleted { t.Errorf("evaluator %s should be auto-accepted to COMPLETED, got %v", ev.ID, got.State) } } arbitrations := store.dependentsWithRole(evaluators[0].ID, "planner") if len(arbitrations) != 1 { t.Fatalf("expected arbitration spawned once all evaluators auto-accept, got %d", len(arbitrations)) } } // TestStoryOrchestrator_AutoAcceptsReadyArbitration proves a READY // arbitration task is auto-accepted to COMPLETED by the orchestrator, with // no external accept call, and that this in turn triggers // finalizeArbitration (KindArbitrationDecided + REVIEW_READY) in the same // tick. func TestStoryOrchestrator_AutoAcceptsReadyArbitration(t *testing.T) { store, st, evaluators := seedStoryWithEvaluators(t, task.StateCompleted) pool := &fakePool{} orch := &StoryOrchestrator{Store: store, Pool: pool} // Tick 1: spawns the arbitration task (starts at QUEUED). orch.Tick(context.Background()) arbitrations := store.dependentsWithRole(evaluators[0].ID, "planner") if len(arbitrations) != 1 { t.Fatalf("expected 1 arbitration task, got %d", len(arbitrations)) } arb := arbitrations[0] // Simulate the arbitration's execution succeeding (RUNNING -> READY), // exactly as executor.Pool.handleRunResult would do for any top-level // task — without ever calling POST /api/tasks/{id}/accept. store.setTaskState(arb.ID, task.StateReady) store.setTaskSummary(arb.ID, "approved") // Tick 2: orchestrator must auto-accept READY -> COMPLETED itself. orch.Tick(context.Background()) got, err := store.GetTask(arb.ID) if err != nil { t.Fatal(err) } if got.State != task.StateCompleted { t.Fatalf("arbitration should be auto-accepted to COMPLETED, got %v", got.State) } decided := store.eventsOfKind(event.KindArbitrationDecided) if len(decided) != 1 { t.Fatalf("expected exactly 1 arbitration_decided event after auto-accept, got %d", len(decided)) } stories, _ := store.ListStories(storage.StoryFilter{}) for _, s := range stories { if s.ID == st.ID && s.Status != "REVIEW_READY" { t.Errorf("story status: want REVIEW_READY after arbitration auto-accepts, got %q", s.Status) } } } // TestStoryOrchestrator_AutoAccept_DoesNotTouchUnrelatedReadyTask proves the // auto-accept behavior is scoped to a story's own pipeline tasks (root task // + its role-matched evaluator/arbitration dependents) and does not sweep up // an unrelated READY task that merely happens to exist in the store. func TestStoryOrchestrator_AutoAccept_DoesNotTouchUnrelatedReadyTask(t *testing.T) { store := newFakeStoryStore() root := builderTask("builder-1", task.StateReady) store.tasks[root.ID] = root st := newStoryWithRoot("story-1", root.ID, "IN_PROGRESS") store.stories[st.ID] = st unrelated := &task.Task{ID: "unrelated-1", Name: "unrelated", Agent: task.AgentConfig{Type: "claude"}, State: task.StateReady} store.tasks[unrelated.ID] = unrelated pool := &fakePool{} orch := &StoryOrchestrator{Store: store, Pool: pool} orch.Tick(context.Background()) got, err := store.GetTask(unrelated.ID) if err != nil { t.Fatal(err) } if got.State != task.StateReady { t.Errorf("unrelated task must not be auto-accepted: want READY, got %v", got.State) } } // TestStoryOrchestrator_SkipsStoriesWithNoRootTask proves a story with no // root_task_id set is left completely untouched. func TestStoryOrchestrator_SkipsStoriesWithNoRootTask(t *testing.T) { store := newFakeStoryStore() st := &story.Story{ID: "story-1", Name: "no root yet", Status: "DISCOVERY"} store.stories[st.ID] = st pool := &fakePool{} orch := &StoryOrchestrator{Store: store, Pool: pool} orch.Tick(context.Background()) // must not panic or error despite no root task existing if pool.submitCount() != 0 { t.Fatalf("expected no submissions, got %d", pool.submitCount()) } } // TestStoryOrchestrator_SkipsTerminalStories proves DONE/CANCELLED stories // are never revisited, even if (hypothetically) their root task is // COMPLETED and evaluators don't yet exist. func TestStoryOrchestrator_SkipsTerminalStories(t *testing.T) { for _, status := range []string{"DONE", "CANCELLED"} { t.Run(status, func(t *testing.T) { store := newFakeStoryStore() root := builderTask("builder-1", task.StateCompleted) store.tasks[root.ID] = root st := newStoryWithRoot("story-1", root.ID, status) store.stories[st.ID] = st pool := &fakePool{} orch := &StoryOrchestrator{Store: store, Pool: pool} orch.Tick(context.Background()) deps, _ := store.ListDependents(root.ID) if len(deps) != 0 { t.Fatalf("status %s: expected no evaluator tasks spawned, got %d", status, len(deps)) } }) } } // TestStoryOrchestrator_EndToEnd drives a story through the full chain — // builder ready -> evaluators -> arbitration -> REVIEW_READY — using only // the fake store/pool, proving the whole ceremony holds together end to end // at the orchestrator level (verification item 5, the higher-level test). // // Every task in the chain is driven to READY (never directly to COMPLETED), // mirroring exactly what executor.Pool.handleRunResult does for a real // top-level task whose execution succeeds. Evaluators and arbitration are // still carried the rest of the way to COMPLETED by the orchestrator's own // auto-accept (not an external POST /api/tasks/{id}/accept call, which this // test never makes) — but the builder itself is the one exception: it stays // READY through the whole cycle and is only promoted to COMPLETED once // arbitration actually approves it, proving "COMPLETED means verified" holds // end to end, not just in isolated unit tests. The only accept call anywhere // in this flow is the story-level one, which isn't part of this test — it's // covered separately in internal/api's story accept-gate tests. func TestStoryOrchestrator_EndToEnd(t *testing.T) { store := newFakeStoryStore() root := builderTask("builder-1", task.StatePending) store.tasks[root.ID] = root st := newStoryWithRoot("story-1", root.ID, "IN_PROGRESS") store.stories[st.ID] = st pool := &fakePool{} orch := &StoryOrchestrator{Store: store, Pool: pool} // Before the builder completes, nothing happens. orch.Tick(context.Background()) if deps, _ := store.ListDependents(root.ID); len(deps) != 0 { t.Fatalf("expected no evaluators before builder completes, got %d", len(deps)) } // Builder's execution succeeds (RUNNING -> READY, exactly like // handleRunResult) — no POST /api/tasks/{id}/accept call here. store.setTaskState(root.ID, task.StateReady) orch.Tick(context.Background()) rootAfter, err := store.GetTask(root.ID) if err != nil { t.Fatal(err) } if rootAfter.State != task.StateReady { t.Fatalf("builder must stay READY until arbitration approves it (not auto-accepted eagerly), got %v", rootAfter.State) } deps, _ := store.ListDependents(root.ID) if len(deps) != 4 { t.Fatalf("expected 4 evaluators, got %d", len(deps)) } statusOf := func() string { stories, _ := store.ListStories(storage.StoryFilter{}) for _, s := range stories { if s.ID == st.ID { return s.Status } } return "" } if statusOf() != "VALIDATING" { t.Fatalf("expected VALIDATING after evaluators spawn, got %q", statusOf()) } // Evaluators' executions succeed one by one (READY, not COMPLETED); // arbitration must not spawn early. for i, d := range deps { store.setTaskState(d.ID, task.StateReady) store.setTaskSummary(d.ID, fmt.Sprintf("verdict %d", i)) orch.Tick(context.Background()) arbs := store.dependentsWithRole(deps[0].ID, "planner") if i < len(deps)-1 && len(arbs) != 0 { t.Fatalf("arbitration spawned too early, after %d/%d evaluators complete", i+1, len(deps)) } } for _, d := range deps { got, err := store.GetTask(d.ID) if err != nil { t.Fatal(err) } if got.State != task.StateCompleted { t.Errorf("evaluator %s should be auto-accepted to COMPLETED, got %v", d.ID, got.State) } } arbs := store.dependentsWithRole(deps[0].ID, "planner") if len(arbs) != 1 { t.Fatalf("expected exactly 1 arbitration task, got %d", len(arbs)) } arb := arbs[0] verdicts := store.eventsOfKind(event.KindEvalVerdict) if len(verdicts) != 4 { t.Fatalf("expected 4 eval_verdict events, got %d", len(verdicts)) } // Arbitration's execution succeeds (READY, not COMPLETED). store.setTaskState(arb.ID, task.StateReady) store.setTaskSummary(arb.ID, "approved") orch.Tick(context.Background()) arbAfter, err := store.GetTask(arb.ID) if err != nil { t.Fatal(err) } if arbAfter.State != task.StateCompleted { t.Fatalf("arbitration should be auto-accepted to COMPLETED, got %v", arbAfter.State) } rootFinal, err := store.GetTask(root.ID) if err != nil { t.Fatal(err) } if rootFinal.State != task.StateCompleted { t.Fatalf("builder should finally be promoted to COMPLETED now that arbitration approved it, got %v", rootFinal.State) } if statusOf() != "REVIEW_READY" { t.Fatalf("expected REVIEW_READY after arbitration completes, got %q", statusOf()) } decided := store.eventsOfKind(event.KindArbitrationDecided) if len(decided) != 1 { t.Fatalf("expected exactly 1 arbitration_decided event, got %d", len(decided)) } } // seedDoneStory wires up a story whose entire Builder->Evaluators-> // Arbitration pipeline has already completed (mirroring what would actually // be true of any real story by the time it reaches DONE) and whose status // is DONE, returning the fakeStoryStore, story, root, evaluators, and // arbitration task. Used by the Phase 8 retro-stage tests below. func seedDoneStory(t *testing.T) (*fakeStoryStore, *story.Story, *task.Task, []*task.Task, *task.Task) { t.Helper() store, st, evaluators := seedStoryWithEvaluators(t, task.StateCompleted) st.Status = "DONE" ids := make([]string, len(evaluators)) for i, ev := range evaluators { ids[i] = ev.ID } arb := &task.Task{ ID: "arbitration-1", Name: "Arbitration", Agent: task.AgentConfig{Role: arbitrationRole}, DependsOn: ids, State: task.StateCompleted, Summary: "ship it", } store.tasks[arb.ID] = arb // By the time a story reaches DONE, its root has already been through a // real (or simulated) approval and is COMPLETED. seedStoryWithEvaluators // seeds root at READY (the pre-arbitration state every other caller of // that helper needs), so promote it here to match this fixture's own // DONE narrative. if err := store.UpdateTaskState(st.RootTaskID, task.StateCompleted); err != nil { t.Fatalf("promote root to completed for DONE fixture: %v", err) } root, err := store.GetTask(st.RootTaskID) if err != nil { t.Fatal(err) } return store, st, root, evaluators, arb } // TestStoryOrchestrator_Retro_SpawnsRetroTask_WhenStoryDone is verification // item (a): a story reaching DONE spawns exactly one retro-role task // depending on the arbitration task, whose instructions contain the story's // context (name and ID, at minimum). func TestStoryOrchestrator_Retro_SpawnsRetroTask_WhenStoryDone(t *testing.T) { store, st, _, _, arb := seedDoneStory(t) pool := &fakePool{} orch := &StoryOrchestrator{Store: store, Pool: pool} orch.Tick(context.Background()) retros := store.dependentsWithRole(arb.ID, retroRole) if len(retros) != 1 { t.Fatalf("expected exactly 1 retro task depending on the arbitration task, got %d", len(retros)) } retro := retros[0] if retro.ParentTaskID != "" { t.Errorf("retro task should be a DAG sibling (no ParentTaskID), got %q", retro.ParentTaskID) } if retro.State != task.StateQueued { t.Errorf("retro task state: want QUEUED, got %v", retro.State) } if !strings.Contains(retro.Agent.Instructions, st.Name) || !strings.Contains(retro.Agent.Instructions, st.ID) { t.Errorf("retro instructions should contain the story's name/ID, got: %s", retro.Agent.Instructions) } } // TestStoryOrchestrator_Retro_DoesNotDuplicate is verification item (b): // checking the same DONE story again does not spawn a second retro task. func TestStoryOrchestrator_Retro_DoesNotDuplicate(t *testing.T) { store, _, _, _, arb := seedDoneStory(t) pool := &fakePool{} orch := &StoryOrchestrator{Store: store, Pool: pool} orch.Tick(context.Background()) orch.Tick(context.Background()) orch.Tick(context.Background()) retros := store.dependentsWithRole(arb.ID, retroRole) if len(retros) != 1 { t.Fatalf("expected still exactly 1 retro task after repeated ticks, got %d", len(retros)) } } // TestStoryOrchestrator_Retro_DoesNothing_UntilPipelineSettled proves // processRetro never backfills a Builder/Evaluator/Arbitration task itself: // a story marked DONE whose evaluators/arbitration don't actually exist yet // (a hypothetical/malformed case — a real story never reaches DONE any // other way) leaves the task tree untouched, exactly like // TestStoryOrchestrator_SkipsTerminalStories already proves for the // evaluator-spawning side of this. func TestStoryOrchestrator_Retro_DoesNothing_UntilPipelineSettled(t *testing.T) { store := newFakeStoryStore() root := builderTask("builder-1", task.StateCompleted) store.tasks[root.ID] = root st := newStoryWithRoot("story-1", root.ID, "DONE") store.stories[st.ID] = st pool := &fakePool{} orch := &StoryOrchestrator{Store: store, Pool: pool} orch.Tick(context.Background()) deps, _ := store.ListDependents(root.ID) if len(deps) != 0 { t.Fatalf("expected no tasks spawned when the pipeline never actually ran, got %d", len(deps)) } } // TestStoryOrchestrator_Retro_EmitsRetroCaptured_WhenRetroTaskCompletes is // verification item (d): once the retro task reaches COMPLETED (via // auto-accept from READY, exactly like every other task in this pipeline), // KindRetroCaptured is emitted attached to the story's ID, aggregating every // role_config_proposed event the retro task itself recorded plus its // reported summary. func TestStoryOrchestrator_Retro_EmitsRetroCaptured_WhenRetroTaskCompletes(t *testing.T) { store, st, _, _, arb := seedDoneStory(t) pool := &fakePool{} orch := &StoryOrchestrator{Store: store, Pool: pool} // Tick 1: spawns the retro task. orch.Tick(context.Background()) retros := store.dependentsWithRole(arb.ID, retroRole) if len(retros) != 1 { t.Fatalf("expected 1 retro task, got %d", len(retros)) } retro := retros[0] // Simulate the retro agent having called propose_role_config twice // (recording KindRoleConfigProposed events attached to its own task ID, // exactly as storeChannel.ProposeRoleConfig does — see // internal/executor/channel.go) before finishing with a summary. mustMarshal := func(v any) json.RawMessage { b, err := json.Marshal(v) if err != nil { t.Fatal(err) } return b } store.CreateEvent(&event.Event{ TaskID: retro.ID, Kind: event.KindRoleConfigProposed, Actor: event.ActorAgent, Payload: mustMarshal(struct { Role string `json:"role"` Version int `json:"version"` }{Role: "builder", Version: 2}), }) store.CreateEvent(&event.Event{ TaskID: retro.ID, Kind: event.KindRoleConfigProposed, Actor: event.ActorAgent, Payload: mustMarshal(struct { Role string `json:"role"` Version int `json:"version"` }{Role: "evaluator_quality", Version: 1}), }) store.setTaskState(retro.ID, task.StateReady) store.setTaskSummary(retro.ID, "The builder over-escalated twice; tightening its system prompt should help.") // Tick 2: retro task auto-accepted to COMPLETED, retro_captured emitted. orch.Tick(context.Background()) retroAfter, err := store.GetTask(retro.ID) if err != nil { t.Fatal(err) } if retroAfter.State != task.StateCompleted { t.Fatalf("retro task should be auto-accepted to COMPLETED, got %v", retroAfter.State) } captured := store.eventsOfKind(event.KindRetroCaptured) if len(captured) != 1 { t.Fatalf("expected exactly 1 retro_captured event, got %d", len(captured)) } if captured[0].TaskID != st.ID { t.Errorf("retro_captured should be attached to the story ID %q, got %q", st.ID, captured[0].TaskID) } var payload struct { TaskID string `json:"task_id"` Proposals []struct { Role string `json:"role"` Version int `json:"version"` } `json:"proposals"` Summary string `json:"summary"` } if err := json.Unmarshal(captured[0].Payload, &payload); err != nil { t.Fatalf("unmarshal payload: %v", err) } if payload.TaskID != retro.ID { t.Errorf("payload.task_id: want %q, got %q", retro.ID, payload.TaskID) } if len(payload.Proposals) != 2 { t.Fatalf("expected 2 proposals in payload, got %d: %+v", len(payload.Proposals), payload.Proposals) } if payload.Proposals[0].Role != "builder" || payload.Proposals[0].Version != 2 { t.Errorf("proposal[0] mismatch: %+v", payload.Proposals[0]) } if payload.Proposals[1].Role != "evaluator_quality" || payload.Proposals[1].Version != 1 { t.Errorf("proposal[1] mismatch: %+v", payload.Proposals[1]) } if !strings.Contains(payload.Summary, "over-escalated") { t.Errorf("expected the retro's summary in the payload, got %q", payload.Summary) } // Tick 3+: must not re-emit. orch.Tick(context.Background()) orch.Tick(context.Background()) captured = store.eventsOfKind(event.KindRetroCaptured) if len(captured) != 1 { t.Fatalf("expected still exactly 1 retro_captured event after repeated ticks, got %d", len(captured)) } } // TestStoryOrchestrator_ProcessesNestedReadyBuilderNode_WhileRootStillBlocked // proves the deadlock the single-root version of processStory had: a nested // builder subtask reaching READY while its parent (root) is still BLOCKED // must still get its own Evaluators spawned this tick -- root can only ever // become READY once this nested node's own arbitration approves it (see // internal/executor.Pool.maybeUnblockParent, which requires every // subtask's task.CurrentAttempt to resolve to COMPLETED before promoting a // BLOCKED parent). Root itself, still BLOCKED, gets nothing spawned for it. func TestStoryOrchestrator_ProcessesNestedReadyBuilderNode_WhileRootStillBlocked(t *testing.T) { store := newFakeStoryStore() root := builderTask("deadlock-root", task.StateBlocked) store.tasks[root.ID] = root st := newStoryWithRoot("deadlock-story", root.ID, "IN_PROGRESS") store.stories[st.ID] = st nested := builderTask("deadlock-nested", task.StateReady) nested.ParentTaskID = root.ID store.tasks[nested.ID] = nested pool := &fakePool{} orch := &StoryOrchestrator{Store: store, Pool: pool} orch.Tick(context.Background()) deps, err := store.ListDependents(nested.ID) if err != nil { t.Fatal(err) } if len(deps) != 4 { t.Fatalf("expected 4 evaluator tasks spawned for the nested READY node even though root is still BLOCKED, got %d", len(deps)) } rootDeps, _ := store.ListDependents(root.ID) if len(rootDeps) != 0 { t.Fatalf("root itself must not get evaluators spawned while still BLOCKED, got %d dependents", len(rootDeps)) } rootAfter, err := store.GetTask(root.ID) if err != nil { t.Fatal(err) } if rootAfter.State != task.StateBlocked { t.Errorf("root state must be untouched: want BLOCKED, got %v", rootAfter.State) } stories, _ := store.ListStories(storage.StoryFilter{}) var gotStory *story.Story for _, s := range stories { if s.ID == st.ID { gotStory = s } } if gotStory.Status != "IN_PROGRESS" { t.Errorf("story Status = %q, want unchanged IN_PROGRESS (nested evaluator spawn must not touch story-level status)", gotStory.Status) } } // TestStoryOrchestrator_ProcessStory_MultipleSiblingNestedNodes_BothProcessedSameTick // proves the tree walk processes every qualifying node in a single tick, not // just the first one it finds -- a real behavioral change from the old // single-root processStory, which returned as soon as it found one node not // ready to progress. func TestStoryOrchestrator_ProcessStory_MultipleSiblingNestedNodes_BothProcessedSameTick(t *testing.T) { store := newFakeStoryStore() root := builderTask("multi-root", task.StateBlocked) store.tasks[root.ID] = root st := newStoryWithRoot("multi-story", root.ID, "IN_PROGRESS") store.stories[st.ID] = st nested1 := builderTask("multi-nested-1", task.StateReady) nested1.ParentTaskID = root.ID store.tasks[nested1.ID] = nested1 nested2 := builderTask("multi-nested-2", task.StateReady) nested2.ParentTaskID = root.ID store.tasks[nested2.ID] = nested2 pool := &fakePool{} orch := &StoryOrchestrator{Store: store, Pool: pool} orch.Tick(context.Background()) deps1, _ := store.ListDependents(nested1.ID) deps2, _ := store.ListDependents(nested2.ID) if len(deps1) != 4 { t.Errorf("nested1: expected 4 evaluator tasks, got %d", len(deps1)) } if len(deps2) != 4 { t.Errorf("nested2: expected 4 evaluator tasks, got %d", len(deps2)) } if pool.submitCount() != 8 { t.Fatalf("expected 8 pool submissions (4 evaluators x 2 sibling nodes) in a single tick, got %d", pool.submitCount()) } } // TestStoryOrchestrator_ProcessStory_RootCompleted_NoOp proves the // short-circuit added for a fully-resolved tree: once root itself is // COMPLETED (only possible after finalizeArbitration approved it, which // itself requires every nested node beneath it to already be COMPLETED -- // see maybeUnblockParent), processStory does nothing further, not even a // tree walk. func TestStoryOrchestrator_ProcessStory_RootCompleted_NoOp(t *testing.T) { store := newFakeStoryStore() root := builderTask("done-root", task.StateCompleted) store.tasks[root.ID] = root st := newStoryWithRoot("done-story", root.ID, "REVIEW_READY") store.stories[st.ID] = st pool := &fakePool{} orch := &StoryOrchestrator{Store: store, Pool: pool} orch.Tick(context.Background()) if pool.submitCount() != 0 { t.Fatalf("expected no submissions once root is COMPLETED, got %d", pool.submitCount()) } } // TestStoryOrchestrator_EndToEnd_NestedSubtask exercises the full recursive // loop this piece adds: a nested builder subtask reaches READY while root is // still BLOCKED, gets its own Evaluators -> Arbitration -> approval cycle // entirely independent of root, reaches COMPLETED -- at which point (in a // real deployment) internal/executor.Pool.maybeUnblockParent would see every // one of root's subtasks resolve to COMPLETED via task.CurrentAttempt and // promote root BLOCKED -> READY. That promotion is simulated directly here // (store.setTaskState) since maybeUnblockParent lives in a different // package/component this fake doesn't exercise. Root then goes through the // exact same cycle for itself, ending at COMPLETED with the story at // REVIEW_READY -- proving the two levels compose without any special-casing // between them. func TestStoryOrchestrator_EndToEnd_NestedSubtask(t *testing.T) { store := newFakeStoryStore() root := builderTask("e2e-nested-root", task.StateBlocked) store.tasks[root.ID] = root st := newStoryWithRoot("e2e-nested-story", root.ID, "IN_PROGRESS") store.stories[st.ID] = st nested := builderTask("e2e-nested-child", task.StateReady) nested.ParentTaskID = root.ID store.tasks[nested.ID] = nested pool := &fakePool{} orch := &StoryOrchestrator{Store: store, Pool: pool} // Tick 1: nested is READY -> its 4 evaluators spawn. Root, still // BLOCKED, gets nothing. orch.Tick(context.Background()) nestedDeps, _ := store.ListDependents(nested.ID) if len(nestedDeps) != 4 { t.Fatalf("expected 4 evaluators for nested, got %d", len(nestedDeps)) } // Evaluators' executions succeed one by one (READY, not COMPLETED). for i, d := range nestedDeps { store.setTaskState(d.ID, task.StateReady) store.setTaskSummary(d.ID, fmt.Sprintf("nested verdict %d", i)) orch.Tick(context.Background()) } for _, d := range nestedDeps { got, err := store.GetTask(d.ID) if err != nil { t.Fatal(err) } if got.State != task.StateCompleted { t.Fatalf("nested evaluator %s should be auto-accepted to COMPLETED, got %v", d.ID, got.State) } } nestedArbs := store.dependentsWithRole(nestedDeps[0].ID, "planner") if len(nestedArbs) != 1 { t.Fatalf("expected exactly 1 arbitration task for nested, got %d", len(nestedArbs)) } nestedArb := nestedArbs[0] // Arbitration approves nested. store.setTaskState(nestedArb.ID, task.StateReady) store.setTaskSummary(nestedArb.ID, "approved") payload, _ := json.Marshal(struct { Approved bool `json:"approved"` Reasoning string `json:"reasoning"` }{Approved: true, Reasoning: "meets criteria"}) if err := store.CreateEvent(&event.Event{TaskID: nestedArb.ID, Kind: event.KindVerdictReported, Actor: event.ActorAgent, Payload: payload}); err != nil { t.Fatalf("seed nested verdict event: %v", err) } orch.Tick(context.Background()) nestedAfter, err := store.GetTask(nested.ID) if err != nil { t.Fatal(err) } if nestedAfter.State != task.StateCompleted { t.Fatalf("nested should be promoted to COMPLETED now that its arbitration approved it, got %v", nestedAfter.State) } // Story status must still be untouched by the nested node's own // approval -- only root's own approval may set REVIEW_READY. statusOf := func() string { stories, _ := store.ListStories(storage.StoryFilter{}) for _, s := range stories { if s.ID == st.ID { return s.Status } } return "" } if statusOf() != "IN_PROGRESS" { t.Fatalf("story status must remain IN_PROGRESS after only the nested node's approval, got %q", statusOf()) } // Simulate what internal/executor.Pool.maybeUnblockParent would do in a // real deployment now that nested's task.CurrentAttempt resolves to // COMPLETED: promote root BLOCKED -> READY. store.setTaskState(root.ID, task.StateReady) // Tick: root is now READY -> its own 4 evaluators spawn. orch.Tick(context.Background()) rootDeps, _ := store.ListDependents(root.ID) if len(rootDeps) != 4 { t.Fatalf("expected 4 evaluators for root, got %d", len(rootDeps)) } if statusOf() != "VALIDATING" { t.Fatalf("expected VALIDATING once root's own evaluators spawn, got %q", statusOf()) } for i, d := range rootDeps { store.setTaskState(d.ID, task.StateReady) store.setTaskSummary(d.ID, fmt.Sprintf("root verdict %d", i)) orch.Tick(context.Background()) } rootArbs := store.dependentsWithRole(rootDeps[0].ID, "planner") if len(rootArbs) != 1 { t.Fatalf("expected exactly 1 arbitration task for root, got %d", len(rootArbs)) } rootArb := rootArbs[0] store.setTaskState(rootArb.ID, task.StateReady) store.setTaskSummary(rootArb.ID, "approved") rootPayload, _ := json.Marshal(struct { Approved bool `json:"approved"` Reasoning string `json:"reasoning"` }{Approved: true, Reasoning: "meets criteria"}) if err := store.CreateEvent(&event.Event{TaskID: rootArb.ID, Kind: event.KindVerdictReported, Actor: event.ActorAgent, Payload: rootPayload}); err != nil { t.Fatalf("seed root verdict event: %v", err) } orch.Tick(context.Background()) rootAfter, err := store.GetTask(root.ID) if err != nil { t.Fatal(err) } if rootAfter.State != task.StateCompleted { t.Fatalf("root should be promoted to COMPLETED now that its own arbitration approved it, got %v", rootAfter.State) } if statusOf() != "REVIEW_READY" { t.Fatalf("expected REVIEW_READY once root's own arbitration approves it, got %q", statusOf()) } }