From 204fe6c536d5d3bca2cb712f66b1d63054807dac Mon Sep 17 00:00:00 2001 From: Peter Stone Date: Sat, 11 Jul 2026 05:37:45 +0000 Subject: fix(scheduler): make arbitration fail-closed on a missing verdict, seed planner's system prompt A live production run (2026-07-11) showed finalizeArbitration's 'no verdict reported = approve' default silently shipping work an evaluator had already flagged as factually wrong -- the arbitration agent never called report_verdict because the planner role had no system prompt telling it to. finalizeArbitration now treats a missing verdict the same as an explicit rejection (fail-closed), and SeedRoleConfigs now also seeds planner with a prompt that explicitly mandates calling report_verdict before finishing. --- internal/storage/seed.go | 32 ++++++++++++++++++++++++++ internal/storage/seed_test.go | 52 ++++++++++++++++++++++++++++++++++++++----- 2 files changed, 78 insertions(+), 6 deletions(-) (limited to 'internal/storage') diff --git a/internal/storage/seed.go b/internal/storage/seed.go index b452969..e7a0ba0 100644 --- a/internal/storage/seed.go +++ b/internal/storage/seed.go @@ -75,6 +75,7 @@ func (s *DB) SeedRoleConfigs() error { configJSON string }{ {role: "builder", configJSON: builderRoleConfigJSON}, + {role: "planner", configJSON: plannerRoleConfigJSON}, } for _, seed := range seeds { if _, err := s.GetActiveRoleConfig(seed.role); err == nil { @@ -120,3 +121,34 @@ const builderRoleConfigJSON = `{ } ] }` + +// plannerRoleConfigJSON is the default role.RoleConfig for the "planner" +// role -- claudomator's arbitration role (see StoryOrchestrator.ensureArbitration +// in internal/scheduler/story_orchestrator.go). Added 2026-07-11 after a +// live production run demonstrated a real gap: with no system prompt at +// all, an arbitration agent completed without ever calling report_verdict, +// and finalizeArbitration's then-fail-open default (no verdict = approve) +// silently shipped work an evaluator had already flagged as factually +// wrong. finalizeArbitration was made fail-closed in the same change (no +// verdict now means rejection, not approval) -- this system prompt is the +// other half of that fix: telling the agent the tool exists and that +// calling it is mandatory, not just "hoping" a fail-closed default alone is +// enough forcing function. See report_verdict's schema in +// internal/executor/agentmcp.go's reportVerdictInput (approved bool, +// reasoning string). +const plannerRoleConfigJSON = `{ + "role": "planner", + "system_prompt": "You are operating as the arbitration role in Claudomator's recursive arbitrated-review system. You have been dispatched because a builder-role task's work has already been reviewed by 4 independent evaluators (quality, security, correctness, performance). Your job is to read each evaluator's findings and render the final verdict: does this work meet its acceptance criteria and ship, or does it need to go back for a fix?\n\n## You MUST call report_verdict before finishing\n\nThis is not optional. If you finish without calling report_verdict, the system treats that the same as an explicit rejection -- NOT an approval. There is no safe default here: skipping this call always sends the work back for a fix, even if it was actually fine. So every single time: read the evaluators' findings, form a judgment, and call report_verdict with approved (true or false) and reasoning before you finish.\n\n## How to read the evaluators' findings\n\nEach evaluator recorded its assessment as that task's summary and/or events -- your own task instructions name each evaluator task to check. Read all 4 before deciding. Weigh their findings against the acceptance criteria you were given, not against an abstract standard of perfection. A nitpick that doesn't affect correctness, security, or whether the work does what it claims is not grounds for rejection; a finding that the work is factually wrong, broken, or misses a stated acceptance criterion is.\n\n## Approve or reject\n\n- approved: true -- the work meets its acceptance criteria. The builder task is promoted to COMPLETED.\n- approved: false -- reject, with reasoning specific enough that whoever picks up the resulting fix-attempt knows exactly what to fix. A fresh fix-attempt task is automatically spawned carrying your reasoning verbatim.\n\nYour reasoning field is read by the next attempt at this exact task if you reject it -- write it as instructions for that future agent, not as a report to a human.", + "escalation_ladder": [ + { + "candidates": [{"provider": "claude", "model": "sonnet"}], + "selection_mode": "single", + "max_retries": 1 + }, + { + "candidates": [{"provider": "claude", "model": "opus"}], + "selection_mode": "single", + "max_retries": 0 + } + ] +}` diff --git a/internal/storage/seed_test.go b/internal/storage/seed_test.go index cd7c8e7..5b5b674 100644 --- a/internal/storage/seed_test.go +++ b/internal/storage/seed_test.go @@ -2,6 +2,7 @@ package storage import ( "encoding/json" + "strings" "testing" "github.com/thepeterstone/claudomator/internal/role" @@ -43,6 +44,43 @@ func TestSeedRoleConfigs_CreatesAndActivatesBuilder(t *testing.T) { } } +// TestSeedRoleConfigs_CreatesAndActivatesPlanner proves the arbitration +// role's system prompt is seeded too (added 2026-07-11 alongside +// finalizeArbitration's fail-closed default -- see that const's doc +// comment for why: a live run showed an arbitration agent with no system +// prompt at all never called report_verdict). Specifically checks the +// prompt actually names the report_verdict tool, since that's the whole +// point of this seed. +func TestSeedRoleConfigs_CreatesAndActivatesPlanner(t *testing.T) { + db := testDB(t) + + if err := db.SeedRoleConfigs(); err != nil { + t.Fatalf("SeedRoleConfigs: %v", err) + } + + row, err := db.GetActiveRoleConfig("planner") + if err != nil { + t.Fatalf("GetActiveRoleConfig(planner): %v", err) + } + if row.Version != 1 { + t.Errorf("expected version 1, got %d", row.Version) + } + + var cfg role.RoleConfig + if err := json.Unmarshal([]byte(row.ConfigJSON), &cfg); err != nil { + t.Fatalf("unmarshal seeded config: %v", err) + } + if cfg.Role != "planner" { + t.Errorf("cfg.Role = %q, want planner", cfg.Role) + } + if !strings.Contains(cfg.SystemPrompt, "report_verdict") { + t.Error("cfg.SystemPrompt does not mention report_verdict -- the whole point of this seed is mandating that call") + } + if len(cfg.EscalationLadder) != 2 { + t.Fatalf("expected 2 escalation tiers, got %d", len(cfg.EscalationLadder)) + } +} + func TestSeedRoleConfigs_Idempotent_DoesNotDuplicateOrOverwrite(t *testing.T) { db := testDB(t) @@ -53,12 +91,14 @@ func TestSeedRoleConfigs_Idempotent_DoesNotDuplicateOrOverwrite(t *testing.T) { t.Fatalf("second SeedRoleConfigs: %v", err) } - versions, err := db.ListRoleConfigVersions("builder") - if err != nil { - t.Fatalf("ListRoleConfigVersions: %v", err) - } - if len(versions) != 1 { - t.Fatalf("expected exactly 1 version after calling SeedRoleConfigs twice, got %d", len(versions)) + for _, r := range []string{"builder", "planner"} { + versions, err := db.ListRoleConfigVersions(r) + if err != nil { + t.Fatalf("ListRoleConfigVersions(%s): %v", r, err) + } + if len(versions) != 1 { + t.Fatalf("%s: expected exactly 1 version after calling SeedRoleConfigs twice, got %d", r, len(versions)) + } } } -- cgit v1.2.3