diff options
| author | Claude <noreply@anthropic.com> | 2026-05-26 07:18:07 +0000 |
|---|---|---|
| committer | Claude <noreply@anthropic.com> | 2026-05-26 07:18:07 +0000 |
| commit | fb7f8a4c07a8bf6539566aacbc8a92310c555992 (patch) | |
| tree | dceb87f179823939c5a11f390921b7b185a56cf0 /cmd/spike | |
| parent | 301e7a66387f99ab76754d08bca42f4a9930d3b1 (diff) | |
test(spike): manual agent-MCP validation harness + Phase 2 validation findings
Adds cmd/spike, a manual (never run by CI) harness that starts the agent MCP
server on the host and points a real agent CLI at it to confirm tool calls reach
the AgentChannel. Validates Phase 2 end-to-end without Docker.
Findings against real claude 2.1.150 (sonnet-4-6):
- PASS: claude discovers the server via --mcp-config (streamable HTTP + bearer)
and calls record_progress + report_summary; both reach the AgentChannel,
zero permission denials.
- PASS: claude calls ask_user, gets the ErrAgentBlocked "end your turn"
response, and ends its turn cleanly (human-in-the-loop path).
- PRODUCTION NOTE: claude rejects --permission-mode bypassPermissions under
root — the in-container agent must run non-root (ContainerRunner sets
--user to the host uid), or tools must be pre-allowed via --allowedTools.
- GEMINI spike could not run (no gemini binary); the harness's gemini path
writes the production mcpServers/httpUrl settings and is ready to run where
a binary exists.
https://claude.ai/code/session_01SESwn7kQ7oP62trWw6pc39
Diffstat (limited to 'cmd/spike')
| -rw-r--r-- | cmd/spike/main.go | 165 |
1 files changed, 165 insertions, 0 deletions
diff --git a/cmd/spike/main.go b/cmd/spike/main.go new file mode 100644 index 0000000..dd87021 --- /dev/null +++ b/cmd/spike/main.go @@ -0,0 +1,165 @@ +// Command spike is a MANUAL validation harness (never run by tests/CI; it +// spawns a real agent CLI and makes paid API calls). It validates the agent MCP +// back-channel against a real binary without Docker: it starts the agent MCP +// server on the host, points the chosen agent CLI at it, and reports whether the +// agent's tool calls reached the AgentChannel. +// +// Usage: +// +// go run ./cmd/spike # claude (default) +// go run ./cmd/spike gemini # gemini (requires a `gemini` binary) +// +// FINDINGS (2026-05, claude 2.1.150 / claude-sonnet-4-6): +// - PASS: claude discovered the server via --mcp-config (streamable HTTP + +// bearer) and called record_progress + report_summary; both reached the +// AgentChannel with zero permission denials (~$0.11/run). +// - PASS: claude called ask_user, received the ErrAgentBlocked "end your turn" +// response, and ended its turn cleanly (the human-in-the-loop path). +// - PRODUCTION NOTE: `claude --permission-mode bypassPermissions` (what +// ContainerRunner.buildInnerCmd emits) is REJECTED when the process is root. +// The in-container agent must run as a non-root user, or tools must be +// pre-allowed via --allowedTools. Worth verifying the container image's user. +// - GEMINI: not validated — no `gemini` binary was available. The gemini path +// below writes the same mcpServers/httpUrl settings the production +// ContainerRunner does; run `go run ./cmd/spike gemini` where a binary exists. +package main + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/thepeterstone/claudomator/internal/executor" +) + +type recordingChannel struct { + mu sync.Mutex + asks []string + summary string + progress []string + subtasks []executor.SubtaskSpec +} + +func (c *recordingChannel) AskUser(_ context.Context, q string) (string, error) { + c.mu.Lock() + c.asks = append(c.asks, q) + c.mu.Unlock() + return "", executor.ErrAgentBlocked // mirror production storeChannel +} +func (c *recordingChannel) ReportSummary(_ context.Context, s string) error { + c.mu.Lock() + c.summary = s + c.mu.Unlock() + return nil +} +func (c *recordingChannel) SpawnSubtask(_ context.Context, spec executor.SubtaskSpec) (string, error) { + c.mu.Lock() + c.subtasks = append(c.subtasks, spec) + c.mu.Unlock() + return "spike-subtask-1", nil +} +func (c *recordingChannel) RecordProgress(_ context.Context, m string) error { + c.mu.Lock() + c.progress = append(c.progress, m) + c.mu.Unlock() + return nil +} + +func main() { + agent := "claude" + if len(os.Args) > 1 { + agent = os.Args[1] + } + + rec := &recordingChannel{} + reg := executor.NewRegistry() + token, err := reg.Mint(rec) + if err != nil { + fmt.Println("mint:", err) + os.Exit(1) + } + + mux := http.NewServeMux() + mux.Handle("/mcp", executor.NewAgentMCPHandler(reg)) + srv := httptest.NewServer(mux) + defer srv.Close() + mcpURL := srv.URL + "/mcp" + + prompt := "You have MCP tools from the 'claudomator' server. Do exactly this and nothing else: " + + "(1) call record_progress with message 'spike: starting'; " + + "(2) call report_summary with summary 'spike validation complete'; " + + "then stop. Do not edit files or run shell commands." + + ctx, cancel := context.WithTimeout(context.Background(), 150*time.Second) + defer cancel() + + var cmd *exec.Cmd + switch agent { + case "claude": + cfg := map[string]any{"mcpServers": map[string]any{"claudomator": map[string]any{ + "type": "http", "url": mcpURL, "headers": map[string]string{"Authorization": "Bearer " + token}, + }}} + data, _ := json.Marshal(cfg) + cfgPath := "/tmp/spike-claude-mcp.json" + _ = os.WriteFile(cfgPath, data, 0600) + allowed := "mcp__claudomator__record_progress,mcp__claudomator__report_summary,mcp__claudomator__ask_user,mcp__claudomator__spawn_subtask" + cmd = exec.CommandContext(ctx, "claude", "-p", prompt, + "--mcp-config", cfgPath, "--allowedTools", allowed, + "--output-format", "stream-json", "--verbose") + case "gemini": + home, _ := os.MkdirTemp("", "spike-gemini-home-*") + geminiDir := filepath.Join(home, ".gemini") + _ = os.MkdirAll(geminiDir, 0755) + cfg := map[string]any{"mcpServers": map[string]any{"claudomator": map[string]any{ + "httpUrl": mcpURL, "headers": map[string]string{"Authorization": "Bearer " + token}, + }}} + data, _ := json.Marshal(cfg) + _ = os.WriteFile(filepath.Join(geminiDir, "settings.json"), data, 0600) + cmd = exec.CommandContext(ctx, "gemini", "-p", prompt) + cmd.Env = append(os.Environ(), "HOME="+home) + default: + fmt.Println("unknown agent:", agent) + os.Exit(1) + } + + var out, errb strings.Builder + cmd.Stdout = &out + cmd.Stderr = &errb + start := time.Now() + runErr := cmd.Run() + + fmt.Printf("=== %s exit: err=%v elapsed=%s ===\n", agent, runErr, time.Since(start).Round(time.Millisecond)) + fmt.Println("--- stderr (tail) ---") + fmt.Println(tail(errb.String(), 1200)) + fmt.Println("--- stdout (tail) ---") + fmt.Println(tail(out.String(), 1500)) + + rec.mu.Lock() + defer rec.mu.Unlock() + fmt.Println("=== AgentChannel received ===") + fmt.Printf("record_progress: %d %v\n", len(rec.progress), rec.progress) + fmt.Printf("report_summary: %q\n", rec.summary) + fmt.Printf("ask_user: %d\n", len(rec.asks)) + fmt.Printf("spawn_subtask: %d\n", len(rec.subtasks)) + + if len(rec.progress) > 0 && rec.summary != "" { + fmt.Println("RESULT: PASS — the agent discovered and called the MCP tools; calls reached the AgentChannel.") + } else { + fmt.Println("RESULT: INCONCLUSIVE — see output above.") + } +} + +func tail(s string, n int) string { + if len(s) <= n { + return s + } + return "…" + s[len(s)-n:] +} |
