summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorClaude <noreply@anthropic.com>2026-05-26 07:18:07 +0000
committerClaude <noreply@anthropic.com>2026-05-26 07:18:07 +0000
commitfb7f8a4c07a8bf6539566aacbc8a92310c555992 (patch)
treedceb87f179823939c5a11f390921b7b185a56cf0
parent301e7a66387f99ab76754d08bca42f4a9930d3b1 (diff)
test(spike): manual agent-MCP validation harness + Phase 2 validation findings
Adds cmd/spike, a manual (never run by CI) harness that starts the agent MCP server on the host and points a real agent CLI at it to confirm tool calls reach the AgentChannel. Validates Phase 2 end-to-end without Docker. Findings against real claude 2.1.150 (sonnet-4-6): - PASS: claude discovers the server via --mcp-config (streamable HTTP + bearer) and calls record_progress + report_summary; both reach the AgentChannel, zero permission denials. - PASS: claude calls ask_user, gets the ErrAgentBlocked "end your turn" response, and ends its turn cleanly (human-in-the-loop path). - PRODUCTION NOTE: claude rejects --permission-mode bypassPermissions under root — the in-container agent must run non-root (ContainerRunner sets --user to the host uid), or tools must be pre-allowed via --allowedTools. - GEMINI spike could not run (no gemini binary); the harness's gemini path writes the production mcpServers/httpUrl settings and is ready to run where a binary exists. https://claude.ai/code/session_01SESwn7kQ7oP62trWw6pc39
-rw-r--r--.gitignore1
-rw-r--r--cmd/spike/main.go165
2 files changed, 166 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore
index 0872ffd..5a6cfb3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -45,3 +45,4 @@ SESSION_STATE.md
.claude/
credentials/
scripts/.claude/
+/spike
diff --git a/cmd/spike/main.go b/cmd/spike/main.go
new file mode 100644
index 0000000..dd87021
--- /dev/null
+++ b/cmd/spike/main.go
@@ -0,0 +1,165 @@
+// Command spike is a MANUAL validation harness (never run by tests/CI; it
+// spawns a real agent CLI and makes paid API calls). It validates the agent MCP
+// back-channel against a real binary without Docker: it starts the agent MCP
+// server on the host, points the chosen agent CLI at it, and reports whether the
+// agent's tool calls reached the AgentChannel.
+//
+// Usage:
+//
+// go run ./cmd/spike # claude (default)
+// go run ./cmd/spike gemini # gemini (requires a `gemini` binary)
+//
+// FINDINGS (2026-05, claude 2.1.150 / claude-sonnet-4-6):
+// - PASS: claude discovered the server via --mcp-config (streamable HTTP +
+// bearer) and called record_progress + report_summary; both reached the
+// AgentChannel with zero permission denials (~$0.11/run).
+// - PASS: claude called ask_user, received the ErrAgentBlocked "end your turn"
+// response, and ended its turn cleanly (the human-in-the-loop path).
+// - PRODUCTION NOTE: `claude --permission-mode bypassPermissions` (what
+// ContainerRunner.buildInnerCmd emits) is REJECTED when the process is root.
+// The in-container agent must run as a non-root user, or tools must be
+// pre-allowed via --allowedTools. Worth verifying the container image's user.
+// - GEMINI: not validated — no `gemini` binary was available. The gemini path
+// below writes the same mcpServers/httpUrl settings the production
+// ContainerRunner does; run `go run ./cmd/spike gemini` where a binary exists.
+package main
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/thepeterstone/claudomator/internal/executor"
+)
+
+type recordingChannel struct {
+ mu sync.Mutex
+ asks []string
+ summary string
+ progress []string
+ subtasks []executor.SubtaskSpec
+}
+
+func (c *recordingChannel) AskUser(_ context.Context, q string) (string, error) {
+ c.mu.Lock()
+ c.asks = append(c.asks, q)
+ c.mu.Unlock()
+ return "", executor.ErrAgentBlocked // mirror production storeChannel
+}
+func (c *recordingChannel) ReportSummary(_ context.Context, s string) error {
+ c.mu.Lock()
+ c.summary = s
+ c.mu.Unlock()
+ return nil
+}
+func (c *recordingChannel) SpawnSubtask(_ context.Context, spec executor.SubtaskSpec) (string, error) {
+ c.mu.Lock()
+ c.subtasks = append(c.subtasks, spec)
+ c.mu.Unlock()
+ return "spike-subtask-1", nil
+}
+func (c *recordingChannel) RecordProgress(_ context.Context, m string) error {
+ c.mu.Lock()
+ c.progress = append(c.progress, m)
+ c.mu.Unlock()
+ return nil
+}
+
+func main() {
+ agent := "claude"
+ if len(os.Args) > 1 {
+ agent = os.Args[1]
+ }
+
+ rec := &recordingChannel{}
+ reg := executor.NewRegistry()
+ token, err := reg.Mint(rec)
+ if err != nil {
+ fmt.Println("mint:", err)
+ os.Exit(1)
+ }
+
+ mux := http.NewServeMux()
+ mux.Handle("/mcp", executor.NewAgentMCPHandler(reg))
+ srv := httptest.NewServer(mux)
+ defer srv.Close()
+ mcpURL := srv.URL + "/mcp"
+
+ prompt := "You have MCP tools from the 'claudomator' server. Do exactly this and nothing else: " +
+ "(1) call record_progress with message 'spike: starting'; " +
+ "(2) call report_summary with summary 'spike validation complete'; " +
+ "then stop. Do not edit files or run shell commands."
+
+ ctx, cancel := context.WithTimeout(context.Background(), 150*time.Second)
+ defer cancel()
+
+ var cmd *exec.Cmd
+ switch agent {
+ case "claude":
+ cfg := map[string]any{"mcpServers": map[string]any{"claudomator": map[string]any{
+ "type": "http", "url": mcpURL, "headers": map[string]string{"Authorization": "Bearer " + token},
+ }}}
+ data, _ := json.Marshal(cfg)
+ cfgPath := "/tmp/spike-claude-mcp.json"
+ _ = os.WriteFile(cfgPath, data, 0600)
+ allowed := "mcp__claudomator__record_progress,mcp__claudomator__report_summary,mcp__claudomator__ask_user,mcp__claudomator__spawn_subtask"
+ cmd = exec.CommandContext(ctx, "claude", "-p", prompt,
+ "--mcp-config", cfgPath, "--allowedTools", allowed,
+ "--output-format", "stream-json", "--verbose")
+ case "gemini":
+ home, _ := os.MkdirTemp("", "spike-gemini-home-*")
+ geminiDir := filepath.Join(home, ".gemini")
+ _ = os.MkdirAll(geminiDir, 0755)
+ cfg := map[string]any{"mcpServers": map[string]any{"claudomator": map[string]any{
+ "httpUrl": mcpURL, "headers": map[string]string{"Authorization": "Bearer " + token},
+ }}}
+ data, _ := json.Marshal(cfg)
+ _ = os.WriteFile(filepath.Join(geminiDir, "settings.json"), data, 0600)
+ cmd = exec.CommandContext(ctx, "gemini", "-p", prompt)
+ cmd.Env = append(os.Environ(), "HOME="+home)
+ default:
+ fmt.Println("unknown agent:", agent)
+ os.Exit(1)
+ }
+
+ var out, errb strings.Builder
+ cmd.Stdout = &out
+ cmd.Stderr = &errb
+ start := time.Now()
+ runErr := cmd.Run()
+
+ fmt.Printf("=== %s exit: err=%v elapsed=%s ===\n", agent, runErr, time.Since(start).Round(time.Millisecond))
+ fmt.Println("--- stderr (tail) ---")
+ fmt.Println(tail(errb.String(), 1200))
+ fmt.Println("--- stdout (tail) ---")
+ fmt.Println(tail(out.String(), 1500))
+
+ rec.mu.Lock()
+ defer rec.mu.Unlock()
+ fmt.Println("=== AgentChannel received ===")
+ fmt.Printf("record_progress: %d %v\n", len(rec.progress), rec.progress)
+ fmt.Printf("report_summary: %q\n", rec.summary)
+ fmt.Printf("ask_user: %d\n", len(rec.asks))
+ fmt.Printf("spawn_subtask: %d\n", len(rec.subtasks))
+
+ if len(rec.progress) > 0 && rec.summary != "" {
+ fmt.Println("RESULT: PASS — the agent discovered and called the MCP tools; calls reached the AgentChannel.")
+ } else {
+ fmt.Println("RESULT: INCONCLUSIVE — see output above.")
+ }
+}
+
+func tail(s string, n int) string {
+ if len(s) <= n {
+ return s
+ }
+ return "…" + s[len(s)-n:]
+}