package executor import ( "context" "encoding/json" "errors" "io" "log/slog" "net/http" "net/http/httptest" "os" "os/exec" "path/filepath" "strings" "sync" "testing" "github.com/google/uuid" "github.com/thepeterstone/claudomator/internal/llm" "github.com/thepeterstone/claudomator/internal/provider/openaicompat" "github.com/thepeterstone/claudomator/internal/storage" "github.com/thepeterstone/claudomator/internal/task" ) // TestNativeRunner_Run_ChangestatsParseable confirms that a git diff --stat // summary line embedded in the model's assistant text (the only way such a // line can reach stdout.log — tool results themselves are never logged) is // still recoverable by task.ParseChangestatFromFile via the new // NativeRunner/agentloop path, exactly as it was via the old LocalRunner // (neither runner logs raw tool output; both only log assistant text + // result lines, so this has always depended on the model itself echoing the // diff stat in its reply). func TestNativeRunner_Run_ChangestatsParseable(t *testing.T) { srv := fakeChatServer(t, []fakeTurn{ {content: "## Summary\nCommitted the fix.\n2 files changed, 30 insertions(+), 4 deletions(-)", promptTok: 3, outTok: 7}, }) defer srv.Close() r := newLocalRunner(t, srv) tt := localTask() exec := &storage.Execution{ID: uuid.New().String(), TaskID: tt.ID} if err := r.Run(context.Background(), tt, exec, newStoreChannel(&fakeChannelStore{}, tt.ID)); err != nil { t.Fatalf("Run: %v", err) } stdoutPath := filepath.Join(r.ExecLogDir(exec.ID), "stdout.log") cs := task.ParseChangestatFromFile(stdoutPath) if cs == nil { t.Fatalf("ParseChangestatFromFile found nothing in %s", stdoutPath) } if cs.FilesChanged != 2 || cs.LinesAdded != 30 || cs.LinesRemoved != 4 { t.Errorf("changestats: got %+v, want {2 30 4}", cs) } } // This file exercises NativeRunner — the provider.Provider/agentloop.Loop // based successor to the former LocalRunner — via a fake OpenAI-compatible // HTTP server, using the same fake-server test pattern LocalRunner's tests // used (fakeChatServer/fakeTurn/toolCall below), so it verifies behavioral // equivalence: same stdout.log stream-json shape, same summary/changestats // extraction, same tool-loop semantics. // fakeTurn is one canned chat-completion response the fake server returns. type fakeTurn struct { content string toolCalls []llm.ToolCall promptTok int outTok int } // fakeChatServer replies to /chat/completions with the supplied turns in order, // one per request (non-streaming JSON), so a tool-use loop can be driven // deterministically. Requests beyond the list repeat the final turn. func fakeChatServer(t *testing.T, turns []fakeTurn) *httptest.Server { t.Helper() var mu sync.Mutex idx := 0 return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { mu.Lock() turn := turns[idx] if idx < len(turns)-1 { idx++ } mu.Unlock() msg := map[string]any{"role": "assistant", "content": turn.content} if len(turn.toolCalls) > 0 { msg["tool_calls"] = turn.toolCalls } resp := map[string]any{ "model": "fake", "choices": []map[string]any{{"message": msg, "finish_reason": "stop"}}, "usage": map[string]int{"prompt_tokens": turn.promptTok, "completion_tokens": turn.outTok}, } _ = json.NewEncoder(w).Encode(resp) })) } func toolCall(id, name, args string) llm.ToolCall { return llm.ToolCall{ID: id, Type: "function", Function: llm.ToolCallFunction{Name: name, Arguments: args}} } func newLocalRunner(t *testing.T, srv *httptest.Server) *NativeRunner { t.Helper() client := &llm.Client{Endpoint: srv.URL + "/v1", Model: "fake"} return &NativeRunner{ Provider: openaicompat.New(client), Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), LogDir: t.TempDir(), } } func localTask() *task.Task { return &task.Task{ ID: "task-1", Name: "test", Agent: task.AgentConfig{ Type: "local", Model: "fake", Instructions: "Do a thing.", SkipPlanning: true, // keep the preamble out of these assertions }, } } func TestNativeRunner_Run_WritesStreamJSON(t *testing.T) { srv := fakeChatServer(t, []fakeTurn{{content: "## Summary\nAll good.", promptTok: 11, outTok: 22}}) defer srv.Close() r := newLocalRunner(t, srv) tt := localTask() exec := &storage.Execution{ID: uuid.New().String(), TaskID: tt.ID} if err := r.Run(context.Background(), tt, exec, newStoreChannel(&fakeChannelStore{}, tt.ID)); err != nil { t.Fatalf("Run: %v", err) } if exec.CostUSD != 0 { t.Errorf("CostUSD should be 0 for native runner, got %v", exec.CostUSD) } if exec.TokensIn != 11 || exec.TokensOut != 22 { t.Errorf("tokens: want 11/22 got %d/%d", exec.TokensIn, exec.TokensOut) } stdoutPath := filepath.Join(r.ExecLogDir(exec.ID), "stdout.log") data, err := os.ReadFile(stdoutPath) if err != nil { t.Fatalf("read stdout: %v", err) } lines := strings.Split(strings.TrimSpace(string(data)), "\n") // One assistant envelope + one result line. if len(lines) != 2 { t.Fatalf("expected 2 lines (assistant + result), got %d:\n%s", len(lines), data) } var env struct { Type string `json:"type"` } if err := json.Unmarshal([]byte(lines[0]), &env); err != nil || env.Type != "assistant" { t.Fatalf("line 0 should be an assistant envelope: %v: %s", err, lines[0]) } if summary := extractSummary(stdoutPath); !strings.Contains(summary, "All good.") { t.Errorf("extractSummary should find 'All good.', got %q", summary) } } func TestNativeRunner_Run_ToolLoop_ReportSummaryAndSpawn(t *testing.T) { // Turn 1: model spawns a subtask. Turn 2: reports summary. Turn 3: finishes. srv := fakeChatServer(t, []fakeTurn{ {toolCalls: []llm.ToolCall{toolCall("c1", "spawn_subtask", `{"name":"sub A","instructions":"do sub"}`)}, promptTok: 5, outTok: 3}, {toolCalls: []llm.ToolCall{toolCall("c2", "report_summary", `{"summary":"all done"}`)}, promptTok: 4, outTok: 2}, {content: "finished", promptTok: 2, outTok: 1}, }) defer srv.Close() r := newLocalRunner(t, srv) tt := localTask() store := &fakeChannelStore{} ch := newStoreChannel(store, tt.ID) exec := &storage.Execution{ID: uuid.New().String(), TaskID: tt.ID} if err := r.Run(context.Background(), tt, exec, ch); err != nil { t.Fatalf("Run: %v", err) } if len(store.createdTasks) != 1 || store.createdTasks[0].Name != "sub A" { t.Errorf("expected one spawned subtask 'sub A', got %+v", store.createdTasks) } if store.createdTasks[0].ParentTaskID != tt.ID { t.Errorf("subtask parent: want %q, got %q", tt.ID, store.createdTasks[0].ParentTaskID) } if sum, ok := ch.ReportedSummary(); !ok || sum != "all done" { t.Errorf("expected reported summary 'all done', got %q (set=%v)", sum, ok) } // Tokens accumulate across all three turns. if exec.TokensIn != 11 || exec.TokensOut != 6 { t.Errorf("tokens should accumulate: want 11/6, got %d/%d", exec.TokensIn, exec.TokensOut) } } func TestNativeRunner_Run_RecordProgress(t *testing.T) { srv := fakeChatServer(t, []fakeTurn{ {toolCalls: []llm.ToolCall{toolCall("c1", "record_progress", `{"message":"halfway there"}`)}}, {content: "done"}, }) defer srv.Close() store := &fakeChannelStore{} tt := localTask() exec := &storage.Execution{ID: uuid.New().String(), TaskID: tt.ID} if err := newLocalRunner(t, srv).Run(context.Background(), tt, exec, newStoreChannel(store, tt.ID)); err != nil { t.Fatalf("Run: %v", err) } if len(store.createdEvents) != 1 { t.Fatalf("expected 1 progress event, got %d", len(store.createdEvents)) } if !strings.Contains(string(store.createdEvents[0].Payload), "halfway there") { t.Errorf("progress event payload: %s", store.createdEvents[0].Payload) } } func TestNativeRunner_Run_AskUser_Blocks(t *testing.T) { srv := fakeChatServer(t, []fakeTurn{ {toolCalls: []llm.ToolCall{toolCall("c1", "ask_user", `{"question":"which branch?"}`)}}, {content: "should not be reached"}, }) defer srv.Close() tt := localTask() exec := &storage.Execution{ID: uuid.New().String(), TaskID: tt.ID} err := newLocalRunner(t, srv).Run(context.Background(), tt, exec, newStoreChannel(&fakeChannelStore{}, tt.ID)) var be *BlockedError if !errors.As(err, &be) { t.Fatalf("expected *BlockedError, got %v", err) } if !strings.Contains(be.QuestionJSON, "which branch?") { t.Errorf("BlockedError question: %q", be.QuestionJSON) } } func TestNativeRunner_Run_NoProvider_Errors(t *testing.T) { r := &NativeRunner{LogDir: t.TempDir()} tt := &task.Task{ID: "x", Agent: task.AgentConfig{Instructions: "hi"}} exec := &storage.Execution{ID: "exec-x"} err := r.Run(context.Background(), tt, exec, newStoreChannel(nil, tt.ID)) if err == nil || !strings.Contains(err.Error(), "no provider configured") { t.Errorf("expected 'no provider configured' error, got %v", err) } } func TestNativeRunner_Run_EmptyInstructions_Errors(t *testing.T) { r := &NativeRunner{ Provider: openaicompat.New(&llm.Client{Endpoint: "http://unused", Model: "x"}), LogDir: t.TempDir(), } tt := &task.Task{ID: "x", Agent: task.AgentConfig{}} exec := &storage.Execution{ID: "exec-x"} err := r.Run(context.Background(), tt, exec, newStoreChannel(nil, tt.ID)) if err == nil || !strings.Contains(err.Error(), "empty instructions") { t.Errorf("expected empty-instructions error, got %v", err) } } func TestNativeRunner_ExecLogDir(t *testing.T) { r := &NativeRunner{LogDir: "/tmp/logs"} if got := r.ExecLogDir("abc"); got != "/tmp/logs/abc" { t.Errorf("ExecLogDir: got %q", got) } r.LogDir = "" if got := r.ExecLogDir("abc"); got != "" { t.Errorf("ExecLogDir empty LogDir: got %q", got) } } // TestNativeRunner_Run_ProjectDir_SandboxToolsAndPush drives a task with // Agent.ProjectDir set through write_file/read_file/glob/run_bash tool calls // against a real HostSandbox (git clone into a temp dir), confirming the // verbatim-ported sandbox.HostSandbox behavior end-to-end: tool dispatch // hits the real filesystem/shell, a commit made via run_bash is pushed back // to the origin repo on a clean finish, and the sandbox temp dir is removed // afterward — exactly the git-clone/push/cleanup sequence LocalRunner used // to run inline. No prior test (LocalRunner's included) exercised this path; // it's new coverage for logic that was previously only exercised manually. func TestNativeRunner_Run_ProjectDir_SandboxToolsAndPush(t *testing.T) { if _, err := exec.LookPath("git"); err != nil { t.Skip("git not available") } // Bare origin repo so `git push origin HEAD` from the sandbox clone // doesn't hit receive.denyCurrentBranch. origin := t.TempDir() run := func(dir string, args ...string) { t.Helper() cmd := exec.Command("git", args...) cmd.Dir = dir cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=test", "GIT_AUTHOR_EMAIL=test@example.com", "GIT_COMMITTER_NAME=test", "GIT_COMMITTER_EMAIL=test@example.com", ) out, err := cmd.CombinedOutput() if err != nil { t.Fatalf("git %v: %v\n%s", args, err, out) } } run(origin, "init", "--bare", "-b", "main") // Seed the bare repo with an initial commit (an empty bare repo has no // HEAD, so LocalRunner's plain `git clone ` needs something // to check out). seed := t.TempDir() run(seed, "init", "-b", "main") run(seed, "remote", "add", "origin", origin) if err := os.WriteFile(filepath.Join(seed, "README.md"), []byte("seed\n"), 0o644); err != nil { t.Fatal(err) } run(seed, "add", "-A") run(seed, "commit", "-m", "seed") run(seed, "push", "origin", "main") srv := fakeChatServer(t, []fakeTurn{ {toolCalls: []llm.ToolCall{toolCall("c1", "write_file", `{"path":"hello.txt","content":"hello world"}`)}}, {toolCalls: []llm.ToolCall{toolCall("c2", "read_file", `{"path":"hello.txt"}`)}}, {toolCalls: []llm.ToolCall{toolCall("c3", "glob", `{"pattern":"*.txt"}`)}}, {toolCalls: []llm.ToolCall{toolCall("c4", "run_bash", `{"command":"git add -A && git -c user.name=test -c user.email=test@example.com commit -m 'add hello'"}`)}}, {content: "## Summary\nAdded hello.txt and committed."}, }) defer srv.Close() r := newLocalRunner(t, srv) tt := localTask() tt.Agent.ProjectDir = origin exec_ := &storage.Execution{ID: uuid.New().String(), TaskID: tt.ID} if err := r.Run(context.Background(), tt, exec_, newStoreChannel(&fakeChannelStore{}, tt.ID)); err != nil { t.Fatalf("Run: %v", err) } if exec_.SandboxDir != "" { t.Errorf("expected SandboxDir cleared after clean finish, got %q", exec_.SandboxDir) } // Verify the commit actually landed on origin. logCmd := exec.Command("git", "-C", origin, "log", "--oneline", "main") out, err := logCmd.CombinedOutput() if err != nil { t.Fatalf("git log on origin: %v\n%s", err, out) } if !strings.Contains(string(out), "add hello") { t.Errorf("expected pushed commit 'add hello' in origin log, got:\n%s", out) } } // TestNativeRunner_Run_GuardrailRejection_IsNonFatal confirms the Phase 3 // guardrail wiring end-to-end: NativeRunner now always wraps its sandbox in // sandbox.Guarded (see NativeRunner.newBaseSandbox), so a denylisted // run_bash command and a write to a protected path both get rejected // without aborting the run — the model sees the rejection reason as // ordinary tool content (via the sandbox.RejectionError handling added to // agentloop.dispatchTool) and the loop continues to a normal finish, // exactly like a failed shell command would (non-zero exit code, not a // crashed execution). func TestNativeRunner_Run_GuardrailRejection_IsNonFatal(t *testing.T) { if _, err := exec.LookPath("git"); err != nil { t.Skip("git not available") } origin := t.TempDir() run := func(dir string, args ...string) { t.Helper() cmd := exec.Command("git", args...) cmd.Dir = dir cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=test", "GIT_AUTHOR_EMAIL=test@example.com", "GIT_COMMITTER_NAME=test", "GIT_COMMITTER_EMAIL=test@example.com", ) out, err := cmd.CombinedOutput() if err != nil { t.Fatalf("git %v: %v\n%s", args, err, out) } } run(origin, "init", "--bare", "-b", "main") seed := t.TempDir() run(seed, "init", "-b", "main") run(seed, "remote", "add", "origin", origin) if err := os.WriteFile(filepath.Join(seed, "README.md"), []byte("seed\n"), 0o644); err != nil { t.Fatal(err) } run(seed, "add", "-A") run(seed, "commit", "-m", "seed") run(seed, "push", "origin", "main") srv := fakeChatServer(t, []fakeTurn{ // Turn 1: model tries something denylisted via run_bash. {toolCalls: []llm.ToolCall{toolCall("c1", "run_bash", `{"command":"rm -rf /"}`)}}, // Turn 2: model tries to write a protected path. {toolCalls: []llm.ToolCall{toolCall("c2", "write_file", `{"path":".env","content":"SECRET=1"}`)}}, // Turn 3: model recovers and does ordinary, allowed work. {toolCalls: []llm.ToolCall{toolCall("c3", "write_file", `{"path":"hello.txt","content":"hello world"}`)}}, {toolCalls: []llm.ToolCall{toolCall("c4", "run_bash", `{"command":"git add -A && git -c user.name=test -c user.email=test@example.com commit -m 'add hello'"}`)}}, {content: "## Summary\nAdded hello.txt and committed."}, }) defer srv.Close() r := newLocalRunner(t, srv) tt := localTask() tt.Agent.ProjectDir = origin exec_ := &storage.Execution{ID: uuid.New().String(), TaskID: tt.ID} if err := r.Run(context.Background(), tt, exec_, newStoreChannel(&fakeChannelStore{}, tt.ID)); err != nil { t.Fatalf("Run should complete successfully despite rejected tool calls: %v", err) } // The commit from turn 4 should still have landed and been pushed — // proof the run continued normally after both rejections. logCmd := exec.Command("git", "-C", origin, "log", "--oneline", "main") out, err := logCmd.CombinedOutput() if err != nil { t.Fatalf("git log on origin: %v\n%s", err, out) } if !strings.Contains(string(out), "add hello") { t.Errorf("expected pushed commit 'add hello' in origin log, got:\n%s", out) } // And .env must never have been written into the sandbox — the write // was rejected before it ever reached the wrapped Sandbox. if _, err := os.Stat(filepath.Join(origin, ".env")); err == nil { t.Error(".env should not exist in origin (write was rejected, nothing to push)") } }