From e38f673edc7428c0c836c39f40707cb681defb14 Mon Sep 17 00:00:00 2001 From: Claude Sonnet 5 Date: Fri, 3 Jul 2026 09:03:01 +0000 Subject: feat(provider): add native Anthropic Messages API adapter (Phase 2) Adds internal/provider/anthropic, the first genuinely new provider.Provider implementation on top of Phase 1's provider-neutral tool-use loop, alongside (not replacing) the existing Docker/CLI-subprocess ContainerRunner path for the "claude" agent type: - internal/provider/anthropic: translates the neutral ChatRequest/ChatResponse shape to/from Anthropic's Messages API content-block format (system as a top-level field, tool_use/tool_result blocks, no "tool" role -- tool results become user-role messages), with a per-model-prefix pricing table for CostUSD - internal/retry: IsRateLimitError additively extended to recognize Anthropic's rate_limit_error/overloaded_error/529 shapes - internal/config: RunnersConfig.Anthropic/AnthropicEnabled() gate - internal/cli/serve.go, run.go: register runners["anthropic"] as a NativeRunner when [providers.anthropic].api_key is set and enabled -- tracked as a distinct executions.agent="anthropic" budget bucket, separate from the CLI-subprocess "claude" runner even though both bill the same Anthropic account go build/vet/test -race all pass. No live Anthropic API key is available in this environment, so verification is via fake-httptest-server adapter tests (12 cases, incl. multi-turn tool_result round-trip and rate-limit error matching) plus a Pool/NativeRunner routing test proving agent.type: "anthropic" actually reaches the new provider. Live end-to-end verification against the real API is a follow-up once a key is configured. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01V1moSNCJRcP6kykA4tyUSs --- internal/provider/anthropic/anthropic_test.go | 416 ++++++++++++++++++++++++++ 1 file changed, 416 insertions(+) create mode 100644 internal/provider/anthropic/anthropic_test.go (limited to 'internal/provider/anthropic/anthropic_test.go') diff --git a/internal/provider/anthropic/anthropic_test.go b/internal/provider/anthropic/anthropic_test.go new file mode 100644 index 0000000..ca7e778 --- /dev/null +++ b/internal/provider/anthropic/anthropic_test.go @@ -0,0 +1,416 @@ +package anthropic + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/thepeterstone/claudomator/internal/provider" + "github.com/thepeterstone/claudomator/internal/retry" +) + +// fakeAnthropicServer replies to POST /v1/messages with the given status and +// raw JSON body, capturing the last decoded request for assertions. Modeled +// on the fake OpenAI-compatible server pattern used in +// internal/llm/client_test.go and internal/executor/nativerunner_test.go +// (fakeChatServer), adapted to the Anthropic Messages API wire shape. +func fakeAnthropicServer(t *testing.T, status int, body string, capture *wireRequest) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/messages" { + t.Errorf("unexpected path %q", r.URL.Path) + } + if r.Header.Get("x-api-key") != "test-key" { + t.Errorf("missing/wrong x-api-key header: %q", r.Header.Get("x-api-key")) + } + if r.Header.Get("anthropic-version") != APIVersion { + t.Errorf("missing/wrong anthropic-version header: %q", r.Header.Get("anthropic-version")) + } + if r.Header.Get("content-type") != "application/json" { + t.Errorf("missing/wrong content-type header: %q", r.Header.Get("content-type")) + } + if capture != nil { + if err := json.NewDecoder(r.Body).Decode(capture); err != nil { + t.Fatalf("decode request body: %v", err) + } + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + w.Write([]byte(body)) + })) +} + +func newTestProvider(url string) *Provider { + return New("test-key", url, 5*time.Second) +} + +// TestChat_PlainTextResponse covers a plain assistant text reply: no tool +// calls, stop_reason "end_turn", usage translated to provider.Usage with a +// non-zero CostUSD computed from the pricing table. +func TestChat_PlainTextResponse(t *testing.T) { + var got wireRequest + srv := fakeAnthropicServer(t, http.StatusOK, `{ + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5", + "content": [{"type": "text", "text": "Hello there!"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 5} + }`, &got) + defer srv.Close() + + p := newTestProvider(srv.URL) + resp, err := p.Chat(context.Background(), provider.ChatRequest{ + Model: "claude-sonnet-5", + System: "You are helpful.", + Messages: []provider.Message{ + {Role: "user", Text: "hi"}, + }, + }) + if err != nil { + t.Fatalf("Chat: %v", err) + } + if resp.Text != "Hello there!" { + t.Errorf("Text: want %q got %q", "Hello there!", resp.Text) + } + if len(resp.ToolCalls) != 0 { + t.Errorf("expected no tool calls, got %+v", resp.ToolCalls) + } + if resp.StopReason != "end_turn" { + t.Errorf("StopReason: want end_turn got %q", resp.StopReason) + } + if resp.Usage.InputTokens != 10 || resp.Usage.OutputTokens != 5 { + t.Errorf("usage tokens: got %+v", resp.Usage) + } + wantCost := float64(10)/1_000_000*3.00 + float64(5)/1_000_000*15.00 + if resp.Usage.CostUSD != wantCost { + t.Errorf("CostUSD: want %v got %v", wantCost, resp.Usage.CostUSD) + } + + // Verify the outgoing request shape: system is top-level, not a message. + if got.System != "You are helpful." { + t.Errorf("system: want %q got %q", "You are helpful.", got.System) + } + if len(got.Messages) != 1 || got.Messages[0].Role != "user" { + t.Fatalf("messages: %+v", got.Messages) + } + if len(got.Messages[0].Content) != 1 || got.Messages[0].Content[0].Type != "text" || got.Messages[0].Content[0].Text != "hi" { + t.Errorf("user content block: %+v", got.Messages[0].Content) + } + if got.MaxTokens != defaultMaxTokens { + t.Errorf("max_tokens should default to %d, got %d", defaultMaxTokens, got.MaxTokens) + } +} + +// TestChat_ToolUseResponse verifies that a response whose content mixes a +// text block and a tool_use block round-trips into ChatResponse.ToolCalls +// correctly (id/name/args), and that tools declared on the request are +// translated to Anthropic's {name, description, input_schema} shape. +func TestChat_ToolUseResponse(t *testing.T) { + var got wireRequest + srv := fakeAnthropicServer(t, http.StatusOK, `{ + "id": "msg_2", + "type": "message", + "role": "assistant", + "model": "claude-opus-4-8", + "content": [ + {"type": "text", "text": "Let me check that."}, + {"type": "tool_use", "id": "toolu_01", "name": "read_file", "input": {"path": "a.txt"}} + ], + "stop_reason": "tool_use", + "usage": {"input_tokens": 20, "output_tokens": 8} + }`, &got) + defer srv.Close() + + p := newTestProvider(srv.URL) + resp, err := p.Chat(context.Background(), provider.ChatRequest{ + Model: "claude-opus-4-8", + Messages: []provider.Message{{Role: "user", Text: "read a.txt"}}, + Tools: []provider.ToolSpec{ + { + Name: "read_file", + Description: "Read a file.", + ParametersJSONSchema: map[string]any{ + "type": "object", + "properties": map[string]any{"path": map[string]any{"type": "string"}}, + "required": []string{"path"}, + }, + }, + }, + }) + if err != nil { + t.Fatalf("Chat: %v", err) + } + if resp.Text != "Let me check that." { + t.Errorf("Text: got %q", resp.Text) + } + if resp.StopReason != "tool_use" { + t.Errorf("StopReason: want tool_use got %q", resp.StopReason) + } + if len(resp.ToolCalls) != 1 { + t.Fatalf("expected 1 tool call, got %d: %+v", len(resp.ToolCalls), resp.ToolCalls) + } + tc := resp.ToolCalls[0] + if tc.ID != "toolu_01" || tc.Name != "read_file" { + t.Errorf("tool call id/name: got %+v", tc) + } + var args struct { + Path string `json:"path"` + } + if err := json.Unmarshal([]byte(tc.ArgsJSON), &args); err != nil { + t.Fatalf("tool call args not valid JSON: %v (%q)", err, tc.ArgsJSON) + } + if args.Path != "a.txt" { + t.Errorf("tool call args.path: got %q", args.Path) + } + + // Verify tools were declared with input_schema (not "parameters"). + if len(got.Tools) != 1 { + t.Fatalf("expected 1 tool declared, got %d", len(got.Tools)) + } + if got.Tools[0].Name != "read_file" || got.Tools[0].Description != "Read a file." { + t.Errorf("tool declaration: %+v", got.Tools[0]) + } + if got.Tools[0].InputSchema == nil || got.Tools[0].InputSchema["type"] != "object" { + t.Errorf("tool input_schema not forwarded: %+v", got.Tools[0].InputSchema) + } +} + +// TestChat_MultiTurnWithPriorToolResult exercises the tool_result-as-a- +// user-message translation: a provider.Message carrying ToolResults (as +// agentloop.Loop emits with Role "tool" after dispatching a tool call) must +// serialize as a user-role message with tool_result content blocks — +// Anthropic has no "tool" role. Also verifies a prior assistant turn with a +// tool call round-trips into a single assistant message with a tool_use +// block matching the id used in the subsequent tool_result. +func TestChat_MultiTurnWithPriorToolResult(t *testing.T) { + var got wireRequest + srv := fakeAnthropicServer(t, http.StatusOK, `{ + "id": "msg_3", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5", + "content": [{"type": "text", "text": "The file says hello."}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 30, "output_tokens": 6} + }`, &got) + defer srv.Close() + + p := newTestProvider(srv.URL) + _, err := p.Chat(context.Background(), provider.ChatRequest{ + Model: "claude-sonnet-5", + Messages: []provider.Message{ + {Role: "user", Text: "read a.txt"}, + { + Role: "assistant", + Text: "", + ToolCalls: []provider.ToolCall{ + {ID: "toolu_01", Name: "read_file", ArgsJSON: `{"path":"a.txt"}`}, + }, + }, + { + Role: "tool", + ToolResults: []provider.ToolResult{ + {ToolCallID: "toolu_01", Name: "read_file", Content: "hello", IsError: false}, + }, + }, + }, + }) + if err != nil { + t.Fatalf("Chat: %v", err) + } + + if len(got.Messages) != 3 { + t.Fatalf("expected 3 wire messages, got %d: %+v", len(got.Messages), got.Messages) + } + + // Turn 2: assistant tool_use, no text block since Text was empty. + asst := got.Messages[1] + if asst.Role != "assistant" { + t.Errorf("turn 2 role: want assistant got %q", asst.Role) + } + if len(asst.Content) != 1 || asst.Content[0].Type != "tool_use" { + t.Fatalf("turn 2 content: %+v", asst.Content) + } + if asst.Content[0].ID != "toolu_01" || asst.Content[0].Name != "read_file" { + t.Errorf("turn 2 tool_use block: %+v", asst.Content[0]) + } + + // Turn 3: the tool result must be a USER-role message with a + // tool_result block (Anthropic has no "tool" role). + toolTurn := got.Messages[2] + if toolTurn.Role != "user" { + t.Errorf("tool result turn role: want user (Anthropic has no tool role) got %q", toolTurn.Role) + } + if len(toolTurn.Content) != 1 || toolTurn.Content[0].Type != "tool_result" { + t.Fatalf("tool result content: %+v", toolTurn.Content) + } + block := toolTurn.Content[0] + if block.ToolUseID != "toolu_01" { + t.Errorf("tool_use_id: want toolu_01 got %q", block.ToolUseID) + } + if block.Content != "hello" { + t.Errorf("tool_result content: want hello got %q", block.Content) + } + if block.IsError { + t.Errorf("is_error should be false") + } +} + +// TestChat_ToolResultIsError verifies IsError round-trips onto the +// tool_result block's is_error field. +func TestChat_ToolResultIsError(t *testing.T) { + var got wireRequest + srv := fakeAnthropicServer(t, http.StatusOK, `{ + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 1, "output_tokens": 1} + }`, &got) + defer srv.Close() + + p := newTestProvider(srv.URL) + _, err := p.Chat(context.Background(), provider.ChatRequest{ + Model: "claude-sonnet-5", + Messages: []provider.Message{ + {Role: "tool", ToolResults: []provider.ToolResult{ + {ToolCallID: "toolu_x", Name: "run_bash", Content: "boom", IsError: true}, + }}, + }, + }) + if err != nil { + t.Fatalf("Chat: %v", err) + } + if len(got.Messages) != 1 || len(got.Messages[0].Content) != 1 { + t.Fatalf("unexpected messages: %+v", got.Messages) + } + if !got.Messages[0].Content[0].IsError { + t.Errorf("expected is_error true") + } +} + +// TestChat_RateLimitError429 verifies a 429 rate_limit_error response +// produces a Go error whose text retry.IsRateLimitError recognizes. +func TestChat_RateLimitError429(t *testing.T) { + srv := fakeAnthropicServer(t, http.StatusTooManyRequests, `{ + "type": "error", + "error": {"type": "rate_limit_error", "message": "Number of requests has exceeded your rate limit."} + }`, nil) + defer srv.Close() + + p := newTestProvider(srv.URL) + _, err := p.Chat(context.Background(), provider.ChatRequest{ + Model: "claude-sonnet-5", + Messages: []provider.Message{{Role: "user", Text: "hi"}}, + }) + if err == nil { + t.Fatal("expected an error for HTTP 429") + } + if !strings.Contains(err.Error(), "429") || !strings.Contains(err.Error(), "rate_limit_error") { + t.Errorf("error should mention 429 and rate_limit_error, got: %v", err) + } + if !retry.IsRateLimitError(err) { + t.Errorf("retry.IsRateLimitError should recognize: %v", err) + } +} + +// TestChat_OverloadedError529 verifies a 529 overloaded_error response is +// also recognized as retryable. +func TestChat_OverloadedError529(t *testing.T) { + srv := fakeAnthropicServer(t, 529, `{ + "type": "error", + "error": {"type": "overloaded_error", "message": "Overloaded"} + }`, nil) + defer srv.Close() + + p := newTestProvider(srv.URL) + _, err := p.Chat(context.Background(), provider.ChatRequest{ + Model: "claude-sonnet-5", + Messages: []provider.Message{{Role: "user", Text: "hi"}}, + }) + if err == nil { + t.Fatal("expected an error for HTTP 529") + } + if !retry.IsRateLimitError(err) { + t.Errorf("retry.IsRateLimitError should recognize overloaded_error: %v", err) + } +} + +// TestChat_InvalidRequestError400 is a non-retryable error case: confirms +// the error text carries the invalid_request_error type and is NOT +// misclassified as a rate-limit error. +func TestChat_InvalidRequestError400(t *testing.T) { + srv := fakeAnthropicServer(t, http.StatusBadRequest, `{ + "type": "error", + "error": {"type": "invalid_request_error", "message": "max_tokens: field required"} + }`, nil) + defer srv.Close() + + p := newTestProvider(srv.URL) + _, err := p.Chat(context.Background(), provider.ChatRequest{ + Model: "claude-sonnet-5", + Messages: []provider.Message{{Role: "user", Text: "hi"}}, + }) + if err == nil { + t.Fatal("expected an error for HTTP 400") + } + if !strings.Contains(err.Error(), "invalid_request_error") { + t.Errorf("error should mention invalid_request_error, got: %v", err) + } + if retry.IsRateLimitError(err) { + t.Errorf("400 invalid_request_error should NOT be classified as a rate-limit error: %v", err) + } +} + +// TestChat_NoAPIKey_Errors confirms Chat fails fast without ever making an +// HTTP request when no API key is configured. +func TestChat_NoAPIKey_Errors(t *testing.T) { + p := New("", "http://127.0.0.1:1", time.Second) + _, err := p.Chat(context.Background(), provider.ChatRequest{Model: "claude-sonnet-5"}) + if err == nil || !strings.Contains(err.Error(), "no API key") { + t.Errorf("expected 'no API key' error, got %v", err) + } +} + +// TestNew_DefaultsEndpointAndTimeout verifies New falls back to +// DefaultEndpoint/DefaultTimeout for zero-value arguments. +func TestNew_DefaultsEndpointAndTimeout(t *testing.T) { + p := New("key", "", 0) + if p.Endpoint != DefaultEndpoint { + t.Errorf("Endpoint: want %q got %q", DefaultEndpoint, p.Endpoint) + } + if p.HTTPClient.Timeout != DefaultTimeout { + t.Errorf("Timeout: want %v got %v", DefaultTimeout, p.HTTPClient.Timeout) + } +} + +// TestName returns the provider's identifier, used as the agent-type/budget +// key ("anthropic") once wired into a NativeRunner. +func TestName(t *testing.T) { + p := New("key", "", 0) + if p.Name() != "anthropic" { + t.Errorf("Name: got %q", p.Name()) + } +} + +// TestPricingFor_KnownAndUnknownModels sanity-checks the pricing table +// lookup: exact/prefix match for known models, and a non-erroring fallback +// for an unrecognized model string. +func TestPricingFor_KnownAndUnknownModels(t *testing.T) { + in, out := pricingFor("claude-opus-4-8") + if in != 5.00 || out != 25.00 { + t.Errorf("opus 4.8 pricing: got %v/%v", in, out) + } + in, out = pricingFor("claude-haiku-4-5-20251001") + if in != 1.00 || out != 5.00 { + t.Errorf("haiku 4.5 dated snapshot pricing: got %v/%v", in, out) + } + in, out = pricingFor("some-future-unknown-model") + if in != defaultPricing.InputPerMTok || out != defaultPricing.OutputPerMTok { + t.Errorf("unknown model should fall back to default pricing, got %v/%v", in, out) + } +} -- cgit v1.2.3