package anthropic import ( "context" "encoding/json" "net/http" "net/http/httptest" "strings" "testing" "time" "github.com/thepeterstone/claudomator/internal/provider" "github.com/thepeterstone/claudomator/internal/retry" ) // fakeAnthropicServer replies to POST /v1/messages with the given status and // raw JSON body, capturing the last decoded request for assertions. Modeled // on the fake OpenAI-compatible server pattern used in // internal/llm/client_test.go and internal/executor/nativerunner_test.go // (fakeChatServer), adapted to the Anthropic Messages API wire shape. func fakeAnthropicServer(t *testing.T, status int, body string, capture *wireRequest) *httptest.Server { t.Helper() return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/v1/messages" { t.Errorf("unexpected path %q", r.URL.Path) } if r.Header.Get("x-api-key") != "test-key" { t.Errorf("missing/wrong x-api-key header: %q", r.Header.Get("x-api-key")) } if r.Header.Get("anthropic-version") != APIVersion { t.Errorf("missing/wrong anthropic-version header: %q", r.Header.Get("anthropic-version")) } if r.Header.Get("content-type") != "application/json" { t.Errorf("missing/wrong content-type header: %q", r.Header.Get("content-type")) } if capture != nil { if err := json.NewDecoder(r.Body).Decode(capture); err != nil { t.Fatalf("decode request body: %v", err) } } w.Header().Set("Content-Type", "application/json") w.WriteHeader(status) w.Write([]byte(body)) })) } func newTestProvider(url string) *Provider { return New("test-key", url, 5*time.Second) } // TestChat_PlainTextResponse covers a plain assistant text reply: no tool // calls, stop_reason "end_turn", usage translated to provider.Usage with a // non-zero CostUSD computed from the pricing table. func TestChat_PlainTextResponse(t *testing.T) { var got wireRequest srv := fakeAnthropicServer(t, http.StatusOK, `{ "id": "msg_1", "type": "message", "role": "assistant", "model": "claude-sonnet-5", "content": [{"type": "text", "text": "Hello there!"}], "stop_reason": "end_turn", "usage": {"input_tokens": 10, "output_tokens": 5} }`, &got) defer srv.Close() p := newTestProvider(srv.URL) resp, err := p.Chat(context.Background(), provider.ChatRequest{ Model: "claude-sonnet-5", System: "You are helpful.", Messages: []provider.Message{ {Role: "user", Text: "hi"}, }, }) if err != nil { t.Fatalf("Chat: %v", err) } if resp.Text != "Hello there!" { t.Errorf("Text: want %q got %q", "Hello there!", resp.Text) } if len(resp.ToolCalls) != 0 { t.Errorf("expected no tool calls, got %+v", resp.ToolCalls) } if resp.StopReason != "end_turn" { t.Errorf("StopReason: want end_turn got %q", resp.StopReason) } if resp.Usage.InputTokens != 10 || resp.Usage.OutputTokens != 5 { t.Errorf("usage tokens: got %+v", resp.Usage) } wantCost := float64(10)/1_000_000*3.00 + float64(5)/1_000_000*15.00 if resp.Usage.CostUSD != wantCost { t.Errorf("CostUSD: want %v got %v", wantCost, resp.Usage.CostUSD) } // Verify the outgoing request shape: system is top-level, not a message. if got.System != "You are helpful." { t.Errorf("system: want %q got %q", "You are helpful.", got.System) } if len(got.Messages) != 1 || got.Messages[0].Role != "user" { t.Fatalf("messages: %+v", got.Messages) } if len(got.Messages[0].Content) != 1 || got.Messages[0].Content[0].Type != "text" || got.Messages[0].Content[0].Text != "hi" { t.Errorf("user content block: %+v", got.Messages[0].Content) } if got.MaxTokens != defaultMaxTokens { t.Errorf("max_tokens should default to %d, got %d", defaultMaxTokens, got.MaxTokens) } } // TestChat_ToolUseResponse verifies that a response whose content mixes a // text block and a tool_use block round-trips into ChatResponse.ToolCalls // correctly (id/name/args), and that tools declared on the request are // translated to Anthropic's {name, description, input_schema} shape. func TestChat_ToolUseResponse(t *testing.T) { var got wireRequest srv := fakeAnthropicServer(t, http.StatusOK, `{ "id": "msg_2", "type": "message", "role": "assistant", "model": "claude-opus-4-8", "content": [ {"type": "text", "text": "Let me check that."}, {"type": "tool_use", "id": "toolu_01", "name": "read_file", "input": {"path": "a.txt"}} ], "stop_reason": "tool_use", "usage": {"input_tokens": 20, "output_tokens": 8} }`, &got) defer srv.Close() p := newTestProvider(srv.URL) resp, err := p.Chat(context.Background(), provider.ChatRequest{ Model: "claude-opus-4-8", Messages: []provider.Message{{Role: "user", Text: "read a.txt"}}, Tools: []provider.ToolSpec{ { Name: "read_file", Description: "Read a file.", ParametersJSONSchema: map[string]any{ "type": "object", "properties": map[string]any{"path": map[string]any{"type": "string"}}, "required": []string{"path"}, }, }, }, }) if err != nil { t.Fatalf("Chat: %v", err) } if resp.Text != "Let me check that." { t.Errorf("Text: got %q", resp.Text) } if resp.StopReason != "tool_use" { t.Errorf("StopReason: want tool_use got %q", resp.StopReason) } if len(resp.ToolCalls) != 1 { t.Fatalf("expected 1 tool call, got %d: %+v", len(resp.ToolCalls), resp.ToolCalls) } tc := resp.ToolCalls[0] if tc.ID != "toolu_01" || tc.Name != "read_file" { t.Errorf("tool call id/name: got %+v", tc) } var args struct { Path string `json:"path"` } if err := json.Unmarshal([]byte(tc.ArgsJSON), &args); err != nil { t.Fatalf("tool call args not valid JSON: %v (%q)", err, tc.ArgsJSON) } if args.Path != "a.txt" { t.Errorf("tool call args.path: got %q", args.Path) } // Verify tools were declared with input_schema (not "parameters"). if len(got.Tools) != 1 { t.Fatalf("expected 1 tool declared, got %d", len(got.Tools)) } if got.Tools[0].Name != "read_file" || got.Tools[0].Description != "Read a file." { t.Errorf("tool declaration: %+v", got.Tools[0]) } if got.Tools[0].InputSchema == nil || got.Tools[0].InputSchema["type"] != "object" { t.Errorf("tool input_schema not forwarded: %+v", got.Tools[0].InputSchema) } } // TestChat_MultiTurnWithPriorToolResult exercises the tool_result-as-a- // user-message translation: a provider.Message carrying ToolResults (as // agentloop.Loop emits with Role "tool" after dispatching a tool call) must // serialize as a user-role message with tool_result content blocks — // Anthropic has no "tool" role. Also verifies a prior assistant turn with a // tool call round-trips into a single assistant message with a tool_use // block matching the id used in the subsequent tool_result. func TestChat_MultiTurnWithPriorToolResult(t *testing.T) { var got wireRequest srv := fakeAnthropicServer(t, http.StatusOK, `{ "id": "msg_3", "type": "message", "role": "assistant", "model": "claude-sonnet-5", "content": [{"type": "text", "text": "The file says hello."}], "stop_reason": "end_turn", "usage": {"input_tokens": 30, "output_tokens": 6} }`, &got) defer srv.Close() p := newTestProvider(srv.URL) _, err := p.Chat(context.Background(), provider.ChatRequest{ Model: "claude-sonnet-5", Messages: []provider.Message{ {Role: "user", Text: "read a.txt"}, { Role: "assistant", Text: "", ToolCalls: []provider.ToolCall{ {ID: "toolu_01", Name: "read_file", ArgsJSON: `{"path":"a.txt"}`}, }, }, { Role: "tool", ToolResults: []provider.ToolResult{ {ToolCallID: "toolu_01", Name: "read_file", Content: "hello", IsError: false}, }, }, }, }) if err != nil { t.Fatalf("Chat: %v", err) } if len(got.Messages) != 3 { t.Fatalf("expected 3 wire messages, got %d: %+v", len(got.Messages), got.Messages) } // Turn 2: assistant tool_use, no text block since Text was empty. asst := got.Messages[1] if asst.Role != "assistant" { t.Errorf("turn 2 role: want assistant got %q", asst.Role) } if len(asst.Content) != 1 || asst.Content[0].Type != "tool_use" { t.Fatalf("turn 2 content: %+v", asst.Content) } if asst.Content[0].ID != "toolu_01" || asst.Content[0].Name != "read_file" { t.Errorf("turn 2 tool_use block: %+v", asst.Content[0]) } // Turn 3: the tool result must be a USER-role message with a // tool_result block (Anthropic has no "tool" role). toolTurn := got.Messages[2] if toolTurn.Role != "user" { t.Errorf("tool result turn role: want user (Anthropic has no tool role) got %q", toolTurn.Role) } if len(toolTurn.Content) != 1 || toolTurn.Content[0].Type != "tool_result" { t.Fatalf("tool result content: %+v", toolTurn.Content) } block := toolTurn.Content[0] if block.ToolUseID != "toolu_01" { t.Errorf("tool_use_id: want toolu_01 got %q", block.ToolUseID) } if block.Content != "hello" { t.Errorf("tool_result content: want hello got %q", block.Content) } if block.IsError { t.Errorf("is_error should be false") } } // TestChat_ToolResultIsError verifies IsError round-trips onto the // tool_result block's is_error field. func TestChat_ToolResultIsError(t *testing.T) { var got wireRequest srv := fakeAnthropicServer(t, http.StatusOK, `{ "content": [{"type": "text", "text": "ok"}], "stop_reason": "end_turn", "usage": {"input_tokens": 1, "output_tokens": 1} }`, &got) defer srv.Close() p := newTestProvider(srv.URL) _, err := p.Chat(context.Background(), provider.ChatRequest{ Model: "claude-sonnet-5", Messages: []provider.Message{ {Role: "tool", ToolResults: []provider.ToolResult{ {ToolCallID: "toolu_x", Name: "run_bash", Content: "boom", IsError: true}, }}, }, }) if err != nil { t.Fatalf("Chat: %v", err) } if len(got.Messages) != 1 || len(got.Messages[0].Content) != 1 { t.Fatalf("unexpected messages: %+v", got.Messages) } if !got.Messages[0].Content[0].IsError { t.Errorf("expected is_error true") } } // TestChat_RateLimitError429 verifies a 429 rate_limit_error response // produces a Go error whose text retry.IsRateLimitError recognizes. func TestChat_RateLimitError429(t *testing.T) { srv := fakeAnthropicServer(t, http.StatusTooManyRequests, `{ "type": "error", "error": {"type": "rate_limit_error", "message": "Number of requests has exceeded your rate limit."} }`, nil) defer srv.Close() p := newTestProvider(srv.URL) _, err := p.Chat(context.Background(), provider.ChatRequest{ Model: "claude-sonnet-5", Messages: []provider.Message{{Role: "user", Text: "hi"}}, }) if err == nil { t.Fatal("expected an error for HTTP 429") } if !strings.Contains(err.Error(), "429") || !strings.Contains(err.Error(), "rate_limit_error") { t.Errorf("error should mention 429 and rate_limit_error, got: %v", err) } if !retry.IsRateLimitError(err) { t.Errorf("retry.IsRateLimitError should recognize: %v", err) } } // TestChat_OverloadedError529 verifies a 529 overloaded_error response is // also recognized as retryable. func TestChat_OverloadedError529(t *testing.T) { srv := fakeAnthropicServer(t, 529, `{ "type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"} }`, nil) defer srv.Close() p := newTestProvider(srv.URL) _, err := p.Chat(context.Background(), provider.ChatRequest{ Model: "claude-sonnet-5", Messages: []provider.Message{{Role: "user", Text: "hi"}}, }) if err == nil { t.Fatal("expected an error for HTTP 529") } if !retry.IsRateLimitError(err) { t.Errorf("retry.IsRateLimitError should recognize overloaded_error: %v", err) } } // TestChat_InvalidRequestError400 is a non-retryable error case: confirms // the error text carries the invalid_request_error type and is NOT // misclassified as a rate-limit error. func TestChat_InvalidRequestError400(t *testing.T) { srv := fakeAnthropicServer(t, http.StatusBadRequest, `{ "type": "error", "error": {"type": "invalid_request_error", "message": "max_tokens: field required"} }`, nil) defer srv.Close() p := newTestProvider(srv.URL) _, err := p.Chat(context.Background(), provider.ChatRequest{ Model: "claude-sonnet-5", Messages: []provider.Message{{Role: "user", Text: "hi"}}, }) if err == nil { t.Fatal("expected an error for HTTP 400") } if !strings.Contains(err.Error(), "invalid_request_error") { t.Errorf("error should mention invalid_request_error, got: %v", err) } if retry.IsRateLimitError(err) { t.Errorf("400 invalid_request_error should NOT be classified as a rate-limit error: %v", err) } } // TestChat_NoAPIKey_Errors confirms Chat fails fast without ever making an // HTTP request when no API key is configured. func TestChat_NoAPIKey_Errors(t *testing.T) { p := New("", "http://127.0.0.1:1", time.Second) _, err := p.Chat(context.Background(), provider.ChatRequest{Model: "claude-sonnet-5"}) if err == nil || !strings.Contains(err.Error(), "no API key") { t.Errorf("expected 'no API key' error, got %v", err) } } // TestNew_DefaultsEndpointAndTimeout verifies New falls back to // DefaultEndpoint/DefaultTimeout for zero-value arguments. func TestNew_DefaultsEndpointAndTimeout(t *testing.T) { p := New("key", "", 0) if p.Endpoint != DefaultEndpoint { t.Errorf("Endpoint: want %q got %q", DefaultEndpoint, p.Endpoint) } if p.HTTPClient.Timeout != DefaultTimeout { t.Errorf("Timeout: want %v got %v", DefaultTimeout, p.HTTPClient.Timeout) } } // TestName returns the provider's identifier, used as the agent-type/budget // key ("anthropic") once wired into a NativeRunner. func TestName(t *testing.T) { p := New("key", "", 0) if p.Name() != "anthropic" { t.Errorf("Name: got %q", p.Name()) } } // TestPricingFor_KnownAndUnknownModels sanity-checks the pricing table // lookup: exact/prefix match for known models, and a non-erroring fallback // for an unrecognized model string. func TestPricingFor_KnownAndUnknownModels(t *testing.T) { in, out := pricingFor("claude-opus-4-8") if in != 5.00 || out != 25.00 { t.Errorf("opus 4.8 pricing: got %v/%v", in, out) } in, out = pricingFor("claude-haiku-4-5-20251001") if in != 1.00 || out != 5.00 { t.Errorf("haiku 4.5 dated snapshot pricing: got %v/%v", in, out) } in, out = pricingFor("some-future-unknown-model") if in != defaultPricing.InputPerMTok || out != defaultPricing.OutputPerMTok { t.Errorf("unknown model should fall back to default pricing, got %v/%v", in, out) } }