package executor import ( "context" "encoding/json" "fmt" "os/exec" "strings" "github.com/thepeterstone/claudomator/internal/llm" ) type Classification struct { AgentType string `json:"agent_type"` Model string `json:"model"` Reason string `json:"reason"` } type SystemStatus struct { ActiveTasks map[string]int RateLimited map[string]bool } // Classifier picks a model for an incoming task. When LLM is non-nil the // classifier routes through the local OpenAI-compatible client (cheap, // private, fast). Otherwise it falls back to invoking the Gemini CLI // at GeminiBinaryPath. type Classifier struct { LLM *llm.Client GeminiBinaryPath string } const classificationPrompt = ` You are a model selector for Claudomator. The agent has already been chosen by the load balancer. Your ONLY job is to select the best model for that agent. REQUIRED agent: %s Available Models: Claude: - claude-sonnet-4-6 (default, balanced, best for most coding tasks) - claude-opus-4-6 (most powerful, expensive, use for hardest tasks only) - claude-haiku-4-5-20251001 (fast, cheap, use for simple tasks) Gemini: - gemini-2.5-flash-lite (fastest, most efficient, best for simple/trivial tasks) - gemini-2.5-flash (fast, balanced) - gemini-2.5-pro (most powerful, use for hardest tasks only) Selection Criteria: - Use powerful models (opus, pro) only for the hardest reasoning/coding tasks. - Use lite/haiku for simple, short, or low-stakes tasks. - Default to the balanced model (sonnet, flash) for everything else. Task: Name: %s Instructions: %s Respond with ONLY a JSON object: { "agent_type": "%s", "model": "", "reason": "brief reason" } ` // validModels are the exact model identifiers classificationPrompt lists as // valid choices. Classify's and classifyViaLLM's output is validated // against this set before being trusted -- see validateClassification's // doc comment for why this exists. var validModels = map[string]bool{ "claude-sonnet-4-6": true, "claude-opus-4-6": true, "claude-haiku-4-5-20251001": true, "gemini-2.5-flash-lite": true, "gemini-2.5-flash": true, "gemini-2.5-pro": true, } // validateClassification rejects a Classification whose Model isn't one of // the exact identifiers classificationPrompt actually lists. Added // 2026-07-11 after two real production failures where the underlying LLM // echoed part of the prompt's own JSON schema literally -- "model-name" // (this prompt's own placeholder text at the time, since fixed to be less // echo-prone) and, separately, "choose-the-best-model" -- instead of // substituting a real identifier. Both were syntactically valid JSON // strings, so nothing before this caught them; the claude CLI then rejected // the literal string as an invalid --model argument and the task failed // outright, with no automatic recovery until internal/scheduler's // retryWithoutLadder was added for the retry side of this same incident. // Rejecting here makes Classify return an error, which callers already // treat as "classification failed" and fall back to dispatching with no // explicit model override (see internal/executor/executor.go's Pool.execute, // which only sets t.Agent.Model when err == nil) -- exactly the documented // "falls back to the default model if Gemini fails" behavior, now also // covering "Gemini succeeded but returned garbage." func validateClassification(cls *Classification) error { if !validModels[cls.Model] { return fmt.Errorf("classifier returned an unrecognized model %q -- not one of the known model identifiers, likely echoed the prompt's own example text instead of substituting a real one", cls.Model) } return nil } func (c *Classifier) Classify(ctx context.Context, taskName, instructions string, _ SystemStatus, agentType string) (*Classification, error) { prompt := fmt.Sprintf(classificationPrompt, agentType, taskName, instructions, agentType, ) if c.LLM != nil { return c.classifyViaLLM(ctx, prompt, agentType) } binary := c.GeminiBinaryPath if binary == "" { binary = "gemini" } // Use a minimal model for classification to be fast and cheap. args := []string{ "--prompt", prompt, "--model", "gemini-2.5-flash-lite", "--output-format", "json", } cmd := exec.CommandContext(ctx, binary, args...) out, err := cmd.Output() if err != nil { if exitErr, ok := err.(*exec.ExitError); ok { return nil, fmt.Errorf("classifier failed (%v): %s", err, string(exitErr.Stderr)) } return nil, fmt.Errorf("classifier failed: %w", err) } // 1. Parse the JSON envelope from the gemini CLI. var cliOut struct { Response string `json:"response"` } if err := json.Unmarshal(out, &cliOut); err != nil { // If it's not JSON, it might be raw text (though we requested JSON). // This can happen if the CLI prints "Loaded cached credentials" or other info. cliOut.Response = string(out) } // 2. Extract the model response from the "response" field if present. // If it was already raw text, cliOut.Response will have it. cleanOut := strings.TrimSpace(cliOut.Response) // 3. Clean up "Loaded cached credentials" or other noise that might be in the string // if we fell back to string(out). if strings.Contains(cleanOut, "Loaded cached credentials.") { lines := strings.Split(cleanOut, "\n") var modelLines []string for _, line := range lines { if !strings.Contains(line, "Loaded cached credentials.") { modelLines = append(modelLines, line) } } cleanOut = strings.TrimSpace(strings.Join(modelLines, "\n")) } // 4. Gemini might wrap the JSON in markdown code blocks. cleanOut = strings.TrimPrefix(cleanOut, "```json") cleanOut = strings.TrimPrefix(cleanOut, "```") // fallback cleanOut = strings.TrimSuffix(cleanOut, "```") cleanOut = strings.TrimSpace(cleanOut) var cls Classification if err := json.Unmarshal([]byte(cleanOut), &cls); err != nil { return nil, fmt.Errorf("failed to parse classification JSON: %w\nOriginal Output: %s\nCleaned Output: %s", err, string(out), cleanOut) } if err := validateClassification(&cls); err != nil { return nil, err } return &cls, nil } // classifyViaLLM routes classification through the local OpenAI-compatible // client with response_format=json_object, so we get clean JSON without the // markdown-fence cleanup needed for the Gemini CLI fallback. func (c *Classifier) classifyViaLLM(ctx context.Context, prompt, agentType string) (*Classification, error) { resp, err := c.LLM.Chat(ctx, llm.ChatRequest{ Messages: []llm.Message{{Role: "user", Content: prompt}}, ResponseJSON: true, }) if err != nil { return nil, fmt.Errorf("classifier (local llm): %w", err) } body := strings.TrimSpace(resp.Content) var cls Classification if err := json.Unmarshal([]byte(body), &cls); err != nil { return nil, fmt.Errorf("classifier (local llm): parse JSON: %w\nbody: %s", err, body) } if cls.AgentType == "" { cls.AgentType = agentType } if err := validateClassification(&cls); err != nil { return nil, err } return &cls, nil }