1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
|
package executor
import (
"context"
"encoding/json"
"fmt"
"os/exec"
"strings"
"github.com/thepeterstone/claudomator/internal/llm"
)
type Classification struct {
AgentType string `json:"agent_type"`
Model string `json:"model"`
Reason string `json:"reason"`
}
type SystemStatus struct {
ActiveTasks map[string]int
RateLimited map[string]bool
}
// Classifier picks a model for an incoming task. When LLM is non-nil the
// classifier routes through the local OpenAI-compatible client (cheap,
// private, fast). Otherwise it falls back to invoking the Gemini CLI
// at GeminiBinaryPath.
type Classifier struct {
LLM *llm.Client
GeminiBinaryPath string
}
const classificationPrompt = `
You are a model selector for Claudomator.
The agent has already been chosen by the load balancer. Your ONLY job is to select the best model for that agent.
REQUIRED agent: %s
Available Models:
Claude:
- claude-sonnet-4-6 (default, balanced, best for most coding tasks)
- claude-opus-4-6 (most powerful, expensive, use for hardest tasks only)
- claude-haiku-4-5-20251001 (fast, cheap, use for simple tasks)
Gemini:
- gemini-2.5-flash-lite (fastest, most efficient, best for simple/trivial tasks)
- gemini-2.5-flash (fast, balanced)
- gemini-2.5-pro (most powerful, use for hardest tasks only)
Selection Criteria:
- Use powerful models (opus, pro) only for the hardest reasoning/coding tasks.
- Use lite/haiku for simple, short, or low-stakes tasks.
- Default to the balanced model (sonnet, flash) for everything else.
Task:
Name: %s
Instructions: %s
Respond with ONLY a JSON object:
{
"agent_type": "%s",
"model": "<one of the exact model identifiers listed above, verbatim>",
"reason": "brief reason"
}
`
// validModels are the exact model identifiers classificationPrompt lists as
// valid choices. Classify's and classifyViaLLM's output is validated
// against this set before being trusted -- see validateClassification's
// doc comment for why this exists.
var validModels = map[string]bool{
"claude-sonnet-4-6": true,
"claude-opus-4-6": true,
"claude-haiku-4-5-20251001": true,
"gemini-2.5-flash-lite": true,
"gemini-2.5-flash": true,
"gemini-2.5-pro": true,
}
// validateClassification rejects a Classification whose Model isn't one of
// the exact identifiers classificationPrompt actually lists. Added
// 2026-07-11 after two real production failures where the underlying LLM
// echoed part of the prompt's own JSON schema literally -- "model-name"
// (this prompt's own placeholder text at the time, since fixed to be less
// echo-prone) and, separately, "choose-the-best-model" -- instead of
// substituting a real identifier. Both were syntactically valid JSON
// strings, so nothing before this caught them; the claude CLI then rejected
// the literal string as an invalid --model argument and the task failed
// outright, with no automatic recovery until internal/scheduler's
// retryWithoutLadder was added for the retry side of this same incident.
// Rejecting here makes Classify return an error, which callers already
// treat as "classification failed" and fall back to dispatching with no
// explicit model override (see internal/executor/executor.go's Pool.execute,
// which only sets t.Agent.Model when err == nil) -- exactly the documented
// "falls back to the default model if Gemini fails" behavior, now also
// covering "Gemini succeeded but returned garbage."
func validateClassification(cls *Classification) error {
if !validModels[cls.Model] {
return fmt.Errorf("classifier returned an unrecognized model %q -- not one of the known model identifiers, likely echoed the prompt's own example text instead of substituting a real one", cls.Model)
}
return nil
}
func (c *Classifier) Classify(ctx context.Context, taskName, instructions string, _ SystemStatus, agentType string) (*Classification, error) {
prompt := fmt.Sprintf(classificationPrompt,
agentType, taskName, instructions, agentType,
)
if c.LLM != nil {
return c.classifyViaLLM(ctx, prompt, agentType)
}
binary := c.GeminiBinaryPath
if binary == "" {
binary = "gemini"
}
// Use a minimal model for classification to be fast and cheap.
args := []string{
"--prompt", prompt,
"--model", "gemini-2.5-flash-lite",
"--output-format", "json",
}
cmd := exec.CommandContext(ctx, binary, args...)
out, err := cmd.Output()
if err != nil {
if exitErr, ok := err.(*exec.ExitError); ok {
return nil, fmt.Errorf("classifier failed (%v): %s", err, string(exitErr.Stderr))
}
return nil, fmt.Errorf("classifier failed: %w", err)
}
// 1. Parse the JSON envelope from the gemini CLI.
var cliOut struct {
Response string `json:"response"`
}
if err := json.Unmarshal(out, &cliOut); err != nil {
// If it's not JSON, it might be raw text (though we requested JSON).
// This can happen if the CLI prints "Loaded cached credentials" or other info.
cliOut.Response = string(out)
}
// 2. Extract the model response from the "response" field if present.
// If it was already raw text, cliOut.Response will have it.
cleanOut := strings.TrimSpace(cliOut.Response)
// 3. Clean up "Loaded cached credentials" or other noise that might be in the string
// if we fell back to string(out).
if strings.Contains(cleanOut, "Loaded cached credentials.") {
lines := strings.Split(cleanOut, "\n")
var modelLines []string
for _, line := range lines {
if !strings.Contains(line, "Loaded cached credentials.") {
modelLines = append(modelLines, line)
}
}
cleanOut = strings.TrimSpace(strings.Join(modelLines, "\n"))
}
// 4. Gemini might wrap the JSON in markdown code blocks.
cleanOut = strings.TrimPrefix(cleanOut, "```json")
cleanOut = strings.TrimPrefix(cleanOut, "```") // fallback
cleanOut = strings.TrimSuffix(cleanOut, "```")
cleanOut = strings.TrimSpace(cleanOut)
var cls Classification
if err := json.Unmarshal([]byte(cleanOut), &cls); err != nil {
return nil, fmt.Errorf("failed to parse classification JSON: %w\nOriginal Output: %s\nCleaned Output: %s", err, string(out), cleanOut)
}
if err := validateClassification(&cls); err != nil {
return nil, err
}
return &cls, nil
}
// classifyViaLLM routes classification through the local OpenAI-compatible
// client with response_format=json_object, so we get clean JSON without the
// markdown-fence cleanup needed for the Gemini CLI fallback.
func (c *Classifier) classifyViaLLM(ctx context.Context, prompt, agentType string) (*Classification, error) {
resp, err := c.LLM.Chat(ctx, llm.ChatRequest{
Messages: []llm.Message{{Role: "user", Content: prompt}},
ResponseJSON: true,
})
if err != nil {
return nil, fmt.Errorf("classifier (local llm): %w", err)
}
body := strings.TrimSpace(resp.Content)
var cls Classification
if err := json.Unmarshal([]byte(body), &cls); err != nil {
return nil, fmt.Errorf("classifier (local llm): parse JSON: %w\nbody: %s", err, body)
}
if cls.AgentType == "" {
cls.AgentType = agentType
}
if err := validateClassification(&cls); err != nil {
return nil, err
}
return &cls, nil
}
|
