diff options
Diffstat (limited to 'internal/executor/classifier.go')
| -rw-r--r-- | internal/executor/classifier.go | 45 |
1 files changed, 44 insertions, 1 deletions
diff --git a/internal/executor/classifier.go b/internal/executor/classifier.go index 049dc4f..05ed021 100644 --- a/internal/executor/classifier.go +++ b/internal/executor/classifier.go @@ -59,11 +59,48 @@ Instructions: %s Respond with ONLY a JSON object: { "agent_type": "%s", - "model": "model-name", + "model": "<one of the exact model identifiers listed above, verbatim>", "reason": "brief reason" } ` +// validModels are the exact model identifiers classificationPrompt lists as +// valid choices. Classify's and classifyViaLLM's output is validated +// against this set before being trusted -- see validateClassification's +// doc comment for why this exists. +var validModels = map[string]bool{ + "claude-sonnet-4-6": true, + "claude-opus-4-6": true, + "claude-haiku-4-5-20251001": true, + "gemini-2.5-flash-lite": true, + "gemini-2.5-flash": true, + "gemini-2.5-pro": true, +} + +// validateClassification rejects a Classification whose Model isn't one of +// the exact identifiers classificationPrompt actually lists. Added +// 2026-07-11 after two real production failures where the underlying LLM +// echoed part of the prompt's own JSON schema literally -- "model-name" +// (this prompt's own placeholder text at the time, since fixed to be less +// echo-prone) and, separately, "choose-the-best-model" -- instead of +// substituting a real identifier. Both were syntactically valid JSON +// strings, so nothing before this caught them; the claude CLI then rejected +// the literal string as an invalid --model argument and the task failed +// outright, with no automatic recovery until internal/scheduler's +// retryWithoutLadder was added for the retry side of this same incident. +// Rejecting here makes Classify return an error, which callers already +// treat as "classification failed" and fall back to dispatching with no +// explicit model override (see internal/executor/executor.go's Pool.execute, +// which only sets t.Agent.Model when err == nil) -- exactly the documented +// "falls back to the default model if Gemini fails" behavior, now also +// covering "Gemini succeeded but returned garbage." +func validateClassification(cls *Classification) error { + if !validModels[cls.Model] { + return fmt.Errorf("classifier returned an unrecognized model %q -- not one of the known model identifiers, likely echoed the prompt's own example text instead of substituting a real one", cls.Model) + } + return nil +} + func (c *Classifier) Classify(ctx context.Context, taskName, instructions string, _ SystemStatus, agentType string) (*Classification, error) { prompt := fmt.Sprintf(classificationPrompt, agentType, taskName, instructions, agentType, @@ -131,6 +168,9 @@ func (c *Classifier) Classify(ctx context.Context, taskName, instructions string if err := json.Unmarshal([]byte(cleanOut), &cls); err != nil { return nil, fmt.Errorf("failed to parse classification JSON: %w\nOriginal Output: %s\nCleaned Output: %s", err, string(out), cleanOut) } + if err := validateClassification(&cls); err != nil { + return nil, err + } return &cls, nil } @@ -154,5 +194,8 @@ func (c *Classifier) classifyViaLLM(ctx context.Context, prompt, agentType strin if cls.AgentType == "" { cls.AgentType = agentType } + if err := validateClassification(&cls); err != nil { + return nil, err + } return &cls, nil } |
