From 50f8fe8c1ff8b82e0bd399e5776e58bda3e57d1c Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Sat, 2 May 2026 08:00:17 +0000
Subject: feat(executor): synthesize execution summary via local LLM fallback
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Phase 4 of "local OSS models as agents" plan. Closes the epic.

When an execution finishes and the agent did NOT write a "## Summary"
heading in its stdout (so the existing extractSummary path returns
empty), and the Pool has a local LLM configured, we now synthesize a
2-4 sentence summary from the assistant text content of the log tail.

Behavior:
- Primary path unchanged: if the agent wrote "## Summary", that wins
  byte-for-byte (TestPool_HandleRunResult_ExtractSummaryWins guards).
- Fallback path: empty extractSummary + Pool.LLM != nil → synthesize.
- All-empty path: when no LLM is configured, summary stays empty —
  identical to pre-Phase-4 behavior.

Implementation:
- Pool gains an LLM *llm.Client field, wired in serve.go and run.go
  alongside Classifier.LLM (same localClient used everywhere).
- New synthesizeSummary in internal/executor/summary.go:
  * 6s timeout so a slow local model can't stall finalization
  * 16 KB tail cap on the stdout log
  * readAssistantTextTail seeks to the last 16 KB and skips the
    first (likely partial) line, parses each line as a stream-json
    event, joins assistant `text` blocks (skips system/result/etc).
  * Returns "" on any error so the caller's behavior never regresses.
- handleRunResult: 3-tier summary resolution — exec.Summary set by
  runner → extractSummary → synthesizeSummary → empty.
- minimalMockStore now records UpdateTaskSummary calls (additive;
  existing tests unaffected) so integration tests can assert.

Tests (9 new):
- synthesizeSummary nil client / empty path / missing file all
  return "" without HTTP calls.
- empty assistant content short-circuits without LLM call.
- success path returns trimmed body, with both assistant texts in
  the user prompt.
- LLM 500 returns "" (caller handles same as no-summary).
- readAssistantTextTail seeks past early content in a large file.
- Pool integration: ## Summary present → LLM not called, agent text
  used. ## Summary absent + LLM set → LLM called, synthesized summary
  recorded against the right task ID.

Plan: docs/plans/local-oss-runner.md.

Epic complete. Post-epic deep cleanup queue captured in the same plan
file for follow-up.

https://claude.ai/code/session_017Edeq947TpSm1vQTxMhi1J
---
 internal/executor/summary.go | 95 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 95 insertions(+)

(limited to 'internal/executor/summary.go')

diff --git a/internal/executor/summary.go b/internal/executor/summary.go
index a942de0..bcf5cfd 100644
--- a/internal/executor/summary.go
+++ b/internal/executor/summary.go
@@ -2,11 +2,26 @@ package executor
 
 import (
 	"bufio"
+	"context"
 	"encoding/json"
+	"io"
 	"os"
 	"strings"
+	"time"
+
+	"github.com/thepeterstone/claudomator/internal/llm"
 )
 
+// synthesizeSummaryMaxBytes caps how much of the stdout log we send to the
+// LLM. Larger values cost more tokens with diminishing returns for a 2-4
+// sentence summary.
+const synthesizeSummaryMaxBytes = 16 * 1024
+
+// synthesizeSummaryTimeout caps the LLM call so a slow local model can't
+// stall executor finalization. On timeout, we return "" (the existing
+// no-summary path takes over).
+const synthesizeSummaryTimeout = 6 * time.Second
+
 // extractSummary reads a stream-json stdout log and returns the text following
 // the last "## Summary" heading found in any assistant text block.
 // Returns empty string if the file cannot be read or no summary is found.
@@ -28,6 +43,86 @@ func extractSummary(stdoutPath string) string {
 	return last
 }
 
+// synthesizeSummary asks the LLM to summarize the assistant text content in
+// stdoutPath when no "## Summary" heading was present. Returns "" on any
+// error, an empty file, or an empty model response — preserving the
+// existing "no summary" behavior so the new path is purely additive.
+func synthesizeSummary(parent context.Context, c *llm.Client, stdoutPath string) string {
+	if c == nil || stdoutPath == "" {
+		return ""
+	}
+	text := readAssistantTextTail(stdoutPath, synthesizeSummaryMaxBytes)
+	if strings.TrimSpace(text) == "" {
+		return ""
+	}
+
+	cctx, cancel := context.WithTimeout(parent, synthesizeSummaryTimeout)
+	defer cancel()
+	resp, err := c.Chat(cctx, llm.ChatRequest{
+		Messages: []llm.Message{
+			{Role: "system", Content: "You summarize what an automated coding agent did. Reply with 2-4 sentences of plain prose. No bullets, no headings, no preamble."},
+			{Role: "user", Content: "Here is the agent's output. Summarize what it accomplished:\n\n" + text},
+		},
+	})
+	if err != nil {
+		return ""
+	}
+	return strings.TrimSpace(resp.Content)
+}
+
+// readAssistantTextTail returns the concatenated `text` blocks from assistant
+// stream-json events in the last maxBytes of the file. Non-assistant events
+// (system, result, tool_use, etc.) are skipped so the LLM sees just what the
+// agent said. Returns "" on any error.
+func readAssistantTextTail(stdoutPath string, maxBytes int64) string {
+	f, err := os.Open(stdoutPath)
+	if err != nil {
+		return ""
+	}
+	defer f.Close()
+
+	stat, err := f.Stat()
+	if err != nil {
+		return ""
+	}
+	size := stat.Size()
+	if size > maxBytes {
+		if _, err := f.Seek(size-maxBytes, io.SeekStart); err != nil {
+			return ""
+		}
+	}
+
+	var sb strings.Builder
+	scanner := bufio.NewScanner(f)
+	scanner.Buffer(make([]byte, 1024*1024), 1024*1024)
+	first := size > maxBytes // if we seeked, drop the first (likely partial) line
+	for scanner.Scan() {
+		if first {
+			first = false
+			continue
+		}
+		var event struct {
+			Type    string `json:"type"`
+			Message struct {
+				Content []struct {
+					Type string `json:"type"`
+					Text string `json:"text"`
+				} `json:"content"`
+			} `json:"message"`
+		}
+		if err := json.Unmarshal(scanner.Bytes(), &event); err != nil || event.Type != "assistant" {
+			continue
+		}
+		for _, block := range event.Message.Content {
+			if block.Type == "text" && block.Text != "" {
+				sb.WriteString(block.Text)
+				sb.WriteString("\n")
+			}
+		}
+	}
+	return sb.String()
+}
+
 // summaryFromLine parses a single stream-json line and returns the text after
 // "## Summary" if the line is an assistant text block containing that heading.
 func summaryFromLine(line []byte) string {
-- 
cgit v1.2.3