From 06c9730441593c191f968f8041a0e76f56368a39 Mon Sep 17 00:00:00 2001
From: Peter Stone <thepeterstone@gmail.com>
Date: Fri, 10 Jul 2026 22:13:49 +0000
Subject: fix(executor): periodically sweep stale dispatch workspace
 directories

ContainerRunner preserves a failed execution's workspace indefinitely for
debugging, with no expiry -- this accumulated 161 stale directories
(~17.6GB) and took the host to 100% disk full on 2026-07-10. Pool.RunWorkspaceCleanup
sweeps claudomator-workspace-* dirs older than 24h every hour, started from
serve.go, mirroring StoryOrchestrator.Run's ticker shape. Never removes a
directory still referenced as a currently-BLOCKED task's sandbox_dir,
regardless of age.
---
 internal/executor/executor.go | 85 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 85 insertions(+)

(limited to 'internal/executor/executor.go')

diff --git a/internal/executor/executor.go b/internal/executor/executor.go
index 659bcda..1025e2b 100644
--- a/internal/executor/executor.go
+++ b/internal/executor/executor.go
@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"log/slog"
+	"os"
 	"path/filepath"
 	"strings"
 	"sync"
@@ -1148,6 +1149,90 @@ func (p *Pool) RecoverStaleBlocked() {
 	}
 }
 
+// DefaultWorkspaceCleanupInterval is how often RunWorkspaceCleanup sweeps for
+// stale dispatch workspaces when no interval is given.
+const DefaultWorkspaceCleanupInterval = 1 * time.Hour
+
+// DefaultWorkspaceCleanupMaxAge is how long a preserved (failed) workspace is
+// kept around for debugging before RunWorkspaceCleanup removes it.
+const DefaultWorkspaceCleanupMaxAge = 24 * time.Hour
+
+// RunWorkspaceCleanup periodically removes stale dispatch workspace
+// directories -- ContainerRunner.Run's os.MkdirTemp("", "claudomator-workspace-*")
+// dirs -- left behind by failed executions. ContainerRunner.Run's own doc
+// comment says "workspace is only removed on success. On failure, it's
+// preserved for debugging" -- that preservation has no expiry of its own, so
+// without this sweep those directories accumulate forever (confirmed in
+// production: 161 stale directories, ~17.6GB, took the host to 100% disk
+// full on 2026-07-10). Call this once, in a goroutine, at server startup --
+// mirrors StoryOrchestrator.Run's ticker shape exactly, including running
+// one pass immediately before the first tick so a server that's been down
+// doesn't wait a full interval before its first sweep.
+func (p *Pool) RunWorkspaceCleanup(ctx context.Context, interval, maxAge time.Duration) {
+	if interval <= 0 {
+		interval = DefaultWorkspaceCleanupInterval
+	}
+	p.cleanupStaleWorkspaces(maxAge)
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			p.cleanupStaleWorkspaces(maxAge)
+		}
+	}
+}
+
+// cleanupStaleWorkspaces removes claudomator-workspace-* directories under
+// os.TempDir() whose mtime is older than maxAge, except any directory still
+// referenced as the sandbox_dir of a currently-BLOCKED task's latest
+// execution -- that one is preserved intentionally for resumption (see
+// ContainerRunner.Run) and must never be swept purely by age. A directory
+// this sweep doesn't recognize (glob mismatch) or can't stat is left alone,
+// not treated as an error worth aborting the whole pass over.
+func (p *Pool) cleanupStaleWorkspaces(maxAge time.Duration) {
+	keep := map[string]bool{}
+	blocked, err := p.store.ListTasks(storage.TaskFilter{State: task.StateBlocked})
+	if err != nil {
+		p.logger.Error("cleanupStaleWorkspaces: list blocked tasks", "error", err)
+		return
+	}
+	for _, t := range blocked {
+		execs, err := p.store.ListExecutions(t.ID)
+		if err != nil {
+			continue
+		}
+		for _, e := range execs {
+			if e.SandboxDir != "" {
+				keep[e.SandboxDir] = true
+			}
+		}
+	}
+
+	matches, err := filepath.Glob(filepath.Join(os.TempDir(), "claudomator-workspace-*"))
+	if err != nil {
+		p.logger.Error("cleanupStaleWorkspaces: glob", "error", err)
+		return
+	}
+	cutoff := time.Now().Add(-maxAge)
+	for _, dir := range matches {
+		if keep[dir] {
+			continue
+		}
+		info, err := os.Stat(dir)
+		if err != nil || info.ModTime().After(cutoff) {
+			continue
+		}
+		if err := os.RemoveAll(dir); err != nil {
+			p.logger.Error("cleanupStaleWorkspaces: remove", "dir", dir, "error", err)
+			continue
+		}
+		p.logger.Info("cleanupStaleWorkspaces: removed stale workspace", "dir", dir, "age", time.Since(info.ModTime()))
+	}
+}
+
 // terminalFailureStates are dependency states that cause the waiting task to fail immediately.
 var terminalFailureStates = map[task.State]bool{
 	task.StateFailed:         true,
-- 
cgit v1.2.3