From 7e8967decbc8221694953abf1435fda8aaf18824 Mon Sep 17 00:00:00 2001 From: Peter Stone Date: Thu, 19 Mar 2026 23:03:56 +0000 Subject: feat: agent status dashboard with availability timeline and Gemini quota detection - Detect Gemini TerminalQuotaError (daily quota) as BUDGET_EXCEEDED, not generic FAILED - Surface container stderr tail in error so quota/rate-limit classifiers can match it - Add agent_events table to persist rate-limit start/recovery events across restarts - Add GET /api/agents/status endpoint returning live agent state + 24h event history - Stats dashboard: agent status cards, 24h availability timeline, per-run execution table Co-Authored-By: Claude Sonnet 4.6 --- web/app.js | 187 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 184 insertions(+), 3 deletions(-) (limited to 'web/app.js') diff --git a/web/app.js b/web/app.js index 6ddb23c..5d99984 100644 --- a/web/app.js +++ b/web/app.js @@ -1175,8 +1175,11 @@ function renderActiveTab(allTasks) { renderAllPanel(allTasks); break; case 'stats': - fetchRecentExecutions(BASE_PATH, fetch) - .then(execs => renderStatsPanel(allTasks, execs)) + Promise.all([ + fetchRecentExecutions(BASE_PATH, fetch), + fetch(`${BASE_PATH}/api/agents/status?since=${encodeURIComponent(new Date(Date.now() - 24*60*60*1000).toISOString())}`).then(r => r.ok ? r.json() : { agents: [], events: [] }), + ]) + .then(([execs, agentData]) => renderStatsPanel(allTasks, execs, agentData)) .catch(() => {}); break; case 'drops': @@ -2432,7 +2435,7 @@ function formatDurationMs(ms) { return rm > 0 ? `${h}h ${rm}m` : `${h}h`; } -function renderStatsPanel(tasks, executions) { +function renderStatsPanel(tasks, executions, agentData = { agents: [], events: [] }) { const panel = document.querySelector('[data-panel="stats"]'); if (!panel) return; @@ -2566,7 +2569,185 @@ function renderStatsPanel(tasks, executions) { execSection.appendChild(chartSection); } + // ── Per-execution detail table ───────────────────────────────────────────── + if (executions.length > 0) { + const tableWrap = document.createElement('div'); + tableWrap.className = 'stats-exec-table-wrap'; + + const tableLabel = document.createElement('p'); + tableLabel.className = 'stats-bar-chart-label'; + tableLabel.textContent = 'Recent runs'; + tableWrap.appendChild(tableLabel); + + const table = document.createElement('table'); + table.className = 'stats-exec-table'; + table.innerHTML = 'TaskOutcomeCostDurationStarted'; + const tbody = document.createElement('tbody'); + for (const ex of executions.slice(0, 20)) { + const tr = document.createElement('tr'); + const durationMs = ex.duration_ms != null ? formatDurationMs(ex.duration_ms) : '—'; + const cost = ex.cost_usd > 0 ? `$${ex.cost_usd.toFixed(3)}` : '—'; + const started = ex.started_at ? new Date(ex.started_at).toLocaleTimeString() : '—'; + const state = (ex.state || '').toUpperCase(); + tr.innerHTML = `${ex.task_name || ex.task_id}${state.replace(/_/g,' ')}${cost}${durationMs}${started}`; + tbody.appendChild(tr); + } + table.appendChild(tbody); + tableWrap.appendChild(table); + execSection.appendChild(tableWrap); + } + panel.appendChild(execSection); + + // ── Agent Status ─────────────────────────────────────────────────────────── + const agentSection = document.createElement('div'); + agentSection.className = 'stats-section'; + + const agentHeading = document.createElement('h2'); + agentHeading.textContent = 'Agent Status'; + agentSection.appendChild(agentHeading); + + const agents = agentData.agents || []; + const agentEvents = agentData.events || []; + + if (agents.length === 0) { + const none = document.createElement('p'); + none.className = 'task-meta'; + none.textContent = 'No agents registered.'; + agentSection.appendChild(none); + } else { + // Status cards row + const cardsRow = document.createElement('div'); + cardsRow.className = 'stats-agent-cards'; + for (const ag of agents) { + const card = document.createElement('div'); + card.className = 'stats-agent-card'; + const statusClass = ag.rate_limited ? 'agent-rate-limited' : 'agent-available'; + card.classList.add(statusClass); + + const nameEl = document.createElement('span'); + nameEl.className = 'stats-agent-name'; + nameEl.textContent = ag.agent; + + const statusEl = document.createElement('span'); + statusEl.className = 'stats-agent-status'; + if (ag.rate_limited && ag.until) { + const untilDate = new Date(ag.until); + const minsLeft = Math.max(0, Math.round((untilDate - Date.now()) / 60000)); + statusEl.textContent = `Rate limited — ${minsLeft}m remaining`; + } else { + statusEl.textContent = ag.active_tasks > 0 ? `Active (${ag.active_tasks} running)` : 'Available'; + } + + card.appendChild(nameEl); + card.appendChild(statusEl); + cardsRow.appendChild(card); + } + agentSection.appendChild(cardsRow); + + // Availability timeline (last 24h) + const now = Date.now(); + const windowMs = 24 * 60 * 60 * 1000; + const windowStart = now - windowMs; + + const timelineHeading = document.createElement('p'); + timelineHeading.className = 'stats-bar-chart-label'; + timelineHeading.textContent = 'Availability last 24h'; + agentSection.appendChild(timelineHeading); + + // Group events by agent + const eventsByAgent = {}; + for (const ev of agentEvents) { + if (!eventsByAgent[ev.agent]) eventsByAgent[ev.agent] = []; + eventsByAgent[ev.agent].push(ev); + } + + for (const ag of agents) { + const evs = (eventsByAgent[ag.agent] || []).slice().sort((a, b) => new Date(a.timestamp) - new Date(b.timestamp)); + + const row = document.createElement('div'); + row.className = 'stats-timeline-row'; + + const label = document.createElement('span'); + label.className = 'stats-timeline-label'; + label.textContent = ag.agent; + row.appendChild(label); + + const track = document.createElement('div'); + track.className = 'stats-timeline-track'; + + // Build segments: walk events and produce [start, end, state] intervals + const segments = []; + let cursor = windowStart; + // Reconstruct: before first event, assume available unless currently rate-limited with an until before window + let inRateLimit = false; + + for (const ev of evs) { + const evTime = Math.max(windowStart, new Date(ev.timestamp).getTime()); + if (evTime > cursor) { + segments.push({ start: cursor, end: evTime, limited: inRateLimit }); + } + cursor = evTime; + if (ev.event === 'rate_limited') { + inRateLimit = true; + } else if (ev.event === 'available') { + inRateLimit = false; + } + } + // Tail to now + if (cursor < now) { + // If currently rate limited use current agent state + segments.push({ start: cursor, end: now, limited: ag.rate_limited || inRateLimit }); + } + + for (const seg of segments) { + const pct = ((seg.end - seg.start) / windowMs) * 100; + if (pct < 0.01) continue; + const span = document.createElement('div'); + span.className = 'stats-timeline-seg'; + span.classList.add(seg.limited ? 'seg-limited' : 'seg-available'); + span.style.width = `${pct.toFixed(2)}%`; + const mins = Math.round((seg.end - seg.start) / 60000); + span.title = `${seg.limited ? 'Rate limited' : 'Available'} — ${mins}m`; + track.appendChild(span); + } + + row.appendChild(track); + + // Legend labels + const timeLabels = document.createElement('div'); + timeLabels.className = 'stats-timeline-timelabels'; + timeLabels.innerHTML = '24h agonow'; + row.appendChild(timeLabels); + + agentSection.appendChild(row); + } + + // Rate-limit event log + if (agentEvents.length > 0) { + const evLogLabel = document.createElement('p'); + evLogLabel.className = 'stats-bar-chart-label'; + evLogLabel.textContent = 'Rate-limit events (last 24h)'; + agentSection.appendChild(evLogLabel); + + const evTable = document.createElement('table'); + evTable.className = 'stats-exec-table'; + evTable.innerHTML = 'AgentEventReasonUntilTime'; + const evTbody = document.createElement('tbody'); + for (const ev of agentEvents.slice(0, 30)) { + const tr = document.createElement('tr'); + const until = ev.until ? new Date(ev.until).toLocaleTimeString() : '—'; + const ts = new Date(ev.timestamp).toLocaleTimeString(); + const eventClass = ev.event === 'rate_limited' ? 'state-badge" data-state="FAILED' : 'state-badge" data-state="COMPLETED'; + tr.innerHTML = `${ev.agent}${ev.event.replace(/_/g,' ')}${ev.reason || '—'}${until}${ts}`; + evTbody.appendChild(tr); + } + evTable.appendChild(evTbody); + agentSection.appendChild(evTable); + } + } + + panel.appendChild(agentSection); } // ── Web Push Notifications ──────────────────────────────────────────────────── -- cgit v1.2.3