diff options
| author | Peter Stone <thepeterstone@gmail.com> | 2026-08-04 20:10:25 +0000 |
|---|---|---|
| committer | Peter Stone <thepeterstone@gmail.com> | 2026-08-04 20:10:25 +0000 |
| commit | 0a410243dea33f204764000be81814e541dcae48 (patch) | |
| tree | f91f8312af0bc70e5da8c254635118efe811dbef /internal/handlers | |
| parent | 2da86b009c76bb7688103da49a9125d35d3a1ed8 (diff) | |
Fix production wedge: propagate context to Google Calendar API calls
Three .Do() calls in google_calendar.go accepted a ctx parameter but
never chained .Context(ctx) into the actual SDK call, so the existing
global 60s request timeout never reached the blocking network call.
One hung Google Calendar request wedged every DB/session-touching
request path in production for three days (2026-08-01 through
2026-08-04), undetected because /health unconditionally returned 200
throughout.
- Wire .Context(ctx) into GetUpcomingEvents, GetEventsByDateRange, and
GetCalendarList.
- Bound aggregateData's four external fetches with a per-fetch
sub-context as defense-in-depth (only effective if the callee
actually honors ctx -- documented as such, not oversold).
- Make GetUpcomingEvents/GetEventsByDateRange fetch calendars
concurrently instead of sequentially: a review of this fix caught
that a shared per-fetch deadline over a sequential loop would starve
calendars past the first under any real latency, silently caching
partial results as complete. Concurrent fetches give every calendar
an equal shot at the same deadline instead.
- /health now does a real PingContext DB check instead of a static
"ok" (Handler.PingDB, tested for both healthy and closed-DB cases).
- Add internal/api/context_audit_test.go: an AST-based structural
guard that fails any future .Do() call in google_*.go missing
.Context(...) anywhere in its chain, so this class of bug can't
silently recur. Verified by deliberately reintroducing the original
bug against a backup and confirming the guard catches it.
- Add scripts/health-watchdog.sh: cron job restarts the service if
/health fails twice in a row, five minutes apart.
go test ./... -race is green. Deployed and live-verified.
Diffstat (limited to 'internal/handlers')
| -rw-r--r-- | internal/handlers/handlers.go | 42 | ||||
| -rw-r--r-- | internal/handlers/handlers_test.go | 22 |
2 files changed, 53 insertions, 11 deletions
diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index 54c6a70..a5bc51a 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -96,6 +96,14 @@ func New(s *store.Store, trello api.TrelloAPI, planToEat api.PlanToEatAPI, googl } } +// PingDB checks database reachability within the given context, for use by +// a deep health check that actually exercises the DB layer (unlike a bare +// "200 ok" liveness check, which stays green even if every DB-touching +// request path is wedged -- see the 2026-08-04 incident). +func (h *Handler) PingDB(ctx context.Context) error { + return h.store.DB().PingContext(ctx) +} + // HandleDashboard renders the main dashboard view func (h *Handler) HandleDashboard(w http.ResponseWriter, r *http.Request) { ctx := r.Context() @@ -202,8 +210,18 @@ func (h *Handler) aggregateData(ctx context.Context, forceRefresh bool) (*models var wg sync.WaitGroup var mu sync.Mutex - // Helper to run fetch in goroutine with error collection - fetch := func(name string, fn func() error) { + // Helper to run fetch in goroutine with error collection. Each fetch gets + // its own bounded sub-context, which bounds wg.Wait() PROVIDED fn actually + // wires fetchCtx into whatever blocking call it makes. This context alone + // does not and cannot guarantee that -- a client that ignores the ctx it's + // handed (as google_calendar.go's .Do() calls did until the 2026-08-04 + // incident, see internal/api/context_audit_test.go for the regression + // guard against that specific class of bug) will still hang forever + // regardless of this timeout. This is defense-in-depth, not a substitute + // for every fn() correctly propagating context into its actual I/O. + // Previously nothing here bounded fn()'s worst case at all, only + // best-effort ctx cancellation checked before the call even started. + fetch := func(name string, fn func(context.Context) error) { wg.Add(1) go func() { defer wg.Done() @@ -212,7 +230,9 @@ func (h *Handler) aggregateData(ctx context.Context, forceRefresh bool) (*models return default: } - if err := fn(); err != nil { + fetchCtx, cancel := context.WithTimeout(ctx, config.HTTPClientTimeout) + defer cancel() + if err := fn(fetchCtx); err != nil { log.Printf("ERROR [%s]: %v", name, err) mu.Lock() data.Errors = append(data.Errors, name+": "+err.Error()) @@ -221,8 +241,8 @@ func (h *Handler) aggregateData(ctx context.Context, forceRefresh bool) (*models }() } - fetch("Trello", func() error { - boards, err := h.fetchBoards(ctx, forceRefresh) + fetch("Trello", func(fetchCtx context.Context) error { + boards, err := h.fetchBoards(fetchCtx, forceRefresh) if err == nil { mu.Lock() data.Boards = boards @@ -232,8 +252,8 @@ func (h *Handler) aggregateData(ctx context.Context, forceRefresh bool) (*models }) if h.planToEatClient != nil { - fetch("PlanToEat", func() error { - meals, err := h.fetchMeals(ctx, forceRefresh) + fetch("PlanToEat", func(fetchCtx context.Context) error { + meals, err := h.fetchMeals(fetchCtx, forceRefresh) if err == nil { mu.Lock() data.Meals = meals @@ -244,8 +264,8 @@ func (h *Handler) aggregateData(ctx context.Context, forceRefresh bool) (*models } if h.googleCalendarClient != nil { - fetch("Google Calendar", func() error { - events, err := h.fetchCalendarEvents(ctx, false) + fetch("Google Calendar", func(fetchCtx context.Context) error { + events, err := h.fetchCalendarEvents(fetchCtx, false) if err == nil { mu.Lock() data.Events = events @@ -256,8 +276,8 @@ func (h *Handler) aggregateData(ctx context.Context, forceRefresh bool) (*models } if h.googleTasksClient != nil { - fetch("Google Tasks", func() error { - tasks, err := h.fetchGoogleTasks(ctx, forceRefresh) + fetch("Google Tasks", func(fetchCtx context.Context) error { + tasks, err := h.fetchGoogleTasks(fetchCtx, forceRefresh) if err == nil { mu.Lock() data.GoogleTasks = tasks diff --git a/internal/handlers/handlers_test.go b/internal/handlers/handlers_test.go index bb383aa..9d92faa 100644 --- a/internal/handlers/handlers_test.go +++ b/internal/handlers/handlers_test.go @@ -2355,3 +2355,25 @@ func TestHandleTimeline_IncludesBudgetStatusWhenTrackedTaskExists(t *testing.T) } } +func TestPingDB_HealthyStore_ReturnsNil(t *testing.T) { + h, cleanup := setupTestHandler(t) + defer cleanup() + + if err := h.PingDB(context.Background()); err != nil { + t.Errorf("PingDB() = %v, want nil for a healthy store", err) + } +} + +// This is the regression test for the actual incident: a health check that +// can't observe a broken database isn't a health check. If PingDB stops +// reflecting real DB reachability -- e.g. someone "simplifies" it back to +// an unconditional nil -- this must fail. +func TestPingDB_ClosedStore_ReturnsError(t *testing.T) { + h, cleanup := setupTestHandler(t) + cleanup() // close the underlying DB before pinging it + + if err := h.PingDB(context.Background()); err == nil { + t.Error("PingDB() = nil, want an error for a closed/unreachable store") + } +} + |
