From 0a410243dea33f204764000be81814e541dcae48 Mon Sep 17 00:00:00 2001 From: Peter Stone Date: Tue, 4 Aug 2026 20:10:25 +0000 Subject: Fix production wedge: propagate context to Google Calendar API calls Three .Do() calls in google_calendar.go accepted a ctx parameter but never chained .Context(ctx) into the actual SDK call, so the existing global 60s request timeout never reached the blocking network call. One hung Google Calendar request wedged every DB/session-touching request path in production for three days (2026-08-01 through 2026-08-04), undetected because /health unconditionally returned 200 throughout. - Wire .Context(ctx) into GetUpcomingEvents, GetEventsByDateRange, and GetCalendarList. - Bound aggregateData's four external fetches with a per-fetch sub-context as defense-in-depth (only effective if the callee actually honors ctx -- documented as such, not oversold). - Make GetUpcomingEvents/GetEventsByDateRange fetch calendars concurrently instead of sequentially: a review of this fix caught that a shared per-fetch deadline over a sequential loop would starve calendars past the first under any real latency, silently caching partial results as complete. Concurrent fetches give every calendar an equal shot at the same deadline instead. - /health now does a real PingContext DB check instead of a static "ok" (Handler.PingDB, tested for both healthy and closed-DB cases). - Add internal/api/context_audit_test.go: an AST-based structural guard that fails any future .Do() call in google_*.go missing .Context(...) anywhere in its chain, so this class of bug can't silently recur. Verified by deliberately reintroducing the original bug against a backup and confirming the guard catches it. - Add scripts/health-watchdog.sh: cron job restarts the service if /health fails twice in a row, five minutes apart. go test ./... -race is green. Deployed and live-verified. --- internal/handlers/handlers.go | 42 +++++++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 11 deletions(-) (limited to 'internal/handlers/handlers.go') diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index 54c6a70..a5bc51a 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -96,6 +96,14 @@ func New(s *store.Store, trello api.TrelloAPI, planToEat api.PlanToEatAPI, googl } } +// PingDB checks database reachability within the given context, for use by +// a deep health check that actually exercises the DB layer (unlike a bare +// "200 ok" liveness check, which stays green even if every DB-touching +// request path is wedged -- see the 2026-08-04 incident). +func (h *Handler) PingDB(ctx context.Context) error { + return h.store.DB().PingContext(ctx) +} + // HandleDashboard renders the main dashboard view func (h *Handler) HandleDashboard(w http.ResponseWriter, r *http.Request) { ctx := r.Context() @@ -202,8 +210,18 @@ func (h *Handler) aggregateData(ctx context.Context, forceRefresh bool) (*models var wg sync.WaitGroup var mu sync.Mutex - // Helper to run fetch in goroutine with error collection - fetch := func(name string, fn func() error) { + // Helper to run fetch in goroutine with error collection. Each fetch gets + // its own bounded sub-context, which bounds wg.Wait() PROVIDED fn actually + // wires fetchCtx into whatever blocking call it makes. This context alone + // does not and cannot guarantee that -- a client that ignores the ctx it's + // handed (as google_calendar.go's .Do() calls did until the 2026-08-04 + // incident, see internal/api/context_audit_test.go for the regression + // guard against that specific class of bug) will still hang forever + // regardless of this timeout. This is defense-in-depth, not a substitute + // for every fn() correctly propagating context into its actual I/O. + // Previously nothing here bounded fn()'s worst case at all, only + // best-effort ctx cancellation checked before the call even started. + fetch := func(name string, fn func(context.Context) error) { wg.Add(1) go func() { defer wg.Done() @@ -212,7 +230,9 @@ func (h *Handler) aggregateData(ctx context.Context, forceRefresh bool) (*models return default: } - if err := fn(); err != nil { + fetchCtx, cancel := context.WithTimeout(ctx, config.HTTPClientTimeout) + defer cancel() + if err := fn(fetchCtx); err != nil { log.Printf("ERROR [%s]: %v", name, err) mu.Lock() data.Errors = append(data.Errors, name+": "+err.Error()) @@ -221,8 +241,8 @@ func (h *Handler) aggregateData(ctx context.Context, forceRefresh bool) (*models }() } - fetch("Trello", func() error { - boards, err := h.fetchBoards(ctx, forceRefresh) + fetch("Trello", func(fetchCtx context.Context) error { + boards, err := h.fetchBoards(fetchCtx, forceRefresh) if err == nil { mu.Lock() data.Boards = boards @@ -232,8 +252,8 @@ func (h *Handler) aggregateData(ctx context.Context, forceRefresh bool) (*models }) if h.planToEatClient != nil { - fetch("PlanToEat", func() error { - meals, err := h.fetchMeals(ctx, forceRefresh) + fetch("PlanToEat", func(fetchCtx context.Context) error { + meals, err := h.fetchMeals(fetchCtx, forceRefresh) if err == nil { mu.Lock() data.Meals = meals @@ -244,8 +264,8 @@ func (h *Handler) aggregateData(ctx context.Context, forceRefresh bool) (*models } if h.googleCalendarClient != nil { - fetch("Google Calendar", func() error { - events, err := h.fetchCalendarEvents(ctx, false) + fetch("Google Calendar", func(fetchCtx context.Context) error { + events, err := h.fetchCalendarEvents(fetchCtx, false) if err == nil { mu.Lock() data.Events = events @@ -256,8 +276,8 @@ func (h *Handler) aggregateData(ctx context.Context, forceRefresh bool) (*models } if h.googleTasksClient != nil { - fetch("Google Tasks", func() error { - tasks, err := h.fetchGoogleTasks(ctx, forceRefresh) + fetch("Google Tasks", func(fetchCtx context.Context) error { + tasks, err := h.fetchGoogleTasks(fetchCtx, forceRefresh) if err == nil { mu.Lock() data.GoogleTasks = tasks -- cgit v1.2.3