From 6a59098c3096f5ebd3a61ef5268cbd480b0f1519 Mon Sep 17 00:00:00 2001 From: Peter Stone Date: Tue, 20 Jan 2026 10:40:29 -1000 Subject: Implement efficient sync for Todoist and Trello APIs - Add Todoist Sync API v9 support with incremental sync tokens - Store sync tokens in SQLite for persistence across restarts - Add field filtering to Trello API calls to reduce payload size - Update handlers to use incremental sync (merge changes vs full replace) Co-Authored-By: Claude Opus 4.5 --- internal/api/interfaces.go | 1 + internal/api/todoist.go | 141 ++++++++++++++++++++++++++++++++++++- internal/api/trello.go | 3 + internal/handlers/handlers.go | 95 +++++++++++++++++++++++-- internal/handlers/handlers_test.go | 27 +++++++ internal/store/sqlite.go | 78 ++++++++++++++++++++ 6 files changed, 337 insertions(+), 8 deletions(-) (limited to 'internal') diff --git a/internal/api/interfaces.go b/internal/api/interfaces.go index db7e6c0..2419707 100644 --- a/internal/api/interfaces.go +++ b/internal/api/interfaces.go @@ -13,6 +13,7 @@ type TodoistAPI interface { GetProjects(ctx context.Context) ([]models.Project, error) CreateTask(ctx context.Context, content, projectID string, dueDate *time.Time, priority int) (*models.Task, error) CompleteTask(ctx context.Context, taskID string) error + Sync(ctx context.Context, syncToken string) (*TodoistSyncResponse, error) } // TrelloAPI defines the interface for Trello operations diff --git a/internal/api/todoist.go b/internal/api/todoist.go index 511753d..b51fffd 100644 --- a/internal/api/todoist.go +++ b/internal/api/todoist.go @@ -13,7 +13,8 @@ import ( ) const ( - todoistBaseURL = "https://api.todoist.com/rest/v2" + todoistBaseURL = "https://api.todoist.com/rest/v2" + todoistSyncBaseURL = "https://api.todoist.com/sync/v9" ) // TodoistClient handles interactions with the Todoist API @@ -56,6 +57,40 @@ type todoistProjectResponse struct { Name string `json:"name"` } +// Sync API v9 response types + +// TodoistSyncResponse represents the Sync API response +type TodoistSyncResponse struct { + SyncToken string `json:"sync_token"` + FullSync bool `json:"full_sync"` + Items []SyncItemResponse `json:"items"` + Projects []SyncProjectResponse `json:"projects"` +} + +// SyncItemResponse represents a task item from Sync API +type SyncItemResponse struct { + ID string `json:"id"` + Content string `json:"content"` + Description string `json:"description"` + ProjectID string `json:"project_id"` + Priority int `json:"priority"` + Labels []string `json:"labels"` + Due *struct { + Date string `json:"date"` + Datetime string `json:"datetime"` + } `json:"due"` + IsCompleted bool `json:"is_completed"` + IsDeleted bool `json:"is_deleted"` + AddedAt string `json:"added_at"` +} + +// SyncProjectResponse represents a project from Sync API +type SyncProjectResponse struct { + ID string `json:"id"` + Name string `json:"name"` + IsDeleted bool `json:"is_deleted"` +} + // GetTasks fetches all active tasks from Todoist func (c *TodoistClient) GetTasks(ctx context.Context) ([]models.Task, error) { req, err := http.NewRequestWithContext(ctx, "GET", c.baseURL+"/tasks", nil) @@ -167,6 +202,110 @@ func (c *TodoistClient) GetProjects(ctx context.Context) ([]models.Project, erro return projects, nil } +// Sync performs an incremental sync using the Sync API v9 +// If syncToken is empty or "*", a full sync is performed +// Returns the new sync token and the sync response +func (c *TodoistClient) Sync(ctx context.Context, syncToken string) (*TodoistSyncResponse, error) { + if syncToken == "" { + syncToken = "*" // Full sync + } + + // Prepare sync request + payload := map[string]interface{}{ + "sync_token": syncToken, + "resource_types": []string{"items", "projects"}, + } + + jsonData, err := json.Marshal(payload) + if err != nil { + return nil, fmt.Errorf("failed to marshal sync request: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, "POST", todoistSyncBaseURL+"/sync", bytes.NewBuffer(jsonData)) + if err != nil { + return nil, fmt.Errorf("failed to create sync request: %w", err) + } + + req.Header.Set("Authorization", "Bearer "+c.apiKey) + req.Header.Set("Content-Type", "application/json") + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to perform sync: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("todoist sync API error (status %d): %s", resp.StatusCode, string(body)) + } + + var syncResp TodoistSyncResponse + if err := json.NewDecoder(resp.Body).Decode(&syncResp); err != nil { + return nil, fmt.Errorf("failed to decode sync response: %w", err) + } + + return &syncResp, nil +} + +// ConvertSyncItemsToTasks converts sync API items to Task models +func ConvertSyncItemsToTasks(items []SyncItemResponse, projectMap map[string]string) []models.Task { + tasks := make([]models.Task, 0, len(items)) + for _, item := range items { + // Skip completed or deleted items + if item.IsCompleted || item.IsDeleted { + continue + } + + task := models.Task{ + ID: item.ID, + Content: item.Content, + Description: item.Description, + ProjectID: item.ProjectID, + ProjectName: projectMap[item.ProjectID], + Priority: item.Priority, + Completed: false, + Labels: item.Labels, + URL: fmt.Sprintf("https://todoist.com/showTask?id=%s", item.ID), + } + + // Parse added_at + if item.AddedAt != "" { + if createdAt, err := time.Parse(time.RFC3339, item.AddedAt); err == nil { + task.CreatedAt = createdAt + } + } + + // Parse due date + if item.Due != nil { + var dueDate time.Time + var err error + if item.Due.Datetime != "" { + dueDate, err = time.Parse(time.RFC3339, item.Due.Datetime) + } else if item.Due.Date != "" { + dueDate, err = time.Parse("2006-01-02", item.Due.Date) + } + if err == nil { + task.DueDate = &dueDate + } + } + + tasks = append(tasks, task) + } + return tasks +} + +// BuildProjectMapFromSync builds a project ID to name map from sync response +func BuildProjectMapFromSync(projects []SyncProjectResponse) map[string]string { + projectMap := make(map[string]string) + for _, proj := range projects { + if !proj.IsDeleted { + projectMap[proj.ID] = proj.Name + } + } + return projectMap +} + // CreateTask creates a new task in Todoist func (c *TodoistClient) CreateTask(ctx context.Context, content, projectID string, dueDate *time.Time, priority int) (*models.Task, error) { // Prepare request payload diff --git a/internal/api/trello.go b/internal/api/trello.go index 91d6d66..037c881 100644 --- a/internal/api/trello.go +++ b/internal/api/trello.go @@ -68,6 +68,7 @@ func (c *TrelloClient) GetBoards(ctx context.Context) ([]models.Board, error) { params.Set("key", c.apiKey) params.Set("token", c.token) params.Set("filter", "open") + params.Set("fields", "id,name") // Only fetch required fields reqURL := fmt.Sprintf("%s/members/me/boards?%s", c.baseURL, params.Encode()) req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil) @@ -111,6 +112,7 @@ func (c *TrelloClient) GetCards(ctx context.Context, boardID string) ([]models.C params.Set("key", c.apiKey) params.Set("token", c.token) params.Set("filter", "visible") + params.Set("fields", "id,name,idList,due,url,idBoard") // Only fetch required fields reqURL := fmt.Sprintf("%s/boards/%s/cards?%s", c.baseURL, boardID, params.Encode()) req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil) @@ -174,6 +176,7 @@ func (c *TrelloClient) getLists(ctx context.Context, boardID string) ([]models.L params := url.Values{} params.Set("key", c.apiKey) params.Set("token", c.token) + params.Set("fields", "id,name") // Only fetch required fields reqURL := fmt.Sprintf("%s/boards/%s/lists?%s", c.baseURL, boardID, params.Encode()) req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil) diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index 0af2bba..f53eced 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -350,9 +350,10 @@ func (h *Handler) aggregateData(ctx context.Context, forceRefresh bool) (*models return data, nil } -// fetchTasks fetches tasks from cache or API +// fetchTasks fetches tasks from cache or API using incremental sync func (h *Handler) fetchTasks(ctx context.Context, forceRefresh bool) ([]models.Task, error) { cacheKey := store.CacheKeyTodoistTasks + syncService := "todoist" // Check cache validity if !forceRefresh { @@ -362,8 +363,20 @@ func (h *Handler) fetchTasks(ctx context.Context, forceRefresh bool) ([]models.T } } - // Fetch from API - tasks, err := h.todoistClient.GetTasks(ctx) + // Get stored sync token (empty string means full sync) + syncToken, err := h.store.GetSyncToken(syncService) + if err != nil { + log.Printf("Failed to get sync token, will do full sync: %v", err) + syncToken = "" + } + + // Force full sync if requested + if forceRefresh { + syncToken = "" + } + + // Fetch using Sync API + syncResp, err := h.todoistClient.Sync(ctx, syncToken) if err != nil { // Try to return cached data even if stale cachedTasks, cacheErr := h.store.GetTasks() @@ -373,9 +386,41 @@ func (h *Handler) fetchTasks(ctx context.Context, forceRefresh bool) ([]models.T return nil, err } - // Save to cache - if err := h.store.SaveTasks(tasks); err != nil { - log.Printf("Failed to save tasks to cache: %v", err) + // Build project map from sync response + projectMap := api.BuildProjectMapFromSync(syncResp.Projects) + + // Process sync response + if syncResp.FullSync { + // Full sync: replace all tasks + tasks := api.ConvertSyncItemsToTasks(syncResp.Items, projectMap) + if err := h.store.SaveTasks(tasks); err != nil { + log.Printf("Failed to save tasks to cache: %v", err) + } + } else { + // Incremental sync: merge changes + var deletedIDs []string + for _, item := range syncResp.Items { + if item.IsDeleted || item.IsCompleted { + deletedIDs = append(deletedIDs, item.ID) + } else { + // Upsert active task + task := h.convertSyncItemToTask(item, projectMap) + if err := h.store.UpsertTask(task); err != nil { + log.Printf("Failed to upsert task %s: %v", item.ID, err) + } + } + } + // Delete removed tasks + if len(deletedIDs) > 0 { + if err := h.store.DeleteTasksByIDs(deletedIDs); err != nil { + log.Printf("Failed to delete tasks: %v", err) + } + } + } + + // Store the new sync token + if err := h.store.SetSyncToken(syncService, syncResp.SyncToken); err != nil { + log.Printf("Failed to save sync token: %v", err) } // Update cache metadata @@ -383,7 +428,43 @@ func (h *Handler) fetchTasks(ctx context.Context, forceRefresh bool) ([]models.T log.Printf("Failed to update cache metadata: %v", err) } - return tasks, nil + return h.store.GetTasks() +} + +// convertSyncItemToTask converts a sync item to a Task model +func (h *Handler) convertSyncItemToTask(item api.SyncItemResponse, projectMap map[string]string) models.Task { + task := models.Task{ + ID: item.ID, + Content: item.Content, + Description: item.Description, + ProjectID: item.ProjectID, + ProjectName: projectMap[item.ProjectID], + Priority: item.Priority, + Completed: false, + Labels: item.Labels, + URL: fmt.Sprintf("https://todoist.com/showTask?id=%s", item.ID), + } + + if item.AddedAt != "" { + if createdAt, err := time.Parse(time.RFC3339, item.AddedAt); err == nil { + task.CreatedAt = createdAt + } + } + + if item.Due != nil { + var dueDate time.Time + var err error + if item.Due.Datetime != "" { + dueDate, err = time.Parse(time.RFC3339, item.Due.Datetime) + } else if item.Due.Date != "" { + dueDate, err = time.Parse("2006-01-02", item.Due.Date) + } + if err == nil { + task.DueDate = &dueDate + } + } + + return task } // fetchNotes fetches notes from cache or filesystem diff --git a/internal/handlers/handlers_test.go b/internal/handlers/handlers_test.go index 3ea2a3e..1aa72cc 100644 --- a/internal/handlers/handlers_test.go +++ b/internal/handlers/handlers_test.go @@ -9,6 +9,7 @@ import ( "testing" "time" + "task-dashboard/internal/api" "task-dashboard/internal/config" "task-dashboard/internal/models" "task-dashboard/internal/store" @@ -82,6 +83,32 @@ func (m *mockTodoistClient) CompleteTask(ctx context.Context, taskID string) err return nil } +func (m *mockTodoistClient) Sync(ctx context.Context, syncToken string) (*api.TodoistSyncResponse, error) { + if m.err != nil { + return nil, m.err + } + // Return a mock sync response with tasks converted to sync items + items := make([]api.SyncItemResponse, 0, len(m.tasks)) + for _, task := range m.tasks { + items = append(items, api.SyncItemResponse{ + ID: task.ID, + Content: task.Content, + Description: task.Description, + ProjectID: task.ProjectID, + Priority: task.Priority, + Labels: task.Labels, + IsCompleted: task.Completed, + IsDeleted: false, + }) + } + return &api.TodoistSyncResponse{ + SyncToken: "test-sync-token", + FullSync: true, + Items: items, + Projects: []api.SyncProjectResponse{}, + }, nil +} + // mockTrelloClient creates a mock Trello client for testing type mockTrelloClient struct { boards []models.Board diff --git a/internal/store/sqlite.go b/internal/store/sqlite.go index a078748..b8d0c97 100644 --- a/internal/store/sqlite.go +++ b/internal/store/sqlite.go @@ -186,6 +186,54 @@ func (s *Store) DeleteTask(id string) error { return err } +// UpsertTask inserts or updates a single task +func (s *Store) UpsertTask(task models.Task) error { + labelsJSON, _ := json.Marshal(task.Labels) + _, err := s.db.Exec(` + INSERT OR REPLACE INTO tasks + (id, content, description, project_id, project_name, due_date, priority, completed, labels, url, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP) + `, + task.ID, + task.Content, + task.Description, + task.ProjectID, + task.ProjectName, + task.DueDate, + task.Priority, + task.Completed, + string(labelsJSON), + task.URL, + task.CreatedAt, + ) + return err +} + +// DeleteTasksByIDs removes multiple tasks by ID +func (s *Store) DeleteTasksByIDs(ids []string) error { + if len(ids) == 0 { + return nil + } + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + stmt, err := tx.Prepare(`DELETE FROM tasks WHERE id = ?`) + if err != nil { + return err + } + defer stmt.Close() + + for _, id := range ids { + if _, err := stmt.Exec(id); err != nil { + return err + } + } + return tx.Commit() +} + // Notes operations // SaveNotes saves multiple notes to the database @@ -578,3 +626,33 @@ func (s *Store) DeleteCard(id string) error { _, err := s.db.Exec(`DELETE FROM cards WHERE id = ?`, id) return err } + +// Sync token operations + +// GetSyncToken retrieves the sync token for a service +func (s *Store) GetSyncToken(service string) (string, error) { + var token string + err := s.db.QueryRow(`SELECT token FROM sync_tokens WHERE service = ?`, service).Scan(&token) + if err == sql.ErrNoRows { + return "", nil + } + if err != nil { + return "", err + } + return token, nil +} + +// SetSyncToken saves the sync token for a service +func (s *Store) SetSyncToken(service, token string) error { + _, err := s.db.Exec(` + INSERT OR REPLACE INTO sync_tokens (service, token, updated_at) + VALUES (?, ?, CURRENT_TIMESTAMP) + `, service, token) + return err +} + +// ClearSyncToken removes the sync token for a service +func (s *Store) ClearSyncToken(service string) error { + _, err := s.db.Exec(`DELETE FROM sync_tokens WHERE service = ?`, service) + return err +} -- cgit v1.2.3