Implement efficient sync for Todoist and Trello APIs

- Add Todoist Sync API v9 support with incremental sync tokens
- Store sync tokens in SQLite for persistence across restarts
- Add field filtering to Trello API calls to reduce payload size
- Update handlers to use incremental sync (merge changes vs full replace)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

8 files changed, 389 insertions, 8 deletions

diff --git a/internal/api/interfaces.go b/internal/api/interfaces.go
index db7e6c0..2419707 100644
--- a/internal/api/interfaces.go
+++ b/internal/api/interfaces.go
@@ -13,6 +13,7 @@ type TodoistAPI interface {
 	GetProjects(ctx context.Context) ([]models.Project, error)
 	CreateTask(ctx context.Context, content, projectID string, dueDate *time.Time, priority int) (*models.Task, error)
 	CompleteTask(ctx context.Context, taskID string) error
+	Sync(ctx context.Context, syncToken string) (*TodoistSyncResponse, error)
 }
 
 // TrelloAPI defines the interface for Trello operations
diff --git a/internal/api/todoist.go b/internal/api/todoist.go
index 511753d..b51fffd 100644
--- a/internal/api/todoist.go
+++ b/internal/api/todoist.go
@@ -13,7 +13,8 @@ import (
 )
 
 const (
-	todoistBaseURL = "https://api.todoist.com/rest/v2"
+	todoistBaseURL     = "https://api.todoist.com/rest/v2"
+	todoistSyncBaseURL = "https://api.todoist.com/sync/v9"
 )
 
 // TodoistClient handles interactions with the Todoist API
@@ -56,6 +57,40 @@ type todoistProjectResponse struct {
 	Name string `json:"name"`
 }
 
+// Sync API v9 response types
+
+// TodoistSyncResponse represents the Sync API response
+type TodoistSyncResponse struct {
+	SyncToken string                `json:"sync_token"`
+	FullSync  bool                  `json:"full_sync"`
+	Items     []SyncItemResponse    `json:"items"`
+	Projects  []SyncProjectResponse `json:"projects"`
+}
+
+// SyncItemResponse represents a task item from Sync API
+type SyncItemResponse struct {
+	ID          string   `json:"id"`
+	Content     string   `json:"content"`
+	Description string   `json:"description"`
+	ProjectID   string   `json:"project_id"`
+	Priority    int      `json:"priority"`
+	Labels      []string `json:"labels"`
+	Due         *struct {
+		Date     string `json:"date"`
+		Datetime string `json:"datetime"`
+	} `json:"due"`
+	IsCompleted bool   `json:"is_completed"`
+	IsDeleted   bool   `json:"is_deleted"`
+	AddedAt     string `json:"added_at"`
+}
+
+// SyncProjectResponse represents a project from Sync API
+type SyncProjectResponse struct {
+	ID        string `json:"id"`
+	Name      string `json:"name"`
+	IsDeleted bool   `json:"is_deleted"`
+}
+
 // GetTasks fetches all active tasks from Todoist
 func (c *TodoistClient) GetTasks(ctx context.Context) ([]models.Task, error) {
 	req, err := http.NewRequestWithContext(ctx, "GET", c.baseURL+"/tasks", nil)
@@ -167,6 +202,110 @@ func (c *TodoistClient) GetProjects(ctx context.Context) ([]models.Project, erro
 	return projects, nil
 }
 
+// Sync performs an incremental sync using the Sync API v9
+// If syncToken is empty or "*", a full sync is performed
+// Returns the new sync token and the sync response
+func (c *TodoistClient) Sync(ctx context.Context, syncToken string) (*TodoistSyncResponse, error) {
+	if syncToken == "" {
+		syncToken = "*" // Full sync
+	}
+
+	// Prepare sync request
+	payload := map[string]interface{}{
+		"sync_token":     syncToken,
+		"resource_types": []string{"items", "projects"},
+	}
+
+	jsonData, err := json.Marshal(payload)
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal sync request: %w", err)
+	}
+
+	req, err := http.NewRequestWithContext(ctx, "POST", todoistSyncBaseURL+"/sync", bytes.NewBuffer(jsonData))
+	if err != nil {
+		return nil, fmt.Errorf("failed to create sync request: %w", err)
+	}
+
+	req.Header.Set("Authorization", "Bearer "+c.apiKey)
+	req.Header.Set("Content-Type", "application/json")
+
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to perform sync: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		body, _ := io.ReadAll(resp.Body)
+		return nil, fmt.Errorf("todoist sync API error (status %d): %s", resp.StatusCode, string(body))
+	}
+
+	var syncResp TodoistSyncResponse
+	if err := json.NewDecoder(resp.Body).Decode(&syncResp); err != nil {
+		return nil, fmt.Errorf("failed to decode sync response: %w", err)
+	}
+
+	return &syncResp, nil
+}
+
+// ConvertSyncItemsToTasks converts sync API items to Task models
+func ConvertSyncItemsToTasks(items []SyncItemResponse, projectMap map[string]string) []models.Task {
+	tasks := make([]models.Task, 0, len(items))
+	for _, item := range items {
+		// Skip completed or deleted items
+		if item.IsCompleted || item.IsDeleted {
+			continue
+		}
+
+		task := models.Task{
+			ID:          item.ID,
+			Content:     item.Content,
+			Description: item.Description,
+			ProjectID:   item.ProjectID,
+			ProjectName: projectMap[item.ProjectID],
+			Priority:    item.Priority,
+			Completed:   false,
+			Labels:      item.Labels,
+			URL:         fmt.Sprintf("https://todoist.com/showTask?id=%s", item.ID),
+		}
+
+		// Parse added_at
+		if item.AddedAt != "" {
+			if createdAt, err := time.Parse(time.RFC3339, item.AddedAt); err == nil {
+				task.CreatedAt = createdAt
+			}
+		}
+
+		// Parse due date
+		if item.Due != nil {
+			var dueDate time.Time
+			var err error
+			if item.Due.Datetime != "" {
+				dueDate, err = time.Parse(time.RFC3339, item.Due.Datetime)
+			} else if item.Due.Date != "" {
+				dueDate, err = time.Parse("2006-01-02", item.Due.Date)
+			}
+			if err == nil {
+				task.DueDate = &dueDate
+			}
+		}
+
+		tasks = append(tasks, task)
+	}
+	return tasks
+}
+
+// BuildProjectMapFromSync builds a project ID to name map from sync response
+func BuildProjectMapFromSync(projects []SyncProjectResponse) map[string]string {
+	projectMap := make(map[string]string)
+	for _, proj := range projects {
+		if !proj.IsDeleted {
+			projectMap[proj.ID] = proj.Name
+		}
+	}
+	return projectMap
+}
+
 // CreateTask creates a new task in Todoist
 func (c *TodoistClient) CreateTask(ctx context.Context, content, projectID string, dueDate *time.Time, priority int) (*models.Task, error) {
 	// Prepare request payload
diff --git a/internal/api/trello.go b/internal/api/trello.go
index 91d6d66..037c881 100644
--- a/internal/api/trello.go
+++ b/internal/api/trello.go
@@ -68,6 +68,7 @@ func (c *TrelloClient) GetBoards(ctx context.Context) ([]models.Board, error) {
 	params.Set("key", c.apiKey)
 	params.Set("token", c.token)
 	params.Set("filter", "open")
+	params.Set("fields", "id,name") // Only fetch required fields
 
 	reqURL := fmt.Sprintf("%s/members/me/boards?%s", c.baseURL, params.Encode())
 	req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil)
@@ -111,6 +112,7 @@ func (c *TrelloClient) GetCards(ctx context.Context, boardID string) ([]models.C
 	params.Set("key", c.apiKey)
 	params.Set("token", c.token)
 	params.Set("filter", "visible")
+	params.Set("fields", "id,name,idList,due,url,idBoard") // Only fetch required fields
 
 	reqURL := fmt.Sprintf("%s/boards/%s/cards?%s", c.baseURL, boardID, params.Encode())
 	req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil)
@@ -174,6 +176,7 @@ func (c *TrelloClient) getLists(ctx context.Context, boardID string) ([]models.L
 	params := url.Values{}
 	params.Set("key", c.apiKey)
 	params.Set("token", c.token)
+	params.Set("fields", "id,name") // Only fetch required fields
 
 	reqURL := fmt.Sprintf("%s/boards/%s/lists?%s", c.baseURL, boardID, params.Encode())
 	req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil)
diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go
index 0af2bba..f53eced 100644
--- a/internal/handlers/handlers.go
+++ b/internal/handlers/handlers.go
@@ -350,9 +350,10 @@ func (h *Handler) aggregateData(ctx context.Context, forceRefresh bool) (*models
 	return data, nil
 }
 
-// fetchTasks fetches tasks from cache or API
+// fetchTasks fetches tasks from cache or API using incremental sync
 func (h *Handler) fetchTasks(ctx context.Context, forceRefresh bool) ([]models.Task, error) {
 	cacheKey := store.CacheKeyTodoistTasks
+	syncService := "todoist"
 
 	// Check cache validity
 	if !forceRefresh {
@@ -362,8 +363,20 @@ func (h *Handler) fetchTasks(ctx context.Context, forceRefresh bool) ([]models.T
 		}
 	}
 
-	// Fetch from API
-	tasks, err := h.todoistClient.GetTasks(ctx)
+	// Get stored sync token (empty string means full sync)
+	syncToken, err := h.store.GetSyncToken(syncService)
+	if err != nil {
+		log.Printf("Failed to get sync token, will do full sync: %v", err)
+		syncToken = ""
+	}
+
+	// Force full sync if requested
+	if forceRefresh {
+		syncToken = ""
+	}
+
+	// Fetch using Sync API
+	syncResp, err := h.todoistClient.Sync(ctx, syncToken)
 	if err != nil {
 		// Try to return cached data even if stale
 		cachedTasks, cacheErr := h.store.GetTasks()
@@ -373,9 +386,41 @@ func (h *Handler) fetchTasks(ctx context.Context, forceRefresh bool) ([]models.T
 		return nil, err
 	}
 
-	// Save to cache
-	if err := h.store.SaveTasks(tasks); err != nil {
-		log.Printf("Failed to save tasks to cache: %v", err)
+	// Build project map from sync response
+	projectMap := api.BuildProjectMapFromSync(syncResp.Projects)
+
+	// Process sync response
+	if syncResp.FullSync {
+		// Full sync: replace all tasks
+		tasks := api.ConvertSyncItemsToTasks(syncResp.Items, projectMap)
+		if err := h.store.SaveTasks(tasks); err != nil {
+			log.Printf("Failed to save tasks to cache: %v", err)
+		}
+	} else {
+		// Incremental sync: merge changes
+		var deletedIDs []string
+		for _, item := range syncResp.Items {
+			if item.IsDeleted || item.IsCompleted {
+				deletedIDs = append(deletedIDs, item.ID)
+			} else {
+				// Upsert active task
+				task := h.convertSyncItemToTask(item, projectMap)
+				if err := h.store.UpsertTask(task); err != nil {
+					log.Printf("Failed to upsert task %s: %v", item.ID, err)
+				}
+			}
+		}
+		// Delete removed tasks
+		if len(deletedIDs) > 0 {
+			if err := h.store.DeleteTasksByIDs(deletedIDs); err != nil {
+				log.Printf("Failed to delete tasks: %v", err)
+			}
+		}
+	}
+
+	// Store the new sync token
+	if err := h.store.SetSyncToken(syncService, syncResp.SyncToken); err != nil {
+		log.Printf("Failed to save sync token: %v", err)
 	}
 
 	// Update cache metadata
@@ -383,7 +428,43 @@ func (h *Handler) fetchTasks(ctx context.Context, forceRefresh bool) ([]models.T
 		log.Printf("Failed to update cache metadata: %v", err)
 	}
 
-	return tasks, nil
+	return h.store.GetTasks()
+}
+
+// convertSyncItemToTask converts a sync item to a Task model
+func (h *Handler) convertSyncItemToTask(item api.SyncItemResponse, projectMap map[string]string) models.Task {
+	task := models.Task{
+		ID:          item.ID,
+		Content:     item.Content,
+		Description: item.Description,
+		ProjectID:   item.ProjectID,
+		ProjectName: projectMap[item.ProjectID],
+		Priority:    item.Priority,
+		Completed:   false,
+		Labels:      item.Labels,
+		URL:         fmt.Sprintf("https://todoist.com/showTask?id=%s", item.ID),
+	}
+
+	if item.AddedAt != "" {
+		if createdAt, err := time.Parse(time.RFC3339, item.AddedAt); err == nil {
+			task.CreatedAt = createdAt
+		}
+	}
+
+	if item.Due != nil {
+		var dueDate time.Time
+		var err error
+		if item.Due.Datetime != "" {
+			dueDate, err = time.Parse(time.RFC3339, item.Due.Datetime)
+		} else if item.Due.Date != "" {
+			dueDate, err = time.Parse("2006-01-02", item.Due.Date)
+		}
+		if err == nil {
+			task.DueDate = &dueDate
+		}
+	}
+
+	return task
 }
 
 // fetchNotes fetches notes from cache or filesystem
diff --git a/internal/handlers/handlers_test.go b/internal/handlers/handlers_test.go
index 3ea2a3e..1aa72cc 100644
--- a/internal/handlers/handlers_test.go
+++ b/internal/handlers/handlers_test.go
@@ -9,6 +9,7 @@ import (
 	"testing"
 	"time"
 
+	"task-dashboard/internal/api"
 	"task-dashboard/internal/config"
 	"task-dashboard/internal/models"
 	"task-dashboard/internal/store"
@@ -82,6 +83,32 @@ func (m *mockTodoistClient) CompleteTask(ctx context.Context, taskID string) err
 	return nil
 }
 
+func (m *mockTodoistClient) Sync(ctx context.Context, syncToken string) (*api.TodoistSyncResponse, error) {
+	if m.err != nil {
+		return nil, m.err
+	}
+	// Return a mock sync response with tasks converted to sync items
+	items := make([]api.SyncItemResponse, 0, len(m.tasks))
+	for _, task := range m.tasks {
+		items = append(items, api.SyncItemResponse{
+			ID:          task.ID,
+			Content:     task.Content,
+			Description: task.Description,
+			ProjectID:   task.ProjectID,
+			Priority:    task.Priority,
+			Labels:      task.Labels,
+			IsCompleted: task.Completed,
+			IsDeleted:   false,
+		})
+	}
+	return &api.TodoistSyncResponse{
+		SyncToken: "test-sync-token",
+		FullSync:  true,
+		Items:     items,
+		Projects:  []api.SyncProjectResponse{},
+	}, nil
+}
+
 // mockTrelloClient creates a mock Trello client for testing
 type mockTrelloClient struct {
 	boards []models.Board
diff --git a/internal/store/sqlite.go b/internal/store/sqlite.go
index a078748..b8d0c97 100644
--- a/internal/store/sqlite.go
+++ b/internal/store/sqlite.go
@@ -186,6 +186,54 @@ func (s *Store) DeleteTask(id string) error {
 	return err
 }
 
+// UpsertTask inserts or updates a single task
+func (s *Store) UpsertTask(task models.Task) error {
+	labelsJSON, _ := json.Marshal(task.Labels)
+	_, err := s.db.Exec(`
+		INSERT OR REPLACE INTO tasks
+		(id, content, description, project_id, project_name, due_date, priority, completed, labels, url, created_at, updated_at)
+		VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
+	`,
+		task.ID,
+		task.Content,
+		task.Description,
+		task.ProjectID,
+		task.ProjectName,
+		task.DueDate,
+		task.Priority,
+		task.Completed,
+		string(labelsJSON),
+		task.URL,
+		task.CreatedAt,
+	)
+	return err
+}
+
+// DeleteTasksByIDs removes multiple tasks by ID
+func (s *Store) DeleteTasksByIDs(ids []string) error {
+	if len(ids) == 0 {
+		return nil
+	}
+	tx, err := s.db.Begin()
+	if err != nil {
+		return err
+	}
+	defer tx.Rollback()
+
+	stmt, err := tx.Prepare(`DELETE FROM tasks WHERE id = ?`)
+	if err != nil {
+		return err
+	}
+	defer stmt.Close()
+
+	for _, id := range ids {
+		if _, err := stmt.Exec(id); err != nil {
+			return err
+		}
+	}
+	return tx.Commit()
+}
+
 // Notes operations
 
 // SaveNotes saves multiple notes to the database
@@ -578,3 +626,33 @@ func (s *Store) DeleteCard(id string) error {
 	_, err := s.db.Exec(`DELETE FROM cards WHERE id = ?`, id)
 	return err
 }
+
+// Sync token operations
+
+// GetSyncToken retrieves the sync token for a service
+func (s *Store) GetSyncToken(service string) (string, error) {
+	var token string
+	err := s.db.QueryRow(`SELECT token FROM sync_tokens WHERE service = ?`, service).Scan(&token)
+	if err == sql.ErrNoRows {
+		return "", nil
+	}
+	if err != nil {
+		return "", err
+	}
+	return token, nil
+}
+
+// SetSyncToken saves the sync token for a service
+func (s *Store) SetSyncToken(service, token string) error {
+	_, err := s.db.Exec(`
+		INSERT OR REPLACE INTO sync_tokens (service, token, updated_at)
+		VALUES (?, ?, CURRENT_TIMESTAMP)
+	`, service, token)
+	return err
+}
+
+// ClearSyncToken removes the sync token for a service
+func (s *Store) ClearSyncToken(service string) error {
+	_, err := s.db.Exec(`DELETE FROM sync_tokens WHERE service = ?`, service)
+	return err
+}
diff --git a/issues/phase4_step2_efficient_sync.md b/issues/phase4_step2_efficient_sync.md
new file mode 100644
index 0000000..92fa03c
--- /dev/null
+++ b/issues/phase4_step2_efficient_sync.md
@@ -0,0 +1,46 @@
+# Phase 4 Step 2: Efficient Sync Research & Implementation
+
+## Status: COMPLETE
+
+## Goal
+Improve the efficiency of data synchronization to reduce API calls and latency. Previously, the dashboard fetched all data on every refresh.
+
+## Implementation
+
+### Todoist Sync API v9
+- **Endpoint**: `https://api.todoist.com/sync/v9/sync`
+- **Method**: POST with `sync_token` and `resource_types`
+- **Full sync**: When token is `*` or empty, returns all items
+- **Incremental sync**: With valid token, returns only changes since last sync
+- **Response**: Contains `sync_token` (for next request), `full_sync` flag, `items`, `projects`
+
+### Storage
+- New `sync_tokens` table in SQLite for persisting tokens across restarts
+- Store methods: `GetSyncToken()`, `SetSyncToken()`, `ClearSyncToken()`
+- Incremental update methods: `UpsertTask()`, `DeleteTasksByIDs()`
+
+### Handler Logic
+- `fetchTasks()` uses Sync API with stored token
+- If `FullSync=true`: Replace all cached tasks
+- If `FullSync=false`: Merge changes (upsert active, delete completed/deleted)
+- Falls back to cached data on API errors
+
+### Trello Optimization
+- Added `fields` parameter to API calls to reduce response payload:
+  - `GetBoards()`: `fields=id,name`
+  - `GetCards()`: `fields=id,name,idList,due,url,idBoard`
+  - `getLists()`: `fields=id,name`
+
+## Files Modified
+- `migrations/003_add_sync_tokens.sql` (new)
+- `internal/store/sqlite.go` - Added sync token and incremental update methods
+- `internal/api/todoist.go` - Added Sync API support
+- `internal/api/interfaces.go` - Added Sync method to interface
+- `internal/api/trello.go` - Added field filtering
+- `internal/handlers/handlers.go` - Updated fetchTasks to use Sync API
+- `internal/handlers/handlers_test.go` - Updated mock to implement Sync
+
+## Benefits
+- **Todoist**: Reduced data transfer on subsequent syncs (only changes returned)
+- **Trello**: Smaller API responses (only required fields)
+- **Persistence**: Sync token survives restarts, enabling incremental sync on startup
diff --git a/migrations/003_add_sync_tokens.sql b/migrations/003_add_sync_tokens.sql
new file mode 100644
index 0000000..dd29576
--- /dev/null
+++ b/migrations/003_add_sync_tokens.sql
@@ -0,0 +1,6 @@
+-- Sync tokens table for incremental API sync
+CREATE TABLE IF NOT EXISTS sync_tokens (
+    service TEXT PRIMARY KEY,
+    token TEXT NOT NULL,
+    updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
+);