perf: update beancore state incrementally instead of full reload

hmans · claude · hmans · commit f3b56cd3afe4 · 2025-12-20T10:40:35.000+01:00
- Accumulate fsnotify events during debounce window - Process only affected files instead of reloading all beans - Update search index incrementally per-bean - Handle edge cases: rapid updates, create+delete, invalid files Tests added: - Multiple changes in single debounce window - Invalid files are skipped gracefully - Rapid updates to same file coalesce properly 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
diff --git a/internal/beancore/core_test.go b/internal/beancore/core_test.go
@@ -1,6 +1,7 @@
 package beancore
 
 import (
+	"fmt"
 	"os"
 	"path/filepath"
 	"sync"
@@ -851,3 +852,200 @@ func TestSubscribersClosedOnUnwatch(t *testing.T) {
 		t.Error("expected channel to be closed after Unwatch")
 	}
 }
+
+func TestMultipleChangesInDebounceWindow(t *testing.T) {
+	core, beansDir := setupTestCore(t)
+
+	// Create an initial bean to update
+	createTestBean(t, core, "upd1", "To Update", "todo")
+
+	if err := core.StartWatching(); err != nil {
+		t.Fatalf("StartWatching() error = %v", err)
+	}
+	defer core.Unwatch()
+
+	ch, unsub := core.Subscribe()
+	defer unsub()
+
+	time.Sleep(50 * time.Millisecond)
+
+	// Make multiple changes rapidly (within debounce window)
+	// 1. Create a new bean
+	content1 := `---
+title: New Bean
+status: todo
+---
+`
+	os.WriteFile(filepath.Join(beansDir, "new1--new.md"), []byte(content1), 0644)
+
+	// 2. Update existing bean
+	content2 := `---
+title: Updated Bean
+status: in-progress
+---
+`
+	os.WriteFile(filepath.Join(beansDir, "upd1--to-update.md"), []byte(content2), 0644)
+
+	// 3. Create another bean then delete it (net effect: nothing)
+	os.WriteFile(filepath.Join(beansDir, "tmp1--temp.md"), []byte(content1), 0644)
+	os.Remove(filepath.Join(beansDir, "tmp1--temp.md"))
+
+	// Wait for debounced events
+	select {
+	case events := <-ch:
+		// Should have events for new1 (created) and upd1 (updated)
+		// tmp1 might or might not appear depending on timing
+		foundNew := false
+		foundUpd := false
+		for _, e := range events {
+			if e.BeanID == "new1" && e.Type == EventCreated {
+				foundNew = true
+			}
+			if e.BeanID == "upd1" && e.Type == EventUpdated {
+				foundUpd = true
+			}
+		}
+		if !foundNew {
+			t.Error("expected EventCreated for new1")
+		}
+		if !foundUpd {
+			t.Error("expected EventUpdated for upd1")
+		}
+	case <-time.After(500 * time.Millisecond):
+		t.Error("timeout waiting for events")
+	}
+
+	// Verify state is correct
+	_, err := core.Get("new1")
+	if err != nil {
+		t.Errorf("new1 should exist: %v", err)
+	}
+
+	upd, err := core.Get("upd1")
+	if err != nil {
+		t.Fatalf("upd1 should exist: %v", err)
+	}
+	if upd.Title != "Updated Bean" {
+		t.Errorf("upd1 title = %q, want %q", upd.Title, "Updated Bean")
+	}
+
+	// tmp1 should not exist
+	_, err = core.Get("tmp1")
+	if err != ErrNotFound {
+		t.Error("tmp1 should not exist (was created then deleted)")
+	}
+}
+
+func TestInvalidFileIgnored(t *testing.T) {
+	core, beansDir := setupTestCore(t)
+
+	// Create a valid bean first
+	createTestBean(t, core, "val1", "Valid Bean", "todo")
+
+	if err := core.StartWatching(); err != nil {
+		t.Fatalf("StartWatching() error = %v", err)
+	}
+	defer core.Unwatch()
+
+	ch, unsub := core.Subscribe()
+	defer unsub()
+
+	time.Sleep(50 * time.Millisecond)
+
+	// Create an invalid bean file (malformed YAML frontmatter)
+	invalidContent := `---
+title: [unclosed bracket
+status: {broken yaml
+---
+`
+	os.WriteFile(filepath.Join(beansDir, "bad1--invalid.md"), []byte(invalidContent), 0644)
+
+	// Also create a valid bean to verify processing continues
+	validContent := `---
+title: Another Valid
+status: todo
+---
+`
+	os.WriteFile(filepath.Join(beansDir, "val2--another.md"), []byte(validContent), 0644)
+
+	// Wait for events
+	select {
+	case events := <-ch:
+		// Should have event for val2 (created), bad1 should be skipped
+		foundVal2 := false
+		for _, e := range events {
+			if e.BeanID == "val2" && e.Type == EventCreated {
+				foundVal2 = true
+			}
+			if e.BeanID == "bad1" {
+				t.Error("bad1 should not produce an event (invalid file)")
+			}
+		}
+		if !foundVal2 {
+			t.Error("expected EventCreated for val2")
+		}
+	case <-time.After(500 * time.Millisecond):
+		t.Error("timeout waiting for events")
+	}
+
+	// Valid beans should still be accessible
+	if _, err := core.Get("val1"); err != nil {
+		t.Errorf("val1 should still exist: %v", err)
+	}
+	if _, err := core.Get("val2"); err != nil {
+		t.Errorf("val2 should exist: %v", err)
+	}
+}
+
+func TestRapidUpdatesToSameFile(t *testing.T) {
+	core, beansDir := setupTestCore(t)
+
+	createTestBean(t, core, "rap1", "Rapid Updates", "todo")
+
+	if err := core.StartWatching(); err != nil {
+		t.Fatalf("StartWatching() error = %v", err)
+	}
+	defer core.Unwatch()
+
+	ch, unsub := core.Subscribe()
+	defer unsub()
+
+	time.Sleep(50 * time.Millisecond)
+
+	// Write to the same file multiple times rapidly
+	for i := 1; i <= 5; i++ {
+		content := fmt.Sprintf(`---
+title: Update %d
+status: todo
+---
+`, i)
+		os.WriteFile(filepath.Join(beansDir, "rap1--rapid-updates.md"), []byte(content), 0644)
+		time.Sleep(10 * time.Millisecond) // Small delay but within debounce
+	}
+
+	// Should get a single batch of events (debounced)
+	select {
+	case events := <-ch:
+		// Count events for rap1 - should be exactly one
+		rap1Count := 0
+		var lastEvent BeanEvent
+		for _, e := range events {
+			if e.BeanID == "rap1" {
+				rap1Count++
+				lastEvent = e
+			}
+		}
+		if rap1Count != 1 {
+			t.Errorf("expected 1 event for rap1, got %d", rap1Count)
+		}
+		if lastEvent.Type != EventUpdated {
+			t.Errorf("expected EventUpdated, got %v", lastEvent.Type)
+		}
+		// Should have the final value
+		if lastEvent.Bean != nil && lastEvent.Bean.Title != "Update 5" {
+			t.Errorf("expected title 'Update 5', got %q", lastEvent.Bean.Title)
+		}
+	case <-time.After(500 * time.Millisecond):
+		t.Error("timeout waiting for events")
+	}
+}
diff --git a/internal/beancore/watcher.go b/internal/beancore/watcher.go
@@ -1,8 +1,10 @@
 package beancore
 
 import (
+	"os"
 	"path/filepath"
 	"strings"
+	"sync"
 	"sync/atomic"
 	"time"
 
@@ -172,6 +174,8 @@ func (c *Core) watchLoop(watcher *fsnotify.Watcher) {
 	defer watcher.Close()
 
 	var debounceTimer *time.Timer
+	var pendingMu sync.Mutex
+	pendingChanges := make(map[string]fsnotify.Op)
 
 	for {
 		select {
@@ -207,12 +211,23 @@ func (c *Core) watchLoop(watcher *fsnotify.Watcher) {
 				continue
 			}
 
+			// Accumulate changes during debounce window
+			pendingMu.Lock()
+			pendingChanges[event.Name] |= event.Op
+			pendingMu.Unlock()
+
 			// Start/reset debounce timer
 			if debounceTimer != nil {
 				debounceTimer.Stop()
 			}
 			debounceTimer = time.AfterFunc(debounceDelay, func() {
-				c.handleChange()
+				// Swap out pending changes atomically
+				pendingMu.Lock()
+				changes := pendingChanges
+				pendingChanges = make(map[string]fsnotify.Op)
+				pendingMu.Unlock()
+
+				c.handleChanges(changes)
 			})
 
 		case err, ok := <-watcher.Errors:
@@ -225,57 +240,83 @@ func (c *Core) watchLoop(watcher *fsnotify.Watcher) {
 	}
 }
 
-// handleChange reloads beans from disk, detects changes, and notifies subscribers.
-func (c *Core) handleChange() {
-	c.mu.Lock()
-
-	// Check if we're still watching
-	if !c.watching {
-		c.mu.Unlock()
+// handleChanges processes only the files that changed, updating state incrementally.
+func (c *Core) handleChanges(changes map[string]fsnotify.Op) {
+	if len(changes) == 0 {
 		return
 	}
 
-	// Capture old state for comparison
-	oldBeans := make(map[string]*bean.Bean, len(c.beans))
-	for id, b := range c.beans {
-		oldBeans[id] = b
-	}
+	c.mu.Lock()
 
-	// Reload from disk
-	if err := c.loadFromDisk(); err != nil {
-		// On error, just continue - the beans map may be stale but that's better than crashing
+	// Check if we're still watching
+	if !c.watching {
 		c.mu.Unlock()
 		return
 	}
 
-	// Compute events by comparing states
 	var events []BeanEvent
 
-	// Check for created and updated beans
-	for id, newBean := range c.beans {
-		if _, existed := oldBeans[id]; !existed {
-			events = append(events, BeanEvent{
-				Type:   EventCreated,
-				Bean:   newBean,
-				BeanID: id,
-			})
-		} else {
-			events = append(events, BeanEvent{
-				Type:   EventUpdated,
-				Bean:   newBean,
-				BeanID: id,
-			})
+	for path, op := range changes {
+		filename := filepath.Base(path)
+		id, _ := bean.ParseFilename(filename)
+
+		// Handle removes/renames (file is gone)
+		if op&fsnotify.Remove != 0 || op&fsnotify.Rename != 0 {
+			// Check if the file actually exists (rename might be followed by create)
+			if _, exists := c.beans[id]; exists {
+				// Only delete if it was in our map and file is actually gone
+				if !c.fileExists(path) {
+					delete(c.beans, id)
+
+					// Update search index
+					if c.searchIndex != nil {
+						if err := c.searchIndex.DeleteBean(id); err != nil {
+							c.logWarn("failed to remove bean %s from search index: %v", id, err)
+						}
+					}
+
+					events = append(events, BeanEvent{
+						Type:   EventDeleted,
+						Bean:   nil,
+						BeanID: id,
+					})
+				}
+			}
+			continue
 		}
-		delete(oldBeans, id)
-	}
 
-	// Remaining oldBeans entries were deleted
-	for id := range oldBeans {
-		events = append(events, BeanEvent{
-			Type:   EventDeleted,
-			Bean:   nil,
-			BeanID: id,
-		})
+		// Handle creates/writes (file exists or was created)
+		if op&fsnotify.Create != 0 || op&fsnotify.Write != 0 {
+			newBean, err := c.loadBean(path)
+			if err != nil {
+				c.logWarn("failed to load bean from %s: %v", path, err)
+				continue
+			}
+
+			_, existed := c.beans[newBean.ID]
+			c.beans[newBean.ID] = newBean
+
+			// Update search index
+			if c.searchIndex != nil {
+				if err := c.searchIndex.IndexBean(newBean); err != nil {
+					c.logWarn("failed to index bean %s: %v", newBean.ID, err)
+				}
+			}
+
+			if existed {
+				events = append(events, BeanEvent{
+					Type:   EventUpdated,
+					Bean:   newBean,
+					BeanID: newBean.ID,
+				})
+			} else {
+				events = append(events, BeanEvent{
+					Type:   EventCreated,
+					Bean:   newBean,
+					BeanID: newBean.ID,
+				})
+			}
+		}
 	}
 
 	callback := c.onChange
@@ -289,3 +330,9 @@ func (c *Core) handleChange() {
 		callback()
 	}
 }
+
+// fileExists checks if a file exists at the given path.
+func (c *Core) fileExists(path string) bool {
+	_, err := os.Stat(path)
+	return err == nil
+}
