From ea31b25ecabf9f170e6f4b1e6b49dee05b34a82d Mon Sep 17 00:00:00 2001 From: Blaine Motsinger Date: Sun, 20 Sep 2026 10:57:19 -0400 Subject: [PATCH 1/3] GH-23: add health check after bleve.Open to detect silent index corruption --- search/index.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/search/index.go b/search/index.go index c1622e1..9ae8d37 100644 --- a/search/index.go +++ b/search/index.go @@ -92,6 +92,24 @@ func NewIndex(indexPath string, indexNames []string) (*Index, error) { if err != nil { return nil, fmt.Errorf("create index after corruption: %w", err) } + } else { + // Verify the index is actually usable (bolt pages can + // be corrupted in ways that don't surface on Open). + testReq := bleve.NewSearchRequest(bleve.NewMatchAllQuery()) + testReq.Size = 1 + if _, searchErr := idx.Search(testReq); searchErr != nil { + log.Printf("warning: index unhealthy (%v), rebuilding", searchErr) + if rmErr := idx.Close(); rmErr != nil { + return nil, fmt.Errorf("close unhealthy index: %w", rmErr) + } + if rmErr := DeleteIndex(indexPath); rmErr != nil { + return nil, fmt.Errorf("remove unhealthy index: %w", rmErr) + } + idx, err = bleve.New(indexPath, mapping) + if err != nil { + return nil, fmt.Errorf("create index after unhealthy: %w", err) + } + } } } From 4ae9b0b2eea686aa71b7ee6278e31249e5a94eee Mon Sep 17 00:00:00 2001 From: Blaine Motsinger Date: Sun, 20 Sep 2026 11:02:03 -0400 Subject: [PATCH 2/3] GH-23: recover from bbolt panics and test truncated root.bolt recovery --- search/index.go | 81 +++++++++++++++++++++++++++++--------------- search/index_test.go | 53 +++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 28 deletions(-) diff --git a/search/index.go b/search/index.go index 9ae8d37..840d80f 100644 --- a/search/index.go +++ b/search/index.go @@ -81,35 +81,9 @@ func NewIndex(indexPath string, indexNames []string) (*Index, error) { return nil, fmt.Errorf("create index: %w", err) } } else { - idx, err = bleve.Open(indexPath) + idx, err = openOrRecover(indexPath, mapping) if err != nil { - // Index corrupted — rebuild from scratch - log.Printf("warning: index corrupted (%v), rebuilding", err) - if rmErr := DeleteIndex(indexPath); rmErr != nil { - return nil, fmt.Errorf("remove corrupted index: %w", rmErr) - } - idx, err = bleve.New(indexPath, mapping) - if err != nil { - return nil, fmt.Errorf("create index after corruption: %w", err) - } - } else { - // Verify the index is actually usable (bolt pages can - // be corrupted in ways that don't surface on Open). - testReq := bleve.NewSearchRequest(bleve.NewMatchAllQuery()) - testReq.Size = 1 - if _, searchErr := idx.Search(testReq); searchErr != nil { - log.Printf("warning: index unhealthy (%v), rebuilding", searchErr) - if rmErr := idx.Close(); rmErr != nil { - return nil, fmt.Errorf("close unhealthy index: %w", rmErr) - } - if rmErr := DeleteIndex(indexPath); rmErr != nil { - return nil, fmt.Errorf("remove unhealthy index: %w", rmErr) - } - idx, err = bleve.New(indexPath, mapping) - if err != nil { - return nil, fmt.Errorf("create index after unhealthy: %w", err) - } - } + return nil, err } } @@ -145,6 +119,57 @@ func metaStale(indexPath string, indexNames []string) bool { return meta.Version != metaFileVersion || !slices.Equal(meta.Names, want) } +// openOrRecover opens an existing bleve index, recovering from corruption +// (including panics from bbolt on truncated or damaged files) and verifying +// the index is actually usable with a test query. +func openOrRecover(indexPath string, mapping mapping.IndexMapping) (bleve.Index, error) { + var idx bleve.Index + var openErr error + + // bbolt can panic on corrupted files (e.g. truncated root.bolt). + func() { + defer func() { + if r := recover(); r != nil { + openErr = fmt.Errorf("panic opening index: %v", r) + } + }() + idx, openErr = bleve.Open(indexPath) + }() + + if openErr != nil { + log.Printf("warning: index corrupted (%v), rebuilding", openErr) + if rmErr := DeleteIndex(indexPath); rmErr != nil { + return nil, fmt.Errorf("remove corrupted index: %w", rmErr) + } + idx, err := bleve.New(indexPath, mapping) + if err != nil { + return nil, fmt.Errorf("create index after corruption: %w", err) + } + return idx, nil + } + + // Verify the index is actually usable — bolt pages can be + // corrupted in ways that don't surface on Open but hang on use. + testReq := bleve.NewSearchRequest(bleve.NewMatchAllQuery()) + testReq.Size = 1 + if _, searchErr := idx.Search(testReq); searchErr != nil { + log.Printf("warning: index unhealthy (%v), rebuilding", searchErr) + if rmErr := idx.Close(); rmErr != nil { + return nil, fmt.Errorf("close unhealthy index: %w", rmErr) + } + if rmErr := DeleteIndex(indexPath); rmErr != nil { + return nil, fmt.Errorf("remove unhealthy index: %w", rmErr) + } + idx, err := bleve.New(indexPath, mapping) + if err != nil { + return nil, fmt.Errorf("create index after unhealthy: %w", err) + } + return idx, nil + } + + return idx, nil +} + // indexMapping builds an explicit mapping: text analysis for content fields, // exact keyword matching for filter fields. func indexMapping() mapping.IndexMapping { diff --git a/search/index_test.go b/search/index_test.go index 4fc82b4..b2c4c58 100644 --- a/search/index_test.go +++ b/search/index_test.go @@ -320,3 +320,56 @@ func TestIndexRecoverFromCorruption(t *testing.T) { t.Errorf("len(results) = %d, want 1", len(results)) } } + +func TestIndexRecoverFromSilentCorruption(t *testing.T) { + dir := t.TempDir() + indexPath := filepath.Join(dir, "test.bleve") + + // Create index and add a document + idx := newIndex(t, indexPath, []string{"test"}) + if err := idx.Add("test/conv-001", SearchDocument{ + Summary: "original doc", Project: "test", + }); err != nil { + t.Fatalf("Add() error: %v", err) + } + idx.Close() + + // Corrupt root.bolt by truncating it — this simulates the case + // where the file is partially written or has stale pages that + // allow Open() to succeed but break internal operations. + boltPath := filepath.Join(indexPath, "store", "root.bolt") + info, err := os.Stat(boltPath) + if err != nil { + t.Fatalf("Stat(%s) error: %v", boltPath, err) + } + if err := os.Truncate(boltPath, info.Size()/2); err != nil { + t.Fatalf("Truncate() error: %v", err) + } + + // Opening should detect the unhealthy index and rebuild + idx2 := newIndex(t, indexPath, []string{"test"}) + defer idx2.Close() + + // Old data is gone (rebuilt), but server should work + results, err := idx2.Query("test", "", "", 10) + if err != nil { + t.Fatalf("Query() after recovery error: %v", err) + } + if len(results) != 0 { + t.Errorf("len(results) = %d, want 0 (old data should be gone after rebuild)", len(results)) + } + + // Can add new data after recovery + if err := idx2.Add("test/conv-003", SearchDocument{ + Summary: "new doc after silent recovery", Project: "test", + }); err != nil { + t.Fatalf("Add() after recovery error: %v", err) + } + results, err = idx2.Query("test", "silent", "", 10) + if err != nil { + t.Fatalf("Query() after recovery error: %v", err) + } + if len(results) != 1 { + t.Errorf("len(results) = %d, want 1", len(results)) + } +} From eb4b7f6057e20877ffc6e819ec6d3a61855e6cc5 Mon Sep 17 00:00:00 2001 From: Blaine Motsinger Date: Sun, 20 Sep 2026 11:05:59 -0400 Subject: [PATCH 3/3] GH-23: fix test to use garbage bytes instead of truncation (SIGBUS is not recoverable) --- search/index_test.go | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/search/index_test.go b/search/index_test.go index b2c4c58..def9ef7 100644 --- a/search/index_test.go +++ b/search/index_test.go @@ -1,6 +1,7 @@ package search import ( + "bytes" "os" "path/filepath" "testing" @@ -334,16 +335,13 @@ func TestIndexRecoverFromSilentCorruption(t *testing.T) { } idx.Close() - // Corrupt root.bolt by truncating it — this simulates the case - // where the file is partially written or has stale pages that - // allow Open() to succeed but break internal operations. + // Overwrite root.bolt with garbage — simulates a partially written + // or corrupted file where bleve.Open() may succeed but the index + // is internally broken. We overwrite rather than truncate because + // truncating an mmap'd file causes SIGBUS which is not recoverable. boltPath := filepath.Join(indexPath, "store", "root.bolt") - info, err := os.Stat(boltPath) - if err != nil { - t.Fatalf("Stat(%s) error: %v", boltPath, err) - } - if err := os.Truncate(boltPath, info.Size()/2); err != nil { - t.Fatalf("Truncate() error: %v", err) + if err := os.WriteFile(boltPath, bytes.Repeat([]byte{0xff}, 4096), 0600); err != nil { + t.Fatalf("WriteFile() error: %v", err) } // Opening should detect the unhealthy index and rebuild