GH-25: add background indexing

.github/workflows/release.yml

@@ -118,7 +118,7 @@ jobs:
         run: |
           git config user.name "github-actions[bot]"
           git config user.email "github-actions[bot]@users.noreply.github.com"
-          git tag -a "v${{ github.event.inputs.version }}" -m "Release v${{ github.event.inputs.version }}"
+          git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}"
           git push origin "v${{ github.event.inputs.version }}"
 
       - name: Create GitHub Release
@@ -127,4 +127,4 @@ jobs:
           tag_name: v${{ github.event.inputs.version }}
           name: knowledge-mcp v${{ github.event.inputs.version }}
           generate_release_notes: true
-          files: release/*
+          files: release/*

main.go

@@ -17,7 +17,6 @@ import (
 
 	"github.com/renderorange/knowledge-mcp/hook"
 	"github.com/renderorange/knowledge-mcp/install"
-	"github.com/renderorange/knowledge-mcp/knowledge"
 	"github.com/renderorange/knowledge-mcp/projects"
 	"github.com/renderorange/knowledge-mcp/search"
 	"github.com/renderorange/knowledge-mcp/tools"
@@ -66,6 +65,7 @@ func main() {
 	globalPath := flag.String("global", "", "Path to a global knowledge store shared across all projects")
 	indexOverride := flag.String("index", "", "Override the search index location")
 	storeDir := flag.String("store", "", "Central directory for all knowledge stores; in-tree .agents/ is ignored when set")
+	noIndexOnStartup := flag.Bool("no-index-on-startup", false, "Skip indexing on startup; index may be stale or empty")
 	flag.Var(&roots, "root", "Org root whose immediate children are projects (repeatable)")
 	flag.Var(&projs, "project", "Single project root (repeatable)")
 	flag.Parse()
@@ -100,7 +100,9 @@ func main() {
 	}
 	defer idx.Close()
 
-	indexAll(resolver, idx)
+	if !*noIndexOnStartup {
+		go idx.IndexAll(resolver)
+	}
 
 	// Create MCP server
 	s := server.NewMCPServer(
@@ -129,70 +131,6 @@ func main() {
 	}
 }
 
-// indexAll indexes every ref known to the resolver.
-func indexAll(res *projects.Resolver, idx *search.Index) {
-	for _, ref := range res.Snapshot() {
-		switch ref.Kind {
-		case projects.KindProject:
-			indexProjectKnowledge(res.AgentsDir(ref), ref.Address, idx)
-		case projects.KindOrg:
-			indexOrgKnowledge(res.OrgKnowledgeDir(ref), ref.Name, idx)
-		case projects.KindGlobal:
-			indexProjectKnowledge(res.AgentsDir(ref), ref.Address, idx)
-		}
-	}
-}
-
-// indexProjectKnowledge indexes all knowledge files under an agents dir
-// under the given addressing name.
-func indexProjectKnowledge(agentsDir, projectName string, idx *search.Index) {
-	for _, cat := range knowledge.ValidCategories() {
-		catPath := knowledge.CategoryFilePath(agentsDir, cat)
-		kf, err := knowledge.Load(catPath)
-		if err != nil {
-			continue
-		}
-		for _, entry := range kf.Entries {
-			doc := search.SearchDocument{
-				Summary:  entry.Summary,
-				Detail:   entry.Detail,
-				Rule:     entry.Rule,
-				Category: cat,
-				Project:  projectName,
-			}
-			if addErr := idx.Add(projectName+"/"+entry.ID, doc); addErr != nil {
-				log.Printf("warning: failed to index %s/%s: %v", projectName, entry.ID, addErr)
-			}
-		}
-	}
-}
-
-// indexOrgKnowledge indexes an org's knowledge files as individual markdown
-// sections, so queries return only the relevant section instead of the
-// entire document.
-func indexOrgKnowledge(knowledgeDir, orgName string, idx *search.Index) {
-	for _, catFile := range []string{"architecture.md", "review.md"} {
-		filePath := filepath.Join(knowledgeDir, catFile)
-		data, err := os.ReadFile(filePath)
-		if err != nil {
-			continue
-		}
-		for _, sec := range knowledge.SplitSections(string(data)) {
-			heading, body := sec[0], sec[1]
-			doc := search.SearchDocument{
-				Summary:  fmt.Sprintf("%s: %s", catFile, heading),
-				Detail:   body,
-				Category: "conventions",
-				Project:  orgName,
-			}
-			id := fmt.Sprintf("%s/org-%s::%s", orgName, catFile, heading)
-			if addErr := idx.Add(id, doc); addErr != nil {
-				log.Printf("warning: failed to index %s: %v", id, addErr)
-			}
-		}
-	}
-}
-
 // runHook executes hook-augment against the recorded install config.
 func runHook() {
 	if err := hook.Run(os.Stdin, os.Stdout); err != nil {

search/index.go

@@ -9,10 +9,13 @@ import (
 	"slices"
 	"sort"
 	"strings"
+	"sync"
 
 	"github.com/blevesearch/bleve/v2"
 	"github.com/blevesearch/bleve/v2/mapping"
 	"github.com/blevesearch/bleve/v2/search/query"
+	"github.com/renderorange/knowledge-mcp/knowledge"
+	"github.com/renderorange/knowledge-mcp/projects"
 )
 
 // metaFileVersion is the on-disk key-format version. A mismatch forces an index rebuild.
@@ -54,6 +57,8 @@ type SearchResult struct {
 type Index struct {
 	indexPath string
 	index     bleve.Index
+	ready     chan struct{} // closed when indexing completes or stale data exists
+	readyOnce sync.Once     // guards close of ready to prevent double-close panic
 }
 
 // NewIndex opens or creates a bleve index at the given path. indexNames is the
@@ -97,7 +102,14 @@ func NewIndex(indexPath string, indexNames []string) (*Index, error) {
 		return nil, fmt.Errorf("write index meta: %w", marshalErr)
 	}
 
-	return &Index{indexPath: indexPath, index: idx}, nil
+	ki := &Index{indexPath: indexPath, index: idx, ready: make(chan struct{})}
+
+	// If index has data, close ready immediately (stale data available)
+	if ki.indexHasData() {
+		ki.CloseReady()
+	}
+
+	return ki, nil
 }
 
 func metaFilePath(indexPath string) string {
@@ -196,6 +208,9 @@ func (i *Index) Add(id string, doc SearchDocument) error {
 // Query searches the index, scoped to one project, with full-text
 // search and optional filters.
 func (i *Index) Query(project, q, category string, limit int) ([]SearchResult, error) {
+	// Block until indexing completes if no stale data available
+	<-i.ready
+
 	if limit <= 0 {
 		limit = 10
 	}
@@ -272,6 +287,92 @@ func (i *Index) Close() error {
 	return i.index.Close()
 }
 
+// CloseReady closes the ready channel exactly once, preventing double-close panics.
+// This is used by tests that add documents directly without IndexAll.
+func (i *Index) CloseReady() {
+	i.readyOnce.Do(func() { close(i.ready) })
+}
+
+// indexHasData reports whether the index contains any documents.
+func (i *Index) indexHasData() bool {
+	req := bleve.NewSearchRequest(bleve.NewMatchAllQuery())
+	req.Size = 0
+	result, err := i.index.Search(req)
+	if err != nil {
+		return false
+	}
+	return result.Total > 0
+}
+
+// WaitReady blocks until the index has data available for queries.
+func (i *Index) WaitReady() {
+	<-i.ready
+}
+
+// IndexAll indexes all projects known to the resolver in the background.
+// It closes the ready channel when complete.
+func (i *Index) IndexAll(res *projects.Resolver) {
+	defer i.CloseReady()
+
+	for _, ref := range res.Snapshot() {
+		switch ref.Kind {
+		case projects.KindProject:
+			i.indexProjectKnowledge(res.AgentsDir(ref), ref.Address)
+		case projects.KindOrg:
+			i.indexOrgKnowledge(res.OrgKnowledgeDir(ref), ref.Name)
+		case projects.KindGlobal:
+			i.indexProjectKnowledge(res.AgentsDir(ref), ref.Address)
+		}
+	}
+}
+
+// indexProjectKnowledge indexes all knowledge files under an agents dir.
+func (i *Index) indexProjectKnowledge(agentsDir, projectName string) {
+	for _, cat := range knowledge.ValidCategories() {
+		catPath := knowledge.CategoryFilePath(agentsDir, cat)
+		kf, err := knowledge.Load(catPath)
+		if err != nil {
+			continue
+		}
+		for _, entry := range kf.Entries {
+			doc := SearchDocument{
+				Summary:  entry.Summary,
+				Detail:   entry.Detail,
+				Rule:     entry.Rule,
+				Category: cat,
+				Project:  projectName,
+			}
+			if addErr := i.Add(projectName+"/"+entry.ID, doc); addErr != nil {
+				log.Printf("warning: failed to index %s/%s: %v", projectName, entry.ID, addErr)
+			}
+		}
+	}
+}
+
+// indexOrgKnowledge indexes an org's knowledge files as sections.
+func (i *Index) indexOrgKnowledge(knowledgeDir, orgName string) {
+	for _, catFile := range []string{"architecture.md", "review.md"} {
+		filePath := filepath.Join(knowledgeDir, catFile)
+		data, err := os.ReadFile(filePath)
+		if err != nil {
+			continue
+		}
+		for _, sec := range knowledge.SplitSections(string(data)) {
+			heading, body := sec[0], sec[1]
+			doc := SearchDocument{
+				Summary:  fmt.Sprintf("%s: %s", catFile, heading),
+				Detail:   body,
+				Category: "conventions",
+				Project:  orgName,
+			}
+			id := fmt.Sprintf("%s/org-%s::%s", orgName, catFile, heading)
+			if addErr := i.Add(id, doc); addErr != nil {
+				log.Printf("warning: failed to index %s: %v", id, addErr)
+			}
+		}
+	}
+}
+
 // DeleteIndex removes a bleve index directory.
 func DeleteIndex(indexPath string) error {
 	return os.RemoveAll(indexPath)