improvement(knowledge): leave the legacy keyword backfill unscoped

Author: waleedlatif1

packages/db/script-migrations-paused-billing-attribution.test.ts

@@ -377,6 +377,7 @@ describe('script migration registry', () => {
       '0022_projection_source_acl_backfill',
       '0023_projection_acl_skip_unfilled',
       '0024_knowledge_projection_async',
+      '0025_scope_keyword_projections',
     ])
   })
 })

packages/db/script-migrations/0016_backfill_search_vectors.integration.ts

@@ -44,11 +44,9 @@ describe('search projection upgrade in PostgreSQL', () => {
     await sql.unsafe(
       `ALTER TABLE embedding_search ${fields.map((field) => `DROP COLUMN ${field}`).join(', ')}`
     )
-    /** A workspace holds one search index, so each fixture index has its own. */
-    await sql`INSERT INTO knowledge_base (id, user_id, name, workspace_id, embedding_model, is_search_index) VALUES
-      ('prefix', 'reader', 'Prefix fixture', 'workspace', 'text-embedding-3-small', true),
-      ('full', 'reader', 'Full fixture', 'full-workspace', 'gemini-embedding-001', true),
-      ('legacy', 'reader', 'Legacy fixture', 'workspace', 'text-embedding-3-small', false)`
+    await sql`INSERT INTO knowledge_base (id, user_id, name, workspace_id, embedding_model) VALUES
+      ('prefix', 'reader', 'Prefix fixture', 'workspace', 'text-embedding-3-small'),
+      ('full', 'reader', 'Full fixture', 'workspace', 'gemini-embedding-001')`
     await backfillEmbeddingSearch(sql)
     await sql.unsafe(`INSERT INTO embedding
       (id, knowledge_base_id, document_id, chunk_index, chunk_hash, content, content_length, token_count, start_offset, end_offset, embedding)
@@ -155,22 +153,6 @@ describe('search projection upgrade in PostgreSQL', () => {
     expect(await backfillSearchKeywords(sql)).toBe(0)
   })
 
-  it('leaves the chunks of a base that is not a search index unfilled', async () => {
-    await sql.unsafe(`INSERT INTO embedding
-      (id, knowledge_base_id, document_id, chunk_index, chunk_hash, content, content_length, token_count, start_offset, end_offset, embedding)
-      VALUES ('legacy-chunk', 'legacy', 'legacy-document', 0, 'legacy-hash', 'Synthetic fixture', 17, 4, 0, 17,
-        array_fill(0.01::real, ARRAY[1536])::vector(1536))`)
-    await sql`DELETE FROM embedding_keyword_search WHERE id = 'legacy-chunk'`
-    try {
-      expect(await backfillSearchKeywords(sql)).toBe(0)
-      expect(
-        await sql`SELECT id FROM embedding_keyword_search WHERE id = 'legacy-chunk'`
-      ).toHaveLength(0)
-    } finally {
-      await sql`DELETE FROM embedding WHERE id = 'legacy-chunk'`
-    }
-  })
-
   it('repairs an interrupted index build and preserves valid indexes on replay', async () => {
     await expect(
       sql.unsafe(

packages/db/script-migrations/0016_backfill_search_vectors.ts

@@ -115,11 +115,7 @@ export async function backfillSearchVectors(sql: Sql): Promise<number> {
   return count
 }
 
-/**
- * Keeps keyword scoring independent of the full-vector and chunk-content storage working sets. Only
- * search-index chunks are filled: they are the only rows the projection holds once
- * `0025_scope_keyword_projections` scopes its triggers, so a rerun cannot refill the rest.
- */
+/** Keeps keyword scoring independent of the full-vector and chunk-content storage working sets. */
 export async function backfillSearchKeywords(sql: Sql): Promise<number> {
   await sql.begin(async (tx) => {
     await tx.unsafe("SET LOCAL lock_timeout = '5s'")
@@ -163,7 +159,6 @@ export async function backfillSearchKeywords(sql: Sql): Promise<number> {
         ), batch AS MATERIALIZED (
           SELECT e.id, e.knowledge_base_id, e.document_id, e.enabled, e.content_tsv
           FROM missing m INNER JOIN embedding e ON e.id = m.id
-          INNER JOIN knowledge_base k ON k.id = e.knowledge_base_id AND k.is_search_index
           ORDER BY e.id FOR KEY SHARE OF e
         ), inserted AS (
           INSERT INTO embedding_keyword_search (id, knowledge_base_id, document_id, enabled, content_tsv)

packages/db/script-migrations/0025_scope_keyword_projections.ts

@@ -117,8 +117,9 @@ async function installDocumentTrigger(tx: TransactionSql): Promise<void> {
  * whichever mode runs. The document trigger stops firing on inserts and on updates that change
  * nothing it carries.
  *
- * Rows already written for other bases are left in place: removing them is a table owner's
- * maintenance, not a deploy's. All of it is installed in one transaction, so no marker change sees
+ * Rows already written for other bases, and those a rerun of `0016_backfill_search_vectors` writes
+ * before this reruns after it, are left in place: nothing reads them, and removing them is a table
+ * owner's maintenance, not a deploy's. All of it is installed in one transaction, so no marker change sees
  * the scoped chunk trigger without the base trigger that backfills it. Each attempt waits at most
  * {@link TRIGGER_LOCK_TIMEOUT} for the trigger DDL's locks and is retried within the budget.
  * Idempotent.