fix(connectors): keep counting every slice until a connector's first pass finishes

A first pass's own unfinished slices wrote a count, so later slices of that
pass saw it and stopped counting. Skip only once lastSyncAt shows a finished
pass, which unfinished slices never advance.

Author: waleedlatif1

apps/sim/lib/knowledge/connectors/sync-engine.test.ts

@@ -1592,11 +1592,22 @@ describe('completeSuccessfulSync', () => {
   })
 
   it.each([
-    { recordedDocCount: 7, written: undefined },
-    { recordedDocCount: null, written: 4 },
+    {
+      scenario: 'after a finished pass',
+      lastSyncAt: new Date(),
+      lastSyncDocCount: 7,
+      written: undefined,
+    },
+    { scenario: 'with no count yet', lastSyncAt: null, lastSyncDocCount: null, written: 4 },
+    {
+      scenario: 'on a later slice of the first pass',
+      lastSyncAt: null,
+      lastSyncDocCount: 3,
+      written: 4,
+    },
   ])(
-    'leaves an unfinished pass on the last finished count unless none was recorded: $recordedDocCount',
-    async ({ recordedDocCount, written }) => {
+    'counts an unfinished slice only until a pass has finished: $scenario',
+    async ({ lastSyncAt, lastSyncDocCount, written }) => {
       const { completeSuccessfulSync } = await import('@/lib/knowledge/connectors/sync-engine')
       queueTableRows(schemaMock.knowledgeBase, [{ id: 'kb-1' }])
       queueTableRows(schemaMock.knowledgeConnector, [{ id: 'c-1', lastSyncDocCount: 7 }])
@@ -1624,7 +1635,7 @@ describe('completeSuccessfulSync', () => {
           },
           null,
           false,
-          recordedDocCount
+          { lastSyncAt, lastSyncDocCount }
         )
       ).resolves.toBe(true)
 

apps/sim/lib/knowledge/connectors/sync-engine.ts

@@ -400,8 +400,8 @@ export async function completeSuccessfulSync(
   directoryNotice: string | null = null,
   /** A pending ACL rewrite this run began but did not finish: the flag stays and the next run resumes it. */
   accessRewriteUnfinished = false,
-  /** The count the connector held when this run took its lock. */
-  recordedDocCount: number | null = null
+  /** The connector as this run locked it: whether a pass has finished, and the count it holds. */
+  lockedConnector: { lastSyncAt: Date | null; lastSyncDocCount: number | null } | null = null
 ): Promise<boolean> {
   const processingDispatchFailed = result.processingDispatch.failed > 0
   const contentNotice =
@@ -434,13 +434,17 @@ export async function completeSuccessfulSync(
    * adds and deletes would drift; when it cannot be read the previous count stands, which the
    * guard already treats as a floor.
    *
-   * A slice that leaves its content pass unfinished keeps the count the last finished pass
-   * recorded, the same pass `lastSyncAt` still names, so a large connector is not counted again
-   * on every slice of a pass whose count would only be replaced when it finishes. A connector
-   * with no count yet is still counted, so a first pass shows its progress.
+   * Once a pass has finished, a slice that leaves the next pass unfinished keeps the count that
+   * finished pass recorded, the same pass `lastSyncAt` still names, so a large connector is not
+   * counted again on every slice of a pass whose count is replaced when it finishes. Unfinished
+   * slices never advance `lastSyncAt`, so until a pass finishes every slice is counted and a
+   * first pass shows its progress.
    */
   const keepRecordedCount =
-    contentPass !== undefined && !contentPass.complete && recordedDocCount !== null
+    contentPass !== undefined &&
+    !contentPass.complete &&
+    lockedConnector?.lastSyncAt != null &&
+    lockedConnector.lastSyncDocCount !== null
   const actualDocCount = keepRecordedCount
     ? null
     : await countLiveConnectorDocuments(connectorId).catch((error: unknown) => {
@@ -1429,7 +1433,7 @@ export async function executeSync(
         contentPass,
         directoryNotice,
         accessRewriteUnfinished,
-        connector.lastSyncDocCount
+        connector
       )
 
       if (!completionLanded) {