1+ import * as NodeSqlite from "node:sqlite" ;
12import * as NodeServices from "@effect/platform-node/NodeServices" ;
2- import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient" ;
3- import * as ServerSettings from "../serverSettings.ts" ;
43import { assert , it } from "@effect/vitest" ;
54import {
65 CheckpointId ,
@@ -27,6 +26,8 @@ import {
2726 ThreadId ,
2827 TurnItemId ,
2928} from "@t3tools/contracts" ;
29+ import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient" ;
30+ import * as Clock from "effect/Clock" ;
3031import * as DateTime from "effect/DateTime" ;
3132import * as Deferred from "effect/Deferred" ;
3233import * as Duration from "effect/Duration" ;
@@ -40,10 +41,16 @@ import * as Ref from "effect/Ref";
4041import * as Scheduler from "effect/Scheduler" ;
4142import * as Schema from "effect/Schema" ;
4243import * as Stream from "effect/Stream" ;
44+ import * as TestClock from "effect/testing/TestClock" ;
4345import * as Tracer from "effect/Tracer" ;
4446import * as SqlClient from "effect/sql/SqlClient" ;
4547import * as Statement from "effect/sql/Statement" ;
4648
49+ import { vi } from "vite-plus/test" ;
50+
51+ import * as ServerConfig from "../config.ts" ;
52+ import * as DatabaseStorage from "../persistence/DatabaseStorage.ts" ;
53+ import * as ServerSettings from "../serverSettings.ts" ;
4754import { LIVE_STREAM_MAX_ITEMS , LiveStreamBufferError } from "./LiveStreamBudget.ts" ;
4855import * as SqlitePersistence from "../persistence/Sqlite.ts" ;
4956import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts" ;
@@ -219,6 +226,173 @@ it.effect("rebuilds event history one bounded page at a time", () =>
219226 } ) . pipe ( Effect . provide ( layerTest ) ) ,
220227) ;
221228
229+ const seedCompaction = Effect . fnUntraced ( function * ( ) {
230+ const sql = yield * SqlClient . SqlClient ;
231+ yield * sql `INSERT INTO orchestration_v2_legacy_imports (
232+ thread_id, source_updated_at, shell_imported_at, transcript_imported_at, imported_message_count
233+ ) VALUES ('imported', '2026-10-17', '2026-10-17', '2026-10-17', 0)` ;
234+ yield * sql `WITH RECURSIVE history(n) AS (
235+ SELECT 1 UNION ALL SELECT n + 1 FROM history WHERE n < 151
236+ ) INSERT INTO orchestration_events (
237+ event_id, aggregate_kind, stream_id, stream_version, event_type,
238+ occurred_at, actor_kind, payload_json, metadata_json, application_event_version
239+ ) SELECT 'event-' || n, 'thread', 'thread', n,
240+ CASE WHEN n IN (1, 51, 101) THEN 'message.updated' ELSE 'thread.visited' END,
241+ '2026-10-17', 'server', '{"id":"message"}', '{}', 2 FROM history` ;
242+ yield * sql `WITH RECURSIVE history(n) AS (
243+ SELECT 1 UNION ALL SELECT n + 1 FROM history WHERE n < 100
244+ ) INSERT INTO orchestration_command_receipts (
245+ rowid, command_id, aggregate_kind, aggregate_id, accepted_at, result_sequence, status, command_type
246+ ) SELECT n * 2, 'command-' || n, 'thread', 'imported', '2026-10-17', 0, 'accepted', 'legacy' FROM history` ;
247+ } ) ;
248+
249+ it . effect ( "discovers on the reader worker and deletes on the writer" , ( ) =>
250+ Effect . gen ( function * ( ) {
251+ const fs = yield * FileSystem . FileSystem ;
252+ const path = yield * Path . Path ;
253+ const filename = path . join ( yield * fs . makeTempDirectoryScoped ( ) , "fixture.sqlite" ) ;
254+ yield * Effect . gen ( function * ( ) {
255+ const sql = yield * SqlClient . SqlClient ;
256+ yield * seedCompaction ( ) ;
257+ // Establish the reader before recording main-thread statements.
258+ yield * sql `SELECT 1` . pipe ( NodeSqliteClient . readOnly ) ;
259+ const reads : string [ ] = [ ] ;
260+ const writes : string [ ] = [ ] ;
261+ const all = NodeSqlite . StatementSync . prototype . all ;
262+ const run = NodeSqlite . StatementSync . prototype . run ;
263+ yield * Effect . acquireRelease (
264+ Effect . sync ( ( ) => {
265+ const readSpy = vi
266+ . spyOn ( NodeSqlite . StatementSync . prototype , "all" )
267+ . mockImplementation ( function ( this : NodeSqlite . StatementSync , ...params ) {
268+ reads . push ( this . sourceSQL ) ;
269+ return all . apply ( this , params ) ;
270+ } ) ;
271+ const writeSpy = vi
272+ . spyOn ( NodeSqlite . StatementSync . prototype , "run" )
273+ . mockImplementation ( function ( this : NodeSqlite . StatementSync , ...params ) {
274+ writes . push ( this . sourceSQL ) ;
275+ return run . apply ( this , params ) ;
276+ } ) ;
277+ return { readSpy, writeSpy } ;
278+ } ) ,
279+ ( { readSpy, writeSpy } ) =>
280+ Effect . sync ( ( ) => {
281+ readSpy . mockRestore ( ) ;
282+ writeSpy . mockRestore ( ) ;
283+ } ) ,
284+ ) ;
285+ const maintenance = yield * ProjectionMaintenance . ProjectionMaintenanceV2 ;
286+ const result = yield * maintenance . compactEventStore ( ) ;
287+ assert . equal ( result . deletedEventCount , 148 ) ;
288+ assert . equal ( result . deletedReceiptCount , 100 ) ;
289+ assert . deepEqual (
290+ reads . filter ( ( query ) => / \b S E L E C T \b / . test ( query ) ) ,
291+ [ ] ,
292+ ) ;
293+ assert ( writes . some ( ( query ) => query . includes ( "DELETE FROM orchestration_events" ) ) ) ;
294+ assert ( writes . some ( ( query ) => query . includes ( "DELETE FROM orchestration_command_receipts" ) ) ) ;
295+ } ) . pipe (
296+ Effect . provide (
297+ ProjectionMaintenance . layer . pipe (
298+ Layer . provide ( Layer . merge ( EventStore . layer , ProjectionStore . layer ) ) ,
299+ Layer . provideMerge ( SqlitePersistence . layerFromPath ( filename ) ) ,
300+ ) ,
301+ ) ,
302+ ) ;
303+ } ) . pipe ( Effect . scoped , Effect . provide ( NodeServices . layer ) ) ,
304+ ) ;
305+
306+ it . effect ( "paces scheduled compaction while preserving retention state and fixed scan bounds" , ( ) =>
307+ Effect . gen ( function * ( ) {
308+ const sql = yield * SqlClient . SqlClient ;
309+ const clock = yield * Clock . Clock ;
310+ yield * seedCompaction ( ) ;
311+ const completed = yield * Deferred . make < void > ( ) ;
312+ const pauseStarted = yield * Deferred . make < void > ( ) ;
313+ let activeNanos = 0n ;
314+ let eventPages = 0 ;
315+ const pauses : number [ ] = [ ] ;
316+ const compactionSpans : Tracer . Span [ ] = [ ] ;
317+ const tracer = Tracer . make ( {
318+ span ( options ) {
319+ const span = new Tracer . NativeSpan ( options ) ;
320+ if ( span . name === "ProjectionMaintenance.compactEventStore" ) compactionSpans . push ( span ) ;
321+ const end = span . end . bind ( span ) ;
322+ span . end = ( time , exit ) => {
323+ end ( time , exit ) ;
324+ const query = span . attributes . get ( "db.query.text" ) ;
325+ if ( typeof query !== "string" || ! query . includes ( "LIMIT" ) ) return ;
326+ if ( query . includes ( "FROM orchestration_events AS event" ) ) {
327+ eventPages ++ ;
328+ } else if ( ! query . includes ( "FROM orchestration_command_receipts AS receipt" ) ) return ;
329+ // Model a page exceeding the active budget, without wall-clock sleeps.
330+ activeNanos += 30_000_000n ;
331+ } ;
332+ return span ;
333+ } ,
334+ } ) ;
335+ yield * Effect . gen ( function * ( ) {
336+ yield * DatabaseStorage . DatabaseStorage ;
337+ yield * TestClock . adjust ( "1 minute" ) ;
338+ yield * Effect . raceFirst ( Deferred . await ( pauseStarted ) , Deferred . await ( completed ) ) ;
339+ yield * TestClock . adjust ( "269 millis" ) ;
340+ assert . equal ( eventPages , 1 ) ;
341+ assert . deepEqual ( pauses , [ 270 ] ) ;
342+ assert . deepEqual (
343+ compactionSpans . map ( ( span ) => span . status . _tag ) ,
344+ [ "Started" ] ,
345+ ) ;
346+ // Foreground commits must not extend the compactor's initial scan bounds.
347+ yield * sql `INSERT INTO orchestration_events (
348+ event_id, aggregate_kind, stream_id, stream_version, event_type,
349+ occurred_at, actor_kind, payload_json, metadata_json, application_event_version
350+ ) VALUES ('after-start', 'thread', 'thread', 152, 'thread.visited', '2026-10-17', 'server', '{}', '{}', 2)` ;
351+ yield * sql `INSERT INTO orchestration_command_receipts (
352+ command_id, aggregate_kind, aggregate_id, accepted_at, result_sequence, status, command_type
353+ ) VALUES ('after-start', 'thread', 'imported', '2026-10-17', 0, 'accepted', 'legacy')` ;
354+ yield * TestClock . adjust ( "10 seconds" ) ;
355+ yield * Deferred . await ( completed ) ;
356+ } ) . pipe (
357+ Effect . provide (
358+ DatabaseStorage . layer . pipe (
359+ Layer . provide ( ServerConfig . layerTest ( process . cwd ( ) , { prefix : "compaction-" } ) ) ,
360+ Layer . provide ( NodeServices . layer ) ,
361+ ) ,
362+ ) ,
363+ Effect . withTracer ( tracer ) ,
364+ Effect . provideService ( Clock . Clock , {
365+ ...clock ,
366+ monotonicTimeNanos : Effect . sync ( ( ) => activeNanos ) ,
367+ sleep : ( duration ) => {
368+ const millis = Duration . toMillis ( duration ) ;
369+ if ( millis < 60_000 ) {
370+ pauses . push ( millis ) ;
371+ Deferred . doneUnsafe ( pauseStarted , Effect . void ) ;
372+ } else if ( millis === 6 * 60 * 60_000 ) {
373+ Deferred . doneUnsafe ( completed , Effect . void ) ;
374+ }
375+ return clock . sleep ( duration ) ;
376+ } ,
377+ } ) ,
378+ ) ;
379+ assert . deepEqual ( pauses , Array ( 5 ) . fill ( 270 ) ) ;
380+ assert . deepEqual (
381+ compactionSpans . map ( ( span ) => span . status . _tag ) ,
382+ [ "Ended" ] ,
383+ ) ;
384+ assert . deepEqual ( yield * sql `SELECT sequence FROM orchestration_events ORDER BY sequence` , [
385+ { sequence : 1 } ,
386+ { sequence : 101 } ,
387+ { sequence : 151 } ,
388+ { sequence : 152 } ,
389+ ] ) ;
390+ assert . deepEqual ( yield * sql `SELECT command_id FROM orchestration_command_receipts` , [
391+ { command_id : "after-start" } ,
392+ ] ) ;
393+ } ) . pipe ( Effect . scoped , Effect . provide ( layerTest ) ) ,
394+ ) ;
395+
222396it . layer ( layerTest ) ( "orchestration V2 foundation persistence" , ( it ) => {
223397 it . effect ( "projects oversized tool bodies before both replay and live RPC retention" , ( ) =>
224398 Effect . scoped (
@@ -803,7 +977,7 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => {
803977 ON CONFLICT(command_id) DO NOTHING
804978 ` ;
805979
806- const summary = yield * maintenance . compactEventStore ;
980+ const summary = yield * maintenance . compactEventStore ( ) ;
807981 assert . isAtLeast ( summary . deletedEventCount , 504 ) ;
808982 assert . isAtLeast ( summary . deletedReceiptCount , 1 ) ;
809983
@@ -3173,34 +3347,56 @@ it.effect.each(["events", "receipts"] as const)(
31733347 yield * sql `CREATE TABLE concurrent_writes (value TEXT)` ;
31743348 const readerHeld = yield * Deferred . make < void > ( ) ;
31753349 const releaseReader = yield * Deferred . make < void > ( ) ;
3176- const reader = yield * sql
3177- . withTransaction (
3178- Deferred . succeed ( readerHeld , undefined ) . pipe (
3179- Effect . andThen ( Deferred . await ( releaseReader ) ) ,
3180- ) ,
3181- )
3182- . pipe ( NodeSqliteClient . readOnly , Effect . forkChild ) ;
3183- yield * Deferred . await ( readerHeld ) ;
31843350 const discoveryStarted = yield * Deferred . make < Tracer . Span > ( ) ;
31853351 const observeDiscovery : Statement . Transformer = ( statement , _sql , _fiber , span ) => {
31863352 const [ query ] = statement . compile ( ) ;
31873353 const pageQuery =
31883354 kind === "events" ? / ^ \s * S E L E C T \s + e v e n t \. s e q u e n c e / : / ^ \s * S E L E C T \s + r e c e i p t \. r o w i d / ;
31893355 return pageQuery . test ( query )
3190- ? Deferred . succeed ( discoveryStarted , span ) . pipe ( Effect . as ( statement ) )
3356+ ? Deferred . succeed ( discoveryStarted , span ) . pipe (
3357+ Effect . andThen ( Deferred . await ( readerHeld ) ) ,
3358+ Effect . as ( statement ) ,
3359+ )
31913360 : Effect . succeed ( statement ) ;
31923361 } ;
31933362 const compacted = yield * Deferred . make < void > ( ) ;
3194- const compaction = yield * maintenance . compactEventStore . pipe (
3363+ const compaction = yield * maintenance . compactEventStore ( ) . pipe (
31953364 Effect . provideService ( Statement . CurrentTransformer , observeDiscovery ) ,
31963365 Effect . provideService ( Statement . SpanPropagationEnabled , true ) ,
31973366 Effect . tap ( ( ) => Deferred . succeed ( compacted , undefined ) ) ,
31983367 Effect . forkChild ,
31993368 ) ;
32003369 const discoverySpan = yield * Deferred . await ( discoveryStarted ) ;
3370+ // Let the initial bounds read finish before holding the reader for this page.
3371+ const reader = yield * sql
3372+ . withTransaction (
3373+ Deferred . succeed ( readerHeld , undefined ) . pipe (
3374+ Effect . andThen ( Deferred . await ( releaseReader ) ) ,
3375+ ) ,
3376+ )
3377+ . pipe ( NodeSqliteClient . readOnly , Effect . forkChild ) ;
3378+ yield * Deferred . await ( readerHeld ) ;
32013379 yield * sql `INSERT INTO concurrent_writes VALUES ('committed')` ;
32023380 assert . isFalse ( yield * Deferred . isDone ( compacted ) ) ;
32033381 assert . deepEqual ( yield * sql `SELECT value FROM concurrent_writes` , [ { value : "committed" } ] ) ;
3382+ if ( kind === "events" ) {
3383+ yield * sql `
3384+ INSERT INTO orchestration_events (
3385+ event_id, aggregate_kind, stream_id, stream_version, event_type,
3386+ occurred_at, actor_kind, payload_json, metadata_json, application_event_version
3387+ ) VALUES (
3388+ 'after-bound', 'thread', 'imported', 2, 'thread.message-appended',
3389+ ${ now } , 'user', '{}', '{}', 1
3390+ )
3391+ ` ;
3392+ } else {
3393+ yield * sql `
3394+ INSERT INTO orchestration_command_receipts (
3395+ command_id, aggregate_kind, aggregate_id, accepted_at, result_sequence,
3396+ status, command_type
3397+ ) VALUES ('after-bound', 'thread', 'imported', ${ now } , 1, 'accepted', 'legacy')
3398+ ` ;
3399+ }
32043400 yield * Deferred . succeed ( releaseReader , undefined ) ;
32053401 yield * Fiber . join ( reader ) ;
32063402 const result = yield * Fiber . join ( compaction ) ;
@@ -3209,9 +3405,9 @@ it.effect.each(["events", "receipts"] as const)(
32093405 assert . equal ( result . deletedReceiptCount , kind === "receipts" ? 1 : 0 ) ;
32103406 const retained =
32113407 kind === "events"
3212- ? yield * sql `SELECT stream_id AS id FROM orchestration_events`
3213- : yield * sql `SELECT aggregate_id AS id FROM orchestration_command_receipts` ;
3214- assert . deepEqual ( retained , [ { id : "pending" } ] ) ;
3408+ ? yield * sql `SELECT event_id AS id FROM orchestration_events ORDER BY sequence `
3409+ : yield * sql `SELECT command_id AS id FROM orchestration_command_receipts ORDER BY rowid ` ;
3410+ assert . deepEqual ( retained , [ { id : "pending" } , { id : "after-bound" } ] ) ;
32153411 } ) . pipe ( Effect . provide ( maintenanceLayer ) ) ;
32163412 } ) . pipe ( Effect . scoped , Effect . provide ( NodeServices . layer ) ) ,
32173413) ;
0 commit comments