Skip to content

Commit d5b921c

Browse files
JustYannicccodexjuliusmarmingeYash-Singh1
authored
fix(server): keep history discovery off the writer and pace compaction (pingdotgg#20877)
Co-authored-by: GPT-6.1 Sol <noreply@openai.com> Co-authored-by: Julius Marminge <julius0216@outlook.com> Co-authored-by: Yash Singh <saiansh2525@gmail.com>
1 parent 1c10563 commit d5b921c

8 files changed

Lines changed: 338 additions & 38 deletions

File tree

‎apps/server/src/orchestration-v2/FoundationPersistence.test.ts‎

Lines changed: 212 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
1+
import * as NodeSqlite from "node:sqlite";
12
import * as NodeServices from "@effect/platform-node/NodeServices";
2-
import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient";
3-
import * as ServerSettings from "../serverSettings.ts";
43
import { assert, it } from "@effect/vitest";
54
import {
65
CheckpointId,
@@ -27,6 +26,8 @@ import {
2726
ThreadId,
2827
TurnItemId,
2928
} from "@t3tools/contracts";
29+
import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient";
30+
import * as Clock from "effect/Clock";
3031
import * as DateTime from "effect/DateTime";
3132
import * as Deferred from "effect/Deferred";
3233
import * as Duration from "effect/Duration";
@@ -40,10 +41,16 @@ import * as Ref from "effect/Ref";
4041
import * as Scheduler from "effect/Scheduler";
4142
import * as Schema from "effect/Schema";
4243
import * as Stream from "effect/Stream";
44+
import * as TestClock from "effect/testing/TestClock";
4345
import * as Tracer from "effect/Tracer";
4446
import * as SqlClient from "effect/sql/SqlClient";
4547
import * as Statement from "effect/sql/Statement";
4648

49+
import { vi } from "vite-plus/test";
50+
51+
import * as ServerConfig from "../config.ts";
52+
import * as DatabaseStorage from "../persistence/DatabaseStorage.ts";
53+
import * as ServerSettings from "../serverSettings.ts";
4754
import { LIVE_STREAM_MAX_ITEMS, LiveStreamBufferError } from "./LiveStreamBudget.ts";
4855
import * as SqlitePersistence from "../persistence/Sqlite.ts";
4956
import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts";
@@ -219,6 +226,173 @@ it.effect("rebuilds event history one bounded page at a time", () =>
219226
}).pipe(Effect.provide(layerTest)),
220227
);
221228

229+
const seedCompaction = Effect.fnUntraced(function* () {
230+
const sql = yield* SqlClient.SqlClient;
231+
yield* sql`INSERT INTO orchestration_v2_legacy_imports (
232+
thread_id, source_updated_at, shell_imported_at, transcript_imported_at, imported_message_count
233+
) VALUES ('imported', '2026-10-17', '2026-10-17', '2026-10-17', 0)`;
234+
yield* sql`WITH RECURSIVE history(n) AS (
235+
SELECT 1 UNION ALL SELECT n + 1 FROM history WHERE n < 151
236+
) INSERT INTO orchestration_events (
237+
event_id, aggregate_kind, stream_id, stream_version, event_type,
238+
occurred_at, actor_kind, payload_json, metadata_json, application_event_version
239+
) SELECT 'event-' || n, 'thread', 'thread', n,
240+
CASE WHEN n IN (1, 51, 101) THEN 'message.updated' ELSE 'thread.visited' END,
241+
'2026-10-17', 'server', '{"id":"message"}', '{}', 2 FROM history`;
242+
yield* sql`WITH RECURSIVE history(n) AS (
243+
SELECT 1 UNION ALL SELECT n + 1 FROM history WHERE n < 100
244+
) INSERT INTO orchestration_command_receipts (
245+
rowid, command_id, aggregate_kind, aggregate_id, accepted_at, result_sequence, status, command_type
246+
) SELECT n * 2, 'command-' || n, 'thread', 'imported', '2026-10-17', 0, 'accepted', 'legacy' FROM history`;
247+
});
248+
249+
it.effect("discovers on the reader worker and deletes on the writer", () =>
250+
Effect.gen(function* () {
251+
const fs = yield* FileSystem.FileSystem;
252+
const path = yield* Path.Path;
253+
const filename = path.join(yield* fs.makeTempDirectoryScoped(), "fixture.sqlite");
254+
yield* Effect.gen(function* () {
255+
const sql = yield* SqlClient.SqlClient;
256+
yield* seedCompaction();
257+
// Establish the reader before recording main-thread statements.
258+
yield* sql`SELECT 1`.pipe(NodeSqliteClient.readOnly);
259+
const reads: string[] = [];
260+
const writes: string[] = [];
261+
const all = NodeSqlite.StatementSync.prototype.all;
262+
const run = NodeSqlite.StatementSync.prototype.run;
263+
yield* Effect.acquireRelease(
264+
Effect.sync(() => {
265+
const readSpy = vi
266+
.spyOn(NodeSqlite.StatementSync.prototype, "all")
267+
.mockImplementation(function (this: NodeSqlite.StatementSync, ...params) {
268+
reads.push(this.sourceSQL);
269+
return all.apply(this, params);
270+
});
271+
const writeSpy = vi
272+
.spyOn(NodeSqlite.StatementSync.prototype, "run")
273+
.mockImplementation(function (this: NodeSqlite.StatementSync, ...params) {
274+
writes.push(this.sourceSQL);
275+
return run.apply(this, params);
276+
});
277+
return { readSpy, writeSpy };
278+
}),
279+
({ readSpy, writeSpy }) =>
280+
Effect.sync(() => {
281+
readSpy.mockRestore();
282+
writeSpy.mockRestore();
283+
}),
284+
);
285+
const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2;
286+
const result = yield* maintenance.compactEventStore();
287+
assert.equal(result.deletedEventCount, 148);
288+
assert.equal(result.deletedReceiptCount, 100);
289+
assert.deepEqual(
290+
reads.filter((query) => /\bSELECT\b/.test(query)),
291+
[],
292+
);
293+
assert(writes.some((query) => query.includes("DELETE FROM orchestration_events")));
294+
assert(writes.some((query) => query.includes("DELETE FROM orchestration_command_receipts")));
295+
}).pipe(
296+
Effect.provide(
297+
ProjectionMaintenance.layer.pipe(
298+
Layer.provide(Layer.merge(EventStore.layer, ProjectionStore.layer)),
299+
Layer.provideMerge(SqlitePersistence.layerFromPath(filename)),
300+
),
301+
),
302+
);
303+
}).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
304+
);
305+
306+
it.effect("paces scheduled compaction while preserving retention state and fixed scan bounds", () =>
307+
Effect.gen(function* () {
308+
const sql = yield* SqlClient.SqlClient;
309+
const clock = yield* Clock.Clock;
310+
yield* seedCompaction();
311+
const completed = yield* Deferred.make<void>();
312+
const pauseStarted = yield* Deferred.make<void>();
313+
let activeNanos = 0n;
314+
let eventPages = 0;
315+
const pauses: number[] = [];
316+
const compactionSpans: Tracer.Span[] = [];
317+
const tracer = Tracer.make({
318+
span(options) {
319+
const span = new Tracer.NativeSpan(options);
320+
if (span.name === "ProjectionMaintenance.compactEventStore") compactionSpans.push(span);
321+
const end = span.end.bind(span);
322+
span.end = (time, exit) => {
323+
end(time, exit);
324+
const query = span.attributes.get("db.query.text");
325+
if (typeof query !== "string" || !query.includes("LIMIT")) return;
326+
if (query.includes("FROM orchestration_events AS event")) {
327+
eventPages++;
328+
} else if (!query.includes("FROM orchestration_command_receipts AS receipt")) return;
329+
// Model a page exceeding the active budget, without wall-clock sleeps.
330+
activeNanos += 30_000_000n;
331+
};
332+
return span;
333+
},
334+
});
335+
yield* Effect.gen(function* () {
336+
yield* DatabaseStorage.DatabaseStorage;
337+
yield* TestClock.adjust("1 minute");
338+
yield* Effect.raceFirst(Deferred.await(pauseStarted), Deferred.await(completed));
339+
yield* TestClock.adjust("269 millis");
340+
assert.equal(eventPages, 1);
341+
assert.deepEqual(pauses, [270]);
342+
assert.deepEqual(
343+
compactionSpans.map((span) => span.status._tag),
344+
["Started"],
345+
);
346+
// Foreground commits must not extend the compactor's initial scan bounds.
347+
yield* sql`INSERT INTO orchestration_events (
348+
event_id, aggregate_kind, stream_id, stream_version, event_type,
349+
occurred_at, actor_kind, payload_json, metadata_json, application_event_version
350+
) VALUES ('after-start', 'thread', 'thread', 152, 'thread.visited', '2026-10-17', 'server', '{}', '{}', 2)`;
351+
yield* sql`INSERT INTO orchestration_command_receipts (
352+
command_id, aggregate_kind, aggregate_id, accepted_at, result_sequence, status, command_type
353+
) VALUES ('after-start', 'thread', 'imported', '2026-10-17', 0, 'accepted', 'legacy')`;
354+
yield* TestClock.adjust("10 seconds");
355+
yield* Deferred.await(completed);
356+
}).pipe(
357+
Effect.provide(
358+
DatabaseStorage.layer.pipe(
359+
Layer.provide(ServerConfig.layerTest(process.cwd(), { prefix: "compaction-" })),
360+
Layer.provide(NodeServices.layer),
361+
),
362+
),
363+
Effect.withTracer(tracer),
364+
Effect.provideService(Clock.Clock, {
365+
...clock,
366+
monotonicTimeNanos: Effect.sync(() => activeNanos),
367+
sleep: (duration) => {
368+
const millis = Duration.toMillis(duration);
369+
if (millis < 60_000) {
370+
pauses.push(millis);
371+
Deferred.doneUnsafe(pauseStarted, Effect.void);
372+
} else if (millis === 6 * 60 * 60_000) {
373+
Deferred.doneUnsafe(completed, Effect.void);
374+
}
375+
return clock.sleep(duration);
376+
},
377+
}),
378+
);
379+
assert.deepEqual(pauses, Array(5).fill(270));
380+
assert.deepEqual(
381+
compactionSpans.map((span) => span.status._tag),
382+
["Ended"],
383+
);
384+
assert.deepEqual(yield* sql`SELECT sequence FROM orchestration_events ORDER BY sequence`, [
385+
{ sequence: 1 },
386+
{ sequence: 101 },
387+
{ sequence: 151 },
388+
{ sequence: 152 },
389+
]);
390+
assert.deepEqual(yield* sql`SELECT command_id FROM orchestration_command_receipts`, [
391+
{ command_id: "after-start" },
392+
]);
393+
}).pipe(Effect.scoped, Effect.provide(layerTest)),
394+
);
395+
222396
it.layer(layerTest)("orchestration V2 foundation persistence", (it) => {
223397
it.effect("projects oversized tool bodies before both replay and live RPC retention", () =>
224398
Effect.scoped(
@@ -803,7 +977,7 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => {
803977
ON CONFLICT(command_id) DO NOTHING
804978
`;
805979

806-
const summary = yield* maintenance.compactEventStore;
980+
const summary = yield* maintenance.compactEventStore();
807981
assert.isAtLeast(summary.deletedEventCount, 504);
808982
assert.isAtLeast(summary.deletedReceiptCount, 1);
809983

@@ -3173,34 +3347,56 @@ it.effect.each(["events", "receipts"] as const)(
31733347
yield* sql`CREATE TABLE concurrent_writes (value TEXT)`;
31743348
const readerHeld = yield* Deferred.make<void>();
31753349
const releaseReader = yield* Deferred.make<void>();
3176-
const reader = yield* sql
3177-
.withTransaction(
3178-
Deferred.succeed(readerHeld, undefined).pipe(
3179-
Effect.andThen(Deferred.await(releaseReader)),
3180-
),
3181-
)
3182-
.pipe(NodeSqliteClient.readOnly, Effect.forkChild);
3183-
yield* Deferred.await(readerHeld);
31843350
const discoveryStarted = yield* Deferred.make<Tracer.Span>();
31853351
const observeDiscovery: Statement.Transformer = (statement, _sql, _fiber, span) => {
31863352
const [query] = statement.compile();
31873353
const pageQuery =
31883354
kind === "events" ? /^\s*SELECT\s+event\.sequence/ : /^\s*SELECT\s+receipt\.rowid/;
31893355
return pageQuery.test(query)
3190-
? Deferred.succeed(discoveryStarted, span).pipe(Effect.as(statement))
3356+
? Deferred.succeed(discoveryStarted, span).pipe(
3357+
Effect.andThen(Deferred.await(readerHeld)),
3358+
Effect.as(statement),
3359+
)
31913360
: Effect.succeed(statement);
31923361
};
31933362
const compacted = yield* Deferred.make<void>();
3194-
const compaction = yield* maintenance.compactEventStore.pipe(
3363+
const compaction = yield* maintenance.compactEventStore().pipe(
31953364
Effect.provideService(Statement.CurrentTransformer, observeDiscovery),
31963365
Effect.provideService(Statement.SpanPropagationEnabled, true),
31973366
Effect.tap(() => Deferred.succeed(compacted, undefined)),
31983367
Effect.forkChild,
31993368
);
32003369
const discoverySpan = yield* Deferred.await(discoveryStarted);
3370+
// Let the initial bounds read finish before holding the reader for this page.
3371+
const reader = yield* sql
3372+
.withTransaction(
3373+
Deferred.succeed(readerHeld, undefined).pipe(
3374+
Effect.andThen(Deferred.await(releaseReader)),
3375+
),
3376+
)
3377+
.pipe(NodeSqliteClient.readOnly, Effect.forkChild);
3378+
yield* Deferred.await(readerHeld);
32013379
yield* sql`INSERT INTO concurrent_writes VALUES ('committed')`;
32023380
assert.isFalse(yield* Deferred.isDone(compacted));
32033381
assert.deepEqual(yield* sql`SELECT value FROM concurrent_writes`, [{ value: "committed" }]);
3382+
if (kind === "events") {
3383+
yield* sql`
3384+
INSERT INTO orchestration_events (
3385+
event_id, aggregate_kind, stream_id, stream_version, event_type,
3386+
occurred_at, actor_kind, payload_json, metadata_json, application_event_version
3387+
) VALUES (
3388+
'after-bound', 'thread', 'imported', 2, 'thread.message-appended',
3389+
${now}, 'user', '{}', '{}', 1
3390+
)
3391+
`;
3392+
} else {
3393+
yield* sql`
3394+
INSERT INTO orchestration_command_receipts (
3395+
command_id, aggregate_kind, aggregate_id, accepted_at, result_sequence,
3396+
status, command_type
3397+
) VALUES ('after-bound', 'thread', 'imported', ${now}, 1, 'accepted', 'legacy')
3398+
`;
3399+
}
32043400
yield* Deferred.succeed(releaseReader, undefined);
32053401
yield* Fiber.join(reader);
32063402
const result = yield* Fiber.join(compaction);
@@ -3209,9 +3405,9 @@ it.effect.each(["events", "receipts"] as const)(
32093405
assert.equal(result.deletedReceiptCount, kind === "receipts" ? 1 : 0);
32103406
const retained =
32113407
kind === "events"
3212-
? yield* sql`SELECT stream_id AS id FROM orchestration_events`
3213-
: yield* sql`SELECT aggregate_id AS id FROM orchestration_command_receipts`;
3214-
assert.deepEqual(retained, [{ id: "pending" }]);
3408+
? yield* sql`SELECT event_id AS id FROM orchestration_events ORDER BY sequence`
3409+
: yield* sql`SELECT command_id AS id FROM orchestration_command_receipts ORDER BY rowid`;
3410+
assert.deepEqual(retained, [{ id: "pending" }, { id: "after-bound" }]);
32153411
}).pipe(Effect.provide(maintenanceLayer));
32163412
}).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
32173413
);

0 commit comments

Comments
 (0)