diff --git a/.changeset/alchemy-local-dev.md b/.changeset/alchemy-local-dev.md new file mode 100644 index 0000000000..5d10a24c3d --- /dev/null +++ b/.changeset/alchemy-local-dev.md @@ -0,0 +1,9 @@ +--- +"@maple-dev/alchemy": minor +--- + +`alchemy dev` can run Maple Local for the stack. Declare `Local("observability")` from `@maple-dev/alchemy/local`, register `localProviders()`, and pass it as `Telemetry({ local })`: under `alchemy dev` every Worker exports to one shared loopback server, with no ingest key and no Maple account, and the dashboard opens at `local.maple.dev`. Deploys ignore `local` and use `ingestKey` and `endpoint` as before. + +The server runs in an Alchemy sidecar process, survives stack reloads, and stores data per stack, stage and resource under `.alchemy/maple-local/`. The first run downloads the pinned chDB library, verifies its sha256 and caches it in `~/.maple/cache/libchdb`; `MAPLE_LIBCHDB` points at a copy you already have. `LocalProps` takes `dataDir`, `port` (default `4318`) and `onDirtyStore`: a store left unclean by a killed sidecar is refused unless it is `"wipe"`. + +Requires Bun, and `@effect/platform-bun` as an optional peer dependency (Alchemy already needs it under Bun). diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dbad2256d6..257b27a702 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -77,6 +77,7 @@ jobs: cli: - *base - 'apps/cli/**' + - 'packages/local-server/**' - 'apps/local-ui/**' # local-ui renders apps/web's source modules. - 'apps/web/src/**' @@ -95,6 +96,7 @@ jobs: archive: - *base - 'apps/cli/**' + - 'packages/local-server/**' - 'apps/local-ui/**' - 'scripts/build-local-binary.sh' - 'scripts/gen-ui-embed.ts' diff --git a/.oxlintrc.json b/.oxlintrc.json index 9813cccb36..0afabbc4cb 100644 --- a/.oxlintrc.json +++ b/.oxlintrc.json @@ -217,6 +217,62 @@ "typescript/no-non-null-assertion": "off" } }, + // Burndown entries relocated with their implementation from apps/cli into + // @maple/local-server, one block per rule. Drop each file as it is converted. + { + "files": [ + "packages/local-server/src/archives/export.ts", + "packages/local-server/src/archives/gc.ts", + "packages/local-server/src/archives/generation.ts", + "packages/local-server/src/archives/journal.ts", + "packages/local-server/src/archives/listing.ts", + "packages/local-server/src/archives/manifest.ts", + "packages/local-server/src/archives/paths.ts", + "packages/local-server/src/chdb.ts", + "packages/local-server/src/checkpoints.ts", + "packages/local-server/src/durable-files.ts", + "packages/local-server/src/eventing/control-store.ts", + "packages/local-server/src/local-address.ts", + "packages/local-server/src/local-store-migration-journal.ts", + "packages/local-server/src/otlp/encode.ts", + "packages/local-server/src/schema-physical.ts", + "packages/local-server/src/serve.ts", + "packages/local-server/src/store-claim.ts", + "packages/local-server/src/store-version.ts" + ], + "rules": { "maple/no-try-catch": "off" } + }, + { + "files": [ + "packages/local-server/src/archives/export.ts", + "packages/local-server/src/archives/gc.ts", + "packages/local-server/src/archives/journal.ts", + "packages/local-server/src/archives/paths.ts", + "packages/local-server/src/archives/retention.ts", + "packages/local-server/src/checkpoints.ts", + "packages/local-server/src/insert-batcher.ts", + "packages/local-server/src/local-store-migration-journal.ts", + "packages/local-server/src/otlp/encode.ts", + "packages/local-server/src/schema-manifest.ts", + "packages/local-server/src/schema-physical.ts", + "packages/local-server/src/scoped-delete.ts" + ], + "rules": { "typescript/no-non-null-assertion": "off" } + }, + { + "files": [ + "packages/local-server/src/archives/generation.ts", + "packages/local-server/src/archives/manifest.ts", + "packages/local-server/src/archives/paths.ts", + "packages/local-server/src/archives/retention.ts", + "packages/local-server/src/checkpoints.ts", + "packages/local-server/src/eventing/control-store.ts", + "packages/local-server/src/identity-schema.ts", + "packages/local-server/src/service-map-rollup.ts", + "packages/local-server/src/store-version.ts" + ], + "rules": { "maple/no-date-parse": "off" } + }, // Test cases that cannot catch a bug: assertion-free, existence-only, constant pins, // Tailwind class pins, source greps, exact command lists and SQL projection pins. // Rationale per rule in scripts/oxlint-plugins/test-slop.mjs. @@ -707,6 +763,24 @@ } ] } + }, + // @maple/local-server is bundled into @maple-dev/alchemy's sidecar; the CLI, its + // migrations and its historical schema snapshots must never be reachable from it. + { + "files": ["packages/local-server/src/**/*.ts"], + "rules": { + "no-restricted-imports": [ + "error", + { + "patterns": [ + { + "regex": "^@maple/cli($|/)|/apps/cli/", + "message": "@maple/local-server must not depend on the CLI; the CLI depends on it." + } + ] + } + ] + } } ], // Vendored third-party source is read as a unit and re-pulled wholesale from its upstream, diff --git a/apps/cli/bench/ingest-chdb.ts b/apps/cli/bench/ingest-chdb.ts index a2d13f4cd1..16d428f2ba 100644 --- a/apps/cli/bench/ingest-chdb.ts +++ b/apps/cli/bench/ingest-chdb.ts @@ -16,10 +16,10 @@ import { mkdtempSync, rmSync } from "node:fs" import { tmpdir } from "node:os" import { join } from "node:path" -import { Chdb, writeIngestConfig } from "../src/server/chdb" -import { buildInsertStatements } from "../src/server/inserts" -import { type EncodedBatch, encodeLogs, encodeMetrics, encodeTraces } from "../src/server/otlp/encode" -import { LOCAL_SCHEMA_SQL } from "../src/server/schema-identity" +import { Chdb, writeIngestConfig } from "@maple/local-server/chdb" +import { buildInsertStatements } from "@maple/local-server/inserts" +import { type EncodedBatch, encodeLogs, encodeMetrics, encodeTraces } from "@maple/local-server/otlp/encode" +import { LOCAL_SCHEMA_SQL } from "@maple/local-server/schema-identity" import { type Signal, exportBody } from "./payloads" import { environment, flag, list, numbers, summarize, type Summary, writeReport } from "./report" diff --git a/apps/cli/package.json b/apps/cli/package.json index 130dbd970f..8d4a95daed 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -10,7 +10,7 @@ "start": "bun run src/bin.ts", "bench:ingest": "bun bench/ingest-http.ts", "bench:ingest:chdb": "bun bench/ingest-chdb.ts", - "test": "bun test --max-concurrency=1 test/*.test.ts src/server/otlp/encode.test.ts src/core/*.test.ts", + "test": "bun test --max-concurrency=1 test/*.test.ts src/core/*.test.ts", "typecheck": "tsrs --noEmit" }, "dependencies": { @@ -18,10 +18,9 @@ "@maple-dev/effect-sdk": "workspace:*", "@maple/domain": "workspace:*", "@maple/eventing-core": "workspace:*", - "@maple/ingest-rules": "workspace:*", + "@maple/local-server": "workspace:*", "@maple/query-engine": "workspace:*", - "effect": "catalog:effect", - "protobufjs": "^8.8.1" + "effect": "catalog:effect" }, "devDependencies": { "@effect/vitest": "catalog:effect", diff --git a/apps/cli/src/bin.ts b/apps/cli/src/bin.ts old mode 100644 new mode 100755 index 35821bf307..2dda38f111 --- a/apps/cli/src/bin.ts +++ b/apps/cli/src/bin.ts @@ -20,11 +20,11 @@ import { maybeNotifyUpdate } from "./core/update" import { maybeNotifyWhatsNew } from "./core/changelog" import { LocalAccessHttpClient } from "./core/local-access" import { WarehouseExecutorFromMode } from "./core/warehouse" -import { debugEnabled } from "./lib/debug" +import { debugEnabled } from "@maple/local-server/debug-log" import { makeHelpCapture, usageHint } from "./lib/help" -import { archiveErrorMessage } from "./server/archives/errors" -import { CHECKPOINT_REOPEN_PROBE_ENV, validateCheckpointDataDir } from "./server/checkpoints" -import { MAPLE_VERSION } from "./version" +import { archiveErrorMessage } from "@maple/local-server/archives/errors" +import { CHECKPOINT_REOPEN_PROBE_ENV, validateCheckpointDataDir } from "@maple/local-server/checkpoints" +import { MAPLE_VERSION } from "@maple/local-server/version" // WarehouseExecutorFromMode needs Mode (which needs MapleConfig). provideMerge // keeps Mode + MapleConfig in the output context too, so the login/logout/whoami diff --git a/apps/cli/src/commands/archive.ts b/apps/cli/src/commands/archive.ts index 06d15fcb7f..1ca378ffb5 100644 --- a/apps/cli/src/commands/archive.ts +++ b/apps/cli/src/commands/archive.ts @@ -5,21 +5,25 @@ import * as Argument from "effect/cli/Argument" import { randomUUID } from "node:crypto" import { homedir } from "node:os" import { join, resolve } from "node:path" -import { createArchiveGeneration, runArchiveReconciliation } from "../server/archives/generation" +import { createArchiveGeneration, runArchiveReconciliation } from "@maple/local-server/archives/generation" import { listActiveGenerations, activeParquetPaths, rebuildCatalogWithMaintenanceLock, verifyActiveGenerations, -} from "../server/archives/listing" -import { runArchiveGc } from "../server/archives/gc" -import { resolveArchiveTuning, tuningRecord } from "../server/archives/config" -import { ARCHIVE_SIGNALS, isArchiveSignalName, type ArchiveSignalName } from "../server/archives/signals" -import { expireArchiveDay, readRetiredDayLedger, retireLiveDay } from "../server/archives/retention" -import { validateRangeDate } from "../server/archives/paths" -import { maintenanceOperation } from "../server/checkpoints" +} from "@maple/local-server/archives/listing" +import { runArchiveGc } from "@maple/local-server/archives/gc" +import { resolveArchiveTuning, tuningRecord } from "@maple/local-server/archives/config" +import { + ARCHIVE_SIGNALS, + isArchiveSignalName, + type ArchiveSignalName, +} from "@maple/local-server/archives/signals" +import { expireArchiveDay, readRetiredDayLedger, retireLiveDay } from "@maple/local-server/archives/retention" +import { validateRangeDate } from "@maple/local-server/archives/paths" +import { maintenanceOperation } from "@maple/local-server/checkpoints" import { amber, bold, dim, green, red } from "../lib/style" -import { ArchiveError } from "../server/archives/errors" +import { ArchiveError } from "@maple/local-server/archives/errors" import { debugLog } from "../lib/debug" import { jsonFormatRequested, writeJson } from "./json-output" diff --git a/apps/cli/src/commands/changelog.ts b/apps/cli/src/commands/changelog.ts index b9483668d8..e3dc7fddfd 100644 --- a/apps/cli/src/commands/changelog.ts +++ b/apps/cli/src/commands/changelog.ts @@ -6,7 +6,7 @@ import { stripV } from "../core/update" import { CliUsageError } from "../lib/errors" import { chosenFormat, printJson } from "../lib/output" import { bold } from "../lib/style" -import { MAPLE_VERSION } from "../version" +import { MAPLE_VERSION } from "@maple/local-server/version" const VERSION_RE = /^v?\d+\.\d+\.\d+$/ diff --git a/apps/cli/src/commands/delete.ts b/apps/cli/src/commands/delete.ts index 4fdd9a7893..374b952329 100644 --- a/apps/cli/src/commands/delete.ts +++ b/apps/cli/src/commands/delete.ts @@ -7,12 +7,13 @@ import { Mode } from "../core/mode" import { parseTimestampMs, sinceToMs } from "../core/time" import { CliUsageError } from "../lib/errors" import { bold, dim, green } from "../lib/style" -import { isLoopbackHostname } from "../lib/local-address" -import { maintenanceTokenPath } from "../server/archives/retention" -import { hasSubject, ScopedDeleteReport, ScopedDeleteRequest } from "../server/scoped-delete" +import { isLoopbackHostname } from "@maple/local-server/local-address" +import { maintenanceTokenPath } from "@maple/local-server/archives/retention" +import { hasSubject, ScopedDeleteReport, ScopedDeleteRequest } from "@maple/local-server/scoped-delete" import { jsonFormatRequested, writeJson } from "./json-output" -import { serverDiscoveryPath } from "../server/store-version" -import { isProcessAlive, LocalStatus, prettyPath, ServerDiscovery } from "./server-args" +import { serverDiscoveryPath } from "@maple/local-server/store-version" +import { isProcessAlive } from "@maple/local-server/store-claim" +import { LocalStatus, prettyPath, ServerDiscovery } from "./server-args" const LOCAL_ONLY = "maple delete removes data from a local store only" diff --git a/apps/cli/src/commands/in-memory-store.ts b/apps/cli/src/commands/in-memory-store.ts index db54f21a8f..d88c6e9472 100644 --- a/apps/cli/src/commands/in-memory-store.ts +++ b/apps/cli/src/commands/in-memory-store.ts @@ -2,9 +2,9 @@ import { Clock, Effect, Option } from "effect" import { mkdtempSync, readdirSync, readFileSync, rmSync, statfsSync, statSync } from "node:fs" import { tmpdir } from "node:os" import { join } from "node:path" -import { SERVER_PID_NAME } from "../server/store-version" -import { ServerStateFileError } from "./server-errors" -import { isProcessAlive } from "./server-args" +import { SERVER_PID_NAME } from "@maple/local-server/store-version" +import { isProcessAlive } from "@maple/local-server/store-claim" +import { ServerStateFileError } from "@maple/local-server/store-errors" /** * `maple start --in-memory`: a throwaway store for one run. diff --git a/apps/cli/src/commands/schema.ts b/apps/cli/src/commands/schema.ts index 703e6bacd4..6e6e744241 100644 --- a/apps/cli/src/commands/schema.ts +++ b/apps/cli/src/commands/schema.ts @@ -14,14 +14,14 @@ import { runLocalStoreMigration, type MigrationPlan, } from "../server/local-store-migrations" -import { maintenanceOperation, releasePreservedCheckpoints } from "../server/checkpoints" +import { maintenanceOperation, releasePreservedCheckpoints } from "@maple/local-server/checkpoints" import { listRetainedLeftovers, pruneRetainedLeftovers, type RetainedInventory, type RetainedLeftover, } from "../server/retained-leftovers" -import { isStoreDirty, readMarker } from "../server/store-version" +import { isStoreDirty, readMarker } from "@maple/local-server/store-version" import { jsonFormatRequested, writeJson } from "./json-output" import { commandScope, mapleCommand } from "./server-args" diff --git a/apps/cli/src/commands/server-args.ts b/apps/cli/src/commands/server-args.ts index 4e0de0609b..7b82e81f1f 100644 --- a/apps/cli/src/commands/server-args.ts +++ b/apps/cli/src/commands/server-args.ts @@ -1,7 +1,7 @@ import { Schema } from "effect" import { homedir } from "node:os" import { join, resolve, sep } from "node:path" -import { DEFAULT_LOCAL_PORT, normalizeHost, resolveBindHost } from "../lib/local-address" +import { DEFAULT_LOCAL_PORT, normalizeHost, resolveBindHost } from "@maple/local-server/local-address" export type DirtyStorePolicy = "wipe" | "fail" | "restore-checkpoint" @@ -16,7 +16,7 @@ export { serverProbeUrl, serverUrl, validateHost, -} from "../lib/local-address" +} from "@maple/local-server/local-address" export const defaultDataDir = (): string => join(homedir(), ".maple", "data") @@ -113,23 +113,6 @@ export const ServerDiscovery = Schema.Struct({ }) export type ServerDiscovery = typeof ServerDiscovery.Type -/** Parse `ps -o etime=` (`[[dd-]hh:]mm:ss`) into whole seconds. */ -export const parseElapsedSeconds = (raw: string): number | undefined => { - const match = raw.trim().match(/^(?:(\d+)-)?(?:(\d+):)?(\d+):(\d+)$/) - if (!match) return undefined - const [, days, hours, minutes, seconds] = match - return Number(days ?? 0) * 86_400 + Number(hours ?? 0) * 3_600 + Number(minutes) * 60 + Number(seconds) -} - -/** - * Could a live process that started at `startedAtMs` have written a PID file - * last modified at `writtenAtMs`? A reused PID belongs to a process started - * after the writer died, so after the write. The slack covers `etime`'s - * whole-second resolution. - */ -export const startedBeforeWrite = (startedAtMs: number, writtenAtMs: number): boolean => - startedAtMs <= writtenAtMs + 2_000 - /** Does telemetry ingested since `sinceMs` need a fresh checkpoint? Unknown * ingest state (`undefined`) always does. */ export const ingestedSince = (lastIngestAtMs: number | null | undefined, sinceMs: number): boolean => @@ -215,24 +198,3 @@ export const buildCheckpointChildArgs = (options: { "--data-dir", options.dataDir, ] - -/** Liveness probe via signal 0: a process primitive with no FileSystem - * equivalent. Never throws (errors mean "not alive"). - * - * A PID file naming *this* process is stale by definition: the previous server - * is the one that wrote it, so if its number is ours it has died and the OS - * reused the PID. In a container that is the common case, not the edge case: - * `maple start` is PID 1 on every restart, the file survives on the data - * volume, and `kill(1, 0)` always succeeds, so without this check a container - * whose server died once refuses to start forever ("already running (PID 1)"). - * Non-positive PIDs are rejected for the same reason: `kill(0, 0)` and - * `kill(-n, 0)` signal process groups and "succeed" for a group that exists. */ -export const isProcessAlive = (pid: number): boolean => { - if (!Number.isInteger(pid) || pid <= 0 || pid === process.pid) return false - try { - process.kill(pid, 0) - return true - } catch { - return false - } -} diff --git a/apps/cli/src/commands/server-errors.ts b/apps/cli/src/commands/server-errors.ts index 5c57e80282..5182b2bd2f 100644 --- a/apps/cli/src/commands/server-errors.ts +++ b/apps/cli/src/commands/server-errors.ts @@ -32,68 +32,6 @@ export class ServerOptionError extends Schema.TaggedError()( }, ) {} -/** The dirty-store policy in force when the store turned out to be unclean. */ -export const DirtyStorePolicy = Schema.Literals(["fail", "restore-checkpoint", "reset"]) - -/** - * The store was left open by a server that died without closing chDB. Reopening - * it can crash chDB natively, so startup refuses. - * - * `checkpointAvailable` is the field worth grouping on: without a checkpoint - * there is nothing to restore and the only way forward is `--reset`, which is a - * materially different user experience from the recoverable case. - */ -export class LocalStoreDirtyError extends Schema.TaggedError()( - "@maple/cli/LocalStoreDirtyError", - { - dataDir: Schema.String, - policy: DirtyStorePolicy, - checkpointAvailable: Schema.Boolean, - message: Schema.String, - }, -) {} - -/** The store was written by a different chDB build; loading it would SIGTRAP. */ -export class LocalStoreIncompatibleError extends Schema.TaggedError()( - "@maple/cli/LocalStoreIncompatibleError", - { - dataDir: Schema.String, - storeBuild: Schema.String, - currentBuild: Schema.String, - message: Schema.String, - }, -) {} - -/** The store was bootstrapped from an older bundled schema and needs migrating. */ -export class LocalStoreSchemaStaleError extends Schema.TaggedError()( - "@maple/cli/LocalStoreSchemaStaleError", - { dataDir: Schema.String, message: Schema.String }, -) {} - -/** - * The store's marker records a schema version newer than this build knows: a - * downgrade. Distinct from a stale schema because the remedy is reinstalling - * the newer maple, never a reset. - */ -export class LocalStoreFromNewerMapleError extends Schema.TaggedError()( - "@maple/cli/LocalStoreFromNewerMapleError", - { - dataDir: Schema.String, - storeSchemaVersion: Schema.Number, - currentSchemaVersion: Schema.Number, - message: Schema.String, - }, -) {} - -/** Which part of the local-store migration journal the command was doing. */ -export const MigrationPhase = Schema.Literals(["read-journal", "resume", "preserve"]) - -/** An unfinished schema migration blocks startup, or its journal is unreadable. */ -export class LocalStoreMigrationError extends Schema.TaggedError()( - "@maple/cli/LocalStoreMigrationError", - { dataDir: Schema.String, phase: MigrationPhase, message: Schema.String }, -) {} - /** Why no checkpoint could be selected: none was ever taken, another build took it, or the registry is unusable. */ export const CheckpointUnavailableReason = Schema.Literals(["none", "other_build", "unusable"]) @@ -123,12 +61,6 @@ export class BackgroundServerTimeoutError extends Schema.TaggedError()( - "@maple/cli/ServerStateFileError", - { path: Schema.String, message: Schema.String }, -) {} - /** The detached child exited before it was ready; its own error is in the log. */ export class BackgroundServerExitedError extends Schema.TaggedError()( "@maple/cli/BackgroundServerExitedError", diff --git a/apps/cli/src/commands/server.ts b/apps/cli/src/commands/server.ts index a001ad72bc..e06e6c6bd6 100644 --- a/apps/cli/src/commands/server.ts +++ b/apps/cli/src/commands/server.ts @@ -3,25 +3,19 @@ import { FileSystem } from "effect/FileSystem" import * as Command from "effect/cli/Command" import * as Flag from "effect/cli/Flag" import { HttpClient, HttpClientResponse } from "effect/http" -import { randomUUID } from "node:crypto" -import { - closeSync, - linkSync, - mkdirSync, - openSync, - readFileSync, - renameSync, - statSync, - unlinkSync, - writeFileSync, -} from "node:fs" +import { closeSync, openSync, readFileSync, renameSync, statSync, unlinkSync } from "node:fs" import { dirname, resolve } from "node:path" -import { ServerBindError, startServer } from "../server/serve" +import { ensurePortFree, ServerBindError, startServer } from "@maple/local-server/serve" +import { TelemetryLayer } from "../core/telemetry" import { loadProjectorModules, resolveProjectorModulePaths } from "../server/eventing/projector-modules" -import { ACCESS_TOKEN_ENV, INGEST_KEY_ENV, type RemoteAuth, remoteAuthFromEnv } from "../server/remote-auth" -import { CURRENT_LOCAL_SCHEMA } from "../server/schema-identity" import { - adoptLegacySidecars, + ACCESS_TOKEN_ENV, + INGEST_KEY_ENV, + type RemoteAuth, + remoteAuthFromEnv, +} from "@maple/local-server/remote-auth" +import { CURRENT_LOCAL_SCHEMA } from "@maple/local-server/schema-identity" +import { checkStoreCompatible, dataDirSidecarPath, isSchemaIdentityStale, @@ -34,9 +28,33 @@ import { serverPidPath, storeMarkerPath, UNVERSIONED_STORE, -} from "../server/store-version" -import { durableWrite } from "../server/durable-files" +} from "@maple/local-server/store-version" +import { durableWrite } from "@maple/local-server/durable-files" import { abandonLocalStoreMigration, localMigrationIsIncomplete } from "../server/local-store-migrations" +import { + adoptLegacyStoreFiles, + claimPidFileExclusive, + ignoreFailure, + isProcessAlive, + locatePidFile, + pidOwner, + readPidSnapshot, + refuseIfRunning, + releasePidFile, + sameSnapshot, + STALE_PID, +} from "@maple/local-server/store-claim" +import { + describeError, + LocalStoreDirtyError, + LocalStoreFromNewerMapleError, + LocalStoreIncompatibleError, + LocalStoreMigrationError, + LocalStoreSchemaStaleError, + ServerStateError, + ServerStateFileError, + stateFileError, +} from "@maple/local-server/store-errors" import { type CheckpointAvailability, checkpointAvailability, @@ -46,11 +64,10 @@ import { parseCheckpointId, reconcileCheckpointRecovery, resetLiveStorePreservingCheckpoints, + resolveChdbConfigFile, restoreCheckpoint, - writeBackupConfig, -} from "../server/checkpoints" -import { writeIngestConfig } from "../server/chdb" -import { resolveUiAssets } from "../server/ui-assets" +} from "@maple/local-server/checkpoints" +import { resolveUiAssets } from "@maple/local-server/ui-assets" import { clearStartFailures, guardStartFailures } from "./start-failures" import { acquireInMemoryStore, @@ -60,7 +77,7 @@ import { inMemoryStartFailuresKey, } from "./in-memory-store" import { debugLog } from "../lib/debug" -import { MAPLE_VERSION } from "../version" +import { MAPLE_VERSION } from "@maple/local-server/version" import { jsonFormatRequested, writeJson } from "./json-output" import { BackgroundServerExitedError, @@ -68,13 +85,7 @@ import { BackgroundServerTimeoutError, CheckpointChildError, CheckpointUnavailableError, - LocalStoreDirtyError, - LocalStoreFromNewerMapleError, - LocalStoreIncompatibleError, - LocalStoreMigrationError, - LocalStoreSchemaStaleError, ServerOptionError, - ServerStateFileError, ServerStopTimeoutError, } from "./server-errors" import { amber, bold, cyan, dim, green, MARK_LINES, MARK_WIDTH, underline } from "../lib/style" @@ -90,10 +101,8 @@ import { hostedDashboardUrl, hostedUiOrigin, storeChangedSince, - isProcessAlive, LocalStatus, mapleCommand, - parseElapsedSeconds, PID_HANDOVER_ENV, prettyPath, resolveAdvertiseHost, @@ -101,39 +110,13 @@ import { ServerDiscovery, serverProbeUrl, serverUrl, - startedBeforeWrite, validateHost, } from "./server-args" -import { DEFAULT_LOCAL_PORT, isLoopbackHostname } from "../lib/local-address" - -/** - * A refused command whose precondition simply wasn't met: the server is already - * running, isn't running at all, or a destructive command was not confirmed. - * The message and the non-zero exit are identical to a genuine failure; the - * separate tag exists so `bin.ts` can close the root span `Ok` for these without - * also swallowing real start failures (`ServerBindError`, - * `BackgroundServerTimeoutError`, `LocalStoreDirtyError`). Same rule the ingest - * gateway follows for expected 4xx. - */ -export class ServerStateError extends Schema.TaggedError()("@maple/cli/ServerStateError", { - message: Schema.String, -}) {} - -/** Public origin of the deployed local-mode dashboard SPA. Overridable - * (`MAPLE_LOCAL_UI_URL`) for testing against another build of it. */ -const DEFAULT_REMOTE_UI_URL = "https://local.maple.dev" - -const describeError = (error: unknown): string => (error instanceof Error ? error.message : String(error)) - -const errnoCode = (error: unknown): string | undefined => - typeof error === "object" && error !== null && "code" in error && typeof error.code === "string" - ? error.code - : undefined - -const stateFileError = - (path: string, action: string) => - (error: unknown): ServerStateFileError => - new ServerStateFileError({ path, message: `could not ${action} ${path}: ${describeError(error)}` }) +import { + DEFAULT_LOCAL_PORT, + DEFAULT_REMOTE_UI_URL, + isLoopbackHostname, +} from "@maple/local-server/local-address" /** An unset or empty env var reads as `undefined`; never fails. */ const optionalEnvString = (key: string): Effect.Effect => @@ -247,141 +230,6 @@ const startBanner = ( return `\n${lines.join("\n")}\n\n` } -// --------------------------------------------------------------------------- -// PID file: the one claim that serializes `start`, `reset` and `restore`. -// --------------------------------------------------------------------------- - -interface PidFileSnapshot { - readonly raw: string - readonly pid: number | undefined - readonly ino: number - readonly mtimeMs: number -} - -const readPidSnapshot = (path: string): Effect.Effect> => - Effect.try({ - try: (): PidFileSnapshot => { - const raw = readFileSync(path, "utf8") - const info = statSync(path) - const pid = Number.parseInt(raw.trim(), 10) - return { - raw, - pid: Number.isInteger(pid) && pid > 0 ? pid : undefined, - ino: info.ino, - mtimeMs: info.mtimeMs, - } - }, - catch: (error) => error, - }).pipe(Effect.option) - -const sameSnapshot = (a: PidFileSnapshot, b: PidFileSnapshot): boolean => - a.raw === b.raw && a.ino === b.ino && a.mtimeMs === b.mtimeMs - -/** Wall-clock start of a live process (`ps -o etime=`), when it can be known. */ -const processStartedAtMs = (pid: number): Effect.Effect> => - Effect.try({ - try: () => - Bun.spawnSync(["ps", "-o", "etime=", "-p", String(pid)], { stdout: "pipe", stderr: "ignore" }), - catch: (error) => error, - }).pipe( - Effect.flatMap((result) => { - const seconds = result.success ? parseElapsedSeconds(result.stdout.toString()) : undefined - return seconds === undefined - ? Effect.succeed(Option.none()) - : Effect.map(Clock.currentTimeMillis, (nowMs) => Option.some(nowMs - seconds * 1000)) - }), - Effect.orElseSucceed(() => Option.none()), - ) - -type PidOwner = - | { readonly kind: "stale" } - /** `verified`: the process provably started before the file was written. */ - | { readonly kind: "live"; readonly pid: number; readonly verified: boolean } - -const STALE_PID: PidOwner = { kind: "stale" } -const livePid = (pid: number, verified: boolean): PidOwner => ({ kind: "live", pid, verified }) - -/** - * Does the process named by a PID file still own it? After a crash or reboot - * the number may belong to an unrelated process, which necessarily started - * after the file was written. Without `ps` the answer is "live, unverified", - * the conservative reading for every caller. - */ -const pidOwner = (snapshot: PidFileSnapshot): Effect.Effect => - Effect.gen(function* () { - const pid = snapshot.pid - if (pid === undefined || !isProcessAlive(pid)) return STALE_PID - const startedAt = yield* processStartedAtMs(pid) - if (Option.isNone(startedAt)) return livePid(pid, false) - return startedBeforeWrite(startedAt.value, snapshot.mtimeMs) ? livePid(pid, true) : STALE_PID - }) - -/** O_EXCL create; `false` when the file already exists. */ -const createExclusive = (path: string, content: string): Effect.Effect => - Effect.try({ - try: () => { - writeFileSync(path, content, { mode: 0o600, flag: "wx" }) - return true - }, - catch: (error) => error, - }).pipe( - Effect.catch((error) => - errnoCode(error) === "EEXIST" - ? Effect.succeed(false) - : Effect.fail(stateFileError(path, "create")(error)), - ), - ) - -/** Atomically replace the file's content (write aside, rename over). */ -const replaceFile = (path: string, content: string): Effect.Effect => - Effect.try({ - try: () => { - const temporary = `${path}.${randomUUID()}.tmp` - writeFileSync(temporary, content, { mode: 0o600, flag: "wx" }) - renameSync(temporary, path) - }, - catch: stateFileError(path, "replace"), - }) - -const ignoreFailure = (thunk: () => void): Effect.Effect => - Effect.try({ try: thunk, catch: (error) => error }).pipe(Effect.ignore) - -/** - * Remove a stale PID file without ever deleting a claim that a concurrent - * start published after we read it: move it aside, and only discard it if it - * is byte-for-byte the file we judged stale; otherwise put it back and yield. - */ -const takeOverStalePidFile = ( - path: string, - observed: PidFileSnapshot, - busy: (pid: number | undefined) => ServerStateError, -): Effect.Effect => - Effect.gen(function* () { - const moved = `${path}.stale-${randomUUID()}` - const renamed = yield* Effect.try({ - try: () => { - renameSync(path, moved) - return true - }, - catch: (error) => error, - }).pipe( - Effect.catch((error) => - errnoCode(error) === "ENOENT" - ? Effect.succeed(false) - : Effect.fail(stateFileError(path, "replace the stale PID file")(error)), - ), - ) - if (!renamed) return - const current = yield* readPidSnapshot(moved) - if (Option.isSome(current) && sameSnapshot(current.value, observed)) { - yield* ignoreFailure(() => unlinkSync(moved)) - return - } - yield* ignoreFailure(() => linkSync(moved, path)) - yield* ignoreFailure(() => unlinkSync(moved)) - return yield* busy(Option.isSome(current) ? current.value.pid : undefined) - }) - const alreadyRunning = (scope: CommandScope) => (pid: number | undefined): ServerStateError => @@ -400,87 +248,13 @@ const runningRefusal = `stop it first with \`${mapleCommand("stop", scope)}\``, }) -/** - * Exclusively claim the PID file so exactly one `start`, `reset` or `restore` - * touches the store. A stale file (dead or reused PID) is taken over safely. - * `handoverFrom` lets a `start -d` child inherit its parent's claim. - */ -export const claimPidFileExclusive = ( - pidPath: string, - options: { - readonly handoverFrom?: number | undefined - readonly busy?: (pid: number | undefined) => ServerStateError - } = {}, -): Effect.Effect => - Effect.gen(function* () { - const busy = options.busy ?? alreadyRunning({}) - yield* Effect.try({ - try: () => mkdirSync(dirname(pidPath), { recursive: true }), - catch: stateFileError(dirname(pidPath), "create"), - }) - const content = String(process.pid) - for (let attempt = 0; attempt < 4; attempt++) { - if (yield* createExclusive(pidPath, content)) return - const observed = yield* readPidSnapshot(pidPath) - if (Option.isNone(observed)) continue - const owner = yield* pidOwner(observed.value) - if (owner.kind === "live") { - if (owner.pid === options.handoverFrom) return yield* replaceFile(pidPath, content) - return yield* busy(owner.pid) - } - yield* takeOverStalePidFile(pidPath, observed.value, busy) - } - return yield* busy(undefined) - }) - -/** Remove the PID file only while it still names this process. */ -const releasePidFile = (pidPath: string): Effect.Effect => - Effect.gen(function* () { - const current = yield* readPidSnapshot(pidPath) - if (Option.isSome(current) && current.value.pid === process.pid) { - yield* ignoreFailure(() => unlinkSync(pidPath)) - } - }) - -/** The PID file of the server that owns `dataDir`: the current-layout one, or - * the one an older maple wrote for a custom `--data-dir` before upgrading. */ -const locatePidFile = ( - dataDir: string, -): Effect.Effect> => - Effect.gen(function* () { - const candidates = [ - serverPidPath(dataDir), - ...(ownsLegacySidecars(dataDir) ? [legacySidecarPath(dataDir, SERVER_PID_NAME)] : []), - ] - for (const path of candidates) { - const snapshot = yield* readPidSnapshot(path) - if (Option.isSome(snapshot)) return Option.some({ path, snapshot: snapshot.value }) - } - return Option.none() - }) - -/** Refuse when a live server owns `dataDir`, including one an older maple - * started from the legacy PID file. */ -const refuseIfRunning = ( - dataDir: string, - refusal: (pid: number | undefined) => ServerStateError, -): Effect.Effect => - Effect.gen(function* () { - const located = yield* locatePidFile(dataDir) - if (Option.isNone(located)) return - const owner = yield* pidOwner(located.value.snapshot) - if (owner.kind === "live") return yield* refusal(owner.pid) - }) - -/** Explicit adoption of pre-namespacing state files, surfaced when it fails. */ +/** Explicit adoption of pre-namespacing state files, logged under `--debug`. */ const adoptSidecars = (dataDir: string): Effect.Effect => - Effect.try({ - try: () => { - const adopted = adoptLegacySidecars(dataDir) + Effect.flatMap(adoptLegacyStoreFiles(dataDir), (adopted) => + Effect.sync(() => { if (adopted.length > 0) debugLog("adopted legacy store files", adopted.join(", ")) - }, - catch: stateFileError(dataDir, "adopt the legacy state files beside"), - }) + }), + ) // --------------------------------------------------------------------------- // Discovery file and status probe. @@ -672,45 +446,6 @@ const logFilePath = (dataDir: string): string => dataDirSidecarPath(dataDir, "ma /** `maple.log` is rotated to `maple.log.1` at a background start once it passes this. */ const LOG_ROTATE_BYTES = 10 * 1024 * 1024 -// Generated chDB config, beside the PID and log files (e.g. ~/.maple/chdb-config.xml). -export const chdbConfigPath = (dataDir: string): string => dataDirSidecarPath(dataDir, "chdb-config.xml") - -/** - * Resolve the chDB config file, generating a backups-enabled default when the - * user did not supply one. - * - * `BACKUP DATABASE default TO Disk('default', …)` (how every checkpoint is - * taken) needs `` in the config of the *running* chDB - * connection. chDB allows one connection per process, acquired once at start and - * held for the process lifetime, and `maple checkpoint` is a separate process - * talking over HTTP: it cannot inject config into a live connection. So a server - * started without a backups config can never checkpoint. A user-supplied - * `--chdb-config-file` is honoured untouched. An in-memory store is never - * checkpointed, so it gets only the MergeTree ingest defaults (`backups: false`). - */ -export const resolveChdbConfigFile = ( - dataDir: string, - supplied: string | undefined, - { backups }: { readonly backups: boolean } = { backups: true }, -) => - Effect.gen(function* () { - if (supplied !== undefined) return supplied - const fs = yield* FileSystem - const path = chdbConfigPath(dataDir) - // Regenerated every start: idempotent, and it self-heals a truncated or - // hand-edited file. Failing to write is not fatal: the server still starts, - // checkpoints just stay unavailable, which is the old behaviour. - yield* fs.makeDirectory(dirname(path), { recursive: true }).pipe(Effect.ignore) - return yield* Effect.try({ - try: () => { - if (backups) writeBackupConfig(path) - else writeIngestConfig(path) - return path - }, - catch: (error) => error, - }).pipe(Effect.orElseSucceed(() => undefined)) - }) - /** * What to tell someone whose local store was left dirty, given whether a * checkpoint is actually restorable. Every branch names at least one command @@ -980,22 +715,6 @@ const checkpointRefreshLoop = ( } }) -/** Fail fast when the port is taken. Not atomic (the real bind can still lose - * a race), but it keeps a busy port from costing a reset or a wipe. */ -const ensurePortFree = (hostname: string, port: number): Effect.Effect => - Effect.try({ - try: () => Bun.listen({ hostname, port, socket: { data() {} } }).stop(true), - catch: (error) => - new ServerBindError({ - hostname, - port, - message: - errnoCode(error) === "EADDRINUSE" - ? `port ${port} on ${hostname} is already in use; choose another with --port, or stop whatever holds it` - : `cannot bind ${hostname}:${port}: ${describeError(error)}`, - }), - }) - const fileSize = (path: string): number => Effect.runSync( Effect.try({ try: () => statSync(path).size, catch: (error) => error }).pipe( @@ -1571,6 +1290,7 @@ export const start = Command.make("start", { assets, auth, projectors, + telemetryLayer: TelemetryLayer, }) started = true yield* clearStartFailures(startFailuresKey) diff --git a/apps/cli/src/commands/start-failures.ts b/apps/cli/src/commands/start-failures.ts index b656208436..af73141f7e 100644 --- a/apps/cli/src/commands/start-failures.ts +++ b/apps/cli/src/commands/start-failures.ts @@ -4,7 +4,7 @@ import { dirname } from "node:path" import { annotateOutcome, EXIT_START_REFUSED, markFailedExit } from "../core/outcomes" import { describeFailure, formatFailure } from "../lib/failure" import { dim } from "../lib/style" -import { dataDirSidecarPath } from "../server/store-version" +import { dataDirSidecarPath } from "@maple/local-server/store-version" /** * Deterministic `maple start` refusals under a supervisor (launchd KeepAlive, diff --git a/apps/cli/src/commands/update.ts b/apps/cli/src/commands/update.ts index 7722d16a35..df6c954934 100644 --- a/apps/cli/src/commands/update.ts +++ b/apps/cli/src/commands/update.ts @@ -14,7 +14,7 @@ import { } from "../core/update" import { step, type StepHandle, type StepLabels } from "../lib/progress" import { amber, bold, cyan, dim, green } from "../lib/style" -import { MAPLE_VERSION } from "../version" +import { MAPLE_VERSION } from "@maple/local-server/version" // NB: this is named `--tag`, not `--version`: the CLI framework reserves a // global `--version` flag (prints the binary version and exits), so a diff --git a/apps/cli/src/core/changelog.ts b/apps/cli/src/core/changelog.ts index 4bf2783b6c..003ba1bf95 100644 --- a/apps/cli/src/core/changelog.ts +++ b/apps/cli/src/core/changelog.ts @@ -5,7 +5,7 @@ import { Effect, Option } from "effect" import changelogMarkdown from "../../CHANGELOG.md" with { type: "text" } import { bold, dim, green } from "../lib/style" -import { MAPLE_VERSION } from "../version" +import { MAPLE_VERSION } from "@maple/local-server/version" import { MapleConfig } from "./config" import { isNewer, shouldRunNotify, stripV } from "./update" diff --git a/apps/cli/src/core/config.ts b/apps/cli/src/core/config.ts index 3bb56c7775..6c729dcc66 100644 --- a/apps/cli/src/core/config.ts +++ b/apps/cli/src/core/config.ts @@ -3,8 +3,8 @@ import { FileSystem } from "effect/FileSystem" import { ChildProcessSpawner } from "effect/process/ChildProcessSpawner" import * as os from "node:os" import * as path from "node:path" -import { defaultLocalUrl } from "../lib/local-address" -import { durableWrite } from "../server/durable-files" +import { defaultLocalUrl } from "@maple/local-server/local-address" +import { durableWrite } from "@maple/local-server/durable-files" import { credentialAccount, deleteNativeCredential, diff --git a/apps/cli/src/core/local-access.ts b/apps/cli/src/core/local-access.ts index 650a3d6c0c..34f55b9cd4 100644 --- a/apps/cli/src/core/local-access.ts +++ b/apps/cli/src/core/local-access.ts @@ -3,8 +3,8 @@ import { Cause, Duration, Effect, Layer, Option, Result, Stream } from "effect" import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/http" import * as ChildProcess from "effect/process/ChildProcess" import { ChildProcessSpawner } from "effect/process/ChildProcessSpawner" -import { defaultLocalUrl, isLoopbackHostname, isRemoteHttps } from "../lib/local-address" -import { ACCESS_TOKEN_ENV } from "../server/remote-auth" +import { defaultLocalUrl, isLoopbackHostname, isRemoteHttps } from "@maple/local-server/local-address" +import { ACCESS_TOKEN_ENV } from "@maple/local-server/remote-auth" const bare = (hostname: string): string => hostname.replace(/^\[(.*)\]$/, "$1").toLowerCase() diff --git a/apps/cli/src/core/telemetry.ts b/apps/cli/src/core/telemetry.ts index 359e38c63f..450a872467 100644 --- a/apps/cli/src/core/telemetry.ts +++ b/apps/cli/src/core/telemetry.ts @@ -2,7 +2,7 @@ import { getAutoPlatformAttributes } from "@maple-dev/effect-sdk/server" import { Config, Effect, Layer, Option, Redacted } from "effect" import { FetchHttpClient, HttpBody } from "effect/http" import { OtlpMetrics, OtlpResource, OtlpSerialization, OtlpTracer } from "effect/observability" -import { MAPLE_VERSION } from "../version" +import { MAPLE_VERSION } from "@maple/local-server/version" // Publishable ("pk") ingest token, baked into the distributed binary so CLI // telemetry works on a fresh install with zero config. The `maple_pk_` class is diff --git a/apps/cli/src/core/update.ts b/apps/cli/src/core/update.ts index 9a7f78cb30..61f866cb7c 100644 --- a/apps/cli/src/core/update.ts +++ b/apps/cli/src/core/update.ts @@ -23,7 +23,7 @@ import { randomUUID } from "node:crypto" import { realpathSync } from "node:fs" import { dirname, join } from "node:path" import { amber, bold, dim, green } from "../lib/style" -import { MAPLE_VERSION } from "../version" +import { MAPLE_VERSION } from "@maple/local-server/version" import { MapleConfig } from "./config" import { releaseChecksum, releaseVerificationFailure, type ReleaseSignatureStatus } from "./release-signature" diff --git a/apps/cli/src/lib/debug.ts b/apps/cli/src/lib/debug.ts index 80584a8746..0d1fdf9c28 100644 --- a/apps/cli/src/lib/debug.ts +++ b/apps/cli/src/lib/debug.ts @@ -3,12 +3,9 @@ // the mode resolver (core/mode.ts), the executor that emits these lines is built // outside the parsed-flag context, so it scans argv rather than threading a flag. +import { debugEnabled } from "@maple/local-server/debug-log" import { dim, gray } from "./style" -export const debugEnabled = (): boolean => - (typeof process !== "undefined" && Array.isArray(process.argv) && process.argv.includes("--debug")) || - process.env.MAPLE_DEBUG === "1" - /** Emit a debug line to stderr (no-op unless debug is enabled). */ export const debugLog = (label: string, detail?: string): void => { if (!debugEnabled()) return diff --git a/apps/cli/src/lib/failure.ts b/apps/cli/src/lib/failure.ts index b0e1e8468c..254018188b 100644 --- a/apps/cli/src/lib/failure.ts +++ b/apps/cli/src/lib/failure.ts @@ -3,8 +3,8 @@ // `--debug`. Pure so the rules are testable (bin.ts runs at import time). import { Predicate } from "effect" -import { SIGN_IN_REQUIRED } from "../server/remote-auth" -import { isRemoteHttps } from "./local-address" +import { SIGN_IN_REQUIRED } from "@maple/local-server/remote-auth" +import { isRemoteHttps } from "@maple/local-server/local-address" export interface FailureReport { /** The failure's `_tag`, recorded as `maple.cli.outcome` on the root span. */ diff --git a/apps/cli/src/server/local-store-migration-module.ts b/apps/cli/src/server/local-store-migration-module.ts index 76ae4154b4..ef7c517a6d 100644 --- a/apps/cli/src/server/local-store-migration-module.ts +++ b/apps/cli/src/server/local-store-migration-module.ts @@ -1,7 +1,10 @@ // BOUNDARY: This module owns unparsed external values and narrows them before domain use. -import type { Chdb } from "./chdb" -import type { LocalSchemaIdentity } from "./schema-identity" -import type { MigrationPhase, MigrationStepJournalSchema } from "./local-store-migrations/journal-schema" +import type { Chdb } from "@maple/local-server/chdb" +import type { LocalSchemaIdentity } from "@maple/local-server/schema-identity" +import type { + MigrationPhase, + MigrationStepJournalSchema, +} from "@maple/local-server/local-store-migrations/journal-schema" /** Coordinator-owned transaction phases. Modules may report progress, but * only the coordinator advances this top-level state machine. The union is diff --git a/apps/cli/src/server/local-store-migrations.ts b/apps/cli/src/server/local-store-migrations.ts index 0f037ba725..8167eae4bf 100644 --- a/apps/cli/src/server/local-store-migrations.ts +++ b/apps/cli/src/server/local-store-migrations.ts @@ -10,8 +10,8 @@ import { existsSync, readFileSync, statfsSync } from "node:fs" import { lstat, readdir, readFile, stat } from "node:fs/promises" import { dirname, join, relative, resolve } from "node:path" import { Effect, Schema } from "effect" -import { Chdb } from "./chdb" -import { decodeJsonEachRow } from "./chdb-rows" +import { Chdb } from "@maple/local-server/chdb" +import { decodeJsonEachRow } from "@maple/local-server/chdb-rows" import { CURRENT_LOCAL_SCHEMA, LEGACY_LOCAL_SCHEMA, @@ -19,7 +19,7 @@ import { LOCAL_SCHEMA_SQL, identityLabel, type LocalSchemaIdentity, -} from "./schema-identity" +} from "@maple/local-server/schema-identity" import { checkStoreCompatible, adoptLegacySidecars, @@ -38,10 +38,24 @@ import { storeOpenMarkerPath, type StoreMarker, type StoreMarkerV2, -} from "./store-version" -import { durableJson, durableRename, ensurePrivateDirectory, syncTree } from "./durable-files" -import { MAPLE_VERSION } from "../version" -import { decodeMigrationJournal, type MigrationJournalSchema } from "./local-store-migrations/journal-schema" +} from "@maple/local-server/store-version" +import { + durableJson, + durableRename, + ensurePrivateDirectory, + syncTree, +} from "@maple/local-server/durable-files" +import { MAPLE_VERSION } from "@maple/local-server/version" +import { + assertJournalPaths, + isMigrationIncomplete, + type MigrationJournal, + migrationHistoryPath, + migrationJournalPath, + migrationRootPath, + readMigrationJournalStructure, + safeMigrationPath, +} from "@maple/local-server/local-store-migration-journal" import { legacyToCurrentModule } from "./local-store-migrations/legacy-to-current" import { stepModule } from "./local-store-migrations/step-executor" import { LOCAL_STORE_STEPS } from "./local-store-migrations/steps" @@ -64,18 +78,6 @@ export { export { legacyToCurrentModule } from "./local-store-migrations/legacy-to-current" -const NONTERMINAL_PHASES = new Set([ - "planned", - "preflight-complete", - "target-created", - "copying", - "copy-verified", - "promotion-started", - "failed", -]) - -export const isMigrationIncomplete = (phase: MigrationPhase): boolean => NONTERMINAL_PHASES.has(phase) - export interface MigrationPlan { readonly source: LocalSchemaIdentity readonly target: LocalSchemaIdentity @@ -87,8 +89,6 @@ export interface MigrationPlan { readonly checkpointDisposition: string } -export type MigrationJournal = typeof MigrationJournalSchema.Type - export interface MigrationResult { readonly migrationId: string readonly phase: MigrationPhase @@ -292,162 +292,6 @@ export const formatMigrationPlan = (plan: MigrationPlan): string => { return `${lines.join("\n")}\n` } -export const migrationJournalPath = (dataDir: string): string => - dataDirSidecarPath(dataDir, "maple-store-migration.json") - -export const migrationRootPath = (dataDir: string, migrationId: string): string => - join(dirname(resolve(dataDir)), ".maple-migrations", migrationId) - -export const migrationHistoryPath = (dataDir: string, migrationId: string): string => - join(migrationRootPath(dataDir, migrationId), "journal.json") - -const safeMigrationPath = (path: string, root: string, label: string): string => { - const absolute = resolve(path) - const relativePath = relative(resolve(root), absolute) - if ( - relativePath === "" || - relativePath === ".." || - relativePath.startsWith("../") || - resolve(root) === absolute - ) { - throw new Error(`${label} escapes migration root`) - } - return absolute -} - -const assertJournalPaths = (dataDir: string, journal: MigrationJournal): void => { - const source = resolve(dataDir) - if (resolve(journal.sourceDataDir) !== source) - throw new Error("local migration journal source does not match the configured data directory") - const root = migrationRootPath(dataDir, journal.migrationId) - safeMigrationPath(journal.targetDataDir, root, "migration target") - if (resolve(dirname(journal.targetDataDir)) !== resolve(join(root, "target"))) - throw new Error("local migration journal target is outside its owned target directory") - if ( - journal.chain.length === 0 || - journal.currentStepIndex < 0 || - journal.currentStepIndex > journal.chain.length - ) - throw new Error("local migration journal chain state is invalid") - const final = journal.chain[journal.chain.length - 1]! - if ( - final.to.version !== journal.targetVersion || - final.to.fingerprint !== journal.targetFingerprint || - final.to.digest !== journal.targetDigest - ) - throw new Error("local migration journal final target does not match its chain") -} - -const sameJournalIdentity = (a: LocalSchemaIdentity, b: LocalSchemaIdentity): boolean => - a.version === b.version && a.fingerprint === b.fingerprint && a.digest === b.digest && a.chdb === b.chdb - -const assertJournalChainInvariants = (journal: MigrationJournal): void => { - const { chain, currentStepIndex: current } = journal - if (current < 0 || current > chain.length) - throw new Error("migration journal currentStepIndex is invalid") - if ( - !sameJournalIdentity(chain[0]!.from, { - version: journal.sourceVersion, - fingerprint: journal.sourceFingerprint, - digest: journal.sourceDigest, - chdb: journal.sourceChdb, - }) - ) - throw new Error("migration journal source identity does not match its first step") - if ( - !sameJournalIdentity(chain[chain.length - 1]!.to, { - version: journal.targetVersion, - fingerprint: journal.targetFingerprint, - digest: journal.targetDigest, - chdb: journal.targetChdb, - }) - ) - throw new Error("migration journal target identity does not match its final step") - for (let index = 1; index < chain.length; index += 1) { - if (!sameJournalIdentity(chain[index - 1]!.to, chain[index]!.from)) - throw new Error(`migration journal step ${index} does not continue the previous step`) - } - for (let index = 0; index < current; index += 1) { - if (chain[index]!.status !== "completed") - throw new Error(`migration journal step ${index} precedes currentStepIndex but is not completed`) - } - for (let index = current + 1; index < chain.length; index += 1) { - if (chain[index]!.status !== "pending") - throw new Error(`migration journal step ${index} follows currentStepIndex but is not pending`) - } - if (current === chain.length) { - if (chain.some((step) => step.status !== "completed")) - throw new Error( - "migration journal currentStepIndex reaches the end before all steps are completed", - ) - } else { - const currentStatus = chain[current]!.status - const plannedStart = journal.phase === "planned" && current === 0 - if (plannedStart) { - if (currentStatus !== "pending" && currentStatus !== "running") - throw new Error("planned migration journal must begin with a pending or running step") - } else if (currentStatus !== "running" && currentStatus !== "verified") { - throw new Error("migration journal current step must be running or verified") - } - } - if (journal.phase === "promotion-started" || journal.phase === "promoted") { - if (current !== chain.length || chain.some((step) => step.status !== "completed")) - throw new Error(`${journal.phase} migration journal must contain an entirely completed chain`) - } - if ( - (journal.phase === "preflight-complete" || - journal.phase === "target-created" || - journal.phase === "copying") && - current === chain.length - ) - throw new Error(`${journal.phase} migration journal must still have an active current step`) - if (journal.phase === "planned" && current !== 0) - throw new Error("planned migration journal must have currentStepIndex zero") - if ( - journal.phase === "failed" && - current === chain.length && - chain.some((step) => step.status !== "completed") - ) - throw new Error("failed migration journal cannot end before all steps are completed") -} - -const parseJournal = (value: unknown): MigrationJournal => { - const journal = decodeMigrationJournal(value) - if (journal.currentStepIndex > journal.chain.length) - throw new Error("migration journal currentStepIndex is invalid") - assertJournalChainInvariants(journal) - for (const [index, step] of journal.chain.entries()) { - if (step.status === "verified" || step.status === "completed") { - if (step.state === undefined) - throw new Error( - `migration journal step ${index} is ${step.status} but has no persisted state`, - ) - if (step.progress === undefined) - throw new Error( - `migration journal step ${index} is ${step.status} but has no persisted progress`, - ) - } - } - return journal -} - -/** Read and validate only the coordinator-owned journal envelope. This path is - * intentionally independent of the executable module registry so target-only - * abandonment remains available when a newer build cannot bind a historical - * module or its persisted state is corrupt. */ -export const readMigrationJournalStructure = async (dataDir: string): Promise => { - try { - // A custom-named store may still keep its journal at the pre-sidecar path. - adoptLegacySidecars(dataDir) - return parseJournal(JSON.parse(await readFile(migrationJournalPath(dataDir), "utf8")) as unknown) - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") return null - throw new Error( - `cannot read local migration journal: ${error instanceof Error ? error.message : String(error)}`, - ) - } -} - /** Read a journal for resume or promotion. Unlike structural quarantine, this * path must bind every step to the executable module and decode its persisted * state before any lifecycle handler can run. */ @@ -1235,7 +1079,7 @@ const withMigrationMaintenanceLock = async ( fn: () => Promise, ): Promise => { const { resetTransactionPath, restoreTransactionPath, withMaintenanceLock } = - await import("./checkpoints") + await import("@maple/local-server/checkpoints") return withMaintenanceLock(dataDir, operationId, async () => { if (existsSync(restoreTransactionPath(dataDir)) || existsSync(resetTransactionPath(dataDir))) throw new Error( diff --git a/apps/cli/src/server/local-store-migrations/backfills.ts b/apps/cli/src/server/local-store-migrations/backfills.ts index 5551443aa7..927990e1ec 100644 --- a/apps/cli/src/server/local-store-migrations/backfills.ts +++ b/apps/cli/src/server/local-store-migrations/backfills.ts @@ -2,9 +2,9 @@ // Backfill SQL and the custom steps that `steps.ts` references. Each projection // is frozen byte-for-byte to the view body its snapshot installs, so backfilled and live rows agree. import { Schema } from "effect" -import { applyRawTelemetryRetentionFloor, type Chdb } from "../chdb" -import { decodeJsonEachRow, decodeRowCounts } from "../chdb-rows" -import { localSchemaSnapshot } from "../schema-identity" +import { applyRawTelemetryRetentionFloor, type Chdb } from "@maple/local-server/chdb" +import { decodeJsonEachRow, decodeRowCounts } from "@maple/local-server/chdb-rows" +import { localSchemaSnapshot } from "../schema-snapshots" import { customStep, LocalMigrationStepError } from "./step-executor" import { UnsignedDecimal } from "./journal-codecs" diff --git a/apps/cli/src/server/local-store-migrations/journal-codecs.ts b/apps/cli/src/server/local-store-migrations/journal-codecs.ts index 6411713d6e..7f8b908f2a 100644 --- a/apps/cli/src/server/local-store-migrations/journal-codecs.ts +++ b/apps/cli/src/server/local-store-migrations/journal-codecs.ts @@ -3,13 +3,13 @@ import { Schema } from "effect" import { cp, mkdir, rm } from "node:fs/promises" import { dirname, join, sep } from "node:path" -import { RAW_TELEMETRY_TTL_COLUMNS, type Chdb } from "../chdb" -import { decodeRowCounts, decodeTableRowCounts } from "../chdb-rows" +import { RAW_TELEMETRY_TTL_COLUMNS, type Chdb } from "@maple/local-server/chdb" +import { decodeRowCounts, decodeTableRowCounts } from "@maple/local-server/chdb-rows" import { ttlDaysFromDefinition, withRawTelemetryRetentionFloor, type LocalSchemaManifest, -} from "../schema-manifest" +} from "@maple/local-server/schema-manifest" const RAW_TABLES_INTERNAL = RAW_TELEMETRY_TTL_COLUMNS.map(([table]) => table) diff --git a/apps/cli/src/server/local-store-migrations/legacy-to-current.ts b/apps/cli/src/server/local-store-migrations/legacy-to-current.ts index 36f2b42a16..becef8e672 100644 --- a/apps/cli/src/server/local-store-migrations/legacy-to-current.ts +++ b/apps/cli/src/server/local-store-migrations/legacy-to-current.ts @@ -1,16 +1,12 @@ // SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. import { createHash } from "node:crypto" import { Schema } from "effect" -import { decodeJsonEachRow, decodeJsonObjectRows } from "../chdb-rows" -import { - LOCAL_SCHEMA_V1_MANIFEST, - LOCAL_SCHEMA_V1_SQL, - LEGACY_LOCAL_SCHEMA, - LOCAL_SCHEMA_V1, -} from "../schema-identity" -import { assertPhysicalSchema } from "../schema-physical" +import { decodeJsonEachRow, decodeJsonObjectRows } from "@maple/local-server/chdb-rows" +import { LEGACY_LOCAL_SCHEMA, LOCAL_SCHEMA_V1 } from "@maple/local-server/schema-identity" +import { LOCAL_SCHEMA_V1_MANIFEST, LOCAL_SCHEMA_V1_SQL } from "../schema-snapshots" +import { assertPhysicalSchema } from "@maple/local-server/schema-physical" import { strictDecoder, UnsignedDecimal } from "./journal-codecs" -import type { LocalSchemaColumn } from "../schema-manifest" +import type { LocalSchemaColumn } from "@maple/local-server/schema-manifest" import type { LocalStoreMigrationModule, MigrationModuleContext, diff --git a/apps/cli/src/server/local-store-migrations/step-executor.ts b/apps/cli/src/server/local-store-migrations/step-executor.ts index 8701f8b261..989804cb75 100644 --- a/apps/cli/src/server/local-store-migrations/step-executor.ts +++ b/apps/cli/src/server/local-store-migrations/step-executor.ts @@ -3,15 +3,16 @@ // the clone, bootstrap the target snapshot over it, verify, and never touch the source. import { resolve } from "node:path" import { Schema } from "effect" -import { readRawTelemetryRetentionDays, type Chdb } from "../chdb" -import { decodeJsonObjectRows } from "../chdb-rows" +import { readRawTelemetryRetentionDays, type Chdb } from "@maple/local-server/chdb" +import { decodeJsonObjectRows } from "@maple/local-server/chdb-rows" import type { LocalStoreMigrationModule, MigrationOperation, StateDispositionEntry, } from "../local-store-migration-module" -import { localSchemaIdentity, localSchemaSnapshot } from "../schema-identity" -import { assertPhysicalSchema } from "../schema-physical" +import { localSchemaIdentity } from "@maple/local-server/schema-identity" +import { localSchemaSnapshot } from "../schema-snapshots" +import { assertPhysicalSchema } from "@maple/local-server/schema-physical" import { cloneStoreForStaging, expectedManifest, diff --git a/apps/cli/src/server/retained-leftovers.ts b/apps/cli/src/server/retained-leftovers.ts index f7cec29e26..cd62cf73e0 100644 --- a/apps/cli/src/server/retained-leftovers.ts +++ b/apps/cli/src/server/retained-leftovers.ts @@ -16,13 +16,13 @@ import { withMaintenanceLock, type CheckpointId, type CheckpointValidation, -} from "./checkpoints" -import { syncDirectory } from "./durable-files" +} from "@maple/local-server/checkpoints" +import { syncDirectory } from "@maple/local-server/durable-files" import { migrationJournalPath, migrationRootPath, readMigrationJournalStructure, -} from "./local-store-migrations" +} from "@maple/local-server/local-store-migration-journal" export type RetainedLeftoverKind = | "restore-replaced-store" diff --git a/apps/cli/src/server/schema-identity.ts b/apps/cli/src/server/schema-snapshots.ts similarity index 54% rename from apps/cli/src/server/schema-identity.ts rename to apps/cli/src/server/schema-snapshots.ts index 27dbe9dc16..2d50e5ff3b 100644 --- a/apps/cli/src/server/schema-identity.ts +++ b/apps/cli/src/server/schema-snapshots.ts @@ -1,4 +1,3 @@ -import schemaSql from "./schema/local-schema.sql" with { type: "text" } import schemaV1Sql from "./schema/local-schema-v1.sql" with { type: "text" } import schemaV2Sql from "./schema/local-schema-v2.sql" with { type: "text" } import schemaV3Sql from "./schema/local-schema-v3.sql" with { type: "text" } @@ -41,40 +40,8 @@ import schemaV39Sql from "./schema/local-schema-v39.sql" with { type: "text" } import schemaV40Sql from "./schema/local-schema-v40.sql" with { type: "text" } import schemaV41Sql from "./schema/local-schema-v41.sql" with { type: "text" } import schemaV42Sql from "./schema/local-schema-v42.sql" with { type: "text" } -import { schemaDigest as digestSchema, schemaFingerprint as fingerprintSchema } from "./store-version" -import { buildLocalSchemaManifest, type LocalSchemaManifest } from "./schema-manifest" -import { LOCAL_SCHEMA_VERSION } from "./local-schema-version" -import { LOCAL_SCHEMA_HISTORY } from "./local-schema-history" -import { CHDB_VERSION } from "../version" - -export { LOCAL_SCHEMA_HISTORY } from "./local-schema-history" +import { buildLocalSchemaManifest, type LocalSchemaManifest } from "@maple/local-server/schema-manifest" -/** - * Local-store schema version. - * - * Version 0 is the fingerprint-only legacy store represented by the recovery - * procedure from issue #297. Any future structural DDL change must increment - * this value and add a registered migration or an explicit unsupported edge. - */ -export { LOCAL_SCHEMA_VERSION } -export const LEGACY_LOCAL_SCHEMA_VERSION = 0 as const - -export const LEGACY_SCHEMA_PROJECT_REVISION = - "d58ce4a83d3ad3f3a29b9bb972272b757547ae793c050194354454634f3abccd" -export const LEGACY_SCHEMA_FINGERPRINT = "428701854f9fd30e" - -export const CURRENT_SCHEMA_PROJECT_REVISION = - "ed74788ef292834069e0ea6ee3b22d68fc604fb66cb54d2d551db67ce8d20b3a" -/** Revision recorded by the issue-297 recovery report. The refreshed upstream - * generator currently emits CURRENT_SCHEMA_PROJECT_REVISION; the structural - * fingerprint is the compatibility identity used by the migration. */ -export const ISSUE_297_TARGET_SCHEMA_PROJECT_REVISION = - "506bc745f7a7eca202ec905a6403a6815e86413faf0cd3cbbf73881023edce91" -export const LOCAL_SCHEMA_SQL = schemaSql -export const SCHEMA_FINGERPRINT = fingerprintSchema(schemaSql) -export const SCHEMA_DIGEST = digestSchema(schemaSql) -export const LOCAL_SCHEMA_MANIFEST: LocalSchemaManifest = buildLocalSchemaManifest(schemaSql) -export const LOCAL_SCHEMA_MANIFEST_DIGEST = LOCAL_SCHEMA_MANIFEST.digest /** * Immutable per-version DDL and manifest snapshots. * @@ -153,7 +120,7 @@ export const localSchemaSnapshot = (version: number): LocalSchemaSnapshot => { } // Per-version constants exist only where the legacy step or the tests pin them; -// table-driven steps look their versions up through the two functions here. +// table-driven steps look their versions up through localSchemaSnapshot and localSchemaIdentity. export const LOCAL_SCHEMA_V1_SQL = localSchemaSnapshot(1).sql export const LOCAL_SCHEMA_V1_MANIFEST = localSchemaSnapshot(1).manifest export const LOCAL_SCHEMA_V2_MANIFEST = localSchemaSnapshot(2).manifest @@ -165,83 +132,3 @@ export const LOCAL_SCHEMA_V10_MANIFEST = localSchemaSnapshot(10).manifest export const LOCAL_SCHEMA_V11_MANIFEST = localSchemaSnapshot(11).manifest export const LOCAL_SCHEMA_V12_MANIFEST = localSchemaSnapshot(12).manifest export const LOCAL_SCHEMA_V13_MANIFEST = localSchemaSnapshot(13).manifest - -export interface LocalSchemaIdentity { - readonly version: number - readonly fingerprint: string - readonly digest: string - readonly manifestDigest?: string - readonly chdb: string - readonly projectRevision?: string -} - -/** - * Per-version identities, frozen and read straight from the append-only - * history. Historical migration edges must never point at - * CURRENT_LOCAL_SCHEMA: when v10 ships, v0 -> v1 must still construct and verify - * v1 rather than silently changing its destination. - */ -export const localSchemaIdentity = (version: number): LocalSchemaIdentity => { - const entry = LOCAL_SCHEMA_HISTORY[version] - if (!entry || entry.version !== version) - throw new Error(`local schema history has no entry for version ${version}`) - return Object.freeze({ - version: entry.version, - fingerprint: entry.fingerprint, - digest: entry.digest, - manifestDigest: entry.manifestDigest, - chdb: CHDB_VERSION, - projectRevision: entry.projectRevision, - }) -} - -export const LOCAL_SCHEMA_V1 = localSchemaIdentity(1) -export const LOCAL_SCHEMA_V2 = localSchemaIdentity(2) -export const LOCAL_SCHEMA_V3 = localSchemaIdentity(3) -export const LOCAL_SCHEMA_V4 = localSchemaIdentity(4) -export const LOCAL_SCHEMA_V5 = localSchemaIdentity(5) -export const LOCAL_SCHEMA_V6 = localSchemaIdentity(6) -export const LOCAL_SCHEMA_V10 = localSchemaIdentity(10) -export const LOCAL_SCHEMA_V11 = localSchemaIdentity(11) -export const LOCAL_SCHEMA_V20 = localSchemaIdentity(20) -export const LOCAL_SCHEMA_V21 = localSchemaIdentity(21) -export const LOCAL_SCHEMA_V22 = localSchemaIdentity(22) -export const LOCAL_SCHEMA_V23 = localSchemaIdentity(23) -export const LOCAL_SCHEMA_V24 = localSchemaIdentity(24) -export const LOCAL_SCHEMA_V25 = localSchemaIdentity(25) -export const LOCAL_SCHEMA_V26 = localSchemaIdentity(26) -export const LOCAL_SCHEMA_V27 = localSchemaIdentity(27) -export const LOCAL_SCHEMA_V28 = localSchemaIdentity(28) -export const LOCAL_SCHEMA_V29 = localSchemaIdentity(29) -export const LOCAL_SCHEMA_V30 = localSchemaIdentity(30) -export const LOCAL_SCHEMA_V31 = localSchemaIdentity(31) -export const LOCAL_SCHEMA_V32 = localSchemaIdentity(32) -export const LOCAL_SCHEMA_V33 = localSchemaIdentity(33) -export const LOCAL_SCHEMA_V34 = localSchemaIdentity(34) -export const LOCAL_SCHEMA_V36 = localSchemaIdentity(36) -export const LOCAL_SCHEMA_V37 = localSchemaIdentity(37) -export const LOCAL_SCHEMA_V38 = localSchemaIdentity(38) -export const LOCAL_SCHEMA_V39 = localSchemaIdentity(39) -export const LOCAL_SCHEMA_V40 = localSchemaIdentity(40) -export const LOCAL_SCHEMA_V41 = localSchemaIdentity(41) -export const LOCAL_SCHEMA_V42 = localSchemaIdentity(42) - -export const CURRENT_LOCAL_SCHEMA: LocalSchemaIdentity = Object.freeze({ - version: LOCAL_SCHEMA_VERSION, - fingerprint: SCHEMA_FINGERPRINT, - digest: SCHEMA_DIGEST, - manifestDigest: LOCAL_SCHEMA_MANIFEST_DIGEST, - chdb: CHDB_VERSION, - projectRevision: CURRENT_SCHEMA_PROJECT_REVISION, -}) - -export const LEGACY_LOCAL_SCHEMA: LocalSchemaIdentity = { - version: LEGACY_LOCAL_SCHEMA_VERSION, - fingerprint: LEGACY_SCHEMA_FINGERPRINT, - digest: "", - chdb: CHDB_VERSION, - projectRevision: LEGACY_SCHEMA_PROJECT_REVISION, -} - -export const identityLabel = (identity: Pick): string => - `v${identity.version} (${identity.fingerprint})` diff --git a/apps/cli/test/archive-config.test.ts b/apps/cli/test/archive-config.test.ts index 2fde26c655..2860dc1ee2 100644 --- a/apps/cli/test/archive-config.test.ts +++ b/apps/cli/test/archive-config.test.ts @@ -1,6 +1,6 @@ import { describe, it } from "@effect/vitest" import { strictEqual, throws } from "node:assert" -import { DEFAULT_ARCHIVE_TUNING, resolveArchiveTuning } from "../src/server/archives/config" +import { DEFAULT_ARCHIVE_TUNING, resolveArchiveTuning } from "@maple/local-server/archives/config" const base = { archiveDir: "/tmp/archive", scratchRoot: "/tmp/scratch" } diff --git a/apps/cli/test/archive-export-round5.test.ts b/apps/cli/test/archive-export-round5.test.ts index 4a8d0cbbd9..e7c364015b 100644 --- a/apps/cli/test/archive-export-round5.test.ts +++ b/apps/cli/test/archive-export-round5.test.ts @@ -7,10 +7,10 @@ import { COMPLEX_DIGEST_ALGORITHM, multisetDigestSql, type ExportSettings, -} from "../src/server/archives/export" -import { parseArchiveGenerationManifest } from "../src/server/archives/manifest" -import { CHDB_VERSION, MAPLE_VERSION } from "../src/version" -import { SCHEMA_FINGERPRINT } from "../src/server/schema-identity" +} from "@maple/local-server/archives/export" +import { parseArchiveGenerationManifest } from "@maple/local-server/archives/manifest" +import { CHDB_VERSION, MAPLE_VERSION } from "@maple/local-server/version" +import { SCHEMA_FINGERPRINT } from "@maple/local-server/schema-identity" // Pure-logic tests for the round-5 fixes. The end-to-end adversarial coverage // (column swap, row reassociation, byte refinement, timezone, recovery) lives diff --git a/apps/cli/test/archive-export-safety.test.ts b/apps/cli/test/archive-export-safety.test.ts index c8bdea3e1d..e613400df1 100644 --- a/apps/cli/test/archive-export-safety.test.ts +++ b/apps/cli/test/archive-export-safety.test.ts @@ -1,6 +1,6 @@ import { describe, it } from "@effect/vitest" import { throws } from "node:assert" -import { exportSignalShards } from "../src/server/archives/export" +import { exportSignalShards } from "@maple/local-server/archives/export" // Pure-logic tests for export safety primitives that don't require chDB. // The full Parquet write+reopen+validation path is exercised by the native diff --git a/apps/cli/test/archive-gc.test.ts b/apps/cli/test/archive-gc.test.ts index 1c157c2daa..f6c94119ce 100644 --- a/apps/cli/test/archive-gc.test.ts +++ b/apps/cli/test/archive-gc.test.ts @@ -19,14 +19,17 @@ import { generationManifestPath, generationRoot, nextMidnightUtc, -} from "../src/server/archives/paths" -import { promoteGeneration, selectActiveGeneration } from "../src/server/archives/generation" -import { type ArchiveGenerationManifest, parseArchiveActivePointer } from "../src/server/archives/manifest" -import { CHDB_VERSION, MAPLE_VERSION } from "../src/version" -import { SCHEMA_FINGERPRINT } from "../src/server/schema-identity" -import { rebuildCatalog } from "../src/server/archives/listing" -import { planArchiveGc, runArchiveGc, verifyCompletedGcInvariants } from "../src/server/archives/gc" -import { writeInitialIntent, type GcOperationIntent } from "../src/server/archives/journal" +} from "@maple/local-server/archives/paths" +import { promoteGeneration, selectActiveGeneration } from "@maple/local-server/archives/generation" +import { + type ArchiveGenerationManifest, + parseArchiveActivePointer, +} from "@maple/local-server/archives/manifest" +import { CHDB_VERSION, MAPLE_VERSION } from "@maple/local-server/version" +import { SCHEMA_FINGERPRINT } from "@maple/local-server/schema-identity" +import { rebuildCatalog } from "@maple/local-server/archives/listing" +import { planArchiveGc, runArchiveGc, verifyCompletedGcInvariants } from "@maple/local-server/archives/gc" +import { writeInitialIntent, type GcOperationIntent } from "@maple/local-server/archives/journal" // Hostile unit tests for archive garbage collection (Gate 3b). These cover the // deterministic deletion-set logic, keep-N retention, fail-closed on diff --git a/apps/cli/test/archive-generation.test.ts b/apps/cli/test/archive-generation.test.ts index 4f4a7ac29a..f4bfd054be 100644 --- a/apps/cli/test/archive-generation.test.ts +++ b/apps/cli/test/archive-generation.test.ts @@ -21,8 +21,11 @@ import { generationRoot, generationManifestPath, shardsRoot, -} from "../src/server/archives/paths" -import { parseArchiveActivePointer, type ArchiveGenerationManifest } from "../src/server/archives/manifest" +} from "@maple/local-server/archives/paths" +import { + parseArchiveActivePointer, + type ArchiveGenerationManifest, +} from "@maple/local-server/archives/manifest" import { assertArchiveScratchFreeSpace, assertNoArchiveShrink, @@ -32,16 +35,16 @@ import { reconcileArchiveGeneration, selectActiveGeneration, ttlDaysFromDefinition, -} from "../src/server/archives/generation" +} from "@maple/local-server/archives/generation" import { advancePhase, listActiveOperationIds, operationDir, writeInitialIntent, type ArchiveOperationPhase, -} from "../src/server/archives/journal" -import { CHDB_VERSION, MAPLE_VERSION } from "../src/version" -import { SCHEMA_FINGERPRINT } from "../src/server/schema-identity" +} from "@maple/local-server/archives/journal" +import { CHDB_VERSION, MAPLE_VERSION } from "@maple/local-server/version" +import { SCHEMA_FINGERPRINT } from "@maple/local-server/schema-identity" import { checkpointPinsRoot, checkpointRoot, @@ -50,10 +53,10 @@ import { newCheckpointId, withMaintenanceLock, type CheckpointId, -} from "../src/server/checkpoints" -import { assertCatalogExact, rebuildCatalog } from "../src/server/archives/listing" -import { RetiredDayAuthority } from "../src/server/archives/retention" -import { ARCHIVE_SIGNALS } from "../src/server/archives/signals" +} from "@maple/local-server/checkpoints" +import { assertCatalogExact, rebuildCatalog } from "@maple/local-server/archives/listing" +import { RetiredDayAuthority } from "@maple/local-server/archives/retention" +import { ARCHIVE_SIGNALS } from "@maple/local-server/archives/signals" // Filesystem-level tests for generation promotion, supersession, and catalog // append. These exercise the durable state machine without a restored chDB; the diff --git a/apps/cli/test/archive-journal.test.ts b/apps/cli/test/archive-journal.test.ts index 9839d7614e..4e992052c7 100644 --- a/apps/cli/test/archive-journal.test.ts +++ b/apps/cli/test/archive-journal.test.ts @@ -27,9 +27,9 @@ import { readActiveOperation, writeInitialIntent, type ArchiveOperationIntent, -} from "../src/server/archives/journal" -import { reconcileArchiveGeneration } from "../src/server/archives/generation" -import { checkpointPinsRoot } from "../src/server/checkpoints" +} from "@maple/local-server/archives/journal" +import { reconcileArchiveGeneration } from "@maple/local-server/archives/generation" +import { checkpointPinsRoot } from "@maple/local-server/checkpoints" // Filesystem-level tests for the archive operation journal (Gate 3). These are // the fast in-process checks of the journal's fail-closed parsing, phase diff --git a/apps/cli/test/archive-legacy-calibration.test.ts b/apps/cli/test/archive-legacy-calibration.test.ts index 3cacf3bcb4..7c1631c750 100644 --- a/apps/cli/test/archive-legacy-calibration.test.ts +++ b/apps/cli/test/archive-legacy-calibration.test.ts @@ -11,11 +11,11 @@ import { checkpointSnapshotDir, checkpointStatePath, newCheckpointId, -} from "../src/server/checkpoints" -import { runArchiveReconciliation } from "../src/server/archives/generation" -import { retireLegacyCalibration } from "../src/server/archives/legacy-calibration" -import { SCHEMA_FINGERPRINT } from "../src/server/schema-identity" -import { CHDB_VERSION, MAPLE_VERSION } from "../src/version" +} from "@maple/local-server/checkpoints" +import { runArchiveReconciliation } from "@maple/local-server/archives/generation" +import { retireLegacyCalibration } from "@maple/local-server/archives/legacy-calibration" +import { SCHEMA_FINGERPRINT } from "@maple/local-server/schema-identity" +import { CHDB_VERSION, MAPLE_VERSION } from "@maple/local-server/version" // Calibration was removed; what an interrupted older run left must not wedge // anything and must be retired by the ordinary reconcile path. diff --git a/apps/cli/test/archive-listing.test.ts b/apps/cli/test/archive-listing.test.ts index 9b4d13cf9a..06bc429b9e 100644 --- a/apps/cli/test/archive-listing.test.ts +++ b/apps/cli/test/archive-listing.test.ts @@ -13,7 +13,7 @@ import { generationsRoot, nextMidnightUtc, shardsRoot, -} from "../src/server/archives/paths" +} from "@maple/local-server/archives/paths" import { ARCHIVE_VERIFY_BUFFER_BYTES, activeParquetPaths, @@ -21,11 +21,11 @@ import { rebuildCatalog, rebuildCatalogWithMaintenanceLock, verifyActiveGenerations, -} from "../src/server/archives/listing" -import { withMaintenanceLock } from "../src/server/checkpoints" -import { type ArchiveGenerationManifest } from "../src/server/archives/manifest" -import { CHDB_VERSION, MAPLE_VERSION } from "../src/version" -import { SCHEMA_FINGERPRINT } from "../src/server/schema-identity" +} from "@maple/local-server/archives/listing" +import { withMaintenanceLock } from "@maple/local-server/checkpoints" +import { type ArchiveGenerationManifest } from "@maple/local-server/archives/manifest" +import { CHDB_VERSION, MAPLE_VERSION } from "@maple/local-server/version" +import { SCHEMA_FINGERPRINT } from "@maple/local-server/schema-identity" const withArchive = async (run: (archiveDir: string) => Promise | void): Promise => { const parent = mkdtempSync(join(tmpdir(), "maple-archive-listing-test-")) diff --git a/apps/cli/test/archive-manifest.test.ts b/apps/cli/test/archive-manifest.test.ts index f90c343d73..24ea3a1a0a 100644 --- a/apps/cli/test/archive-manifest.test.ts +++ b/apps/cli/test/archive-manifest.test.ts @@ -9,10 +9,13 @@ import { generationManifestPath, generationsRoot, rangeRoot, -} from "../src/server/archives/paths" -import { parseArchiveActivePointer, parseArchiveGenerationManifest } from "../src/server/archives/manifest" -import { CHDB_VERSION, MAPLE_VERSION } from "../src/version" -import { SCHEMA_FINGERPRINT } from "../src/server/schema-identity" +} from "@maple/local-server/archives/paths" +import { + parseArchiveActivePointer, + parseArchiveGenerationManifest, +} from "@maple/local-server/archives/manifest" +import { CHDB_VERSION, MAPLE_VERSION } from "@maple/local-server/version" +import { SCHEMA_FINGERPRINT } from "@maple/local-server/schema-identity" import { randomUUID } from "node:crypto" const withArchive = async (run: (archiveDir: string) => Promise | void): Promise => { @@ -169,7 +172,7 @@ describe("archive generation manifest parser", () => { it("reads a manifest from disk bound to its location", async () => { await withArchive(async (archiveDir) => { - const { readArchiveGenerationManifest } = await import("../src/server/archives/manifest") + const { readArchiveGenerationManifest } = await import("@maple/local-server/archives/manifest") const generationId = randomUUID() const manifestPath = generationManifestPath(archiveDir, "traces", "2026-06-01", generationId) mkdirSync(dirname(manifestPath), { recursive: true }) diff --git a/apps/cli/test/archive-path-safety.test.ts b/apps/cli/test/archive-path-safety.test.ts index f008d4407e..2b9dfc7112 100644 --- a/apps/cli/test/archive-path-safety.test.ts +++ b/apps/cli/test/archive-path-safety.test.ts @@ -15,12 +15,12 @@ import { generationManifestPath, rangeRoot, shardsRoot, -} from "../src/server/archives/paths" -import { promoteGeneration } from "../src/server/archives/generation" -import { rebuildCatalog, listActiveGenerations } from "../src/server/archives/listing" -import { type ArchiveGenerationManifest } from "../src/server/archives/manifest" -import { CHDB_VERSION, MAPLE_VERSION } from "../src/version" -import { SCHEMA_FINGERPRINT } from "../src/server/schema-identity" +} from "@maple/local-server/archives/paths" +import { promoteGeneration } from "@maple/local-server/archives/generation" +import { rebuildCatalog, listActiveGenerations } from "@maple/local-server/archives/listing" +import { type ArchiveGenerationManifest } from "@maple/local-server/archives/manifest" +import { CHDB_VERSION, MAPLE_VERSION } from "@maple/local-server/version" +import { SCHEMA_FINGERPRINT } from "@maple/local-server/schema-identity" // External-sentinel tests proving archive-root path escapes are closed (C-1). // These mirror the reviewer's deterministic probes: a symlinked descendant must @@ -233,7 +233,7 @@ describe("archive path safety — symlink escapes (C-1)", () => { describe("archive path safety — read-side symlink escapes (HIGH-1/HIGH-2)", () => { it("readArchiveGenerationManifest refuses a symlinked generation dir and does not read outside content", async () => { await withArchive(async (archiveDir, outside) => { - const { readArchiveGenerationManifest } = await import("../src/server/archives/manifest") + const { readArchiveGenerationManifest } = await import("@maple/local-server/archives/manifest") const generationId = randomUUID() // Plant a real generation outside the root with an attacker manifest. const outsideGen = join(outside, "evil-gen") diff --git a/apps/cli/test/archive-pins.test.ts b/apps/cli/test/archive-pins.test.ts index d5fdecdfe2..fe8a8867e9 100644 --- a/apps/cli/test/archive-pins.test.ts +++ b/apps/cli/test/archive-pins.test.ts @@ -25,9 +25,9 @@ import { readCheckpointState, withMaintenanceLock, type CheckpointId, -} from "../src/server/checkpoints" -import { SCHEMA_FINGERPRINT } from "../src/server/schema-identity" -import { CHDB_VERSION, MAPLE_VERSION } from "../src/version" +} from "@maple/local-server/checkpoints" +import { SCHEMA_FINGERPRINT } from "@maple/local-server/schema-identity" +import { CHDB_VERSION, MAPLE_VERSION } from "@maple/local-server/version" // Pin and maintenance-lock API for the dependent archive branch. These tests // mirror the checkpoint suite's conventions: a throwaway parent/data dir, diff --git a/apps/cli/test/archive-probe-helpers.ts b/apps/cli/test/archive-probe-helpers.ts index 342fd721c0..fd77e448cb 100644 --- a/apps/cli/test/archive-probe-helpers.ts +++ b/apps/cli/test/archive-probe-helpers.ts @@ -15,7 +15,7 @@ import { mkdtempSync, rmSync } from "node:fs" import { tmpdir } from "node:os" import { join } from "node:path" -import { Chdb } from "../src/server/chdb" +import { Chdb } from "@maple/local-server/chdb" export interface ArchiveProbeHandle { /** The probe's owned root directory (created via mkdtemp, unique per run). */ diff --git a/apps/cli/test/archive-reconcile-decision.test.ts b/apps/cli/test/archive-reconcile-decision.test.ts index 96d320574a..d21f7851ef 100644 --- a/apps/cli/test/archive-reconcile-decision.test.ts +++ b/apps/cli/test/archive-reconcile-decision.test.ts @@ -6,8 +6,8 @@ import { digestOfIntent, type ReconciliationInspection, type ReconciliationSnapshot, -} from "../src/server/archives/reconcile" -import type { CreateOperationIntent, GcOperationIntent } from "../src/server/archives/journal" +} from "@maple/local-server/archives/reconcile" +import type { CreateOperationIntent, GcOperationIntent } from "@maple/local-server/archives/journal" // Exhaustive transition-table tests for the pure decision function (Gate 3b r5). // These test the SOLE branch logic with NO I/O: every (kind, phase, topology) diff --git a/apps/cli/test/archive-reconcile.test.ts b/apps/cli/test/archive-reconcile.test.ts index 6035043b1b..e2aa933f80 100644 --- a/apps/cli/test/archive-reconcile.test.ts +++ b/apps/cli/test/archive-reconcile.test.ts @@ -1,8 +1,8 @@ import { describe, it } from "@effect/vitest" import { ok, rejects, strictEqual } from "node:assert" import { createHash, randomUUID } from "node:crypto" -import { MAPLE_VERSION, CHDB_VERSION } from "../src/version" -import { SCHEMA_FINGERPRINT } from "../src/server/schema-identity" +import { MAPLE_VERSION, CHDB_VERSION } from "@maple/local-server/version" +import { SCHEMA_FINGERPRINT } from "@maple/local-server/schema-identity" import { existsSync, lstatSync, @@ -18,8 +18,8 @@ import { } from "node:fs" import { tmpdir } from "node:os" import { dirname, join } from "node:path" -import { runArchiveReconciliation } from "../src/server/archives/generation" -import type { ReconciliationDecision } from "../src/server/archives/reconcile" +import { runArchiveReconciliation } from "@maple/local-server/archives/generation" +import type { ReconciliationDecision } from "@maple/local-server/archives/reconcile" const withRoots = async ( run: (archiveDir: string, dataDir: string, scratchRoot: string) => Promise | void, @@ -272,7 +272,7 @@ describe("archive reconciliation protocol (Gate 3b r5)", () => { }) it("dry-run fails nonzero while a live owner holds the lock", async () => { await withRoots(async (archiveDir, dataDir, scratchRoot) => { - const { withMaintenanceLock } = await import("../src/server/checkpoints") + const { withMaintenanceLock } = await import("@maple/local-server/checkpoints") const { record } = validV2(archiveDir, dataDir, scratchRoot) writeActiveIntent(archiveDir, record.operationId as string, record) await withMaintenanceLock(dataDir, randomUUID(), async () => { diff --git a/apps/cli/test/archive-retention.test.ts b/apps/cli/test/archive-retention.test.ts index c6b47b594e..21c3b701db 100644 --- a/apps/cli/test/archive-retention.test.ts +++ b/apps/cli/test/archive-retention.test.ts @@ -12,20 +12,20 @@ import { retiredDayLedgerPath, RetiredDayAuthority, retireLiveDayInServer, -} from "../src/server/archives/retention" -import { ARCHIVE_SIGNALS, type ArchiveSignalName } from "../src/server/archives/signals" +} from "@maple/local-server/archives/retention" +import { ARCHIVE_SIGNALS, type ArchiveSignalName } from "@maple/local-server/archives/signals" import { activePointerPath, generationManifestPath, nextMidnightUtc, rangeRoot, shardsRoot, -} from "../src/server/archives/paths" -import { rebuildCatalog } from "../src/server/archives/listing" -import type { ArchiveGenerationManifest } from "../src/server/archives/manifest" -import { CHDB_VERSION, MAPLE_VERSION } from "../src/version" -import { RequestQuiescenceGate } from "../src/server/serve" -import { SCHEMA_FINGERPRINT } from "../src/server/schema-identity" +} from "@maple/local-server/archives/paths" +import { rebuildCatalog } from "@maple/local-server/archives/listing" +import type { ArchiveGenerationManifest } from "@maple/local-server/archives/manifest" +import { CHDB_VERSION, MAPLE_VERSION } from "@maple/local-server/version" +import { RequestQuiescenceGate } from "@maple/local-server/serve" +import { SCHEMA_FINGERPRINT } from "@maple/local-server/schema-identity" const sha256 = (value: string): string => createHash("sha256").update(value).digest("hex") diff --git a/apps/cli/test/chdb-args.test.ts b/apps/cli/test/chdb-args.test.ts index 5bb7bc83f1..105112534d 100644 --- a/apps/cli/test/chdb-args.test.ts +++ b/apps/cli/test/chdb-args.test.ts @@ -9,7 +9,7 @@ import { configureRawTelemetryRetentionDays, rawTelemetryTtlStatements, readRawTelemetryRetentionDays, -} from "../src/server/chdb" +} from "@maple/local-server/chdb" describe("embedded chDB arguments", () => { it("keeps an explicit config after the loader and restore safety settings", () => { diff --git a/apps/cli/test/chdb-ingest-config.test.ts b/apps/cli/test/chdb-ingest-config.test.ts index c15c7409d1..1ffeb7c997 100644 --- a/apps/cli/test/chdb-ingest-config.test.ts +++ b/apps/cli/test/chdb-ingest-config.test.ts @@ -2,7 +2,7 @@ import { afterAll, beforeAll, expect, test } from "bun:test" import { mkdtempSync, rmSync } from "node:fs" import { tmpdir } from "node:os" import { join } from "node:path" -import { Chdb, writeIngestConfig } from "../src/server/chdb" +import { Chdb, writeIngestConfig } from "@maple/local-server/chdb" import { describeWithChdb } from "./libchdb-test-support" // chDB silently ignores `--merge_tree.*` arguments, so the only proof the ingest diff --git a/apps/cli/test/chdb-timezone.test.ts b/apps/cli/test/chdb-timezone.test.ts index 8dee585794..2c3f800db9 100644 --- a/apps/cli/test/chdb-timezone.test.ts +++ b/apps/cli/test/chdb-timezone.test.ts @@ -2,7 +2,7 @@ import { describe, it } from "@effect/vitest" import { dlopen, FFIType, ptr } from "bun:ffi" import { CString } from "bun:ffi" import { strictEqual } from "node:assert" -import { pinProcessTimezoneToUtc } from "../src/server/chdb" +import { pinProcessTimezoneToUtc } from "@maple/local-server/chdb" /** * libchdb resolves its SERVER timezone from the process environment at init, diff --git a/apps/cli/test/checkpoint-merge-freeze.test.ts b/apps/cli/test/checkpoint-merge-freeze.test.ts index ae238b7a44..07fa337dc8 100644 --- a/apps/cli/test/checkpoint-merge-freeze.test.ts +++ b/apps/cli/test/checkpoint-merge-freeze.test.ts @@ -3,9 +3,9 @@ import { ok, strictEqual } from "node:assert" import { mkdtempSync, rmSync } from "node:fs" import { tmpdir } from "node:os" import { join } from "node:path" -import { Chdb } from "../src/server/chdb" -import { validateCheckpointDataDir } from "../src/server/checkpoints" -import schemaSql from "../src/server/schema/local-schema.sql" with { type: "text" } +import { Chdb } from "@maple/local-server/chdb" +import { validateCheckpointDataDir } from "@maple/local-server/checkpoints" +import schemaSql from "@maple/local-server/schema/local-schema.sql" with { type: "text" } import { describeWithChdb } from "./libchdb-test-support" const countRows = (db: Chdb, table: string): number => { diff --git a/apps/cli/test/checkpoint-retained-days.test.ts b/apps/cli/test/checkpoint-retained-days.test.ts index 5729091a2a..a35dd1bb37 100644 --- a/apps/cli/test/checkpoint-retained-days.test.ts +++ b/apps/cli/test/checkpoint-retained-days.test.ts @@ -1,6 +1,6 @@ import { describe, it } from "@effect/vitest" import { strictEqual } from "node:assert" -import { retainedDaysMatch, type RetainedDays } from "../src/server/checkpoints" +import { retainedDaysMatch, type RetainedDays } from "@maple/local-server/checkpoints" const TABLES = [ "logs", diff --git a/apps/cli/test/checkpoints.test.ts b/apps/cli/test/checkpoints.test.ts index 6677735710..e0a250b436 100644 --- a/apps/cli/test/checkpoints.test.ts +++ b/apps/cli/test/checkpoints.test.ts @@ -1,7 +1,7 @@ // BOUNDARY: Test doubles preserve opaque values so the consuming boundary can be exercised. import { describe, it } from "@effect/vitest" import { createHash } from "node:crypto" -import { sha256File } from "../src/server/checkpoint-digest" +import { sha256File } from "@maple/local-server/checkpoint-digest" import { Clock, Duration, Effect, Exit, Option } from "effect" import { HttpClient, HttpClientResponse } from "effect/http" import { deepStrictEqual, match, ok, rejects, strictEqual, throws } from "node:assert" @@ -56,17 +56,17 @@ import { retireCheckpointIfEligible, validateCheckpointDataDir, writeBackupConfig, -} from "../src/server/checkpoints" +} from "@maple/local-server/checkpoints" import { durableWrite, isUnsupportedDirectorySyncError, syncDirectory, syncTree, -} from "../src/server/durable-files" -import { SCHEMA_FINGERPRINT } from "../src/server/schema-identity" -import { storeMarkerPath, storeOpenMarkerPath } from "../src/server/store-version" -import { CHDB_VERSION, MAPLE_VERSION } from "../src/version" -import { eventingControlSnapshotPath } from "../src/server/eventing/control-store" +} from "@maple/local-server/durable-files" +import { SCHEMA_FINGERPRINT } from "@maple/local-server/schema-identity" +import { storeMarkerPath, storeOpenMarkerPath } from "@maple/local-server/store-version" +import { CHDB_VERSION, MAPLE_VERSION } from "@maple/local-server/version" +import { eventingControlSnapshotPath } from "@maple/local-server/eventing/control-store" import { openStore, runAsync } from "./eventing-test-support" const withDataDir = async (run: (dataDir: string) => Promise | void): Promise => { diff --git a/apps/cli/test/durable-files.test.ts b/apps/cli/test/durable-files.test.ts index 88ecbe7a6c..dcd2178358 100644 --- a/apps/cli/test/durable-files.test.ts +++ b/apps/cli/test/durable-files.test.ts @@ -8,7 +8,7 @@ import { durableWrite, durableWriteFileSync, ensurePrivateDirectory, -} from "../src/server/durable-files" +} from "@maple/local-server/durable-files" const withRoot = async (run: (root: string) => Promise): Promise => { const root = mkdtempSync(join(tmpdir(), "maple-durable-files-test-")) diff --git a/apps/cli/test/eventing-test-support.ts b/apps/cli/test/eventing-test-support.ts index 5a7a035ac6..5784995a28 100644 --- a/apps/cli/test/eventing-test-support.ts +++ b/apps/cli/test/eventing-test-support.ts @@ -5,12 +5,12 @@ import { openControlStore, type LocalEventingControlLimits, type LocalEventingControlStoreApi, -} from "../src/server/eventing/control-store" +} from "@maple/local-server/eventing/control-store" import { LocalEventingProjectors, LocalEventingRuntime, type LocalEventingRuntimeApi, -} from "../src/server/eventing/runtime" +} from "@maple/local-server/eventing/runtime" /** Runs a synchronous store or runtime step, throwing its typed failure like the old API did. */ export const run = (effect: Effect.Effect): A => Effect.runSync(effect) diff --git a/apps/cli/test/expected-outcomes.test.ts b/apps/cli/test/expected-outcomes.test.ts index a5b35362ba..a723ceb09c 100644 --- a/apps/cli/test/expected-outcomes.test.ts +++ b/apps/cli/test/expected-outcomes.test.ts @@ -6,25 +6,30 @@ import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs" import { tmpdir } from "node:os" import { join } from "node:path" import { - chdbConfigPath, dirtyStoreRecoveryAdvice, needsInitialCheckpoint, parseCheckpointInterval, - resolveChdbConfigFile, } from "../src/commands/server" -import { CheckpointPreconditionError, parseCheckpointId } from "../src/server/checkpoints" +import { + CheckpointPreconditionError, + chdbConfigPath, + parseCheckpointId, + resolveChdbConfigFile, +} from "@maple/local-server/checkpoints" import { recoverExpected } from "../src/core/outcomes" import { BackgroundServerSpawnError, BackgroundServerTimeoutError, CheckpointUnavailableError, + ServerOptionError, + ServerStopTimeoutError, +} from "../src/commands/server-errors" +import { LocalStoreDirtyError, LocalStoreIncompatibleError, LocalStoreMigrationError, LocalStoreSchemaStaleError, - ServerOptionError, - ServerStopTimeoutError, -} from "../src/commands/server-errors" +} from "@maple/local-server/store-errors" /** Run an effect, capturing anything written to stderr and restoring the real * writer + exit code afterwards. */ diff --git a/apps/cli/test/host-filesystem-utilization.test.ts b/apps/cli/test/host-filesystem-utilization.test.ts index d3125f5041..b9b3edd195 100644 --- a/apps/cli/test/host-filesystem-utilization.test.ts +++ b/apps/cli/test/host-filesystem-utilization.test.ts @@ -4,10 +4,10 @@ import { tmpdir } from "node:os" import { join } from "node:path" import * as CH from "@maple/query-engine/ch" import { Effect, Schema } from "effect" -import schemaSql from "../src/server/schema/local-schema.sql" with { type: "text" } -import { Chdb } from "../src/server/chdb" +import schemaSql from "@maple/local-server/schema/local-schema.sql" with { type: "text" } +import { Chdb } from "@maple/local-server/chdb" import { LOCAL_ORG_ID } from "../src/core/executor" -import { decodeJsonEachRow } from "../src/server/chdb-rows" +import { decodeJsonEachRow } from "@maple/local-server/chdb-rows" import { describeWithChdb } from "./libchdb-test-support" const iso = (ms: number) => new Date(ms).toISOString().replace("T", " ").replace("Z", "") diff --git a/apps/cli/test/in-memory-store.test.ts b/apps/cli/test/in-memory-store.test.ts index 3c197af63d..cb7d1d16ae 100644 --- a/apps/cli/test/in-memory-store.test.ts +++ b/apps/cli/test/in-memory-store.test.ts @@ -18,7 +18,7 @@ import { sweepStaleDiscoveryFiles, } from "../src/commands/in-memory-store" import { guardStartFailures, readStartFailureRecord } from "../src/commands/start-failures" -import { ChdbError } from "../src/server/chdb" +import { ChdbError } from "@maple/local-server/chdb" const NO_FLAGS = { background: false, diff --git a/apps/cli/test/insert-batcher.test.ts b/apps/cli/test/insert-batcher.test.ts index d2d7eba5d5..de1726e467 100644 --- a/apps/cli/test/insert-batcher.test.ts +++ b/apps/cli/test/insert-batcher.test.ts @@ -2,12 +2,12 @@ import { afterAll, beforeAll, describe, expect, test } from "bun:test" import { mkdtempSync, rmSync } from "node:fs" import { tmpdir } from "node:os" import { join } from "node:path" -import schemaSql from "../src/server/schema/local-schema.sql" with { type: "text" } +import schemaSql from "@maple/local-server/schema/local-schema.sql" with { type: "text" } import { Effect, Result } from "effect" -import { Chdb } from "../src/server/chdb" -import { type InsertBatch, InsertBatcher } from "../src/server/insert-batcher" -import { MAX_ESCAPED_PAYLOAD_BYTES } from "../src/server/inserts" -import { encodeLogs } from "../src/server/otlp/encode" +import { Chdb } from "@maple/local-server/chdb" +import { type InsertBatch, InsertBatcher } from "@maple/local-server/insert-batcher" +import { MAX_ESCAPED_PAYLOAD_BYTES } from "@maple/local-server/inserts" +import { encodeLogs } from "@maple/local-server/otlp/encode" import { describeWithChdb } from "./libchdb-test-support" const rows = (...bodies: string[]) => bodies.map((body) => JSON.stringify({ body })).join("\n") diff --git a/apps/cli/test/inserts.test.ts b/apps/cli/test/inserts.test.ts index 02abd7c9fe..31382d54c2 100644 --- a/apps/cli/test/inserts.test.ts +++ b/apps/cli/test/inserts.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it } from "vitest" -import { buildBatchedInsertStatements, buildInsertSql, buildInsertStatements } from "../src/server/inserts" +import { + buildBatchedInsertStatements, + buildInsertSql, + buildInsertStatements, +} from "@maple/local-server/inserts" const line = (i: number, pad = 0) => JSON.stringify({ body: `row-${i}${"x".repeat(pad)}` }) diff --git a/apps/cli/test/journal-schema.test.ts b/apps/cli/test/journal-schema.test.ts index f1bc07bdf4..e43a471a18 100644 --- a/apps/cli/test/journal-schema.test.ts +++ b/apps/cli/test/journal-schema.test.ts @@ -9,7 +9,7 @@ import { describe, expect, it } from "vitest" import { decodeMigrationJournal, MigrationJournalSchema, -} from "../src/server/local-store-migrations/journal-schema" +} from "@maple/local-server/local-store-migrations/journal-schema" const golden = { formatVersion: 2, diff --git a/apps/cli/test/local-eventing-consumer-auth.test.ts b/apps/cli/test/local-eventing-consumer-auth.test.ts index 24fe47feff..225377cdb9 100644 --- a/apps/cli/test/local-eventing-consumer-auth.test.ts +++ b/apps/cli/test/local-eventing-consumer-auth.test.ts @@ -8,7 +8,7 @@ import { ensureEventConsumerToken, eventConsumerTokenMatches, eventConsumerTokenPath, -} from "../src/server/eventing/consumer-auth" +} from "@maple/local-server/eventing/consumer-auth" describe("local event consumer authorization", () => { it("creates a stable private token separate from the data directory", async () => { diff --git a/apps/cli/test/local-eventing-control-store.test.ts b/apps/cli/test/local-eventing-control-store.test.ts index a05cb6d710..415141e6db 100644 --- a/apps/cli/test/local-eventing-control-store.test.ts +++ b/apps/cli/test/local-eventing-control-store.test.ts @@ -20,7 +20,7 @@ import { restoreControlSnapshot, validateControlSnapshot, writeControlSnapshot, -} from "../src/server/eventing/control-store" +} from "@maple/local-server/eventing/control-store" import { metricRecorder, openStore, run, runAsync } from "./eventing-test-support" const withDataDir = async (run: (dataDir: string) => Promise): Promise => { diff --git a/apps/cli/test/local-eventing-ingest.test.ts b/apps/cli/test/local-eventing-ingest.test.ts index f6044c5fb4..aee626623c 100644 --- a/apps/cli/test/local-eventing-ingest.test.ts +++ b/apps/cli/test/local-eventing-ingest.test.ts @@ -2,15 +2,15 @@ import { deepStrictEqual, ok, rejects, strictEqual } from "node:assert" import { gzipSync } from "node:zlib" import { Effect, Result } from "effect" import { describe, it } from "vitest" -import { EventingControlStoreError, OutboxEventRejected } from "../src/server/eventing/control-store" -import { normalizeOtlpLogs } from "../src/server/eventing/otlp" +import { EventingControlStoreError, OutboxEventRejected } from "@maple/local-server/eventing/control-store" +import { normalizeOtlpLogs } from "@maple/local-server/eventing/otlp" import { type LocalEventingRuntimeApi, ProjectionActivationConflict, SourceOccurrenceCollision, -} from "../src/server/eventing/runtime" -import { InsertBatcher } from "../src/server/insert-batcher" -import { __testables } from "../src/server/serve" +} from "@maple/local-server/eventing/runtime" +import { InsertBatcher } from "@maple/local-server/insert-batcher" +import { __testables } from "@maple/local-server/serve" import { run, serveCheckpointWith, serveWith, stubEventing } from "./eventing-test-support" describe("Local eventing ingest seam", () => { diff --git a/apps/cli/test/local-eventing-overflow.test.ts b/apps/cli/test/local-eventing-overflow.test.ts index 8ced7c090b..9cffd3a864 100644 --- a/apps/cli/test/local-eventing-overflow.test.ts +++ b/apps/cli/test/local-eventing-overflow.test.ts @@ -5,9 +5,9 @@ import { mkdtempSync, mkdirSync, rmSync } from "node:fs" import { tmpdir } from "node:os" import { join } from "node:path" import { describe, it } from "vitest" -import { RetiredDayAuthority } from "../src/server/archives/retention" -import { InsertBatcher } from "../src/server/insert-batcher" -import { __testables } from "../src/server/serve" +import { RetiredDayAuthority } from "@maple/local-server/archives/retention" +import { InsertBatcher } from "@maple/local-server/insert-batcher" +import { __testables } from "@maple/local-server/serve" import { makeRuntime, openStore, run, serveWith } from "./eventing-test-support" describe("Durable outbox overflow", () => { diff --git a/apps/cli/test/local-eventing-runtime.test.ts b/apps/cli/test/local-eventing-runtime.test.ts index fde94adb0f..525b08033a 100644 --- a/apps/cli/test/local-eventing-runtime.test.ts +++ b/apps/cli/test/local-eventing-runtime.test.ts @@ -13,9 +13,9 @@ import { type SignalProjectionSpec, type SignalScalar, } from "@maple/eventing-core" -import { normalizeOtlpLogs, normalizeOtlpLogsWithDiagnostics } from "../src/server/eventing/otlp" -import { sourceOccurrenceFingerprint } from "../src/server/eventing/runtime" -import { encodeLogs } from "../src/server/otlp/encode" +import { normalizeOtlpLogs, normalizeOtlpLogsWithDiagnostics } from "@maple/local-server/eventing/otlp" +import { sourceOccurrenceFingerprint } from "@maple/local-server/eventing/runtime" +import { encodeLogs } from "@maple/local-server/otlp/encode" import { makeRuntime, metricRecorder, openStore, run } from "./eventing-test-support" const withDataDir = async (run: (dataDir: string) => Promise): Promise => { diff --git a/apps/cli/test/local-eventing-telemetry.test.ts b/apps/cli/test/local-eventing-telemetry.test.ts index 0d0eaa3091..7977f9c814 100644 --- a/apps/cli/test/local-eventing-telemetry.test.ts +++ b/apps/cli/test/local-eventing-telemetry.test.ts @@ -2,7 +2,7 @@ import { strictEqual, ok } from "node:assert" import { describe, it } from "vitest" import { Effect, ManagedRuntime } from "effect" import { Maple } from "@maple-dev/effect-sdk/server" -import { observeEventing } from "../src/server/eventing/telemetry" +import { observeEventing } from "@maple/local-server/eventing/telemetry" describe("eventing metric export", () => { it("exports eventing counters through the CLI's server SDK layer", async () => { diff --git a/apps/cli/test/local-ingest-rules.test.ts b/apps/cli/test/local-ingest-rules.test.ts index d68a82f224..262256b7d7 100644 --- a/apps/cli/test/local-ingest-rules.test.ts +++ b/apps/cli/test/local-ingest-rules.test.ts @@ -6,8 +6,8 @@ import { applyLocalIngestRules, makeLocalIngestRulesSource, parseLocalIngestRules, -} from "../src/server/ingest-rules" -import { encodeLogs, encodeTraces } from "../src/server/otlp/encode" +} from "@maple/local-server/ingest-rules" +import { encodeLogs, encodeTraces } from "@maple/local-server/otlp/encode" const rules = parseLocalIngestRules( JSON.stringify([ diff --git a/apps/cli/test/local-query-guard.test.ts b/apps/cli/test/local-query-guard.test.ts index 87c3546b69..05f0017a92 100644 --- a/apps/cli/test/local-query-guard.test.ts +++ b/apps/cli/test/local-query-guard.test.ts @@ -3,15 +3,15 @@ import { existsSync, mkdtempSync, rmSync } from "node:fs" import { tmpdir } from "node:os" import { join } from "node:path" import { Result } from "effect" -import { Chdb, ChdbClosedError } from "../src/server/chdb" +import { Chdb, ChdbClosedError } from "@maple/local-server/chdb" import { describeWithChdb } from "./libchdb-test-support" import { countArrayRows, prepareLocalQuery, READ_ONLY_REJECTION_PREFIX, ReadOnlyQueryRejected, -} from "../src/server/query-guard" -import { __testables } from "../src/server/serve" +} from "@maple/local-server/query-guard" +import { __testables } from "@maple/local-server/serve" /** The engine's canonical form, as `formatQuerySingleLine` would print it. */ const engine = (canonical: string) => ({ query: () => canonical }) diff --git a/apps/cli/test/local-service-map-rollup.test.ts b/apps/cli/test/local-service-map-rollup.test.ts index 3c21cccdea..7345376d49 100644 --- a/apps/cli/test/local-service-map-rollup.test.ts +++ b/apps/cli/test/local-service-map-rollup.test.ts @@ -4,17 +4,17 @@ import { tmpdir } from "node:os" import { join } from "node:path" import * as CH from "@maple/query-engine/ch" import { Effect, Schema } from "effect" -import schemaSql from "../src/server/schema/local-schema.sql" with { type: "text" } -import { Chdb } from "../src/server/chdb" +import schemaSql from "@maple/local-server/schema/local-schema.sql" with { type: "text" } +import { Chdb } from "@maple/local-server/chdb" import { LOCAL_ORG_ID } from "../src/core/executor" -import { decodeJsonEachRow } from "../src/server/chdb-rows" +import { decodeJsonEachRow } from "@maple/local-server/chdb-rows" import { describeWithChdb } from "./libchdb-test-support" import { type LocalServiceMapRollupDeps, type LocalServiceMapRollupState, ROLLUP_SETTLE_MS, runLocalServiceMapRollupTick, -} from "../src/server/service-map-rollup" +} from "@maple/local-server/service-map-rollup" const HOUR_MS = 3_600_000 const iso = (ms: number) => new Date(ms).toISOString().replace("T", " ").replace("Z", "") diff --git a/apps/cli/test/local-store-migrations.test.ts b/apps/cli/test/local-store-migrations.test.ts index be08d2b081..5b4115f756 100644 --- a/apps/cli/test/local-store-migrations.test.ts +++ b/apps/cli/test/local-store-migrations.test.ts @@ -8,21 +8,12 @@ import { LOCAL_SCHEMA_MANIFEST, LOCAL_SCHEMA_V1, LOCAL_SCHEMA_V2, - LOCAL_SCHEMA_V2_MANIFEST, LOCAL_SCHEMA_V3, - LOCAL_SCHEMA_V3_MANIFEST, LOCAL_SCHEMA_V4, - LOCAL_SCHEMA_V4_MANIFEST, LOCAL_SCHEMA_V5, - LOCAL_SCHEMA_V5_MANIFEST, - LOCAL_SCHEMA_V7_MANIFEST, LOCAL_SCHEMA_V6, LOCAL_SCHEMA_V10, - LOCAL_SCHEMA_V10_MANIFEST, LOCAL_SCHEMA_V11, - LOCAL_SCHEMA_V11_MANIFEST, - LOCAL_SCHEMA_V12_MANIFEST, - LOCAL_SCHEMA_V13_MANIFEST, LOCAL_SCHEMA_V20, LOCAL_SCHEMA_V21, LOCAL_SCHEMA_V22, @@ -46,7 +37,18 @@ import { LOCAL_SCHEMA_V42, SCHEMA_DIGEST, SCHEMA_FINGERPRINT, -} from "../src/server/schema-identity" +} from "@maple/local-server/schema-identity" +import { + LOCAL_SCHEMA_V2_MANIFEST, + LOCAL_SCHEMA_V3_MANIFEST, + LOCAL_SCHEMA_V4_MANIFEST, + LOCAL_SCHEMA_V5_MANIFEST, + LOCAL_SCHEMA_V7_MANIFEST, + LOCAL_SCHEMA_V10_MANIFEST, + LOCAL_SCHEMA_V11_MANIFEST, + LOCAL_SCHEMA_V12_MANIFEST, + LOCAL_SCHEMA_V13_MANIFEST, +} from "../src/server/schema-snapshots" import { abandonLocalStoreMigration, abandonLocalStoreMigrationPreservingSource, @@ -56,34 +58,36 @@ import { identityFromMarker, legacyToCurrentModule, localStoreMigrations, - migrationJournalPath, - migrationHistoryPath, - migrationRootPath, planMigration, readMigrationJournal, - readMigrationJournalStructure, resolveMigrationChain, runLocalStoreMigration, promoteLocalStoreMigration, validateMigrationRegistry, type LocalStoreMigration, type MigrationModuleContext, - type MigrationJournal, } from "../src/server/local-store-migrations" +import { + type MigrationJournal, + migrationHistoryPath, + migrationJournalPath, + migrationRootPath, + readMigrationJournalStructure, +} from "@maple/local-server/local-store-migration-journal" import type { MigrationDb } from "../src/server/local-store-migration-module" import { comparePhysicalSchema, type LocalSchemaManifest, withRawTelemetryRetentionFloor, -} from "../src/server/schema-manifest" +} from "@maple/local-server/schema-manifest" import { ensureStoreMarkerDurable, isStoreDirty, readMarker, storeMarkerPath, storeOpenMarkerPath, -} from "../src/server/store-version" -import { durableJson } from "../src/server/durable-files" +} from "@maple/local-server/store-version" +import { durableJson } from "@maple/local-server/durable-files" import { __testables as legacyTestables, advanceDuplicateKeyProgress, @@ -93,7 +97,7 @@ import { type CopyProgress, type RawReplayProgress, } from "../src/server/local-store-migrations/legacy-to-current" -import { LOCAL_SCHEMA_HISTORY } from "../src/server/local-schema-history" +import { LOCAL_SCHEMA_HISTORY } from "@maple/local-server/local-schema-history" import { LOCAL_STORE_STEPS } from "../src/server/local-store-migrations/steps" import { stepStatements } from "../src/server/local-store-migrations/step-executor" import { mkdir, mkdtemp, rm } from "node:fs/promises" @@ -1810,13 +1814,13 @@ describe("migration journal creation is lock-serialized", () => { await writeFile( storeMarkerPath(dataDir), `${JSON.stringify({ - chdb: (await import("../src/version")).CHDB_VERSION, + chdb: (await import("@maple/local-server/version")).CHDB_VERSION, maple: "test", createdAt: "2026-08-30T00:00:00.000Z", schema: LEGACY_LOCAL_SCHEMA.fingerprint, })}\n`, ) - const { withMaintenanceLock } = await import("../src/server/checkpoints") + const { withMaintenanceLock } = await import("@maple/local-server/checkpoints") const { randomUUID } = await import("node:crypto") await withMaintenanceLock(dataDir, randomUUID(), async () => { // A concurrent migrate must fail at the lock WITHOUT having written diff --git a/apps/cli/test/local-store-service-map-rekey.test.ts b/apps/cli/test/local-store-service-map-rekey.test.ts index 0f34abf1df..1b1cbe8df5 100644 --- a/apps/cli/test/local-store-service-map-rekey.test.ts +++ b/apps/cli/test/local-store-service-map-rekey.test.ts @@ -3,10 +3,10 @@ import { mkdtemp, rm } from "node:fs/promises" import { tmpdir } from "node:os" import { join } from "node:path" import { Schema } from "effect" -import { Chdb } from "../src/server/chdb" -import { decodeJsonEachRow } from "../src/server/chdb-rows" +import { Chdb } from "@maple/local-server/chdb" +import { decodeJsonEachRow } from "@maple/local-server/chdb-rows" import type { MigrationModuleContext } from "../src/server/local-store-migration-module" -import { localSchemaSnapshot } from "../src/server/schema-identity" +import { localSchemaSnapshot } from "../src/server/schema-snapshots" import { describeWithChdb } from "./libchdb-test-support" import { migrationStep, openStore } from "./local-store-step-test-support" diff --git a/apps/cli/test/local-store-step-test-support.ts b/apps/cli/test/local-store-step-test-support.ts index d4dab9a8eb..521a396524 100644 --- a/apps/cli/test/local-store-step-test-support.ts +++ b/apps/cli/test/local-store-step-test-support.ts @@ -1,4 +1,4 @@ -import { Chdb } from "../src/server/chdb" +import { Chdb } from "@maple/local-server/chdb" import type { MigrationDbOptions, MigrationModuleContext } from "../src/server/local-store-migration-module" import { localStoreMigrations } from "../src/server/local-store-migrations" diff --git a/apps/cli/test/local-store-traces-rekey.test.ts b/apps/cli/test/local-store-traces-rekey.test.ts index cb9b5519d0..0656ff5e08 100644 --- a/apps/cli/test/local-store-traces-rekey.test.ts +++ b/apps/cli/test/local-store-traces-rekey.test.ts @@ -3,10 +3,10 @@ import { mkdtemp, rm } from "node:fs/promises" import { tmpdir } from "node:os" import { join } from "node:path" import { Schema } from "effect" -import { Chdb, configureRawTelemetryRetentionDays } from "../src/server/chdb" -import { decodeJsonEachRow } from "../src/server/chdb-rows" +import { Chdb, configureRawTelemetryRetentionDays } from "@maple/local-server/chdb" +import { decodeJsonEachRow } from "@maple/local-server/chdb-rows" import type { MigrationModuleContext } from "../src/server/local-store-migration-module" -import { localSchemaSnapshot } from "../src/server/schema-identity" +import { localSchemaSnapshot } from "../src/server/schema-snapshots" import { describeWithChdb } from "./libchdb-test-support" import { migrationStep, openStore } from "./local-store-step-test-support" diff --git a/apps/cli/test/maintenance-lock-recovery.test.ts b/apps/cli/test/maintenance-lock-recovery.test.ts index 4973aa2298..3a1d5445c9 100644 --- a/apps/cli/test/maintenance-lock-recovery.test.ts +++ b/apps/cli/test/maintenance-lock-recovery.test.ts @@ -3,7 +3,7 @@ import { randomUUID } from "node:crypto" import { existsSync, mkdirSync, mkdtempSync, rmSync, statSync, utimesSync, writeFileSync } from "node:fs" import { tmpdir } from "node:os" import { join, resolve } from "node:path" -import { withMaintenanceLock } from "../src/server/checkpoints" +import { withMaintenanceLock } from "@maple/local-server/checkpoints" const lockPathOf = (dataDir: string): string => `${resolve(dataDir)}.maple-maintenance-lock` diff --git a/apps/cli/test/maintenance-operation.test.ts b/apps/cli/test/maintenance-operation.test.ts index 3d5f8335af..982d32b817 100644 --- a/apps/cli/test/maintenance-operation.test.ts +++ b/apps/cli/test/maintenance-operation.test.ts @@ -9,7 +9,7 @@ import { ok, strictEqual } from "node:assert" import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs" import { tmpdir } from "node:os" import { join, resolve } from "node:path" -import { maintenanceOperation, withMaintenanceLock } from "../src/server/checkpoints" +import { maintenanceOperation, withMaintenanceLock } from "@maple/local-server/checkpoints" // Deliberately recomputed from the on-disk contract rather than imported: this // pins the sibling path a *different* process must find and reconcile. diff --git a/apps/cli/test/merge-injection-probe.ts b/apps/cli/test/merge-injection-probe.ts index 35f841d203..63b59565c8 100644 --- a/apps/cli/test/merge-injection-probe.ts +++ b/apps/cli/test/merge-injection-probe.ts @@ -10,9 +10,9 @@ import { rmSync, mkdirSync } from "node:fs" import { join } from "node:path" -import { Chdb } from "../src/server/chdb" -import { exportSignalShards } from "../src/server/archives/export" -import { archiveSignal } from "../src/server/archives/signals" +import { Chdb } from "@maple/local-server/chdb" +import { exportSignalShards } from "@maple/local-server/archives/export" +import { archiveSignal } from "@maple/local-server/archives/signals" const SCHEMA = ` CREATE DATABASE IF NOT EXISTS default; diff --git a/apps/cli/test/native-local-store-migration-fixture.ts b/apps/cli/test/native-local-store-migration-fixture.ts index 65383ad2d7..7f271463b6 100644 --- a/apps/cli/test/native-local-store-migration-fixture.ts +++ b/apps/cli/test/native-local-store-migration-fixture.ts @@ -4,8 +4,8 @@ // the migration probe must prove that a physical legacy source can be opened // by native chDB before the current target schema is created. -import { Chdb } from "../src/server/chdb" -import { markStoreClosedDurable, markStoreOpenDurable } from "../src/server/store-version" +import { Chdb } from "@maple/local-server/chdb" +import { markStoreClosedDurable, markStoreOpenDurable } from "@maple/local-server/store-version" const [dataDir, configFile] = process.argv.slice(2) if (!dataDir || !configFile) diff --git a/apps/cli/test/native-local-store-migration.sh b/apps/cli/test/native-local-store-migration.sh index 488a5aec20..a279765914 100755 --- a/apps/cli/test/native-local-store-migration.sh +++ b/apps/cli/test/native-local-store-migration.sh @@ -140,7 +140,7 @@ grep -q "local store migrated" "$ROOT/migrate.out" || fail "native migration did # Pinned to the CURRENT head identity on purpose — this is the gate that proves # the native migration promotes all the way to head, not merely that it ran. It # must be bumped in lockstep with LOCAL_SCHEMA_VERSION and the matching -# LOCAL_SCHEMA_V.fingerprint in apps/cli/src/server/schema-identity.ts; +# LOCAL_SCHEMA_V.fingerprint in packages/local-server/src/schema-identity.ts; # leaving it on the previous version is what makes this step fail after a bump. jq -e '.formatVersion == 2 and .activation == "active" and .schemaVersion == 42 and .schema == "d03ed074b19ce990"' \ "$ROOT/maple-store-version.json" >/dev/null || fail "native migration wrote the wrong active identity" diff --git a/apps/cli/test/native-projector-modules.ts b/apps/cli/test/native-projector-modules.ts index 94a0c285c1..f70ef40e73 100644 --- a/apps/cli/test/native-projector-modules.ts +++ b/apps/cli/test/native-projector-modules.ts @@ -14,7 +14,7 @@ import { readlinkSync, } from "node:fs" import { createHash } from "node:crypto" -import { serverPidPath, storeMarkerPath, storeOpenMarkerPath } from "../src/server/store-version" +import { serverPidPath, storeMarkerPath, storeOpenMarkerPath } from "@maple/local-server/store-version" import { tmpdir } from "node:os" import { Schema } from "effect" import { join, resolve } from "node:path" diff --git a/apps/cli/test/native-service-map-ingest-bridge-probe.ts b/apps/cli/test/native-service-map-ingest-bridge-probe.ts index 5ae1b5ca60..4c88e12952 100644 --- a/apps/cli/test/native-service-map-ingest-bridge-probe.ts +++ b/apps/cli/test/native-service-map-ingest-bridge-probe.ts @@ -1,7 +1,7 @@ // SAFETY-FILE: JSON in this test is emitted by the fixture or unit under test before its fields are asserted. -import { Chdb } from "../src/server/chdb" -import { LOCAL_SCHEMA_SQL } from "../src/server/schema-identity" -import { markStoreClosedDurable, markStoreOpenDurable } from "../src/server/store-version" +import { Chdb } from "@maple/local-server/chdb" +import { LOCAL_SCHEMA_SQL } from "@maple/local-server/schema-identity" +import { markStoreClosedDurable, markStoreOpenDurable } from "@maple/local-server/store-version" const [dataDir, configFile] = process.argv.slice(2) if (!dataDir || !configFile) diff --git a/apps/cli/test/probes/archive-crash-worker.ts b/apps/cli/test/probes/archive-crash-worker.ts index 5376510fa3..08db0a5106 100644 --- a/apps/cli/test/probes/archive-crash-worker.ts +++ b/apps/cli/test/probes/archive-crash-worker.ts @@ -23,8 +23,11 @@ import { writeFileSync, existsSync, mkdirSync } from "node:fs" import { join } from "node:path" -import { createArchiveGeneration, type ArchiveGenerationFaults } from "../../src/server/archives/generation" -import { resolveArchiveTuning } from "../../src/server/archives/config" +import { + createArchiveGeneration, + type ArchiveGenerationFaults, +} from "@maple/local-server/archives/generation" +import { resolveArchiveTuning } from "@maple/local-server/archives/config" interface Args { boundary: string diff --git a/apps/cli/test/probes/archive-gc-worker.ts b/apps/cli/test/probes/archive-gc-worker.ts index a6453cc0db..eb3bd9533c 100644 --- a/apps/cli/test/probes/archive-gc-worker.ts +++ b/apps/cli/test/probes/archive-gc-worker.ts @@ -13,7 +13,7 @@ // --scratch-root --keep [--block-ms ] import { writeFileSync, existsSync, mkdirSync } from "node:fs" -import { runArchiveGc } from "../../src/server/archives/gc" +import { runArchiveGc } from "@maple/local-server/archives/gc" interface Args { boundary: string diff --git a/apps/cli/test/probes/archive-probe-byte-heterogeneous.ts b/apps/cli/test/probes/archive-probe-byte-heterogeneous.ts index 4ab8df20b0..fe4c6492be 100644 --- a/apps/cli/test/probes/archive-probe-byte-heterogeneous.ts +++ b/apps/cli/test/probes/archive-probe-byte-heterogeneous.ts @@ -11,8 +11,8 @@ import { mkdirSync } from "node:fs" import { join } from "node:path" import { ArchiveProbe, firstRow } from "../archive-probe-helpers" -import { archiveSignal } from "../../src/server/archives/signals" -import { exportSignalShards } from "../../src/server/archives/export" +import { archiveSignal } from "@maple/local-server/archives/signals" +import { exportSignalShards } from "@maple/local-server/archives/export" const SCHEMA = ` CREATE DATABASE IF NOT EXISTS default; diff --git a/apps/cli/test/probes/archive-probe-byte-single-row.ts b/apps/cli/test/probes/archive-probe-byte-single-row.ts index fc71e66ab3..82590000cc 100644 --- a/apps/cli/test/probes/archive-probe-byte-single-row.ts +++ b/apps/cli/test/probes/archive-probe-byte-single-row.ts @@ -9,8 +9,8 @@ import { mkdirSync } from "node:fs" import { join } from "node:path" import { ArchiveProbe } from "../archive-probe-helpers" -import { archiveSignal } from "../../src/server/archives/signals" -import { exportSignalShards } from "../../src/server/archives/export" +import { archiveSignal } from "@maple/local-server/archives/signals" +import { exportSignalShards } from "@maple/local-server/archives/export" const SCHEMA = ` CREATE DATABASE IF NOT EXISTS default; diff --git a/apps/cli/test/probes/archive-probe-byte-uniform.ts b/apps/cli/test/probes/archive-probe-byte-uniform.ts index 7fdd20f54c..df5796447e 100644 --- a/apps/cli/test/probes/archive-probe-byte-uniform.ts +++ b/apps/cli/test/probes/archive-probe-byte-uniform.ts @@ -8,8 +8,8 @@ import { mkdirSync } from "node:fs" import { join } from "node:path" import { ArchiveProbe, firstRow, readRows } from "../archive-probe-helpers" -import { archiveSignal } from "../../src/server/archives/signals" -import { exportSignalShards } from "../../src/server/archives/export" +import { archiveSignal } from "@maple/local-server/archives/signals" +import { exportSignalShards } from "@maple/local-server/archives/export" const SCHEMA = ` CREATE DATABASE IF NOT EXISTS default; diff --git a/apps/cli/test/probes/archive-probe-complex-alter.ts b/apps/cli/test/probes/archive-probe-complex-alter.ts index 7bbc4d9415..a671a1978d 100644 --- a/apps/cli/test/probes/archive-probe-complex-alter.ts +++ b/apps/cli/test/probes/archive-probe-complex-alter.ts @@ -8,8 +8,8 @@ import { mkdirSync, rmSync } from "node:fs" import { join } from "node:path" import { ArchiveProbe } from "../archive-probe-helpers" -import { archiveSignal } from "../../src/server/archives/signals" -import { exportSignalShards } from "../../src/server/archives/export" +import { archiveSignal } from "@maple/local-server/archives/signals" +import { exportSignalShards } from "@maple/local-server/archives/export" const SCHEMA = ` CREATE DATABASE IF NOT EXISTS default; diff --git a/apps/cli/test/probes/archive-probe-digest-column-swap.ts b/apps/cli/test/probes/archive-probe-digest-column-swap.ts index 88495a68c7..6a26394989 100644 --- a/apps/cli/test/probes/archive-probe-digest-column-swap.ts +++ b/apps/cli/test/probes/archive-probe-digest-column-swap.ts @@ -20,8 +20,8 @@ import { mkdirSync, rmSync } from "node:fs" import { join } from "node:path" import { ArchiveProbe } from "../archive-probe-helpers" -import { archiveSignal } from "../../src/server/archives/signals" -import { exportSignalShards } from "../../src/server/archives/export" +import { archiveSignal } from "@maple/local-server/archives/signals" +import { exportSignalShards } from "@maple/local-server/archives/export" // Two Map columns of the SAME type — the precondition for the commutative // collision. (logs/traces/metrics all carry several Map(LowCardinality(String), diff --git a/apps/cli/test/probes/archive-probe-digest-dup-drop.ts b/apps/cli/test/probes/archive-probe-digest-dup-drop.ts index ba71680be0..06ecb06134 100644 --- a/apps/cli/test/probes/archive-probe-digest-dup-drop.ts +++ b/apps/cli/test/probes/archive-probe-digest-dup-drop.ts @@ -9,8 +9,8 @@ import { mkdirSync, rmSync } from "node:fs" import { join } from "node:path" import { ArchiveProbe } from "../archive-probe-helpers" -import { archiveSignal } from "../../src/server/archives/signals" -import { exportSignalShards } from "../../src/server/archives/export" +import { archiveSignal } from "@maple/local-server/archives/signals" +import { exportSignalShards } from "@maple/local-server/archives/export" const SCHEMA = ` CREATE DATABASE IF NOT EXISTS default; diff --git a/apps/cli/test/probes/archive-probe-digest-row-swap.ts b/apps/cli/test/probes/archive-probe-digest-row-swap.ts index 8011f0497c..ea74768639 100644 --- a/apps/cli/test/probes/archive-probe-digest-row-swap.ts +++ b/apps/cli/test/probes/archive-probe-digest-row-swap.ts @@ -9,8 +9,8 @@ import { mkdirSync, rmSync } from "node:fs" import { join } from "node:path" import { ArchiveProbe } from "../archive-probe-helpers" -import { archiveSignal } from "../../src/server/archives/signals" -import { exportSignalShards } from "../../src/server/archives/export" +import { archiveSignal } from "@maple/local-server/archives/signals" +import { exportSignalShards } from "@maple/local-server/archives/export" const SCHEMA = ` CREATE DATABASE IF NOT EXISTS default; diff --git a/apps/cli/test/probes/archive-probe-duckdb-oracle.ts b/apps/cli/test/probes/archive-probe-duckdb-oracle.ts index d661e293c7..1a8d7ac51c 100644 --- a/apps/cli/test/probes/archive-probe-duckdb-oracle.ts +++ b/apps/cli/test/probes/archive-probe-duckdb-oracle.ts @@ -14,8 +14,8 @@ import { mkdirSync } from "node:fs" import { join } from "node:path" import { execSync } from "node:child_process" import { ArchiveProbe, readRows } from "../archive-probe-helpers" -import { archiveSignal } from "../../src/server/archives/signals" -import { exportSignalShards } from "../../src/server/archives/export" +import { archiveSignal } from "@maple/local-server/archives/signals" +import { exportSignalShards } from "@maple/local-server/archives/export" const SCHEMA = `CREATE DATABASE IF NOT EXISTS default; CREATE TABLE IF NOT EXISTS default.traces ( diff --git a/apps/cli/test/probes/archive-probe-merge-freeze-leak.ts b/apps/cli/test/probes/archive-probe-merge-freeze-leak.ts index 1e74f0ef58..cf66b07041 100644 --- a/apps/cli/test/probes/archive-probe-merge-freeze-leak.ts +++ b/apps/cli/test/probes/archive-probe-merge-freeze-leak.ts @@ -8,8 +8,8 @@ import { mkdirSync, writeFileSync } from "node:fs" import { join } from "node:path" import { ArchiveProbe } from "../archive-probe-helpers" -import { archiveSignal } from "../../src/server/archives/signals" -import { exportSignalShards } from "../../src/server/archives/export" +import { archiveSignal } from "@maple/local-server/archives/signals" +import { exportSignalShards } from "@maple/local-server/archives/export" const SCHEMA = ` CREATE DATABASE IF NOT EXISTS default; diff --git a/apps/cli/test/probes/archive-probe-merge-injection.ts b/apps/cli/test/probes/archive-probe-merge-injection.ts index 85bb12b639..f657935466 100644 --- a/apps/cli/test/probes/archive-probe-merge-injection.ts +++ b/apps/cli/test/probes/archive-probe-merge-injection.ts @@ -8,8 +8,8 @@ import { mkdirSync } from "node:fs" import { join } from "node:path" import { ArchiveProbe, readRows } from "../archive-probe-helpers" -import { archiveSignal } from "../../src/server/archives/signals" -import { exportSignalShards } from "../../src/server/archives/export" +import { archiveSignal } from "@maple/local-server/archives/signals" +import { exportSignalShards } from "@maple/local-server/archives/export" const SCHEMA = ` CREATE DATABASE IF NOT EXISTS default; diff --git a/apps/cli/test/probes/archive-probe-mixed-hour.ts b/apps/cli/test/probes/archive-probe-mixed-hour.ts index 888be9d4e9..fdee0d2f5d 100644 --- a/apps/cli/test/probes/archive-probe-mixed-hour.ts +++ b/apps/cli/test/probes/archive-probe-mixed-hour.ts @@ -9,8 +9,8 @@ import { mkdirSync } from "node:fs" import { join } from "node:path" import { ArchiveProbe, firstRow, readRows } from "../archive-probe-helpers" -import { archiveSignal } from "../../src/server/archives/signals" -import { exportSignalShards } from "../../src/server/archives/export" +import { archiveSignal } from "@maple/local-server/archives/signals" +import { exportSignalShards } from "@maple/local-server/archives/export" const SCHEMA = ` CREATE DATABASE IF NOT EXISTS default; diff --git a/apps/cli/test/probes/archive-probe-multipart.ts b/apps/cli/test/probes/archive-probe-multipart.ts index 7559d407d9..5c38244168 100644 --- a/apps/cli/test/probes/archive-probe-multipart.ts +++ b/apps/cli/test/probes/archive-probe-multipart.ts @@ -7,8 +7,8 @@ import { mkdirSync } from "node:fs" import { join } from "node:path" import { ArchiveProbe, readRows } from "../archive-probe-helpers" -import { archiveSignal } from "../../src/server/archives/signals" -import { exportSignalShards } from "../../src/server/archives/export" +import { archiveSignal } from "@maple/local-server/archives/signals" +import { exportSignalShards } from "@maple/local-server/archives/export" const SCHEMA = ` CREATE DATABASE IF NOT EXISTS default; diff --git a/apps/cli/test/probes/archive-probe-null-digest.ts b/apps/cli/test/probes/archive-probe-null-digest.ts index 4945c3e883..505be5364f 100644 --- a/apps/cli/test/probes/archive-probe-null-digest.ts +++ b/apps/cli/test/probes/archive-probe-null-digest.ts @@ -8,8 +8,8 @@ import { mkdirSync } from "node:fs" import { join } from "node:path" import { ArchiveProbe } from "../archive-probe-helpers" -import { archiveSignal } from "../../src/server/archives/signals" -import { exportSignalShards } from "../../src/server/archives/export" +import { archiveSignal } from "@maple/local-server/archives/signals" +import { exportSignalShards } from "@maple/local-server/archives/export" const SCHEMA = ` CREATE DATABASE IF NOT EXISTS default; diff --git a/apps/cli/test/probes/archive-probe-null-flag-binding.ts b/apps/cli/test/probes/archive-probe-null-flag-binding.ts index f8989fd84f..21b6821474 100644 --- a/apps/cli/test/probes/archive-probe-null-flag-binding.ts +++ b/apps/cli/test/probes/archive-probe-null-flag-binding.ts @@ -12,8 +12,8 @@ import { mkdirSync, rmSync } from "node:fs" import { join } from "node:path" import { ArchiveProbe } from "../archive-probe-helpers" -import { archiveSignal } from "../../src/server/archives/signals" -import { exportSignalShards } from "../../src/server/archives/export" +import { archiveSignal } from "@maple/local-server/archives/signals" +import { exportSignalShards } from "@maple/local-server/archives/export" // A traces-shaped table with an added Nullable(String) column to expose the // NULL-vs-sentinel-string collision generically. diff --git a/apps/cli/test/probes/archive-probe-null-value-sensitivity.ts b/apps/cli/test/probes/archive-probe-null-value-sensitivity.ts index 70b586d023..ad06ef46a3 100644 --- a/apps/cli/test/probes/archive-probe-null-value-sensitivity.ts +++ b/apps/cli/test/probes/archive-probe-null-value-sensitivity.ts @@ -13,8 +13,8 @@ import { mkdirSync, rmSync } from "node:fs" import { join } from "node:path" import { ArchiveProbe } from "../archive-probe-helpers" -import { archiveSignal } from "../../src/server/archives/signals" -import { exportSignalShards } from "../../src/server/archives/export" +import { archiveSignal } from "@maple/local-server/archives/signals" +import { exportSignalShards } from "@maple/local-server/archives/export" // metrics_histogram carries Nullable(Float64) Min/Max, NULL when unset — the // exact production shape that defeated the round-5 digest. diff --git a/apps/cli/test/probes/archive-probe-schema-substitution.ts b/apps/cli/test/probes/archive-probe-schema-substitution.ts index 9546e46d74..7bb268d1a8 100644 --- a/apps/cli/test/probes/archive-probe-schema-substitution.ts +++ b/apps/cli/test/probes/archive-probe-schema-substitution.ts @@ -7,7 +7,7 @@ // Run: MAPLE_LIBCHDB=/libchdb.so bun apps/cli/test/probes/archive-probe-schema-substitution.ts import { ArchiveProbe } from "../archive-probe-helpers" -import { compareSchema } from "../../src/server/archives/export" +import { compareSchema } from "@maple/local-server/archives/export" const h = ArchiveProbe.create("schema-substitution") diff --git a/apps/cli/test/probes/archive-probe-timezone-bound.ts b/apps/cli/test/probes/archive-probe-timezone-bound.ts index 7b7e8fbac7..1dc27e313a 100644 --- a/apps/cli/test/probes/archive-probe-timezone-bound.ts +++ b/apps/cli/test/probes/archive-probe-timezone-bound.ts @@ -11,10 +11,10 @@ // Run: TZ=America/New_York MAPLE_LIBCHDB=/libchdb.so bun apps/cli/test/probes/archive-probe-timezone-bound.ts import { ArchiveProbe } from "../archive-probe-helpers" -import { parseArchiveGenerationManifest } from "../../src/server/archives/manifest" +import { parseArchiveGenerationManifest } from "@maple/local-server/archives/manifest" import { randomUUID } from "node:crypto" -import { CHDB_VERSION, MAPLE_VERSION } from "../../src/version" -import { SCHEMA_FINGERPRINT } from "../../src/server/schema-identity" +import { CHDB_VERSION, MAPLE_VERSION } from "@maple/local-server/version" +import { SCHEMA_FINGERPRINT } from "@maple/local-server/schema-identity" const h = ArchiveProbe.create("timezone-bound") diff --git a/apps/cli/test/probes/archive-reconcile-worker.ts b/apps/cli/test/probes/archive-reconcile-worker.ts index cf6e83494c..1fae271c5b 100644 --- a/apps/cli/test/probes/archive-reconcile-worker.ts +++ b/apps/cli/test/probes/archive-reconcile-worker.ts @@ -17,8 +17,8 @@ // 1 = reconciliation threw (ambiguous/corrupt state — surfaced to the harness) import { randomUUID } from "node:crypto" -import { withMaintenanceLock } from "../../src/server/checkpoints" -import { reconcileArchiveGeneration } from "../../src/server/archives/generation" +import { withMaintenanceLock } from "@maple/local-server/checkpoints" +import { reconcileArchiveGeneration } from "@maple/local-server/archives/generation" const get = (name: string): string => { const i = process.argv.indexOf(`--${name}`) diff --git a/apps/cli/test/probes/maintenance-sigint-worker.ts b/apps/cli/test/probes/maintenance-sigint-worker.ts index 4ce7a1f7dc..b614cb2523 100644 --- a/apps/cli/test/probes/maintenance-sigint-worker.ts +++ b/apps/cli/test/probes/maintenance-sigint-worker.ts @@ -21,7 +21,7 @@ import { BunRuntime } from "@effect/platform-bun" import { Effect } from "effect" -import { maintenanceOperation, withMaintenanceLock } from "../../src/server/checkpoints" +import { maintenanceOperation, withMaintenanceLock } from "@maple/local-server/checkpoints" const arg = (name: string): string | undefined => { const index = process.argv.indexOf(`--${name}`) diff --git a/apps/cli/test/remote-auth.test.ts b/apps/cli/test/remote-auth.test.ts index 1d01a95b06..46c2afdbd2 100644 --- a/apps/cli/test/remote-auth.test.ts +++ b/apps/cli/test/remote-auth.test.ts @@ -14,8 +14,8 @@ import { SESSION_COOKIE, SESSION_TTL_MS, sessionValid, -} from "../src/server/remote-auth" -import { __testables, isBrowserOriginAllowed, makeBrowserOriginPolicy } from "../src/server/serve" +} from "@maple/local-server/remote-auth" +import { __testables, isBrowserOriginAllowed, makeBrowserOriginPolicy } from "@maple/local-server/serve" const INGEST_KEY = "ingest-key-0123456789abcdef" const ACCESS_TOKEN = "access-token-0123456789abcdef" diff --git a/apps/cli/test/retained-leftovers.test.ts b/apps/cli/test/retained-leftovers.test.ts index 67c934902d..f37bf14557 100644 --- a/apps/cli/test/retained-leftovers.test.ts +++ b/apps/cli/test/retained-leftovers.test.ts @@ -4,8 +4,8 @@ import { randomUUID } from "node:crypto" import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs" import { tmpdir } from "node:os" import { dirname, join } from "node:path" -import { checkpointQuarantineRoot, restoreTransactionPath } from "../src/server/checkpoints" -import { migrationJournalPath, migrationRootPath } from "../src/server/local-store-migrations" +import { checkpointQuarantineRoot, restoreTransactionPath } from "@maple/local-server/checkpoints" +import { migrationJournalPath, migrationRootPath } from "@maple/local-server/local-store-migration-journal" import { listRetainedLeftovers, pruneRetainedLeftovers } from "../src/server/retained-leftovers" const withParent = async (run: (dataDir: string) => Promise): Promise => { diff --git a/apps/cli/test/scoped-delete.test.ts b/apps/cli/test/scoped-delete.test.ts index 0b7f128b89..d08785fceb 100644 --- a/apps/cli/test/scoped-delete.test.ts +++ b/apps/cli/test/scoped-delete.test.ts @@ -3,12 +3,12 @@ import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs" import { tmpdir } from "node:os" import { join } from "node:path" import { Effect, Schema } from "effect" -import schemaSql from "../src/server/schema/local-schema.sql" with { type: "text" } -import { Chdb } from "../src/server/chdb" -import { decodeJsonEachRow } from "../src/server/chdb-rows" -import { buildLocalSchemaManifest } from "../src/server/schema-manifest" -import { LOCAL_SCHEMA_MANIFEST } from "../src/server/schema-identity" -import { serviceMapRollupInserts } from "../src/server/service-map-rollup" +import schemaSql from "@maple/local-server/schema/local-schema.sql" with { type: "text" } +import { Chdb } from "@maple/local-server/chdb" +import { decodeJsonEachRow } from "@maple/local-server/chdb-rows" +import { buildLocalSchemaManifest } from "@maple/local-server/schema-manifest" +import { LOCAL_SCHEMA_MANIFEST } from "@maple/local-server/schema-identity" +import { serviceMapRollupInserts } from "@maple/local-server/service-map-rollup" import { describeWithChdb } from "./libchdb-test-support" import { pendingDeletePath, @@ -17,7 +17,7 @@ import { TABLE_DELETE_PLAN, tablePlan, validateDeletePlan, -} from "../src/server/scoped-delete" +} from "@maple/local-server/scoped-delete" describe("scoped delete plan", () => { // Fails when a table is added to the local schema without a delete strategy. diff --git a/apps/cli/test/serve-asset-headers.test.ts b/apps/cli/test/serve-asset-headers.test.ts index 6e01010bea..748b565d0e 100644 --- a/apps/cli/test/serve-asset-headers.test.ts +++ b/apps/cli/test/serve-asset-headers.test.ts @@ -1,6 +1,6 @@ import { describe, it } from "@effect/vitest" import { strictEqual } from "node:assert" -import { type AssetResolver, serveAsset } from "../src/server/serve" +import { type AssetResolver, serveAsset } from "@maple/local-server/serve" // The local server hands out the dashboard shell; it must not be frameable or run injected script. const assets: AssetResolver = (path) => diff --git a/apps/cli/test/serve-describe-thrown.test.ts b/apps/cli/test/serve-describe-thrown.test.ts index 3bae05e4d5..22b507b125 100644 --- a/apps/cli/test/serve-describe-thrown.test.ts +++ b/apps/cli/test/serve-describe-thrown.test.ts @@ -1,6 +1,6 @@ import { describe, it } from "@effect/vitest" import { strictEqual } from "node:assert" -import { describeThrown } from "../src/server/serve" +import { describeThrown } from "@maple/local-server/serve" // Production carried 94 spans reading exactly `Error: {}` at `POST /v1/traces`: // one issue, no message, no type, no stack past the span name. Every case below diff --git a/apps/cli/test/server-args.test.ts b/apps/cli/test/server-args.test.ts index ac236cea38..7475bf0f39 100644 --- a/apps/cli/test/server-args.test.ts +++ b/apps/cli/test/server-args.test.ts @@ -10,13 +10,10 @@ import { ingestedSince, storeChangedSince, mapleCommand, - parseElapsedSeconds, prettyPath, shellWord, - startedBeforeWrite, hostedDashboardUrl, hostedUiOrigin, - isProcessAlive, type DirtyStorePolicy, resolveAdvertiseHost, resolveBindHost, @@ -24,6 +21,7 @@ import { serverUrl, validateHost, } from "../src/commands/server-args" +import { isProcessAlive, parseElapsedSeconds, startedBeforeWrite } from "@maple/local-server/store-claim" describe("local server bind host", () => { it("defaults to loopback and accepts a trimmed environment override", () => { diff --git a/apps/cli/test/server-network.test.ts b/apps/cli/test/server-network.test.ts index 008623eedb..b3cca4af05 100644 --- a/apps/cli/test/server-network.test.ts +++ b/apps/cli/test/server-network.test.ts @@ -1,13 +1,13 @@ import { describe, it } from "@effect/vitest" import { deepStrictEqual, ok, rejects, strictEqual } from "node:assert" import { Effect, Exit, Result, Tracer } from "effect" -import { checkpointQueryUrl } from "../src/server/checkpoints" +import { checkpointQueryUrl } from "@maple/local-server/checkpoints" import { __testables, corsHeadersForAllowedOrigin, isBrowserOriginAllowed, makeBrowserOriginPolicy, -} from "../src/server/serve" +} from "@maple/local-server/serve" import { serverProbeUrl } from "../src/commands/server-args" const makeRecordingTracer = () => { diff --git a/apps/cli/test/server-pid.test.ts b/apps/cli/test/server-pid.test.ts index c09f3ba8f4..3373cd7e9f 100644 --- a/apps/cli/test/server-pid.test.ts +++ b/apps/cli/test/server-pid.test.ts @@ -3,7 +3,10 @@ import { Effect, Exit } from "effect" import { mkdtempSync, readFileSync, readdirSync, rmSync, utimesSync, writeFileSync } from "node:fs" import { tmpdir } from "node:os" import { join } from "node:path" -import { claimPidFileExclusive } from "../src/commands/server" +import { claimPidFileExclusive } from "@maple/local-server/store-claim" + +/** The host's refusal, carrying the live owner it was given. */ +const busy = (pid: number | undefined) => ({ busyPid: pid }) const withPidPath = async (run: (pidPath: string, root: string) => Promise): Promise => { const root = mkdtempSync(join(tmpdir(), "maple-pid-claim-")) @@ -20,7 +23,7 @@ const deadPid = (): number => Bun.spawnSync(["true"]).pid describe("start PID claim is exclusive", () => { it("writes exactly the bare PID, readable by older builds", async () => { await withPidPath(async (pidPath) => { - await Effect.runPromise(claimPidFileExclusive(pidPath)) + await Effect.runPromise(claimPidFileExclusive(pidPath, { busy })) expect(readFileSync(pidPath, "utf8")).toBe(String(process.pid)) }) }) @@ -29,9 +32,8 @@ describe("start PID claim is exclusive", () => { await withPidPath(async (pidPath) => { // The runner's parent started before this write, so it provably owns it. writeFileSync(pidPath, String(process.ppid)) - const second = await Effect.runPromiseExit(claimPidFileExclusive(pidPath)) - expect(Exit.isFailure(second)).toBe(true) - expect(JSON.stringify(second)).toContain("already running or starting") + const second = await Effect.runPromiseExit(claimPidFileExclusive(pidPath, { busy })) + expect(second).toEqual(Exit.fail({ busyPid: process.ppid })) expect(readFileSync(pidPath, "utf8")).toBe(String(process.ppid)) }) }) @@ -39,7 +41,7 @@ describe("start PID claim is exclusive", () => { it("takes over a PID file whose process is gone, leaving no debris", async () => { await withPidPath(async (pidPath, root) => { writeFileSync(pidPath, String(deadPid())) - await Effect.runPromise(claimPidFileExclusive(pidPath)) + await Effect.runPromise(claimPidFileExclusive(pidPath, { busy })) expect(readFileSync(pidPath, "utf8")).toBe(String(process.pid)) expect(readdirSync(root)).toEqual(["maple.pid"]) }) @@ -51,7 +53,7 @@ describe("start PID claim is exclusive", () => { writeFileSync(pidPath, String(process.ppid)) const longAgo = new Date("2001-01-01T00:00:00Z") utimesSync(pidPath, longAgo, longAgo) - await Effect.runPromise(claimPidFileExclusive(pidPath)) + await Effect.runPromise(claimPidFileExclusive(pidPath, { busy })) expect(readFileSync(pidPath, "utf8")).toBe(String(process.pid)) }) }) @@ -59,7 +61,7 @@ describe("start PID claim is exclusive", () => { it("hands a `start -d` parent's claim to its child", async () => { await withPidPath(async (pidPath) => { writeFileSync(pidPath, String(process.ppid)) - await Effect.runPromise(claimPidFileExclusive(pidPath, { handoverFrom: process.ppid })) + await Effect.runPromise(claimPidFileExclusive(pidPath, { busy, handoverFrom: process.ppid })) expect(readFileSync(pidPath, "utf8")).toBe(String(process.pid)) }) }) diff --git a/apps/cli/test/start-failures.test.ts b/apps/cli/test/start-failures.test.ts index 0ad60c2210..aa7061b749 100644 --- a/apps/cli/test/start-failures.test.ts +++ b/apps/cli/test/start-failures.test.ts @@ -14,14 +14,11 @@ import { startFailureFingerprint, startFailuresPath, } from "../src/commands/start-failures" -import { - BackgroundServerExitedError, - LocalStoreDirtyError, - ServerOptionError, -} from "../src/commands/server-errors" +import { BackgroundServerExitedError, ServerOptionError } from "../src/commands/server-errors" +import { LocalStoreDirtyError } from "@maple/local-server/store-errors" import { ProjectorModuleError } from "../src/server/eventing/projector-modules" -import { ServerBindError } from "../src/server/serve" -import { ChdbError, describeChdbOpenFailure } from "../src/server/chdb" +import { ServerBindError } from "@maple/local-server/serve" +import { ChdbError, describeChdbOpenFailure } from "@maple/local-server/chdb" const MINUTE = 60 * 1000 diff --git a/apps/cli/test/store-marker-schema.test.ts b/apps/cli/test/store-marker-schema.test.ts index c69b9bb188..20c7fa29c1 100644 --- a/apps/cli/test/store-marker-schema.test.ts +++ b/apps/cli/test/store-marker-schema.test.ts @@ -7,8 +7,8 @@ import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs" import { tmpdir } from "node:os" import { join } from "node:path" import { describe, expect, it } from "vitest" -import { makeStoreMarker, readMarkerState, storeMarkerPath } from "../src/server/store-version" -import { CHDB_VERSION } from "../src/version" +import { makeStoreMarker, readMarkerState, storeMarkerPath } from "@maple/local-server/store-version" +import { CHDB_VERSION } from "@maple/local-server/version" const goldenV1 = { chdb: "1.4.0", diff --git a/apps/cli/test/store-version.test.ts b/apps/cli/test/store-version.test.ts index 08c42bbc53..0d1d0e08ec 100644 --- a/apps/cli/test/store-version.test.ts +++ b/apps/cli/test/store-version.test.ts @@ -20,7 +20,7 @@ import { serverPidPath, storeMarkerPath, storeOpenMarkerPath, -} from "../src/server/store-version" +} from "@maple/local-server/store-version" // Each test gets a throwaway parent dir; the data dir is a child of it so the // markers (written beside the data dir) land in the temp tree, not on $HOME. @@ -96,7 +96,7 @@ describe("clean-shutdown sentinel", () => { seedData(dataDir) let child: ChildProcess | undefined try { - const moduleUrl = new URL("../src/server/store-version.ts", import.meta.url).href + const moduleUrl = import.meta.resolve("@maple/local-server/store-version") child = spawn( process.execPath, [ diff --git a/apps/cli/test/tenant-isolation.test.ts b/apps/cli/test/tenant-isolation.test.ts index 053f51caef..c621b44546 100644 --- a/apps/cli/test/tenant-isolation.test.ts +++ b/apps/cli/test/tenant-isolation.test.ts @@ -5,9 +5,9 @@ import { join } from "node:path" import * as CH from "@maple/query-engine/ch" import { OrgId } from "@maple/domain" import { DateTime, Effect, Schema } from "effect" -import schemaSql from "../src/server/schema/local-schema.sql" with { type: "text" } -import { Chdb } from "../src/server/chdb" -import { decodeJsonEachRow } from "../src/server/chdb-rows" +import schemaSql from "@maple/local-server/schema/local-schema.sql" with { type: "text" } +import { Chdb } from "@maple/local-server/chdb" +import { decodeJsonEachRow } from "@maple/local-server/chdb-rows" import { describeWithChdb } from "./libchdb-test-support" // Two orgs with the same telemetry shape, told apart only by OrgId and a name diff --git a/apps/landing/src/components/live/LiveLocalTerminal.astro b/apps/landing/src/components/live/LiveLocalTerminal.astro index 8b6cde1f29..fdb150d080 100644 --- a/apps/landing/src/components/live/LiveLocalTerminal.astro +++ b/apps/landing/src/components/live/LiveLocalTerminal.astro @@ -5,7 +5,7 @@ * The banner is a faithful reproduction of `startBanner()` in * `apps/cli/src/commands/server.ts`: same rows, same order, same 11-column key * gutter (`key.padEnd(11)`), and the defaults are the real ones — - * `DEFAULT_LOCAL_HOST` / `DEFAULT_LOCAL_PORT` in `apps/cli/src/lib/local-address.ts`. + * `DEFAULT_LOCAL_HOST` / `DEFAULT_LOCAL_PORT` in `packages/local-server/src/local-address.ts`. * The CLI's ANSI helpers (`apps/cli/src/lib/style.ts`) map onto tokens rather * than being restyled: amber() → --primary, green() → --chart-3, cyan() → * --chart-4, dim() → --muted-foreground. Chart tokens, not severity or service diff --git a/apps/landing/src/content/docs/iac/alchemy.md b/apps/landing/src/content/docs/iac/alchemy.md index ff18c825a6..add742f012 100644 --- a/apps/landing/src/content/docs/iac/alchemy.md +++ b/apps/landing/src/content/docs/iac/alchemy.md @@ -100,11 +100,57 @@ Send a request to the Worker and open [Traces](/docs/explore/traces) in Maple. T | `ingestKey` | `MAPLE_INGEST_KEY` (secret) | A `Maple.IngestKeys` resource (its private key is used) or a `Redacted` key | | `endpoint` | `MAPLE_ENDPOINT` | Defaults to `https://ingest.maple.dev`. Use `https://ingest.eu.maple.dev` for EU orgs | | `environment` | `MAPLE_ENVIRONMENT` | Becomes `deployment.environment.name`, e.g. your Alchemy stage | +| `local` | `MAPLE_ENDPOINT` | A `Local` resource. Under `alchemy dev` only, replaces `ingestKey` and `endpoint` | Every other option (`serviceName`, `serviceNamespace`, `dropSpanNames`, `anticipatedErrorIdentifiers`, …) is the Effect SDK's and is forwarded unchanged. See [Effect SDK on Cloudflare Workers](/docs/sdks/effect-cloudflare) for the full list. Import `Telemetry` from the `/telemetry` subpath, not the package root, so the Maple API client stays out of the Worker bundle. +## Local development + +Under `alchemy dev` your Workers can report to Maple Local instead of your Maple organization: an embedded collector and store that runs inside `alchemy dev`, needs no account or keys, and keeps everything on your machine. Declare one `Local` resource, register `localProviders()`, and pass it to every Worker's `Telemetry`: + +```typescript +// observability.ts +import { Local } from "@maple-dev/alchemy/local" + +export const Observability = Local("observability") +``` + +```typescript +// alchemy.run.ts +import { localProviders } from "@maple-dev/alchemy/local" + +export default Alchemy.Stack( + "my-app", + { providers: Layer.mergeAll(Cloudflare.providers(), localProviders()) }, + Effect.gen(function* () { + const local = yield* Observability + const api = yield* Api + return { api: api.url, dashboard: local.dashboardUrl } + }), +) +``` + +```typescript +// api.ts +}).pipe(Effect.provide(Telemetry({ serviceName: "api", local: Observability }))), +``` + +Run `alchemy dev` and open the `dashboard` output. Workers sharing one `Local` report into one store, so their traces join up across services. Outside `alchemy dev`, `Local` starts nothing and its outputs are `undefined`. + +To use the same stack for hosted deploys, also pass `ingestKey` (or `endpoint`): `Telemetry` ignores both under `alchemy dev` and binds them on deploy. `Maple.providers()` reads `MAPLE_API_KEY` when the stack loads, so either keep that key set while developing or register `Maple.providers()` only when deploying. + +| Prop | Default | Notes | +| -------------- | ---------------------------------- | ------------------------------------------------------------------------------ | +| `dataDir` | `.alchemy/maple-local//data` | One store per stack, stage and resource; it persists across `alchemy dev` runs | +| `port` | `4318` | An occupied port fails startup | +| `onDirtyStore` | `"fail"` | `"wipe"` discards a store left unclean by a crashed server and starts fresh | + +The first run downloads the chDB library Maple pins (the same release the `maple` CLI ships), verifies its checksum and caches it in `~/.maple/cache/libchdb` (about 330 MB, shared across projects). Set `MAPLE_LIBCHDB` to point at a copy you already have. Run `alchemy dev` with Bun, on macOS or Linux. + +Stopping `alchemy dev` always closes the store cleanly, even when the terminal closes or the process is killed. The store is only left unclean if the server process itself dies (out of memory, power loss). Reopening it then could crash chDB, so it is refused; set `onDirtyStore: "wipe"` to start fresh. + ## Add alerts and dashboards The same stack can declare what you watch the Worker with. Add a notification channel, an alert rule that delivers to it, and a dashboard: diff --git a/bun.lock b/bun.lock index 601860fc3d..7a57708053 100644 --- a/bun.lock +++ b/bun.lock @@ -148,10 +148,9 @@ "@maple-dev/effect-sdk": "workspace:*", "@maple/domain": "workspace:*", "@maple/eventing-core": "workspace:*", - "@maple/ingest-rules": "workspace:*", + "@maple/local-server": "workspace:*", "@maple/query-engine": "workspace:*", "effect": "catalog:effect", - "protobufjs": "^8.8.1", }, "devDependencies": { "@effect/vitest": "catalog:effect", @@ -522,8 +521,11 @@ "@maple-dev/effect-sdk": "workspace:*", }, "devDependencies": { + "@effect/platform-bun": "catalog:effect", "@effect/vitest": "catalog:effect", "@maple/domain": "workspace:*", + "@maple/local-server": "workspace:*", + "@types/bun": "^1.4.4", "@types/node": "catalog:tooling", "alchemy": "catalog:alchemy", "effect": "catalog:effect", @@ -532,9 +534,13 @@ "vitest": "catalog:", }, "peerDependencies": { + "@effect/platform-bun": "^4.0.0", "alchemy": ">=2.0.0-beta.80 <3", "effect": "^4.0.0", }, + "optionalPeers": [ + "@effect/platform-bun", + ], }, "packages/alerting-core": { "name": "@maple/alerting-core", @@ -801,6 +807,25 @@ "vitest": "catalog:", }, }, + "packages/local-server": { + "name": "@maple/local-server", + "version": "0.0.0", + "dependencies": { + "@effect/platform-bun": "catalog:effect", + "@maple/domain": "workspace:*", + "@maple/eventing-core": "workspace:*", + "@maple/ingest-rules": "workspace:*", + "@maple/query-engine": "workspace:*", + "effect": "catalog:effect", + "protobufjs": "^8.8.1", + }, + "devDependencies": { + "@types/bun": "^1.4.4", + "@types/node": "catalog:tooling", + "typescript": "catalog:tooling", + "vitest": "catalog:", + }, + }, "packages/otel-collector-maple-exporter": { "name": "otel-collector-maple-exporter", "version": "0.0.0", @@ -1566,6 +1591,8 @@ "@maple/landing": ["@maple/landing@workspace:apps/landing"], + "@maple/local-server": ["@maple/local-server@workspace:packages/local-server"], + "@maple/local-ui": ["@maple/local-ui@workspace:apps/local-ui"], "@maple/primitives": ["@maple/primitives@workspace:packages/primitives"], diff --git a/docs/eventing-extension-guide.md b/docs/eventing-extension-guide.md index 94272c08f6..1433fd437b 100644 --- a/docs/eventing-extension-guide.md +++ b/docs/eventing-extension-guide.md @@ -335,7 +335,7 @@ Recovery must complete the original durable obligation. It must not substitute whatever the current registry would produce. Maple Local implements this with its SQLite eventing control store -(`apps/cli/src/server/eventing/control-store.ts`) and the chDB commit seam. A +(`packages/local-server/src/eventing/control-store.ts`) and the chDB commit seam. A hosted implementation may use a database transaction or another durable outbox, as long as it gives the same ordering and recovery guarantees. @@ -344,13 +344,13 @@ as long as it gives the same ordering and recovery guarantees. ### Maple Local Maple Local normalizes OTLP logs (source kind `otel.log`) in -`apps/cli/src/server/eventing/otlp.ts`. +`packages/local-server/src/eventing/otlp.ts`. When those logs already carry the new fact: 1. register the projector in a `ProjectorRegistry` and pass it as the third constructor argument of `LocalEventingRuntime` - (`apps/cli/src/server/eventing/runtime.ts`). `apps/cli/src/server/serve.ts` + (`packages/local-server/src/eventing/runtime.ts`). `packages/local-server/src/serve.ts` constructs the runtime and currently passes no registry, so Local ships with no projectors installed; and 2. activate a durable projection revision through the authenticated diff --git a/docs/local-event-consumers.md b/docs/local-event-consumers.md index 9dc720d509..c8778a0e99 100644 --- a/docs/local-event-consumers.md +++ b/docs/local-event-consumers.md @@ -1,8 +1,8 @@ # Maple Local event consumer protocol Status: version 1 of the durable downstream-consumer boundary for the Maple Local event outbox. -Routes live in `apps/cli/src/server/serve.ts`; state lives in -`apps/cli/src/server/eventing/control-store.ts`. +Routes live in `packages/local-server/src/serve.ts`; state lives in +`packages/local-server/src/eventing/control-store.ts`. This protocol lets a local consumer deliver ready Maple CloudEvents without destructive reads or a second delivery database. It is transport-neutral. Maple does not pick a downstream transport, store @@ -170,8 +170,8 @@ delivery history on purpose. Ordinary inspection and acknowledgement never aband Reconcile or replay source facts separately when needed. The eventing control schema is version 1 (`LOCAL_CONTROL_SCHEMA_VERSION` in -`apps/cli/src/server/local-schema-version.ts`). Its DDL digest is recorded in -`LOCAL_CONTROL_SCHEMA_HISTORY` (`apps/cli/src/server/local-schema-history.ts`). Snapshot validation rejects staged source-backed rows +`packages/local-server/src/local-schema-version.ts`). Its DDL digest is recorded in +`LOCAL_CONTROL_SCHEMA_HISTORY` (`packages/local-server/src/local-schema-history.ts`). Snapshot validation rejects staged source-backed rows with missing or malformed fingerprints. Consumer cursors and leases are in the same SQLite backup as projection and outbox state. Consumer diff --git a/docs/local-mode.md b/docs/local-mode.md index 41fb087e40..1cd799c5c2 100644 --- a/docs/local-mode.md +++ b/docs/local-mode.md @@ -65,7 +65,7 @@ MAPLE_LOCAL_ADVERTISE_HOST=maple.home.arpa \ # Equivalent: maple start --host 0.0.0.0 --advertise-host maple.home.arpa --offline ``` -Off loopback, set two secrets (`apps/cli/src/server/remote-auth.ts`), each at +Off loopback, set two secrets (`packages/local-server/src/remote-auth.ts`), each at least 16 characters: - `MAPLE_LOCAL_INGEST_KEY` guards `/v1/*` (`Authorization: Bearer` or @@ -163,16 +163,19 @@ or remove the old PATH symlink so your shell resolves Homebrew's `maple`. There is a single binary, `maple`, compiled from **`apps/cli`** (package `@maple/cli`, Effect + Bun) with `bun build --compile`. It is both the CLI and -the server. It talks to the embedded ClickHouse engine **directly via -`bun:ffi`**, with no subprocess and no second language in front: - -| Concern | Where | How | -| -------------------- | -------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| CLI commands | `apps/cli/src/commands` | `maple services`, `traces`, `errors`, … run against **either** the local server **or** a remote workspace. `apps/cli/src/core/operations.ts` picks the path per [mode](#local-vs-remote-mode). | -| `maple start` server | `apps/cli/src/server/serve.ts` | A `Bun.serve` hosting OTLP/HTTP ingest (`POST /v1/{traces,logs,metrics}`), the query API (`POST /local/query`), and the bundled SPA, all on one port. | -| Embedded ClickHouse | `apps/cli/src/server/chdb.ts` | `dlopen`s `libchdb` via `bun:ffi` (the `chdb_*` accessor C API) and holds a single connection for the process. | -| OTLP → rows | `apps/cli/src/server/otlp/` | Decodes OTLP protobuf/JSON (protobufjs) and encodes each signal to per-table NDJSON, matching the generated `local-inserts.json` schema exactly. Ported from the production Rust encoders so row shapes can't diverge. | -| UI (SPA) | `apps/local-ui` (Vite + Foldkit) | Pages compile queries with `CH.compile(...)` and POST to `/local/query`. The same build is deployed to `local.maple.dev` (the default) **and** inlined into the binary as the `--offline` fallback (see [release bundle](#release-bundle)); it picks its query base URL from `window.location` at runtime (see [Where the UI comes from](#where-the-ui-comes-from)). | +the server; the server, store and archive code lives in the private workspace +package **`packages/local-server`** (`@maple/local-server`), which the CLI and +the `@maple-dev/alchemy` local sidecar both build on. It talks to the embedded +ClickHouse engine **directly via `bun:ffi`**, with no subprocess and no second +language in front: + +| Concern | Where | How | +| -------------------- | ------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| CLI commands | `apps/cli/src/commands` | `maple services`, `traces`, `errors`, … run against **either** the local server **or** a remote workspace. `apps/cli/src/core/operations.ts` picks the path per [mode](#local-vs-remote-mode). | +| `maple start` server | `packages/local-server/src/serve.ts` | A `Bun.serve` hosting OTLP/HTTP ingest (`POST /v1/{traces,logs,metrics}`), the query API (`POST /local/query`), and the bundled SPA, all on one port. | +| Embedded ClickHouse | `packages/local-server/src/chdb.ts` | `dlopen`s `libchdb` via `bun:ffi` (the `chdb_*` accessor C API) and holds a single connection for the process. | +| OTLP → rows | `packages/local-server/src/otlp/` | Decodes OTLP protobuf/JSON (protobufjs) and encodes each signal to per-table NDJSON, matching the generated `local-inserts.json` schema exactly. Ported from the production Rust encoders so row shapes can't diverge. | +| UI (SPA) | `apps/local-ui` (Vite + Foldkit) | Pages compile queries with `CH.compile(...)` and POST to `/local/query`. The same build is deployed to `local.maple.dev` (the default) **and** inlined into the binary as the `--offline` fallback (see [release bundle](#release-bundle)); it picks its query base URL from `window.location` at runtime (see [Where the UI comes from](#where-the-ui-comes-from)). | chDB allows exactly one connection per process and isn't safe to call concurrently. The long-lived `maple start` process owns the connection, and @@ -184,7 +187,7 @@ chDB's single-writer requirement. ### Store lifecycle & recovery The on-disk store at `~/.maple/data` is guarded by two sentinels beside it -(`apps/cli/src/server/store-version.ts`). Every per-store file lives beside the +(`packages/local-server/src/store-version.ts`). Every per-store file lives beside the data directory and is named by `dataDirSidecarPath`: a data directory named `data` keeps the plain names (`~/.maple/maple.pid`, `maple-store-open`, ...), and any other name gets a `.` prefix (`/var/lib/maple` uses @@ -271,7 +274,7 @@ killed with `kill -9` leaves its temp directory behind, and the next `service_map_edges_hourly` has no materialized view: an edge needs a Client/Producer span joined to its child Server/Consumer span. In cloud a scheduled job fills it; locally `maple start` runs the same rollup in-process -(`apps/cli/src/server/service-map-rollup.ts`), sharing its hour planning and +(`packages/local-server/src/service-map-rollup.ts`), sharing its hour planning and queries with the cloud job. The first tick runs 60 seconds after start, then every 5 minutes. Each tick seals every completed hour of the trailing 6 hours not yet in the table, once the hour has been over for 2 minutes. On the first @@ -314,7 +317,7 @@ inside the admission gate's exclusive section, so ingest, queries, the service map rollup and checkpoint backups wait for it. **Which tables.** Every table in the local schema is classified in -`TABLE_DELETE_PLAN` (`apps/cli/src/server/scoped-delete.ts`). A table either is +`TABLE_DELETE_PLAN` (`packages/local-server/src/scoped-delete.ts`). A table either is `excluded` or declares which of the three dimensions (service, namespace, env) it can filter exactly, plus how to recompute it when a request names one it lacks. `validateDeletePlan` checks the map against the bundled schema manifest @@ -445,15 +448,16 @@ seam exercises a two-edge chain, a resumed verified edge, and one final promotion. Structural identities live in the append-only history at -`apps/cli/src/server/local-schema-history.ts`. The schema gate checks the +`packages/local-server/src/local-schema-history.ts`. The schema gate checks the current identity against the history tip, preserves the base branch's prior entries in CI, and requires every historical identity to reach the current one through registered migration edges. Changing a schema digest or manifest therefore requires a new versioned entry and an executable edge together. Everything that pairing demands except the DDL is derived from the new version -number, so `bun run local-schema:bump ` writes it: the retained snapshot, -the version constant, the `schema-identity.ts` edit sites, the history entry's +number, so `bun run local-schema:bump ` writes it: the retained snapshot +(kept with the migrations in `apps/cli/src/server/schema/`), the version +constant, the `schema-snapshots.ts` and `schema-identity.ts` edit sites, the history entry's hashes, a new row in `steps.ts`, and the ids and identities pinned in `apps/cli/test/local-store-migrations.test.ts` and the native probe. It leaves the row's operations, plan line and dispositions to be written. @@ -514,7 +518,7 @@ See [external projector modules](local-projector-modules.md) for the module cont Clients POST `{ "sql": "..." }` and get back a bare JSON array of rows. -The endpoint is **read-only** (`apps/cli/src/server/query-guard.ts`). The +The endpoint is **read-only** (`packages/local-server/src/query-guard.ts`). The engine's own parser canonicalizes the text and the canonical statement is what runs. A body must be exactly one `SELECT`, `WITH`, `SHOW`, `DESCRIBE` (of a plain table), `EXISTS` or `EXPLAIN` statement; `INTO OUTFILE`, the `file()` function @@ -571,7 +575,7 @@ The dashboard SPA is a single build served two ways, and it decides which Because the remote UI auto-updates independently of the binary, keep the `/local/query` contract and the local chDB schema -([apps/cli/src/server/schema/local-schema.sql](../apps/cli/src/server/schema/local-schema.sql)) +([packages/local-server/src/schema/local-schema.sql](../packages/local-server/src/schema/local-schema.sql)) backward compatible. A newer UI may run against an older binary. `MAPLE_LOCAL_UI_URL` overrides the default UI origin (e.g. point a binary at a @@ -750,12 +754,12 @@ The build: 1. builds the workspace packages consumed through gitignored `dist/` paths; 2. builds the SPA; -3. inlines `apps/local-ui/dist` into `apps/cli/src/server/ui-embed.gen.ts`, so +3. inlines `apps/local-ui/dist` into `packages/local-server/src/ui-embed.gen.ts`, so `bun build --compile` bakes it into the binary as the `--offline` fallback (the default UI is served from `local.maple.dev`); 4. compiles `apps/cli`; -5. downloads the pinned `libchdb` (`LIBCHDB_VERSION`, checksum-verified) beside - the binary; +5. downloads the pinned `libchdb` (`LIBCHDB_VERSION`, defaulting to the release in + `packages/local-server/libchdb.json`; sha256-verified against the same manifest) beside the binary; 6. restores the committed `ui-embed.gen.ts` stub. At runtime `maple` `dlopen`s the sibling `libchdb`, resolved relative to its own diff --git a/docs/local-telemetry-archives.md b/docs/local-telemetry-archives.md index f444a7cec8..74ce2bc1fc 100644 --- a/docs/local-telemetry-archives.md +++ b/docs/local-telemetry-archives.md @@ -356,7 +356,7 @@ PRAGMA temp_directory='/Volumes/External/duckdb-spill'; ## Export tuning Archive export uses one fixed set of values (`DEFAULT_ARCHIVE_TUNING` in -`apps/cli/src/server/archives/config.ts`): one writer thread, 10 000-row row +`packages/local-server/src/archives/config.ts`): one writer thread, 10 000-row row groups, and shards of at most 500 000 rows or 256 MiB uncompressed. There is no per-deployment tuning. diff --git a/docs/profiling.md b/docs/profiling.md index 2c8c66b179..81a2f5b6ba 100644 --- a/docs/profiling.md +++ b/docs/profiling.md @@ -183,11 +183,11 @@ Everything a new signal touches, from the code as it is on `main` today: | Decode + enrich | `decode_and_enrich_payload` (6-arm match), `otlp_json::normalize` root field | `resourceProfiles` arm for protobuf + JSON; `enrich_profiles_request` writes `maple_org_id` etc. into resource attributes exactly like the others | | Flatten | new `telemetry.rs::encode_profiles` | the real work: resolve dictionary indices → frame name/file/line arrays, `cityHash64` stack, one row per `(sample, value_index)`, link → `TraceId`/`SpanId`, `routing_key = hash64(trace_id)` when linked else `hash64(profile_id)` so a span's samples land in one lane | | Pipeline | `accept_native_decoded_payload` arm → `accept_profiles_to` (copy of `accept_logs_to`) | export lanes are destination-major, so profiles ride the same WAL/lanes for free | -| Insert mapping | `scripts/generate-clickhouse-insert-mappings.ts` `OTLP_DATASOURCES` | add `"profiles"`; regenerates `apps/ingest/src/clickhouse_insert_mappings.rs` and `apps/cli/src/server/schema/local-inserts.json` | +| Insert mapping | `scripts/generate-clickhouse-insert-mappings.ts` `OTLP_DATASOURCES` | add `"profiles"`; regenerates `apps/ingest/src/clickhouse_insert_mappings.rs` and `packages/local-server/src/schema/local-inserts.json` | | BYO ClickHouse | `packages/domain/src/clickhouse/migrations/00NN_profiles.ts` + `migrations/index.ts` | new delta migration; bumps `clickHouseSchemaVersion` → ingest `SCHEMA_VERSION` gate | | Billing | `usage_metrics.rs` (`"logs" \| "traces" \| "metrics"`), `metered_enqueue`, Autumn feature ids | new feature id `profiles`; re-size `USAGE_CARDINALITY_LIMIT` (sized for 3 signals × orgs) | | Forward mode | `forward_to_collector` | forward profiles too; collectors ≥0.148 accept them | -| Local CLI | `apps/cli/src/server/serve.ts` `Signal` union, decode/encode/route, `local-schema.sql` | same table in chDB so `maple local` shows profiles too | +| Local CLI | `packages/local-server/src/serve.ts` `Signal` union, decode/encode/route, `local-schema.sql` | same table in chDB so `maple local` shows profiles too | | Self-obs | `metrics::native_rows` etc. keyed by `signal.path()` | falls out once the enum exists; add `maple.ingest.frame_count` to the encode span | Frames come from the producer as-is. eBPF/native frames arrive **unsymbolized** (addresses + diff --git a/docs/service-map-architecture.md b/docs/service-map-architecture.md index c35fdc0e02..97ebc3d445 100644 --- a/docs/service-map-architecture.md +++ b/docs/service-map-architecture.md @@ -205,7 +205,7 @@ per-org atoms, the 3D renderer and the integration panels. Maple Local storage. In local mode the `maple start` server fills `service_map_edges_hourly` itself -(`apps/cli/src/server/service-map-rollup.ts`). It shares the hour planning and +(`packages/local-server/src/service-map-rollup.ts`). It shares the hour planning and per-hour queries with `ServiceMapRollupService` (`service-map-rollup.ts` in `@maple/query-engine`) and writes the same edge and address-resolution rows with `INSERT ... SELECT` into chDB. It ticks 60 seconds after start (so exporters diff --git a/docs/signal-to-event-projection.md b/docs/signal-to-event-projection.md index 893edec320..46cff228cf 100644 --- a/docs/signal-to-event-projection.md +++ b/docs/signal-to-event-projection.md @@ -6,7 +6,7 @@ Related work: [issue #222](https://github.com/MapleTechLabs/maple/issues/222), Audience: Maple maintainers and implementers of hosted or Maple Local runtimes Status (2026-09-25): slices 1, 2, 3 and 5 shipped. `packages/eventing-core` -landed in #363, the Local runtime lives in `apps/cli/src/server/eventing`, +landed in #363, the Local runtime lives in `packages/local-server/src/eventing`, PlanetScale uses `packages/backend/src/services/integrations/planetscale/webhook-events.ts`, and alert lifecycle events come from `packages/alerting-core`. The slice 4 example @@ -602,7 +602,7 @@ field keys by JavaScript code-unit order, not locale collation, so checkpoint recovery does not depend on host locale. The control schema (`LOCAL_CONTROL_SCHEMA_VERSION = 1` in -`apps/cli/src/server/local-schema-version.ts`) includes source fingerprints. +`packages/local-server/src/local-schema-version.ts`) includes source fingerprints. Opening a store or validating a snapshot rejects staged source-backed rows with missing or malformed fingerprints. Restore verifies the control snapshot against its checkpoint manifest, validates it, and copies it through a private scratch @@ -964,7 +964,7 @@ The TypeScript reference implementation settles the remaining host choices: - Maple Local stores projection revisions, failures, and the staged/ready outbox in SQLite at `/control/eventing.sqlite`, with WAL and - `synchronous = FULL` (`apps/cli/src/server/eventing/control-store.ts`). While + `synchronous = FULL` (`packages/local-server/src/eventing/control-store.ts`). While ingest is quiesced, backup first runs and verifies a blocking `wal_checkpoint(TRUNCATE)`, so the serialized database holds every committed control-store transaction and not only the main SQLite file. A version-2 @@ -982,7 +982,7 @@ The TypeScript reference implementation settles the remaining host choices: event type and schema names; neutral fixtures use `dev.maple.example.record.observed.v1` with `urn:maple:event-schema:example-record:v1`. -- Limits (`apps/cli/src/server/eventing/otlp.ts`, `MAX_CLOUD_EVENT_BYTES` in +- Limits (`packages/local-server/src/eventing/otlp.ts`, `MAX_CLOUD_EVENT_BYTES` in eventing-core): attribute strings 16 KiB; source and event identities 256 characters (longer stable inputs become a SHA-256 URN); 256 entries per attribute namespace; nested values depth 8 and 1,024 nodes; normalized source @@ -1022,7 +1022,7 @@ Maple Local activates immutable revisions with authenticated `POST /local/eventing/projections`. The same maintenance credential protects `GET /local/eventing/projections`, `/local/eventing/health`, `/local/eventing/outbox`, and consumer administration (routes in -`apps/cli/src/server/serve.ts`). +`packages/local-server/src/serve.ts`). A ready record gets a separate, append-only readiness `sequence` on its first staged-to-ready transition. `?after=&limit=` therefore cannot skip diff --git a/examples/alchemy-maple/README.md b/examples/alchemy-maple/README.md index 3e190b03fd..18548a99b7 100644 --- a/examples/alchemy-maple/README.md +++ b/examples/alchemy-maple/README.md @@ -3,10 +3,11 @@ A Cloudflare Worker and the Maple resources that observe it, in one stack — via [`@maple-dev/alchemy`](../../packages/alchemy-maple) and [`@maple-dev/effect-sdk`](../../packages/effect-sdk). -| | | -| ---------------------------------- | -------------------------------------------------------------------------- | -| [`src/Api.ts`](src/Api.ts) | The Worker: Alchemy's Effect-native style, instrumented with the Maple SDK | -| [`alchemy.run.ts`](alchemy.run.ts) | A PagerDuty destination, two alert rules, a dashboard, a scoped API key | +| | | +| -------------------------------------- | -------------------------------------------------------------------------- | +| [`src/Api.ts`](src/Api.ts) | The Worker: Alchemy's Effect-native style, instrumented with the Maple SDK | +| [`alchemy.run.ts`](alchemy.run.ts) | A PagerDuty destination, two alert rules, a dashboard, a scoped API key | +| [`local.alchemy.ts`](local.alchemy.ts) | Two Workers exporting to one Maple Local server under `alchemy dev` | ## The seam @@ -48,3 +49,32 @@ MAPLE_API_KEY=maple_ak_… bun alchemy destroy Hit the Worker's URL a few times, then look for the `checkout` service in Maple: one span per request, plus the handler's log line, both drained through `ctx.waitUntil` after the response. + +## Local runtime + +`local.alchemy.ts` runs two Workers with distinct service names sharing +`Local("observability")` through `Telemetry({ local: Observability })`. Its independent +`localProviders()` needs no Maple or Cloudflare deployment credentials for local dev. + +From the repository root: + +```bash +bun install --frozen-lockfile +bun run --cwd packages/effect-sdk build +bun run --cwd packages/alchemy-maple build +``` + +Then, from this example directory: + +```bash +bun alchemy dev --config local.alchemy.ts +``` + +The first run downloads and verifies the pinned chDB library into `~/.maple/cache/libchdb`; +set `MAPLE_LIBCHDB` to use a copy you already have. The `Local` resource, its storage, port +and unclean-shutdown behavior are documented in +[`@maple-dev/alchemy`'s README](../../packages/alchemy-maple/README.md#local-development). + +This local-only example has no hosted ingest key. For hosted deploys, also pass `ingestKey` to +`Telemetry`; it is resolved only outside local dev. `Maple.providers()` reads `MAPLE_API_KEY` +when the stack loads, so register it only where you deploy. diff --git a/examples/alchemy-maple/local.alchemy.ts b/examples/alchemy-maple/local.alchemy.ts new file mode 100644 index 0000000000..fd2bc5aec2 --- /dev/null +++ b/examples/alchemy-maple/local.alchemy.ts @@ -0,0 +1,21 @@ +import * as Alchemy from "alchemy" +import * as Cloudflare from "@alchemy.run/cloudflare" +import { localProviders } from "@maple-dev/alchemy/local" +import { Effect, Layer } from "effect" +import Api from "./src/LocalApi.ts" +import Jobs from "./src/LocalJobs.ts" +import { Observability } from "./src/Observability.ts" + +export default Alchemy.Stack( + "maple-local-example", + { + state: Alchemy.localState(), + providers: Layer.mergeAll(Cloudflare.providers(), localProviders()), + }, + Effect.gen(function* () { + const local = yield* Observability + const api = yield* Api + const jobs = yield* Jobs + return { api: api.url, jobs: jobs.url, endpoint: local.endpoint, dashboard: local.dashboardUrl } + }), +) diff --git a/examples/alchemy-maple/src/LocalApi.ts b/examples/alchemy-maple/src/LocalApi.ts new file mode 100644 index 0000000000..0b51ead1ca --- /dev/null +++ b/examples/alchemy-maple/src/LocalApi.ts @@ -0,0 +1,18 @@ +import * as Cloudflare from "@alchemy.run/cloudflare" +import { Telemetry } from "@maple-dev/alchemy/telemetry" +import { Effect } from "effect" +import * as HttpServerResponse from "effect/http/HttpServerResponse" +import { Observability } from "./Observability.ts" + +export default Cloudflare.Worker( + "local-api", + { main: import.meta.url }, + Effect.succeed({ + fetch: Effect.gen(function* () { + yield* Effect.log("handling local API request") + return HttpServerResponse.text("api") + }), + // Worker init is the composition root: Telemetry registers the per-request layer here. + // oxlint-disable-next-line effecttsgo/strict-effect-provide + }).pipe(Effect.provide(Telemetry({ serviceName: "local-api", local: Observability }))), +) diff --git a/examples/alchemy-maple/src/LocalJobs.ts b/examples/alchemy-maple/src/LocalJobs.ts new file mode 100644 index 0000000000..272dd11ae0 --- /dev/null +++ b/examples/alchemy-maple/src/LocalJobs.ts @@ -0,0 +1,18 @@ +import * as Cloudflare from "@alchemy.run/cloudflare" +import { Telemetry } from "@maple-dev/alchemy/telemetry" +import { Effect } from "effect" +import * as HttpServerResponse from "effect/http/HttpServerResponse" +import { Observability } from "./Observability.ts" + +export default Cloudflare.Worker( + "local-jobs", + { main: import.meta.url }, + Effect.succeed({ + fetch: Effect.gen(function* () { + yield* Effect.log("handling local jobs request") + return HttpServerResponse.text("jobs") + }), + // Worker init is the composition root: Telemetry registers the per-request layer here. + // oxlint-disable-next-line effecttsgo/strict-effect-provide + }).pipe(Effect.provide(Telemetry({ serviceName: "local-jobs", local: Observability }))), +) diff --git a/examples/alchemy-maple/src/Observability.ts b/examples/alchemy-maple/src/Observability.ts new file mode 100644 index 0000000000..3a88ba2ac7 --- /dev/null +++ b/examples/alchemy-maple/src/Observability.ts @@ -0,0 +1,4 @@ +import { Local } from "@maple-dev/alchemy/local" + +// One declaration, shared by both Workers. Storage is isolated under .alchemy. +export const Observability = Local("observability") diff --git a/packages/alchemy-maple/README.md b/packages/alchemy-maple/README.md index fb8c0fe60f..b608cd9339 100644 --- a/packages/alchemy-maple/README.md +++ b/packages/alchemy-maple/README.md @@ -196,6 +196,49 @@ Cloudflare options, forwarded unchanged. Full docs: [maple.dev/docs/iac/alchemy](https://maple.dev/docs/iac/alchemy). +## Local development + +`Local` runs Maple Local inside `alchemy dev`: an embedded OTLP collector, query API and +chDB store that every Worker in the stack exports to, with no Maple account or keys. Declare it +once and pass it to each Worker's `Telemetry`: + +```typescript +// observability.ts +import { Local } from "@maple-dev/alchemy/local" +export const Observability = Local("observability") + +// alchemy.run.ts +import { localProviders } from "@maple-dev/alchemy/local" +// providers: Layer.mergeAll(Cloudflare.providers(), localProviders()) + +// api.ts (and every other Worker) +}).pipe(Effect.provide(Telemetry({ serviceName: "api", local: Observability }))), +``` + +Open the dashboard at the `dashboardUrl` output (`https://local.maple.dev`, reading from your +machine); ingest and queries never leave it. + +- **Dev only.** Under `alchemy dev` the Workers export keylessly to the local endpoint and + `ingestKey`/`endpoint` are ignored. On deploy `Local` starts nothing, its outputs are + `undefined`, and `Telemetry` uses `ingestKey` and `endpoint` as usual, so one stack can keep both. + `localProviders()` needs no Maple credentials. +- **One server per stack.** Every Worker that passes the same `Local` shares it; the server + survives stack hot reloads and closes its store when `alchemy dev` exits. +- **Storage.** Data persists per stack, stage and resource under + `.alchemy/maple-local//data`. Set `dataDir` to keep it elsewhere and `port` (default + `4318`) to move the listener; an occupied port fails startup rather than picking another. +- **libchdb.** On first use the sidecar downloads the chDB release Maple pins (the same one the + `maple` CLI ships with), verifies its sha256, and caches it in `~/.maple/cache/libchdb` + (~330 MB, shared by every project). Set `MAPLE_LIBCHDB` to use a library you already have; a + different chDB release is refused before the store is touched. +- **Unclean shutdowns.** Ending `alchemy dev` any way it can be ended (Ctrl-C, a closed terminal, + even `kill -9` of `alchemy dev` itself) closes the store first. Only the sidecar process dying + outright (out of memory, `kill -9` on the sidecar, power loss) leaves it unclean, and + reopening such a store can crash chDB, so it is refused. Set `onDirtyStore: "wipe"` on `Local` + to discard its data and start fresh; that change restarts the server and is safe to leave set. +- **Requirements.** Bun (the sidecar uses Bun's FFI), on macOS or Linux, x64 or arm64. + Periodic checkpoints are a `maple start` feature; this store keeps none. + ## Notes - Alert rules reserve one of Maple's 20 tag slots for an `alchemy:` ownership tag, leaving up to 19 user tags. Keep the ownership tag intact. To take over an existing rule deliberately, wrap the resource in Alchemy's `adopt(true)` or deploy with `--adopt`. Use distinct rule names for different stages in the same Maple organization. diff --git a/packages/alchemy-maple/package.json b/packages/alchemy-maple/package.json index 4a5581ac5a..73f9a94c4f 100644 --- a/packages/alchemy-maple/package.json +++ b/packages/alchemy-maple/package.json @@ -33,6 +33,14 @@ "./telemetry": { "types": "./dist/telemetry.d.mts", "import": "./dist/telemetry.mjs" + }, + "./local": { + "types": "./dist/local.d.mts", + "import": "./dist/local.mjs" + }, + "./local-sidecar": { + "types": "./dist/local-sidecar.d.mts", + "import": "./dist/local-sidecar.mjs" } }, "publishConfig": { @@ -48,8 +56,11 @@ "@maple-dev/effect-sdk": "workspace:*" }, "devDependencies": { + "@effect/platform-bun": "catalog:effect", "@effect/vitest": "catalog:effect", "@maple/domain": "workspace:*", + "@maple/local-server": "workspace:*", + "@types/bun": "^1.4.4", "@types/node": "catalog:tooling", "alchemy": "catalog:alchemy", "effect": "catalog:effect", @@ -58,7 +69,13 @@ "vitest": "catalog:" }, "peerDependencies": { + "@effect/platform-bun": "^4.0.0", "alchemy": ">=2.0.0-beta.80 <3", "effect": "^4.0.0" + }, + "peerDependenciesMeta": { + "@effect/platform-bun": { + "optional": true + } } } diff --git a/packages/alchemy-maple/src/Local.ts b/packages/alchemy-maple/src/Local.ts new file mode 100644 index 0000000000..50d7a15d84 --- /dev/null +++ b/packages/alchemy-maple/src/Local.ts @@ -0,0 +1,67 @@ +import type { AlchemyContext } from "alchemy/AlchemyContext" +import type { PropsInput } from "alchemy/Input" +import * as LocalProvider from "alchemy/Local/LocalProvider" +import * as ProviderLayer from "alchemy/Local/ProviderLayer" +import * as Provider from "alchemy/Provider" +import { Resource } from "alchemy/Resource" +import * as Effect from "effect/Effect" +import type { LocalConfig } from "./LocalConfig" + +export interface LocalProps { + /** Persistent store override. Defaults to an identity-specific directory inside `.alchemy`. */ + readonly dataDir?: string | undefined + /** Loopback listener port. Defaults to 4318; an occupied port fails startup. */ + readonly port?: number | undefined + /** + * A store whose sidecar was killed before closing it (OOM, `kill -9`, power + * loss) is refused by default. `"wipe"` discards its live data and starts + * fresh instead. Clean `alchemy dev` exits, including a closed terminal or a + * killed `alchemy dev`, always close the store first. + */ + readonly onDirtyStore?: "fail" | "wipe" | undefined +} + +export interface LocalAttributes { + readonly endpoint: string | undefined + readonly dashboardUrl: string | undefined + readonly dataDir: string | undefined +} + +/** A shared, scoped Maple Local server during `alchemy dev`; unavailable on deploy. */ +export type Local = Resource<"Maple.Local", LocalProps, LocalAttributes> +const LocalResource = Resource("Maple.Local") +export const Local = Object.assign( + (id: string, props: PropsInput = {}) => LocalResource(id, props), + LocalResource, +) + +export type LocalSpec = Effect.Effect< + LocalProvider.LocalProviderSpec, + never, + AlchemyContext +> + +export const LocalProviderLive = () => + Provider.succeed(Local, { + diff: () => Effect.succeed({ action: "noop" as const }), + reconcile: () => Effect.succeed({ endpoint: undefined, dashboardUrl: undefined, dataDir: undefined }), + delete: () => Effect.void, + list: () => Effect.succeed([]), + }) + +// Resolve the package entry rather than a source-relative filename: the sidecar +// is emitted separately and must also resolve from a bundled provider chunk. +export const LocalProviderLocal = () => { + const sidecar = import.meta.resolve("@maple-dev/alchemy/local-sidecar") + return LocalProvider.make( + Local, + sidecar, + Effect.promise(() => import(sidecar) as Promise<{ localSpec: LocalSpec }>).pipe( + Effect.flatMap((module) => module.localSpec), + ), + ) +} + +/** Register local dev and no-op deploy providers without Maple cloud credentials. */ +export const localProviders = () => + ProviderLayer.dual(Local, { live: LocalProviderLive, local: LocalProviderLocal }) diff --git a/packages/alchemy-maple/src/LocalConfig.ts b/packages/alchemy-maple/src/LocalConfig.ts new file mode 100644 index 0000000000..aa2c5da3ea --- /dev/null +++ b/packages/alchemy-maple/src/LocalConfig.ts @@ -0,0 +1,31 @@ +import { createHash } from "node:crypto" +import { resolve } from "node:path" +import type { LocalProps } from "./Local" + +export interface LocalConfig { + readonly dataDir: string + readonly port: number + readonly onDirtyStore: "fail" | "wipe" +} + +export const localConfig = ( + props: LocalProps, + identity: { + readonly dotAlchemy: string + readonly stack: string + readonly stage: string + readonly fqn: string + }, +): LocalConfig => { + const key = createHash("sha256") + .update(JSON.stringify([identity.stack, identity.stage, identity.fqn])) + .digest("hex") + return { + dataDir: + props.dataDir !== undefined + ? resolve(props.dataDir) + : resolve(identity.dotAlchemy, "maple-local", key, "data"), + port: props.port ?? 4318, + onDirtyStore: props.onDirtyStore ?? "fail", + } +} diff --git a/packages/alchemy-maple/src/LocalSidecar.ts b/packages/alchemy-maple/src/LocalSidecar.ts new file mode 100644 index 0000000000..83ed73086b --- /dev/null +++ b/packages/alchemy-maple/src/LocalSidecar.ts @@ -0,0 +1,44 @@ +// Infrastructure-only entry. The Worker-safe telemetry module never imports it. +import { startLocal } from "@maple/local-server" +import { provisionLibchdb } from "@maple/local-server/libchdb-provision" +import { AlchemyContext } from "alchemy/AlchemyContext" +import type * as LocalProvider from "alchemy/Local/LocalProvider" +import type * as RpcServer from "alchemy/Local/RpcServer" +import { Stack } from "alchemy/Stack" +import * as Effect from "effect/Effect" +import { FetchHttpClient } from "effect/http" +import { LocalProviderLocal, type Local, type LocalSpec } from "./Local" +import { localConfig, type LocalConfig } from "./LocalConfig" + +export const localSpec: LocalSpec = Effect.gen(function* () { + const context = yield* AlchemyContext + return { + resolveConfig: Effect.fn(function* ({ news, fqn }) { + const stack = yield* Stack + return localConfig(news, { + dotAlchemy: context.dotAlchemy, + stack: stack.name, + stage: stack.stage, + fqn, + }) + }), + start: Effect.fn(function* ({ config }) { + // MAPLE_LIBCHDB wins; otherwise the pinned release, downloaded and verified + // once into ~/.maple/cache, exactly as the `maple` release bundle gets it. + const libchdb = + process.env.MAPLE_LIBCHDB || + // The sidecar process's entry point: it owns its HTTP transport. + // oxlint-disable-next-line effecttsgo/strict-effect-provide + (yield* provisionLibchdb().pipe(Effect.provide(FetchHttpClient.layer))).path + const local = yield* startLocal({ ...config, libchdb }) + return { + endpoint: local.endpoint, + dashboardUrl: local.dashboardUrl, + dataDir: local.dataDir, + } + }), + } satisfies LocalProvider.LocalProviderSpec +}) + +const providers: RpcServer.ProviderLayer = LocalProviderLocal() +export default providers diff --git a/packages/alchemy-maple/src/Telemetry.ts b/packages/alchemy-maple/src/Telemetry.ts index bef7b0620b..080c1c208f 100644 --- a/packages/alchemy-maple/src/Telemetry.ts +++ b/packages/alchemy-maple/src/Telemetry.ts @@ -25,13 +25,16 @@ * when `endpoint` is a local `maple start` or your own collector. */ import * as MapleCloudflareSDK from "@maple-dev/effect-sdk/cloudflare" +import { AlchemyContext } from "alchemy/AlchemyContext" import * as Output from "alchemy/Output" import { CurrentRuntimeContext, unpackEnvValue } from "alchemy/RuntimeContext" import * as AlchemyTelemetry from "alchemy/Telemetry" import * as Effect from "effect/Effect" import * as Layer from "effect/Layer" +import * as Option from "effect/Option" import * as Redacted from "effect/Redacted" import type { IngestKeys } from "./IngestKeys" +import type { Local } from "./Local" // Alchemy's bundler folds this to `true` in a deployed bundle. Alchemy declares // it in a module this file does not import, so the declaration is repeated @@ -50,6 +53,8 @@ export type ResourceInput = T | Effect.Effect export type TelemetrySdkOptions = Omit export interface TelemetryProps extends TelemetrySdkOptions { + /** Shared Maple Local resource used only during `alchemy dev`; hosted settings remain deploy-only. */ + readonly local?: ResourceInput | undefined /** * Bound onto the host as the `MAPLE_INGEST_KEY` secret: the org's * {@link IngestKeys} resource (its private key is used), or a key in hand. @@ -110,7 +115,7 @@ const sdkRequestLayer = (telemetry: MapleCloudflareSDK.Telemetry) => ) export const Telemetry = (props: TelemetryProps): Layer.Layer => { - const { ingestKey, endpoint, environment, ...config } = props + const { local, ingestKey, endpoint, environment, ...config } = props const telemetry = MapleCloudflareSDK.make(config) const bind = Layer.effectDiscard( Effect.gen(function* () { @@ -118,8 +123,16 @@ export const Telemetry = (props: TelemetryProps): Layer.Layer => { // Plan-time only: in the deployed bundle the bound values are already // in the env, and the SDK reads them from there. if (rc === undefined || globalThis.__ALCHEMY_RUNTIME__) return - if (ingestKey !== undefined) yield* rc.set("MAPLE_INGEST_KEY", yield* ingestKeyOutput(ingestKey)) - if (endpoint !== undefined) yield* rc.set("MAPLE_ENDPOINT", Output.asOutput(endpoint)) + const context = yield* Effect.serviceOption(AlchemyContext) + if (local !== undefined && Option.isSome(context) && context.value.dev) { + // Local dev exports keylessly to the shared Maple Local server. + const server = Effect.isEffect(local) ? yield* local : local + yield* rc.set("MAPLE_ENDPOINT", Output.asOutput(server.endpoint)) + } else { + if (ingestKey !== undefined) + yield* rc.set("MAPLE_INGEST_KEY", yield* ingestKeyOutput(ingestKey)) + if (endpoint !== undefined) yield* rc.set("MAPLE_ENDPOINT", Output.asOutput(endpoint)) + } if (environment !== undefined) yield* rc.set("MAPLE_ENVIRONMENT", Output.asOutput(environment)) }), ) diff --git a/packages/alchemy-maple/test/local.test.ts b/packages/alchemy-maple/test/local.test.ts new file mode 100644 index 0000000000..93575a9049 --- /dev/null +++ b/packages/alchemy-maple/test/local.test.ts @@ -0,0 +1,95 @@ +import { describe, expect, it } from "@effect/vitest" +import { AlchemyContext } from "alchemy/AlchemyContext" +import { ArtifactStore, createArtifactStore } from "alchemy/Artifacts" +import { Stack } from "alchemy/Stack" +import { Provider } from "alchemy/Provider" +import * as Core from "alchemy/Test/Core" +import * as State from "alchemy/State" +import { ConfigProvider, Effect } from "effect" +import { resolve } from "node:path" +import { Local, localProviders } from "../src/Local" +import { localConfig } from "../src/LocalConfig" + +const identity = { dotAlchemy: "/project/.alchemy", stack: "app", stage: "dev", fqn: "observability" } + +describe("Maple.Local configuration", () => { + it("uses stable isolated storage per project, stack, stage and namespace", () => { + const config = localConfig({}, identity) + expect(config).toEqual(localConfig({}, identity)) + expect(config.port).toBe(4318) + expect(config.onDirtyStore).toBe("fail") + expect(config.dataDir.startsWith("/project/.alchemy/maple-local/")).toBe(true) + for (const changed of [ + { dotAlchemy: "/another-project/.alchemy" }, + { stack: "another-app" }, + { stage: "preview" }, + { fqn: "namespace/observability" }, + ]) { + expect(localConfig({}, { ...identity, ...changed }).dataDir).not.toBe(config.dataDir) + } + }) + + it("resolves explicit storage, port and dirty-store policy", () => { + expect(localConfig({ dataDir: "./local-maple", port: 4320, onDirtyStore: "wipe" }, identity)).toEqual( + { + dataDir: resolve("./local-maple"), + port: 4320, + onDirtyStore: "wipe", + }, + ) + }) +}) + +describe("Maple.Local providers", () => { + it.effect("registers independently of cloud auth and lazily selects local mode", () => + Effect.gen(function* () { + const provider = yield* Provider(Local.Type) + expect(provider.mode).toBe("local") + // Registration must not resolve a runtime, open a store or acquire cloud credentials. + }).pipe( + Effect.provide(localProviders()), + Effect.provideService(AlchemyContext, { ...identity, dev: true, adopt: false }), + Effect.provideService(Stack, { + name: "test", + stage: "test", + resources: {}, + bindings: {}, + actions: {}, + }), + Effect.provideServiceEffect(ArtifactStore, Effect.sync(createArtifactStore)), + Effect.provide(ConfigProvider.layer(ConfigProvider.fromUnknown({}))), + Effect.scoped, + ), + ) + + it.live("shares one declaration and returns unavailable local outputs on deploy", () => { + const options = { + providers: localProviders(), + state: State.inMemoryState(), + stage: "test", + adopt: false, + dev: false, + } + const stack = Core.scratchStack(options, "maple-local-live") + const shared = Local("observability") + return Core.toEffect( + Effect.gen(function* () { + const outputs = yield* stack.deploy( + Effect.gen(function* () { + const first = yield* shared + const second = yield* shared + expect(first).toBe(second) + return { + endpoint: first.endpoint, + dashboardUrl: first.dashboardUrl, + dataDir: first.dataDir, + } + }), + ) + expect(outputs).toEqual({ endpoint: undefined, dashboardUrl: undefined, dataDir: undefined }) + yield* stack.destroy() + }), + options, + ) + }) +}) diff --git a/packages/alchemy-maple/test/telemetry.test.ts b/packages/alchemy-maple/test/telemetry.test.ts index a614d89dab..93f8944a32 100644 --- a/packages/alchemy-maple/test/telemetry.test.ts +++ b/packages/alchemy-maple/test/telemetry.test.ts @@ -2,6 +2,7 @@ import { assert, describe, it } from "@effect/vitest" import * as MapleCloudflareSDK from "@maple-dev/effect-sdk/cloudflare" import { Effect, Exit, Layer, Redacted, Scope } from "effect" import * as Output from "alchemy/Output" +import { AlchemyContext } from "alchemy/AlchemyContext" import { type BaseRuntimeContext, packEnvValue, @@ -9,6 +10,7 @@ import { RuntimeContext, } from "alchemy/RuntimeContext" import { Telemetry, unpackBoundEnv } from "../src/Telemetry" +import { Local } from "../src/Local" /** A recording host: what the layer binds onto it at plan time. */ const host = () => { @@ -60,6 +62,51 @@ describe("Maple.Telemetry", () => { assert.isDefined(context.telemetry) }).pipe(Effect.scoped), ) + + it.effect("binds one shared local resource for two hosts without evaluating the cloud key", () => + Effect.gen(function* () { + const shared = yield* Local.ref("observability") + for (const serviceName of ["api", "jobs"]) { + const { bound, context } = host() + yield* Layer.build( + Telemetry({ + serviceName, + local: shared, + ingestKey: Effect.die("cloud ingest keys must not be evaluated in dev"), + endpoint: "https://ingest.maple.dev", + }), + ).pipe( + Effect.provideService(RuntimeContext, context), + Effect.provideService(AlchemyContext, { + dotAlchemy: ".alchemy", + dev: true, + adopt: false, + }), + ) + assert.deepStrictEqual([...bound.keys()], ["MAPLE_ENDPOINT"]) + assert.isTrue(Output.isOutput(bound.get("MAPLE_ENDPOINT"))) + assert.isDefined(context.telemetry) + } + }).pipe(Effect.scoped), + ) + + it.effect("does not evaluate the local declaration on deploy and retains hosted bindings", () => + Effect.gen(function* () { + const { bound, context } = host() + yield* Layer.build( + Telemetry({ + serviceName: "api", + local: Effect.die("local must not be evaluated on deploy"), + ingestKey: Redacted.make("maple_sk_deployed"), + endpoint: "https://ingest.maple.dev", + }), + ).pipe( + Effect.provideService(RuntimeContext, context), + Effect.provideService(AlchemyContext, { dotAlchemy: ".alchemy", dev: false, adopt: false }), + ) + assert.deepStrictEqual([...bound.keys()].sort(), ["MAPLE_ENDPOINT", "MAPLE_INGEST_KEY"]) + }).pipe(Effect.scoped), + ) }) describe("unpackBoundEnv", () => { @@ -87,10 +134,13 @@ describe("Maple.Telemetry at runtime", () => { Effect.gen(function* () { const authorizations: Array = [] const realFetch = globalThis.fetch - globalThis.fetch = async (input, init) => { - authorizations.push(new Headers(init?.headers).get("authorization")) - return new Response(null, { status: 200 }) - } + globalThis.fetch = Object.assign( + async (_input: string | Request | URL, init?: RequestInit) => { + authorizations.push(new Headers(init?.headers).get("authorization")) + return new Response(null, { status: 200 }) + }, + { preconnect: realFetch.preconnect }, + ) yield* Effect.addFinalizer(() => Effect.sync(() => { globalThis.fetch = realFetch diff --git a/packages/alchemy-maple/tsconfig.json b/packages/alchemy-maple/tsconfig.json index a9596d622c..ace3bada50 100644 --- a/packages/alchemy-maple/tsconfig.json +++ b/packages/alchemy-maple/tsconfig.json @@ -3,8 +3,8 @@ "compilerOptions": { "target": "ES2022", "module": "ESNext", - "lib": ["ES2022", "DOM"], - "types": ["node"], + "lib": ["ES2024"], + "types": ["bun", "node"], "moduleResolution": "bundler", "verbatimModuleSyntax": true, "noEmit": true, diff --git a/packages/alchemy-maple/tsdown.config.ts b/packages/alchemy-maple/tsdown.config.ts index 6dc697e3fd..6cb8001951 100644 --- a/packages/alchemy-maple/tsdown.config.ts +++ b/packages/alchemy-maple/tsdown.config.ts @@ -1,16 +1,28 @@ import { defineConfig } from "tsdown" +import libchdb from "../local-server/libchdb.json" with { type: "json" } + +// The chDB release the sidecar pairs with: the manifest scripts/build-local-binary.sh +// also reads. The runtime refuses to start against any other libchdb. +const libchdbVersion = process.env.LIBCHDB_VERSION ?? libchdb.version export default defineConfig({ entry: { index: "./src/index.ts", telemetry: "./src/Telemetry.ts", + local: "./src/Local.ts", + "local-sidecar": "./src/LocalSidecar.ts", }, format: "esm", dts: { tsconfig: "../../tsconfig.alchemy-maple.json" }, outDir: "dist", + // The sidecar bundles @maple/local-server (private, never published); its + // schema DDL is imported as text. Bun built-ins and every @effect/* package + // stay external: they must be the same versions as the user's `effect`. + loader: { ".sql": "text" }, + define: { __CHDB_VERSION__: JSON.stringify(libchdbVersion) }, deps: { - neverBundle: ["effect", "alchemy", "@maple-dev/effect-sdk"], + neverBundle: ["effect", /^@effect\//, "alchemy", "@maple-dev/effect-sdk", /^bun:/], alwaysBundle: [/^@maple\//, /^@maple-dev\/effect-orm/], - onlyImport: ["effect", "alchemy", "@maple-dev/effect-sdk", /^node:/], + onlyImport: ["effect", /^@effect\//, "alchemy", "@maple-dev/effect-sdk", /^node:/, /^bun:/], }, }) diff --git a/packages/eventing-core/README.md b/packages/eventing-core/README.md index 8621803e86..be93d579b3 100644 --- a/packages/eventing-core/README.md +++ b/packages/eventing-core/README.md @@ -29,7 +29,7 @@ perform the mandatory predicate depth/node preflight described in the generated predicate definition; JSON Schema validation alone does not enforce those whole-tree limits. -The first host adapter is Maple Local in `apps/cli/src/server/eventing`. It uses +The first host adapter is Maple Local in `packages/local-server/src/eventing`. It uses an authenticated configuration endpoint, a SQLite projection/outbox store, and the pre-chDB OTLP seam. The package itself deliberately contains none of those host decisions. diff --git a/packages/local-server/libchdb.json b/packages/local-server/libchdb.json new file mode 100644 index 0000000000..ee88a26395 --- /dev/null +++ b/packages/local-server/libchdb.json @@ -0,0 +1,9 @@ +{ + "version": "v26.7.3", + "sha256": { + "linux-x86_64-libchdb.tar.gz": "bc33260c32acf78eade2ac41a9115f38e00404651fa42e3bb4c419e4f011c031", + "linux-aarch64-libchdb.tar.gz": "d153adad1ff39b2e3caf0417f09d8bd9edd41939c7c67a3c4978a61e73fb9227", + "macos-x86_64-libchdb.tar.gz": "af5ded3ed3e84c31af1cd198dcf459f11d2b6aad4f6ddeccc04b8a519b0300fc", + "macos-arm64-libchdb.tar.gz": "5640e50dccf711bf3dd5551333d08e43f433edf7bd94b2289f36c2539e627762" + } +} diff --git a/packages/local-server/package.json b/packages/local-server/package.json new file mode 100644 index 0000000000..e4ee44682e --- /dev/null +++ b/packages/local-server/package.json @@ -0,0 +1,30 @@ +{ + "name": "@maple/local-server", + "version": "0.0.0", + "private": true, + "type": "module", + "exports": { + ".": "./src/runtime.ts", + "./schema/*": "./src/schema/*", + "./*": "./src/*.ts" + }, + "scripts": { + "test": "bun test --max-concurrency=1 src/otlp/encode.test.ts test/*.test.ts", + "typecheck": "tsrs --noEmit" + }, + "dependencies": { + "@effect/platform-bun": "catalog:effect", + "@maple/domain": "workspace:*", + "@maple/eventing-core": "workspace:*", + "@maple/ingest-rules": "workspace:*", + "@maple/query-engine": "workspace:*", + "effect": "catalog:effect", + "protobufjs": "^8.8.1" + }, + "devDependencies": { + "@types/bun": "^1.4.4", + "@types/node": "catalog:tooling", + "typescript": "catalog:tooling", + "vitest": "catalog:" + } +} diff --git a/apps/cli/src/server/archives/config.ts b/packages/local-server/src/archives/config.ts similarity index 100% rename from apps/cli/src/server/archives/config.ts rename to packages/local-server/src/archives/config.ts diff --git a/apps/cli/src/server/archives/errors.ts b/packages/local-server/src/archives/errors.ts similarity index 100% rename from apps/cli/src/server/archives/errors.ts rename to packages/local-server/src/archives/errors.ts diff --git a/apps/cli/src/server/archives/export.ts b/packages/local-server/src/archives/export.ts similarity index 100% rename from apps/cli/src/server/archives/export.ts rename to packages/local-server/src/archives/export.ts diff --git a/apps/cli/src/server/archives/gc.ts b/packages/local-server/src/archives/gc.ts similarity index 100% rename from apps/cli/src/server/archives/gc.ts rename to packages/local-server/src/archives/gc.ts diff --git a/apps/cli/src/server/archives/generation.ts b/packages/local-server/src/archives/generation.ts similarity index 99% rename from apps/cli/src/server/archives/generation.ts rename to packages/local-server/src/archives/generation.ts index ed027dd7dc..15e45c896f 100644 --- a/apps/cli/src/server/archives/generation.ts +++ b/packages/local-server/src/archives/generation.ts @@ -4,7 +4,7 @@ import { existsSync, readFileSync, statSync } from "node:fs" import { lstat, rm, statfs } from "node:fs/promises" import { dirname, join, parse, relative, resolve, sep } from "node:path" import { Effect, Option, Schema } from "effect" -import { CHDB_VERSION, MAPLE_VERSION } from "../../version" +import { CHDB_VERSION, MAPLE_VERSION } from "../version" import { readRawTelemetryRetentionDays } from "../chdb" import { LOCAL_SCHEMA_MANIFEST, SCHEMA_FINGERPRINT } from "../schema-identity" import { ttlDaysFromDefinition } from "../schema-manifest" diff --git a/apps/cli/src/server/archives/journal.ts b/packages/local-server/src/archives/journal.ts similarity index 100% rename from apps/cli/src/server/archives/journal.ts rename to packages/local-server/src/archives/journal.ts diff --git a/apps/cli/src/server/archives/legacy-calibration.ts b/packages/local-server/src/archives/legacy-calibration.ts similarity index 98% rename from apps/cli/src/server/archives/legacy-calibration.ts rename to packages/local-server/src/archives/legacy-calibration.ts index 5a4f27b848..6e8280ecd1 100644 --- a/apps/cli/src/server/archives/legacy-calibration.ts +++ b/packages/local-server/src/archives/legacy-calibration.ts @@ -6,8 +6,8 @@ import { type Dirent, readdirSync } from "node:fs" import { rm } from "node:fs/promises" import { join } from "node:path" import { Effect, Option, Result } from "effect" -import { debugLog } from "../../lib/debug" import { assertCheckpointPinIdentity, checkpointPinsRoot, releaseCheckpointPin } from "../checkpoints" +import { debugLog } from "../debug-log" import { ArchiveError } from "./errors" import { classifyArchivePathSync } from "./paths" @@ -104,12 +104,11 @@ export const retireLegacyCalibration = ( else if (removed.success) removedDirectories++ } yield* Effect.sync(() => { - if (releasedPins + removedDirectories > 0) { + if (releasedPins + removedDirectories > 0) debugLog( "retired legacy archive calibration state", `${releasedPins} pin(s), ${removedDirectories} dir(s)`, ) - } for (const failure of failures) debugLog("legacy archive calibration cleanup skipped", failure) }) return { releasedPins, removedDirectories, failures } diff --git a/apps/cli/src/server/archives/listing.ts b/packages/local-server/src/archives/listing.ts similarity index 100% rename from apps/cli/src/server/archives/listing.ts rename to packages/local-server/src/archives/listing.ts diff --git a/apps/cli/src/server/archives/manifest.ts b/packages/local-server/src/archives/manifest.ts similarity index 100% rename from apps/cli/src/server/archives/manifest.ts rename to packages/local-server/src/archives/manifest.ts diff --git a/apps/cli/src/server/archives/paths.ts b/packages/local-server/src/archives/paths.ts similarity index 100% rename from apps/cli/src/server/archives/paths.ts rename to packages/local-server/src/archives/paths.ts diff --git a/apps/cli/src/server/archives/reconcile.ts b/packages/local-server/src/archives/reconcile.ts similarity index 100% rename from apps/cli/src/server/archives/reconcile.ts rename to packages/local-server/src/archives/reconcile.ts diff --git a/apps/cli/src/server/archives/retention.ts b/packages/local-server/src/archives/retention.ts similarity index 100% rename from apps/cli/src/server/archives/retention.ts rename to packages/local-server/src/archives/retention.ts diff --git a/apps/cli/src/server/archives/schemas.ts b/packages/local-server/src/archives/schemas.ts similarity index 100% rename from apps/cli/src/server/archives/schemas.ts rename to packages/local-server/src/archives/schemas.ts diff --git a/apps/cli/src/server/archives/signals.ts b/packages/local-server/src/archives/signals.ts similarity index 100% rename from apps/cli/src/server/archives/signals.ts rename to packages/local-server/src/archives/signals.ts diff --git a/packages/local-server/src/assets.d.ts b/packages/local-server/src/assets.d.ts new file mode 100644 index 0000000000..34f059079e --- /dev/null +++ b/packages/local-server/src/assets.d.ts @@ -0,0 +1,15 @@ +// Bun embeds text-typed imports (`import s from "./x.sql" with { type: "text" }`) +// into the `--compile`d binary and resolves them from disk in dev. These ambient +// declarations give those imports a `string` type. +declare module "*.sql" { + const content: string + export default content +} +declare module "*.proto" { + const content: string + export default content +} +declare module "*.md" { + const content: string + export default content +} diff --git a/apps/cli/src/server/chdb-rows.ts b/packages/local-server/src/chdb-rows.ts similarity index 100% rename from apps/cli/src/server/chdb-rows.ts rename to packages/local-server/src/chdb-rows.ts diff --git a/apps/cli/src/server/chdb.ts b/packages/local-server/src/chdb.ts similarity index 95% rename from apps/cli/src/server/chdb.ts rename to packages/local-server/src/chdb.ts index 04c326989c..2ff3084ead 100644 --- a/apps/cli/src/server/chdb.ts +++ b/packages/local-server/src/chdb.ts @@ -18,7 +18,7 @@ import { homedir } from "node:os" import { dirname, join, resolve } from "node:path" import { durableJson } from "./durable-files" import { markStoreClosed, markStoreOpen, storeHasData } from "./store-version" -import { MAPLE_VERSION } from "../version" +import { MAPLE_VERSION } from "./version" /** A chDB failure — locating libchdb, opening the connection, or bootstrapping * the schema. Carries the underlying message verbatim. */ @@ -36,12 +36,25 @@ export class ChdbClosedError extends Schema.TaggedError()("@map message: Schema.String, }) {} +let configuredLibchdb: string | undefined + +/** + * Load libchdb from exactly `path` (a host that provisioned it, e.g. the + * @maple-dev/alchemy sidecar). Must run before the first connection: chDB is + * loaded once per process. + */ +export const configureLibchdb = (path: string): void => { + configuredLibchdb = path +} + /** Locate `libchdb` at runtime, in priority order: + * 0. the path a host passed to {@link configureLibchdb} * 1. `MAPLE_LIBCHDB` env (explicit override) * 2. sibling of the executable (the shipped 2-file bundle: `maple` + `libchdb`) * 3. `~/.maple/bin/libchdb.{so,dylib}` (dev / installed location) */ function resolveLibchdb(): string { + if (configuredLibchdb !== undefined) return configuredLibchdb const candidates: string[] = [] const override = process.env.MAPLE_LIBCHDB if (override) candidates.push(override) @@ -62,6 +75,21 @@ function resolveLibchdb(): string { return found } +/** + * The release tag (`v26.7.3`) of the libchdb this process would load, read from + * the library itself via `chdb_version`. Loads only that symbol and opens no + * connection, so it is safe before any store is touched. Throws when the + * library is missing or predates `chdb_version`. + */ +export const loadedLibchdbRelease = (): string => { + const lib = dlopen(resolveLibchdb(), { chdb_version: { args: [], returns: FFIType.ptr } }) + const version = lib.symbols.chdb_version() + const release = version === null ? undefined : new CString(version).toString() + lib.close() + if (release === undefined) throw new Error("libchdb reported no version") + return `v${release}` +} + type ChdbSymbols = ReturnType["symbols"] function openLib(libPath: string) { diff --git a/apps/cli/src/server/checkpoint-digest.ts b/packages/local-server/src/checkpoint-digest.ts similarity index 100% rename from apps/cli/src/server/checkpoint-digest.ts rename to packages/local-server/src/checkpoint-digest.ts diff --git a/apps/cli/src/server/checkpoints.ts b/packages/local-server/src/checkpoints.ts similarity index 98% rename from apps/cli/src/server/checkpoints.ts rename to packages/local-server/src/checkpoints.ts index 99a7ceb724..ad64d9b0e9 100644 --- a/apps/cli/src/server/checkpoints.ts +++ b/packages/local-server/src/checkpoints.ts @@ -6,10 +6,11 @@ import { existsSync, lstatSync, readFileSync, rmSync, writeFileSync } from "node import { lstat, mkdir, readFile, readdir, rm, rmdir, stat } from "node:fs/promises" import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path" import { Duration, Effect, Option, Schema } from "effect" +import { FileSystem } from "effect/FileSystem" import { HttpClient, HttpClientRequest } from "effect/http" -import { CHDB_VERSION, MAPLE_VERSION } from "../version" -import { serverUrl } from "../lib/local-address" -import { Chdb, MERGE_TREE_CONFIG_XML, RAW_TELEMETRY_TTL_COLUMNS } from "./chdb" +import { CHDB_VERSION, MAPLE_VERSION } from "./version" +import { serverUrl } from "./local-address" +import { Chdb, MERGE_TREE_CONFIG_XML, RAW_TELEMETRY_TTL_COLUMNS, writeIngestConfig } from "./chdb" import { ttlDaysFromDefinition } from "./schema-manifest" import { sha256File } from "./checkpoint-digest" import { @@ -31,6 +32,7 @@ import { import { CURRENT_LOCAL_SCHEMA, SCHEMA_FINGERPRINT } from "./schema-identity" import schemaSql from "./schema/local-schema.sql" with { type: "text" } import { + dataDirSidecarPath, markStoreClosedDurable, storeMarkerPath, storeOpenMarkerPath, @@ -2927,3 +2929,42 @@ export const assertCheckpointRootSafe = (dataDir: string): void => { } if (basename(root) !== "backups") throw new Error("invalid checkpoint root") } + +// Generated chDB config, beside the PID and log files (e.g. ~/.maple/chdb-config.xml). +export const chdbConfigPath = (dataDir: string): string => dataDirSidecarPath(dataDir, "chdb-config.xml") + +/** + * Resolve the chDB config file, generating a backups-enabled default when the + * user did not supply one. + * + * `BACKUP DATABASE default TO Disk('default', …)` (how every checkpoint is + * taken) needs `` in the config of the *running* chDB + * connection. chDB allows one connection per process, acquired once at start and + * held for the process lifetime, and `maple checkpoint` is a separate process + * talking over HTTP: it cannot inject config into a live connection. So a server + * started without a backups config can never checkpoint. A user-supplied + * `--chdb-config-file` is honoured untouched. An in-memory store is never + * checkpointed, so it gets only the MergeTree ingest defaults (`backups: false`). + */ +export const resolveChdbConfigFile = ( + dataDir: string, + supplied: string | undefined, + { backups }: { readonly backups: boolean } = { backups: true }, +) => + Effect.gen(function* () { + if (supplied !== undefined) return supplied + const fs = yield* FileSystem + const path = chdbConfigPath(dataDir) + // Regenerated every start: idempotent, and it self-heals a truncated or + // hand-edited file. Failing to write is not fatal: the server still starts, + // checkpoints just stay unavailable, which is the old behaviour. + yield* fs.makeDirectory(dirname(path), { recursive: true }).pipe(Effect.ignore) + return yield* Effect.try({ + try: () => { + if (backups) writeBackupConfig(path) + else writeIngestConfig(path) + return path + }, + catch: (error) => error, + }).pipe(Effect.orElseSucceed(() => undefined)) + }) diff --git a/packages/local-server/src/debug-log.ts b/packages/local-server/src/debug-log.ts new file mode 100644 index 0000000000..9d034bbe3c --- /dev/null +++ b/packages/local-server/src/debug-log.ts @@ -0,0 +1,14 @@ +// `--debug` (or MAPLE_DEBUG=1) surfaces diagnostics on stderr, so stdout stays +// clean. Read straight from argv/env: the code that emits these lines runs +// outside any parsed-flag context. The CLI's styled printer shares this switch. + +export const debugEnabled = (): boolean => + (typeof process !== "undefined" && Array.isArray(process.argv) && process.argv.includes("--debug")) || + process.env.MAPLE_DEBUG === "1" + +/** Emit a plain debug line to stderr (no-op unless debug is enabled). */ +export const debugLog = (label: string, detail?: string): void => { + if (!debugEnabled()) return + process.stderr.write(`[debug] ${label}\n`) + if (detail) process.stderr.write(`${detail}\n`) +} diff --git a/apps/cli/src/server/durable-files.ts b/packages/local-server/src/durable-files.ts similarity index 98% rename from apps/cli/src/server/durable-files.ts rename to packages/local-server/src/durable-files.ts index e215fffa0b..a602ad10f3 100644 --- a/apps/cli/src/server/durable-files.ts +++ b/packages/local-server/src/durable-files.ts @@ -60,7 +60,9 @@ const warnedDirectories = new Set() const warnUnhardenedDirectory = (path: string): void => { if (warnedDirectories.has(path)) return warnedDirectories.add(path) - process.emitWarning(`cannot chmod ${path} to 0700 (owned by another user); continuing, it is already owner-only`) + process.emitWarning( + `cannot chmod ${path} to 0700 (owned by another user); continuing, it is already owner-only`, + ) } /** Like {@link ensurePrivateDirectory}, but an already-existing directory keeps diff --git a/apps/cli/src/server/eventing/consumer-auth.ts b/packages/local-server/src/eventing/consumer-auth.ts similarity index 100% rename from apps/cli/src/server/eventing/consumer-auth.ts rename to packages/local-server/src/eventing/consumer-auth.ts diff --git a/apps/cli/src/server/eventing/control-store.ts b/packages/local-server/src/eventing/control-store.ts similarity index 100% rename from apps/cli/src/server/eventing/control-store.ts rename to packages/local-server/src/eventing/control-store.ts diff --git a/apps/cli/src/server/eventing/otlp.ts b/packages/local-server/src/eventing/otlp.ts similarity index 100% rename from apps/cli/src/server/eventing/otlp.ts rename to packages/local-server/src/eventing/otlp.ts diff --git a/apps/cli/src/server/eventing/runtime.ts b/packages/local-server/src/eventing/runtime.ts similarity index 100% rename from apps/cli/src/server/eventing/runtime.ts rename to packages/local-server/src/eventing/runtime.ts diff --git a/apps/cli/src/server/eventing/telemetry.ts b/packages/local-server/src/eventing/telemetry.ts similarity index 100% rename from apps/cli/src/server/eventing/telemetry.ts rename to packages/local-server/src/eventing/telemetry.ts diff --git a/apps/cli/src/server/identity-schema.ts b/packages/local-server/src/identity-schema.ts similarity index 100% rename from apps/cli/src/server/identity-schema.ts rename to packages/local-server/src/identity-schema.ts diff --git a/apps/cli/src/server/ingest-rules.ts b/packages/local-server/src/ingest-rules.ts similarity index 100% rename from apps/cli/src/server/ingest-rules.ts rename to packages/local-server/src/ingest-rules.ts diff --git a/apps/cli/src/server/insert-batcher.ts b/packages/local-server/src/insert-batcher.ts similarity index 100% rename from apps/cli/src/server/insert-batcher.ts rename to packages/local-server/src/insert-batcher.ts diff --git a/apps/cli/src/server/inserts.ts b/packages/local-server/src/inserts.ts similarity index 100% rename from apps/cli/src/server/inserts.ts rename to packages/local-server/src/inserts.ts diff --git a/packages/local-server/src/libchdb-provision.ts b/packages/local-server/src/libchdb-provision.ts new file mode 100644 index 0000000000..d29bd7129d --- /dev/null +++ b/packages/local-server/src/libchdb-provision.ts @@ -0,0 +1,158 @@ +// Acquire the pinned libchdb the way scripts/build-local-binary.sh does for the +// `maple` bundle: download the release asset, verify its sha256 BEFORE +// extracting (release assets are mutable even for a fixed tag), extract, and +// install it. Hosts without a bundled library (the @maple-dev/alchemy sidecar) +// keep one verified copy per release in a per-user cache. +import { Duration, Effect, Schedule, Schema } from "effect" +import { HttpClient } from "effect/http" +import { createHash, randomUUID } from "node:crypto" +import { existsSync } from "node:fs" +import { mkdir, readdir, rename, rm, writeFile } from "node:fs/promises" +import { homedir } from "node:os" +import { join } from "node:path" +import manifest from "../libchdb.json" with { type: "json" } + +export class LibchdbProvisionError extends Schema.TaggedError()( + "@maple/local-server/LibchdbProvisionError", + { message: Schema.String }, +) {} + +export interface LibchdbRelease { + readonly version: string + /** sha256 of each platform's release asset, by asset file name. */ + readonly sha256: Readonly> +} + +/** The release `maple` ships with and the sidecar build pins (`libchdb.json`). */ +export const PINNED_LIBCHDB: LibchdbRelease = manifest + +const RELEASES_URL = "https://github.com/chdb-io/chdb-core/releases/download" + +/** Release asset per `${process.platform}-${process.arch}`, as the binary build names them. */ +const PLATFORM_ASSETS = { + "darwin-arm64": "macos-arm64-libchdb.tar.gz", + "darwin-x64": "macos-x86_64-libchdb.tar.gz", + "linux-arm64": "linux-aarch64-libchdb.tar.gz", + "linux-x64": "linux-x86_64-libchdb.tar.gz", +} as const + +const isSupportedPlatform = (key: string): key is keyof typeof PLATFORM_ASSETS => + Object.hasOwn(PLATFORM_ASSETS, key) + +/** The release asset this machine needs, or `undefined` where chdb-core ships none. */ +export const platformLibchdbAsset = (): string | undefined => { + const platform = `${process.platform}-${process.arch}` + return isSupportedPlatform(platform) ? PLATFORM_ASSETS[platform] : undefined +} + +/** One verified copy per release, shared by every project on the machine (~330 MB). */ +export const defaultLibchdbCacheRoot = (): string => join(homedir(), ".maple", "cache", "libchdb") + +export interface ProvisionOptions { + readonly cacheRoot?: string + readonly release?: LibchdbRelease + /** Base of `//`; defaults to chdb-core's GitHub releases. */ + readonly baseUrl?: string +} + +export interface ProvisionedLibchdb { + readonly path: string + readonly version: string + /** Whether this call downloaded it (false: already cached). */ + readonly downloaded: boolean +} + +const fail = (message: string) => new LibchdbProvisionError({ message }) + +/** Internal: a failure worth retrying (5xx or network); never escapes {@link download}. */ +class RetryableDownload extends Schema.TaggedError()( + "@maple/local-server/RetryableDownload", + { message: Schema.String }, +) {} + +/** Release downloads 500 intermittently: retry 5xx and network failures, never a 404. */ +const download = (url: string): Effect.Effect => + Effect.gen(function* () { + const retryable = (detail: string) => new RetryableDownload({ message: `${url}: ${detail}` }) + const client = yield* HttpClient.HttpClient + const response = yield* client.get(url).pipe(Effect.mapError((error) => retryable(error.message))) + if (response.status >= 500) return yield* retryable(`HTTP ${response.status}`) + if (response.status < 200 || response.status >= 300) + return yield* fail(`${url}: HTTP ${response.status}`) + return yield* response.arrayBuffer.pipe(Effect.mapError((error) => retryable(error.message))) + }).pipe( + Effect.retry({ + schedule: Schedule.spaced(Duration.seconds(5)), + times: 5, + while: (error) => error._tag === "@maple/local-server/RetryableDownload", + }), + Effect.catchTag("@maple/local-server/RetryableDownload", (error) => Effect.fail(fail(error.message))), + ) + +const findLibrary = (root: string) => + Effect.tryPromise({ + try: () => readdir(root, { recursive: true }), + catch: (error) => fail(`cannot list the extracted archive: ${String(error)}`), + }).pipe( + Effect.map((entries) => + entries.find((entry) => entry.endsWith("libchdb.so") || entry.endsWith("libchdb.dylib")), + ), + ) + +/** + * The path of a verified copy of the pinned libchdb for this platform, + * downloading it into the cache on first use. Concurrent callers each stage in + * their own directory and install with an atomic rename. + */ +export const provisionLibchdb = (options: ProvisionOptions = {}) => + Effect.gen(function* () { + const release = options.release ?? PINNED_LIBCHDB + const asset = platformLibchdbAsset() + if (asset === undefined) + return yield* fail(`no libchdb release asset for ${process.platform}-${process.arch}`) + const expected = release.sha256[asset] + if (expected === undefined) + return yield* fail(`no pinned sha256 for ${asset} at libchdb ${release.version}`) + + const cacheRoot = options.cacheRoot ?? defaultLibchdbCacheRoot() + const installDir = join(cacheRoot, release.version, asset.replace(/\.tar\.gz$/, "")) + const target = join(installDir, "libchdb.so") + if (existsSync(target)) return { path: target, version: release.version, downloaded: false } + + const url = `${options.baseUrl ?? RELEASES_URL}/${release.version}/${asset}` + yield* Effect.logInfo(`downloading libchdb ${release.version} (${asset}) into ${installDir}`) + const archive = yield* download(url) + const actual = createHash("sha256").update(new Uint8Array(archive)).digest("hex") + if (actual !== expected) + return yield* fail(`libchdb checksum mismatch for ${asset}: expected ${expected}, got ${actual}`) + + // Staged beside the cache so the final rename never crosses filesystems. + const staging = join(cacheRoot, `.staging-${randomUUID()}`) + yield* Effect.acquireRelease( + Effect.tryPromise({ + try: () => mkdir(staging, { recursive: true }), + catch: (error) => fail(`cannot create ${staging}: ${String(error)}`), + }), + () => Effect.promise(() => rm(staging, { recursive: true, force: true })), + ) + const tarball = join(staging, asset) + yield* Effect.tryPromise({ + try: () => writeFile(tarball, new Uint8Array(archive)), + catch: (error) => fail(`cannot write ${tarball}: ${String(error)}`), + }) + const exitCode = yield* Effect.tryPromise({ + try: () => Bun.spawn(["tar", "-xzf", tarball, "-C", staging], { stderr: "ignore" }).exited, + catch: (error) => fail(`cannot run tar: ${String(error)}`), + }) + if (exitCode !== 0) return yield* fail(`tar could not extract ${asset} (exit ${exitCode})`) + const library = yield* findLibrary(staging) + if (library === undefined) return yield* fail(`libchdb not found in ${asset}`) + yield* Effect.tryPromise({ + try: async () => { + await mkdir(installDir, { recursive: true }) + await rename(join(staging, library), target) + }, + catch: (error) => fail(`cannot install libchdb into ${installDir}: ${String(error)}`), + }) + return { path: target, version: release.version, downloaded: true } + }).pipe(Effect.scoped) diff --git a/apps/cli/src/lib/local-address.ts b/packages/local-server/src/local-address.ts similarity index 95% rename from apps/cli/src/lib/local-address.ts rename to packages/local-server/src/local-address.ts index 8dc026e144..ba556b648c 100644 --- a/apps/cli/src/lib/local-address.ts +++ b/packages/local-server/src/local-address.ts @@ -1,6 +1,10 @@ export const DEFAULT_LOCAL_HOST = "127.0.0.1" export const DEFAULT_LOCAL_PORT = 4318 +/** Public origin of the deployed local-mode dashboard SPA. Overridable + * (`MAPLE_LOCAL_UI_URL`) for testing against another build of it. */ +export const DEFAULT_REMOTE_UI_URL = "https://local.maple.dev" + /** Bun expects bare IPv6 listener addresses, while users commonly copy the * bracketed form used in URLs. Accept either form at the CLI boundary. */ export const normalizeHost = (host: string): string => { diff --git a/apps/cli/src/server/local-schema-history.ts b/packages/local-server/src/local-schema-history.ts similarity index 100% rename from apps/cli/src/server/local-schema-history.ts rename to packages/local-server/src/local-schema-history.ts diff --git a/apps/cli/src/server/local-schema-version.ts b/packages/local-server/src/local-schema-version.ts similarity index 100% rename from apps/cli/src/server/local-schema-version.ts rename to packages/local-server/src/local-schema-version.ts diff --git a/packages/local-server/src/local-store-migration-journal.ts b/packages/local-server/src/local-store-migration-journal.ts new file mode 100644 index 0000000000..000610c228 --- /dev/null +++ b/packages/local-server/src/local-store-migration-journal.ts @@ -0,0 +1,182 @@ +// The coordinator-owned migration journal envelope. Deliberately independent of +// the executable module registry and its schema snapshots, so a host that only +// needs to know whether a migration is unfinished can read it without them. +import { readFile } from "node:fs/promises" +import { dirname, join, relative, resolve } from "node:path" +import { + decodeMigrationJournal, + type MigrationJournalSchema, + type MigrationPhase, +} from "./local-store-migrations/journal-schema" +import type { LocalSchemaIdentity } from "./schema-identity" +import { adoptLegacySidecars, dataDirSidecarPath } from "./store-version" + +const NONTERMINAL_PHASES = new Set([ + "planned", + "preflight-complete", + "target-created", + "copying", + "copy-verified", + "promotion-started", + "failed", +]) + +export const isMigrationIncomplete = (phase: MigrationPhase): boolean => NONTERMINAL_PHASES.has(phase) + +export type MigrationJournal = typeof MigrationJournalSchema.Type + +export const migrationJournalPath = (dataDir: string): string => + dataDirSidecarPath(dataDir, "maple-store-migration.json") + +export const migrationRootPath = (dataDir: string, migrationId: string): string => + join(dirname(resolve(dataDir)), ".maple-migrations", migrationId) + +export const migrationHistoryPath = (dataDir: string, migrationId: string): string => + join(migrationRootPath(dataDir, migrationId), "journal.json") + +export const safeMigrationPath = (path: string, root: string, label: string): string => { + const absolute = resolve(path) + const relativePath = relative(resolve(root), absolute) + if ( + relativePath === "" || + relativePath === ".." || + relativePath.startsWith("../") || + resolve(root) === absolute + ) { + throw new Error(`${label} escapes migration root`) + } + return absolute +} + +export const assertJournalPaths = (dataDir: string, journal: MigrationJournal): void => { + const source = resolve(dataDir) + if (resolve(journal.sourceDataDir) !== source) + throw new Error("local migration journal source does not match the configured data directory") + const root = migrationRootPath(dataDir, journal.migrationId) + safeMigrationPath(journal.targetDataDir, root, "migration target") + if (resolve(dirname(journal.targetDataDir)) !== resolve(join(root, "target"))) + throw new Error("local migration journal target is outside its owned target directory") + if ( + journal.chain.length === 0 || + journal.currentStepIndex < 0 || + journal.currentStepIndex > journal.chain.length + ) + throw new Error("local migration journal chain state is invalid") + const final = journal.chain[journal.chain.length - 1]! + if ( + final.to.version !== journal.targetVersion || + final.to.fingerprint !== journal.targetFingerprint || + final.to.digest !== journal.targetDigest + ) + throw new Error("local migration journal final target does not match its chain") +} + +const sameJournalIdentity = (a: LocalSchemaIdentity, b: LocalSchemaIdentity): boolean => + a.version === b.version && a.fingerprint === b.fingerprint && a.digest === b.digest && a.chdb === b.chdb + +const assertJournalChainInvariants = (journal: MigrationJournal): void => { + const { chain, currentStepIndex: current } = journal + if (current < 0 || current > chain.length) + throw new Error("migration journal currentStepIndex is invalid") + if ( + !sameJournalIdentity(chain[0]!.from, { + version: journal.sourceVersion, + fingerprint: journal.sourceFingerprint, + digest: journal.sourceDigest, + chdb: journal.sourceChdb, + }) + ) + throw new Error("migration journal source identity does not match its first step") + if ( + !sameJournalIdentity(chain[chain.length - 1]!.to, { + version: journal.targetVersion, + fingerprint: journal.targetFingerprint, + digest: journal.targetDigest, + chdb: journal.targetChdb, + }) + ) + throw new Error("migration journal target identity does not match its final step") + for (let index = 1; index < chain.length; index += 1) { + if (!sameJournalIdentity(chain[index - 1]!.to, chain[index]!.from)) + throw new Error(`migration journal step ${index} does not continue the previous step`) + } + for (let index = 0; index < current; index += 1) { + if (chain[index]!.status !== "completed") + throw new Error(`migration journal step ${index} precedes currentStepIndex but is not completed`) + } + for (let index = current + 1; index < chain.length; index += 1) { + if (chain[index]!.status !== "pending") + throw new Error(`migration journal step ${index} follows currentStepIndex but is not pending`) + } + if (current === chain.length) { + if (chain.some((step) => step.status !== "completed")) + throw new Error( + "migration journal currentStepIndex reaches the end before all steps are completed", + ) + } else { + const currentStatus = chain[current]!.status + const plannedStart = journal.phase === "planned" && current === 0 + if (plannedStart) { + if (currentStatus !== "pending" && currentStatus !== "running") + throw new Error("planned migration journal must begin with a pending or running step") + } else if (currentStatus !== "running" && currentStatus !== "verified") { + throw new Error("migration journal current step must be running or verified") + } + } + if (journal.phase === "promotion-started" || journal.phase === "promoted") { + if (current !== chain.length || chain.some((step) => step.status !== "completed")) + throw new Error(`${journal.phase} migration journal must contain an entirely completed chain`) + } + if ( + (journal.phase === "preflight-complete" || + journal.phase === "target-created" || + journal.phase === "copying") && + current === chain.length + ) + throw new Error(`${journal.phase} migration journal must still have an active current step`) + if (journal.phase === "planned" && current !== 0) + throw new Error("planned migration journal must have currentStepIndex zero") + if ( + journal.phase === "failed" && + current === chain.length && + chain.some((step) => step.status !== "completed") + ) + throw new Error("failed migration journal cannot end before all steps are completed") +} + +const parseJournal = (value: unknown): MigrationJournal => { + const journal = decodeMigrationJournal(value) + if (journal.currentStepIndex > journal.chain.length) + throw new Error("migration journal currentStepIndex is invalid") + assertJournalChainInvariants(journal) + for (const [index, step] of journal.chain.entries()) { + if (step.status === "verified" || step.status === "completed") { + if (step.state === undefined) + throw new Error( + `migration journal step ${index} is ${step.status} but has no persisted state`, + ) + if (step.progress === undefined) + throw new Error( + `migration journal step ${index} is ${step.status} but has no persisted progress`, + ) + } + } + return journal +} + +/** Read and validate only the coordinator-owned journal envelope. This path is + * intentionally independent of the executable module registry so target-only + * abandonment remains available when a newer build cannot bind a historical + * module or its persisted state is corrupt. */ +export const readMigrationJournalStructure = async (dataDir: string): Promise => { + try { + // A custom-named store may still keep its journal at the pre-sidecar path. + adoptLegacySidecars(dataDir) + return parseJournal(JSON.parse(await readFile(migrationJournalPath(dataDir), "utf8")) as unknown) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null + throw new Error( + `cannot read local migration journal: ${error instanceof Error ? error.message : String(error)}`, + ) + } +} diff --git a/apps/cli/src/server/local-store-migrations/journal-schema.ts b/packages/local-server/src/local-store-migrations/journal-schema.ts similarity index 100% rename from apps/cli/src/server/local-store-migrations/journal-schema.ts rename to packages/local-server/src/local-store-migrations/journal-schema.ts diff --git a/apps/cli/src/server/local-token.ts b/packages/local-server/src/local-token.ts similarity index 100% rename from apps/cli/src/server/local-token.ts rename to packages/local-server/src/local-token.ts diff --git a/apps/cli/src/server/otlp/encode.test.ts b/packages/local-server/src/otlp/encode.test.ts similarity index 99% rename from apps/cli/src/server/otlp/encode.test.ts rename to packages/local-server/src/otlp/encode.test.ts index 455e668724..c03f1e44b2 100644 --- a/apps/cli/src/server/otlp/encode.test.ts +++ b/packages/local-server/src/otlp/encode.test.ts @@ -273,7 +273,10 @@ describe("value-level spot checks", () => { { name: "h", histogram: { - dataPoints: [histogramPoint("0"), histogramPoint("18446744073709551615")], + dataPoints: [ + histogramPoint("0"), + histogramPoint("18446744073709551615"), + ], }, }, ], diff --git a/apps/cli/src/server/otlp/encode.ts b/packages/local-server/src/otlp/encode.ts similarity index 100% rename from apps/cli/src/server/otlp/encode.ts rename to packages/local-server/src/otlp/encode.ts diff --git a/apps/cli/src/server/otlp/proto.ts b/packages/local-server/src/otlp/proto.ts similarity index 100% rename from apps/cli/src/server/otlp/proto.ts rename to packages/local-server/src/otlp/proto.ts diff --git a/apps/cli/src/server/query-guard.ts b/packages/local-server/src/query-guard.ts similarity index 100% rename from apps/cli/src/server/query-guard.ts rename to packages/local-server/src/query-guard.ts diff --git a/apps/cli/src/server/remote-auth.ts b/packages/local-server/src/remote-auth.ts similarity index 100% rename from apps/cli/src/server/remote-auth.ts rename to packages/local-server/src/remote-auth.ts diff --git a/packages/local-server/src/runtime.ts b/packages/local-server/src/runtime.ts new file mode 100644 index 0000000000..9e46e3393a --- /dev/null +++ b/packages/local-server/src/runtime.ts @@ -0,0 +1,231 @@ +/// +import * as BunServices from "@effect/platform-bun/BunServices" +import { Effect, Exit, Scope } from "effect" +import { FileSystem } from "effect/FileSystem" +import { resolve } from "node:path" +import { DEFAULT_REMOTE_UI_URL, hostedDashboardUrl, hostedUiOrigin, serverUrl } from "./local-address" +import { ChdbError, configureLibchdb, loadedLibchdbRelease } from "./chdb" +import { + checkpointAvailability, + reconcileCheckpointRecovery, + resetLiveStorePreservingCheckpoints, + resolveChdbConfigFile, +} from "./checkpoints" +import { isMigrationIncomplete, readMigrationJournalStructure } from "./local-store-migration-journal" +import { NO_REMOTE_AUTH } from "./remote-auth" +import { CURRENT_LOCAL_SCHEMA } from "./schema-identity" +import { ensurePortFree, startServer } from "./serve" +import { acquireStoreClaim } from "./store-claim" +import { + describeError, + LocalStoreDirtyError, + LocalStoreFromNewerMapleError, + LocalStoreIncompatibleError, + LocalStoreMigrationError, + LocalStoreSchemaStaleError, + ServerStateError, + stateFileError, +} from "./store-errors" +import { + checkStoreCompatible, + isSchemaIdentityStale, + isStoreDirty, + newerStoreSchemaVersion, +} from "./store-version" +import { resolveUiAssets } from "./ui-assets" +import { CHDB_VERSION } from "./version" + +/** + * What to do with a store whose last process was killed before it could close + * chDB (reopening such a store can crash chDB natively). `"fail"` refuses; + * `"wipe"` discards its live data, keeping any checkpoints, and starts fresh. + */ +export type DirtyStorePolicy = "fail" | "wipe" + +export interface LocalOptions { + readonly dataDir: string + readonly port?: number + /** Load libchdb from this path instead of `MAPLE_LIBCHDB` or the default locations. */ + readonly libchdb?: string + /** Default `"fail"`: an unclean store is never wiped implicitly. */ + readonly onDirtyStore?: DirtyStorePolicy +} + +export interface LocalServer { + readonly port: number + readonly endpoint: string + readonly dashboardUrl: string + readonly dataDir: string +} + +const HOSTNAME = "127.0.0.1" + +/** Fail closed on any store the CLI would only open after an explicit choice. */ +const refuseUnsafeStore = (dataDir: string, onDirtyStore: DirtyStorePolicy) => + Effect.gen(function* () { + const journal = yield* Effect.tryPromise({ + try: () => readMigrationJournalStructure(dataDir), + catch: (error) => + new LocalStoreMigrationError({ + dataDir, + phase: "read-journal", + message: describeError(error), + }), + }) + if (journal !== null && isMigrationIncomplete(journal.phase)) + return yield* new LocalStoreMigrationError({ + dataDir, + phase: "resume", + message: `the local store at ${dataDir} has an unfinished schema migration; resume or abandon it with maple before starting Maple Local`, + }) + const newerSchema = yield* Effect.try({ + try: () => newerStoreSchemaVersion(dataDir, CURRENT_LOCAL_SCHEMA.version), + catch: stateFileError(dataDir, "read store schema for"), + }) + if (newerSchema !== undefined) + return yield* new LocalStoreFromNewerMapleError({ + dataDir, + storeSchemaVersion: newerSchema, + currentSchemaVersion: CURRENT_LOCAL_SCHEMA.version, + message: `the local store at ${dataDir} was written by a newer maple (schema v${newerSchema}; this build supports up to v${CURRENT_LOCAL_SCHEMA.version}). Nothing was changed.`, + }) + const compatibility = yield* Effect.try({ + try: () => checkStoreCompatible(dataDir), + catch: stateFileError(dataDir, "read store identity for"), + }) + if (!compatibility.compatible) + return yield* new LocalStoreIncompatibleError({ + dataDir, + storeBuild: compatibility.found, + currentBuild: compatibility.current, + message: `the local store at ${dataDir} is incompatible with this build's chDB (store: ${compatibility.found}; build: ${compatibility.current}). Reset it with maple.`, + }) + const dirty = yield* Effect.try({ + try: () => isStoreDirty(dataDir), + catch: stateFileError(dataDir, "read open marker for"), + }) + if (dirty) { + if (onDirtyStore === "wipe") { + yield* Effect.logWarning( + `the local store at ${dataDir} was not cleanly closed; onDirtyStore is "wipe", so its live data is discarded`, + ) + yield* resetLiveStorePreservingCheckpoints(dataDir) + } else { + const availability = yield* Effect.promise(() => checkpointAvailability(dataDir)) + return yield* new LocalStoreDirtyError({ + dataDir, + policy: "fail", + checkpointAvailable: availability.available, + message: + `the local store at ${dataDir} was not cleanly closed: the process that owned it was killed before closing chDB, and reopening it can crash chDB. ` + + `Start with onDirtyStore: "wipe" to discard its live data and start fresh` + + (availability.available + ? ", or run `maple restore` on it to recover its last checkpoint." + : "."), + }) + } + } + const stale = yield* Effect.try({ + try: () => isSchemaIdentityStale(dataDir, CURRENT_LOCAL_SCHEMA), + catch: stateFileError(dataDir, "read schema identity for"), + }) + if (stale) + return yield* new LocalStoreSchemaStaleError({ + dataDir, + message: `the local store at ${dataDir} has a different schema identity; migrate or reset it with maple`, + }) + }) + +/** + * A build that pinned its chDB release (`__CHDB_VERSION__`) stamps it on every + * store it writes, so the library actually loaded must be that release: a + * different one would write stores under a false label and could crash opening + * them later. Source runs (`"dev"`) carry no pin to check. + */ +const requirePinnedLibchdb = Effect.gen(function* () { + if (CHDB_VERSION === "dev") return + const loaded = yield* Effect.try({ + try: loadedLibchdbRelease, + catch: (error) => + new ChdbError({ message: `cannot read the libchdb release: ${describeError(error)}` }), + }) + if (loaded !== CHDB_VERSION) + return yield* new ChdbError({ + message: `libchdb ${loaded} is loaded, but this build is pinned to ${CHDB_VERSION}; point MAPLE_LIBCHDB at libchdb ${CHDB_VERSION}`, + }) +}) + +const processState = globalThis as typeof globalThis & { __mapleLocalRuntimeOwner?: string } + +/** Acquires into the ambient scope; {@link startLocal} owns that scope's failure path. */ +const acquireLocal = (options: LocalOptions) => + Effect.gen(function* () { + const dataDir = resolve(options.dataDir) + const port = options.port ?? 4318 + + // chDB holds process-global native state, even across re-imported modules (HMR). + yield* Effect.acquireRelease( + Effect.gen(function* () { + if (processState.__mapleLocalRuntimeOwner !== undefined) + return yield* new ServerStateError({ + message: `this process already owns Maple Local at ${processState.__mapleLocalRuntimeOwner}; share that resource instead of opening another chDB store`, + }) + processState.__mapleLocalRuntimeOwner = dataDir + }), + () => + Effect.sync(() => { + delete processState.__mapleLocalRuntimeOwner + }), + ) + yield* acquireStoreClaim( + dataDir, + (pid) => + new ServerStateError({ + message: `the local store at ${dataDir} is already in use${pid === undefined ? "" : ` by PID ${pid}`}`, + }), + ) + yield* ensurePortFree(HOSTNAME, port) + if (options.libchdb !== undefined) configureLibchdb(options.libchdb) + yield* requirePinnedLibchdb + yield* reconcileCheckpointRecovery(dataDir) + yield* refuseUnsafeStore(dataDir, options.onDirtyStore ?? "fail") + + yield* (yield* FileSystem).makeDirectory(dataDir, { recursive: true }) + const { port: boundPort } = yield* startServer({ + hostname: HOSTNAME, + browserHosts: [HOSTNAME], + advertiseHost: HOSTNAME, + corsOrigin: hostedUiOrigin(DEFAULT_REMOTE_UI_URL), + port, + dataDir, + configFile: yield* resolveChdbConfigFile(dataDir, undefined), + auth: NO_REMOTE_AUTH, + assets: yield* resolveUiAssets(), + }) + return { + port: boundPort, + endpoint: serverUrl(HOSTNAME, boundPort), + dashboardUrl: hostedDashboardUrl(DEFAULT_REMOTE_UI_URL, boundPort), + dataDir, + } satisfies LocalServer + // Native-host entry point: the caller's Scope still owns all acquired resources. + // oxlint-disable-next-line effecttsgo/strict-effect-provide + }).pipe(Effect.provide(BunServices.layer)) + +/** + * Own one embedded loopback Maple server until the caller's scope closes. No CLI + * command or Maple executable runs. Unsafe stores fail closed; recovery stays + * explicit. Periodic checkpoints belong to `maple start`. + * + * A failed start releases everything it acquired (process ownership, the PID + * claim) before failing, even when the caller keeps its scope open: Alchemy + * retries a failed local resource in the same process without closing it. + */ +export const startLocal = (options: LocalOptions) => + Effect.gen(function* () { + const attempt = yield* Scope.fork(yield* Scope.Scope) + return yield* acquireLocal(options).pipe( + Scope.provide(attempt), + Effect.onExit((exit) => (Exit.isFailure(exit) ? Scope.close(attempt, exit) : Effect.void)), + ) + }) diff --git a/packages/local-server/src/schema-identity.ts b/packages/local-server/src/schema-identity.ts new file mode 100644 index 0000000000..21f357cfb4 --- /dev/null +++ b/packages/local-server/src/schema-identity.ts @@ -0,0 +1,115 @@ +import schemaSql from "./schema/local-schema.sql" with { type: "text" } +import { schemaDigest as digestSchema, schemaFingerprint as fingerprintSchema } from "./store-version" +import { buildLocalSchemaManifest, type LocalSchemaManifest } from "./schema-manifest" +import { LOCAL_SCHEMA_VERSION } from "./local-schema-version" +import { LOCAL_SCHEMA_HISTORY } from "./local-schema-history" +import { CHDB_VERSION } from "./version" + +export { LOCAL_SCHEMA_HISTORY } from "./local-schema-history" + +/** + * Local-store schema version. + * + * Version 0 is the fingerprint-only legacy store represented by the recovery + * procedure from issue #297. Any future structural DDL change must increment + * this value and add a registered migration or an explicit unsupported edge. + */ +export { LOCAL_SCHEMA_VERSION } +export const LEGACY_LOCAL_SCHEMA_VERSION = 0 as const + +export const LEGACY_SCHEMA_PROJECT_REVISION = + "d58ce4a83d3ad3f3a29b9bb972272b757547ae793c050194354454634f3abccd" +export const LEGACY_SCHEMA_FINGERPRINT = "428701854f9fd30e" + +export const CURRENT_SCHEMA_PROJECT_REVISION = + "ed74788ef292834069e0ea6ee3b22d68fc604fb66cb54d2d551db67ce8d20b3a" +/** Revision recorded by the issue-297 recovery report. The refreshed upstream + * generator currently emits CURRENT_SCHEMA_PROJECT_REVISION; the structural + * fingerprint is the compatibility identity used by the migration. */ +export const ISSUE_297_TARGET_SCHEMA_PROJECT_REVISION = + "506bc745f7a7eca202ec905a6403a6815e86413faf0cd3cbbf73881023edce91" +export const LOCAL_SCHEMA_SQL = schemaSql +export const SCHEMA_FINGERPRINT = fingerprintSchema(schemaSql) +export const SCHEMA_DIGEST = digestSchema(schemaSql) +export const LOCAL_SCHEMA_MANIFEST: LocalSchemaManifest = buildLocalSchemaManifest(schemaSql) +export const LOCAL_SCHEMA_MANIFEST_DIGEST = LOCAL_SCHEMA_MANIFEST.digest + +export interface LocalSchemaIdentity { + readonly version: number + readonly fingerprint: string + readonly digest: string + readonly manifestDigest?: string + readonly chdb: string + readonly projectRevision?: string +} + +/** + * Per-version identities, frozen and read straight from the append-only + * history. Historical migration edges must never point at + * CURRENT_LOCAL_SCHEMA: when v10 ships, v0 -> v1 must still construct and verify + * v1 rather than silently changing its destination. + */ +export const localSchemaIdentity = (version: number): LocalSchemaIdentity => { + const entry = LOCAL_SCHEMA_HISTORY[version] + if (!entry || entry.version !== version) + throw new Error(`local schema history has no entry for version ${version}`) + return Object.freeze({ + version: entry.version, + fingerprint: entry.fingerprint, + digest: entry.digest, + manifestDigest: entry.manifestDigest, + chdb: CHDB_VERSION, + projectRevision: entry.projectRevision, + }) +} + +export const LOCAL_SCHEMA_V1 = localSchemaIdentity(1) +export const LOCAL_SCHEMA_V2 = localSchemaIdentity(2) +export const LOCAL_SCHEMA_V3 = localSchemaIdentity(3) +export const LOCAL_SCHEMA_V4 = localSchemaIdentity(4) +export const LOCAL_SCHEMA_V5 = localSchemaIdentity(5) +export const LOCAL_SCHEMA_V6 = localSchemaIdentity(6) +export const LOCAL_SCHEMA_V10 = localSchemaIdentity(10) +export const LOCAL_SCHEMA_V11 = localSchemaIdentity(11) +export const LOCAL_SCHEMA_V20 = localSchemaIdentity(20) +export const LOCAL_SCHEMA_V21 = localSchemaIdentity(21) +export const LOCAL_SCHEMA_V22 = localSchemaIdentity(22) +export const LOCAL_SCHEMA_V23 = localSchemaIdentity(23) +export const LOCAL_SCHEMA_V24 = localSchemaIdentity(24) +export const LOCAL_SCHEMA_V25 = localSchemaIdentity(25) +export const LOCAL_SCHEMA_V26 = localSchemaIdentity(26) +export const LOCAL_SCHEMA_V27 = localSchemaIdentity(27) +export const LOCAL_SCHEMA_V28 = localSchemaIdentity(28) +export const LOCAL_SCHEMA_V29 = localSchemaIdentity(29) +export const LOCAL_SCHEMA_V30 = localSchemaIdentity(30) +export const LOCAL_SCHEMA_V31 = localSchemaIdentity(31) +export const LOCAL_SCHEMA_V32 = localSchemaIdentity(32) +export const LOCAL_SCHEMA_V33 = localSchemaIdentity(33) +export const LOCAL_SCHEMA_V34 = localSchemaIdentity(34) +export const LOCAL_SCHEMA_V36 = localSchemaIdentity(36) +export const LOCAL_SCHEMA_V37 = localSchemaIdentity(37) +export const LOCAL_SCHEMA_V38 = localSchemaIdentity(38) +export const LOCAL_SCHEMA_V39 = localSchemaIdentity(39) +export const LOCAL_SCHEMA_V40 = localSchemaIdentity(40) +export const LOCAL_SCHEMA_V41 = localSchemaIdentity(41) +export const LOCAL_SCHEMA_V42 = localSchemaIdentity(42) + +export const CURRENT_LOCAL_SCHEMA: LocalSchemaIdentity = Object.freeze({ + version: LOCAL_SCHEMA_VERSION, + fingerprint: SCHEMA_FINGERPRINT, + digest: SCHEMA_DIGEST, + manifestDigest: LOCAL_SCHEMA_MANIFEST_DIGEST, + chdb: CHDB_VERSION, + projectRevision: CURRENT_SCHEMA_PROJECT_REVISION, +}) + +export const LEGACY_LOCAL_SCHEMA: LocalSchemaIdentity = { + version: LEGACY_LOCAL_SCHEMA_VERSION, + fingerprint: LEGACY_SCHEMA_FINGERPRINT, + digest: "", + chdb: CHDB_VERSION, + projectRevision: LEGACY_SCHEMA_PROJECT_REVISION, +} + +export const identityLabel = (identity: Pick): string => + `v${identity.version} (${identity.fingerprint})` diff --git a/apps/cli/src/server/schema-manifest.ts b/packages/local-server/src/schema-manifest.ts similarity index 100% rename from apps/cli/src/server/schema-manifest.ts rename to packages/local-server/src/schema-manifest.ts diff --git a/apps/cli/src/server/schema-physical.ts b/packages/local-server/src/schema-physical.ts similarity index 100% rename from apps/cli/src/server/schema-physical.ts rename to packages/local-server/src/schema-physical.ts diff --git a/apps/cli/src/server/schema/control-schema-v1.sql b/packages/local-server/src/schema/control-schema-v1.sql similarity index 100% rename from apps/cli/src/server/schema/control-schema-v1.sql rename to packages/local-server/src/schema/control-schema-v1.sql diff --git a/apps/cli/src/server/schema/control-schema.sql b/packages/local-server/src/schema/control-schema.sql similarity index 100% rename from apps/cli/src/server/schema/control-schema.sql rename to packages/local-server/src/schema/control-schema.sql diff --git a/apps/cli/src/server/schema/local-inserts.json b/packages/local-server/src/schema/local-inserts.json similarity index 100% rename from apps/cli/src/server/schema/local-inserts.json rename to packages/local-server/src/schema/local-inserts.json diff --git a/apps/cli/src/server/schema/local-schema.sql b/packages/local-server/src/schema/local-schema.sql similarity index 100% rename from apps/cli/src/server/schema/local-schema.sql rename to packages/local-server/src/schema/local-schema.sql diff --git a/apps/cli/src/server/scoped-delete.ts b/packages/local-server/src/scoped-delete.ts similarity index 100% rename from apps/cli/src/server/scoped-delete.ts rename to packages/local-server/src/scoped-delete.ts diff --git a/apps/cli/src/server/serve.ts b/packages/local-server/src/serve.ts similarity index 98% rename from apps/cli/src/server/serve.ts rename to packages/local-server/src/serve.ts index 84185bc47b..eb246e1ee9 100644 --- a/apps/cli/src/server/serve.ts +++ b/packages/local-server/src/serve.ts @@ -8,9 +8,8 @@ import { Clock, Context, Effect, Exit, Layer, Metric, Result, Schema, type Scope import * as ManagedRuntime from "effect/ManagedRuntime" import { resolve } from "node:path" import { gunzipSync } from "node:zlib" -import { TelemetryLayer } from "../core/telemetry" -import { connectionHostForBindHost, isLoopbackHostname, serverUrl } from "../lib/local-address" -import { MAPLE_VERSION } from "../version" +import { connectionHostForBindHost, isLoopbackHostname, serverUrl } from "./local-address" +import { MAPLE_VERSION } from "./version" import { acquireChdb, type Chdb, @@ -74,6 +73,7 @@ import { import { applyLocalIngestRules, localIngestRulesPath, makeLocalIngestRulesSource } from "./ingest-rules" import { localServiceMapRollupLoop } from "./service-map-rollup" import { ensureStoreMarkerDurable, storeHasData } from "./store-version" +import { describeError, errnoCode } from "./store-errors" import { ensureMaintenanceToken, maintenanceTokenMatches, @@ -108,6 +108,8 @@ export interface ServerOptions { readonly assets?: AssetResolver /** Ingest key and access token; omit for an open loopback server. */ readonly auth?: RemoteAuth + /** Host-owned request instrumentation. The embedded runtime never enables cloud exporters itself. */ + readonly telemetryLayer?: Layer.Layer } export class ServerBindError extends Schema.TaggedError()("@maple/cli/ServerBindError", { @@ -122,6 +124,22 @@ export class LocalServerConfigError extends Schema.TaggedError => + Effect.try({ + try: () => Bun.listen({ hostname, port, socket: { data() {} } }).stop(true), + catch: (error) => + new ServerBindError({ + hostname, + port, + message: + errnoCode(error) === "EADDRINUSE" + ? `port ${port} on ${hostname} is already in use; choose another with --port, or stop whatever holds it` + : `cannot bind ${hostname}:${port}: ${describeError(error)}`, + }), + }) + /** Vite dev server origins for apps/local-ui (`vite --port 4319`), which proxies here. */ export const LOCAL_UI_DEV_ORIGINS = [ "http://127.0.0.1:4319", @@ -1730,7 +1748,9 @@ export const startServer = ( ), ) const runtime = yield* Effect.acquireRelease( - Effect.sync(() => ManagedRuntime.make(Layer.mergeAll(TelemetryLayer, eventingLayer))), + Effect.sync(() => + ManagedRuntime.make(Layer.mergeAll(options.telemetryLayer ?? Layer.empty, eventingLayer)), + ), (rt) => Effect.promise(() => rt.dispose()), ) const services = yield* runtime.contextEffect.pipe( diff --git a/apps/cli/src/server/service-map-rollup.ts b/packages/local-server/src/service-map-rollup.ts similarity index 100% rename from apps/cli/src/server/service-map-rollup.ts rename to packages/local-server/src/service-map-rollup.ts diff --git a/packages/local-server/src/store-claim.ts b/packages/local-server/src/store-claim.ts new file mode 100644 index 0000000000..c4825124b1 --- /dev/null +++ b/packages/local-server/src/store-claim.ts @@ -0,0 +1,283 @@ +// Ownership of a local store: the PID file that serializes every process +// opening it, with stale-claim takeover and legacy-sidecar adoption. +import { Clock, Effect, Option } from "effect" +import { randomUUID } from "node:crypto" +import { linkSync, mkdirSync, readFileSync, renameSync, statSync, unlinkSync, writeFileSync } from "node:fs" +import { dirname } from "node:path" +import { errnoCode, type ServerStateFileError, stateFileError } from "./store-errors" +import { + adoptLegacySidecars, + legacySidecarPath, + ownsLegacySidecars, + SERVER_PID_NAME, + serverPidPath, +} from "./store-version" + +// --------------------------------------------------------------------------- +// Process liveness: is the owner named by a PID file still that process? +// --------------------------------------------------------------------------- + +/** Parse `ps -o etime=` (`[[dd-]hh:]mm:ss`) into whole seconds. */ +export const parseElapsedSeconds = (raw: string): number | undefined => { + const match = raw.trim().match(/^(?:(\d+)-)?(?:(\d+):)?(\d+):(\d+)$/) + if (!match) return undefined + const [, days, hours, minutes, seconds] = match + return Number(days ?? 0) * 86_400 + Number(hours ?? 0) * 3_600 + Number(minutes) * 60 + Number(seconds) +} + +/** + * Could a live process that started at `startedAtMs` have written a PID file + * last modified at `writtenAtMs`? A reused PID belongs to a process started + * after the writer died, so after the write. The slack covers `etime`'s + * whole-second resolution. + */ +export const startedBeforeWrite = (startedAtMs: number, writtenAtMs: number): boolean => + startedAtMs <= writtenAtMs + 2_000 + +/** Liveness probe via signal 0: a process primitive with no FileSystem + * equivalent. Never throws (errors mean "not alive"). + * + * A PID file naming *this* process is stale by definition: the previous server + * is the one that wrote it, so if its number is ours it has died and the OS + * reused the PID. In a container that is the common case, not the edge case: + * `maple start` is PID 1 on every restart, the file survives on the data + * volume, and `kill(1, 0)` always succeeds, so without this check a container + * whose server died once refuses to start forever ("already running (PID 1)"). + * Non-positive PIDs are rejected for the same reason: `kill(0, 0)` and + * `kill(-n, 0)` signal process groups and "succeed" for a group that exists. */ +export const isProcessAlive = (pid: number): boolean => { + if (!Number.isInteger(pid) || pid <= 0 || pid === process.pid) return false + try { + process.kill(pid, 0) + return true + } catch { + return false + } +} + +// --------------------------------------------------------------------------- +// PID file: the one claim that serializes every process opening a store. +// --------------------------------------------------------------------------- + +export interface PidFileSnapshot { + readonly raw: string + readonly pid: number | undefined + readonly ino: number + readonly mtimeMs: number +} + +export const readPidSnapshot = (path: string): Effect.Effect> => + Effect.try({ + try: (): PidFileSnapshot => { + const raw = readFileSync(path, "utf8") + const info = statSync(path) + const pid = Number.parseInt(raw.trim(), 10) + return { + raw, + pid: Number.isInteger(pid) && pid > 0 ? pid : undefined, + ino: info.ino, + mtimeMs: info.mtimeMs, + } + }, + catch: (error) => error, + }).pipe(Effect.option) + +export const sameSnapshot = (a: PidFileSnapshot, b: PidFileSnapshot): boolean => + a.raw === b.raw && a.ino === b.ino && a.mtimeMs === b.mtimeMs + +/** Wall-clock start of a live process (`ps -o etime=`), when it can be known. */ +const processStartedAtMs = (pid: number): Effect.Effect> => + Effect.try({ + try: () => + Bun.spawnSync(["ps", "-o", "etime=", "-p", String(pid)], { stdout: "pipe", stderr: "ignore" }), + catch: (error) => error, + }).pipe( + Effect.flatMap((result) => { + const seconds = result.success ? parseElapsedSeconds(result.stdout.toString()) : undefined + return seconds === undefined + ? Effect.succeed(Option.none()) + : Effect.map(Clock.currentTimeMillis, (nowMs) => Option.some(nowMs - seconds * 1000)) + }), + Effect.orElseSucceed(() => Option.none()), + ) + +export type PidOwner = + | { readonly kind: "stale" } + /** `verified`: the process provably started before the file was written. */ + | { readonly kind: "live"; readonly pid: number; readonly verified: boolean } + +export const STALE_PID: PidOwner = { kind: "stale" } +const livePid = (pid: number, verified: boolean): PidOwner => ({ kind: "live", pid, verified }) + +/** + * Does the process named by a PID file still own it? After a crash or reboot + * the number may belong to an unrelated process, which necessarily started + * after the file was written. Without `ps` the answer is "live, unverified", + * the conservative reading for every caller. + */ +export const pidOwner = (snapshot: PidFileSnapshot): Effect.Effect => + Effect.gen(function* () { + const pid = snapshot.pid + if (pid === undefined || !isProcessAlive(pid)) return STALE_PID + const startedAt = yield* processStartedAtMs(pid) + if (Option.isNone(startedAt)) return livePid(pid, false) + return startedBeforeWrite(startedAt.value, snapshot.mtimeMs) ? livePid(pid, true) : STALE_PID + }) + +/** O_EXCL create; `false` when the file already exists. */ +const createExclusive = (path: string, content: string): Effect.Effect => + Effect.try({ + try: () => { + writeFileSync(path, content, { mode: 0o600, flag: "wx" }) + return true + }, + catch: (error) => error, + }).pipe( + Effect.catch((error) => + errnoCode(error) === "EEXIST" + ? Effect.succeed(false) + : Effect.fail(stateFileError(path, "create")(error)), + ), + ) + +/** Atomically replace the file's content (write aside, rename over). */ +const replaceFile = (path: string, content: string): Effect.Effect => + Effect.try({ + try: () => { + const temporary = `${path}.${randomUUID()}.tmp` + writeFileSync(temporary, content, { mode: 0o600, flag: "wx" }) + renameSync(temporary, path) + }, + catch: stateFileError(path, "replace"), + }) + +export const ignoreFailure = (thunk: () => void): Effect.Effect => + Effect.try({ try: thunk, catch: (error) => error }).pipe(Effect.ignore) + +/** + * Remove a stale PID file without ever deleting a claim that a concurrent + * start published after we read it: move it aside, and only discard it if it + * is byte-for-byte the file we judged stale; otherwise put it back and yield. + */ +const takeOverStalePidFile = ( + path: string, + observed: PidFileSnapshot, + busy: (pid: number | undefined) => E, +): Effect.Effect => + Effect.gen(function* () { + const moved = `${path}.stale-${randomUUID()}` + const renamed = yield* Effect.try({ + try: () => { + renameSync(path, moved) + return true + }, + catch: (error) => error, + }).pipe( + Effect.catch((error) => + errnoCode(error) === "ENOENT" + ? Effect.succeed(false) + : Effect.fail(stateFileError(path, "replace the stale PID file")(error)), + ), + ) + if (!renamed) return + const current = yield* readPidSnapshot(moved) + if (Option.isSome(current) && sameSnapshot(current.value, observed)) { + yield* ignoreFailure(() => unlinkSync(moved)) + return + } + yield* ignoreFailure(() => linkSync(moved, path)) + yield* ignoreFailure(() => unlinkSync(moved)) + return yield* Effect.fail(busy(Option.isSome(current) ? current.value.pid : undefined)) + }) + +/** + * Exclusively claim the PID file so exactly one `start`, `reset` or `restore` + * touches the store. A stale file (dead or reused PID) is taken over safely. + * `handoverFrom` lets a `start -d` child inherit its parent's claim; `busy` + * builds the host's refusal when another live process holds it. + */ +export const claimPidFileExclusive = ( + pidPath: string, + options: { + readonly handoverFrom?: number | undefined + readonly busy: (pid: number | undefined) => E + }, +): Effect.Effect => + Effect.gen(function* () { + const { busy } = options + yield* Effect.try({ + try: () => mkdirSync(dirname(pidPath), { recursive: true }), + catch: stateFileError(dirname(pidPath), "create"), + }) + const content = String(process.pid) + for (let attempt = 0; attempt < 4; attempt++) { + if (yield* createExclusive(pidPath, content)) return + const observed = yield* readPidSnapshot(pidPath) + if (Option.isNone(observed)) continue + const owner = yield* pidOwner(observed.value) + if (owner.kind === "live") { + if (owner.pid === options.handoverFrom) return yield* replaceFile(pidPath, content) + return yield* Effect.fail(busy(owner.pid)) + } + yield* takeOverStalePidFile(pidPath, observed.value, busy) + } + return yield* Effect.fail(busy(undefined)) + }) + +/** Remove the PID file only while it still names this process. */ +export const releasePidFile = (pidPath: string): Effect.Effect => + Effect.gen(function* () { + const current = yield* readPidSnapshot(pidPath) + if (Option.isSome(current) && current.value.pid === process.pid) { + yield* ignoreFailure(() => unlinkSync(pidPath)) + } + }) + +/** The PID file of the server that owns `dataDir`: the current-layout one, or + * the one an older maple wrote for a custom `--data-dir` before upgrading. */ +export const locatePidFile = ( + dataDir: string, +): Effect.Effect> => + Effect.gen(function* () { + const candidates = [ + serverPidPath(dataDir), + ...(ownsLegacySidecars(dataDir) ? [legacySidecarPath(dataDir, SERVER_PID_NAME)] : []), + ] + for (const path of candidates) { + const snapshot = yield* readPidSnapshot(path) + if (Option.isSome(snapshot)) return Option.some({ path, snapshot: snapshot.value }) + } + return Option.none() + }) + +/** Refuse when a live server owns `dataDir`, including one an older maple + * started from the legacy PID file. */ +export const refuseIfRunning = ( + dataDir: string, + refusal: (pid: number | undefined) => E, +): Effect.Effect => + Effect.gen(function* () { + const located = yield* locatePidFile(dataDir) + if (Option.isNone(located)) return + const owner = yield* pidOwner(located.value.snapshot) + if (owner.kind === "live") return yield* Effect.fail(refusal(owner.pid)) + }) + +/** Explicit adoption of pre-namespacing state files; returns what it adopted. */ +export const adoptLegacyStoreFiles = ( + dataDir: string, +): Effect.Effect, ServerStateFileError> => + Effect.try({ + try: () => adoptLegacySidecars(dataDir), + catch: stateFileError(dataDir, "adopt the legacy state files beside"), + }) + +/** Own `dataDir` until the scope closes, with the same protocol as `maple start`. */ +export const acquireStoreClaim = (dataDir: string, busy: (pid: number | undefined) => E) => + Effect.gen(function* () { + yield* adoptLegacyStoreFiles(dataDir) + yield* refuseIfRunning(dataDir, busy) + yield* Effect.acquireRelease(claimPidFileExclusive(serverPidPath(dataDir), { busy }), () => + releasePidFile(serverPidPath(dataDir)), + ) + }) diff --git a/packages/local-server/src/store-errors.ts b/packages/local-server/src/store-errors.ts new file mode 100644 index 0000000000..32f32c3f33 --- /dev/null +++ b/packages/local-server/src/store-errors.ts @@ -0,0 +1,99 @@ +import { Schema } from "effect" + +// Failures any host hits while claiming or opening a local store. The tags keep +// their `@maple/cli/` prefix: the CLI's exit handling and its telemetry group +// failures by tag, and these are the same failures `maple start` has always raised. + +export const describeError = (error: unknown): string => + error instanceof Error ? error.message : String(error) + +export const errnoCode = (error: unknown): string | undefined => + typeof error === "object" && error !== null && "code" in error && typeof error.code === "string" + ? error.code + : undefined + +export const stateFileError = + (path: string, action: string) => + (error: unknown): ServerStateFileError => + new ServerStateFileError({ path, message: `could not ${action} ${path}: ${describeError(error)}` }) + +/** + * A refused command whose precondition simply wasn't met: the server is already + * running, isn't running at all, or a destructive command was not confirmed. + * The message and the non-zero exit are identical to a genuine failure; the + * separate tag exists so `bin.ts` can close the root span `Ok` for these without + * also swallowing real start failures (`ServerBindError`, + * `BackgroundServerTimeoutError`, `LocalStoreDirtyError`). Same rule the ingest + * gateway follows for expected 4xx. + */ +export class ServerStateError extends Schema.TaggedError()("@maple/cli/ServerStateError", { + message: Schema.String, +}) {} + +/** A PID, discovery, or legacy state file beside the data dir could not be read or written. */ +export class ServerStateFileError extends Schema.TaggedError()( + "@maple/cli/ServerStateFileError", + { path: Schema.String, message: Schema.String }, +) {} + +/** The dirty-store policy in force when the store turned out to be unclean. */ +export const DirtyStorePolicy = Schema.Literals(["fail", "restore-checkpoint", "reset"]) + +/** + * The store was left open by a server that died without closing chDB. Reopening + * it can crash chDB natively, so startup refuses. + * + * `checkpointAvailable` is the field worth grouping on: without a checkpoint + * there is nothing to restore and the only way forward is `--reset`, which is a + * materially different user experience from the recoverable case. + */ +export class LocalStoreDirtyError extends Schema.TaggedError()( + "@maple/cli/LocalStoreDirtyError", + { + dataDir: Schema.String, + policy: DirtyStorePolicy, + checkpointAvailable: Schema.Boolean, + message: Schema.String, + }, +) {} + +/** The store was written by a different chDB build; loading it would SIGTRAP. */ +export class LocalStoreIncompatibleError extends Schema.TaggedError()( + "@maple/cli/LocalStoreIncompatibleError", + { + dataDir: Schema.String, + storeBuild: Schema.String, + currentBuild: Schema.String, + message: Schema.String, + }, +) {} + +/** The store was bootstrapped from an older bundled schema and needs migrating. */ +export class LocalStoreSchemaStaleError extends Schema.TaggedError()( + "@maple/cli/LocalStoreSchemaStaleError", + { dataDir: Schema.String, message: Schema.String }, +) {} + +/** + * The store's marker records a schema version newer than this build knows: a + * downgrade. Distinct from a stale schema because the remedy is reinstalling + * the newer maple, never a reset. + */ +export class LocalStoreFromNewerMapleError extends Schema.TaggedError()( + "@maple/cli/LocalStoreFromNewerMapleError", + { + dataDir: Schema.String, + storeSchemaVersion: Schema.Number, + currentSchemaVersion: Schema.Number, + message: Schema.String, + }, +) {} + +/** Which part of the local-store migration journal the command was doing. */ +export const MigrationPhase = Schema.Literals(["read-journal", "resume", "preserve"]) + +/** An unfinished schema migration blocks startup, or its journal is unreadable. */ +export class LocalStoreMigrationError extends Schema.TaggedError()( + "@maple/cli/LocalStoreMigrationError", + { dataDir: Schema.String, phase: MigrationPhase, message: Schema.String }, +) {} diff --git a/apps/cli/src/server/store-version.ts b/packages/local-server/src/store-version.ts similarity index 99% rename from apps/cli/src/server/store-version.ts rename to packages/local-server/src/store-version.ts index dbf306890a..95039b48b7 100644 --- a/apps/cli/src/server/store-version.ts +++ b/packages/local-server/src/store-version.ts @@ -10,7 +10,7 @@ import { createHash, randomUUID } from "node:crypto" import { existsSync, readFileSync, statSync, unlinkSync } from "node:fs" import { basename, dirname, join, resolve } from "node:path" import { Effect, Schema, SchemaGetter } from "effect" -import { CHDB_VERSION } from "../version" +import { CHDB_VERSION } from "./version" import { Digest64, Fingerprint16, IsoOrUnknown } from "./identity-schema" import { durableRemove, durableRenameSync, durableWrite, durableWriteFileSync } from "./durable-files" diff --git a/apps/cli/src/server/ui-assets.ts b/packages/local-server/src/ui-assets.ts similarity index 97% rename from apps/cli/src/server/ui-assets.ts rename to packages/local-server/src/ui-assets.ts index 7953c98f96..ed9ff02fa5 100644 --- a/apps/cli/src/server/ui-assets.ts +++ b/packages/local-server/src/ui-assets.ts @@ -55,7 +55,7 @@ export const resolveUiAssets = (): Effect.Effect false)))) return undefined const entries = yield* fs diff --git a/apps/cli/src/server/ui-embed.gen.ts b/packages/local-server/src/ui-embed.gen.ts similarity index 100% rename from apps/cli/src/server/ui-embed.gen.ts rename to packages/local-server/src/ui-embed.gen.ts diff --git a/apps/cli/src/version.ts b/packages/local-server/src/version.ts similarity index 61% rename from apps/cli/src/version.ts rename to packages/local-server/src/version.ts index 0047f65acf..68f3b87427 100644 --- a/apps/cli/src/version.ts +++ b/packages/local-server/src/version.ts @@ -5,6 +5,11 @@ // `typeof` guards the reference so an un-defined identifier (the dev path, // `bun run src/bin.ts`, where no --define is passed) reports "dev" instead of // throwing a ReferenceError. +// +// Only a `maple` release may set it: stores, checkpoints and chDB-load errors +// turn any other value into `maple update --tag v` advice. Embedded +// hosts that are not a `maple` release (the @maple-dev/alchemy sidecar) leave +// it "dev" on purpose. declare const __MAPLE_VERSION__: string | undefined const raw = typeof __MAPLE_VERSION__ !== "undefined" ? __MAPLE_VERSION__ : "dev" @@ -13,11 +18,12 @@ const raw = typeof __MAPLE_VERSION__ !== "undefined" ? __MAPLE_VERSION__ : "dev" // release tags like "v0.5.0" to avoid a doubled "vv0.5.0". export const MAPLE_VERSION: string = raw.replace(/^v/, "") -// The libchdb release the binary was built against (e.g. "v26.1.0"), baked in by -// `scripts/build-local-binary.sh` via `bun build --define`. It identifies the -// chDB on-disk-store format, so the store-version guard can refuse a directory -// written by an incompatible build. Dev runs (`bun run src/bin.ts`, no --define) -// report "dev". +// The libchdb release the build pairs with (e.g. "v26.7.3"), baked in via +// `--define` from packages/local-server/libchdb.json by both the `maple` build +// and the @maple-dev/alchemy sidecar build. It identifies the chDB +// on-disk-store format, so the store-version guard can refuse a directory +// written by an incompatible build. Dev runs (`bun run src/bin.ts`, no +// --define) report "dev". declare const __CHDB_VERSION__: string | undefined export const CHDB_VERSION: string = typeof __CHDB_VERSION__ !== "undefined" ? __CHDB_VERSION__ : "dev" diff --git a/packages/local-server/test/libchdb-provision.test.ts b/packages/local-server/test/libchdb-provision.test.ts new file mode 100644 index 0000000000..6abad93e5d --- /dev/null +++ b/packages/local-server/test/libchdb-provision.test.ts @@ -0,0 +1,99 @@ +import { describe, expect, it } from "vitest" +import { Effect, Exit } from "effect" +import { FetchHttpClient } from "effect/http" +import { createHash } from "node:crypto" +import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { platformLibchdbAsset, provisionLibchdb, type ProvisionOptions } from "../src/libchdb-provision" + +const provision = (options: ProvisionOptions) => + provisionLibchdb(options).pipe(Effect.provide(FetchHttpClient.layer)) + +const asset = platformLibchdbAsset() +if (asset === undefined) throw new Error(`no libchdb asset for ${process.platform}-${process.arch}`) + +/** A release archive shaped like chdb-core's: the library nested in a directory. */ +const releaseArchive = (root: string, library: string): Uint8Array => { + const tree = join(root, "tree") + mkdirSync(join(tree, "lib"), { recursive: true }) + writeFileSync(join(tree, "lib", "libchdb.so"), library) + const archive = join(root, asset) + const tar = Bun.spawnSync(["tar", "-czf", archive, "-C", tree, "lib"]) + if (tar.exitCode !== 0) throw new Error("tar failed") + return new Uint8Array(readFileSync(archive)) +} + +const withReleaseServer = async ( + run: (ctx: { + baseUrl: string + cacheRoot: string + archive: Uint8Array + requests: string[] + }) => Promise, + respond?: (path: string) => Response | undefined, +) => { + const root = mkdtempSync(join(tmpdir(), "maple-libchdb-provision-")) + const archive = releaseArchive(root, "fake libchdb bytes") + const requests: string[] = [] + const server = Bun.serve({ + port: 0, + fetch: (request) => { + const path = new URL(request.url).pathname + requests.push(path) + return ( + respond?.(path) ?? + (path === `/v1.0.0/${asset}` ? new Response(archive) : new Response(null, { status: 404 })) + ) + }, + }) + try { + await run({ + baseUrl: `http://127.0.0.1:${server.port}`, + cacheRoot: join(root, "cache"), + archive, + requests, + }) + } finally { + server.stop(true) + rmSync(root, { recursive: true, force: true }) + } +} + +const sha256 = (bytes: Uint8Array) => createHash("sha256").update(bytes).digest("hex") + +describe("provisionLibchdb", () => { + it("installs a verified release once and reuses the cached copy", async () => { + await withReleaseServer(async ({ baseUrl, cacheRoot, archive, requests }) => { + const release = { version: "v1.0.0", sha256: { [asset]: sha256(archive) } } + const first = await Effect.runPromise(provision({ cacheRoot, baseUrl, release })) + expect(first.downloaded).toBe(true) + expect(readFileSync(first.path, "utf8")).toBe("fake libchdb bytes") + + const second = await Effect.runPromise(provision({ cacheRoot, baseUrl, release })) + expect(second).toEqual({ ...first, downloaded: false }) + expect(requests).toEqual([`/v1.0.0/${asset}`]) + // No staging directories survive a successful install. + expect(readdirSync(cacheRoot)).toEqual(["v1.0.0"]) + }) + }) + + it("refuses an archive whose checksum does not match the pin and installs nothing", async () => { + await withReleaseServer(async ({ baseUrl, cacheRoot }) => { + const release = { version: "v1.0.0", sha256: { [asset]: "0".repeat(64) } } + const exit = await Effect.runPromiseExit(provision({ cacheRoot, baseUrl, release })) + expect(Exit.isFailure(exit)).toBe(true) + expect(JSON.stringify(exit)).toContain("checksum mismatch") + expect(existsSync(join(cacheRoot, "v1.0.0"))).toBe(false) + }) + }) + + it("fails on a missing release without retrying", async () => { + await withReleaseServer(async ({ baseUrl, cacheRoot, requests }) => { + const release = { version: "v2.0.0", sha256: { [asset]: "0".repeat(64) } } + const exit = await Effect.runPromiseExit(provision({ cacheRoot, baseUrl, release })) + expect(JSON.stringify(exit)).toContain("HTTP 404") + expect(requests).toEqual([`/v2.0.0/${asset}`]) + }) + }) +}) diff --git a/packages/local-server/test/local-runtime.test.ts b/packages/local-server/test/local-runtime.test.ts new file mode 100644 index 0000000000..a1dd2ffad3 --- /dev/null +++ b/packages/local-server/test/local-runtime.test.ts @@ -0,0 +1,110 @@ +import { describe, expect, it } from "vitest" +import { Effect, Exit, Scope } from "effect" +import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { startLocal } from "../src/runtime" +import { CURRENT_LOCAL_SCHEMA } from "../src/schema-identity" +import { markStoreOpen, serverPidPath, storeMarkerPath, storeOpenMarkerPath } from "../src/store-version" +import { CHDB_VERSION } from "../src/version" + +const withStore = async (run: (dataDir: string) => Promise) => { + const root = mkdtempSync(join(tmpdir(), "maple-local-runtime-")) + const dataDir = join(root, "data") + mkdirSync(join(dataDir, "metadata"), { recursive: true }) + writeFileSync( + storeMarkerPath(dataDir), + JSON.stringify({ + formatVersion: 2, + storeId: "8b7d9f1e-0000-4000-8000-000000000001", + chdb: CHDB_VERSION, + maple: "dev", + createdAt: "2026-01-01T00:00:00.000Z", + schemaVersion: CURRENT_LOCAL_SCHEMA.version, + schemaDigest: CURRENT_LOCAL_SCHEMA.digest, + schema: CURRENT_LOCAL_SCHEMA.fingerprint, + activation: "active", + }), + ) + try { + await run(dataDir) + } finally { + rmSync(root, { recursive: true, force: true }) + } +} + +// These refusal paths deliberately require no native library. Native opening, +// ingest and finalization are covered by the host's end-to-end smoke scenario. +describe("scoped local runtime safety", () => { + it("refuses dirty stores before opening chDB and releases its claim", async () => { + await withStore(async (dataDir) => { + markStoreOpen(dataDir) + const error = await Effect.runPromise( + Effect.scoped(startLocal({ dataDir, port: 0 })).pipe(Effect.flip), + ) + expect(error._tag).toBe("@maple/cli/LocalStoreDirtyError") + expect(existsSync(storeOpenMarkerPath(dataDir))).toBe(true) + expect(existsSync(serverPidPath(dataDir))).toBe(false) + }) + }) + + it("releases a failed start even when the host keeps its scope open, so a retry can proceed", async () => { + await withStore(async (dataDir) => { + markStoreOpen(dataDir) + // Alchemy retries a failed local resource in the same process without + // closing the scope the failed attempt ran in. + const held = await Effect.runPromise(Scope.make()) + const attempt = startLocal({ dataDir, port: 0 }).pipe(Scope.provide(held), Effect.flip) + const first = await Effect.runPromise(attempt) + expect(existsSync(serverPidPath(dataDir))).toBe(false) + const retry = await Effect.runPromise(attempt) + expect([first._tag, retry._tag]).toEqual([ + "@maple/cli/LocalStoreDirtyError", + "@maple/cli/LocalStoreDirtyError", + ]) + await Effect.runPromise(Scope.close(held, Exit.void)) + }) + }) + + it("wipes a dirty store's live data only when asked to", async () => { + await withStore(async (dataDir) => { + const liveData = join(dataDir, "metadata", "default.sql") + writeFileSync(liveData, "ATTACH DATABASE default") + markStoreOpen(dataDir) + // A library path that cannot load stops startup right after the store + // decision, so the test needs no native chDB. + const exit = await Effect.runPromiseExit( + Effect.scoped( + startLocal({ + dataDir, + port: 0, + onDirtyStore: "wipe", + libchdb: join(dataDir, "missing.so"), + }), + ), + ) + expect(JSON.stringify(exit)).not.toContain("LocalStoreDirtyError") + expect(existsSync(liveData)).toBe(false) + expect(existsSync(serverPidPath(dataDir))).toBe(false) + }) + }) + + it("refuses an incompatible populated store without loading its native metadata", async () => { + await withStore(async (dataDir) => { + writeFileSync( + storeMarkerPath(dataDir), + JSON.stringify({ + chdb: "incompatible-test-build", + maple: "test", + createdAt: "2026-01-01", + schema: CURRENT_LOCAL_SCHEMA.fingerprint, + }), + ) + const error = await Effect.runPromise( + Effect.scoped(startLocal({ dataDir, port: 0 })).pipe(Effect.flip), + ) + expect(error._tag).toBe("@maple/cli/LocalStoreIncompatibleError") + expect(existsSync(serverPidPath(dataDir))).toBe(false) + }) + }) +}) diff --git a/packages/local-server/tsconfig.json b/packages/local-server/tsconfig.json new file mode 100644 index 0000000000..1df40e1c1a --- /dev/null +++ b/packages/local-server/tsconfig.json @@ -0,0 +1,24 @@ +{ + "include": ["src/**/*.ts", "src/**/*.d.ts", "test/**/*.ts"], + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "lib": ["ES2022"], + "types": ["bun", "node"], + "moduleResolution": "bundler", + "resolveJsonModule": true, + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "noEmit": true, + "skipLibCheck": true, + "strict": true, + "noFallthroughCasesInSwitch": true, + "noUncheckedSideEffectImports": true, + "plugins": [ + { + "name": "@effect/language-service", + "reportSuggestionsAsWarningsInTsc": true + } + ] + } +} diff --git a/packages/query-engine/src/local.ts b/packages/query-engine/src/local.ts index ff30c1cfdc..cdf465bad0 100644 --- a/packages/query-engine/src/local.ts +++ b/packages/query-engine/src/local.ts @@ -3,7 +3,7 @@ // The endpoint runs raw SQL through the in-process chDB session and returns a // bare JSON array. // -// The output FORMAT is owned by the server: `apps/cli/src/server/query-guard.ts` +// The output FORMAT is owned by the server: `packages/local-server/src/query-guard.ts` // peels whatever trailing `FORMAT ` the compiler emitted (`CH.compile(...)` // appends `FORMAT JSON`) and runs the single read-only statement as JSON rows. // So callers POST `compiled.sql` verbatim. diff --git a/scripts/build-local-binary.sh b/scripts/build-local-binary.sh index 18192f26f0..e85bdf08d2 100755 --- a/scripts/build-local-binary.sh +++ b/scripts/build-local-binary.sh @@ -5,12 +5,12 @@ # Pipeline: # 1. Build the workspace packages consumed through gitignored `dist/` paths. # 2. Build the lightweight SPA (`apps/local-ui` → its `dist/`). -# 3. Inline that dist into apps/cli/src/server/ui-embed.gen.ts so +# 3. Inline that dist into packages/local-server/src/ui-embed.gen.ts so # `bun build --compile` bakes the SPA into the binary. # 4. Compile apps/cli (the CLI + the OTLP-ingest/query server) into a single # executable with `bun build --compile`. The schema artifacts and SPA are # embedded; the OTLP encoders run in-process; chDB is reached via bun:ffi. -# 5. Download libchdb (v26.7.3, matching what we test against) for the host +# 5. Download libchdb (pinned in packages/local-server/libchdb.json) for the host # platform and place it beside the binary. At runtime `maple` dlopens the # sibling libchdb (resolved relative to its own path) — no rpath tricks. # 6. Restore the committed ui-embed.gen.ts stub so the tree stays clean. @@ -26,8 +26,11 @@ set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" OUT_DIR="${OUT_DIR:-$REPO_ROOT/dist}" -LIBCHDB_VERSION="${LIBCHDB_VERSION:-v26.7.3}" -UI_EMBED="$REPO_ROOT/apps/cli/src/server/ui-embed.gen.ts" +# The chDB release this build pairs with, and each platform asset's sha256. The +# @maple-dev/alchemy sidecar builds and downloads from the same manifest. +LIBCHDB_MANIFEST="$REPO_ROOT/packages/local-server/libchdb.json" +LIBCHDB_VERSION="${LIBCHDB_VERSION:-$(bun -p "require(process.argv[1]).version" "$LIBCHDB_MANIFEST")}" +UI_EMBED="$REPO_ROOT/packages/local-server/src/ui-embed.gen.ts" # Version baked into the binary via `bun build --define`. The release workflow # passes the tag; a local build defaults to `git describe` (or "dev"). @@ -79,20 +82,17 @@ case "$MAPLE_TARGET" in darwin-arm64) ASSET="macos-arm64-libchdb.tar.gz" ;; *) echo "ERROR: unsupported MAPLE_TARGET $MAPLE_TARGET" >&2; exit 1 ;; esac -# Expected sha256 per (LIBCHDB_VERSION, platform asset): release assets are +# Expected sha256 per platform asset comes from the manifest: release assets are # mutable even for a fixed tag, so this is verified before extraction. Bumping -# LIBCHDB_VERSION means adding each asset's hash here. A `case` rather than an -# associative array because macOS still ships bash 3.2. +# the release means updating its version and every asset hash there. # v26.1.0's macos-arm64 dylib put its LINKEDIT string pool at a non-8-byte-aligned # offset, which newer dyld rejects ("mis-aligned LINKEDIT string pool"). Check # `otool -l libchdb.so` (LC_SYMTAB stroff % 8 == 0) before pinning a new release. -case "$LIBCHDB_VERSION/$ASSET" in - v26.7.3/linux-x86_64-libchdb.tar.gz) EXPECTED_SHA256="bc33260c32acf78eade2ac41a9115f38e00404651fa42e3bb4c419e4f011c031" ;; - v26.7.3/linux-aarch64-libchdb.tar.gz) EXPECTED_SHA256="d153adad1ff39b2e3caf0417f09d8bd9edd41939c7c67a3c4978a61e73fb9227" ;; - v26.7.3/macos-x86_64-libchdb.tar.gz) EXPECTED_SHA256="af5ded3ed3e84c31af1cd198dcf459f11d2b6aad4f6ddeccc04b8a519b0300fc" ;; - v26.7.3/macos-arm64-libchdb.tar.gz) EXPECTED_SHA256="5640e50dccf711bf3dd5551333d08e43f433edf7bd94b2289f36c2539e627762" ;; - *) echo "ERROR: no pinned sha256 for $ASSET at LIBCHDB_VERSION=$LIBCHDB_VERSION" >&2; exit 1 ;; -esac +EXPECTED_SHA256="$(bun -p 'const m = require(process.argv[1]); m.version === process.argv[2] ? (m.sha256[process.argv[3]] ?? "") : ""' "$LIBCHDB_MANIFEST" "$LIBCHDB_VERSION" "$ASSET")" +if [ -z "$EXPECTED_SHA256" ]; then + echo "ERROR: no pinned sha256 for $ASSET at LIBCHDB_VERSION=$LIBCHDB_VERSION (see $LIBCHDB_MANIFEST)" >&2 + exit 1 +fi URL="https://github.com/chdb-io/chdb-core/releases/download/$LIBCHDB_VERSION/$ASSET" TMP="$(mktemp -d)" diff --git a/scripts/bump-local-control-schema.ts b/scripts/bump-local-control-schema.ts index cff1a399e9..e1c8d13326 100644 --- a/scripts/bump-local-control-schema.ts +++ b/scripts/bump-local-control-schema.ts @@ -1,9 +1,9 @@ /** Scaffold an independent SQLite control-schema identity; the author supplies its migration edge. */ import { createHash } from "node:crypto" import { existsSync, readFileSync, writeFileSync } from "node:fs" -import { LOCAL_CONTROL_SCHEMA_VERSION } from "../apps/cli/src/server/local-schema-version" +import { LOCAL_CONTROL_SCHEMA_VERSION } from "../packages/local-server/src/local-schema-version" -const root = "apps/cli/src/server" +const root = "packages/local-server/src" const next = LOCAL_CONTROL_SCHEMA_VERSION + 1 const currentPath = `${root}/schema/control-schema.sql` const snapshotPath = `${root}/schema/control-schema-v${next}.sql` diff --git a/scripts/bump-local-schema.ts b/scripts/bump-local-schema.ts index c98d5993e4..0b7eb4b5b2 100644 --- a/scripts/bump-local-schema.ts +++ b/scripts/bump-local-schema.ts @@ -2,9 +2,10 @@ * Scaffold a local-store schema version bump. * * Everything the append-only gate demands except the DDL itself is a pure - * function of one integer: the snapshot copy, the version constant, three edit - * sites in `schema-identity.ts`, the history entry's four hashes, a new row in - * the step table, and the pinned literals in both the bun test and the native + * function of one integer: the snapshot copy, the version constant, two edit + * sites in the CLI's `schema-snapshots.ts`, one in `schema-identity.ts`, the + * history entry's four hashes, a new row in the step table, and the pinned + * literals in both the bun test and the native * probe. This script writes all of them, then leaves the row's operations to a * human, which is the only part that was ever real work. * @@ -16,12 +17,16 @@ */ import { execFileSync } from "node:child_process" import { existsSync, readFileSync, writeFileSync } from "node:fs" -import { buildLocalSchemaManifest } from "../apps/cli/src/server/schema-manifest" -import { schemaDigest, schemaFingerprint } from "../apps/cli/src/server/store-version" +import { buildLocalSchemaManifest } from "../packages/local-server/src/schema-manifest" +import { schemaDigest, schemaFingerprint } from "../packages/local-server/src/store-version" -const SERVER_DIR = "apps/cli/src/server" -const STEPS_FILE = `${SERVER_DIR}/local-store-migrations/steps.ts` +// The current schema and its identity live in @maple/local-server; the frozen +// snapshots and the migration steps that read them stay in the CLI. +const SERVER_DIR = "packages/local-server/src" +const CLI_SERVER_DIR = "apps/cli/src/server" +const STEPS_FILE = `${CLI_SERVER_DIR}/local-store-migrations/steps.ts` const SCHEMA_DIR = `${SERVER_DIR}/schema` +const SNAPSHOT_DIR = `${CLI_SERVER_DIR}/schema` const TEST_FILE = "apps/cli/test/local-store-migrations.test.ts" const NATIVE_PROBE = "apps/cli/test/native-local-store-migration.sh" @@ -96,11 +101,11 @@ if (!versionMatch) fail("could not read LOCAL_SCHEMA_VERSION") const from = Number(versionMatch[1]) const to = from + 1 -const snapshotPath = `${SCHEMA_DIR}/local-schema-v${to}.sql` +const snapshotPath = `${SNAPSHOT_DIR}/local-schema-v${to}.sql` if (existsSync(snapshotPath)) fail(`${snapshotPath} already exists; v${to} looks already bumped`) // The identity is computed from the generated DDL directly rather than through -// `schema-identity.ts`, which cannot even be imported until the v{to} snapshot +// `schema-snapshots.ts`, which cannot even be imported until the v{to} snapshot // it statically imports exists. const currentSql = read(`${SCHEMA_DIR}/local-schema.sql`) const identity = { @@ -163,12 +168,15 @@ const plannedVersion: PlannedWrite = plan(`${SERVER_DIR}/local-schema-version.ts ], ]) -const plannedIdentity: PlannedWrite = plan(`${SERVER_DIR}/schema-identity.ts`, [ +const plannedSnapshots: PlannedWrite = plan(`${CLI_SERVER_DIR}/schema-snapshots.ts`, [ [ `import schemaV${from}Sql from "./schema/local-schema-v${from}.sql" with { type: "text" }`, `import schemaV${from}Sql from "./schema/local-schema-v${from}.sql" with { type: "text" }\nimport schemaV${to}Sql from "./schema/local-schema-v${to}.sql" with { type: "text" }`, ], [`\tschemaV${from}Sql,\n]`, `\tschemaV${from}Sql,\n\tschemaV${to}Sql,\n]`], +]) + +const plannedIdentity: PlannedWrite = plan(`${SERVER_DIR}/schema-identity.ts`, [ // The test pins CURRENT_LOCAL_SCHEMA to this constant; steps look versions up by number. [ `export const LOCAL_SCHEMA_V${from} = localSchemaIdentity(${from})`, @@ -237,6 +245,7 @@ const writes: ReadonlyArray = [ // The retained snapshot is the generated DDL, verbatim. { path: snapshotPath, content: currentSql }, plannedVersion, + plannedSnapshots, plannedIdentity, plannedHistory, plannedSteps, @@ -256,7 +265,7 @@ console.log(`bumped local schema v${from} -> v${to} (${identity.fingerprint}) ${snapshotPath} ${STEPS_FILE} <- fill in the ${moduleId} row -edited: local-schema-version.ts, schema-identity.ts, local-schema-history.ts, +edited: local-schema-version.ts, schema-snapshots.ts, schema-identity.ts, local-schema-history.ts, ${TEST_FILE}, ${NATIVE_PROBE} next: diff --git a/scripts/check-local-schema-manifest.ts b/scripts/check-local-schema-manifest.ts index 04bdceb002..3822fe4d6e 100644 --- a/scripts/check-local-schema-manifest.ts +++ b/scripts/check-local-schema-manifest.ts @@ -1,18 +1,18 @@ import { createHash } from "node:crypto" import { readFileSync } from "node:fs" -import { LOCAL_CONTROL_SCHEMA_HISTORY } from "../apps/cli/src/server/local-schema-history" -import { LOCAL_CONTROL_SCHEMA_VERSION } from "../apps/cli/src/server/local-schema-version" +import { LOCAL_CONTROL_SCHEMA_HISTORY } from "../packages/local-server/src/local-schema-history" +import { LOCAL_CONTROL_SCHEMA_VERSION } from "../packages/local-server/src/local-schema-version" import { execFileSync } from "node:child_process" import { CURRENT_LOCAL_SCHEMA, LOCAL_SCHEMA_HISTORY, LOCAL_SCHEMA_MANIFEST, LOCAL_SCHEMA_MANIFEST_DIGEST, - LOCAL_SCHEMA_SNAPSHOTS, LOCAL_SCHEMA_VERSION, -} from "../apps/cli/src/server/schema-identity" +} from "../packages/local-server/src/schema-identity" +import { LOCAL_SCHEMA_SNAPSHOTS } from "../apps/cli/src/server/schema-snapshots" import { resolveMigrationChain } from "../apps/cli/src/server/local-store-migrations" -import { schemaDigest, schemaFingerprint } from "../apps/cli/src/server/store-version" +import { schemaDigest, schemaFingerprint } from "../packages/local-server/src/store-version" const fail = (message: string): never => { console.error(message) @@ -68,7 +68,7 @@ if (!sameIdentity(latest, CURRENT_LOCAL_SCHEMA)) { ) else console.error( - `\nAppend this entry to LOCAL_SCHEMA_HISTORY in apps/cli/src/server/local-schema-history.ts,\nthen register a migration edge reaching v${CURRENT_LOCAL_SCHEMA.version}:\n\n${historyEntrySource(CURRENT_LOCAL_SCHEMA)}\n`, + `\nAppend this entry to LOCAL_SCHEMA_HISTORY in packages/local-server/src/local-schema-history.ts,\nthen register a migration edge reaching v${CURRENT_LOCAL_SCHEMA.version}:\n\n${historyEntrySource(CURRENT_LOCAL_SCHEMA)}\n`, ) fail("append a new versioned identity and migration edge before changing the schema") } @@ -122,7 +122,7 @@ for (const entry of LOCAL_SCHEMA_HISTORY) { } } -const historyPath = "apps/cli/src/server/local-schema-history.ts" +const historyPath = "packages/local-server/src/local-schema-history.ts" const parseHistorySource = (source: string) => { const entries: Array> = [] const pattern = @@ -207,7 +207,7 @@ if (baseRefExists) { } const controlTip = LOCAL_CONTROL_SCHEMA_HISTORY.at(-1) -const controlSql = readFileSync("apps/cli/src/server/schema/control-schema.sql", "utf8") +const controlSql = readFileSync("packages/local-server/src/schema/control-schema.sql", "utf8") const controlDigest = (sql: string) => createHash("sha256").update(sql).digest("hex") if (controlTip?.version !== LOCAL_CONTROL_SCHEMA_VERSION || controlTip.digest !== controlDigest(controlSql)) fail("control schema changed without a versioned identity; run local-schema:bump --control") @@ -215,7 +215,10 @@ if (!controlSql.includes(`PRAGMA user_version = ${LOCAL_CONTROL_SCHEMA_VERSION}; fail("control DDL pragma does not match its version identity") for (const [index, entry] of LOCAL_CONTROL_SCHEMA_HISTORY.entries()) { if (entry.version !== index + 1) fail("control schema history versions must be sequential from 1") - const snapshot = readFileSync(`apps/cli/src/server/schema/control-schema-v${entry.version}.sql`, "utf8") + const snapshot = readFileSync( + `packages/local-server/src/schema/control-schema-v${entry.version}.sql`, + "utf8", + ) if (controlDigest(snapshot) !== entry.digest) fail(`immutable control schema v${entry.version} drifted`) } diff --git a/scripts/gen-ui-embed.ts b/scripts/gen-ui-embed.ts index c78ba700d3..1cfdcf355a 100644 --- a/scripts/gen-ui-embed.ts +++ b/scripts/gen-ui-embed.ts @@ -1,4 +1,4 @@ -// Inline apps/local-ui/dist into apps/cli/src/server/ui-embed.gen.ts so +// Inline apps/local-ui/dist into packages/local-server/src/ui-embed.gen.ts so // `bun build --compile` bakes the SPA into the `maple` binary. Run by // scripts/build-local-binary.sh, which restores the committed stub afterwards. @@ -7,7 +7,7 @@ import { join, relative } from "node:path" import { fileURLToPath } from "node:url" const distDir = fileURLToPath(new URL("../apps/local-ui/dist/", import.meta.url)) -const outPath = fileURLToPath(new URL("../apps/cli/src/server/ui-embed.gen.ts", import.meta.url)) +const outPath = fileURLToPath(new URL("../packages/local-server/src/ui-embed.gen.ts", import.meta.url)) const MIME: Record = { html: "text/html", diff --git a/scripts/generate-clickhouse-insert-mappings.ts b/scripts/generate-clickhouse-insert-mappings.ts index 7c981b04b9..7e61996591 100644 --- a/scripts/generate-clickhouse-insert-mappings.ts +++ b/scripts/generate-clickhouse-insert-mappings.ts @@ -32,7 +32,7 @@ const INGEST_DATASOURCES = [ const ORG_PLACEHOLDER = "__ORG__" const localOutputPath = fileURLToPath( - new URL("../apps/cli/src/server/schema/local-inserts.json", import.meta.url), + new URL("../packages/local-server/src/schema/local-inserts.json", import.meta.url), ) const rustOutputPath = fileURLToPath( new URL("../apps/ingest/src/clickhouse_insert_mappings.rs", import.meta.url), diff --git a/scripts/generate-clickhouse-schema-sql.ts b/scripts/generate-clickhouse-schema-sql.ts index d88d219564..c3b95f9526 100644 --- a/scripts/generate-clickhouse-schema-sql.ts +++ b/scripts/generate-clickhouse-schema-sql.ts @@ -1,10 +1,12 @@ import { mkdirSync, readFileSync, writeFileSync } from "node:fs" import { dirname } from "node:path" import { fileURLToPath } from "node:url" -import { LOCAL_SCHEMA_VERSION } from "../apps/cli/src/server/local-schema-version" +import { LOCAL_SCHEMA_VERSION } from "../packages/local-server/src/local-schema-version" import { latestSnapshotStatements, projectRevision } from "../packages/domain/src/generated/clickhouse-schema" -const outputPath = fileURLToPath(new URL("../apps/cli/src/server/schema/local-schema.sql", import.meta.url)) +const outputPath = fileURLToPath( + new URL("../packages/local-server/src/schema/local-schema.sql", import.meta.url), +) const checkMode = process.argv.includes("--check") const rendered = render(latestSnapshotStatements, projectRevision, LOCAL_SCHEMA_VERSION) diff --git a/tsconfig.alchemy-maple.json b/tsconfig.alchemy-maple.json index f562059470..741175aca5 100644 --- a/tsconfig.alchemy-maple.json +++ b/tsconfig.alchemy-maple.json @@ -2,6 +2,9 @@ "extends": "./packages/alchemy-maple/tsconfig.json", "include": ["packages/alchemy-maple/src/**/*.ts"], "compilerOptions": { + // The package's own @types (bun is a devDependency there): this config lives at + // the repo root, so default type resolution would look only in ./node_modules/@types. + "typeRoots": ["./packages/alchemy-maple/node_modules/@types", "./node_modules/@types"], "paths": { "@maple/domain/*": ["./packages/domain/src/*"], "@maple/widgets": ["./packages/widgets/src/index.ts"],