From 8d0abc8171b84f06066baf5aa1a6f1145945dbe7 Mon Sep 17 00:00:00 2001 From: Shawn Chen Date: Tue, 8 Sep 2026 10:26:34 +1200 Subject: [PATCH] Use chdb npm runtime for local mode --- .github/workflows/ci.yml | 4 +- .github/workflows/local-binary-release.yml | 25 +- apps/cli/package.json | 1 + apps/cli/src/core/update.ts | 29 +- apps/cli/src/server/chdb.ts | 295 ++++++++++-------- apps/cli/src/server/local-store-migrations.ts | 2 +- .../legacy-to-current.ts | 2 +- apps/cli/src/server/schema-manifest.ts | 2 +- apps/cli/src/server/schema-physical.ts | 2 +- apps/cli/src/server/store-version.ts | 4 +- apps/cli/src/version.ts | 10 +- apps/cli/test/chdb-args.test.ts | 15 + apps/cli/test/chdb-timezone.test.ts | 4 +- apps/cli/test/merge-injection-probe.ts | 2 +- .../test/native-archive-adversarial-probe.sh | 13 +- .../test/native-archive-calibrate-probe.sh | 4 +- .../native-archive-crash-recovery-probe.sh | 17 +- apps/cli/test/native-archive-gc-probe.sh | 11 +- apps/cli/test/native-local-store-migration.sh | 21 +- apps/cli/test/probes/archive-crash-worker.ts | 2 +- apps/cli/test/probes/archive-gc-worker.ts | 2 +- .../archive-probe-byte-heterogeneous.ts | 2 +- .../probes/archive-probe-byte-single-row.ts | 2 +- .../test/probes/archive-probe-byte-uniform.ts | 2 +- .../probes/archive-probe-complex-alter.ts | 2 +- .../archive-probe-digest-column-swap.ts | 2 +- .../probes/archive-probe-digest-dup-drop.ts | 2 +- .../probes/archive-probe-digest-row-swap.ts | 2 +- .../probes/archive-probe-duckdb-oracle.ts | 2 +- .../probes/archive-probe-merge-freeze-leak.ts | 2 +- .../probes/archive-probe-merge-injection.ts | 2 +- .../test/probes/archive-probe-mixed-hour.ts | 2 +- .../test/probes/archive-probe-multipart.ts | 2 +- .../test/probes/archive-probe-null-digest.ts | 2 +- .../probes/archive-probe-null-flag-binding.ts | 2 +- .../archive-probe-null-value-sensitivity.ts | 2 +- .../archive-probe-schema-substitution.ts | 2 +- .../probes/archive-probe-timezone-bound.ts | 2 +- .../test/probes/archive-reconcile-worker.ts | 2 +- apps/cli/test/update.test.ts | 24 +- .../src/components/local/InstallTabs.astro | 4 +- apps/landing/src/content/docs/local-mode.mdx | 4 +- .../content/docs/local-mode/cli-reference.md | 30 +- bun.lock | 69 +--- docs/local-mode.md | 59 ++-- scripts/build-local-binary.sh | 108 ++++--- scripts/install.sh | 22 +- scripts/lint-clickhouse-schema.ts | 4 +- scripts/resolve-chdb-version.ts | 35 +++ scripts/uninstall.sh | 4 +- 50 files changed, 470 insertions(+), 396 deletions(-) create mode 100644 scripts/resolve-chdb-version.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 76b5fdd1a..431dddafc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -677,8 +677,8 @@ jobs: # probe (crash-recovery, ~125s) plus setup. # # Each leg rebuilds the bundle rather than sharing one via upload-artifact: - # dist/ is ~403MB (libchdb.so 334MB + maple 69MB), and build-local-binary.sh - # takes ~14s including the libchdb download. Uploading once and downloading + # dist/ carries the native chdb npm sidecar plus maple, and build-local-binary.sh + # is already quick after bun install. Uploading once and downloading # seven times is strictly slower. name: Local archive native (${{ matrix.probe }}) needs: changes diff --git a/.github/workflows/local-binary-release.yml b/.github/workflows/local-binary-release.yml index aa772458e..3341e3306 100644 --- a/.github/workflows/local-binary-release.yml +++ b/.github/workflows/local-binary-release.yml @@ -2,13 +2,12 @@ name: Local Binary Release # Builds the distributable `maple` local binary (a single Bun-compiled # executable: CLI + OTLP-ingest/query server + bundled SPA) for macOS and Linux, -# then attaches a 2-file bundle (`maple` + `libchdb.so`) per platform to a +# then attaches a bundle (`maple` + npm `chdb` runtime sidecar) per platform to a # GitHub release. # -# Why native runners (no cross-compilation): the bundle ships a prebuilt, -# platform-specific `libchdb.so` (downloaded from chdb-io/chdb-core releases), -# which `maple` dlopens via bun:ffi at runtime. We fetch the engine matching the -# host, so each target is built on a matching native runner. +# Why native runners (no cross-compilation): the bundle ships the platform-specific +# `@chdb/lib-*` npm package installed for the host, so each target is built on a +# matching native runner. # # Each build job publishes its own tarball immediately after packaging — no # separate aggregation step — so a slow or queued runner can never block the @@ -91,8 +90,8 @@ jobs: - name: Build local binary bundle # Builds the SPA, inlines it into the binary, compiles apps/cli with - # `bun build --compile`, and downloads the matching libchdb beside it. - # Output bundle lands in ./dist (maple + libchdb.so). MAPLE_BUILD_VERSION + # `bun build --compile`, and copies the matching npm chdb runtime beside it. + # Output bundle lands in ./dist (maple + node_modules). MAPLE_BUILD_VERSION # bakes the resolved tag into `maple --version`. env: MAPLE_BUILD_VERSION: ${{ steps.version.outputs.version }} @@ -109,13 +108,15 @@ jobs: - name: Ad-hoc codesign (macOS) # bun-compiled binaries run locally once Gatekeeper quarantine is - # cleared; ad-hoc sign both files for good measure. Production + # cleared; ad-hoc sign the executable and native sidecar for good measure. Production # distribution should layer Developer ID signing + notarization on # top (requires APPLE_* secrets not configured here). if: ${{ matrix.os == 'macos' }} run: | set -euo pipefail - codesign --force --sign - dist/libchdb.so + while IFS= read -r -d '' f; do + codesign --force --sign - "$f" + done < <(find dist/node_modules/@chdb -type f \( -name '*.node' -o -name 'libchdb.so' -o -name '*.dylib' \) -print0) codesign --force --sign - dist/maple codesign --verify --verbose dist/maple @@ -130,8 +131,8 @@ jobs: rm -rf "$staging" mkdir -p "$staging" cp dist/maple "$staging/maple" - cp dist/libchdb.so "$staging/libchdb.so" - # Smoke-check the relocatable 2-file bundle (maple + libchdb.so). + cp -R dist/node_modules "$staging/node_modules" + # Smoke-check the relocatable bundle (maple + node_modules/chdb). ( cd "$staging" && ./maple --help >/dev/null 2>&1 ) || \ echo "::warning::'maple --help' smoke check did not exit cleanly on $target" tarball="$RUNNER_TEMP/${name}.tar.gz" @@ -181,7 +182,7 @@ jobs: 'Local Maple (`maple start`) bundles.' \ '**Install (macOS / Linux):**' \ 'curl -fsSL https://maple.dev/cli/install | sh' \ - 'Or download a bundle below. Each archive contains `maple` + `libchdb.so` — keep both in the same directory. macOS: run `xattr -dr com.apple.quarantine maple libchdb.so` after extracting.' \ + 'Or download a bundle below. Each archive contains `maple` + `node_modules/chdb` — keep the sidecar next to the binary. macOS: run `xattr -dr com.apple.quarantine maple node_modules` after extracting.' \ > "$RUNNER_TEMP/release-notes.md" gh release create "$TAG" --repo "$REPO" \ --title "$TAG" \ diff --git a/apps/cli/package.json b/apps/cli/package.json index 9072e41f9..d7ced91bb 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -15,6 +15,7 @@ "@maple-dev/effect-sdk": "workspace:*", "@maple/domain": "workspace:*", "@maple/query-engine": "workspace:*", + "chdb": "3.3.0", "effect": "catalog:effect", "protobufjs": "^8.6.1" }, diff --git a/apps/cli/src/core/update.ts b/apps/cli/src/core/update.ts index 67e852005..2d22eccc6 100644 --- a/apps/cli/src/core/update.ts +++ b/apps/cli/src/core/update.ts @@ -6,12 +6,12 @@ // (driven by `maple update`, see commands/update.ts). // // We mirror scripts/install.sh's conventions (target triples, release URLs, -// 2-file bundle, checksum, macOS quarantine clear) rather than shelling out to +// runtime sidecar, checksum, macOS quarantine clear) rather than shelling out to // it: the installer uses `cp`, which fails with ETXTBSY when overwriting the // *running* executable on Linux. Self-update instead downloads into a temp dir // on the same filesystem as the install dir and `rename()`s over the targets — // rename swaps the directory entry, so the running process keeps its old inode -// while new invocations pick up the new binary. Keep the triple/URL logic here +// while new invocations pick up the new binary and sidecar. Keep the triple/URL logic here // in sync with install.sh. import { Clock, Duration, Effect, Option, Schema, Stream } from "effect" import { FileSystem } from "effect/FileSystem" @@ -152,7 +152,7 @@ export const fetchLatestTag = (timeoutMs = 5000): Effect.Effect dirname(realpathSync(process.execPath)) @@ -298,10 +298,10 @@ const clearQuarantine = (paths: ReadonlyArray): Effect.Effect restorePrevious)) }) @@ -409,9 +409,8 @@ export const performUpdate = ( Effect.mapError((e) => mapFsError(e, installDir)), ) - if (process.platform === "darwin") { - yield* clearQuarantine([join(installDir, "maple"), join(installDir, "libchdb.so")]) - } + if (process.platform === "darwin") + yield* clearQuarantine([join(installDir, "maple"), join(installDir, "node_modules")]) }), ) diff --git a/apps/cli/src/server/chdb.ts b/apps/cli/src/server/chdb.ts index 92089e3a8..29b1aa128 100644 --- a/apps/cli/src/server/chdb.ts +++ b/apps/cli/src/server/chdb.ts @@ -1,79 +1,88 @@ // SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. -// Embedded chDB (in-process ClickHouse) via `bun:ffi` → `libchdb`. +// Embedded chDB (in-process ClickHouse) through the published `chdb` npm package. // // Replaces the Rust `apps/ingest/src/chdb.rs`. chDB allows exactly one // connection per process and is not safe to call concurrently, so the local -// server holds a single `Chdb` and `bun:ffi` calls — which are synchronous and +// server holds a single `Chdb` and npm Session calls — which are synchronous and // block the calling thread — serialize naturally on the JS thread. -// -// We use the modern `chdb_*` accessor C API (opaque result handles read via -// `chdb_result_buffer`/`_length`/`_error`), not the older `local_result_v2` -// struct, so there is no struct-offset fragility across libchdb versions. -import { CString, dlopen, FFIType, type Pointer, ptr, read, toArrayBuffer } from "bun:ffi" +import { dlopen, FFIType, ptr } from "bun:ffi" import { Effect, Schema, type Scope } from "effect" -import { existsSync } from "node:fs" -import { lstatSync, readFileSync } from "node:fs" -import { homedir } from "node:os" +import { createRequire } from "node:module" +import { + existsSync, + lstatSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs" +import { tmpdir } from "node:os" import { dirname, join, resolve } from "node:path" import { durableJson } from "./durable-files" import { markStoreClosed, markStoreOpen, storeHasData } from "./store-version" -/** A chDB failure — locating libchdb, opening the connection, or bootstrapping +/** A chDB failure — loading the npm addon, opening the connection, or bootstrapping * the schema. Carries the underlying message verbatim. */ export class ChdbError extends Schema.TaggedError()("@maple/cli/ChdbError", { message: Schema.String, }) {} -/** Locate `libchdb` at runtime, in priority order: - * 1. `MAPLE_LIBCHDB` env (explicit override) - * 2. sibling of the executable (the shipped 2-file bundle: `maple` + `libchdb`) - * 3. `~/.maple/bin/libchdb.{so,dylib}` (dev / installed location) - */ -function resolveLibchdb(): string { - const candidates: string[] = [] - const override = process.env.MAPLE_LIBCHDB - if (override) candidates.push(override) +interface ChdbSession { + query(sql: string, format?: string): string + close(): void +} - const execDir = dirname(process.execPath) - candidates.push(join(execDir, "libchdb.so"), join(execDir, "libchdb.dylib")) +interface ChdbPackage { + Session: new (path?: string) => ChdbSession +} - const binDir = join(homedir(), ".maple", "bin") - candidates.push(join(binDir, "libchdb.so"), join(binDir, "libchdb.dylib")) +let chdbPackage: ChdbPackage | undefined - const found = candidates.find((p) => existsSync(p)) - if (!found) { - throw new Error( - `libchdb not found. Looked in:\n ${candidates.join("\n ")}\n` + - `Set MAPLE_LIBCHDB to its path, or keep libchdb next to the maple binary.`, - ) - } - return found +const requireFromNodeModules = (nodeModulesDir: string): ChdbPackage => { + const require = createRequire(join(nodeModulesDir, ".maple-chdb-runtime.cjs")) + return require("chdb") as ChdbPackage } -type ChdbSymbols = ReturnType["symbols"] - -function openLib(libPath: string) { - return dlopen(libPath, { - // chdb_connection* chdb_connect(int argc, char** argv) - chdb_connect: { args: [FFIType.int, FFIType.ptr], returns: FFIType.ptr }, - // void chdb_close_conn(chdb_connection* conn) - chdb_close_conn: { args: [FFIType.ptr], returns: FFIType.void }, - // chdb_result* chdb_query(chdb_connection conn, const char* query, const char* format) - chdb_query: { args: [FFIType.ptr, FFIType.ptr, FFIType.ptr], returns: FFIType.ptr }, - chdb_result_buffer: { args: [FFIType.ptr], returns: FFIType.ptr }, - chdb_result_length: { args: [FFIType.ptr], returns: FFIType.u64 }, - // const char* chdb_result_error(chdb_result*) — NULL or EMPTY string means success - chdb_result_error: { args: [FFIType.ptr], returns: FFIType.ptr }, - chdb_destroy_query_result: { args: [FFIType.ptr], returns: FFIType.void }, - }) -} +const loadChdbPackage = (): ChdbPackage => { + if (chdbPackage) return chdbPackage + const failures: string[] = [] + const override = process.env.MAPLE_CHDB_NODE_MODULES + if (override) { + try { + chdbPackage = requireFromNodeModules(resolve(override)) + return chdbPackage + } catch (error) { + failures.push( + `MAPLE_CHDB_NODE_MODULES=${override}: ${error instanceof Error ? error.message : String(error)}`, + ) + } + } + + const siblingNodeModules = join(dirname(process.execPath), "node_modules") + if (existsSync(join(siblingNodeModules, "chdb", "package.json"))) { + try { + chdbPackage = requireFromNodeModules(siblingNodeModules) + return chdbPackage + } catch (error) { + failures.push(`${siblingNodeModules}: ${error instanceof Error ? error.message : String(error)}`) + } + } -let lib: { symbols: ChdbSymbols; close: () => void } | undefined + try { + chdbPackage = createRequire(import.meta.url)("chdb") as ChdbPackage + return chdbPackage + } catch (error) { + failures.push(`workspace dependency: ${error instanceof Error ? error.message : String(error)}`) + } -function symbols(): ChdbSymbols { - if (!lib) lib = openLib(resolveLibchdb()) - return lib.symbols + throw new Error( + "chdb npm package not found. Expected node_modules/chdb next to the maple binary " + + "or installed in the development workspace." + + (failures.length > 0 ? ` Tried:\n ${failures.join("\n ")}` : ""), + ) } const encoder = new TextEncoder() @@ -200,19 +209,25 @@ export const applyRawTelemetryRetentionFloor = (db: Pick } } -/** Build the embedded ClickHouse argv. Keep table metadata loading and restore - * work serialized: +/** Keep table metadata loading and restore work serialized: * chDB v26.1.0 can otherwise fail nondeterministically while its loader resolves * Maple's materialized-view dependency graph (`recursive_mutex lock failed` / * `ASYNC_LOAD_WAIT_FAILED`). `async_load_databases=0` waits for loading, but it * does not make the loader pools single-threaded. RESTORE uses a separate * 16-thread pool by default and can trip the same invalid recursive-mutex state - * while restoring that dependency graph. */ + * while restoring that dependency graph. + * + * The npm `chdb` 3.3.x Session API does not expose clickhouse-local argv. chDB + * does, however, read `config.xml` from the process cwd while opening the first + * connection, and ClickHouse merges `config.d/*.xml` after that. `Chdb.open` + * uses a temporary cwd with the user config symlinked as `00-user.xml` and these + * Maple-required values written as `99-maple-required.xml`, so explicit user + * config is still honoured while Maple's safety settings win on conflicts. */ /** * Parser limit for one statement, applied as a session setting at open. * - * The C API has no separate data stream: every INSERT inlines its NDJSON as a - * string literal, and the parser reads the whole statement against + * The Session API has no separate data stream: every INSERT inlines its NDJSON + * as a string literal, and the parser reads the whole statement against * `max_query_size` (default 256 KiB). `buildInsertStatements` chunks batches * under that, but it cannot split a single row — a span carrying a large * attribute (a request body, a stack, a prompt) still arrives as one line and @@ -220,20 +235,54 @@ export const applyRawTelemetryRetentionFloor = (db: Pick * limit is a parser guard, not a buffer allocation, so raising it well past any * single OTLP row costs nothing. * - * A `SET`, not an argv flag: `chdb_connect` accepts `--=` for some - * settings but `--max_query_size` measurably does not take (system.settings - * still reports 262144), while the session `SET` — the same path - * `session_timezone` uses — does, and holds for the connection's lifetime. + * A session `SET`, rather than a server config value, is enough here and holds + * for the connection's lifetime. */ export const MAX_QUERY_SIZE_BYTES = 64 * 1024 * 1024 +const CHDB_REQUIRED_SERVER_SETTINGS = { + async_load_databases: "0", + async_load_system_database: "0", + tables_loader_foreground_pool_size: "1", + tables_loader_background_pool_size: "1", + restore_threads: "1", +} as const + +export const requiredChdbConfigXml = (): string => { + const settings = Object.entries(CHDB_REQUIRED_SERVER_SETTINGS) + .map(([name, value]) => ` <${name}>${value}`) + .join("\n") + return `\n${settings}\n\n` +} + +interface PreparedChdbConfig { + readonly cwd: string + cleanup(): void +} + +const prepareChdbConfig = (configFile?: string): PreparedChdbConfig => { + const root = mkdtempChdbConfigRoot() + const configDir = join(root, "config.d") + mkdirSync(configDir, { recursive: true }) + writeFileSync(join(root, "config.xml"), "\n", { mode: 0o600 }) + if (configFile) { + symlinkSync(resolve(configFile), join(configDir, "00-user.xml")) + } + writeFileSync(join(configDir, "99-maple-required.xml"), requiredChdbConfigXml(), { mode: 0o600 }) + return { + cwd: root, + cleanup: () => rmSync(root, { recursive: true, force: true }), + } +} + +const mkdtempChdbConfigRoot = (): string => { + const root = join(tmpdir(), "maple-chdb-config-") + return mkdtempSync(root) +} + export const chdbArgv = (options: Pick): string[] => [ "clickhouse", - "--async_load_databases=0", - "--async_load_system_database=0", - "--tables_loader_foreground_pool_size=1", - "--tables_loader_background_pool_size=1", - "--restore_threads=1", + ...Object.entries(CHDB_REQUIRED_SERVER_SETTINGS).map(([name, value]) => `--${name}=${value}`), // Wire-format parity with the cloud read paths (BackendDialect // unquote64BitIntegers): emit 64-bit ints as JSON numbers, not strings, so // local mode decodes rows exactly like managed Tinybird / BYO ClickHouse. @@ -250,12 +299,13 @@ const LIBC_CANDIDATES = let timezonePinned = false /** - * Force the embedded engine's SERVER timezone to UTC, before libchdb loads. + * Force the embedded engine's SERVER timezone to UTC, before the native chDB + * runtime loads. * * `SET session_timezone = 'UTC'` (in `Chdb.open`) is not enough: every * `DateTime64(n)` column in the local schema is declared without an explicit - * zone, and ClickHouse resolves *those* against the server timezone, which - * libchdb reads from the host environment when it initialises. On a machine in, + * zone. ClickHouse resolves *those* against the server timezone, which the + * native chDB runtime reads from the host environment when it initialises. On a machine in, * say, `Europe/Berlin` that meant stored timestamps rendered in local time * and — the part that actually broke — every datetime **string literal** in a * `WHERE` clause parsed as local time, while the UI and CLI build their window @@ -300,61 +350,58 @@ export const pinProcessTimezoneToUtc = (): void => { * a statement and discards output. Both throw on a non-empty chDB error. */ export class Chdb { - readonly #sym: ChdbSymbols - #connPtrPtr: Pointer | null - readonly #conn: Pointer - - private constructor(sym: ChdbSymbols, connPtrPtr: Pointer, conn: Pointer) { - this.#sym = sym - this.#connPtrPtr = connPtrPtr - this.#conn = conn + #session: ChdbSession | null + + private constructor(session: ChdbSession) { + this.#session = session } static open(options: ChdbOptions): Chdb { pinProcessTimezoneToUtc() - const sym = symbols() - const args = chdbArgv(options) - const argBufs = args.map(cstr) - const argv = new BigUint64Array(args.length) - argBufs.forEach((b, i) => { - argv[i] = BigInt(ptr(b)) - }) - const connPtrPtr = sym.chdb_connect(args.length, ptr(argv)) - if (!connPtrPtr) { - throw new Error( - Chdb.#connectFailure(options.dataDir, "chdb_connect returned NULL", options.configFile), - ) - } - // chdb_connect returns chdb_connection* (a double pointer); chdb_query - // wants chdb_connection — dereference once. - const conn = read.ptr(connPtrPtr, 0) as Pointer - if (!conn) + const { Session } = loadChdbPackage() + const preparedConfig = prepareChdbConfig(options.configFile) + const previousCwd = process.cwd() + let session: ChdbSession + try { + process.chdir(preparedConfig.cwd) + session = new Session(options.dataDir) + } catch (error) { throw new Error( Chdb.#connectFailure( options.dataDir, - "chdb_connect produced a NULL connection", + error instanceof Error ? error.message : String(error), options.configFile, ), ) + } finally { + process.chdir(previousCwd) + preparedConfig.cleanup() + } - const db = new Chdb(sym, connPtrPtr, conn) - // Partition expressions, ingest conversions, and retention predicates must - // never inherit a host-specific timezone. - db.exec("SET session_timezone = 'UTC'") - db.exec(`SET max_query_size = ${MAX_QUERY_SIZE_BYTES}`) - if (options.bootstrapSchema !== false) { - db.#bootstrap(options.schemaSql) - if (options.rawTelemetryRetentionDays !== undefined) - applyRawTelemetryRetentionFloor(db, options.rawTelemetryRetentionDays) + const db = new Chdb(session) + try { + // Partition expressions, ingest conversions, and retention predicates must + // never inherit a host-specific timezone. + db.exec("SET session_timezone = 'UTC'") + db.exec("SET output_format_json_quote_64bit_integers = 0") + db.exec(`SET max_query_size = ${MAX_QUERY_SIZE_BYTES}`) + if (options.bootstrapSchema !== false) { + db.#bootstrap(options.schemaSql) + if (options.rawTelemetryRetentionDays !== undefined) + applyRawTelemetryRetentionFloor(db, options.rawTelemetryRetentionDays) + } + } catch (error) { + db.close() + throw error } return db } - // chdb_connect failing over a *populated* store almost always means an + // Session construction failing over a *populated* store almost always means an // unloadable on-disk state (e.g. a pipeline left inconsistent by an unclean - // kill); point the user at the recovery path rather than the raw libchdb + // kill); point the user at the recovery path rather than the raw native addon // message. A failure over an empty dir is a different problem (missing/broken - // libchdb), so keep the generic message there. + // chdb install), so keep the generic message there. static #connectFailure(dataDir: string, raw: string, configFile?: string): string { if (configFile) { return ( @@ -372,25 +419,8 @@ export class Chdb { /** Run a query and return the result bytes decoded as UTF-8 text. */ query(sql: string, format = "JSONEachRow"): string { - const q = cstr(sql) - const f = cstr(format) - const res = this.#sym.chdb_query(this.#conn, ptr(q), ptr(f)) - if (!res) throw new Error("chdb_query returned NULL") - try { - const errPtr = this.#sym.chdb_result_error(res) - // chdb returns a non-null pointer to an EMPTY string on success; only a - // non-empty message is a real error (matches chdb-rust `check_error`). - const errMsg = errPtr ? new CString(errPtr).toString() : "" - if (errMsg.length > 0) throw new Error(errMsg) - const len = Number(this.#sym.chdb_result_length(res)) - if (len === 0) return "" - const bufPtr = this.#sym.chdb_result_buffer(res) - if (!bufPtr) return "" - // Copy out of the chDB-owned buffer before it is destroyed. - return new TextDecoder().decode(toArrayBuffer(bufPtr, 0, len).slice(0)) - } finally { - this.#sym.chdb_destroy_query_result(res) - } + if (!this.#session) throw new Error("No active chDB connection available") + return this.#session.query(sql, format) } /** Run a statement and discard its output. */ @@ -399,13 +429,12 @@ export class Chdb { } close(): void { - if (this.#connPtrPtr !== null) { - this.#sym.chdb_close_conn(this.#connPtrPtr) - this.#connPtrPtr = null - } + const session = this.#session + this.#session = null + session?.close() } - // chDB executes a multi-statement script in a single call. If a given libchdb + // chDB executes a multi-statement script in a single call. If a given engine // build rejects that, fall back to running each statement on its own. The // generated schema joins statements with a blank line, so splitting on blank // lines is safe (no statement body contains a blank line). @@ -427,7 +456,7 @@ export class Chdb { /** * Acquire a chDB connection as a scoped resource: `Chdb.open` (which bootstraps * the schema) on acquire, `close()` as a finalizer. Open failures — a missing - * libchdb, a NULL connection, a rejected bootstrap — surface as a typed + * native addon load, connection construction, and rejected bootstrap — surface as a typed * `ChdbError` instead of an unhandled throw. The synchronous `query`/`exec` * methods are unchanged; only the lifecycle is Effect-managed. */ diff --git a/apps/cli/src/server/local-store-migrations.ts b/apps/cli/src/server/local-store-migrations.ts index f47ca57a6..f14df3fd9 100644 --- a/apps/cli/src/server/local-store-migrations.ts +++ b/apps/cli/src/server/local-store-migrations.ts @@ -547,7 +547,7 @@ const assertNoLiveServer = (dataDir: string): void => { } /** Total bytes on disk. A `string | number` because the setting that unquotes - * 64-bit integers is not in force for every libchdb build this CLI supports. */ + * 64-bit integers is not in force for every chDB engine build this CLI supports. */ const DiskUsageRow = Schema.Struct({ bytes: Schema.Union([Schema.String, Schema.Number]) }) const decodeDiskUsageRows = decodeJsonEachRow(DiskUsageRow) diff --git a/apps/cli/src/server/local-store-migrations/legacy-to-current.ts b/apps/cli/src/server/local-store-migrations/legacy-to-current.ts index 665508a65..72eb23473 100644 --- a/apps/cli/src/server/local-store-migrations/legacy-to-current.ts +++ b/apps/cli/src/server/local-store-migrations/legacy-to-current.ts @@ -243,7 +243,7 @@ const asProgress = (value: RawReplayProgress | undefined): RawReplayProgress => * Row shapes read back out of chDB. * * `rowCount` and the hash aggregates are `string | number` on purpose: a UInt64 - * arrives quoted or unquoted depending on the libchdb build, and `numberString` + * arrives quoted or unquoted depending on the chDB engine build, and `numberString` * normalizes it. What must not happen is a `Number` decode above 2^53, which is * why nothing here is `Schema.Number` alone. */ diff --git a/apps/cli/src/server/schema-manifest.ts b/apps/cli/src/server/schema-manifest.ts index f959c88a3..7680b9115 100644 --- a/apps/cli/src/server/schema-manifest.ts +++ b/apps/cli/src/server/schema-manifest.ts @@ -309,7 +309,7 @@ export const viewBody = (definition: string): string | undefined => { /** Compare a bundled structural manifest with an inspected chDB schema. This * is deliberately data-only so migration and startup can share the same - * fail-closed gate and unit tests do not require native libchdb. + * fail-closed gate and unit tests do not require the native chDB runtime. * * `normalizeSql` renders a view body through a parser so the bundled * source text and ClickHouse's own rewritten text become comparable. Without diff --git a/apps/cli/src/server/schema-physical.ts b/apps/cli/src/server/schema-physical.ts index c9ac11880..d6667287e 100644 --- a/apps/cli/src/server/schema-physical.ts +++ b/apps/cli/src/server/schema-physical.ts @@ -60,7 +60,7 @@ export const inspectPhysicalSchema = (db: Chdb): PhysicalSchema => { db.query("SELECT table, name FROM system.data_skipping_indices WHERE database = 'default'"), ) } catch { - // Older supported libchdb builds may not expose this system table. The + // Older supported chDB engine builds may not expose this system table. The // CREATE TABLE definitions below still provide a strict index-name fallback. } const columnsByTable = new Map() diff --git a/apps/cli/src/server/store-version.ts b/apps/cli/src/server/store-version.ts index 95485e525..099cc7d17 100644 --- a/apps/cli/src/server/store-version.ts +++ b/apps/cli/src/server/store-version.ts @@ -29,7 +29,7 @@ export interface StoreMigrationStamp { export interface StoreMarkerV2 { readonly formatVersion: typeof STORE_MARKER_FORMAT_VERSION readonly storeId: string - /** libchdb package/release identity that owns the on-disk format. */ + /** chDB engine identity that owns the on-disk format. */ readonly chdb: string /** Maple version that first created the store. */ readonly maple: string @@ -409,7 +409,7 @@ export type StoreCompatibility = | { readonly compatible: true } | { readonly compatible: false; readonly found: string; readonly current: string } -/** Decide whether the current libchdb may open the store at `dataDir`. */ +/** Decide whether the current chDB engine may open the store at `dataDir`. */ export const checkStoreCompatible = (dataDir: string): StoreCompatibility => { if (!storeHasData(dataDir)) return { compatible: true } const state = readMarkerState(dataDir) diff --git a/apps/cli/src/version.ts b/apps/cli/src/version.ts index 0047f65ac..ea62ee2ee 100644 --- a/apps/cli/src/version.ts +++ b/apps/cli/src/version.ts @@ -13,11 +13,11 @@ const raw = typeof __MAPLE_VERSION__ !== "undefined" ? __MAPLE_VERSION__ : "dev" // release tags like "v0.5.0" to avoid a doubled "vv0.5.0". export const MAPLE_VERSION: string = raw.replace(/^v/, "") -// The libchdb release the binary was built against (e.g. "v26.1.0"), baked in by -// `scripts/build-local-binary.sh` via `bun build --define`. It identifies the -// chDB on-disk-store format, so the store-version guard can refuse a directory -// written by an incompatible build. Dev runs (`bun run src/bin.ts`, no --define) -// report "dev". +// The chDB engine version bundled by the `chdb` npm runtime sidecar, baked in +// by `scripts/build-local-binary.sh` via `bun build --define`. It identifies +// the chDB on-disk-store format, so the store-version guard can refuse a +// directory written by an incompatible build. Dev runs (`bun run src/bin.ts`, +// no --define) report "dev". declare const __CHDB_VERSION__: string | undefined export const CHDB_VERSION: string = typeof __CHDB_VERSION__ !== "undefined" ? __CHDB_VERSION__ : "dev" diff --git a/apps/cli/test/chdb-args.test.ts b/apps/cli/test/chdb-args.test.ts index d94153201..713370a79 100644 --- a/apps/cli/test/chdb-args.test.ts +++ b/apps/cli/test/chdb-args.test.ts @@ -9,6 +9,7 @@ import { configureRawTelemetryRetentionDays, rawTelemetryTtlStatements, readRawTelemetryRetentionDays, + requiredChdbConfigXml, } from "../src/server/chdb" describe("embedded chDB arguments", () => { @@ -38,6 +39,20 @@ describe("embedded chDB arguments", () => { "--config-file=/tmp/backups.xml", ]) }) + + it("renders the required server settings as a config overlay", () => { + strictEqual( + requiredChdbConfigXml(), + ` + 0 + 0 + 1 + 1 + 1 + +`, + ) + }) }) describe("persistent raw telemetry retention floor", () => { diff --git a/apps/cli/test/chdb-timezone.test.ts b/apps/cli/test/chdb-timezone.test.ts index 8dee58579..7b8ddd66d 100644 --- a/apps/cli/test/chdb-timezone.test.ts +++ b/apps/cli/test/chdb-timezone.test.ts @@ -5,7 +5,7 @@ import { strictEqual } from "node:assert" import { pinProcessTimezoneToUtc } from "../src/server/chdb" /** - * libchdb resolves its SERVER timezone from the process environment at init, + * The embedded chDB engine resolves its SERVER timezone from the process environment at init, * and every `DateTime64(n)` column in the local schema is declared without an * explicit zone — so on a non-UTC host, datetime string literals in a WHERE * clause parsed as local time while the UI built its bounds in UTC. Every time @@ -13,7 +13,7 @@ import { pinProcessTimezoneToUtc } from "../src/server/chdb" * * The write has to reach *libc*, not just Bun's `process.env` map: Bun keeps its * own copy and never calls `setenv`, so a plain assignment is invisible to a - * dlopened library. This test asserts the value libchdb would actually read. + * native addon. This test asserts the value the chDB runtime would actually read. */ describe("embedded engine timezone", () => { it("pins TZ=UTC in libc, where the engine reads it", () => { diff --git a/apps/cli/test/merge-injection-probe.ts b/apps/cli/test/merge-injection-probe.ts index 35f841d20..e5b5d063d 100644 --- a/apps/cli/test/merge-injection-probe.ts +++ b/apps/cli/test/merge-injection-probe.ts @@ -6,7 +6,7 @@ // must be blocked (or harmless), and the exported shards must contain the exact // source row set with no duplicates or omissions. // -// Run: MAPLE_LIBCHDB=/libchdb.so bun apps/cli/test/merge-injection-probe.ts +// Run: MAPLE_CHDB_NODE_MODULES=/node_modules bun apps/cli/test/merge-injection-probe.ts import { rmSync, mkdirSync } from "node:fs" import { join } from "node:path" diff --git a/apps/cli/test/native-archive-adversarial-probe.sh b/apps/cli/test/native-archive-adversarial-probe.sh index ee97d9165..ad47182f3 100755 --- a/apps/cli/test/native-archive-adversarial-probe.sh +++ b/apps/cli/test/native-archive-adversarial-probe.sh @@ -8,16 +8,17 @@ # # See apps/cli/test/archive-adversarial-matrix.md for the invariant matrix. # -# Usage: apps/cli/test/native-archive-adversarial-probe.sh [libchdb-path] +# Usage: apps/cli/test/native-archive-adversarial-probe.sh [node-modules-dir] set -uo pipefail -BUNDLE="${1:?usage: $0 [libchdb-path]}" -LIBCHDB="${2:-$BUNDLE/libchdb.so}" -export MAPLE_LIBCHDB="$LIBCHDB" +BUNDLE="${1:?usage: $0 [node-modules-dir]}" +CHDB_NODE_MODULES="${2:-$BUNDLE/node_modules}" +export MAPLE_CHDB_NODE_MODULES="$CHDB_NODE_MODULES" cd "$(dirname "$0")/../../.." || exit 1 # apps/cli/test/x.sh -> apps/cli/test -> apps/cli -> apps -> repo root PROBE_DIR="apps/cli/test/probes" +[ -d "$CHDB_NODE_MODULES/chdb" ] || { echo "FAIL: chdb runtime not found at $CHDB_NODE_MODULES/chdb" >&2; exit 1; } pass=0 fail=0 @@ -29,7 +30,7 @@ declare -a FAILURES=() run_probe() { local label="$1" probe="$2"; shift 2 local out rc - out="$(env MAPLE_LIBCHDB="$LIBCHDB" "$@" bun "$PROBE_DIR/$probe" 2>&1)" && rc=0 || rc=$? + out="$(env MAPLE_CHDB_NODE_MODULES="$CHDB_NODE_MODULES" "$@" bun "$PROBE_DIR/$probe" 2>&1)" && rc=0 || rc=$? if [ "$rc" -eq 0 ]; then printf ' OK %s\n' "$label" pass=$((pass+1)) @@ -41,7 +42,7 @@ run_probe() { fi } -echo "=== Archive adversarial probe suite (libchdb=$(basename "$LIBCHDB")) ===" +echo "=== Archive adversarial probe suite (chdb runtime=$CHDB_NODE_MODULES) ===" echo echo "--- sharding correctness ---" diff --git a/apps/cli/test/native-archive-calibrate-probe.sh b/apps/cli/test/native-archive-calibrate-probe.sh index 53a2763f5..856a21007 100755 --- a/apps/cli/test/native-archive-calibrate-probe.sh +++ b/apps/cli/test/native-archive-calibrate-probe.sh @@ -25,6 +25,8 @@ set -euo pipefail BUNDLE_DIR="${1:?usage: native-archive-calibrate-probe.sh [port]}" MAPLE="$BUNDLE_DIR/maple" +CHDB_NODE_MODULES="${MAPLE_CHDB_NODE_MODULES:-$BUNDLE_DIR/node_modules}" +export MAPLE_CHDB_NODE_MODULES="$CHDB_NODE_MODULES" PORT="${2:-45261}" ROOT="$(realpath "$(mktemp -d "${TMPDIR:-/tmp}/maple-native-calib.XXXXXX")")" DATA="$ROOT/data" @@ -331,7 +333,7 @@ jq -nc \ '{peakRssBytes:$rss, wallMs:$wall, physicalBytes:$phys, logicalBytes:$logical, compressionRatio:$comp, writeThroughputBytesPerSec:$tput, peakTempDiskBytes:$temp, rowCount:$rows}' \ > "$OBSERVED_JSON" COMPARISON_OUT="$ROOT/comparison.txt" -if ! MAPLE_LIBCHDB="$BUNDLE_DIR/libchdb.so" bun apps/cli/test/probes/calibration-validation-compare.ts \ +if ! MAPLE_CHDB_NODE_MODULES="$CHDB_NODE_MODULES" bun apps/cli/test/probes/calibration-validation-compare.ts \ "$CFG" "$OBSERVED_JSON" "$TRIAL_OP" "logs" "$OBSERVED_ROWS" 1 \ >"$VALREPORT" 2>"$COMPARISON_OUT"; then cat "$COMPARISON_OUT" >&2 diff --git a/apps/cli/test/native-archive-crash-recovery-probe.sh b/apps/cli/test/native-archive-crash-recovery-probe.sh index e04b3f9ee..9e865c2d4 100755 --- a/apps/cli/test/native-archive-crash-recovery-probe.sh +++ b/apps/cli/test/native-archive-crash-recovery-probe.sh @@ -20,23 +20,24 @@ set -uo pipefail BUNDLE_DIR="${1:?usage: $0 [port]}" MAPLE="$BUNDLE_DIR/maple" -LIBCHDB="${MAPLE_LIBCHDB:-$BUNDLE_DIR/libchdb.so}" +CHDB_NODE_MODULES="${MAPLE_CHDB_NODE_MODULES:-$BUNDLE_DIR/node_modules}" +export MAPLE_CHDB_NODE_MODULES="$CHDB_NODE_MODULES" PORT="${2:-45291}" REPO="$(cd "$(dirname "$0")/../../.." && pwd)" WORKER="$REPO/apps/cli/test/probes/archive-crash-worker.ts" RECONCILE="$REPO/apps/cli/test/probes/archive-reconcile-worker.ts" -# The bundled `maple` binary bakes __CHDB_VERSION__=v26.1.0 via bun --define at +# The bundled `maple` binary bakes __CHDB_VERSION__ via bun --define at # compile time; running the workers from SOURCE defaults CHDB_VERSION to "dev", # which makes the restore version-check reject a checkpoint made by the bundle. # Define it to match so the source-tree workers are version-consistent with the # bundle's checkpoints. (The deployed binary is consistent by construction.) CHDB_VER="$("$MAPLE" --version 2>/dev/null | grep -oE 'chdb v[^ ]+' | sed 's/chdb //')" -[ -z "$CHDB_VER" ] && CHDB_VER="v26.1.0" +[ -z "$CHDB_VER" ] && CHDB_VER="dev" BUN=(bun --define "__CHDB_VERSION__=\"${CHDB_VER}\"") command -v duckdb >/dev/null 2>&1 || { echo "FAIL: duckdb required" >&2; exit 1; } [ -x "$MAPLE" ] || { echo "FAIL: maple binary not found at $MAPLE" >&2; exit 1; } -[ -f "$LIBCHDB" ] || { echo "FAIL: libchdb not found at $LIBCHDB" >&2; exit 1; } +[ -d "$CHDB_NODE_MODULES/chdb" ] || { echo "FAIL: chdb runtime not found at $CHDB_NODE_MODULES/chdb" >&2; exit 1; } pass=0 fail=0 @@ -108,7 +109,7 @@ spawn_and_kill() { archive="$ROOT/archive" scratch="$ROOT/scratch" : >"$ROOT/worker-$boundary.out" - MAPLE_LIBCHDB="$LIBCHDB" "${BUN[@]}" "$WORKER" \ + "${BUN[@]}" "$WORKER" \ --boundary "$boundary" --marker-dir "$marker" \ --data-dir "$data" --archive-dir "$archive" --scratch-root "$scratch" \ --range-date "$RANGE_DATE" --signal "$SIGNAL" --block-ms 60000 \ @@ -321,7 +322,7 @@ run_boundary() { data="$(cat "$ROOT/data.path")" archive="$ROOT/archive" # Reconcile WITHOUT a fresh export. - if ! MAPLE_LIBCHDB="$LIBCHDB" "${BUN[@]}" "$RECONCILE" --data-dir "$data" --archive-dir "$archive" --scratch-root "$ROOT/scratch" >"$ROOT/reconcile-$boundary.out" 2>&1; then + if ! "${BUN[@]}" "$RECONCILE" --data-dir "$data" --archive-dir "$archive" --scratch-root "$ROOT/scratch" >"$ROOT/reconcile-$boundary.out" 2>&1; then echo " !! reconcile threw for $boundary:" >&2; tail -3 "$ROOT/reconcile-$boundary.out" >&2 fail=$((fail+1)); FAILURES+=("$boundary"); return fi @@ -333,14 +334,14 @@ run_boundary() { # op must leave telemetry intact. verify_live_store_unchanged "$boundary" || { fail=$((fail+1)); FAILURES+=("$boundary:live-store"); return; } # Idempotence: reconcile AGAIN, expect the same converged state + exit 0. - if ! MAPLE_LIBCHDB="$LIBCHDB" "${BUN[@]}" "$RECONCILE" --data-dir "$data" --archive-dir "$archive" --scratch-root "$ROOT/scratch" >"$ROOT/reconcile2-$boundary.out" 2>&1; then + if ! "${BUN[@]}" "$RECONCILE" --data-dir "$data" --archive-dir "$archive" --scratch-root "$ROOT/scratch" >"$ROOT/reconcile2-$boundary.out" 2>&1; then echo " !! second reconcile (idempotence) threw for $boundary" >&2; fail=$((fail+1)); FAILURES+=("$boundary:idempotence"); return fi verify_post_crash "$boundary" "$expect_published" "$expect_quarantine" "$quarantine_layout" >/dev/null || { echo " !! state drifted after second reconcile for $boundary" >&2; fail=$((fail+1)); FAILURES+=("$boundary:idempotence"); return; } pass=$((pass+1)) } -echo "=== Archive crash-recovery probe (libchdb=$(basename "$LIBCHDB")) ===" +echo "=== Archive crash-recovery probe (chdb runtime=$CHDB_NODE_MODULES) ===" echo " boundary crash via real SIGKILL; oracle = reconcile then verify exact state" echo diff --git a/apps/cli/test/native-archive-gc-probe.sh b/apps/cli/test/native-archive-gc-probe.sh index 9f433eb9a..88c2ca6ce 100755 --- a/apps/cli/test/native-archive-gc-probe.sh +++ b/apps/cli/test/native-archive-gc-probe.sh @@ -21,7 +21,8 @@ set -uo pipefail BUNDLE_DIR="${1:?usage: $0 [port]}" MAPLE="$BUNDLE_DIR/maple" -LIBCHDB="${MAPLE_LIBCHDB:-$BUNDLE_DIR/libchdb.so}" +CHDB_NODE_MODULES="${MAPLE_CHDB_NODE_MODULES:-$BUNDLE_DIR/node_modules}" +export MAPLE_CHDB_NODE_MODULES="$CHDB_NODE_MODULES" PORT="${2:-45401}" REPO="$(cd "$(dirname "$0")/../../.." && pwd)" WORKER="$REPO/apps/cli/test/probes/archive-gc-worker.ts" @@ -36,10 +37,10 @@ SIGNAL="traces" command -v duckdb >/dev/null 2>&1 || { echo "FAIL: duckdb required" >&2; exit 1; } [ -x "$MAPLE" ] || { echo "FAIL: maple binary not found at $MAPLE" >&2; exit 1; } -[ -f "$LIBCHDB" ] || { echo "FAIL: libchdb not found at $LIBCHDB" >&2; exit 1; } +[ -d "$CHDB_NODE_MODULES/chdb" ] || { echo "FAIL: chdb runtime not found at $CHDB_NODE_MODULES/chdb" >&2; exit 1; } CHDB_VER="$("$MAPLE" --version 2>/dev/null | grep -oE 'chdb v[^ ]+' | sed 's/chdb //')" -[ -z "$CHDB_VER" ] && CHDB_VER="v26.1.0" +[ -z "$CHDB_VER" ] && CHDB_VER="dev" BUN=(bun --define "__CHDB_VERSION__=\"${CHDB_VER}\"") pass=0 @@ -128,7 +129,7 @@ spawn_and_kill_gc() { local marker="$1" boundary="$2" local data archive data="$(cat "$ROOT/data.path")"; archive="$ROOT/archive" - MAPLE_LIBCHDB="$LIBCHDB" "${BUN[@]}" "$WORKER" \ + "${BUN[@]}" "$WORKER" \ --boundary "$boundary" --marker-dir "$marker" \ --data-dir "$data" --archive-dir "$archive" --scratch-root "$ROOT/scratch" \ --keep 0 --block-ms 60000 >"$ROOT/gc-worker.out" 2>&1 & @@ -274,7 +275,7 @@ run_gc_crash() { echo " create-after: OK (crashed GC did not block future work)" } -echo "=== Archive interrupted-GC crash-recovery probe (libchdb=$(basename "$LIBCHDB")) ===" +echo "=== Archive interrupted-GC crash-recovery probe (chdb runtime=$CHDB_NODE_MODULES) ===" echo " real SIGKILL mid-collection → real reconcile CLI → verify convergence + idempotence + create-after" echo diff --git a/apps/cli/test/native-local-store-migration.sh b/apps/cli/test/native-local-store-migration.sh index 989f10811..67ead8572 100755 --- a/apps/cli/test/native-local-store-migration.sh +++ b/apps/cli/test/native-local-store-migration.sh @@ -11,11 +11,12 @@ set -euo pipefail BUNDLE_DIR="${1:?usage: native-local-store-migration.sh [port]}" REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" MAPLE="$BUNDLE_DIR/maple" -LIBCHDB="${MAPLE_LIBCHDB:-$BUNDLE_DIR/libchdb.so}" +CHDB_NODE_MODULES="${MAPLE_CHDB_NODE_MODULES:-$BUNDLE_DIR/node_modules}" +export MAPLE_CHDB_NODE_MODULES="$CHDB_NODE_MODULES" PORT="${2:-45241}" -if [[ ! -f "$LIBCHDB" ]]; then - echo "SKIP: native local-store migration requires libchdb at $LIBCHDB" >&2 +if [[ ! -d "$CHDB_NODE_MODULES/chdb" ]]; then + echo "SKIP: native local-store migration requires chdb runtime at $CHDB_NODE_MODULES/chdb" >&2 exit 0 fi [[ -x "$MAPLE" ]] || { echo "FAIL: maple binary not found at $MAPLE" >&2; exit 1; } @@ -80,7 +81,7 @@ wait_health() { } start_server() { - MAPLE_LIBCHDB="$LIBCHDB" "$MAPLE" start \ + "$MAPLE" start \ --port "$PORT" \ --data-dir "$DATA" \ --chdb-config-file "$CONFIG" \ @@ -91,7 +92,7 @@ start_server() { } stop_server() { - bounded 60 "maple stop" env MAPLE_LIBCHDB="$LIBCHDB" "$MAPLE" stop --data-dir "$DATA" >/dev/null 2>&1 || true + bounded 60 "maple stop" "$MAPLE" stop --data-dir "$DATA" >/dev/null 2>&1 || true # Bounded wait: a server that ignores the stop must not eat the job budget. for _ in $(seq 1 300); do kill -0 "$SERVER_PID" 2>/dev/null || break @@ -116,14 +117,14 @@ chmod 600 "$CONFIG" step "creating legacy v0 source store" bounded 60 "legacy source fixture" \ - env MAPLE_LIBCHDB="$LIBCHDB" bun "$REPO_ROOT/apps/cli/test/native-local-store-migration-fixture.ts" "$DATA" "$CONFIG" + bun "$REPO_ROOT/apps/cli/test/native-local-store-migration-fixture.ts" "$DATA" "$CONFIG" # Install the historical fingerprint-only marker for the v0 physical source. # The physical source was created by native chDB; the marker is the only # compatibility evidence the v0 resolver is allowed to use. step "installing legacy marker" -CHDB_VERSION="$(bounded 60 "maple --version" env MAPLE_LIBCHDB="$LIBCHDB" "$MAPLE" --version 2>/dev/null | sed -n 's/.*chdb \([^ ]*\).*/\1/p')" -[[ -n "$CHDB_VERSION" ]] || CHDB_VERSION="v26.1.0" +CHDB_VERSION="$(bounded 60 "maple --version" "$MAPLE" --version 2>/dev/null | sed -n 's/.*chdb \([^ ]*\).*/\1/p')" +[[ -n "$CHDB_VERSION" ]] || CHDB_VERSION="dev" printf '%s\n' "{\"chdb\":\"$CHDB_VERSION\",\"maple\":\"native-probe\",\"createdAt\":\"unknown\",\"schema\":\"428701854f9fd30e\"}" >"$ROOT/maple-store-version.json" chmod 600 "$ROOT/maple-store-version.json" @@ -131,7 +132,7 @@ step "running maple schema migrate" # 120s bound: healthy runs take ~3s; the job budget is <3 minutes, so a stall # has to fail the step in seconds-not-minutes with migrate.out as evidence. bounded 120 "maple schema migrate" \ - env MAPLE_LIBCHDB="$LIBCHDB" "$MAPLE" schema migrate --data-dir "$DATA" --yes >"$ROOT/migrate.out" 2>&1 || { + "$MAPLE" schema migrate --data-dir "$DATA" --yes >"$ROOT/migrate.out" 2>&1 || { cat "$ROOT/migrate.out" >&2 fail "native schema migration failed" } @@ -165,7 +166,7 @@ stop_server step "verifying the service-map ingress bridge" bounded 60 "service-map ingress bridge probe" \ - env MAPLE_LIBCHDB="$LIBCHDB" bun "$REPO_ROOT/apps/cli/test/native-service-map-ingest-bridge-probe.ts" "$DATA" "$CONFIG" + bun "$REPO_ROOT/apps/cli/test/native-service-map-ingest-bridge-probe.ts" "$DATA" "$CONFIG" rollback="$(sed -n 's/^.*rollback *//p' "$ROOT/migrate.out" | tail -1)" [[ -n "$rollback" && -d "$rollback" ]] || fail "native migration did not retain a rollback source" diff --git a/apps/cli/test/probes/archive-crash-worker.ts b/apps/cli/test/probes/archive-crash-worker.ts index 10cafaeac..22ee2f416 100644 --- a/apps/cli/test/probes/archive-crash-worker.ts +++ b/apps/cli/test/probes/archive-crash-worker.ts @@ -10,7 +10,7 @@ // so the post-crash state is exactly what a real crash leaves. // // Usage (parent harness): -// MAPLE_LIBCHDB=/libchdb.so \ +// MAPLE_CHDB_NODE_MODULES=/node_modules \ // bun apps/cli/test/probes/archive-crash-worker.ts \ // --boundary --marker-dir --data-dir --archive-dir \ // --scratch-root --range --signal [--block-ms ] diff --git a/apps/cli/test/probes/archive-gc-worker.ts b/apps/cli/test/probes/archive-gc-worker.ts index a6453cc0d..b8958c582 100644 --- a/apps/cli/test/probes/archive-gc-worker.ts +++ b/apps/cli/test/probes/archive-gc-worker.ts @@ -8,7 +8,7 @@ // gone, one remaining, journal mid-progress) is exactly what a real crash leaves. // // Usage: -// MAPLE_LIBCHDB=/libchdb.so bun apps/cli/test/probes/archive-gc-worker.ts \ +// MAPLE_CHDB_NODE_MODULES=/node_modules bun apps/cli/test/probes/archive-gc-worker.ts \ // --boundary --marker-dir --data-dir --archive-dir \ // --scratch-root --keep [--block-ms ] diff --git a/apps/cli/test/probes/archive-probe-byte-heterogeneous.ts b/apps/cli/test/probes/archive-probe-byte-heterogeneous.ts index 15dd45bd5..044ad0837 100644 --- a/apps/cli/test/probes/archive-probe-byte-heterogeneous.ts +++ b/apps/cli/test/probes/archive-probe-byte-heterogeneous.ts @@ -6,7 +6,7 @@ // Contract: exit 0 (PASS) when every shard's actual uncompressed size <= the // bound; exit nonzero (FAIL) when export aborts with "recalibrate"/"exceeds". // -// Run: MAPLE_LIBCHDB=/libchdb.so bun apps/cli/test/probes/archive-probe-byte-heterogeneous.ts +// Run: MAPLE_CHDB_NODE_MODULES=/node_modules bun apps/cli/test/probes/archive-probe-byte-heterogeneous.ts import { mkdirSync } from "node:fs" import { join } from "node:path" diff --git a/apps/cli/test/probes/archive-probe-byte-single-row.ts b/apps/cli/test/probes/archive-probe-byte-single-row.ts index 426fab007..22274cea1 100644 --- a/apps/cli/test/probes/archive-probe-byte-single-row.ts +++ b/apps/cli/test/probes/archive-probe-byte-single-row.ts @@ -4,7 +4,7 @@ // export fails with the distinct single-row message; exit nonzero (FAIL) when it // aborts with a generic message or (worse) succeeds. // -// Run: MAPLE_LIBCHDB=/libchdb.so bun apps/cli/test/probes/archive-probe-byte-single-row.ts +// Run: MAPLE_CHDB_NODE_MODULES=/node_modules bun apps/cli/test/probes/archive-probe-byte-single-row.ts import { mkdirSync } from "node:fs" import { join } from "node:path" diff --git a/apps/cli/test/probes/archive-probe-byte-uniform.ts b/apps/cli/test/probes/archive-probe-byte-uniform.ts index 7fdd20f54..7a0540f9b 100644 --- a/apps/cli/test/probes/archive-probe-byte-uniform.ts +++ b/apps/cli/test/probes/archive-probe-byte-uniform.ts @@ -3,7 +3,7 @@ // relocated hermetic. Contract: exit 0 (PASS) when uniform wide rows split into // multiple byte-bounded shards with the exact source set. // -// Run: MAPLE_LIBCHDB=/libchdb.so bun apps/cli/test/probes/archive-probe-byte-uniform.ts +// Run: MAPLE_CHDB_NODE_MODULES=/node_modules bun apps/cli/test/probes/archive-probe-byte-uniform.ts import { mkdirSync } from "node:fs" import { join } from "node:path" diff --git a/apps/cli/test/probes/archive-probe-complex-alter.ts b/apps/cli/test/probes/archive-probe-complex-alter.ts index 7bbc4d941..0988f9c28 100644 --- a/apps/cli/test/probes/archive-probe-complex-alter.ts +++ b/apps/cli/test/probes/archive-probe-complex-alter.ts @@ -3,7 +3,7 @@ // column change (this passed in round 4); the round-5 multiset must preserve it. // Contract: exit 0 (PASS) when the altered export's digest DIFFERS. // -// Run: MAPLE_LIBCHDB=/libchdb.so bun apps/cli/test/probes/archive-probe-complex-alter.ts +// Run: MAPLE_CHDB_NODE_MODULES=/node_modules bun apps/cli/test/probes/archive-probe-complex-alter.ts import { mkdirSync, rmSync } from "node:fs" import { join } from "node:path" diff --git a/apps/cli/test/probes/archive-probe-digest-column-swap.ts b/apps/cli/test/probes/archive-probe-digest-column-swap.ts index 88495a68c..b72ee4868 100644 --- a/apps/cli/test/probes/archive-probe-digest-column-swap.ts +++ b/apps/cli/test/probes/archive-probe-digest-column-swap.ts @@ -15,7 +15,7 @@ // Contract: exit 0 (PASS) when the column-swapped export's digest DIFFERS from // the original's; exit nonzero (FAIL) when equal (the commutative bug). // -// Run: MAPLE_LIBCHDB=/libchdb.so bun apps/cli/test/probes/archive-probe-digest-column-swap.ts +// Run: MAPLE_CHDB_NODE_MODULES=/node_modules bun apps/cli/test/probes/archive-probe-digest-column-swap.ts import { mkdirSync, rmSync } from "node:fs" import { join } from "node:path" diff --git a/apps/cli/test/probes/archive-probe-digest-dup-drop.ts b/apps/cli/test/probes/archive-probe-digest-dup-drop.ts index ba71680be..1bbfe3fee 100644 --- a/apps/cli/test/probes/archive-probe-digest-dup-drop.ts +++ b/apps/cli/test/probes/archive-probe-digest-dup-drop.ts @@ -4,7 +4,7 @@ // preserves duplicates detects this; a count-only or sum digest does not. // Contract: exit 0 (PASS) when the dup/drop export's digest DIFFERS. // -// Run: MAPLE_LIBCHDB=/libchdb.so bun apps/cli/test/probes/archive-probe-digest-dup-drop.ts +// Run: MAPLE_CHDB_NODE_MODULES=/node_modules bun apps/cli/test/probes/archive-probe-digest-dup-drop.ts import { mkdirSync, rmSync } from "node:fs" import { join } from "node:path" diff --git a/apps/cli/test/probes/archive-probe-digest-row-swap.ts b/apps/cli/test/probes/archive-probe-digest-row-swap.ts index 8011f0497..ef177e910 100644 --- a/apps/cli/test/probes/archive-probe-digest-row-swap.ts +++ b/apps/cli/test/probes/archive-probe-digest-row-swap.ts @@ -4,7 +4,7 @@ // hashes) cannot detect this. Contract: exit 0 (PASS) when the reassociated // export's digest DIFFERS; exit nonzero (FAIL) when equal (the bug). // -// Run: MAPLE_LIBCHDB=/libchdb.so bun apps/cli/test/probes/archive-probe-digest-row-swap.ts +// Run: MAPLE_CHDB_NODE_MODULES=/node_modules bun apps/cli/test/probes/archive-probe-digest-row-swap.ts import { mkdirSync, rmSync } from "node:fs" import { join } from "node:path" diff --git a/apps/cli/test/probes/archive-probe-duckdb-oracle.ts b/apps/cli/test/probes/archive-probe-duckdb-oracle.ts index d661e293c..022a49ce3 100644 --- a/apps/cli/test/probes/archive-probe-duckdb-oracle.ts +++ b/apps/cli/test/probes/archive-probe-duckdb-oracle.ts @@ -7,7 +7,7 @@ // // Contract: exit 0 (PASS) when every oracle matches; exit nonzero otherwise. // -// Run: MAPLE_LIBCHDB=/libchdb.so bun apps/cli/test/probes/archive-probe-duckdb-oracle.ts +// Run: MAPLE_CHDB_NODE_MODULES=/node_modules bun apps/cli/test/probes/archive-probe-duckdb-oracle.ts // (duckdb must be on PATH) import { mkdirSync } from "node:fs" diff --git a/apps/cli/test/probes/archive-probe-merge-freeze-leak.ts b/apps/cli/test/probes/archive-probe-merge-freeze-leak.ts index 1e74f0ef5..e25e286c9 100644 --- a/apps/cli/test/probes/archive-probe-merge-freeze-leak.ts +++ b/apps/cli/test/probes/archive-probe-merge-freeze-leak.ts @@ -3,7 +3,7 @@ // merges are confirmed restarted (a later OPTIMIZE is NOT rejected with code // 236) after a forced mid-export failure. // -// Run: MAPLE_LIBCHDB=/libchdb.so bun apps/cli/test/probes/archive-probe-merge-freeze-leak.ts +// Run: MAPLE_CHDB_NODE_MODULES=/node_modules bun apps/cli/test/probes/archive-probe-merge-freeze-leak.ts import { mkdirSync, writeFileSync } from "node:fs" import { join } from "node:path" diff --git a/apps/cli/test/probes/archive-probe-merge-injection.ts b/apps/cli/test/probes/archive-probe-merge-injection.ts index 85bb12b63..3f675d287 100644 --- a/apps/cli/test/probes/archive-probe-merge-injection.ts +++ b/apps/cli/test/probes/archive-probe-merge-injection.ts @@ -3,7 +3,7 @@ // source set. Contract: exit 0 (PASS) when OPTIMIZE is blocked (code 236) every // time and the archived IDs exactly equal the source IDs. // -// Run: MAPLE_LIBCHDB=/libchdb.so bun apps/cli/test/probes/archive-probe-merge-injection.ts +// Run: MAPLE_CHDB_NODE_MODULES=/node_modules bun apps/cli/test/probes/archive-probe-merge-injection.ts import { mkdirSync } from "node:fs" import { join } from "node:path" diff --git a/apps/cli/test/probes/archive-probe-mixed-hour.ts b/apps/cli/test/probes/archive-probe-mixed-hour.ts index 888be9d4e..73d89c9fc 100644 --- a/apps/cli/test/probes/archive-probe-mixed-hour.ts +++ b/apps/cli/test/probes/archive-probe-mixed-hour.ts @@ -4,7 +4,7 @@ // {tid0,tid9} (offsets 0 and 9, with hour-13 at offsets 1..8) archive exactly // with no cross-hour bleed and the full day's source set is archived once. // -// Run: MAPLE_LIBCHDB=/libchdb.so bun apps/cli/test/probes/archive-probe-mixed-hour.ts +// Run: MAPLE_CHDB_NODE_MODULES=/node_modules bun apps/cli/test/probes/archive-probe-mixed-hour.ts import { mkdirSync } from "node:fs" import { join } from "node:path" diff --git a/apps/cli/test/probes/archive-probe-multipart.ts b/apps/cli/test/probes/archive-probe-multipart.ts index 7559d407d..7af77e4a1 100644 --- a/apps/cli/test/probes/archive-probe-multipart.ts +++ b/apps/cli/test/probes/archive-probe-multipart.ts @@ -2,7 +2,7 @@ // the exact source set with no duplicates or omissions. Contract: exit 0 (PASS) // when 8 rows across 2 parts archive as the exact source set. // -// Run: MAPLE_LIBCHDB=/libchdb.so bun apps/cli/test/probes/archive-probe-multipart.ts +// Run: MAPLE_CHDB_NODE_MODULES=/node_modules bun apps/cli/test/probes/archive-probe-multipart.ts import { mkdirSync } from "node:fs" import { join } from "node:path" diff --git a/apps/cli/test/probes/archive-probe-null-digest.ts b/apps/cli/test/probes/archive-probe-null-digest.ts index 4945c3e88..12c0a11dc 100644 --- a/apps/cli/test/probes/archive-probe-null-digest.ts +++ b/apps/cli/test/probes/archive-probe-null-digest.ts @@ -3,7 +3,7 @@ // preserve it. Contract: exit 0 (PASS) when the export succeeds with a non-empty // digest for a row containing NULL columns. // -// Run: MAPLE_LIBCHDB=/libchdb.so bun apps/cli/test/probes/archive-probe-null-digest.ts +// Run: MAPLE_CHDB_NODE_MODULES=/node_modules bun apps/cli/test/probes/archive-probe-null-digest.ts import { mkdirSync } from "node:fs" import { join } from "node:path" diff --git a/apps/cli/test/probes/archive-probe-null-flag-binding.ts b/apps/cli/test/probes/archive-probe-null-flag-binding.ts index f8989fd84..07231ef94 100644 --- a/apps/cli/test/probes/archive-probe-null-flag-binding.ts +++ b/apps/cli/test/probes/archive-probe-null-flag-binding.ts @@ -7,7 +7,7 @@ // Contract: exit 0 (PASS) when a NULL and the sentinel string produce DIFFERENT // digests; exit nonzero (FAIL) when they collide. // -// Run: MAPLE_LIBCHDB=/libchdb.so bun apps/cli/test/probes/archive-probe-null-flag-binding.ts +// Run: MAPLE_CHDB_NODE_MODULES=/node_modules bun apps/cli/test/probes/archive-probe-null-flag-binding.ts import { mkdirSync, rmSync } from "node:fs" import { join } from "node:path" diff --git a/apps/cli/test/probes/archive-probe-null-value-sensitivity.ts b/apps/cli/test/probes/archive-probe-null-value-sensitivity.ts index 70b586d02..2a358ba09 100644 --- a/apps/cli/test/probes/archive-probe-null-value-sensitivity.ts +++ b/apps/cli/test/probes/archive-probe-null-value-sensitivity.ts @@ -8,7 +8,7 @@ // Contract: exit 0 (PASS) when the two datasets produce DIFFERENT digests; // exit nonzero (FAIL) when they are equal (the bare-NULL collapse). // -// Run: MAPLE_LIBCHDB=/libchdb.so bun apps/cli/test/probes/archive-probe-null-value-sensitivity.ts +// Run: MAPLE_CHDB_NODE_MODULES=/node_modules bun apps/cli/test/probes/archive-probe-null-value-sensitivity.ts import { mkdirSync, rmSync } from "node:fs" import { join } from "node:path" diff --git a/apps/cli/test/probes/archive-probe-schema-substitution.ts b/apps/cli/test/probes/archive-probe-schema-substitution.ts index 9546e46d7..40b7699c4 100644 --- a/apps/cli/test/probes/archive-probe-schema-substitution.ts +++ b/apps/cli/test/probes/archive-probe-schema-substitution.ts @@ -4,7 +4,7 @@ // is accepted. Contract: exit 0 (PASS) when the substitution is rejected AND the // valid schema is accepted; exit nonzero if either check is wrong. // -// Run: MAPLE_LIBCHDB=/libchdb.so bun apps/cli/test/probes/archive-probe-schema-substitution.ts +// Run: MAPLE_CHDB_NODE_MODULES=/node_modules bun apps/cli/test/probes/archive-probe-schema-substitution.ts import { ArchiveProbe } from "../archive-probe-helpers" import { compareSchema } from "../../src/server/archives/export" diff --git a/apps/cli/test/probes/archive-probe-timezone-bound.ts b/apps/cli/test/probes/archive-probe-timezone-bound.ts index 7b7e8fbac..ddc07363f 100644 --- a/apps/cli/test/probes/archive-probe-timezone-bound.ts +++ b/apps/cli/test/probes/archive-probe-timezone-bound.ts @@ -8,7 +8,7 @@ // This probe parses a manifest shard record directly (no chDB needed) under // TZ=America/New_York to reproduce the host-timezone dependence deterministically. // -// Run: TZ=America/New_York MAPLE_LIBCHDB=/libchdb.so bun apps/cli/test/probes/archive-probe-timezone-bound.ts +// Run: TZ=America/New_York MAPLE_CHDB_NODE_MODULES=/node_modules bun apps/cli/test/probes/archive-probe-timezone-bound.ts import { ArchiveProbe } from "../archive-probe-helpers" import { parseArchiveGenerationManifest } from "../../src/server/archives/manifest" diff --git a/apps/cli/test/probes/archive-reconcile-worker.ts b/apps/cli/test/probes/archive-reconcile-worker.ts index cf6e83494..b20a6216b 100644 --- a/apps/cli/test/probes/archive-reconcile-worker.ts +++ b/apps/cli/test/probes/archive-reconcile-worker.ts @@ -8,7 +8,7 @@ // export. // // Usage: -// MAPLE_LIBCHDB=/libchdb.so \ +// MAPLE_CHDB_NODE_MODULES=/node_modules \ // bun apps/cli/test/probes/archive-reconcile-worker.ts \ // --data-dir --archive-dir // diff --git a/apps/cli/test/update.test.ts b/apps/cli/test/update.test.ts index 9b6d8d4d0..78ed9aeda 100644 --- a/apps/cli/test/update.test.ts +++ b/apps/cli/test/update.test.ts @@ -264,23 +264,28 @@ describe("swapBundlePair", () => { await mkdir(installDir, { recursive: true }) await mkdir(srcDir, { recursive: true }) await mkdir(tmpDir, { recursive: true }) + await mkdir(join(installDir, "node_modules", "chdb"), { recursive: true }) await writeFile(join(installDir, "maple"), "old-maple") - await writeFile(join(installDir, "libchdb.so"), "old-lib") + await writeFile(join(installDir, "node_modules", "chdb", "package.json"), "old-runtime") return { root, installDir, srcDir, tmpDir } } // Plain `it` + `Effect.runPromise` (see archive-candidate-child.test.ts): // `it.effect` hangs on real fs work under bun test. - it("installs both files together", () => + it("installs the executable and runtime sidecar together", () => Effect.runPromise( Effect.gen(function* () { const { root, installDir, srcDir, tmpDir } = yield* Effect.promise(layout) const fs = yield* FileSystem yield* fs.writeFileString(join(srcDir, "maple"), "new-maple") - yield* fs.writeFileString(join(srcDir, "libchdb.so"), "new-lib") + yield* fs.makeDirectory(join(srcDir, "node_modules", "chdb"), { recursive: true }) + yield* fs.writeFileString(join(srcDir, "node_modules", "chdb", "package.json"), "new-runtime") yield* __testables.swapBundlePair(srcDir, installDir, tmpDir) strictEqual(yield* fs.readFileString(join(installDir, "maple")), "new-maple") - strictEqual(yield* fs.readFileString(join(installDir, "libchdb.so")), "new-lib") + strictEqual( + yield* fs.readFileString(join(installDir, "node_modules", "chdb", "package.json")), + "new-runtime", + ) yield* fs.remove(root, { recursive: true, force: true }) }).pipe(Effect.provide(BunServices.layer)), )) @@ -290,14 +295,17 @@ describe("swapBundlePair", () => { Effect.gen(function* () { const { root, installDir, srcDir, tmpDir } = yield* Effect.promise(layout) const fs = yield* FileSystem - // Only the executable extracted — the library rename will fail after - // the maple swap already happened. The old code left new-maple beside - // old-lib; the swap must put the matched old pair back instead. + // Only the executable extracted — the runtime rename will fail after + // the maple swap already happened. The swap must put the matched old + // pair back instead. yield* fs.writeFileString(join(srcDir, "maple"), "new-maple") const exit = yield* __testables.swapBundlePair(srcDir, installDir, tmpDir).pipe(Effect.exit) ok(Exit.isFailure(exit), "swap must report the failure") strictEqual(yield* fs.readFileString(join(installDir, "maple")), "old-maple") - strictEqual(yield* fs.readFileString(join(installDir, "libchdb.so")), "old-lib") + strictEqual( + yield* fs.readFileString(join(installDir, "node_modules", "chdb", "package.json")), + "old-runtime", + ) yield* fs.remove(root, { recursive: true, force: true }) }).pipe(Effect.provide(BunServices.layer)), )) diff --git a/apps/landing/src/components/local/InstallTabs.astro b/apps/landing/src/components/local/InstallTabs.astro index 71b42a1ec..52ab4ed63 100644 --- a/apps/landing/src/components/local/InstallTabs.astro +++ b/apps/landing/src/components/local/InstallTabs.astro @@ -30,7 +30,7 @@ const methods: Method[] = [ recommended: true, command: "brew install Makisuo/tap/maple", blurb: - "Homebrew downloads the matching release bundle, verifies its checksum, installs maple and libchdb together, and links maple onto your PATH.", + "Homebrew downloads the matching release bundle, verifies its checksum, installs maple and the node_modules/chdb runtime sidecar together, and links maple onto your PATH.", upgrade: "brew upgrade maple", uninstall: "brew uninstall maple", notes: [ @@ -43,7 +43,7 @@ const methods: Method[] = [ label: "Install script", command: "curl -fsSL https://maple.dev/cli/install | sh", blurb: - "The script is scripts/install.sh — read it first. It detects your OS/arch, downloads the matching bundle from the latest GitHub release, verifies its checksum, installs maple + libchdb into ~/.maple/bin, clears the macOS Gatekeeper quarantine, and symlinks maple onto your PATH.", + "The script is scripts/install.sh — read it first. It detects your OS/arch, downloads the matching bundle from the latest GitHub release, verifies its checksum, installs maple + node_modules/chdb into ~/.maple/bin, clears the macOS Gatekeeper quarantine, and symlinks maple onto your PATH.", uninstall: "curl -fsSL https://maple.dev/cli/uninstall | sh", notes: [ "Your ~/.maple/data is kept on uninstall.", diff --git a/apps/landing/src/content/docs/local-mode.mdx b/apps/landing/src/content/docs/local-mode.mdx index d4a57e7a2..1ef6c3f56 100644 --- a/apps/landing/src/content/docs/local-mode.mdx +++ b/apps/landing/src/content/docs/local-mode.mdx @@ -20,7 +20,7 @@ Pick your package manager. Homebrew is the easiest path on macOS and Linux; the -> Prefer to do it by hand? Download a release bundle (both `maple` + `libchdb`) from [GitHub Releases](https://github.com/MapleTechLabs/maple/releases), or read [`scripts/install.sh`](https://github.com/MapleTechLabs/maple/blob/main/scripts/install.sh) first before piping it to a shell. +> Prefer to do it by hand? Download a release bundle (both `maple` + `node_modules/chdb`) from [GitHub Releases](https://github.com/MapleTechLabs/maple/releases), or read [`scripts/install.sh`](https://github.com/MapleTechLabs/maple/blob/main/scripts/install.sh) first before piping it to a shell. ## Start the server @@ -82,4 +82,4 @@ The same CLI talks to either your local server or a hosted Maple workspace. The ## How it works -One Bun-compiled binary is both the CLI and the server, talking to ClickHouse in-process via `bun:ffi` → `libchdb` (no subprocess, no second language). For the architecture, the `/local/query` contract, the UI-origin model, and the release bundle, see the [local-mode design doc](https://github.com/MapleTechLabs/maple/blob/main/docs/local-mode.md). +One Bun-compiled binary is both the CLI and the server, talking to ClickHouse in-process through the published `chdb` npm package and its native runtime sidecar (no subprocess, no second language). For the architecture, the `/local/query` contract, the UI-origin model, and the release bundle, see the [local-mode design doc](https://github.com/MapleTechLabs/maple/blob/main/docs/local-mode.md). diff --git a/apps/landing/src/content/docs/local-mode/cli-reference.md b/apps/landing/src/content/docs/local-mode/cli-reference.md index 661c46e87..a51c06f4d 100644 --- a/apps/landing/src/content/docs/local-mode/cli-reference.md +++ b/apps/landing/src/content/docs/local-mode/cli-reference.md @@ -451,19 +451,19 @@ OTLP bodies may be protobuf (default) or JSON, optionally gzip-encoded. The `/lo **Runtime** (CLI + server): -| Variable | Default | Purpose | -| ---------------------------- | -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `MAPLE_LOCAL_BIND_HOST` | `127.0.0.1` | Server bind host and default same-machine CLI target; wildcards map to loopback | -| `MAPLE_LOCAL_ADVERTISE_HOST` | connection-safe bind host | Host printed for clients and the bundled UI | -| `MAPLE_LOCAL_URL` | derived bind host + `4318` | Explicit base URL override for CLI query and mode detection | -| `MAPLE_LOCAL_UI_URL` | `https://local.maple.dev` | Exact separately hosted UI origin linked by `maple start` and allowed by CORS | -| `MAPLE_LIBCHDB` | _(auto)_ | Explicit path to `libchdb`. Otherwise resolved beside the binary (Homebrew keeps it in the same `libexec` dir), then `~/.maple/bin/libchdb.{so,dylib}` | -| `MAPLE_API_URL` | `https://api.maple.dev` | Remote API base URL | -| `MAPLE_API_TOKEN` | | Remote bearer token (overrides the stored value) | -| `MAPLE_ORG_ID` | | Remote org override | -| `MAPLE_DEBUG` | | Set to `1` to enable `--debug` | -| `MAPLE_FORMAT` | `json` | `json` or `table` — same as `--format` | -| `MAPLE_NO_UPDATE_CHECK` | | Set to `1` to disable startup update checks (the Homebrew wrapper sets this automatically) | +| Variable | Default | Purpose | +| ---------------------------- | -------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- | +| `MAPLE_LOCAL_BIND_HOST` | `127.0.0.1` | Server bind host and default same-machine CLI target; wildcards map to loopback | +| `MAPLE_LOCAL_ADVERTISE_HOST` | connection-safe bind host | Host printed for clients and the bundled UI | +| `MAPLE_LOCAL_URL` | derived bind host + `4318` | Explicit base URL override for CLI query and mode detection | +| `MAPLE_LOCAL_UI_URL` | `https://local.maple.dev` | Exact separately hosted UI origin linked by `maple start` and allowed by CORS | +| `MAPLE_CHDB_NODE_MODULES` | _(auto)_ | Explicit `node_modules` directory containing `chdb`. Otherwise resolved beside the binary, then from the source workspace during development. | +| `MAPLE_API_URL` | `https://api.maple.dev` | Remote API base URL | +| `MAPLE_API_TOKEN` | | Remote bearer token (overrides the stored value) | +| `MAPLE_ORG_ID` | | Remote org override | +| `MAPLE_DEBUG` | | Set to `1` to enable `--debug` | +| `MAPLE_FORMAT` | `json` | `json` or `table` — same as `--format` | +| `MAPLE_NO_UPDATE_CHECK` | | Set to `1` to disable startup update checks (the Homebrew wrapper sets this automatically) | **Homebrew**: @@ -481,7 +481,7 @@ If Homebrew asks you to trust the third-party tap, run `brew trust Makisuo/tap` | Variable | Default | Purpose | | --------------------- | -------------- | ----------------------------------------------------------------- | | `MAPLE_VERSION` | `latest` | Release tag to install | -| `MAPLE_INSTALL_DIR` | `~/.maple/bin` | Where the 2-file bundle is installed | +| `MAPLE_INSTALL_DIR` | `~/.maple/bin` | Where the bundle directory is installed | | `MAPLE_BIN_DIR` | _(auto)_ | Where `maple` is symlinked onto `PATH` | | `MAPLE_SKIP_CHECKSUM` | `0` | Set to `1` to skip SHA-256 verification (air-gapped mirrors only) | @@ -489,7 +489,7 @@ The on-disk config at `~/.maple/config.json` stores `apiUrl`, `token`, `orgId`, ## Troubleshooting -**`libchdb` not found.** The binary `dlopen`s `libchdb` relative to its own path, then falls back to `~/.maple/bin`. Homebrew keeps `maple` and `libchdb` together in its Cellar; the manual installer keeps them in `~/.maple/bin`. If you move files by hand, keep `libchdb.so`/`.dylib` beside `maple`, or set `MAPLE_LIBCHDB` to its full path. (Running from source has no sibling library — set `MAPLE_LIBCHDB` or drop one in `~/.maple/bin`.) +**`chdb` runtime not found.** The binary resolves `node_modules/chdb` beside `maple`; the manual installer keeps both under `~/.maple/bin`. If you move files by hand, keep the `node_modules` sidecar next to `maple`, or set `MAPLE_CHDB_NODE_MODULES` to a `node_modules` directory that contains `chdb`. **Homebrew installed but `maple` still runs the old binary.** You probably have a manual-installer symlink earlier on `PATH`. Run `command -v maple` to confirm, then remove the old symlink or run `curl -fsSL https://maple.dev/cli/uninstall | sh` before reinstalling with Homebrew. diff --git a/bun.lock b/bun.lock index c2bfb9dbe..af7663625 100644 --- a/bun.lock +++ b/bun.lock @@ -96,6 +96,7 @@ "@maple-dev/effect-sdk": "workspace:*", "@maple/domain": "workspace:*", "@maple/query-engine": "workspace:*", + "chdb": "3.3.0", "effect": "catalog:effect", "protobufjs": "^8.6.1", }, @@ -983,6 +984,14 @@ "@capsizecss/unpack": ["@capsizecss/unpack@4.0.1", "", { "dependencies": { "fontkitten": "^1.0.3" } }, "sha512-CuNiSqg7+e1cO/GjffyMOm5Tt2jUF9CWHHnvQ/UkqvtkGfHdgwEC0wpmq7fkN3gxwpRnrAN0WzO3vREKmNolMQ=="], + "@chdb/lib-darwin-arm64": ["@chdb/lib-darwin-arm64@26.7.0-stable.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-jCcGXEqntSdEoJW7fE/EcLX474Z1gynUosJktxiNtNLtxdbLFdyXOtAMy4vCsirLfAkY3mIZVNGzRu3Quzc31A=="], + + "@chdb/lib-darwin-x64": ["@chdb/lib-darwin-x64@26.7.0-stable.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-1leKMckvGnvPgXZMgfowQLiwJ6a/Dx8Sm4yfZFO34ZnVP8sSsWY1TjTpEJfQc9QCmBEUHWo0Wt5L8/Q+dIWybQ=="], + + "@chdb/lib-linux-arm64-gnu": ["@chdb/lib-linux-arm64-gnu@26.7.0-stable.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-IWeWofDCNtYXu8GHEZIGvt5gk7KHMILDlKNqMlFa6S+29pJdL3ROGkMrWW76yWe99kfsjTh4Lfx7BcTJ/ooAnA=="], + + "@chdb/lib-linux-x64-gnu": ["@chdb/lib-linux-x64-gnu@26.7.0-stable.1", "", { "os": "linux", "cpu": "x64" }, "sha512-9DYZmFeyuh5uvDSQ0Eh7ff4EwxdIP1WmUGTX6DeaSWtQWVn8+ELs84uKxalN2nM5yZxvs82NzW+snOE0gk8Fzg=="], + "@chevrotain/cst-dts-gen": ["@chevrotain/cst-dts-gen@10.5.0", "", { "dependencies": { "@chevrotain/gast": "10.5.0", "@chevrotain/types": "10.5.0", "lodash": "4.17.21" } }, "sha512-lhmC/FyqQ2o7pGK4Om+hzuDrm9rhFYIJ/AXoQBeongmn870Xeb0L6oGEiuR8nohFNL5sMaQEJWCxr1oIVIVXrw=="], "@chevrotain/gast": ["@chevrotain/gast@10.5.0", "", { "dependencies": { "@chevrotain/types": "10.5.0", "lodash": "4.17.21" } }, "sha512-pXdMJ9XeDAbgOWKuD1Fldz4ieCs6+nLNmyVhe2gZVqoO7v8HXuHYs5OV2EzUtbuai37TlOAQHrTDvxMnvMJz3A=="], @@ -1003,6 +1012,8 @@ "@clerk/themes": ["@clerk/themes@2.4.57", "", { "dependencies": { "@clerk/shared": "^3.47.2", "tslib": "2.8.1" } }, "sha512-Nb3bO79rMTU/MPVTC/dde6LG27/IgOMKIYi5KSvAmO4ZUHlj0OWufu6CMvz5OYVZ0YdyMnTBU2aPGRUiRzO+2w=="], + "@clickhouse/client-common": ["@clickhouse/client-common@1.23.0", "", {}, "sha512-OkuxPkQ+ph8CCrt/4DYBvlJ9D9Q9q7FwvP8FdwER+7UDNO3jN4Ee6SZl9iLupKJIvKeuS68+x86ZyqFuHdPK3g=="], + "@clickhouse/client-web": ["@clickhouse/client-web@1.23.1", "", {}, "sha512-VpYPSnBD7PWmEucp8wfd24u9tkyLwFH2ATyMT9R6J8b4U9snbVX1QjanJt/92YtSjam1GblhM7t3oChmJF/m0Q=="], "@cloudflare/kv-asset-handler": ["@cloudflare/kv-asset-handler@0.5.0", "", {}, "sha512-jxQYkj8dSIzc0cD6cMMNdOc1UVjqSqu8BZdor5s8cGjW2I8BjODt/kWPVdY+u9zj3ms75Q5qaZgnxUad83+eAg=="], @@ -1761,56 +1772,8 @@ "@rolldown/pluginutils": ["@rolldown/pluginutils@1.0.1", "", {}, "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw=="], - "@rollup/rollup-android-arm-eabi": ["@rollup/rollup-android-arm-eabi@4.62.3", "", { "os": "android", "cpu": "arm" }, "sha512-c0wdcekXtQvvn5Tsrk/+op/gUArrbWaFduBnTLP2l1cKLSQs4diMWjJw3m6A0DdzT8dAAX95KpkJ3qynCePbmw=="], - - "@rollup/rollup-android-arm64": ["@rollup/rollup-android-arm64@4.62.3", "", { "os": "android", "cpu": "arm64" }, "sha512-3YjElDdWN+qXAFbJ/CzPV+0wspLqh54k/I6GfdYtEJRqg7buSgc1yPM3B+93j1M4neobtkATHZTmxK2AMVGfnA=="], - "@rollup/rollup-darwin-arm64": ["@rollup/rollup-darwin-arm64@4.62.3", "", { "os": "darwin", "cpu": "arm64" }, "sha512-Pch2pFNOxxz1hTjypIdPyRTR6riiwRl84+VcN9djS680fw+Co1nAJINrdpqp7KV0NvyuU8ilZXZCjd7ykJl1GQ=="], - "@rollup/rollup-darwin-x64": ["@rollup/rollup-darwin-x64@4.62.3", "", { "os": "darwin", "cpu": "x64" }, "sha512-LEuncFUHFiF8t4yZVZvvZA1wk0pjAscRnsrn1EfTEmN4HXotBi2YtcnLRyaK6UbuczW7xZS5ES+81Rdz8Z0T6g=="], - - "@rollup/rollup-freebsd-arm64": ["@rollup/rollup-freebsd-arm64@4.62.3", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-zvBUvsQUpOWALdDsk6qbS8bXf2VxmPisuudNDrY7x0p0jBdsoZl8HsHczIOgkQiZldmcacMKtBzpoGVNeIe2bQ=="], - - "@rollup/rollup-freebsd-x64": ["@rollup/rollup-freebsd-x64@4.62.3", "", { "os": "freebsd", "cpu": "x64" }, "sha512-C2KmNrcSem/AMg984H/dev+si0lieQGdXdR/lYGJnuumXnFb9Y7QdiI62obFdLlxRYLBv4P0eUVIDbD4c1vVvw=="], - - "@rollup/rollup-linux-arm-gnueabihf": ["@rollup/rollup-linux-arm-gnueabihf@4.62.3", "", { "os": "linux", "cpu": "arm" }, "sha512-ggXnsTAEzNQx74XpunRsiZ9aBZDsI7XIa0hm2nzR9f4WzH5/f/d73ZSDaC5ejJ8YLY4NW+V3wr0tjOaeCq8hqA=="], - - "@rollup/rollup-linux-arm-musleabihf": ["@rollup/rollup-linux-arm-musleabihf@4.62.3", "", { "os": "linux", "cpu": "arm" }, "sha512-2vng+FlzNUhKZxtej3IUqJgbZoQk2M/dwQM20+ULV0R/E/8tr9/P6uEf2iiGIk4HL0zMKh5Jry7mUHdUOvyGgA=="], - - "@rollup/rollup-linux-arm64-gnu": ["@rollup/rollup-linux-arm64-gnu@4.62.3", "", { "os": "linux", "cpu": "arm64" }, "sha512-LLLFZKt4/Nraf9rxDkhiU8QVgLF4WmCkfr0L4fj0fPfIZFBib0DeiFk1hhaYKd03LFAFJcxHslhDFlNJLylf5Q=="], - - "@rollup/rollup-linux-arm64-musl": ["@rollup/rollup-linux-arm64-musl@4.62.3", "", { "os": "linux", "cpu": "arm64" }, "sha512-WJkdQCvS9sWNOUBJZfQRKpZGFBztRzcowI+nndmflKgU4XY+3a420FgTOSKTsVqJbnzSxeT4vaJalpOaPo2YCQ=="], - - "@rollup/rollup-linux-loong64-gnu": ["@rollup/rollup-linux-loong64-gnu@4.62.3", "", { "os": "linux", "cpu": "none" }, "sha512-PwHXCCS2n64/1Ot6rP1YEYA02MGYBcQlr8CSZZyrUG2O7NH6NklYmvr9v3Jy+5e/eDeNchc/ukmKJi9LuflMIQ=="], - - "@rollup/rollup-linux-loong64-musl": ["@rollup/rollup-linux-loong64-musl@4.62.3", "", { "os": "linux", "cpu": "none" }, "sha512-vUjxINQu3RC8NZS3ykk1gN65gIz8pAopOq2HXuZhiIxHdx7TFvDG+jgrdSgInu1Eza4/Rfi2VzZgyIgEH4WOaw=="], - - "@rollup/rollup-linux-ppc64-gnu": ["@rollup/rollup-linux-ppc64-gnu@4.62.3", "", { "os": "linux", "cpu": "ppc64" }, "sha512-wzko4aJ13+0G3kGnviCg5gnXFKd40izKsrf2uOw12US4XqprkDrmwOpeW14aSNa37V8bfPcz5Fkob6LZ3BAPmA=="], - - "@rollup/rollup-linux-ppc64-musl": ["@rollup/rollup-linux-ppc64-musl@4.62.3", "", { "os": "linux", "cpu": "ppc64" }, "sha512-8120ue0JUMSwy11stlwnfdX3pPd+WZYGCDBwEHWtIHi6pOpZmsEF5QKB7a/UN+XFdqvobxz98kv8RTqikyCEBw=="], - - "@rollup/rollup-linux-riscv64-gnu": ["@rollup/rollup-linux-riscv64-gnu@4.62.3", "", { "os": "linux", "cpu": "none" }, "sha512-XLFHnR3tXMjbOCh2vtVJHmxt+995uJsTERQyseFDRA0xxMxyTZPLa3OIUlyFaO4mF/Lu0FjmWHCuPXJT1n/IOg=="], - - "@rollup/rollup-linux-riscv64-musl": ["@rollup/rollup-linux-riscv64-musl@4.62.3", "", { "os": "linux", "cpu": "none" }, "sha512-se6yXvNGMIl0f+RQzyh7XAmia8/9kplQx424wnG2w0C1oi6XgO6Y8otKhdXFHbHs88Ihavzmvh1NWjuovE76BQ=="], - - "@rollup/rollup-linux-s390x-gnu": ["@rollup/rollup-linux-s390x-gnu@4.62.3", "", { "os": "linux", "cpu": "s390x" }, "sha512-gNoxRefktVIiGflpONuxWWXZAzIQG++z9qHO3xKwk4WdDMuQja3JHGfE1u0i3PfPDyvhypdk+WrgIJqLhGG7sg=="], - - "@rollup/rollup-linux-x64-gnu": ["@rollup/rollup-linux-x64-gnu@4.62.3", "", { "os": "linux", "cpu": "x64" }, "sha512-V4KtWtQfAFMU7+9/A/VDps/VI8CHd3cYz0L8sgJzz8qK7eY7wI4ruFD82UYIYvW9Z4DtlTfhQcsl4XyPHW5uSg=="], - - "@rollup/rollup-linux-x64-musl": ["@rollup/rollup-linux-x64-musl@4.62.3", "", { "os": "linux", "cpu": "x64" }, "sha512-LBx9LYXvj2CBkMkjLdNAWLwH0MLMin7do2VcVo9kVPibGLkY0BQQut2fv7NVqkXqZ/CrAu9LqDHVV1xHCMpCPw=="], - - "@rollup/rollup-openbsd-x64": ["@rollup/rollup-openbsd-x64@4.62.3", "", { "os": "openbsd", "cpu": "x64" }, "sha512-ABVf3Q0RCu7NcyCCOZQI0pJ3GuSdfSl8EXcy88QtdceIMIoCUdfhsJChZ64L9zVM2aJHjde1Bhn5uqSRcX9ySA=="], - - "@rollup/rollup-openharmony-arm64": ["@rollup/rollup-openharmony-arm64@4.62.3", "", { "os": "none", "cpu": "arm64" }, "sha512-+2Cy/ldweGBLlPIKsQLF8U5N44a0KDdbrk1rAjHOM9M2K+kGdIVjHLmmrZIcx+9Ny3ke/1JomCsDI1ocb11+sg=="], - - "@rollup/rollup-win32-arm64-msvc": ["@rollup/rollup-win32-arm64-msvc@4.62.3", "", { "os": "win32", "cpu": "arm64" }, "sha512-dtZvzc8BedpSaFNy75x6uiWwAGTH+aZHDtdrqP6qk+WcLJrfti6sGje1ZJ9UxyzDLF23d/mV+PaMwuC0hL7UVA=="], - - "@rollup/rollup-win32-ia32-msvc": ["@rollup/rollup-win32-ia32-msvc@4.62.3", "", { "os": "win32", "cpu": "ia32" }, "sha512-Rj8Ra4noo+aYy7sKBggCx0407mws34kAb1ySyWuq5DAtFBQdkSwnsjCgPrhPe9cvgBKZIukpE+CVHvORCS93kQ=="], - - "@rollup/rollup-win32-x64-gnu": ["@rollup/rollup-win32-x64-gnu@4.62.3", "", { "os": "win32", "cpu": "x64" }, "sha512-vp7N084ew/odXn2gi/mzm9mUkQu9l6AiN6dt4IeUM2Uvm9o+cVmP+YkqbMOteLbiGgqBBlJZjIMYVCfOOIVbVQ=="], - - "@rollup/rollup-win32-x64-msvc": ["@rollup/rollup-win32-x64-msvc@4.62.3", "", { "os": "win32", "cpu": "x64" }, "sha512-MOG/3gTOn4Fwf574RVOaY61I5o6P90legkFADiTyn1hyjNydT+cerU2rLUwPdZkKKyJ+iT+K9p7WXK4LM1Ka6g=="], - "@rrweb/replay": ["@rrweb/replay@2.1.1", "", { "dependencies": { "@rrweb/types": "^2.1.1", "rrweb": "^2.1.1" } }, "sha512-jq64eyJvrh/ioBS1MpDrs/Lj+7QY2TW4xGxp4g4LJKdltSkqMjMbUmhTDcpzaEDZt66S0gasGcgqCulsksSXnw=="], "@rrweb/types": ["@rrweb/types@2.1.1", "", {}, "sha512-g0I3nCNL1S7slDXwhunxOuOoswkY8WVZJQrPFiwixOc6xoD514d1JLTuyAzCKIox8g/PaLKtV5g31Uk42inQSQ=="], @@ -2455,6 +2418,8 @@ "chardet": ["chardet@2.2.0", "", {}, "sha512-rddelWYNPRrXq6PtNEN2S3f6t9ILzvqaN5pVgi4kqt9jHQaXIial9PznB5iSPVlQSLNaaH22ItWz3EJtQ10+OA=="], + "chdb": ["chdb@3.3.0", "", { "dependencies": { "@clickhouse/client-common": "^1.22.0", "node-addon-api": "^6.1.0", "node-gyp-build": "^4.6.0" }, "optionalDependencies": { "@chdb/lib-darwin-arm64": "26.7.0-stable.1", "@chdb/lib-darwin-x64": "26.7.0-stable.1", "@chdb/lib-linux-arm64-gnu": "26.7.0-stable.1", "@chdb/lib-linux-x64-gnu": "26.7.0-stable.1" }, "peerDependencies": { "@clickhouse/client": ">=1.23.0", "@hypequery/clickhouse": ">=2.1.2", "@mastra/core": ">=0.10", "ai": ">=5", "apache-arrow": ">=14", "zod": ">=3" }, "optionalPeers": ["@clickhouse/client", "@hypequery/clickhouse", "@mastra/core", "ai", "apache-arrow", "zod"], "bin": { "chdb-gen-types": "dist/layer3/codegen/cli.js" } }, "sha512-Vg/quPGJgNo7+I4W21g2aKOrz52VCN0aixZwKXjUrV3EUhpu0/hJDvlFdVOvyahhgH5vPKEH9mNjxM7Qmc0SWQ=="], + "cheerio": ["cheerio@1.2.0", "", { "dependencies": { "cheerio-select": "^2.1.0", "dom-serializer": "^2.0.0", "domhandler": "^5.0.3", "domutils": "^3.2.2", "encoding-sniffer": "^0.2.1", "htmlparser2": "^10.1.0", "parse5": "^7.3.0", "parse5-htmlparser2-tree-adapter": "^7.1.0", "parse5-parser-stream": "^7.1.2", "undici": "^7.19.0", "whatwg-mimetype": "^4.0.0" } }, "sha512-WDrybc/gKFpTYQutKIK6UvfcuxijIZfMfXaYm8NMsPQxSYvf+13fXUJ4rztGGbJcBQ/GF55gvrZ0Bc0bj/mqvg=="], "cheerio-select": ["cheerio-select@2.1.0", "", { "dependencies": { "boolbase": "^1.0.0", "css-select": "^5.1.0", "css-what": "^6.1.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.0.1" } }, "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g=="], @@ -3379,12 +3344,16 @@ "nlcst-to-string": ["nlcst-to-string@4.0.0", "", { "dependencies": { "@types/nlcst": "^2.0.0" } }, "sha512-YKLBCcUYKAg0FNlOBT6aI91qFmSiFKiluk655WzPF+DDMA02qIyy8uiRqI8QXtcFpEvll12LpL5MXqEmAZ+dcA=="], + "node-addon-api": ["node-addon-api@6.1.0", "", {}, "sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA=="], + "node-domexception": ["node-domexception@1.0.0", "", {}, "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ=="], "node-fetch": ["node-fetch@3.3.2", "", { "dependencies": { "data-uri-to-buffer": "^4.0.0", "fetch-blob": "^3.1.4", "formdata-polyfill": "^4.0.10" } }, "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA=="], "node-fetch-native": ["node-fetch-native@1.6.7", "", {}, "sha512-g9yhqoedzIUm0nTnTqAQvueMPVOuIY16bqgAJJC8XOOubYFNwz6IER9qs0Gq2Xd0+CecCKFjtdDTMA4u4xG06Q=="], + "node-gyp-build": ["node-gyp-build@4.8.4", "", { "bin": { "node-gyp-build": "bin.js", "node-gyp-build-optional": "optional.js", "node-gyp-build-test": "build-test.js" } }, "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ=="], + "node-gyp-build-optional-packages": ["node-gyp-build-optional-packages@5.2.2", "", { "dependencies": { "detect-libc": "^2.0.1" }, "bin": { "node-gyp-build-optional-packages": "bin.js", "node-gyp-build-optional-packages-optional": "optional.js", "node-gyp-build-optional-packages-test": "build-test.js" } }, "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw=="], "node-mock-http": ["node-mock-http@1.0.5", "", {}, "sha512-KQyt/wLjG3TAc7DOUhpqWzgd4ERxR80JOlTK5VE5R1S12IaPVN5qkj4klBce9HPG1Njuup4Sb5bljaT34lIyjw=="], @@ -3681,8 +3650,6 @@ "rolldown-plugin-dts": ["rolldown-plugin-dts@0.27.14", "", { "dependencies": { "dts-resolver": "^3.0.0", "get-tsconfig": "5.0.0-beta.5", "obug": "^2.1.4", "yuku-ast": "^0.8.0", "yuku-codegen": "^0.8.0", "yuku-parser": "^0.8.0" }, "peerDependencies": { "@typescript/native-preview": "*", "@volar/typescript": "~2.4.0", "rolldown": "^1.0.0", "typescript": "^5.0.0 || ^6.0.0 || ~7.0.0", "vue-tsc": "~3.2.0 || ~3.3.0" }, "optionalPeers": ["@typescript/native-preview", "@volar/typescript", "typescript", "vue-tsc"] }, "sha512-ZvuDDwoIpRK9RPxDXratCpklFO9QZZWndf/sd0VBFb4LEj0jj07UcHK9OCh7V4XiFz2Z89ziyBC2K6tJiDjrbw=="], - "rollup": ["rollup@4.62.3", "", { "dependencies": { "@types/estree": "1.0.9" }, "optionalDependencies": { "@rollup/rollup-android-arm-eabi": "4.62.3", "@rollup/rollup-android-arm64": "4.62.3", "@rollup/rollup-darwin-arm64": "4.62.3", "@rollup/rollup-darwin-x64": "4.62.3", "@rollup/rollup-freebsd-arm64": "4.62.3", "@rollup/rollup-freebsd-x64": "4.62.3", "@rollup/rollup-linux-arm-gnueabihf": "4.62.3", "@rollup/rollup-linux-arm-musleabihf": "4.62.3", "@rollup/rollup-linux-arm64-gnu": "4.62.3", "@rollup/rollup-linux-arm64-musl": "4.62.3", "@rollup/rollup-linux-loong64-gnu": "4.62.3", "@rollup/rollup-linux-loong64-musl": "4.62.3", "@rollup/rollup-linux-ppc64-gnu": "4.62.3", "@rollup/rollup-linux-ppc64-musl": "4.62.3", "@rollup/rollup-linux-riscv64-gnu": "4.62.3", "@rollup/rollup-linux-riscv64-musl": "4.62.3", "@rollup/rollup-linux-s390x-gnu": "4.62.3", "@rollup/rollup-linux-x64-gnu": "4.62.3", "@rollup/rollup-linux-x64-musl": "4.62.3", "@rollup/rollup-openbsd-x64": "4.62.3", "@rollup/rollup-openharmony-arm64": "4.62.3", "@rollup/rollup-win32-arm64-msvc": "4.62.3", "@rollup/rollup-win32-ia32-msvc": "4.62.3", "@rollup/rollup-win32-x64-gnu": "4.62.3", "@rollup/rollup-win32-x64-msvc": "4.62.3", "fsevents": "~2.3.2" }, "bin": { "rollup": "dist/bin/rollup" } }, "sha512-Gu0c0iH9FzgX1L1t7ByIbbS3Vmdz+6KHm/EsqmmC71gUQ82yvZRkTK6XzrFObSka91WUVdynqp6nsfilzr5k6Q=="], - "rou3": ["rou3@0.6.3", "", {}, "sha512-1HSG1ENTj7Kkm5muMnXuzzfdDOf7CFnbSYFA+H3Fp/rB9lOCxCPgy1jlZxTKyFoC5jJay8Mmc+VbPLYRjzYLrA=="], "roughjs": ["roughjs@4.6.6", "", { "dependencies": { "hachure-fill": "^0.5.2", "path-data-parser": "^0.1.0", "points-on-curve": "^0.2.0", "points-on-path": "^0.2.1" } }, "sha512-ZUz/69+SYpFN/g/lUlo2FXcIjRkSu3nDarreVdGGndHEBJ6cXPdKguS8JGxwj5HA5xIbVKSmLgr5b3AWxtRfvQ=="], @@ -3751,8 +3718,6 @@ "smol-toml": ["smol-toml@1.7.1", "", {}, "sha512-PPlsspAZ4jbMBu5DMFhfUGDQLu/vrL4SyBROVS37x8ynnVmFIs1VPBz1Co8Xks3TvpIaZXmU85y4DrQ+UyVFoQ=="], - "solid-js": ["solid-js@1.6.12", "", { "dependencies": { "csstype": "^3.1.0" } }, "sha512-JFqRobfG3q5r1l4RYVOAukk6+FWtHpXGIjgh/GEsHKweN/kK+iHOtzUALE6+P5t/jIcSNeGiVitX8gmJg+cYvQ=="], - "sort-css-media-queries": ["sort-css-media-queries@3.0.5", "", {}, "sha512-wRgTa9kOgx5nV+lp/uwT0XBlH/WN5dpsOxyIkbtQud65Ie66TYjHceWH/8d1C0siMjcdSjjXg4zV022QmvLsaw=="], "sorted-btree": ["sorted-btree@1.8.1", "", {}, "sha512-395+XIP+wqNn3USkFSrNz7G3Ss/MXlZEqesxvzCRFwL14h6e8LukDHdLBePn5pwbm5OQ9vGu8mDyz2lLDIqamQ=="], diff --git a/docs/local-mode.md b/docs/local-mode.md index bf3515e98..26a22a2bb 100644 --- a/docs/local-mode.md +++ b/docs/local-mode.md @@ -17,8 +17,9 @@ brew install Makisuo/tap/maple ``` Homebrew downloads the matching release bundle, verifies its checksum, installs -`maple` and `libchdb.so` together in the Homebrew Cellar, and links `maple` onto -your PATH. If Homebrew asks you to trust the third-party tap, run +`maple` and its `node_modules/chdb` runtime sidecar together in the Homebrew +Cellar, and links `maple` onto your PATH. If Homebrew asks you to trust the +third-party tap, run `brew trust Makisuo/tap` once and retry the install. The tap lives in `Makisuo/homebrew-tap`, not this repo, so its platform coverage is set there — Intel macOS currently installs via the manual installer below. @@ -34,9 +35,9 @@ curl -fsSL https://maple.dev/cli/install | sh `https://raw.githubusercontent.com/MapleTechLabs/maple/main/scripts/install.sh` works too.) The manual installer detects your OS/arch, downloads the matching bundle from -the latest GitHub release, verifies its checksum, installs the two files into -`~/.maple/bin`, clears the macOS Gatekeeper quarantine, and symlinks `maple` -onto your PATH. +the latest GitHub release, verifies its checksum, installs `maple` and its +`node_modules/chdb` runtime sidecar into `~/.maple/bin`, clears the macOS +Gatekeeper quarantine, and symlinks `maple` onto your PATH. Released targets: macOS (Apple Silicon & Intel) and Linux (x86_64 & arm64). Intel macOS builds on GitHub's `macos-15-intel` runner, which is the scarcest @@ -116,7 +117,8 @@ Manual-installer builds keep themselves current: (CI/pipes), and the `--version`/`--help`/`update` paths. Opt out entirely with `MAPLE_NO_UPDATE_CHECK=1`. - **`maple update`** downloads the latest release bundle, verifies its SHA-256, - and installs it **in place** — an atomic rename over both files, safe even + and installs it **in place** — an atomic rename over the binary and runtime + sidecar, safe even though the running binary is being replaced (the install dir's `cp`-based installer can't overwrite a running executable; the rename swaps the directory entry while the live process keeps its old inode). It then clears the macOS @@ -151,27 +153,28 @@ as the installer. If you migrate from the manual installer to Homebrew, run the manual uninstaller or remove the old PATH symlink so your shell resolves Homebrew's `maple`. -## Architecture: one Bun binary + libchdb +## Architecture: one Bun binary + chdb npm runtime There is a single binary, `maple`, compiled from **`apps/cli`** (package `@maple/cli`, Effect + Bun) with `bun build --compile`. It is both the CLI and -the server, and it talks to the embedded ClickHouse engine **directly via -`bun:ffi`** — no subprocess, no second language at the front: +the server, and it talks to the embedded ClickHouse engine through the +published **`chdb` npm package** — no subprocess, no second language at the +front: | Concern | Where | How | | -------------------- | ------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | CLI commands | `apps/cli/src/commands` | `maple services`, `traces`, `errors`, … run against **either** the local server **or** a remote workspace — every command bottoms out at the shared `WarehouseExecutor`, and only the executor layer swaps per [mode](#local-vs-remote-mode). | | `maple start` server | `apps/cli/src/server/serve.ts` | A `Bun.serve` hosting OTLP/HTTP ingest (`POST /v1/{traces,logs,metrics}`), the query API (`POST /local/query`), and the bundled SPA — all on one port. | -| Embedded ClickHouse | `apps/cli/src/server/chdb.ts` | `dlopen`s `libchdb` via `bun:ffi` (the `chdb_*` accessor C API) and holds a single connection for the process. | +| Embedded ClickHouse | `apps/cli/src/server/chdb.ts` | Opens one `chdb` npm `Session` against the Maple data directory and holds a single connection for the process. | | OTLP → rows | `apps/cli/src/server/otlp/` | Decodes OTLP protobuf/JSON (protobufjs) and encodes each signal to per-table NDJSON, matching the generated `local-inserts.json` schema exactly. Ported from the production Rust encoders so row shapes can't diverge. | | UI (SPA) | `apps/local-ui` (Vite + React) | Hooks compile queries with `CH.compile(...)` and POST to `/local/query`. The same build is deployed to `local.maple.dev` (the default) **and** inlined into the binary as the `--offline` fallback (see [release bundle](#release-bundle)); it picks its query base URL from `window.location` at runtime (see [Where the UI comes from](#where-the-ui-comes-from)). | chDB allows exactly one connection per process and isn't safe to call concurrently — so the long-lived `maple start` process owns the connection, and short-lived query commands (`maple traces`, …) reach it over HTTP via -[`executeLocalQuery`](../packages/query-engine/src/local.ts). `bun:ffi` calls are -synchronous and serialize naturally on the single JS thread, which preserves -chDB's single-writer requirement. +[`executeLocalQuery`](../packages/query-engine/src/local.ts). The local server +keeps chDB access on the single JS thread, preserving chDB's single-writer +requirement. ### Store lifecycle & recovery @@ -285,8 +288,8 @@ The Linux native probe `apps/cli/test/native-local-store-migration.sh` uses a native chDB setup helper to create a stopped historical raw-table fixture, applies the legacy marker, runs the public migration command, checks rebuilt service-namespace and database aggregates, and reopens the promoted store in -a fresh process. It reports `SKIP` when no native `libchdb` is available (for -example, on a development machine without the platform bundle); the Linux CI +a fresh process. It reports `SKIP` when no native `chdb` npm runtime is +available (for example, on a development machine without the platform bundle); the Linux CI bundle runs it alongside the checkpoint smoke test. The fixture covers the authoritative v0 raw tables; retained derived objects remain rollback-only rather than being treated as migrated history. @@ -359,7 +362,7 @@ No Rust toolchain needed. Run the server and the SPA dev server in two terminals ```bash # Terminal 1 — the server (OTLP ingest + query API + chDB) on :4318. -# Needs libchdb: set MAPLE_LIBCHDB, or keep libchdb.so in ~/.maple/bin. +# Needs the @maple/cli workspace dependencies installed. bun run apps/cli/src/bin.ts start # Terminal 2 — the Vite SPA dev server on :4319, proxying /local → :4318 @@ -382,10 +385,10 @@ mapping wildcard binds to matching loopback. `MAPLE_LOCAL_URL` remains the explicit override and is required when the server was started with a one-off `--host` or non-default `--port` that later CLI processes cannot infer. -> **libchdb in dev.** `chdb.ts` resolves `libchdb` from, in order: `MAPLE_LIBCHDB`, -> a sibling of the executable, then `~/.maple/bin/libchdb.{so,dylib}`. Running from -> source uses the Bun executable's directory (no sibling libchdb), so either set -> `MAPLE_LIBCHDB` or drop a `libchdb.so` in `~/.maple/bin`. +> **chDB runtime in dev.** Source runs resolve the workspace `@maple/cli` +> dependency on `chdb`. Compiled bundles resolve `node_modules/chdb` beside the +> `maple` binary; set `MAPLE_CHDB_NODE_MODULES` only when testing a moved or +> non-standard runtime sidecar. ## Local vs remote mode @@ -468,22 +471,22 @@ against the emitting service's logs. ## Release bundle -`scripts/build-local-binary.sh` produces a relocatable **2-file bundle** (also built +`scripts/build-local-binary.sh` produces a relocatable bundle (also built per-platform by `.github/workflows/local-binary-release.yml`): ``` -maple # single Bun-compiled binary: CLI + ingest/query server + embedded SPA -libchdb.so # the chDB engine (~320 MB), downloaded from chdb-io/chdb-core releases +maple # single Bun-compiled binary: CLI + ingest/query server + embedded SPA +node_modules/ # chdb npm package plus the installed @chdb/lib-* platform package ``` The build (1) builds the SPA, (2) inlines `apps/local-ui/dist` into `apps/cli/src/server/ui-embed.gen.ts` so `bun build --compile` bakes it into the binary as the `--offline` fallback (the default UI is served from -`local.maple.dev`), (3) compiles `apps/cli`, and (4) downloads the matching `libchdb` beside -the binary. At runtime `maple` `dlopen`s the sibling `libchdb` (resolved relative -to its own path), so keep both files in the same directory — no `LD_LIBRARY_PATH` -or rpath tricks. +`local.maple.dev`), (3) compiles `apps/cli`, and (4) copies the npm `chdb` +runtime sidecar beside the binary. At runtime `maple` resolves +`node_modules/chdb` relative to its own path, so keep the sidecar next to the +binary — no `LD_LIBRARY_PATH` or rpath tricks. ```bash -scripts/build-local-binary.sh # full 2-file bundle into ./dist +scripts/build-local-binary.sh # full bundle into ./dist ``` diff --git a/scripts/build-local-binary.sh b/scripts/build-local-binary.sh index 1556a0e7b..8932ade98 100755 --- a/scripts/build-local-binary.sh +++ b/scripts/build-local-binary.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Build the distributable `maple` local binary — a single Bun-compiled -# executable plus libchdb. No Rust/cargo involved. +# executable plus the npm chdb runtime sidecar. No Rust/cargo involved. # # Pipeline: # 1. Build the workspace packages consumed through gitignored `dist/` paths. @@ -9,14 +9,17 @@ # `bun build --compile` bakes the SPA into the binary. # 4. Compile apps/cli (the CLI + the OTLP-ingest/query server) into a single # executable with `bun build --compile`. The schema artifacts and SPA are -# embedded; the OTLP encoders run in-process; chDB is reached via bun:ffi. -# 5. Download libchdb (v26.1.0, matching what we test against) for the host -# platform and place it beside the binary. At runtime `maple` dlopens the -# sibling libchdb (resolved relative to its own path) — no rpath tricks. +# embedded; the OTLP encoders run in-process; chDB is reached via the +# published `chdb` npm package. +# 5. Copy the npm chdb runtime (`chdb` plus the installed platform package) +# beside the binary under node_modules. At runtime compiled Maple resolves +# `chdb` from that sidecar, while source/dev runs resolve workspace +# node_modules normally. # 6. Restore the committed ui-embed.gen.ts stub so the tree stays clean. # -# The distributable is a 2-file bundle: `maple` + `libchdb.so`. Keep them in the -# same directory. +# The distributable is a bundle directory: `maple` + `node_modules/chdb` and its +# matching `node_modules/@chdb/lib-*` platform package. Keep the sidecar next to +# the binary. # # Usage: # scripts/build-local-binary.sh # release build into ./dist @@ -25,12 +28,43 @@ set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" OUT_DIR="${OUT_DIR:-$REPO_ROOT/dist}" -LIBCHDB_VERSION="${LIBCHDB_VERSION:-v26.1.0}" UI_EMBED="$REPO_ROOT/apps/cli/src/server/ui-embed.gen.ts" +CLI_NODE_MODULES="$REPO_ROOT/apps/cli/node_modules" +ROOT_NODE_MODULES="$REPO_ROOT/node_modules" # Version baked into the binary via `bun build --define`. The release workflow # passes the tag; a local build defaults to `git describe` (or "dev"). MAPLE_BUILD_VERSION="${MAPLE_BUILD_VERSION:-$(git -C "$REPO_ROOT" describe --tags --always 2>/dev/null || echo dev)}" +CHDB_VERSION="${CHDB_VERSION:-$(bun "$REPO_ROOT/scripts/resolve-chdb-version.ts")}" + +copy_runtime_pkg() { + local pkg="$1" + local src="" + for candidate in "$CLI_NODE_MODULES/$pkg" "$ROOT_NODE_MODULES/$pkg" "$ROOT_NODE_MODULES/.bun/node_modules/$pkg"; do + if [ -d "$candidate" ]; then + src="$candidate" + break + fi + done + local dst="$OUT_DIR/node_modules/$pkg" + if [ -z "$src" ]; then + echo "ERROR: runtime package not installed: $pkg" >&2 + echo " run bun install --frozen-lockfile before building" >&2 + exit 1 + fi + mkdir -p "$(dirname "$dst")" + rm -rf "$dst" + cp -RL "$src" "$dst" +} + +copy_runtime_pkg_from() { + local pkg="$1" + local src="$2" + local dst="$OUT_DIR/node_modules/$pkg" + mkdir -p "$(dirname "$dst")" + rm -rf "$dst" + cp -RL "$src" "$dst" +} mkdir -p "$OUT_DIR" @@ -48,53 +82,29 @@ restore_stub() { git -C "$REPO_ROOT" checkout -- "$UI_EMBED" 2>/dev/null || true trap restore_stub EXIT bun run "$REPO_ROOT/scripts/gen-ui-embed.ts" -echo "==> Compiling maple binary (bun build --compile) — version $MAPLE_BUILD_VERSION" +echo "==> Compiling maple binary (bun build --compile) — version $MAPLE_BUILD_VERSION, chDB $CHDB_VERSION" ( cd "$REPO_ROOT" && bun build apps/cli/src/bin.ts --compile \ --define "__MAPLE_VERSION__=\"$MAPLE_BUILD_VERSION\"" \ - --define "__CHDB_VERSION__=\"$LIBCHDB_VERSION\"" \ + --define "__CHDB_VERSION__=\"$CHDB_VERSION\"" \ --outfile "$OUT_DIR/maple" ) -echo "==> Downloading libchdb $LIBCHDB_VERSION for this platform" -case "$(uname -s)-$(uname -m)" in - Linux-x86_64) ASSET="linux-x86_64-libchdb.tar.gz" ;; - Linux-aarch64) ASSET="linux-aarch64-libchdb.tar.gz" ;; - Darwin-x86_64) ASSET="macos-x86_64-libchdb.tar.gz" ;; - Darwin-arm64) ASSET="macos-arm64-libchdb.tar.gz" ;; - *) echo "ERROR: unsupported platform $(uname -s)-$(uname -m)" >&2; exit 1 ;; -esac -# Expected sha256 per platform asset, pinned to LIBCHDB_VERSION: release assets -# are mutable even for a fixed tag, so this is verified before extraction. -# Bumping LIBCHDB_VERSION means re-hashing each asset here. A `case` rather than -# an associative array because macOS still ships bash 3.2. -case "$ASSET" in - linux-x86_64-libchdb.tar.gz) EXPECTED_SHA256="3f192cb85cbdb6153622db8ad6f6a25d7163bbda57f283e5ba8fd22cadd0da81" ;; - linux-aarch64-libchdb.tar.gz) EXPECTED_SHA256="08ee80488822dfa04776841bb5a4b7ddce35e161ef579b5615b80a88b177428d" ;; - macos-x86_64-libchdb.tar.gz) EXPECTED_SHA256="425db8a197288e1826b60958b3e233d0a7a5b4b46b8065c8bb7489366dbaf47f" ;; - macos-arm64-libchdb.tar.gz) EXPECTED_SHA256="78cfd42203cd33cb0304952f4a6c39921fd9a583c614c2eef96ef6060e9677eb" ;; - *) echo "ERROR: no pinned sha256 for $ASSET (LIBCHDB_VERSION=$LIBCHDB_VERSION)" >&2; exit 1 ;; -esac - -URL="https://github.com/chdb-io/chdb-core/releases/download/$LIBCHDB_VERSION/$ASSET" -TMP="$(mktemp -d)" -curl -fsSL "$URL" -o "$TMP/libchdb.tar.gz" - -# Verify BEFORE extracting: release assets are mutable even for a fixed tag, -# so this authenticates the download rather than whatever happened to arrive. -ACTUAL_SHA256="$(if command -v sha256sum >/dev/null 2>&1; then sha256sum "$TMP/libchdb.tar.gz"; else shasum -a 256 "$TMP/libchdb.tar.gz"; fi | awk '{print $1}')" -if [ "$ACTUAL_SHA256" != "$EXPECTED_SHA256" ]; then - echo "ERROR: libchdb checksum mismatch for $ASSET" >&2 - echo " expected: $EXPECTED_SHA256" >&2 - echo " actual: $ACTUAL_SHA256" >&2 - rm -rf "$TMP" +echo "==> Copying npm chdb runtime sidecar" +rm -rf "$OUT_DIR/node_modules" +copy_runtime_pkg chdb +copy_runtime_pkg node-addon-api +copy_runtime_pkg node-gyp-build +copy_runtime_pkg @clickhouse/client-common +found_platform_pkg=0 +for pkg in "$ROOT_NODE_MODULES"/.bun/node_modules/@chdb/lib-* "$ROOT_NODE_MODULES"/.bun/@chdb+lib-*/node_modules/@chdb/lib-*; do + [ -d "$pkg" ] || continue + copy_runtime_pkg_from "@chdb/$(basename "$pkg")" "$pkg" + found_platform_pkg=1 +done +if [ "$found_platform_pkg" != 1 ]; then + echo "ERROR: no @chdb/lib-* platform package installed" >&2 exit 1 fi -tar -xzf "$TMP/libchdb.tar.gz" -C "$TMP" -LIB="$(find "$TMP" -name 'libchdb.so' -o -name 'libchdb.dylib' | head -1)" -[ -n "$LIB" ] || { echo "ERROR: libchdb not found in $ASSET" >&2; exit 1; } -cp "$LIB" "$OUT_DIR/libchdb.so" -rm -rf "$TMP" - echo "==> Done. Bundle in $OUT_DIR:" echo " maple ($(du -h "$OUT_DIR/maple" | cut -f1))" -echo " libchdb.so ($(du -h "$OUT_DIR/libchdb.so" | cut -f1))" +echo " node_modules ($(du -sh "$OUT_DIR/node_modules" | cut -f1))" diff --git a/scripts/install.sh b/scripts/install.sh index 1b05bc13a..b797bfe53 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -8,18 +8,17 @@ # works too.) # # Downloads the platform bundle from the latest GitHub release, verifies its -# checksum, and installs the 2-file bundle (`maple` + `libchdb.so`) into +# checksum, and installs `maple` plus its npm `chdb` runtime sidecar into # ~/.maple/bin, then puts `maple` on your PATH. # # Already installed? `maple update` upgrades in place (same artifact, atomic # swap) — re-running this installer is only needed for a first install. # -# The two files MUST stay in the same directory: `maple` (a single Bun-compiled +# The sidecar MUST stay next to the binary: `maple` (a single Bun-compiled # binary that does everything — CLI, OTLP-ingest/query server, and UI host) -# dlopens `libchdb.so` via bun:ffi, resolving it relative to its own executable -# path. We install both into ~/.maple/bin and symlink only `maple` onto PATH; -# the binary also falls back to ~/.maple/bin when resolving libchdb, so the -# symlink works regardless of how the path resolves. +# loads `node_modules/chdb`, whose platform package carries the native addon. +# We install both into ~/.maple/bin and symlink only `maple` onto PATH; resolving +# the symlink still lands back in the install dir, so the sidecar is found. # # Env overrides: # MAPLE_VERSION release tag to install (default: latest) @@ -139,17 +138,20 @@ else say "Checksum verified." fi -# --- install the 2-file bundle ------------------------------------------------ +# --- install the bundle ------------------------------------------------------- tar -xzf "$tmp/bundle.tar.gz" -C "$tmp" [ -d "$tmp/$name" ] || die "unexpected archive layout (no $name/ directory)" +[ -d "$tmp/$name/node_modules/chdb" ] || die "unexpected archive layout (no node_modules/chdb runtime)" mkdir -p "$INSTALL_DIR" -cp "$tmp/$name/maple" "$tmp/$name/libchdb.so" "$INSTALL_DIR/" +rm -rf "$INSTALL_DIR/node_modules" +cp "$tmp/$name/maple" "$INSTALL_DIR/" +cp -R "$tmp/$name/node_modules" "$INSTALL_DIR/node_modules" chmod +x "$INSTALL_DIR/maple" # macOS: clear the Gatekeeper quarantine flag set on downloaded files. if [ "$os" = "Darwin" ] && command -v xattr >/dev/null 2>&1; then - xattr -dr com.apple.quarantine "$INSTALL_DIR/maple" "$INSTALL_DIR/libchdb.so" 2>/dev/null || true + xattr -dr com.apple.quarantine "$INSTALL_DIR/maple" "$INSTALL_DIR/node_modules" 2>/dev/null || true fi # --- put `maple` on PATH ------------------------------------------------------ @@ -165,7 +167,7 @@ mkdir -p "$link_dir" ln -sf "$INSTALL_DIR/maple" "$link_dir/maple" say "" -say "✓ Installed to $INSTALL_DIR (maple + libchdb.so)" +say "✓ Installed to $INSTALL_DIR (maple + node_modules/chdb)" say "✓ Linked $link_dir/maple" case ":$PATH:" in *":$link_dir:"*) ;; diff --git a/scripts/lint-clickhouse-schema.ts b/scripts/lint-clickhouse-schema.ts index 68ce11f6e..236da74c6 100644 --- a/scripts/lint-clickhouse-schema.ts +++ b/scripts/lint-clickhouse-schema.ts @@ -50,7 +50,7 @@ for (const mv of materializedViews) { } // Rule: no ICU-gated functions. `lowerUTF8`/`upperUTF8` are only available in -// ClickHouse builds with ICU (24.8+ gates them), and the libchdb embedded by +// ClickHouse builds with ICU (24.8+ gates them), and the chDB engine embedded by // local mode is built without it — a schema using them bootstraps fine on // Tinybird but crashes every fresh `maple start` with UNKNOWN_FUNCTION. // For ASCII inputs `lower`/`upper` are byte-identical; for genuinely non-ASCII @@ -61,7 +61,7 @@ for (const stmt of latestSnapshotStatements) { if (match) { violations.push({ kind: "icu_gated_function", - message: `Statement uses ${match[1]}(), which is missing from local mode's non-ICU libchdb. Use ${match[1] === "upperUTF8" ? "upper" : "lower"}() (identical for ASCII) or an ICU-free formulation. Statement: ${stmt.split("\n")[0]?.slice(0, 120)}`, + message: `Statement uses ${match[1]}(), which is missing from local mode's non-ICU chDB engine. Use ${match[1] === "upperUTF8" ? "upper" : "lower"}() (identical for ASCII) or an ICU-free formulation. Statement: ${stmt.split("\n")[0]?.slice(0, 120)}`, }) } } diff --git a/scripts/resolve-chdb-version.ts b/scripts/resolve-chdb-version.ts new file mode 100644 index 000000000..d8f283051 --- /dev/null +++ b/scripts/resolve-chdb-version.ts @@ -0,0 +1,35 @@ +import { createRequire } from "node:module" +import { mkdtempSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { dirname, join } from "node:path" +import { fileURLToPath } from "node:url" + +interface ChdbSession { + query(sql: string, format?: string): string + close(): void +} + +interface ChdbPackage { + Session: new (path?: string) => ChdbSession +} + +const unquoteCsvScalar = (value: string): string => { + const trimmed = value.trim() + if (trimmed.startsWith('"') && trimmed.endsWith('"')) { + return trimmed.slice(1, -1).replace(/""/g, '"') + } + return trimmed +} + +const repoRoot = dirname(dirname(fileURLToPath(import.meta.url))) +const requireFromCliWorkspace = createRequire(join(repoRoot, "apps", "cli", "package.json")) +const { Session } = requireFromCliWorkspace("chdb") as ChdbPackage +const root = mkdtempSync(join(tmpdir(), "maple-chdb-version-")) +const session = new Session(join(root, "data")) + +try { + process.stdout.write(`${unquoteCsvScalar(session.query("SELECT version()", "CSV"))}\n`) +} finally { + session.close() + rmSync(root, { recursive: true, force: true }) +} diff --git a/scripts/uninstall.sh b/scripts/uninstall.sh index 2c6c9e3c8..b451b7495 100755 --- a/scripts/uninstall.sh +++ b/scripts/uninstall.sh @@ -3,8 +3,8 @@ # # curl -fsSL https://maple.dev/cli/uninstall | sh # -# Removes the `maple` PATH symlink and the `~/.maple/bin` bundle (maple + -# libchdb.so). Your data dir (`~/.maple/data`) and config (`~/.maple/config.json`, +# Removes the `maple` PATH symlink and the `~/.maple/bin` bundle (maple plus +# its npm chdb runtime sidecar). Your data dir (`~/.maple/data`) and config (`~/.maple/config.json`, # holding any remote token) are KEPT unless you opt in — the data dir prompts # (or set MAPLE_REMOVE_DATA=1 for non-interactive removal). #