From b29f1ddb005608f559bc6c4babfc39c3ca80395e Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Thu, 9 Jul 2026 20:25:24 -0700 Subject: [PATCH 1/3] fix(mcp): map caller-fixable identifier errors to INVALID_ARGUMENT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An invalid KV `store`/`key` (a disallowed byte or over the 512-byte limit) surfaced from the hyperdb-api layer as `Error::InvalidName` and fell through the catch-all arm to `INTERNAL_ERROR` — telling an LLM caller "server bug, give up" for something it supplied and can fix. Map `InvalidName` and `InvalidTableDefinition` to `INVALID_ARGUMENT`, which carries a self-correction suggestion; the human-readable message naming the offending byte or length is unchanged. `InvalidOperation` is deliberately left out: it is hyperdb-api caller-API misuse where the caller is this MCP's own Rust code, not the LLM, so it correctly stays `INTERNAL_ERROR`. A regression test locks in that distinction. Caught during the KV MCP smoke run. --- hyperdb-mcp/CHANGELOG.md | 9 +++++ hyperdb-mcp/src/error.rs | 16 +++++++++ hyperdb-mcp/tests/error_tests.rs | 62 ++++++++++++++++++++++++++++++++ 3 files changed, 87 insertions(+) diff --git a/hyperdb-mcp/CHANGELOG.md b/hyperdb-mcp/CHANGELOG.md index 826729d..25a689d 100644 --- a/hyperdb-mcp/CHANGELOG.md +++ b/hyperdb-mcp/CHANGELOG.md @@ -27,6 +27,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Fixed +- **Caller-fixable argument errors now return `INVALID_ARGUMENT`, not + `INTERNAL_ERROR`.** An invalid identifier from the `hyperdb-api` layer — + such as a KV `store`/`key` containing a disallowed byte or exceeding the + 512-byte limit — previously fell through to the catch-all `INTERNAL_ERROR` + code, mislabeling a validation failure the caller can fix as a server-side + bug. These (`InvalidName`, `InvalidTableDefinition`) now map to + `INVALID_ARGUMENT`, which carries a self-correction suggestion. The + human-readable message (naming the offending byte or the length) is + unchanged. - **TCP keepalive on the `hyperd` connection.** Connections to `hyperd` now enable TCP keepalive (60s idle, 10s interval, ~90s to declare a dead peer) instead of relying on the OS default 2-hour idle timeout. Without it, a diff --git a/hyperdb-mcp/src/error.rs b/hyperdb-mcp/src/error.rs index d71fd57..e483b65 100644 --- a/hyperdb-mcp/src/error.rs +++ b/hyperdb-mcp/src/error.rs @@ -207,6 +207,22 @@ impl From for McpError { // Configuration errors are caller-visible setup mistakes. hyperdb_api::Error::Config(_) => McpError::new(ErrorCode::InvalidArgument, msg), + // Caller-fixable argument errors: an invalid identifier (e.g. a + // KV store/key with a disallowed byte or over the length limit) + // or a malformed table definition (zero columns, conflicting + // attributes). These are triggered by the tool arguments an LLM + // supplies, and the message names what's wrong, so they are + // InvalidArgument, not an opaque InternalError. + // + // NOTE: `InvalidOperation` is deliberately NOT included — it is + // hyperdb-api "caller-API misuse" where the *caller* is this + // MCP's own Rust code (e.g. mixing inserter modes), not the LLM. + // If it ever fired it would signal an MCP bug the model can't fix + // by changing arguments, so it correctly stays InternalError. + hyperdb_api::Error::InvalidName(_) | hyperdb_api::Error::InvalidTableDefinition(_) => { + McpError::new(ErrorCode::InvalidArgument, msg) + } + // Connection / Closed / Timeout — surface as ConnectionLost // so the engine recycles. is_connection_lost above already // catches most of these via message; this is a fallback. diff --git a/hyperdb-mcp/tests/error_tests.rs b/hyperdb-mcp/tests/error_tests.rs index 7f0d5f7..40b8f63 100644 --- a/hyperdb-mcp/tests/error_tests.rs +++ b/hyperdb-mcp/tests/error_tests.rs @@ -176,3 +176,65 @@ fn connection_lost_suggestion_mentions_both_triggers() { "suggestion should tell the caller the server will recover; got: {suggestion}", ); } + +/// A caller-supplied bad identifier — the shape the KV tools hit when a +/// `store`/`key` has a disallowed byte or exceeds the 512-byte limit — +/// arrives as `hyperdb_api::Error::InvalidName`. It must map to +/// `InvalidArgument`, not `InternalError`: the caller supplied something +/// wrong and can fix it, and the message already names the offending byte +/// or the length. Before this mapping the variant fell through to the +/// catch-all `InternalError` arm, mislabeling a validation failure as a +/// server-side bug (caught during the KV MCP smoke run). +#[test] +fn maps_invalid_name_to_invalid_argument() { + for upstream in [ + // The exact messages `validate_kv_name` produces for the KV tools. + hyperdb_api::Error::invalid_name( + "KV key contains an invalid byte 0x20; allowed: A-Z a-z 0-9 _ . -", + ), + hyperdb_api::Error::invalid_name("KV store name must not be empty"), + hyperdb_api::Error::invalid_name("KV key exceeds 512-byte limit (630 bytes)"), + ] { + let original = upstream.to_string(); + let mcp: McpError = upstream.into(); + assert_eq!( + mcp.code, + ErrorCode::InvalidArgument, + "invalid name must be InvalidArgument, not InternalError: {original}", + ); + // The message must survive so the caller learns what to fix. + assert!( + mcp.message.contains("invalid name"), + "message should carry the validation detail; got: {}", + mcp.message, + ); + assert!( + mcp.suggestion.is_some(), + "InvalidArgument carries a self-correction suggestion", + ); + } +} + +/// `InvalidTableDefinition` shares the `InvalidName` reasoning — a +/// malformed table definition is something the caller (the LLM's tool +/// arguments) supplied and can correct — so it maps to `InvalidArgument`, +/// not the opaque `InternalError` the catch-all arm used to assign. +#[test] +fn maps_invalid_table_definition_to_invalid_argument() { + let def: McpError = + hyperdb_api::Error::invalid_table_definition("table must have at least one column").into(); + assert_eq!(def.code, ErrorCode::InvalidArgument); +} + +/// `InvalidOperation` is deliberately NOT mapped to `InvalidArgument`. It +/// is hyperdb-api "caller-API misuse" where the caller is this MCP's own +/// Rust code (e.g. an inserter used out of sequence), not the LLM — an +/// occurrence signals an MCP bug the model cannot fix by changing tool +/// arguments, so it stays `InternalError`. This guards against a naive +/// "group it with the other caller-fixable variants" regression that +/// would hand the model a misleading "check your arguments" suggestion. +#[test] +fn keeps_invalid_operation_as_internal_error() { + let op: McpError = hyperdb_api::Error::invalid_operation("inserter already finalized").into(); + assert_eq!(op.code, ErrorCode::InternalError); +} From 5e7466d31db161d8049925f840edc6060f0a1431 Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Thu, 9 Jul 2026 20:25:34 -0700 Subject: [PATCH 2/3] docs(mcp): add on-demand KV smoke-test guide MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Capture the manual, tool-driven smoke tests that exercise the eight `kv_*` tools against a live MCP server: CRUD/upsert, listing and discovery, value fidelity, destructive semantics, input validation (now INVALID_ARGUMENT), read-only mode, database routing/isolation, the LEFT JOIN enrichment pattern, the hidden backing table, and single-process atomicity. Includes a safety section: the persistent database may hold real data, so scratch names are `smoke_`-prefixed and every run ends by verifying the persistent DB is untouched. The existence-check safety rule is scoped to the target database — a bare `describe` inspects only the ephemeral primary and `status` never lists table names, so neither alone protects a persistent write. Cross-linked from DEVELOPMENT.md's Test section. --- hyperdb-mcp/DEVELOPMENT.md | 4 + hyperdb-mcp/SMOKE_TESTS.md | 356 +++++++++++++++++++++++++++++++++++++ 2 files changed, 360 insertions(+) create mode 100644 hyperdb-mcp/SMOKE_TESTS.md diff --git a/hyperdb-mcp/DEVELOPMENT.md b/hyperdb-mcp/DEVELOPMENT.md index c8649c7..9033b65 100644 --- a/hyperdb-mcp/DEVELOPMENT.md +++ b/hyperdb-mcp/DEVELOPMENT.md @@ -128,6 +128,10 @@ cargo test -p hyperdb-mcp Most tests require a running `hyperd`. Set `HYPERD_PATH` before running. +For manual, tool-driven checks against a **live** MCP server (useful after a +build or before a release — they exercise real session state the unit tests +don't), see [SMOKE_TESTS.md](SMOKE_TESTS.md). + ### Lint ```bash diff --git a/hyperdb-mcp/SMOKE_TESTS.md b/hyperdb-mcp/SMOKE_TESTS.md new file mode 100644 index 0000000..57b8aa3 --- /dev/null +++ b/hyperdb-mcp/SMOKE_TESTS.md @@ -0,0 +1,356 @@ +# hyperdb-mcp — On-Demand Smoke Tests + +Manual, tool-driven smoke tests for the `hyperdb-mcp` server. Run these +against a **live, running MCP** (e.g. from an LLM client that has the +`hyperdb` tools connected) to confirm end-to-end behavior after a build, +a config change, or before shipping a release. + +These complement — they do **not** replace — the automated suites: + +| Layer | Where | Runs in CI | +|---|---|---| +| End-to-end tool dispatch | [`tests/kv_tools_tests.rs`](tests/kv_tools_tests.rs), [`tests/end_to_end_mcp_tests.rs`](tests/end_to_end_mcp_tests.rs) | yes | +| Error-code mapping | [`tests/error_tests.rs`](tests/error_tests.rs) | yes | +| Tool-name coverage in the LLM README | [`tests/readme_tests.rs`](tests/readme_tests.rs) | yes | + +The automated tests prove correctness in isolation with a fresh temp +server. The smoke tests here prove the **wired-up, running** server behaves +— useful because a live server carries real state (a populated persistent +database, an attached alias, read-only mode, a shared daemon) that the unit +tests deliberately don't. + +--- + +## Safety first — the persistent database is real + +The MCP has two databases per session: + +- the **ephemeral** primary (the default target; a fresh temp `.hyper` that + is deleted on server restart), and +- the **persistent** database (`database: "persistent"` / `persist: true`) + which is the user's durable workspace and **may already hold real data**. + +**Rules for smoke testing:** + +1. **Default to the ephemeral store.** Omit `database` on every call unless + you are explicitly testing routing. Ephemeral writes cost nothing and + vanish on restart. +2. **Never create, drop, or overwrite a table without checking first — + scoped to the database you're about to write to.** A real `products` + table with a thousand rows can already be sitting in the persistent DB. + Before any `CREATE`/`DROP`, confirm the name is free *in that database*: + run `describe table= database=persistent` (or a + `SELECT COUNT(*) FROM ` via `query database=persistent`) when the + target is persistent — a bare `describe` inspects only the **ephemeral** + primary and would miss a persistent collision, and `status` never lists + table *names* (only aggregate counts), so neither alone protects you. + Always use a `smoke_`-prefixed name for any scratch table you create. +3. **Clean up persistent scratch immediately.** If a test writes to + `persistent` (routing/isolation checks), `kv_clear` those stores and drop + any scratch tables the moment the assertion is made. +4. **Prefer `--ephemeral-only`** when you can start the server yourself — it + skips the persistent attachment entirely, so there is nothing real to + touch. + +Every smoke run should end with the persistent database in exactly the state +it started in. The final section is a verification checklist for that. + +--- + +## Preconditions + +- `hyperd` available (`HYPERD_PATH` set, or on `PATH`). +- The `hyperdb` MCP tools connected and responding. +- Confirm the server is up and note its mode before you start: + +``` +status +``` + +Expected: `{"hyperd_running": true, ..., "read_only": false, "engine": {"mode": "daemon"|"local", ...}}`. +Note `read_only` — if `true`, the four KV **mutators** (`kv_set`, +`kv_delete`, `kv_pop`, `kv_clear`) are expected to be **rejected** (see +§7); the four readers still work. + +Throughout, `→` shows the expected JSON the tool returns. Store/key names +below all begin with `smoke` so they're easy to spot and purge. + +--- + +## 1. Server + KV surface present + +The server should expose 8 `kv_*` tools and the `hyper://schema/kv` +resource. + +- `kv_*` tools: `kv_set`, `kv_get`, `kv_delete`, `kv_list`, + `kv_list_stores`, `kv_size`, `kv_pop`, `kv_clear`. +- Reading `hyper://schema/kv` returns text mentioning `_hyperdb_kv_store` + and a `LEFT JOIN` template. + +--- + +## 2. Create / read / overwrite (upsert) + +``` +kv_set store=smoke key=greeting value="hello world" → {"stored": true, "store": "smoke", "key": "greeting"} +kv_get store=smoke key=greeting → {"found": true, "value": "hello world"} +kv_get store=smoke key=does_not_exist → {"found": false, "value": null} +``` + +A miss is **not** an error — `found: false` with a `null` value. + +**Overwrite must not create a duplicate row** (the backing table is +indexless; `kv_set` is an app-side upsert): + +``` +kv_size store=smoke → {"store": "smoke", "size": 1} +kv_set store=smoke key=greeting value="HELLO AGAIN" → {"stored": true, ...} +kv_size store=smoke → {"store": "smoke", "size": 1} # still 1, not 2 +kv_get store=smoke key=greeting → {"found": true, "value": "HELLO AGAIN"} +``` + +--- + +## 3. Listing, size, store discovery + +Seed a few keys, then list: + +``` +kv_set store=smoke key=alpha value=1 +kv_set store=smoke key=bravo value=2 +kv_set store=smoke key=charlie value=3 + +kv_list store=smoke → {"store": "smoke", "count": 4, "keys": ["alpha","bravo","charlie","greeting"]} # sorted ascending +kv_size store=smoke → {"store": "smoke", "size": 4} +kv_list_stores → {"count": 1, "stores": ["smoke"]} +``` + +`kv_list` keys are always sorted ascending. `kv_list_stores` reflects only +stores that currently hold rows (there is no separate store registry — an +emptied store disappears from the list; see §5). + +--- + +## 4. Value fidelity — JSON, empty, large + +``` +kv_set store=smoke key=config value='{"retries": 3, "nested": {"flag": true}}' +kv_get store=smoke key=config → {"found": true, "value": "{\"retries\": 3, \"nested\": {\"flag\": true}}"} # byte-for-byte + +kv_set store=smoke key=empty_val value="" +kv_get store=smoke key=empty_val → {"found": true, "value": ""} # empty string, NOT a miss + +kv_set store=smoke key=big_blob value="" +kv_get store=smoke key=big_blob → {"found": true, "value": ""} +``` + +The empty-string case is the important one: `{"found": true, "value": ""}` +must stay distinct from a miss `{"found": false, "value": null}`. + +--- + +## 5. Destructive semantics — delete, pop, clear + +**Delete is idempotent and reports whether the key existed:** + +``` +kv_delete store=smoke key=greeting → {"deleted": true, ...} # existed +kv_delete store=smoke key=greeting → {"deleted": false, ...} # already gone — no error +kv_delete store=smoke key=never_existed → {"deleted": false, ...} +``` + +**`kv_pop` destructively removes the lowest-keyed entry** (a work-queue +drain in ascending key order): + +``` +# with keys [alpha, bravo, charlie, config, empty_val, big_blob] present +kv_pop store=smoke → {"found": true, "key": "alpha", "value": "1"} +kv_pop store=smoke → {"found": true, "key": "big_blob", "value": "..."} # 'b' < 'c' +kv_pop store=smoke → {"found": true, "key": "bravo", "value": "2"} +``` + +**`kv_clear` empties the store and returns the count removed:** + +``` +kv_size store=smoke → {"store": "smoke", "size": N} +kv_clear store=smoke → {"store": "smoke", "removed": N} +kv_size store=smoke → {"store": "smoke", "size": 0} +``` + +**Empty-store edge cases:** + +``` +kv_pop store=smoke → {"found": false} # nothing to pop +kv_clear store=smoke → {"store": "smoke", "removed": 0} # idempotent +kv_list_stores → {"count": 0, "stores": []} # emptied store drops out +``` + +--- + +## 6. Input validation + +`store` and `key` must be ASCII `[A-Za-z0-9_.-]`, non-empty, ≤ 512 bytes. +Violations are rejected as **`INVALID_ARGUMENT`** (not `INTERNAL_ERROR`) +with a message that names the offending byte or the actual length: + +``` +kv_set store=smoke key="has a space" value=x + → error INVALID_ARGUMENT: "invalid name: KV key contains an invalid byte 0x20; allowed: A-Z a-z 0-9 _ . -" + +kv_set store="bad/store" key=k value=x + → error INVALID_ARGUMENT: "invalid name: KV store name contains an invalid byte 0x2f; ..." + +kv_set store=smoke key="<630-byte key>" value=x + → error INVALID_ARGUMENT: "invalid name: KV key exceeds 512-byte limit (630 bytes)" +``` + +Boundary check: a 499-byte key is **accepted**; a 630-byte key is +**rejected**. (Automated in `error_tests.rs::maps_invalid_name_to_invalid_argument`.) + +--- + +## 7. Read-only mode + +Only relevant when the server runs with `--read-only` (`status` shows +`"read_only": true`). Start such a server yourself for this check — do not +assume the shared daemon is read-only. + +``` +# readers work: +kv_get store=smoke key=k → {"found": ...} +kv_list store=smoke → {...} +kv_size store=smoke → {...} +kv_list_stores → {...} + +# mutators are blocked: +kv_set store=smoke key=k value=v → error READ_ONLY_VIOLATION ("... not permitted in read-only mode") +kv_delete store=smoke key=k → error READ_ONLY_VIOLATION +kv_pop store=smoke → error READ_ONLY_VIOLATION +kv_clear store=smoke → error READ_ONLY_VIOLATION +``` + +--- + +## 8. Database routing + isolation + +**⚠ Touches the persistent database. Clean up after (§12).** + +Each database keeps its own isolated set of stores. The same store name in +two databases holds independent values. `persist: true` and +`database: "persistent"` target the same place. + +``` +kv_set store=smoke_routing key=where value="ephemeral" # → ephemeral (default) +kv_set store=smoke_routing key=where value="persistent" database=persistent # → persistent +kv_set store=smoke_routing key=where2 value="via-flag" persist=true # → persistent (same DB) + +kv_get store=smoke_routing key=where → {"found": true, "value": "ephemeral"} +kv_get store=smoke_routing key=where database=persistent → {"found": true, "value": "persistent"} +kv_get store=smoke_routing key=where2 persist=true → {"found": true, "value": "via-flag"} + +kv_list store=smoke_routing → {"store": "smoke_routing", "count": 1, "keys": ["where"]} # ephemeral +kv_list store=smoke_routing database=persistent → {"store": "smoke_routing", "count": 2, "keys": ["where","where2"]} # persistent +``` + +The ephemeral and persistent `where` values differ → isolation holds. +`persist=true` and `database=persistent` landed in the same store → both +keys present in persistent. + +**Ephemeral-only guard:** if the server was started with `--ephemeral-only`, +`kv_set ... persist=true` returns `INVALID_ARGUMENT` (a clear error, not a +panic). + +--- + +## 9. The `LEFT JOIN` enrichment pattern + +The backing table `_hyperdb_kv_store(store_name, key, value)` is hidden from +`describe`/`status` but queryable directly. This is the point of the KV +store: annotate analytical rows with scratchpad metadata via a plain SQL +join. **Run this in the ephemeral DB** (create a `smoke_`-prefixed table): + +``` +kv_set store=product_notes key=P1 value="flagship - review pricing Q3" +kv_set store=product_notes key=P3 value="discontinue candidate" + +execute ["CREATE TABLE smoke_products (id TEXT, name TEXT, revenue INTEGER)"] +execute ["INSERT INTO smoke_products (id,name,revenue) VALUES ('P1','Widget',5000),('P2','Gadget',3000),('P3','Gizmo',800)"] + +query + SELECT p.id, p.name, p.revenue, kv.value AS note + FROM smoke_products p + LEFT JOIN _hyperdb_kv_store kv + ON kv.store_name = 'product_notes' AND kv.key = p.id + ORDER BY p.id +``` + +Expected: P1 and P3 carry their notes; **P2 survives with `note: null`** +(the `LEFT` join keeps unannotated rows). + +--- + +## 10. Table is hidden but accessible + +``` +describe → table list does NOT include _hyperdb_kv_store +query SELECT COUNT(*) FROM _hyperdb_kv_store → succeeds (directly queryable) +``` + +Hidden-but-accessible, exactly like `_hyperdb_saved_queries`. + +--- + +## 11. Concurrency / atomicity (optional, deeper) + +The backing table has **no index**; uniqueness on overwrite and +single-serve on pop rely on the engine serializing writes within one server +process. To stress this against a live server, fan out concurrent calls +(e.g. from a script or a fleet of parallel tool calls) to a scratch store +named `smoke_concurrency` (keep it ephemeral — omit `database` — and purge +it in §12): + +- **N concurrent `kv_set` to the same key** → the store ends with exactly + **one** row for that key (no duplicates in the indexless table). +- **M concurrent `kv_set` to distinct keys** → exactly M rows, none lost. +- **P concurrent `kv_pop` draining the store** → every found key is + returned **at most once** (no double-serve); surplus poppers get + `{"found": false}`. + +This validates the "atomic within a single server process" guarantee +documented on `kv_pop` and the `hyper://schema/kv` resource. (Cross-process +writes to a shared persistent store via the daemon are **not** guarded by a +DB constraint — that limitation is documented, not a smoke-test failure.) + +--- + +## 12. Cleanup + verification + +Purge every scratch store and table, then confirm the databases are back to +their starting state: + +``` +kv_clear store=smoke +kv_clear store=smoke_routing +kv_clear store=smoke_routing database=persistent +kv_clear store=smoke_concurrency # only if you ran §11 +kv_clear store=product_notes +execute ["DROP TABLE IF EXISTS smoke_products"] + +# verify nothing of ours remains: +kv_list_stores → {"count": 0, "stores": []} # (or only pre-existing non-smoke stores) +kv_list_stores database=persistent → no smoke_* / product_notes stores +describe database=persistent → only the real, pre-existing tables (no smoke_*) +``` + +If the persistent database shows any `smoke_`/`product_notes` remnant, the +run left debris — clear it before finishing. + +--- + +## Expanding this doc + +Add a numbered section per new capability or regression you want covered. +Keep the format: the exact tool calls, the `→` expected JSON, and one line +on *why* the check matters. When a smoke check hardens into something CI +should enforce, promote it into `tests/kv_tools_tests.rs` (or the relevant +suite) and note the automated equivalent here. From 39f3c5e6059c11cf1c3565c6268595524134ba20 Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Thu, 9 Jul 2026 21:17:22 -0700 Subject: [PATCH 3/3] docs(mcp): surface the KV store across all doc surfaces The key-value scratchpad tools shipped without documentation in the user-facing MCP README, the LLM-facing get_readme, or the workspace READMEs. Add coverage in the right places so an LLM told "remember X" considers kv_set instead of always reaching for CREATE TABLE: - hyperdb-mcp/README.md: new "### Key-Value Store" tool section; a "Table or key-value store?" decision rule in Queryable Memory for AI; the hyper://schema/kv Resources-table row; KV tools in the Features bullet; and the KV mutator/reader split across all three read-only enumerations (flag table, Allowed, Blocked). - src/readme.rs (get_readme): "KV store vs. a custom table" decision subsection; kv_pop lexicographic-order clarification; a describe persistent-DB signpost (status reports counts, not names); a back-link completing the bidirectional cross-link. - src/server.rs: kv_pop and kv_list_stores #[tool(description)] strings clarify lexicographic pop order and the no-store-registry behavior. - README.md (top-level): KvStore / AsyncKvStore Key Features bullet. Also fix a pre-existing false claim on the --read-only flag-table row (carried forward on a line this change edited): Hyper-format export is NOT disabled in read-only mode (export has no writable gate; it's a read-only file copy), matching the Read-Only "Allowed" list and the HyperMcpServer::new doc comment. --- README.md | 1 + hyperdb-mcp/CHANGELOG.md | 6 +++++ hyperdb-mcp/README.md | 46 ++++++++++++++++++++++++++++++++++++--- hyperdb-mcp/src/readme.rs | 26 ++++++++++++++++++++-- hyperdb-mcp/src/server.rs | 7 +++--- 5 files changed, 78 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 40174ed..0594b1b 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ database files (`.hyper`) without any C library dependencies. - **Typed Row Mapping** — `#[derive(FromRow)]` structs, including streaming `stream_as` for constant-memory typed queries - **Compile-time SQL Validation** — opt-in `query_as!` macro checks SQL against your schema at build time (red squigglies in VS Code) - **Connection Pooling** — async pooling via `deadpool` for high-concurrency applications +- **Key-Value Store** — string-native `KvStore` / `AsyncKvStore` backed by a single fixed table - **Arrow Integration** — insert and read data in Arrow IPC stream format - **gRPC Transport** — read-only access with Arrow IPC and load balancing support - **Full Type Support** — all Hyper types including Numeric, Geography, Intervals diff --git a/hyperdb-mcp/CHANGELOG.md b/hyperdb-mcp/CHANGELOG.md index 25a689d..cefbbf2 100644 --- a/hyperdb-mcp/CHANGELOG.md +++ b/hyperdb-mcp/CHANGELOG.md @@ -36,6 +36,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/). `INVALID_ARGUMENT`, which carries a self-correction suggestion. The human-readable message (naming the offending byte or the length) is unchanged. +- **README `--read-only` flag table no longer claims Hyper-format export is + disabled.** The `--read-only` row wrongly listed "Hyper-format export" among + the disabled operations, contradicting both the actual behavior (`export` + has no read-only gate — a `.hyper` export is a harmless read-only file copy + and stays allowed) and the same document's Read-Only Mode "Allowed" list. + Docs-only correction; no behavior change. - **TCP keepalive on the `hyperd` connection.** Connections to `hyperd` now enable TCP keepalive (60s idle, 10s interval, ~90s to declare a dead peer) instead of relying on the OS default 2-hour idle timeout. Without it, a diff --git a/hyperdb-mcp/README.md b/hyperdb-mcp/README.md index 7136483..769d160 100644 --- a/hyperdb-mcp/README.md +++ b/hyperdb-mcp/README.md @@ -25,6 +25,8 @@ This means an LLM can: The ephemeral database is scratch space (think: a whiteboard). The persistent database is long-term memory (think: a filing cabinet you can query). Multiple AI clients sharing the same daemon see the same persistent data — so Claude Code, Cursor, and VS Code Copilot can all read from and contribute to the same knowledge base. +**Table or key-value store?** For a handful of small facts, notes, or flags, prefer the built-in key-value store (`kv_set` with `persist: true`) over `CREATE TABLE` + `load_data` — it needs no schema and no DDL. Reach for a real table when you need typed columns, JOINs, or aggregation. See [Working with both databases](#working-with-both-databases) for the `persist` / `database` mechanics that apply to both paths. + --- ## Features @@ -48,6 +50,7 @@ The ephemeral database is scratch space (think: a whiteboard). The persistent da - **Partial schema overrides** — supply just the columns you want to correct (e.g. `{"population":"BIGINT"}`) — the rest keep their inferred type - **Rich resource surface** — workspace readme, per-table JSON and CSV samples, and one JSON + one CSV resource per table so LLMs can orient themselves via `resources/list` without any tool calls - **Saved queries** — register named read-only SQL with `save_query`; each query becomes `hyper://queries/{name}/definition` (metadata) + `hyper://queries/{name}/result` (live re-run). Persisted in the persistent attachment, session-only when `--ephemeral-only` +- **Key-value scratchpad** — lightweight `kv_set` / `kv_get` / `kv_list` / `kv_delete` / `kv_pop` / `kv_size` / `kv_clear` / `kv_list_stores` store for small notes and state without a `CREATE TABLE`. Ephemeral by default (lost on restart); pass `persist: true` (or `database: "persistent"`) to make a store durable across sessions - **Live resource-update notifications** — MCP clients can `resources/subscribe` to any `hyper://...` URI; the server fires `notifications/resources/updated` after every ingest, DDL, watcher event, or saved-query mutation --- @@ -270,7 +273,7 @@ If hyperd repeatedly fails to start (3 attempts within 60 seconds — e.g., misc | Flag | Behavior | |---|---| -| `--read-only` | Disables `execute`, `load_data`, `load_file`, `watch_directory`, `save_query`, `delete_query`, and Hyper-format export. See [Read-Only Mode](#read-only-mode). | +| `--read-only` | Disables `execute`, `load_data`, `load_file`, `watch_directory`, `save_query`, `delete_query`, and the KV mutators (`kv_set`, `kv_delete`, `kv_pop`, `kv_clear`). Export (including `.hyper`) stays allowed — it's a read-only file copy. See [Read-Only Mode](#read-only-mode). | --- @@ -507,6 +510,42 @@ delete_query(name: 'top_5_customers') Returns `{ "deleted": true }` when the query existed, `{ "deleted": false }` when it did not (no error on unknown names). Disabled in read-only mode. +### Key-Value Store + +Lightweight named scratchpad for stashing a value under `store` + `key` and +recalling it later — remember a variable, a summary, a JSON config, or a +work-queue entry without creating a table or running `load_data`. + +> **Stores default to the EPHEMERAL database and are LOST on server restart.** +> Pass `database="persistent"` (or `persist=true`) to make a store durable +> across restarts, or an attached alias to target that database. Each database +> has its own isolated set of stores; a store in one database is invisible from +> another. + +Eight tools cover the surface: + +| Tool | Purpose | Parameters | +|---|---|---| +| `kv_set` | Write/overwrite a value (upsert) | `store`, `key`, `value`, `database`, `persist` | +| `kv_get` | Read a value by store + key (`value` is null when absent, not an error) | `store`, `key`, `database`, `persist` | +| `kv_delete` | Remove one key (`{deleted: true/false}`, no error on unknown key) | `store`, `key`, `database`, `persist` | +| `kv_list` | List all keys in a store, sorted ascending | `store`, `database`, `persist` | +| `kv_list_stores` | List store namespaces that currently hold data | `database`, `persist` | +| `kv_size` | Count keys in a store | `store`, `database`, `persist` | +| `kv_pop` | Destructively read-and-remove the lowest-keyed entry (atomic) | `store`, `database`, `persist` | +| `kv_clear` | Delete all keys in a store (returns count removed) | `store`, `database`, `persist` | + +``` +kv_set(store: 'session', key: 'last_report', value: '{"rows": 4210}', database: 'persistent') +kv_get(store: 'session', key: 'last_report') +``` + +Key properties: +- **Read-only mode** — the four mutators (`kv_set`, `kv_delete`, `kv_pop`, `kv_clear`) are disabled and return `READ_ONLY_VIOLATION`; the four readers (`kv_get`, `kv_list`, `kv_size`, `kv_list_stores`) always work. +- **Pop order** — `kv_pop` removes and returns the **lowest-keyed** entry in lexicographic key order (not insertion order), making a store usable as a simple work queue. +- **No store registry** — a store that becomes empty simply **drops out** of `kv_list_stores`; there is no separate registry of store names. +- **Backing table** — values live in `_hyperdb_kv_store(store_name, key, value)`, which is indexless (Hyper has no indexes) and hidden from `describe` by its `_hyperdb_` prefix, but is directly queryable — e.g. `LEFT JOIN` it to enrich an analytical table (always filter on `kv.store_name`). Uniqueness of `(store_name, key)` is enforced by the tool layer's upsert, atomic within a single server process. See the `hyper://schema/kv` resource for the schema and join pattern. + ### Export Tools #### `export` @@ -601,6 +640,7 @@ can route it appropriately (LLM context vs. file download vs. chart). | `hyper://tables/{name}/csv-sample` | `text/csv` | First 20 rows of a table as CSV, header-first | | `hyper://queries/{name}/definition` | `application/json` | Stored SQL + metadata for a saved query | | `hyper://queries/{name}/result` | `application/json` | Live result of a saved query — re-runs on every read | +| `hyper://schema/kv` | `text/plain` | KV scratchpad schema: the `_hyperdb_kv_store(store_name, key, value)` backing table, its indexless shape, the ephemeral-vs-persistent durability rule, and the `LEFT JOIN` enrichment pattern | Resource templates (discoverable via `resources/templates/list`): @@ -666,8 +706,8 @@ Four guided analytical workflows registered as MCP **Prompts**. hyperdb-mcp --persistent-db ~/analytics.hyper --read-only ``` -- **Allowed:** `query`, `query_data`, `query_file`, `describe`, `sample`, `inspect_file`, `status`, `export` -- **Blocked:** `execute`, `load_data`, `load_file`, `watch_directory`, `save_query`, `delete_query` — return `READ_ONLY_VIOLATION` +- **Allowed:** `query`, `query_data`, `query_file`, `describe`, `sample`, `inspect_file`, `status`, `export`, and the KV readers `kv_get`, `kv_list`, `kv_size`, `kv_list_stores` +- **Blocked:** `execute`, `load_data`, `load_file`, `watch_directory`, `save_query`, `delete_query`, and the KV mutators `kv_set`, `kv_delete`, `kv_pop`, `kv_clear` — return `READ_ONLY_VIOLATION` - **Resources, prompts, and resource subscriptions** work normally — read-only clients can still subscribe to `hyper://...` URIs and receive notifications when other (non-read-only) connections mutate state The `query` tool also enforces read-only at the SQL level — only `SELECT`/`WITH`/`EXPLAIN`/`SHOW`/`VALUES` are accepted. diff --git a/hyperdb-mcp/src/readme.rs b/hyperdb-mcp/src/readme.rs index 3ec4133..9bccdee 100644 --- a/hyperdb-mcp/src/readme.rs +++ b/hyperdb-mcp/src/readme.rs @@ -72,6 +72,22 @@ query({ sql: \"SELECT s.*, p.decision FROM scratch_analysis s \ ON s.topic = p.topic\" }) ``` +### KV store vs. a custom table — which to remember with + +When you need to remember something, pick the lighter tool: + +- **A few scraps** — a variable, a flag, a summary, a JSON blob, a + work-queue entry — use the key-value store (`kv_set` / `kv_get`). No + schema, no DDL, no `load_data`. See `Tool index → Key-value store`. +- **Structured rows** you'll filter, JOIN, or aggregate — use a real + table (`load_data` / `execute CREATE TABLE`), so SQL can reason over + typed columns. + +Both persist the same way: default is the EPHEMERAL database (lost on +restart); pass `database: \"persistent\"` to keep either one across +sessions. The KV and `load_*` tools also accept the `persist: true` +shorthand; `execute` takes `database` only. + ### Routing data to a destination - **`database` parameter** (preferred for tools that build their own @@ -124,6 +140,10 @@ watcher targets the alias; call `unwatch_directory` first. ### Inspect - `describe` — list workspace tables (no args) or describe one table (`table` arg) with columns, types, row count, and prose metadata. + **Defaults to the ephemeral primary** — pass `database: \"persistent\"` + (or an attached alias) to list/inspect durable tables. `status` reports + table *counts* only, never names, so check the right database here + before assuming a persistent table is missing. - `sample` — return schema + first N rows of a table. Use this before writing a non-trivial query. - `inspect_file` — dry-run schema inference on a CSV / Parquet / Arrow @@ -172,7 +192,8 @@ watcher targets the alias; call `unwatch_directory` first. - `kv_list` — list keys in a store. - `kv_list_stores` — list store namespaces that hold data in a database. - `kv_size` — count keys in a store. -- `kv_pop` — destructively read-and-remove the lowest-keyed entry. +- `kv_pop` — destructively read-and-remove the lowest-keyed entry + (lexicographic key order, not insertion order). - `kv_clear` — delete all keys in a store. Every kv_* tool takes the same optional `database` parameter as the data @@ -183,7 +204,8 @@ has its own isolated set of stores. Enrich analytical tables with KV metadata via LEFT JOIN — always filter `kv.store_name = ''` to avoid row multiplication, and keep the KV table in the same database as the joined table. See the `hyper://schema/kv` resource for the join -template. +template, and `KV store vs. a custom table` above for when to reach for +this instead of a real table. ### Introspection - `get_readme` — this document. Call once at the start of a session. diff --git a/hyperdb-mcp/src/server.rs b/hyperdb-mcp/src/server.rs index 0e5f0e6..86c1da4 100644 --- a/hyperdb-mcp/src/server.rs +++ b/hyperdb-mcp/src/server.rs @@ -3185,7 +3185,7 @@ impl HyperMcpServer { /// List all scratchpad store namespaces that hold data in a database. #[tool( - description = "List all KV scratchpad store namespaces that currently hold data in a database. Omit `database` for the ephemeral store, or route with \"persistent\"/persist=true/an attached alias. Each database has its own isolated set of stores." + description = "List all KV scratchpad store namespaces that currently hold data in a database. Omit `database` for the ephemeral store, or route with \"persistent\"/persist=true/an attached alias. Each database has its own isolated set of stores. A store drops off this list once its last key is removed — there is no separate registry." )] fn kv_list_stores( &self, @@ -3224,9 +3224,10 @@ impl HyperMcpServer { } } - /// Destructively read-and-remove the lowest-keyed entry (atomic). + /// Destructively read-and-remove the lowest-keyed entry in lexicographic + /// key order (atomic). #[tool( - description = "Destructively read-and-remove the lowest-keyed entry from a KV store (peek+delete in one transaction, atomic within a single server process — useful as a work queue for one session; two separate server processes popping a shared persistent store could double-serve an entry). Returns {found, key, value}; {found: false} on an empty store. Omit `database` for the ephemeral store, or route with \"persistent\"/persist=true/an attached alias." + description = "Destructively read-and-remove the lowest-keyed entry (lexicographic key order, not insertion order) from a KV store (peek+delete in one transaction, atomic within a single server process — useful as a work queue for one session; two separate server processes popping a shared persistent store could double-serve an entry). Returns {found, key, value}; {found: false} on an empty store. Omit `database` for the ephemeral store, or route with \"persistent\"/persist=true/an attached alias." )] fn kv_pop( &self,