From 37394335e12d372dac00ce459f92d880c9279327 Mon Sep 17 00:00:00 2001 From: Haider Rizvi <86354776+haiderRizvii@users.noreply.github.com> Date: Fri, 23 Oct 2026 12:01:00 +0500 Subject: [PATCH 1/2] Feature/neo4j backend (#1) * feat: add Neo4j as a graph backend index_repository accepts graph_db (http://host:7474), graph_user, graph_password and graph_database (CLI: --graph-db ...), with NEO4J_USER / NEO4J_PASSWORD / NEO4J_DATABASE as fallbacks. - src/neo4j: HTTP client for Neo4j's transactional endpoint (WinHTTP for https on Windows), config + in-process credential cache, sync of a published project database into Neo4j as an atomic generation swap, and a remote store that answers the graph read primitives. - The pipeline mirrors every publish (full, incremental, no-op) into Neo4j; a .db.graph.json marker (no password) keeps later re-indexes syncing to the same target. An unchanged re-run re-uploads when Neo4j no longer holds the published generation. - store: a remote-ops seam routes node/edge lookups, degrees, counts and every BFS to Neo4j; reads fall back to the identical local copy when Neo4j is unreachable or holds another generation. - query_graph native:true runs read-only Cypher in Neo4j; index_status reports graph_backend; delete_project drops the Neo4j graph. - The index supervisor passes the password to the worker through its environment, never on its command line. Co-Authored-By: Claude Opus 5.5 * fix(java): never resolve a call onto a Field Java has no callable fields: `lock.lock()` invokes ReentrantLock.lock, not the `lock` field, but the unique_name strategy bound it to the field. cbm_call_onto_field_policy now drops every Java call that lands on a Field (sequential and parallel resolvers share the policy). Co-Authored-By: Claude Opus 5.5 * feat(docker): cbm-graph image to build code graphs into Neo4j docker/graph builds the binary in a container and indexes a mounted repository (read-only) into Neo4j. cbm-graph.cmd / cbm-graph.ps1 wrap it on Windows; an extra .cbmignore (e.g. a fleet-wide one) is merged in and re-runs are incremental through a named cache volume. Co-Authored-By: Claude Opus 5.5 * build: make the fork installable from its own releases - install.sh / install.ps1 / scripts/setup* download from haiderRizvii/codebase-memory-mcp; `update` re-runs the installer shipped beside the binary, so it updates from the fork too. - fork-release.yml: build all eight targets (VirusTotal optional), smoke-test, publish a full "latest" release with checksums.txt. - release.yml stops at preflight outside the upstream repository; it publishes through upstream's VirusTotal/npm/PyPI/MCP Registry accounts. - UI issue link points at the fork; URL allowlists updated. - README: fork notice, install commands, Neo4j backend section. Co-Authored-By: Claude Opus 5.5 * ci: satisfy the security and no-skips gates for the Neo4j backend - security-network-source-audit.py: new configured-tcp transport, pinned to src/neo4j/neo4j_http.c and to a getaddrinfo() walk over the parsed graph_db host with a stream-socket hint; self-tests cover the accepted shape and reject other files, other hosts, datagram sockets and addresses outside the resolved list. - security-allowlist.txt: the two configured-tcp calls and the graph-backend marker write. - mcp.c: describe graph_db without a literal example URL. - test_neo4j.c: register the live test only when a server is configured instead of SKIP(), which the no-skips policy forbids. Co-Authored-By: Claude Opus 5.5 * fix(neo4j): narrow a variable's scope (cppcheck variableScope) Co-Authored-By: Claude Opus 5.5 * fix(neo4j): reject malformed responses; make fallback visible Review follow-up (Copilot): - A 2xx body is accepted only when "results" holds exactly one entry per statement, each with columns/data arrays and rows matching the columns (cbm_neo4j_result_shape_error). A malformed reply used to read as "no rows": an empty graph to readers, and generation 0 to the sync, whose stale-generation cleanup could then delete the live graph. The sync additionally requires exactly one integer generation row. - A Neo4j-backed project answered from the local index now says so in every tool reply: a content notice plus structuredContent.graph_backend {status: sqlite_fallback, reason}. (The local index is the newest generation; the fallback is current, Neo4j is not.) - A failed background sync is recorded as last_sync_error in the project's marker, keeping the last good synced_generation; index_status reports local_generation, in_sync and last_sync_error. - clang-format 20 over the Neo4j sources and the store routing. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .dockerignore | 4 + .github/workflows/fork-release.yml | 168 ++++ .github/workflows/release.yml | 13 + Makefile.cbm | 12 +- README.md | 42 +- docker/graph/Dockerfile | 33 + docker/graph/README.md | 38 + docker/graph/cbm-graph.cmd | 6 + docker/graph/cbm-graph.ps1 | 69 ++ docker/graph/cbm-graph.sh | 85 ++ install.ps1 | 2 +- install.sh | 4 +- scripts/security-allowlist.txt | 10 + scripts/security-network-source-audit.py | 91 ++ scripts/security-strings.sh | 2 + scripts/setup-windows.ps1 | 2 +- scripts/setup.sh | 2 +- src/mcp/index_supervisor.c | 69 +- src/mcp/mcp.c | 234 ++++- src/mcp/mcp.h | 6 + src/neo4j/neo4j_client.c | 494 ++++++++++ src/neo4j/neo4j_client.h | 118 +++ src/neo4j/neo4j_http.c | 791 ++++++++++++++++ src/neo4j/neo4j_http.h | 48 + src/neo4j/neo4j_store.c | 1044 ++++++++++++++++++++++ src/neo4j/neo4j_store.h | 34 + src/neo4j/neo4j_sync.c | 693 ++++++++++++++ src/neo4j/neo4j_sync.h | 64 ++ src/pipeline/pipeline.c | 55 ++ src/pipeline/pipeline.h | 17 + src/pipeline/registry.c | 7 + src/store/store.c | 113 ++- src/store/store.h | 48 + src/ui/http_server.c | 2 +- tests/test_main.c | 4 + tests/test_neo4j.c | 614 +++++++++++++ tests/test_registry.c | 8 + 37 files changed, 5024 insertions(+), 22 deletions(-) create mode 100644 .dockerignore create mode 100644 .github/workflows/fork-release.yml create mode 100644 docker/graph/Dockerfile create mode 100644 docker/graph/README.md create mode 100644 docker/graph/cbm-graph.cmd create mode 100644 docker/graph/cbm-graph.ps1 create mode 100644 docker/graph/cbm-graph.sh create mode 100644 src/neo4j/neo4j_client.c create mode 100644 src/neo4j/neo4j_client.h create mode 100644 src/neo4j/neo4j_http.c create mode 100644 src/neo4j/neo4j_http.h create mode 100644 src/neo4j/neo4j_store.c create mode 100644 src/neo4j/neo4j_store.h create mode 100644 src/neo4j/neo4j_sync.c create mode 100644 src/neo4j/neo4j_sync.h create mode 100644 tests/test_neo4j.c diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..b33edafad1 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +.git +build +graph-ui/node_modules +**/node_modules diff --git a/.github/workflows/fork-release.yml b/.github/workflows/fork-release.yml new file mode 100644 index 0000000000..2890cb2ae7 --- /dev/null +++ b/.github/workflows/fork-release.yml @@ -0,0 +1,168 @@ +# Fork release: build every platform, smoke-test, publish a GitHub release. +# +# release.yml is upstream's pipeline and publishes through accounts a fork does +# not have (VirusTotal, npm, PyPI, MCP Registry). This workflow keeps exactly +# what the installers need: the eight platform archives (+ MCPB bundles) and +# checksums.txt on a NON-prerelease "latest" release, which is what +# install.sh / install.ps1 -> https://github.com//releases/latest/download/... +# resolve. GitHub never treats a prerelease as "latest", so every release made +# here is a full release, including versions like v0.12.1-neo4j.1. +# +# Run: Actions -> Fork Release -> Run workflow (on main). +name: Fork Release + +on: + workflow_dispatch: + inputs: + version: + description: "Release version, v-prefixed (e.g. v0.12.1-neo4j.1)" + required: true + type: string + release_notes: + description: "Release notes (optional; generated from commits when empty)" + required: false + type: string + virustotal: + description: "VirusTotal-scan the binaries (needs the VIRUS_TOTAL_SCANNER_API_KEY secret)" + required: false + type: boolean + default: false + skip_smoke: + description: "Skip the smoke tests of the built binaries" + required: false + type: boolean + default: false + replace: + description: "Replace an existing release with the same version" + required: false + type: boolean + default: false + +permissions: + contents: read + +concurrency: + group: fork-release + cancel-in-progress: false + +jobs: + preflight: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + # The installers fetch releases/latest/download/; the tag is the + # version verbatim, so a bare "0.12.1" would publish an unreachable tag. + - name: Enforce v-prefixed semver version input + env: + VERSION: ${{ inputs.version }} + run: | + if [[ "$VERSION" =~ ^v[0-9]+\.[0-9]+\.[0-9]+([-.][0-9A-Za-z.]+)?$ ]]; then + echo "version OK: $VERSION" + else + echo "::error::version must be v-prefixed semver (vX.Y.Z[-suffix]), got '$VERSION'" + exit 1 + fi + + build: + needs: [preflight] + permissions: + contents: read + id-token: write + attestations: write + uses: ./.github/workflows/_build.yml + with: + version: ${{ inputs.version }} + attest: false + scan_candidates: ${{ inputs.virustotal }} + secrets: inherit + + smoke: + needs: [build] + if: ${{ !cancelled() && needs.build.result == 'success' && !inputs.skip_smoke }} + uses: ./.github/workflows/_smoke.yml + with: + broad_platforms: false + + publish: + needs: [build, smoke] + # A skipped smoke is accepted only when it was skipped on request. + if: >- + ${{ !cancelled() + && needs.build.result == 'success' + && (needs.smoke.result == 'success' + || (inputs.skip_smoke && needs.smoke.result == 'skipped')) }} + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Download the packaged containers + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: binaries-* + merge-multiple: true + path: dist + + - name: List artifacts and require every platform + working-directory: dist + run: | + ls -la + for target in linux-amd64 linux-arm64 linux-amd64-portable linux-arm64-portable \ + darwin-amd64 darwin-arm64; do + test -f "codebase-memory-mcp-$target.tar.gz" || { echo "::error::missing $target archive"; exit 1; } + done + for target in windows-amd64 windows-arm64; do + test -f "codebase-memory-mcp-$target.zip" || { echo "::error::missing $target archive"; exit 1; } + done + + # install.sh / install.ps1 refuse any archive whose SHA-256 is not listed + # here, so this file is what makes the release installable. + - name: Generate checksums + working-directory: dist + run: | + shopt -s nullglob + sha256sum *.tar.gz *.zip *.mcpb > checksums.txt + cat checksums.txt + + - name: Delete existing release + if: ${{ inputs.replace }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VERSION: ${{ inputs.version }} + run: gh release delete "$VERSION" --yes --cleanup-tag --repo "$GITHUB_REPOSITORY" || true + + # Tag the dispatched commit (the one the binaries were built from), not + # whatever main points at by now. + - name: Publish the release + working-directory: dist + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VERSION: ${{ inputs.version }} + NOTES: ${{ inputs.release_notes }} + run: | + shopt -s nullglob + notes_args=(--generate-notes) + if [ -n "$NOTES" ]; then notes_args=(--notes "$NOTES"); fi + gh release create "$VERSION" \ + --repo "$GITHUB_REPOSITORY" \ + --target "$GITHUB_SHA" \ + --title "$VERSION" \ + --latest \ + "${notes_args[@]}" \ + *.tar.gz *.zip *.mcpb checksums.txt + { + echo "### Released $VERSION" + echo + echo '```bash' + echo "curl -fsSL https://raw.githubusercontent.com/$GITHUB_REPOSITORY/main/install.sh | bash" + echo '```' + echo '```powershell' + echo "Invoke-WebRequest -Uri https://raw.githubusercontent.com/$GITHUB_REPOSITORY/main/install.ps1 -OutFile install.ps1" + echo 'Unblock-File .\install.ps1' + echo 'PowerShell -ExecutionPolicy Bypass -File .\install.ps1' + echo '```' + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 707c699246..3cea781fed 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -44,6 +44,8 @@ permissions: jobs: # ── Security (starts immediately, blocks only verify) ─────────── security: + # Upstream-only: forks release through fork-release.yml (see preflight). + if: ${{ github.repository == 'DeusData/codebase-memory-mcp' }} uses: ./.github/workflows/_security.yml # No job in _security.yml reads code scanning or the Actions API any more # (the CodeQL gate moved to codeql.yml), so the token stays read-only. @@ -57,6 +59,7 @@ jobs: # a called workflow cannot request more than its caller grants, so # withholding either permission fails the whole run at startup. codeql: + if: ${{ github.repository == 'DeusData/codebase-memory-mcp' }} uses: ./.github/workflows/codeql.yml permissions: contents: read @@ -72,6 +75,15 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 steps: + # This pipeline publishes through upstream's VirusTotal, npm, PyPI and + # MCP Registry accounts, which a fork does not have. Stop at once with a + # pointer instead of failing hours later. + - name: Refuse to run outside the upstream repository + if: ${{ !(github.repository == 'DeusData/codebase-memory-mcp') }} + run: | + echo "::error::release.yml is upstream's pipeline. On this fork run the 'Fork Release' workflow (.github/workflows/fork-release.yml) instead." + exit 1 + - name: Enforce v-prefixed semver version input env: VERSION: ${{ inputs.version }} @@ -160,6 +172,7 @@ jobs: # release-draft's needs: the report is evidence for the release notes, never # a publication gate. memwaste: + if: ${{ github.repository == 'DeusData/codebase-memory-mcp' }} uses: ./.github/workflows/_memwaste.yml with: mode: full diff --git a/Makefile.cbm b/Makefile.cbm index 3b93146f1f..0614650390 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -184,7 +184,7 @@ ifeq ($(IS_MINGW),yes) # two Windows builds of identical source are never byte-identical and every # release is a brand-new file to a reputation system. Same motivation as the # mimalloc __DATE__ patch; ELF and Mach-O have no equivalent field to clear. -WIN32_LIBS := -lws2_32 -lpsapi -lshell32 -ladvapi32 -lbcrypt -Wl,--allow-multiple-definition -Wl,--stack,8388608 -Wl,--no-insert-timestamp -static +WIN32_LIBS := -lws2_32 -lwinhttp -lpsapi -lshell32 -ladvapi32 -lbcrypt -Wl,--allow-multiple-definition -Wl,--stack,8388608 -Wl,--no-insert-timestamp -static endif # STATIC=1 produces a fully static binary (for Alpine/musl portable builds) @@ -454,6 +454,9 @@ STORE_SRCS = src/store/store.c src/store/graph_mem.c src/store/architecture_proj # Cypher module (new) CYPHER_SRCS = src/cypher/cypher.c +# Neo4j graph backend (HTTP API client, sync, remote store reads) +NEO4J_SRCS = src/neo4j/neo4j_http.c src/neo4j/neo4j_client.c src/neo4j/neo4j_sync.c src/neo4j/neo4j_store.c + # MCP server module (new) MCP_SRCS = src/mcp/mcp.c src/mcp/index_supervisor.c src/mcp/compact_out.c src/mcp/architecture_jobs.c \ src/mcp/test_impact.c src/mcp/test_impact_diff.c src/mcp/test_impact_classify.c src/mcp/test_impact_source.c src/mcp/test_impact_seed.c src/mcp/test_impact_engine.c src/mcp/test_impact_engine_snapshot.c src/mcp/test_impact_engine_reach.c src/mcp/test_impact_engine_artifact.c src/mcp/test_impact_cli.c src/mcp/test_impact_artifact.c src/mcp/test_impact_changes.c src/mcp/test_impact_origins.c src/mcp/test_impact_coverage.c src/mcp/test_impact_config.c src/mcp/test_impact_select.c src/mcp/test_impact_policy.c src/mcp/test_impact_git.c src/mcp/test_impact_inventory.c src/mcp/test_impact_inventory_input.c src/mcp/test_impact_tree.c src/mcp/test_impact_tree_audit.c src/mcp/test_impact_tree_read.c src/mcp/test_impact_tree_batch.c src/mcp/test_impact_tree_digest.c src/mcp/test_impact_tree_names.c src/mcp/test_impact_tree_plan.c src/mcp/test_impact_tree_posix.c src/mcp/test_impact_tree_posix_acl.c src/mcp/test_impact_tree_posix_io.c src/mcp/test_impact_tree_util.c src/mcp/test_impact_tree_windows.c src/mcp/test_impact_tree_windows_acl.c src/mcp/test_impact_tree_windows_io.c \ @@ -689,7 +692,7 @@ TRE_CFLAGS = -std=c11 -g -O1 -w -Ivendored/tre YYJSON_SRC = vendored/yyjson/yyjson.c # All production sources -PROD_SRCS = $(MEMWASTE_SRCS) $(FOUNDATION_SRCS) $(STORE_SRCS) $(CYPHER_SRCS) $(MCP_SRCS) $(DAEMON_SRCS) $(DISCOVER_SRCS) $(GRAPH_BUFFER_SRCS) $(PIPELINE_SRCS) $(SIMHASH_SRCS) $(SEMANTIC_SRCS) $(TRACES_SRCS) $(WATCHER_SRCS) $(GIT_SRCS) $(CLI_SRCS) $(UI_SRCS) $(YYJSON_SRC) +PROD_SRCS = $(MEMWASTE_SRCS) $(FOUNDATION_SRCS) $(STORE_SRCS) $(CYPHER_SRCS) $(NEO4J_SRCS) $(MCP_SRCS) $(DAEMON_SRCS) $(DISCOVER_SRCS) $(GRAPH_BUFFER_SRCS) $(PIPELINE_SRCS) $(SIMHASH_SRCS) $(SEMANTIC_SRCS) $(TRACES_SRCS) $(WATCHER_SRCS) $(GIT_SRCS) $(CLI_SRCS) $(UI_SRCS) $(YYJSON_SRC) EXISTING_C_SRCS = $(EXTRACTION_SRCS) $(LSP_SRCS) $(TS_RUNTIME_SRC) \ $(GRAMMAR_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) @@ -754,7 +757,8 @@ TEST_STORE_SRCS = \ tests/test_store_bulk.c \ tests/test_store_pragmas.c \ tests/test_store_checkpoint.c \ - tests/test_dump_verify_io.c + tests/test_dump_verify_io.c \ + tests/test_neo4j.c TEST_CYPHER_SRCS = \ tests/test_cypher.c @@ -1392,7 +1396,7 @@ SYSROOT = $(shell xcrun --show-sdk-path 2>/dev/null) SYSROOT_FLAG = $(if $(SYSROOT),-isysroot $(SYSROOT),) # Our source files (excluding vendored, grammars, tree-sitter runtime) -LINT_SRCS = $(FOUNDATION_SRCS) $(STORE_SRCS) $(CYPHER_SRCS) $(MCP_SRCS) $(DAEMON_SRCS) \ +LINT_SRCS = $(FOUNDATION_SRCS) $(STORE_SRCS) $(CYPHER_SRCS) $(NEO4J_SRCS) $(MCP_SRCS) $(DAEMON_SRCS) \ $(DISCOVER_SRCS) $(GRAPH_BUFFER_SRCS) $(PIPELINE_SRCS) $(SIMHASH_SRCS) $(SEMANTIC_SRCS) \ $(TRACES_SRCS) $(WATCHER_SRCS) $(CLI_SRCS) $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) \ $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) $(MAIN_SRC) diff --git a/README.md b/README.md index 7cdac83bf1..244072f2af 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,13 @@ # codebase-memory-mcp: local code intelligence for AI agents via MCP +> **Fork notice.** This is [haiderRizvii/codebase-memory-mcp](https://github.com/haiderRizvii/codebase-memory-mcp), a fork of +> [DeusData/codebase-memory-mcp](https://github.com/DeusData/codebase-memory-mcp) that adds a +> [Neo4j graph backend](#neo4j-graph-backend) and [Docker-based graph builds](docker/graph/README.md). +> The install commands below install **this fork**, and `codebase-memory-mcp update` keeps +> updating from it. Releases are built by `.github/workflows/fork-release.yml`; unlike +> upstream releases they are not VirusTotal-scanned unless that workflow's `virustotal` +> option is used. The badges below describe the upstream project. + [![GitHub Release](https://img.shields.io/github/v/release/DeusData/codebase-memory-mcp?style=flat&color=blue)](https://github.com/DeusData/codebase-memory-mcp/releases/latest) [![License](https://img.shields.io/badge/license-MIT-green)](LICENSE) [![CI](https://img.shields.io/github/actions/workflow/status/DeusData/codebase-memory-mcp/release.yml?branch=main&label=main%20CI)](https://github.com/DeusData/codebase-memory-mcp/actions/workflows/release.yml) @@ -70,7 +78,7 @@ The high-resolution GIFs show CBM Lens Beta on 20 October 2026; the still images **Install the latest release** (macOS / Linux): ```bash -curl -fsSL https://raw.githubusercontent.com/DeusData/codebase-memory-mcp/main/install.sh | bash +curl -fsSL https://raw.githubusercontent.com/haiderRizvii/codebase-memory-mcp/main/install.sh | bash ``` The standard native release includes CBM Lens. Start it after installation with `codebase-memory-mcp daemon start --open --port=9749`. The [Nix packages](#nix-flake) and [source builds](#build-from-source) also offer builds without the UI. @@ -78,7 +86,7 @@ The standard native release includes CBM Lens. Start it after installation with **Windows** (PowerShell): ```powershell # 1. Download the installer -Invoke-WebRequest -Uri https://raw.githubusercontent.com/DeusData/codebase-memory-mcp/main/install.ps1 -OutFile install.ps1 +Invoke-WebRequest -Uri https://raw.githubusercontent.com/haiderRizvii/codebase-memory-mcp/main/install.ps1 -OutFile install.ps1 # 2. (Optional but recommended) Inspect the script notepad install.ps1 @@ -108,7 +116,7 @@ Restart your coding agent. Say **"Index this project"**: done.
Manual install -1. **Download** the archive for your platform from the [latest release](https://github.com/DeusData/codebase-memory-mcp/releases/latest): +1. **Download** the archive for your platform from the [latest release](https://github.com/haiderRizvii/codebase-memory-mcp/releases/latest): - `codebase-memory-mcp--.tar.gz` (macOS/Linux) or `.zip` (Windows) 2. **Extract and install** (each archive includes `install.sh` or `install.ps1`): @@ -438,6 +446,29 @@ Share a compressed graph snapshot so teammates can reuse an indexed baseline. The artifact provides a shared baseline while each developer keeps a local query database. +## Neo4j Graph Backend + +Store a project's graph in Neo4j and have the graph tools read it from there. + +```bash +codebase-memory-mcp cli index_repository --repo-path /path/to/repo \ + --graph-db http://neo4j-host:7474 --graph-user neo4j --graph-password '***' \ + [--graph-database neo4j] +``` + +- **Transport**: Neo4j's HTTP API (`http://host:7474`, or `https://host:7473` on Windows builds). `bolt://` and `neo4j://` URLs are refused with a hint to use the HTTP port. Neo4j 4.4 and 5.x, Community or Enterprise. +- **Credentials**: flags win; otherwise `NEO4J_USER` (or `NEO4J_USERNAME`), `NEO4J_PASSWORD` and `NEO4J_DATABASE` are used, defaulting to user `neo4j` and database `neo4j`. Indexing fails fast, before any work starts, when Neo4j is unreachable or rejects the login. +- **Password handling**: the password is never written to disk and is never put on the index worker's command line. The daemon that ran the index keeps it in memory for later reads. After a daemon restart, reads need `NEO4J_PASSWORD` in the environment of the MCP server (or of whichever session starts the daemon); see the note under Environment Variables. +- **What lives where**: indexing still builds the local SQLite index, which serves incremental re-indexing, coverage, ADRs, full-text and semantic search. After every successful publish (a full index, an incremental update, or the watcher's auto-refresh), the graph is mirrored into Neo4j. The marker `/.db.graph.json` (URL, user and database, no password) keeps later re-indexes syncing to the same target with no flags. +- **Reads**: the graph primitives come from Neo4j. These are node lookups by id, qualified name, name, label and file, edge lookups, degrees, node/edge counts, and every BFS (shortest-path, multi-seed, and the trail expansion behind variable-length Cypher). That covers `query_graph`, `trace_path`, and the lookups inside `get_code_snippet`, `index_status` and `detect_changes`. Some reads still use the local copy, which holds exactly the same generation: QN-suffix and overload matching, `search_graph` (filters, BM25, semantic), `get_graph_schema`, `get_architecture`, the impact walk, and the UI. +- **Consistency**: each sync writes a new generation, flips `(:CbmProject {name}).gen`, then deletes the old one, so readers never see a half-written graph. If Neo4j is down, rejects the login, or holds an older generation (a failed sync), reads fall back to the identical local copy. `index_status` reports `graph_backend.status` as `serving` or as `sqlite_fallback` with the reason. +- **Native Cypher**: `query_graph` with `native: true` runs the query in Neo4j itself (read-only; write clauses and `CALL` are refused), with `$project`, `$db` and `$gen` bound: + ```cypher + MATCH (f:Function {db:$db, gen:$gen})-[:CALLS]->(g) RETURN f.name, count(g) AS fan_out ORDER BY fan_out DESC LIMIT 10 + ``` +- **Model**: nodes are `(:CbmNode:
@@ -991,6 +1022,7 @@ is preserved. See [Index resource limits](docs/INDEX_RESOURCE_LIMITS.md). | `CBM_LOG_LEVEL` | role-aware | Set the minimum log level. Thin MCP/CLI/hook frontends default to `warn`; the detached daemon and its supervised index workers default to `info` so lifecycle and liveness records remain available. Accepted values (case-insensitive): `debug`, `info`, `warn`, `error`, `none`; or their numeric equivalents `0` to `4`. A physical worker retains INFO liveness records even under a stricter override because its private log drives the supervisor's no-progress timeout. Frontend messages go to that session's stderr; detached daemon events go to `${CBM_CACHE_DIR}/logs/cbm-daemon.log`. Stdout is reserved for MCP JSON-RPC. | | `CBM_WORKERS` | *(detected)* | Override the parallel-indexing worker count returned by `cbm_default_worker_count`. Useful inside containers where `sysconf(_SC_NPROCESSORS_ONLN)` reports host CPUs rather than the cgroup's effective quota. Range 1 to 256; invalid values are ignored with a warning. | | `CBM_MEM_BUDGET_MB` | *(detected)* | Override the in-memory graph budget with an explicit cap in MiB, taking precedence over the derived default (`ram_fraction × total_RAM`, adjusted for available memory and daemon worker caps). Useful on bare-metal hosts without a cgroup limit, or to pin a budget *below* the cgroup limit so headroom is left for sibling processes. Must be a positive integer; it is clamped to detected total RAM (logged as `mem.budget.clamped`), and non-numeric or non-positive values are ignored with a warning (`mem.budget.env.invalid`). | +| `NEO4J_USER` / `NEO4J_PASSWORD` / `NEO4J_DATABASE` | `neo4j` / *(unset)* / `neo4j` | Fallback credentials for the [Neo4j graph backend](#neo4j-graph-backend) when `index_repository` is called without `graph_user` / `graph_password` / `graph_database`. `NEO4J_PASSWORD` is also what lets a restarted daemon read a Neo4j-backed project. | | `CBM_DUMP_VERIFY_MIN_RATIO` | `0.5` | After indexing, compare persisted SQLite node count to the in-memory dump count. When persisted nodes fall below this fraction of committed nodes (and committed > 50), `index_repository` returns `status:"degraded"` instead of silent `indexed`. Range 0 to 1; set `0` to disable. Invalid values are ignored with a warning. | Environment used by daemon-owned components (such as diagnostics, daemon logging, and process-wide indexing resource limits) is captured from the first daemon-backed session that starts the daemon. Later sessions join that process and cannot replace those values. To change them, close all daemon-backed sessions, update the relevant agent configurations consistently, and restart a session. `CBM_ALLOWED_ROOT` remains session-specific, a conflicting `CBM_CACHE_DIR` is rejected, and a one-shot CLI command is not exempt from the rule above: it connects to the coordination daemon like any other session, starting one if none is running. Its own environment becomes the captured daemon-owned environment only when its invocation is the one that starts the daemon; joining an already-running daemon, it inherits that daemon's already-captured values instead. diff --git a/docker/graph/Dockerfile b/docker/graph/Dockerfile new file mode 100644 index 0000000000..1dda804819 --- /dev/null +++ b/docker/graph/Dockerfile @@ -0,0 +1,33 @@ +# cbm-graph: build a repository's code graph into Neo4j from a container. +# +# docker build -f docker/graph/Dockerfile -t cbm-graph:local . +# +# See docker/graph/README.md for running it (cbm-graph.ps1 on Windows). + +# ── Build ──────────────────────────────────────────────────────────── +FROM gcc:14 AS build +RUN apt-get update -qq \ + && apt-get install -y -qq --no-install-recommends zlib1g-dev \ + && rm -rf /var/lib/apt/lists/* +WORKDIR /src +COPY . . +RUN make -f Makefile.cbm cbm BUILD_DIR=/build -j"$(nproc)" \ + && /build/codebase-memory-mcp --version + +# ── Runtime ────────────────────────────────────────────────────────── +# gcc:14 is Debian 13; the binary needs its newer libstdc++/libgcc_s. +FROM debian:trixie-slim +RUN apt-get update -qq \ + && apt-get install -y -qq --no-install-recommends git rsync zlib1g ca-certificates curl \ + && rm -rf /var/lib/apt/lists/* +COPY --from=build /usr/local/lib64/libstdc++.so.6* /usr/local/lib64/libgcc_s.so.1 /usr/local/lib64/ +RUN echo /usr/local/lib64 > /etc/ld.so.conf.d/gcc14.conf && ldconfig +COPY --from=build /build/codebase-memory-mcp /usr/local/bin/codebase-memory-mcp +COPY docker/graph/cbm-graph.sh /usr/local/bin/cbm-graph +RUN sed -i 's/\r$//' /usr/local/bin/cbm-graph && chmod 0755 /usr/local/bin/cbm-graph + +# Index cache (SQLite sidecars + working copies) lives in /data: mount a +# named volume there so re-runs are incremental. +ENV HOME=/data/home +VOLUME ["/data"] +ENTRYPOINT ["cbm-graph"] diff --git a/docker/graph/README.md b/docker/graph/README.md new file mode 100644 index 0000000000..da38f59e9c --- /dev/null +++ b/docker/graph/README.md @@ -0,0 +1,38 @@ +# cbm-graph: code graphs into Neo4j from Docker + +Builds a repository's code graph and stores it in Neo4j. No local C toolchain needed. + +## One-time setup + +```powershell +# 1. Build the image (from the fork root, ~10-15 min) +cd D:\ai-skills\codebase-memory-mcp-fork +docker build -f docker/graph/Dockerfile -t cbm-graph:local . + +# 2. Store the Neo4j password for new terminals +setx NEO4J_PASSWORD "ChangeMe_Fleet123" +``` + +## Graph a repository + +```powershell +D:\ai-skills\codebase-memory-mcp-fork\docker\graph\cbm-graph.ps1 -RepoPath "D:\path\to\repo" +``` + +| Option | Default | Meaning | +|---|---|---| +| `-RepoPath` | required | Repository folder. It is mounted read-only and never modified. | +| `-Name` | folder name | Project name in Neo4j (`CbmProject.name`, `CbmNode.db`). | +| `-Fresh` | off | Drop the project's graph and rebuild from scratch. Without it, re-runs are incremental. | +| `-IgnoreFile` | `D:\cbm-fleet\.cbmignore` | Extra ignore rules, merged with the repo's own `.cbmignore`. Pass `''` to skip. | +| `-Neo4jUri` | `http://host.docker.internal:7474` | Neo4j HTTP endpoint (not Bolt). | +| `-Neo4jUser` / `-Neo4jDatabase` | `neo4j` / `neo4j` | | + +The script prints node and edge counts plus a Cypher query to paste into Neo4j Browser (http://localhost:7474). + +## How it works + +- The container syncs the repo into the `cbm-graph-data` volume. Paths matched by the ignore rules are skipped. It then writes the merged `.cbmignore` at the copy's root and indexes the copy. Your repo is never touched, and repeat runs only copy what changed. +- The graph is published to Neo4j generation by generation, so a re-run never leaves a half-written graph. +- To remove a project's graph, run in Neo4j Browser: + `MATCH (n:CbmNode {db:''}) DETACH DELETE n` and `MATCH (p:CbmProject {name:''}) DELETE p`. diff --git a/docker/graph/cbm-graph.cmd b/docker/graph/cbm-graph.cmd new file mode 100644 index 0000000000..686468d4b6 --- /dev/null +++ b/docker/graph/cbm-graph.cmd @@ -0,0 +1,6 @@ +@echo off +rem Launcher for cbm-graph.ps1 that works from cmd.exe and PowerShell alike +rem (a .ps1 double-clicked or typed in cmd opens in an editor instead of running). +rem Usage: cbm-graph -RepoPath "D:\path\to\repo" [-Name name] [-Fresh] +powershell.exe -NoProfile -ExecutionPolicy Bypass -File "%~dp0cbm-graph.ps1" %* +exit /b %ERRORLEVEL% diff --git a/docker/graph/cbm-graph.ps1 b/docker/graph/cbm-graph.ps1 new file mode 100644 index 0000000000..be5a12bf7f --- /dev/null +++ b/docker/graph/cbm-graph.ps1 @@ -0,0 +1,69 @@ +<# +.SYNOPSIS + Build a repository's code graph into Neo4j using the cbm-graph Docker image. + +.EXAMPLE + .\cbm-graph.ps1 -RepoPath "D:\batch applications\alias-directory-sync-service" + +.EXAMPLE + .\cbm-graph.ps1 -RepoPath "D:\repos\my-service" -Name my-service -Fresh + +.NOTES + The Neo4j password is read from $env:NEO4J_PASSWORD (set it once with + setx NEO4J_PASSWORD "..." and open a new terminal). It is handed to the + container through the environment, never on a command line. +#> +param( + [Parameter(Mandatory = $true)][string]$RepoPath, + [string]$Name, + [switch]$Fresh, + [string]$IgnoreFile = "D:\cbm-fleet\.cbmignore", + [string]$Neo4jUri = "http://host.docker.internal:7474", + [string]$Neo4jUser = "neo4j", + [string]$Neo4jDatabase = "neo4j", + [string]$Image = "cbm-graph:local", + [string]$CacheVolume = "cbm-graph-data" +) +$ErrorActionPreference = "Stop" + +if (-not (Test-Path -LiteralPath $RepoPath -PathType Container)) { + throw "Repository folder not found: $RepoPath" +} +$RepoPath = (Resolve-Path -LiteralPath $RepoPath).Path +if (-not $env:NEO4J_PASSWORD) { + throw 'Set the Neo4j password first: $env:NEO4J_PASSWORD = "..." (or setx NEO4J_PASSWORD "..." for every new terminal)' +} +if (-not $Name) { + $Name = (Split-Path -Leaf $RepoPath).ToLowerInvariant() -replace '[^a-z0-9._-]+', '-' + $Name = $Name.Trim('-') +} +if ($Name -notmatch '^[A-Za-z0-9._-]+$') { throw "Invalid -Name '$Name'" } + +docker image inspect $Image *> $null +if ($LASTEXITCODE -ne 0) { + throw "Docker image '$Image' not found. Build it from the fork root: docker build -f docker/graph/Dockerfile -t $Image ." +} + +$dockerArgs = @( + "run", "--rm", + "--add-host=host.docker.internal:host-gateway", + "-e", "CBM_NAME=$Name", + "-e", "CBM_FRESH=$(if ($Fresh) { '1' } else { '0' })", + "-e", "NEO4J_URI=$Neo4jUri", + "-e", "NEO4J_USER=$Neo4jUser", + "-e", "NEO4J_DATABASE=$Neo4jDatabase", + "-e", "NEO4J_PASSWORD", + "-v", "${RepoPath}:/input/repo:ro", + "-v", "${CacheVolume}:/data" +) +if ($IgnoreFile) { + if (-not (Test-Path -LiteralPath $IgnoreFile -PathType Leaf)) { + throw "Ignore file not found: $IgnoreFile (pass -IgnoreFile '' to skip it)" + } + $dockerArgs += @("-v", "$((Resolve-Path -LiteralPath $IgnoreFile).Path):/input/cbmignore:ro") +} +$dockerArgs += $Image + +Write-Host "Graphing '$RepoPath' as project '$Name' -> $Neo4jUri" -ForegroundColor Cyan +& docker @dockerArgs +exit $LASTEXITCODE diff --git a/docker/graph/cbm-graph.sh b/docker/graph/cbm-graph.sh new file mode 100644 index 0000000000..d514c34d4c --- /dev/null +++ b/docker/graph/cbm-graph.sh @@ -0,0 +1,85 @@ +#!/bin/bash +# cbm-graph — index the repository mounted at /input/repo into Neo4j. +# +# Mounts: +# /input/repo the repository (read-only is fine; it is never modified) +# /input/cbmignore optional extra .cbmignore (e.g. a fleet-wide one) +# /data named volume: index cache + working copies (incremental) +# Environment: +# CBM_NAME project name (required) +# CBM_FRESH=1 drop the project first and rebuild from scratch +# NEO4J_URI default http://host.docker.internal:7474 +# NEO4J_USER default neo4j +# NEO4J_PASSWORD required +# NEO4J_DATABASE default neo4j +set -euo pipefail + +die() { echo "cbm-graph: $*" >&2; exit 2; } + +NAME=${CBM_NAME:-} +[ -n "$NAME" ] || die "CBM_NAME is required" +[[ "$NAME" =~ ^[A-Za-z0-9._-]+$ ]] || die "CBM_NAME may contain only letters, digits, '.', '_' and '-'" +[ -d /input/repo ] || die "mount the repository at /input/repo" +[ -n "${NEO4J_PASSWORD:-}" ] || die "NEO4J_PASSWORD is required" +NEO4J_URI=${NEO4J_URI:-http://host.docker.internal:7474} +NEO4J_USER=${NEO4J_USER:-neo4j} +NEO4J_DATABASE=${NEO4J_DATABASE:-neo4j} + +WORK=/data/work/$NAME +mkdir -p "$HOME" "$WORK" + +# ── Working copy ───────────────────────────────────────────────────── +# The indexer reads .cbmignore only from the repo root, and the source must +# stay untouched, so index a synced copy with the extra rules merged in. +# Paths an ignore rule excludes are also skipped by the copy (a jar-heavy +# repo is not copied for nothing) -- unless a rule uses '!' negation, whose +# gitignore meaning rsync cannot reproduce; then everything is copied and +# only the indexer applies the rules. +IGNORE_RULES=$(mktemp) +[ -f /input/repo/.cbmignore ] && cat /input/repo/.cbmignore >> "$IGNORE_RULES" +if [ -f /input/cbmignore ]; then + printf '\n# --- merged from the mounted cbmignore ---\n' >> "$IGNORE_RULES" + tr -d '\r' < /input/cbmignore >> "$IGNORE_RULES" +fi +RSYNC_ARGS=(-a --delete --delete-excluded --include='/.git/***' --exclude='/.cbmignore') +if ! grep -qE '^[[:space:]]*!' "$IGNORE_RULES"; then + # gitignore trims trailing blanks ('target ' means 'target'); rsync would + # take them literally. + RSYNC_RULES=$(mktemp) + sed 's/[[:space:]]*$//' "$IGNORE_RULES" > "$RSYNC_RULES" + RSYNC_ARGS+=(--exclude='node_modules/' --exclude-from="$RSYNC_RULES") +fi +echo "== syncing working copy" +rsync "${RSYNC_ARGS[@]}" /input/repo/ "$WORK/" +cp "$IGNORE_RULES" "$WORK/.cbmignore" +rm -f "$IGNORE_RULES" +if [ -d "$WORK/.git" ]; then + git config --global --add safe.directory "$WORK" + grep -qx '.cbmignore' "$WORK/.git/info/exclude" 2>/dev/null || + { mkdir -p "$WORK/.git/info"; echo '.cbmignore' >> "$WORK/.git/info/exclude"; } +fi + +# ── Index ──────────────────────────────────────────────────────────── +if [ "${CBM_FRESH:-0}" = 1 ]; then + echo "== fresh: dropping previous graph of $NAME" + codebase-memory-mcp cli delete_project --project "$NAME" || true +fi + +echo "== indexing $NAME into $NEO4J_URI (database $NEO4J_DATABASE)" +OUT=$(codebase-memory-mcp cli index_repository --repo-path "$WORK" --name "$NAME" \ + --graph-db "$NEO4J_URI" --graph-user "$NEO4J_USER" --graph-database "$NEO4J_DATABASE" 2>&1) || true +STATUS=$(printf '%s' "$OUT" | grep -o '"status":"[^"]*"' | tail -1 | cut -d'"' -f4) +if [ "$STATUS" != "indexed" ] && [ "$STATUS" != "degraded" ]; then + printf '%s\n' "$OUT" >&2 + die "indexing failed (status: ${STATUS:-unknown})" +fi + +echo "== done: $STATUS" +codebase-memory-mcp cli index_status --project "$NAME" 2>&1 | + awk '/^[a-z_]+:/ { in_gb = ($1 == "graph_backend:") } + /^(project|nodes|edges|status|indexed_at):/ || in_gb' | head -14 +cat <(m) RETURN n,r,m LIMIT 300 +EOF diff --git a/install.ps1 b/install.ps1 index abd203fb25..4321a725f8 100644 --- a/install.ps1 +++ b/install.ps1 @@ -25,7 +25,7 @@ if ([Environment]::OSVersion.Version.Build -ge 20348 -and [Net.ServicePointManager]::SecurityProtocol = $CbmProtocols Add-Type -AssemblyName System.Net.Http -$Repo = "DeusData/codebase-memory-mcp" +$Repo = "haiderRizvii/codebase-memory-mcp" $InstallDir = "$env:LOCALAPPDATA\Programs\codebase-memory-mcp" $BinName = "codebase-memory-mcp.exe" $WindowsArchiveNames = @( diff --git a/install.sh b/install.sh index 9f4b4ec999..e440d35298 100755 --- a/install.sh +++ b/install.sh @@ -4,7 +4,7 @@ set -euo pipefail # install.sh — One-line installer for codebase-memory-mcp. # # Usage: -# curl -fsSL https://raw.githubusercontent.com/DeusData/codebase-memory-mcp/main/install.sh | bash +# curl -fsSL https://raw.githubusercontent.com/haiderRizvii/codebase-memory-mcp/main/install.sh | bash # curl -fsSL ... | bash -s -- --dir /path # Custom install directory # # Environment: @@ -16,7 +16,7 @@ set -euo pipefail # called because the final line hasn't arrived yet. main() { -REPO="DeusData/codebase-memory-mcp" +REPO="haiderRizvii/codebase-memory-mcp" INSTALL_DIR="$HOME/.local/bin" SKIP_CONFIG=false CLIENTS_SET=false diff --git a/scripts/security-allowlist.txt b/scripts/security-allowlist.txt index 720c602112..414324af7a 100644 --- a/scripts/security-allowlist.txt +++ b/scripts/security-allowlist.txt @@ -25,6 +25,14 @@ NETWORK:src/daemon/ipc.c:connect:1:unix:local_socket_connect:single sockaddr_un- NETWORK:src/main.c:socket:1:loopback-v4:main_daemon_ctl_ui_endpoint_ready:single AF_INET stream client for authenticated daemon UI readiness NETWORK:src/main.c:connect:1:loopback-v4:main_daemon_ctl_socket_connected:single sockaddr_in connect whose address is hardcoded to 127.0.0.1 and whose response must prove the daemon generation HMAC +# ── Neo4j graph backend: the one outbound TCP client (opt-in) ────────────── +# Connects only when the user passes index_repository graph_db (or a project +# was indexed with it), to that URL's host:port. configured-tcp is pinned by +# the source audit to src/neo4j/neo4j_http.c and to a getaddrinfo() walk over +# the parsed graph_db host with a stream-socket hint. +NETWORK:src/neo4j/neo4j_http.c:socket:1:configured-tcp:n4_connect:stream socket for one getaddrinfo() result of the user-configured graph_db host +NETWORK:src/neo4j/neo4j_http.c:connect:1:configured-tcp:n4_connect:connect to that same resolved graph_db address (Neo4j HTTP API) + # ── Reviewed count-bounded file writes: private or caller-selected paths ─── # Format: WRITE:file:expected-count:justification WRITE:src/main.c:3:supervised worker response path; cache-scoped hook notice marker; test-seam-only browser-open marker at the caller-supplied test path @@ -38,6 +46,7 @@ WRITE:src/mcp/test_impact_artifact.c:1:receipt staging file in the caller-select WRITE:src/mcp/test_impact_engine.c:1:merge-base configuration staged inside the engine's private working directory WRITE:src/mcp/test_impact_engine_artifact.c:1:reviewed coverage bytes copied into the newly-created output bundle WRITE:src/mcp/test_impact_git.c:1:bounded Git batch stdin staged in the private capture directory +WRITE:src/neo4j/neo4j_sync.c:1:graph-backend marker .db.graph.json (URL, user, database; never the password) staged as .tmp beside the project database in the cache directory, then atomically renamed # ── CLI: update command (user-initiated, interactive) ────────────────────── src/cli/cli.c:cbm_popen:sha256 checksum verification (update cmd) @@ -84,4 +93,5 @@ URL:https://github.com/DeusData/codebase-memory-mcp/releases/latest:version chec URL:http://127.0.0.1:UI server binding (localhost only) URL:https://www.sqlite.org/c3ref/c_checkpoint_full.html:sqlite WAL checkpoint API doc reference (comment only, not a network call) URL:https://github.com/DeusData/codebase-memory-mcp:project repository self-reference in update/star notice (src/mcp/mcp.c) +URL:https://github.com/haiderRizvii/codebase-memory-mcp/issues/new:fork issue link in the UI about panel (src/ui/http_server.c) URL:https://%s:GitHub blob-URL construction in src/ui/http_server.c cbm_ui_git_web_base — https-forces the repo web base for frontend deep-links (not a network call) diff --git a/scripts/security-network-source-audit.py b/scripts/security-network-source-audit.py index 5955cc622a..53783c28ec 100644 --- a/scripts/security-network-source-audit.py +++ b/scripts/security-network-source-audit.py @@ -189,6 +189,36 @@ def _fullmatch(pattern: str, call: Call) -> bool: ) +# configured-tcp: the one outbound TCP client, the Neo4j graph backend +# (src/neo4j/neo4j_http.c). It connects only when the user configures a +# graph_db URL, to exactly that URL's host and port, so the allowance is +# pinned to that file and to a getaddrinfo() walk over the parsed URL host: +# both calls must take their arguments from that result list, and the hints +# must ask for a stream socket. +CONFIGURED_TCP_FILES = ("src/neo4j/neo4j_http.c",) +SOCKET_CONFIGURED_TCP = ( + r"socket\s*\(\s*ai\s*->\s*ai_family\s*,\s*ai\s*->\s*ai_socktype\s*," + r"\s*ai\s*->\s*ai_protocol\s*\)" +) +CONNECT_CONFIGURED_TCP = ( + r"connect\s*\(\s*s\s*,\s*ai\s*->\s*ai_addr\s*,\s*\(\s*int\s*\)\s*" + r"ai\s*->\s*ai_addrlen\s*\)" +) + + +def _configured_tcp_witness(masked: str, call: Call, scope: FunctionRange) -> bool: + before = masked[scope.body_start + 1 : call.start] + witness = re.compile( + r"\bhints\s*\.\s*ai_socktype\s*=\s*SOCK_STREAM\s*;.*?" + r"\bgetaddrinfo\s*\(\s*h\s*->\s*url\s*\.\s*host\s*,\s*port_s\s*,\s*" + r"&\s*hints\s*,\s*&\s*res\s*\).*?" + r"\bfor\s*\(\s*struct\s+addrinfo\s*\*\s*ai\s*=\s*res\s*;\s*ai\s*;\s*" + r"ai\s*=\s*ai\s*->\s*ai_next\s*\)", + re.DOTALL, + ) + return witness.search(before) is not None + + def _loopback_connect_witness(masked: str, call: Call, scope: FunctionRange) -> bool: body = masked[scope.body_start + 1 : scope.body_end - 1] call_offset = call.start - scope.body_start - 1 @@ -226,6 +256,17 @@ def _semantic_ok(masked: str, calls: Sequence[Call], allowance: Allowance) -> bo return ( re.search(r"const\s+struct\s+sockaddr_un\s*\*\s*address", header) is not None ) + if allowance.transport == "configured-tcp": + if allowance.file not in CONFIGURED_TCP_FILES or len(calls) != 1: + return False + pattern = {"socket": SOCKET_CONFIGURED_TCP, "connect": CONNECT_CONFIGURED_TCP}.get( + allowance.function + ) + return ( + pattern is not None + and _fullmatch(pattern, calls[0]) + and _configured_tcp_witness(masked, calls[0], scopes[0]) + ) if allowance.transport == "loopback-v4" and allowance.function == "socket": return all(_fullmatch(SOCKET_LOOPBACK, call) for call in calls) if allowance.transport == "loopback-v4" and allowance.function == "connect": @@ -424,9 +465,59 @@ def _self_test() -> None: ) +def _self_test_configured_tcp() -> None: + client = r''' +static int n4_connect(cbm_neo4j_http_t *h, char *err, size_t err_sz) { + struct addrinfo hints; + memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + struct addrinfo *res = NULL; + if (getaddrinfo(h->url.host, port_s, &hints, &res) != 0 || !res) { + return -1; + } + for (struct addrinfo *ai = res; ai; ai = ai->ai_next) { + s = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol); + if (connect(s, ai->ai_addr, (int)ai->ai_addrlen) == 0) { + break; + } + } + return 0; +} +''' + + def allowance(function: str, file: str = "src/neo4j/neo4j_http.c") -> Allowance: + return Allowance(file, function, 1, "configured-tcp", "n4_connect", "self-test") + + def ok(source: str, function: str, file: str = "src/neo4j/neo4j_http.c") -> bool: + masked = _mask_non_code(source) + calls = [call for call in _scan_calls(masked) if call.name == function] + return _semantic_ok(masked, calls, allowance(function, file)) + + _require(ok(client, "socket"), "configured-tcp socket witness was rejected") + _require(ok(client, "connect"), "configured-tcp connect witness was rejected") + _require( + not ok(client, "connect", "src/mcp/mcp.c"), + "configured-tcp was accepted outside the Neo4j client", + ) + _require( + not ok(client.replace("getaddrinfo(h->url.host", "getaddrinfo(other_host"), "connect"), + "configured-tcp accepted a host that is not the configured URL host", + ) + _require( + not ok(client.replace("SOCK_STREAM", "SOCK_DGRAM"), "socket"), + "configured-tcp accepted a non-stream socket", + ) + _require( + not ok(client.replace("connect(s, ai->ai_addr", "connect(s, other_addr"), "connect"), + "configured-tcp accepted an address outside the resolved list", + ) + + def main(argv: Sequence[str]) -> int: try: _self_test() + _self_test_configured_tcp() except SelfTestFailure as error: print(f"BLOCKED: network source audit self-test failed: {error}", file=sys.stderr) return 1 diff --git a/scripts/security-strings.sh b/scripts/security-strings.sh index 9d6587e5b6..22c362be06 100755 --- a/scripts/security-strings.sh +++ b/scripts/security-strings.sh @@ -115,6 +115,8 @@ echo "--- URL audit ---" ALLOWED_URLS=( "https://api.github.com/repos/DeusData/codebase-memory-mcp" "https://github.com/DeusData/codebase-memory-mcp" + # This fork's own repository (installers, update handoff, UI issue link). + "https://github.com/haiderRizvii/codebase-memory-mcp" # Our own org root and documentation site: the .mcpb bundle manifest carries # them as homepage/documentation fields, and the graph UI links the docs. "https://github.com/DeusData" diff --git a/scripts/setup-windows.ps1 b/scripts/setup-windows.ps1 index 4a836f95e6..270ec2c210 100644 --- a/scripts/setup-windows.ps1 +++ b/scripts/setup-windows.ps1 @@ -9,7 +9,7 @@ param( $ErrorActionPreference = "Stop" -$Repo = "DeusData/codebase-memory-mcp" +$Repo = "haiderRizvii/codebase-memory-mcp" $BinaryName = "codebase-memory-mcp" $InstallDir = Join-Path $env:LOCALAPPDATA "codebase-memory-mcp" diff --git a/scripts/setup.sh b/scripts/setup.sh index ee1a4d4247..dbef73e6d1 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -5,7 +5,7 @@ set -euo pipefail # Default: install the latest pre-built binary through install.sh # --from-source: build from source (requires Go + C compiler) -REPO="DeusData/codebase-memory-mcp" +REPO="haiderRizvii/codebase-memory-mcp" INSTALL_DIR="$HOME/.local/bin" BINARY_NAME="codebase-memory-mcp" SOURCE_DIR="$HOME/.local/share/codebase-memory-mcp" diff --git a/src/mcp/index_supervisor.c b/src/mcp/index_supervisor.c index 223945d3c3..b6643b4342 100644 --- a/src/mcp/index_supervisor.c +++ b/src/mcp/index_supervisor.c @@ -10,7 +10,9 @@ #include "foundation/log.h" #include "foundation/platform.h" /* cbm_safe_getenv, path normalization */ #include "foundation/profile.h" /* cbm_profile_active (keep worker log under CBM_PROFILE) */ +#include "neo4j/neo4j_client.h" /* graph_password hand-off */ #include "ui/http_server.h" /* cbm_http_server_resolve_binary_path */ +#include "yyjson/yyjson.h" #include #include @@ -739,6 +741,60 @@ size_t cbm_index_worker_job_memory_limit(size_t memory_budget_bytes) { return memory_budget_bytes > SIZE_MAX - headroom ? SIZE_MAX : memory_budget_bytes + headroom; } +/* Move index_repository's graph_password out of the worker's argv (visible in + * the process list) into a CBM_NEO4J_PASSWORD=... environment entry for that + * child only. The spawning process (the daemon serving later reads) also + * remembers the credential so it can open the project's Neo4j graph. Returns + * the args to pass (args_json itself when there is no password, otherwise a + * malloc'd copy in *owned_args) and fills env_entry (malloc'd, or NULL). */ +static const char *worker_split_graph_password(const char *args_json, char **owned_args, + char **env_entry) { + *owned_args = NULL; + *env_entry = NULL; + yyjson_doc *doc = yyjson_read(args_json, strlen(args_json), 0); + yyjson_val *root = doc ? yyjson_doc_get_root(doc) : NULL; + const char *pw = yyjson_get_str(yyjson_obj_get(root, "graph_password")); + if (!pw) { + yyjson_doc_free(doc); + return args_json; + } + const char *uri = yyjson_get_str(yyjson_obj_get(root, "graph_db")); + if (uri && uri[0]) { + cbm_neo4j_config_t cfg; + char err[CBM_SZ_256]; + if (cbm_neo4j_config_resolve(uri, yyjson_get_str(yyjson_obj_get(root, "graph_user")), pw, + yyjson_get_str(yyjson_obj_get(root, "graph_database")), &cfg, + err, sizeof(err)) == 0) { + cbm_neo4j_config_remember(&cfg); + } + cbm_neo4j_config_wipe(&cfg); + } + size_t env_len = strlen(CBM_NEO4J_PASSWORD_ENV) + strlen(pw) + 2; + *env_entry = malloc(env_len); + if (*env_entry) { + (void)snprintf(*env_entry, env_len, "%s=%s", CBM_NEO4J_PASSWORD_ENV, pw); + } + yyjson_mut_doc *copy = yyjson_doc_mut_copy(doc, NULL); + yyjson_doc_free(doc); + yyjson_mut_val *croot = copy ? yyjson_mut_doc_get_root(copy) : NULL; + while (croot && yyjson_mut_obj_get(croot, "graph_password")) { + (void)yyjson_mut_obj_remove_key(croot, "graph_password"); + } + *owned_args = croot ? yyjson_mut_write(copy, 0, NULL) : NULL; + yyjson_mut_doc_free(copy); + return *owned_args ? *owned_args : args_json; +} + +static void worker_free_secret(char *s) { + if (s) { + volatile char *p = s; + while (*p) { + *p++ = 0; + } + free(s); + } +} + int cbm_index_worker_start_with_log(const char *args_json, size_t memory_budget_bytes, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, @@ -799,7 +855,9 @@ int cbm_index_worker_start_with_log(const char *args_json, size_t memory_budget_ argv[argc++] = CBM_INDEX_WORKER_BUILD_ARG; argv[argc++] = build_fingerprint; argv[argc++] = "index_repository"; - argv[argc++] = args_json; + char *owned_args = NULL; + char *password_env = NULL; + argv[argc++] = worker_split_graph_password(args_json, &owned_args, &password_env); argv[argc++] = "--response-out"; argv[argc++] = handle->response_path; char memory_budget_text[CBM_SZ_32]; @@ -833,7 +891,14 @@ int cbm_index_worker_start_with_log(const char *args_json, size_t memory_budget_ options.quiet_timeout_ms = worker_quiet_timeout_ms(); options.memory_limit_bytes = cbm_index_worker_job_memory_limit(memory_budget_bytes); options.delete_log_on_exit = false; - if (cbm_subprocess_spawn(&options, &handle->process) != 0) { + const char *env_overrides[2] = {password_env, NULL}; + if (password_env) { + options.env_overrides = env_overrides; + } + int spawn_rc = cbm_subprocess_spawn(&options, &handle->process); + free(owned_args); + worker_free_secret(password_env); + if (spawn_rc != 0) { (void)cbm_unlink(handle->response_path); (void)cbm_unlink(handle->log_path); free(handle); diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 5bd02e4e38..4cfe9ed8ca 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -65,6 +65,9 @@ enum { #include #include "cypher/cypher.h" #include "discover/discover.h" +#include "neo4j/neo4j_client.h" +#include "neo4j/neo4j_store.h" +#include "neo4j/neo4j_sync.h" #include "pipeline/pipeline.h" #include "callable_sig.h" /* cbm_qn_callable_base_len */ #include "pipeline/doc_links.h" /* cbm_doclink_reason_name: the reasons the layer writes */ @@ -559,7 +562,12 @@ static const tool_def_t TOOLS[] = { "one-shot cli call that is the only client of a temporary daemon (run daemon start).\"}," "\"status\":{\"type\":\"boolean\",\"default\":false,\"description\":" "\"Report the running or last index job (queued/running/succeeded/failed/cancelled) " - "instead of indexing. Pass the same repo_path (and name, if any) as the index call.\"}" + "instead of indexing. Pass the same repo_path (and name, if any) as the index call.\"}," + "\"graph_db\":{\"type\":\"string\",\"description\":" + "\"Neo4j HTTP endpoint URL (port 7474, not bolt); graph tools read it. sqlite " + "reverts.\"}," + "\"graph_user\":{\"type\":\"string\"},\"graph_password\":{\"type\":\"string\"}," + "\"graph_database\":{\"type\":\"string\"}" "},\"required\":[\"repo_path\"]}"}, {"search_graph", @@ -599,6 +607,8 @@ static const tool_def_t TOOLS[] = { "query\"},\"project\":{\"type\":\"string\"}," "\"graph\":{\"type\":\"string\",\"enum\":[\"code\",\"missed\"],\"default\":\"code\"," "\"description\":\"code graph (default) or missed coverage-gap file tree.\"}," + "\"native\":{\"type\":\"boolean\",\"description\":\"Neo4j projects: raw read-only " + "Neo4j Cypher; $db $gen $project bound.\"}," "\"max_rows\":{\"type\":\"integer\"," "\"description\":" "\"Visible rows (default 200; max 99998); 0 uses the legacy maximum. Evaluation is " @@ -1867,6 +1877,10 @@ struct cbm_mcp_server { * "project not found". */ bool readonly_resolve_hit_corrupt; char readonly_corrupt_project[CBM_SZ_1K]; + /* Set when this request's project is Neo4j-backed but its graph reads + * fell back to the local index; cbm_mcp_handle_tool attaches it to the + * reply so a fallback is never silent. Cleared at request start. */ + char graph_fallback_reason[CBM_SZ_512]; /* Session + auto-index state */ char session_root[CBM_SZ_1K]; /* detected project root path */ @@ -2658,6 +2672,15 @@ static cbm_store_t *resolve_store_internal(cbm_mcp_server_t *srv, const char *pr srv->owns_store = true; free(srv->current_project); srv->current_project = heap_strdup(project); + /* A Neo4j-backed project answers graph reads from Neo4j; the + * SQLite file keeps serving everything else (and the graph too + * when Neo4j is unusable — logged by the attach). */ + char graph_err[CBM_SZ_512]; + if (cbm_neo4j_store_attach(srv->store, path, project, graph_err, sizeof(graph_err)) == + CBM_NEO4J_ATTACH_FALLBACK) { + (void)snprintf(srv->graph_fallback_reason, sizeof(srv->graph_fallback_reason), "%s", + graph_err[0] ? graph_err : "Neo4j unavailable"); + } return srv->store; /* fast path: filename == internal name */ } /* #704: .db exists but its INTERNAL project name differs from @@ -6109,6 +6132,8 @@ static char *handle_query_graph(cbm_mcp_server_t *srv, const char *args) { char *graph_arg = cbm_mcp_get_string_arg(args, "graph"); bool missed_graph = graph_arg && strcmp(graph_arg, "missed") == 0; free(graph_arg); + /* native (Neo4j-backed projects): run the query in Neo4j itself. */ + bool native = cbm_mcp_get_bool_arg(args, "native"); if (!query) { free(cursor_arg); @@ -6157,8 +6182,8 @@ static char *handle_query_graph(cbm_mcp_server_t *srv, const char *args) { } bool generation_available = generation[0] != '\0'; const char *effective_project = project ? project : srv->current_project; - uint64_t params_hash = - query_graph_params_hash(effective_project, query, missed_graph ? "missed" : "code"); + uint64_t params_hash = query_graph_params_hash( + effective_project, query, missed_graph ? "missed" : (native ? "native" : "code")); query_graph_cursor_t cursor = {0}; if (cursor_arg) { if (requested_offset != 0) { @@ -6193,7 +6218,14 @@ static char *handle_query_graph(cbm_mcp_server_t *srv, const char *args) { * Executing at visible_limit+1 corrupts ORDER BY, DISTINCT, aggregation, * and SKIP semantics before serialization gets a chance to trim rows. * Zero preserves the engine's established 100k safety ceiling. */ - int rc = cbm_cypher_execute(store, query, cypher_project, 0, &result); + int rc; + if (native) { + char native_db_path[CBM_SZ_1K]; + project_db_path(effective_project, native_db_path, sizeof(native_db_path)); + rc = cbm_neo4j_native_query(native_db_path, effective_project, query, 0, &result); + } else { + rc = cbm_cypher_execute(store, query, cypher_project, 0, &result); + } if (rc < 0) { char *err_msg = result.error ? result.error : "query execution failed"; @@ -7412,6 +7444,55 @@ static void add_watch_status_json(cbm_mcp_server_t *srv, yyjson_mut_doc *doc, yy yyjson_mut_obj_add_val(doc, root, "watch", watch); } +/* graph_backend: where graph reads come from. For a Neo4j-backed project + * whose Neo4j copy cannot serve right now, status says sqlite_fallback and + * names why (reads stay correct: the local copy is the same graph). */ +static void add_graph_backend_report(yyjson_mut_doc *doc, yyjson_mut_val *root, cbm_store_t *store, + const char *project) { + char path[CBM_SZ_1K]; + project_db_path(project, path, sizeof(path)); + cbm_neo4j_config_t cfg; + char err[CBM_SZ_512] = ""; + char synced[CBM_SZ_128] = ""; + int rc = cbm_neo4j_sidecar_read(path, &cfg, synced, sizeof(synced), err, sizeof(err)); + yyjson_mut_val *gb = yyjson_mut_obj(doc); + if (rc == 1) { + yyjson_mut_obj_add_str(doc, gb, "kind", "sqlite"); + } else { + yyjson_mut_obj_add_str(doc, gb, "kind", "neo4j"); + if (rc == 0) { + yyjson_mut_obj_add_strcpy(doc, gb, "uri", cfg.uri); + yyjson_mut_obj_add_strcpy(doc, gb, "database", cfg.database); + yyjson_mut_obj_add_strcpy(doc, gb, "synced_generation", synced); + cbm_neo4j_config_wipe(&cfg); + /* in_sync: the last generation sent to Neo4j is the one the local + * index publishes now. A background (watcher) re-index whose sync + * failed leaves it false, with last_sync_error naming why. */ + char local[CBM_SZ_128] = ""; + (void)cbm_store_generation(store, local, sizeof(local)); + yyjson_mut_obj_add_strcpy(doc, gb, "local_generation", local); + yyjson_mut_obj_add_bool(doc, gb, "in_sync", local[0] && strcmp(local, synced) == 0); + char last_error[CBM_SZ_512] = ""; + (void)cbm_neo4j_sidecar_field(path, "last_sync_error", last_error, sizeof(last_error)); + if (last_error[0]) { + yyjson_mut_obj_add_strcpy(doc, gb, "last_sync_error", last_error); + } + } + bool attached = cbm_store_is_remote(store); + yyjson_mut_obj_add_str(doc, gb, "status", attached ? "serving" : "sqlite_fallback"); + if (!attached) { + /* Re-run the attach checks on a throwaway store to name the cause. */ + cbm_store_t *probe = cbm_store_open_path_query(path); + if (probe) { + (void)cbm_neo4j_store_attach(probe, path, project, err, sizeof(err)); + cbm_store_close(probe); + } + yyjson_mut_obj_add_strcpy(doc, gb, "reason", err[0] ? err : "unknown"); + } + } + yyjson_mut_obj_add_val(doc, root, "graph_backend", gb); +} + static char *handle_index_status(cbm_mcp_server_t *srv, const char *args) { char *project = get_project_arg(args); cbm_store_t *store = resolve_store(srv, project); @@ -7465,6 +7546,7 @@ static char *handle_index_status(cbm_mcp_server_t *srv, const char *args) { add_git_context_json(doc, root, proj_info.root_path); } } + add_graph_backend_report(doc, root, store, project); add_coverage_report(doc, root, store, project, have_proj_info ? proj_info.indexed_at : NULL, coverage_samples); (void)cbm_cross_repo_add_status_json(doc, root, store, project); @@ -7547,6 +7629,23 @@ static char *handle_delete_project(cbm_mcp_server_t *srv, const char *args) { const char *error_detail = NULL; bool is_error = false; + /* A Neo4j-backed project also loses its mirrored graph. A Neo4j failure + * is reported but does not keep the local index alive. */ + char graph_status[CBM_SZ_512] = ""; + cbm_neo4j_config_t graph_cfg; + char graph_err[CBM_SZ_512] = ""; + if (cbm_neo4j_sidecar_read(path, &graph_cfg, NULL, 0, graph_err, sizeof(graph_err)) == 0) { + cbm_neo4j_t *gc = cbm_neo4j_connect(&graph_cfg, graph_err, sizeof(graph_err)); + if (gc && cbm_neo4j_drop_project(gc, name, graph_err, sizeof(graph_err)) == 0) { + (void)snprintf(graph_status, sizeof(graph_status), "deleted"); + } else { + (void)snprintf(graph_status, sizeof(graph_status), "delete_failed: %s", graph_err); + } + cbm_neo4j_close(gc); + cbm_neo4j_config_wipe(&graph_cfg); + } + (void)cbm_neo4j_sidecar_remove(path); + if (exists) { int rc = cbm_unlink(path); (void)cbm_unlink(wal); @@ -7579,6 +7678,9 @@ static char *handle_delete_project(cbm_mcp_server_t *srv, const char *args) { if (error_detail) { yyjson_mut_obj_add_str(doc, root, "error", error_detail); } + if (graph_status[0]) { + yyjson_mut_obj_add_strcpy(doc, root, "neo4j", graph_status); + } char *json = yy_doc_to_str(doc); yyjson_mut_doc_free(doc); @@ -11938,6 +12040,62 @@ static void index_doc_candidates_apply(cbm_pipeline_t *p, index_doc_candidates_t } } +static void index_free_secret(char *s) { + if (s) { + volatile char *p = s; + while (*p) { + *p++ = 0; + } + free(s); + } +} + +/* graph_db / graph_user / graph_password / graph_database: resolve the Neo4j + * target, prove it accepts the credentials before any indexing work, and hand + * it to the pipeline. graph_db "sqlite" / "none" returns the project to the + * local graph. Returns 0, or -1 with err filled. */ +static int index_graph_backend_apply(cbm_pipeline_t *p, const char *args, char *err, + size_t err_sz) { + char *uri = cbm_mcp_get_string_arg(args, "graph_db"); + if (!uri || !uri[0]) { + free(uri); + return 0; + } + if (strcmp(uri, "sqlite") == 0 || strcmp(uri, "none") == 0) { + free(uri); + cbm_pipeline_disable_graph_backend(p); + return 0; + } + char *user = cbm_mcp_get_string_arg(args, "graph_user"); + char *password = cbm_mcp_get_string_arg(args, "graph_password"); + char *database = cbm_mcp_get_string_arg(args, "graph_database"); + cbm_neo4j_config_t cfg; + int rc = cbm_neo4j_config_resolve(uri, user, password, database, &cfg, err, err_sz); + free(uri); + free(user); + index_free_secret(password); + free(database); + if (rc == 0) { + cbm_neo4j_t *c = cbm_neo4j_connect(&cfg, err, err_sz); + if (!c) { + rc = -1; + } else { + if (cbm_neo4j_ping(c) != 0) { + (void)snprintf(err, err_sz, "cannot use Neo4j at %s: %s", cfg.uri, + cbm_neo4j_error(c)); + rc = -1; + } + cbm_neo4j_close(c); + } + } + if (rc == 0) { + cbm_neo4j_config_remember(&cfg); + cbm_pipeline_set_graph_backend(p, &cfg); + } + cbm_neo4j_config_wipe(&cfg); + return rc; +} + /* Preserve every index option while replacing all caller-supplied repo_path * keys with the one canonical path that was actually authorized. A non-empty * project_name likewise replaces any caller "name" (#2134: the project key the @@ -12460,6 +12618,14 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { cbm_pipeline_set_persistence(p, persistence); cbm_pipeline_set_resource_policy(p, &resource_policy); index_doc_candidates_apply(p, doc_candidates); + char graph_err[CBM_SZ_1K]; + if (index_graph_backend_apply(p, args, graph_err, sizeof(graph_err)) != 0) { + cbm_pipeline_free(p); + mcp_project_mutation_end(srv, mutation_project); + free(mutation_project); + free(repo_path); + return cbm_mcp_text_result(graph_err, true); + } char *project_name = heap_strdup(cbm_pipeline_project_name(p)); @@ -12617,6 +12783,15 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { serving_index_was_servable && project_db_is_servable(project_name, serving_db_path)); yyjson_mut_obj_add_strcpy(doc, root, "message", message); + } else if (rc == CBM_PIPELINE_GRAPH_SYNC_FAILED) { + char hint[CBM_SZ_2K]; + (void)snprintf(hint, sizeof(hint), + "The local index was published, but mirroring it into Neo4j failed: %s. " + "Graph tools keep serving the previous Neo4j generation (if any). Fix " + "the Neo4j connection and re-run index_repository with graph_db.", + cbm_pipeline_graph_error(p)); + yyjson_mut_obj_add_str(doc, root, "status", "graph_sync_failed"); + yyjson_mut_obj_add_strcpy(doc, root, "hint", hint); } else { yyjson_mut_obj_add_str(doc, root, "status", "error"); /* #1665: a post-publish artifact export failure (read-only repo, etc.) @@ -20605,6 +20780,53 @@ static bool mcp_tool_reads_in_batch(const char *tool_name) { return false; } +/* A Neo4j-backed project whose graph reads fell back to the local index + * (Neo4j unreachable, credentials rejected, or holding another generation): + * the answer is still correct -- the local index is the newest published + * generation and Neo4j only mirrors it -- but the caller must be able to see + * that Neo4j did not serve it. Appends a text notice to content and, for a + * structured reply, a graph_backend object. Takes ownership of result; on any + * failure the original reply is returned unchanged. */ +char *cbm_mcp_result_add_graph_fallback(char *result, const char *reason) { + if (!result) { + return NULL; + } + yyjson_doc *in = yyjson_read(result, strlen(result), 0); + yyjson_mut_doc *doc = in ? yyjson_doc_mut_copy(in, NULL) : NULL; + yyjson_doc_free(in); + yyjson_mut_val *root = doc ? yyjson_mut_doc_get_root(doc) : NULL; + yyjson_mut_val *content = root ? yyjson_mut_obj_get(root, "content") : NULL; + if (!yyjson_mut_is_arr(content)) { + yyjson_mut_doc_free(doc); + return result; + } + char notice[CBM_SZ_1K]; + (void)snprintf(notice, sizeof(notice), + "graph_backend: sqlite_fallback -- this Neo4j-backed project was answered from " + "the local index, not Neo4j (%s). The answer is current; Neo4j is not. Re-run " + "index_repository with graph_db to resync, and see index_status.", + reason); + yyjson_mut_val *item = yyjson_mut_obj(doc); + yyjson_mut_obj_add_str(doc, item, "type", "text"); + yyjson_mut_obj_add_strcpy(doc, item, "text", notice); + yyjson_mut_arr_append(content, item); + yyjson_mut_val *structured = yyjson_mut_obj_get(root, "structuredContent"); + if (yyjson_mut_is_obj(structured) && !yyjson_mut_obj_get(structured, "graph_backend")) { + yyjson_mut_val *gb = yyjson_mut_obj(doc); + yyjson_mut_obj_add_str(doc, gb, "kind", "neo4j"); + yyjson_mut_obj_add_str(doc, gb, "status", "sqlite_fallback"); + yyjson_mut_obj_add_strcpy(doc, gb, "reason", reason); + yyjson_mut_obj_add_val(doc, structured, "graph_backend", gb); + } + char *out = yy_doc_to_str(doc); + yyjson_mut_doc_free(doc); + if (!out) { + return result; + } + free(result); + return out; +} + /* File-backed query stores are request-scoped. Keeping one open between MCP * calls pins an old database generation after another process atomically * replaces the project DB. On Windows it can also prevent that replacement @@ -20642,8 +20864,12 @@ char *cbm_mcp_handle_tool(cbm_mcp_server_t *srv, const char *tool_name, const ch cbm_mem_phase_mark("request.dispatch_tool"); if (srv) { srv->read_batch_tool = mcp_tool_reads_in_batch(tool_name); + srv->graph_fallback_reason[0] = '\0'; } char *result = dispatch_tool(srv, tool_name, args_json); + if (srv && srv->graph_fallback_reason[0]) { + result = cbm_mcp_result_add_graph_fallback(result, srv->graph_fallback_reason); + } if (srv) { /* An embedded store outlives the request: end its batch here. */ srv->read_batch_tool = false; diff --git a/src/mcp/mcp.h b/src/mcp/mcp.h index 3a5fce5ff8..f6167892f3 100644 --- a/src/mcp/mcp.h +++ b/src/mcp/mcp.h @@ -56,6 +56,12 @@ char *cbm_jsonrpc_format_error(int64_t id, int code, const char *message); /* Format an MCP tool result with text content. Returns heap-allocated JSON. */ char *cbm_mcp_text_result(const char *text, bool is_error); +/* Mark a tool reply as answered from the local index although its project is + * Neo4j-backed (graph_backend sqlite_fallback): appends a text notice to + * content and a graph_backend object to structuredContent. Takes ownership of + * result; returns it unchanged when it is not an MCP tool-result object. */ +char *cbm_mcp_result_add_graph_fallback(char *result, const char *reason); + /* Attach a notice to a complete MCP tool result: a JSON-object payload gains a * "notice" key (content text and structuredContent stay in step), a text * payload gains a trailing paragraph. Takes diff --git a/src/neo4j/neo4j_client.c b/src/neo4j/neo4j_client.c new file mode 100644 index 0000000000..32e79a304e --- /dev/null +++ b/src/neo4j/neo4j_client.c @@ -0,0 +1,494 @@ +/* + * neo4j_client.c — Neo4j config, credential cache and Cypher execution. + */ +#include "neo4j/neo4j_client.h" +#include "neo4j/neo4j_http.h" + +#include "foundation/compat_thread.h" +#include "foundation/platform.h" + +#include +#include +#include +#include +#include + +#define N4_DEFAULT_USER "neo4j" +#define N4_DEFAULT_DATABASE "neo4j" +#define N4_CRED_CACHE_MAX 16 + +struct cbm_neo4j { + cbm_neo4j_http_t *http; + char uri[CBM_NEO4J_URI_MAX]; + char commit_path[CBM_NEO4J_DATABASE_MAX + 32]; + char err[CBM_NEO4J_ERR_MAX]; +}; + +/* ── Credential cache ───────────────────────────────────────────── */ + +typedef struct { + char uri[CBM_NEO4J_URI_MAX]; + char user[CBM_NEO4J_USER_MAX]; + char password[CBM_NEO4J_PASSWORD_MAX]; +} n4_cred_t; + +static cbm_mutex_t g_cred_mu; +static atomic_int g_cred_mu_state; /* 0 = uninitialized, 1 = initializing, 2 = ready */ +static n4_cred_t g_creds[N4_CRED_CACHE_MAX]; +static int g_cred_count; + +static void cred_lock(void) { + /* Daemon request threads can race to the first use. */ + int expected = 0; + if (atomic_compare_exchange_strong(&g_cred_mu_state, &expected, 1)) { + cbm_mutex_init(&g_cred_mu); + atomic_store(&g_cred_mu_state, 2); + } else { + while (atomic_load(&g_cred_mu_state) != 2) { + /* another thread is finishing the one-time init */ + } + } + cbm_mutex_lock(&g_cred_mu); +} + +static void copy_str(char *dst, size_t dst_sz, const char *src) { + (void)snprintf(dst, dst_sz, "%s", src ? src : ""); +} + +static bool cred_lookup(const char *uri, const char *user, char *out, size_t out_sz) { + bool found = false; + cred_lock(); + for (int i = 0; i < g_cred_count; i++) { + if (strcmp(g_creds[i].uri, uri) == 0 && strcmp(g_creds[i].user, user) == 0) { + copy_str(out, out_sz, g_creds[i].password); + found = true; + break; + } + } + cbm_mutex_unlock(&g_cred_mu); + return found; +} + +void cbm_neo4j_config_remember(const cbm_neo4j_config_t *cfg) { + if (!cfg || !cfg->uri[0] || !cfg->password[0]) { + return; + } + cred_lock(); + int slot = -1; + for (int i = 0; i < g_cred_count; i++) { + if (strcmp(g_creds[i].uri, cfg->uri) == 0 && strcmp(g_creds[i].user, cfg->user) == 0) { + slot = i; + break; + } + } + if (slot < 0) { + slot = g_cred_count < N4_CRED_CACHE_MAX ? g_cred_count++ : 0; + } + copy_str(g_creds[slot].uri, sizeof(g_creds[slot].uri), cfg->uri); + copy_str(g_creds[slot].user, sizeof(g_creds[slot].user), cfg->user); + copy_str(g_creds[slot].password, sizeof(g_creds[slot].password), cfg->password); + cbm_mutex_unlock(&g_cred_mu); +} + +void cbm_neo4j_config_wipe(cbm_neo4j_config_t *cfg) { + if (cfg) { + volatile char *p = cfg->password; + for (size_t i = 0; i < sizeof(cfg->password); i++) { + p[i] = 0; + } + } +} + +/* ── Config ─────────────────────────────────────────────────────── */ + +static bool n4_nonempty(const char *s) { + return s && *s; +} + +static void resolve_password(cbm_neo4j_config_t *out, const char *explicit_pw) { + char buf[CBM_NEO4J_PASSWORD_MAX]; + /* CBM_NEO4J_PASSWORD is only ever set by the index supervisor for one + * worker and carries that request's --graph-password, so it outranks the + * user's ambient NEO4J_PASSWORD. */ + if (n4_nonempty(explicit_pw)) { + copy_str(out->password, sizeof(out->password), explicit_pw); + } else if (cbm_safe_getenv(CBM_NEO4J_PASSWORD_ENV, buf, sizeof(buf), NULL) && buf[0]) { + copy_str(out->password, sizeof(out->password), buf); + } else if (cbm_safe_getenv("NEO4J_PASSWORD", buf, sizeof(buf), NULL) && buf[0]) { + copy_str(out->password, sizeof(out->password), buf); + } else { + (void)cred_lookup(out->uri, out->user, out->password, sizeof(out->password)); + } + memset(buf, 0, sizeof(buf)); +} + +static void resolve_from_env(char *dst, size_t dst_sz, const char *explicit_val, + const char *env_name, const char *dflt) { + char buf[CBM_NEO4J_URI_MAX]; + if (n4_nonempty(explicit_val)) { + copy_str(dst, dst_sz, explicit_val); + } else if (cbm_safe_getenv(env_name, buf, sizeof(buf), NULL) && buf[0]) { + copy_str(dst, dst_sz, buf); + } else { + copy_str(dst, dst_sz, dflt); + } +} + +int cbm_neo4j_config_resolve(const char *uri, const char *user, const char *password, + const char *database, cbm_neo4j_config_t *out, char *err, + size_t err_sz) { + memset(out, 0, sizeof(*out)); + char uri_buf[CBM_NEO4J_URI_MAX]; + resolve_from_env(uri_buf, sizeof(uri_buf), uri, "NEO4J_URI", ""); + if (!uri_buf[0]) { + (void)snprintf(err, err_sz, "graph_db URL is required"); + return -1; + } + if (strlen(uri_buf) >= sizeof(out->uri)) { + (void)snprintf(err, err_sz, "graph_db URL too long"); + return -1; + } + cbm_neo4j_url_t parsed; + if (cbm_neo4j_url_parse(uri_buf, &parsed, err, err_sz) != 0) { + return -1; + } + copy_str(out->uri, sizeof(out->uri), uri_buf); + resolve_from_env(out->user, sizeof(out->user), user, "NEO4J_USER", ""); + if (!out->user[0]) { + /* NEO4J_USERNAME is what the official drivers and Docker image use. */ + resolve_from_env(out->user, sizeof(out->user), NULL, "NEO4J_USERNAME", N4_DEFAULT_USER); + } + resolve_from_env(out->database, sizeof(out->database), database, "NEO4J_DATABASE", + N4_DEFAULT_DATABASE); + for (const char *p = out->database; *p; p++) { + if (!isalnum((unsigned char)*p) && *p != '.' && *p != '-' && *p != '_') { + (void)snprintf(err, err_sz, "invalid graph_database name '%s'", out->database); + return -1; + } + } + resolve_password(out, password); + return 0; +} + +char *cbm_neo4j_config_to_meta(const cbm_neo4j_config_t *cfg) { + yyjson_mut_doc *d = yyjson_mut_doc_new(NULL); + if (!d) { + return NULL; + } + yyjson_mut_val *o = yyjson_mut_obj(d); + yyjson_mut_doc_set_root(d, o); + yyjson_mut_obj_add_str(d, o, "kind", "neo4j"); + yyjson_mut_obj_add_strcpy(d, o, "uri", cfg->uri); + yyjson_mut_obj_add_strcpy(d, o, "user", cfg->user); + yyjson_mut_obj_add_strcpy(d, o, "database", cfg->database); + char *json = yyjson_mut_write(d, 0, NULL); + yyjson_mut_doc_free(d); + return json; +} + +int cbm_neo4j_config_from_meta(const char *meta_json, cbm_neo4j_config_t *out, char *err, + size_t err_sz) { + if (!meta_json || !*meta_json) { + (void)snprintf(err, err_sz, "project has no graph backend metadata"); + return -1; + } + yyjson_doc *d = yyjson_read(meta_json, strlen(meta_json), 0); + yyjson_val *o = d ? yyjson_doc_get_root(d) : NULL; + const char *kind = yyjson_get_str(yyjson_obj_get(o, "kind")); + if (!o || !kind || strcmp(kind, "neo4j") != 0) { + yyjson_doc_free(d); + (void)snprintf(err, err_sz, "unsupported graph backend metadata"); + return -1; + } + /* Use stored uri/user/database verbatim — env overrides apply only to the + * password, so a stale NEO4J_URI can never redirect an indexed project. */ + memset(out, 0, sizeof(*out)); + copy_str(out->uri, sizeof(out->uri), yyjson_get_str(yyjson_obj_get(o, "uri"))); + copy_str(out->user, sizeof(out->user), yyjson_get_str(yyjson_obj_get(o, "user"))); + copy_str(out->database, sizeof(out->database), yyjson_get_str(yyjson_obj_get(o, "database"))); + yyjson_doc_free(d); + if (!out->uri[0]) { + (void)snprintf(err, err_sz, "graph backend metadata has no uri"); + return -1; + } + if (!out->database[0]) { + copy_str(out->database, sizeof(out->database), N4_DEFAULT_DATABASE); + } + resolve_password(out, NULL); + return 0; +} + +/* ── Client ─────────────────────────────────────────────────────── */ + +cbm_neo4j_t *cbm_neo4j_connect(const cbm_neo4j_config_t *cfg, char *err, size_t err_sz) { + cbm_neo4j_url_t url; + if (cbm_neo4j_url_parse(cfg->uri, &url, err, err_sz) != 0) { + return NULL; + } + cbm_neo4j_t *c = calloc(1, sizeof(*c)); + if (!c) { + return NULL; + } + c->http = cbm_neo4j_http_open(&url, cfg->user, cfg->password, err, err_sz); + if (!c->http) { + free(c); + return NULL; + } + copy_str(c->uri, sizeof(c->uri), cfg->uri); + (void)snprintf(c->commit_path, sizeof(c->commit_path), "/db/%s/tx/commit", + cfg->database[0] ? cfg->database : N4_DEFAULT_DATABASE); + return c; +} + +void cbm_neo4j_close(cbm_neo4j_t *c) { + if (!c) { + return; + } + cbm_neo4j_http_close(c->http); + free(c); +} + +const char *cbm_neo4j_error(const cbm_neo4j_t *c) { + return c ? c->err : "no Neo4j client"; +} + +const char *cbm_neo4j_uri(const cbm_neo4j_t *c) { + return c ? c->uri : ""; +} + +int cbm_neo4j_batch_init(cbm_neo4j_batch_t *b) { + memset(b, 0, sizeof(*b)); + b->doc = yyjson_mut_doc_new(NULL); + if (!b->doc) { + return -1; + } + yyjson_mut_val *root = yyjson_mut_obj(b->doc); + yyjson_mut_doc_set_root(b->doc, root); + b->statements = yyjson_mut_arr(b->doc); + yyjson_mut_obj_add_val(b->doc, root, "statements", b->statements); + return 0; +} + +yyjson_mut_val *cbm_neo4j_batch_add(cbm_neo4j_batch_t *b, const char *cypher) { + yyjson_mut_val *st = yyjson_mut_obj(b->doc); + yyjson_mut_obj_add_strcpy(b->doc, st, "statement", cypher); + yyjson_mut_val *params = yyjson_mut_obj(b->doc); + yyjson_mut_obj_add_val(b->doc, st, "parameters", params); + yyjson_mut_val *contents = yyjson_mut_arr(b->doc); + yyjson_mut_arr_add_str(b->doc, contents, "row"); + yyjson_mut_obj_add_val(b->doc, st, "resultDataContents", contents); + yyjson_mut_arr_append(b->statements, st); + b->count++; + return params; +} + +void cbm_neo4j_batch_free(cbm_neo4j_batch_t *b) { + if (b && b->doc) { + yyjson_mut_doc_free(b->doc); + b->doc = NULL; + } +} + +static void set_err(cbm_neo4j_t *c, const char *fmt, const char *a, const char *b2) { + (void)snprintf(c->err, sizeof(c->err), fmt, a ? a : "", b2 ? b2 : ""); +} + +/* NULL when results holds exactly one well-formed entry per statement sent: + * {"columns": [...], "data": [{"row": [one value per column]}, ...]}. + * Otherwise a short description of the first defect. */ +const char *cbm_neo4j_result_shape_error(yyjson_val *results, int statements) { + if (!yyjson_is_arr(results)) { + return "no results array"; + } + if (yyjson_arr_size(results) != (size_t)statements) { + return "result count does not match the statements sent"; + } + size_t idx; + size_t max; + yyjson_val *res; + yyjson_arr_foreach(results, idx, max, res) { + yyjson_val *columns = yyjson_obj_get(res, "columns"); + yyjson_val *data = yyjson_obj_get(res, "data"); + if (!yyjson_is_arr(columns) || !yyjson_is_arr(data)) { + return "result without columns/data arrays"; + } + size_t ncols = yyjson_arr_size(columns); + size_t ri; + size_t rmax; + yyjson_val *entry; + yyjson_arr_foreach(data, ri, rmax, entry) { + yyjson_val *row = yyjson_obj_get(entry, "row"); + if (!yyjson_is_arr(row) || yyjson_arr_size(row) != ncols) { + return "row does not match its columns"; + } + } + } + return NULL; +} + +int cbm_neo4j_batch_run(cbm_neo4j_t *c, cbm_neo4j_batch_t *b, cbm_neo4j_result_t *out) { + if (out) { + memset(out, 0, sizeof(*out)); + } + if (!c || !b || !b->doc) { + return -1; + } + c->err[0] = '\0'; + size_t body_len = 0; + char *body = yyjson_mut_write(b->doc, 0, &body_len); + if (!body) { + set_err(c, "failed to serialize Neo4j request%s%s", NULL, NULL); + return -1; + } + int status = 0; + char *resp = NULL; + size_t resp_len = 0; + char terr[CBM_NEO4J_ERR_MAX]; + terr[0] = '\0'; + int rc = cbm_neo4j_http_post(c->http, c->commit_path, body, body_len, &status, &resp, &resp_len, + terr, sizeof(terr)); + free(body); + if (rc != 0) { + set_err(c, "%s%s", terr[0] ? terr : "Neo4j request failed", NULL); + free(resp); + return -1; + } + if (status == 401 || status == 403) { + set_err(c, + "Neo4j rejected the credentials for %s (HTTP 401/403); check " + "--graph-user/--graph-password or NEO4J_USER/NEO4J_PASSWORD%s", + c->uri, NULL); + free(resp); + return -1; + } + if (status == 404) { + set_err(c, + "Neo4j endpoint %s not found (HTTP 404): wrong database name or not a " + "Neo4j HTTP port%s", + c->commit_path, NULL); + free(resp); + return -1; + } + yyjson_doc *d = yyjson_read(resp, resp_len, 0); + free(resp); + yyjson_val *root = d ? yyjson_doc_get_root(d) : NULL; + if (!root) { + char sc[16]; + (void)snprintf(sc, sizeof(sc), "%d", status); + set_err(c, "unparseable Neo4j response (HTTP %s)%s", sc, NULL); + yyjson_doc_free(d); + return -1; + } + yyjson_val *errors = yyjson_obj_get(root, "errors"); + if (errors && yyjson_arr_size(errors) > 0) { + yyjson_val *e0 = yyjson_arr_get_first(errors); + set_err(c, "Neo4j error %s: %s", yyjson_get_str(yyjson_obj_get(e0, "code")), + yyjson_get_str(yyjson_obj_get(e0, "message"))); + yyjson_doc_free(d); + return -1; + } + if (status < 200 || status >= 300) { + char sc[16]; + (void)snprintf(sc, sizeof(sc), "%d", status); + set_err(c, "Neo4j returned HTTP %s%s", sc, NULL); + yyjson_doc_free(d); + return -1; + } + /* A 2xx body must still be a complete transactional result. Anything + * else (a proxy or another service answering 200, a truncated reply) + * would read as "no rows" -- an empty graph to every reader, and a + * generation of 0 to the sync -- so it is an error, never a success. */ + const char *shape_err = cbm_neo4j_result_shape_error(yyjson_obj_get(root, "results"), b->count); + if (shape_err) { + set_err(c, "malformed Neo4j response from %s: %s", c->uri, shape_err); + yyjson_doc_free(d); + return -1; + } + if (out) { + out->doc = d; + out->results = yyjson_obj_get(root, "results"); + } else { + yyjson_doc_free(d); + } + return 0; +} + +int cbm_neo4j_ping(cbm_neo4j_t *c) { + cbm_neo4j_batch_t b; + if (cbm_neo4j_batch_init(&b) != 0) { + return -1; + } + (void)cbm_neo4j_batch_add(&b, "RETURN 1"); + int rc = cbm_neo4j_batch_run(c, &b, NULL); + cbm_neo4j_batch_free(&b); + return rc; +} + +yyjson_val *cbm_neo4j_result_data(const cbm_neo4j_result_t *r, size_t stmt_index) { + if (!r || !r->results) { + return NULL; + } + yyjson_val *res = yyjson_arr_get(r->results, stmt_index); + return res ? yyjson_obj_get(res, "data") : NULL; +} + +yyjson_val *cbm_neo4j_row_col(yyjson_val *data_entry, size_t col) { + return yyjson_arr_get(yyjson_obj_get(data_entry, "row"), col); +} + +void cbm_neo4j_result_free(cbm_neo4j_result_t *r) { + if (r && r->doc) { + yyjson_doc_free(r->doc); + r->doc = NULL; + r->results = NULL; + } +} + +int64_t cbm_neo4j_val_int(yyjson_val *v, int64_t fallback) { + if (yyjson_is_int(v)) { + return yyjson_get_sint(v); + } + if (yyjson_is_real(v)) { + return (int64_t)yyjson_get_real(v); + } + if (yyjson_is_str(v)) { + return strtoll(yyjson_get_str(v), NULL, 10); + } + return fallback; +} + +char *cbm_neo4j_val_strdup(yyjson_val *v, const char *fallback) { + const char *s = NULL; + if (yyjson_is_str(v)) { + s = yyjson_get_str(v); + } else if (v && !yyjson_is_null(v)) { + /* Non-string values (numbers, lists, maps) come back as their JSON. */ + return yyjson_val_write(v, 0, NULL); + } else { + s = fallback; + } + if (!s) { + return NULL; + } + size_t n = strlen(s); + char *d = malloc(n + 1); + if (d) { + memcpy(d, s, n + 1); + } + return d; +} + +bool cbm_neo4j_ident_ok(const char *s) { + if (!s || !*s || strlen(s) > 128) { + return false; + } + if (!isalpha((unsigned char)s[0]) && s[0] != '_') { + return false; + } + for (const char *p = s; *p; p++) { + if (!isalnum((unsigned char)*p) && *p != '_') { + return false; + } + } + return true; +} diff --git a/src/neo4j/neo4j_client.h b/src/neo4j/neo4j_client.h new file mode 100644 index 0000000000..c4923f78dd --- /dev/null +++ b/src/neo4j/neo4j_client.h @@ -0,0 +1,118 @@ +/* + * neo4j_client.h — Neo4j connection config + Cypher execution over the + * transactional HTTP endpoint (POST /db//tx/commit). + * + * A request carries one or more statements that Neo4j runs in a single + * transaction; results come back per statement as rows of JSON values. + */ +#ifndef CBM_NEO4J_CLIENT_H +#define CBM_NEO4J_CLIENT_H + +#include +#include +#include + +#include "yyjson/yyjson.h" + +#define CBM_NEO4J_URI_MAX 1024 +#define CBM_NEO4J_USER_MAX 256 +#define CBM_NEO4J_PASSWORD_MAX 512 +#define CBM_NEO4J_DATABASE_MAX 128 +#define CBM_NEO4J_ERR_MAX 1024 + +/* Worker hand-off: the daemon passes the password to the index worker in + * this environment variable instead of on the worker's command line. */ +#define CBM_NEO4J_PASSWORD_ENV "CBM_NEO4J_PASSWORD" + +/* store_meta key holding the project's graph backend (no password). */ +#define CBM_NEO4J_META_KEY "graph_backend" + +typedef struct cbm_neo4j_config { + char uri[CBM_NEO4J_URI_MAX]; + char user[CBM_NEO4J_USER_MAX]; + char password[CBM_NEO4J_PASSWORD_MAX]; + char database[CBM_NEO4J_DATABASE_MAX]; +} cbm_neo4j_config_t; + +/* Build a config from explicit values (index_repository's graph_* args). + * uri is required. Missing user/password/database fall back to NEO4J_USER / + * NEO4J_PASSWORD (then CBM_NEO4J_PASSWORD, then the in-process credential + * cache) / NEO4J_DATABASE, then to "neo4j" / "" / "neo4j". + * Returns 0, or -1 with err filled. */ +int cbm_neo4j_config_resolve(const char *uri, const char *user, const char *password, + const char *database, cbm_neo4j_config_t *out, char *err, + size_t err_sz); + +/* Serialize the config for store_meta (password omitted). malloc'd. */ +char *cbm_neo4j_config_to_meta(const cbm_neo4j_config_t *cfg); + +/* Parse a store_meta value back into a config, resolving the password the + * same way cbm_neo4j_config_resolve does. Returns 0, or -1 with err. */ +int cbm_neo4j_config_from_meta(const char *meta_json, cbm_neo4j_config_t *out, char *err, + size_t err_sz); + +/* Remember the password for (uri, user) for the life of the process so read + * tools served by this daemon can reconnect after an index call. */ +void cbm_neo4j_config_remember(const cbm_neo4j_config_t *cfg); + +/* Overwrite the password in memory. */ +void cbm_neo4j_config_wipe(cbm_neo4j_config_t *cfg); + +typedef struct cbm_neo4j cbm_neo4j_t; + +/* Create a client (no network I/O yet). Returns NULL with err on bad URL. */ +cbm_neo4j_t *cbm_neo4j_connect(const cbm_neo4j_config_t *cfg, char *err, size_t err_sz); +void cbm_neo4j_close(cbm_neo4j_t *c); +const char *cbm_neo4j_error(const cbm_neo4j_t *c); +const char *cbm_neo4j_uri(const cbm_neo4j_t *c); + +/* Round-trip "RETURN 1" to verify reachability, credentials and database. */ +int cbm_neo4j_ping(cbm_neo4j_t *c); + +/* ── Statement batches ─────────────────────────────────────────── */ + +typedef struct { + yyjson_mut_doc *doc; + yyjson_mut_val *statements; + int count; +} cbm_neo4j_batch_t; + +typedef struct { + yyjson_doc *doc; + yyjson_val *results; /* array, one entry per statement */ +} cbm_neo4j_result_t; + +int cbm_neo4j_batch_init(cbm_neo4j_batch_t *b); +/* Append a statement; returns its parameters object (owned by b->doc). */ +yyjson_mut_val *cbm_neo4j_batch_add(cbm_neo4j_batch_t *b, const char *cypher); +void cbm_neo4j_batch_free(cbm_neo4j_batch_t *b); + +/* Run all statements in one transaction. out may be NULL. Returns 0, or -1 + * with cbm_neo4j_error() describing the transport or Cypher failure. */ +int cbm_neo4j_batch_run(cbm_neo4j_t *c, cbm_neo4j_batch_t *b, cbm_neo4j_result_t *out); + +/* Convenience: one statement. params may be NULL; it must belong to b's doc, + * so build it via cbm_neo4j_batch_add and run with cbm_neo4j_batch_run. */ + +/* NULL when a 2xx body's "results" holds exactly one well-formed entry per + * statement ({"columns":[...], "data":[{"row":[one value per column]}]}); + * otherwise a short description of the defect. cbm_neo4j_batch_run refuses + * any response this rejects, so a malformed reply is never "no rows". */ +const char *cbm_neo4j_result_shape_error(yyjson_val *results, int statements); + +/* Rows of statement i: array of {"row":[...]} objects, or NULL. */ +yyjson_val *cbm_neo4j_result_data(const cbm_neo4j_result_t *r, size_t stmt_index); +/* Column j of a data entry. */ +yyjson_val *cbm_neo4j_row_col(yyjson_val *data_entry, size_t col); +void cbm_neo4j_result_free(cbm_neo4j_result_t *r); + +/* Value helpers tolerant of null / wrong type. */ +int64_t cbm_neo4j_val_int(yyjson_val *v, int64_t fallback); +/* malloc'd copy of a string value; non-strings are serialized as JSON; + * null → NULL (or strdup(fallback) when fallback != NULL). */ +char *cbm_neo4j_val_strdup(yyjson_val *v, const char *fallback); + +/* True when s is a safe bare identifier for a label / relationship type. */ +bool cbm_neo4j_ident_ok(const char *s); + +#endif /* CBM_NEO4J_CLIENT_H */ diff --git a/src/neo4j/neo4j_http.c b/src/neo4j/neo4j_http.c new file mode 100644 index 0000000000..6360ef1fc8 --- /dev/null +++ b/src/neo4j/neo4j_http.c @@ -0,0 +1,791 @@ +/* + * neo4j_http.c — Minimal HTTP/1.1 client for the Neo4j HTTP API. + */ +#include "neo4j/neo4j_http.h" + +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#include +#include +#include +typedef SOCKET cbm_n4_sock_t; +#define N4_SOCK_BAD INVALID_SOCKET +#define n4_sock_close closesocket +#else +#include +#include +#include +#include +#include +#include +#include +typedef int cbm_n4_sock_t; +#define N4_SOCK_BAD (-1) +#define n4_sock_close close +#endif + +#ifdef MSG_NOSIGNAL +#define N4_SEND_FLAGS MSG_NOSIGNAL +#else +#define N4_SEND_FLAGS 0 +#endif + +#define N4_DEFAULT_HTTP_PORT 7474 +#define N4_DEFAULT_HTTPS_PORT 7473 +#define N4_IO_TIMEOUT_MS 120000 +#define N4_HDR_MAX (64U * 1024U) +#define N4_RESP_MAX ((size_t)1 << 30) /* 1 GiB: refuse anything larger */ + +struct cbm_neo4j_http { + cbm_neo4j_url_t url; + char *auth_header; /* "Basic xxx" or NULL */ + cbm_n4_sock_t sock; + /* Bytes read past the end of the previous response (keep-alive). */ + char *carry; + size_t carry_len; +#ifdef _WIN32 + HINTERNET session; + HINTERNET connect; +#endif +}; + +static void n4_err(char *err, size_t err_sz, const char *fmt, const char *arg) { + if (err && err_sz > 0) { + (void)snprintf(err, err_sz, fmt, arg ? arg : ""); + } +} + +/* ── URL / base64 ───────────────────────────────────────────────── */ + +static bool n4_has_prefix_ci(const char *s, const char *p) { + for (; *p; s++, p++) { + if (tolower((unsigned char)*s) != tolower((unsigned char)*p)) { + return false; + } + } + return true; +} + +int cbm_neo4j_url_parse(const char *url, cbm_neo4j_url_t *out, char *err, size_t err_sz) { + if (!url || !out || !*url) { + n4_err(err, err_sz, "graph_db URL is empty%s", NULL); + return -1; + } + memset(out, 0, sizeof(*out)); + const char *p = NULL; + if (n4_has_prefix_ci(url, "http://")) { + p = url + strlen("http://"); + out->port = N4_DEFAULT_HTTP_PORT; + } else if (n4_has_prefix_ci(url, "https://")) { + p = url + strlen("https://"); + out->tls = true; + out->port = N4_DEFAULT_HTTPS_PORT; + } else if (n4_has_prefix_ci(url, "bolt") || n4_has_prefix_ci(url, "neo4j")) { + n4_err(err, err_sz, + "unsupported graph_db scheme in '%s': codebase-memory-mcp talks to Neo4j over its " + "HTTP API; use http://:7474 (or https://:7473) instead of bolt/neo4j", + url); + return -1; + } else { + n4_err(err, err_sz, "graph_db URL must start with http:// or https:// (got '%s')", url); + return -1; + } + + const char *host_end = p; + if (*p == '[') { /* IPv6 literal */ + host_end = strchr(p, ']'); + if (!host_end) { + n4_err(err, err_sz, "malformed IPv6 host in '%s'", url); + return -1; + } + size_t hl = (size_t)(host_end - (p + 1)); + if (hl == 0 || hl >= sizeof(out->host)) { + n4_err(err, err_sz, "bad host in '%s'", url); + return -1; + } + memcpy(out->host, p + 1, hl); + host_end++; + } else { + while (*host_end && *host_end != ':' && *host_end != '/') { + host_end++; + } + size_t hl = (size_t)(host_end - p); + if (hl == 0 || hl >= sizeof(out->host)) { + n4_err(err, err_sz, "bad host in '%s'", url); + return -1; + } + memcpy(out->host, p, hl); + } + p = host_end; + if (*p == ':') { + p++; + char *end = NULL; + long port = strtol(p, &end, 10); + if (end == p || port <= 0 || port > 65535) { + n4_err(err, err_sz, "bad port in '%s'", url); + return -1; + } + out->port = (int)port; + p = end; + } + if (*p && *p != '/') { + n4_err(err, err_sz, "malformed graph_db URL '%s'", url); + return -1; + } + size_t pl = strlen(p); + while (pl > 0 && p[pl - 1] == '/') { + pl--; + } + if (pl >= sizeof(out->base_path)) { + n4_err(err, err_sz, "URL path too long in '%s'", url); + return -1; + } + memcpy(out->base_path, p, pl); + out->base_path[pl] = '\0'; + return 0; +} + +char *cbm_neo4j_base64(const char *data, size_t len) { + static const char tbl[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + size_t out_len = ((len + 2) / 3) * 4; + char *out = malloc(out_len + 1); + if (!out) { + return NULL; + } + const unsigned char *in = (const unsigned char *)data; + size_t o = 0; + size_t i = 0; + for (; i + 2 < len; i += 3) { + uint32_t v = ((uint32_t)in[i] << 16) | ((uint32_t)in[i + 1] << 8) | in[i + 2]; + out[o++] = tbl[(v >> 18) & 63]; + out[o++] = tbl[(v >> 12) & 63]; + out[o++] = tbl[(v >> 6) & 63]; + out[o++] = tbl[v & 63]; + } + if (i < len) { + uint32_t v = (uint32_t)in[i] << 16; + if (i + 1 < len) { + v |= (uint32_t)in[i + 1] << 8; + } + out[o++] = tbl[(v >> 18) & 63]; + out[o++] = tbl[(v >> 12) & 63]; + out[o++] = (i + 1 < len) ? tbl[(v >> 6) & 63] : '='; + out[o++] = '='; + } + out[o] = '\0'; + return out; +} + +/* ── Plain-socket transport ─────────────────────────────────────── */ + +static void n4_net_init(void) { +#ifdef _WIN32 + static volatile LONG done = 0; + if (InterlockedCompareExchange(&done, 1, 0) == 0) { + WSADATA wsa; + (void)WSAStartup(MAKEWORD(2, 2), &wsa); + } +#endif +} + +static void n4_set_timeouts(cbm_n4_sock_t s) { +#ifdef _WIN32 + DWORD tv = N4_IO_TIMEOUT_MS; + (void)setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, (const char *)&tv, sizeof(tv)); + (void)setsockopt(s, SOL_SOCKET, SO_SNDTIMEO, (const char *)&tv, sizeof(tv)); +#else + struct timeval tv; + tv.tv_sec = N4_IO_TIMEOUT_MS / 1000; + tv.tv_usec = 0; + (void)setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); + (void)setsockopt(s, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv)); +#endif + int one = 1; + (void)setsockopt(s, IPPROTO_TCP, TCP_NODELAY, (const char *)&one, sizeof(one)); +} + +static void n4_disconnect(cbm_neo4j_http_t *h) { + if (h->sock != N4_SOCK_BAD) { + n4_sock_close(h->sock); + h->sock = N4_SOCK_BAD; + } + free(h->carry); + h->carry = NULL; + h->carry_len = 0; +} + +static int n4_connect(cbm_neo4j_http_t *h, char *err, size_t err_sz) { + n4_net_init(); + char port_s[16]; + (void)snprintf(port_s, sizeof(port_s), "%d", h->url.port); + struct addrinfo hints; + memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + struct addrinfo *res = NULL; + if (getaddrinfo(h->url.host, port_s, &hints, &res) != 0 || !res) { + n4_err(err, err_sz, "cannot resolve Neo4j host '%s'", h->url.host); + return -1; + } + cbm_n4_sock_t s = N4_SOCK_BAD; + for (struct addrinfo *ai = res; ai; ai = ai->ai_next) { + s = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol); + if (s == N4_SOCK_BAD) { + continue; + } + if (connect(s, ai->ai_addr, (int)ai->ai_addrlen) == 0) { + break; + } + n4_sock_close(s); + s = N4_SOCK_BAD; + } + freeaddrinfo(res); + if (s == N4_SOCK_BAD) { + char buf[CBM_NEO4J_HOST_MAX + 32]; + (void)snprintf(buf, sizeof(buf), "%s:%d", h->url.host, h->url.port); + n4_err(err, err_sz, "cannot connect to Neo4j at %s (is the server running?)", buf); + return -1; + } + n4_set_timeouts(s); + h->sock = s; + return 0; +} + +static int n4_send_all(cbm_n4_sock_t s, const char *buf, size_t len) { + while (len > 0) { + int chunk = len > (size_t)(1 << 30) ? (1 << 30) : (int)len; + int n = (int)send(s, buf, chunk, N4_SEND_FLAGS); + if (n <= 0) { + return -1; + } + buf += n; + len -= (size_t)n; + } + return 0; +} + +typedef struct { + char *data; + size_t len; + size_t cap; +} n4_buf_t; + +static int n4_buf_reserve(n4_buf_t *b, size_t extra) { + if (b->len + extra + 1 <= b->cap) { + return 0; + } + size_t cap = b->cap ? b->cap : 8192; + while (cap < b->len + extra + 1) { + cap *= 2; + } + if (cap > N4_RESP_MAX + N4_HDR_MAX) { + return -1; + } + char *d = realloc(b->data, cap); + if (!d) { + return -1; + } + b->data = d; + b->cap = cap; + return 0; +} + +/* Read more bytes from the socket into b. Returns bytes read, 0 on EOF, -1 on error. */ +static int n4_recv_more(cbm_neo4j_http_t *h, n4_buf_t *b) { + if (n4_buf_reserve(b, 16384) != 0) { + return -1; + } + int n = (int)recv(h->sock, b->data + b->len, 16384, 0); + if (n > 0) { + b->len += (size_t)n; + b->data[b->len] = '\0'; + } + return n; +} + +static const char *n4_find_hdr_end(const char *d, size_t len) { + for (size_t i = 0; i + 3 < len; i++) { + if (d[i] == '\r' && d[i + 1] == '\n' && d[i + 2] == '\r' && d[i + 3] == '\n') { + return d + i; + } + } + return NULL; +} + +/* Case-insensitive header lookup within [hdrs, end). Copies value into out. */ +static bool n4_header(const char *hdrs, const char *end, const char *name, char *out, + size_t out_sz) { + size_t nl = strlen(name); + const char *line = hdrs; + while (line < end) { + const char *eol = line; + while (eol < end && *eol != '\r') { + eol++; + } + if ((size_t)(eol - line) > nl && line[nl] == ':' && n4_has_prefix_ci(line, name)) { + const char *v = line + nl + 1; + while (v < eol && (*v == ' ' || *v == '\t')) { + v++; + } + size_t vl = (size_t)(eol - v); + if (vl >= out_sz) { + vl = out_sz - 1; + } + memcpy(out, v, vl); + out[vl] = '\0'; + return true; + } + line = eol + 2; + } + return false; +} + +/* Decode a chunked body in place. Returns decoded length, or -1 when the + * buffer does not yet hold the complete body (*need_more set) or is malformed. */ +static long long n4_dechunk(char *body, size_t len, size_t *consumed, bool *need_more) { + size_t r = 0; + size_t w = 0; + *need_more = false; + for (;;) { + const char *crlf = NULL; + for (size_t i = r; i + 1 < len; i++) { + if (body[i] == '\r' && body[i + 1] == '\n') { + crlf = body + i; + break; + } + } + if (!crlf) { + *need_more = true; + return -1; + } + char *endp = NULL; + unsigned long long sz = strtoull(body + r, &endp, 16); + if (endp == body + r) { + return -1; + } + size_t data_start = (size_t)(crlf - body) + 2; + if (sz == 0) { + /* Trailer section ends with an empty line. */ + size_t t = data_start; + for (;;) { + if (t + 1 >= len) { + *need_more = true; + return -1; + } + if (body[t] == '\r' && body[t + 1] == '\n') { + *consumed = t + 2; + return (long long)w; + } + while (t + 1 < len && !(body[t] == '\r' && body[t + 1] == '\n')) { + t++; + } + if (t + 1 >= len) { + *need_more = true; + return -1; + } + t += 2; + } + } + if (data_start + sz + 2 > len) { + *need_more = true; + return -1; + } + memmove(body + w, body + data_start, (size_t)sz); + w += (size_t)sz; + r = data_start + (size_t)sz + 2; + } +} + +/* One request/response exchange over the plain socket. Returns 0 on success, + * -1 on transport failure; *retryable is set when the failure happened on a + * reused keep-alive connection before any response byte arrived. */ +static int n4_socket_exchange(cbm_neo4j_http_t *h, const char *req, size_t req_len, + const char *body, size_t body_len, int *status, char **resp, + size_t *resp_len, bool *retryable, char *err, size_t err_sz) { + *retryable = false; + bool reused = h->sock != N4_SOCK_BAD; + if (!reused && n4_connect(h, err, err_sz) != 0) { + return -1; + } + if (n4_send_all(h->sock, req, req_len) != 0 || + (body_len > 0 && n4_send_all(h->sock, body, body_len) != 0)) { + n4_disconnect(h); + *retryable = reused; + n4_err(err, err_sz, "send to Neo4j failed%s", NULL); + return -1; + } + + n4_buf_t b = {0}; + if (h->carry_len > 0) { + if (n4_buf_reserve(&b, h->carry_len) != 0) { + n4_disconnect(h); + return -1; + } + memcpy(b.data, h->carry, h->carry_len); + b.len = h->carry_len; + b.data[b.len] = '\0'; + free(h->carry); + h->carry = NULL; + h->carry_len = 0; + } + const char *hdr_end = NULL; + while (!(hdr_end = n4_find_hdr_end(b.data ? b.data : "", b.len))) { + if (b.len > N4_HDR_MAX) { + free(b.data); + n4_disconnect(h); + n4_err(err, err_sz, "Neo4j response headers too large%s", NULL); + return -1; + } + int n = n4_recv_more(h, &b); + if (n <= 0) { + *retryable = reused && b.len == 0; + free(b.data); + n4_disconnect(h); + n4_err(err, err_sz, "no response from Neo4j (connection closed or timed out)%s", NULL); + return -1; + } + } + size_t hdr_len = (size_t)(hdr_end - b.data); + int code = 0; + if (sscanf(b.data, "HTTP/%*d.%*d %d", &code) != 1) { + free(b.data); + n4_disconnect(h); + n4_err(err, err_sz, "malformed HTTP status line from Neo4j%s", NULL); + return -1; + } + char val[128]; + bool chunked = n4_header(b.data, hdr_end + 2, "Transfer-Encoding", val, sizeof(val)) && + strstr(val, "chunked") != NULL; + bool conn_close = n4_header(b.data, hdr_end + 2, "Connection", val, sizeof(val)) && + n4_has_prefix_ci(val, "close"); + long long content_len = -1; + if (!chunked && n4_header(b.data, hdr_end + 2, "Content-Length", val, sizeof(val))) { + content_len = strtoll(val, NULL, 10); + } + size_t body_off = hdr_len + 4; + + size_t out_len = 0; + size_t consumed = 0; + if (chunked) { + for (;;) { + /* Work on a scratch copy so a partial decode never corrupts b. */ + size_t avail = b.len - body_off; + char *tmp = malloc(avail + 1); + if (!tmp) { + free(b.data); + n4_disconnect(h); + return -1; + } + memcpy(tmp, b.data + body_off, avail); + bool need_more = false; + long long dl = n4_dechunk(tmp, avail, &consumed, &need_more); + if (dl >= 0) { + memcpy(b.data + body_off, tmp, (size_t)dl); + free(tmp); + out_len = (size_t)dl; + /* Decoded bytes sit before `consumed`, so any surplus after it + * is still intact for the carry below. */ + consumed += body_off; + break; + } + free(tmp); + if (!need_more) { + free(b.data); + n4_disconnect(h); + n4_err(err, err_sz, "malformed chunked response from Neo4j%s", NULL); + return -1; + } + if (n4_recv_more(h, &b) <= 0) { + free(b.data); + n4_disconnect(h); + n4_err(err, err_sz, "truncated response from Neo4j%s", NULL); + return -1; + } + } + } else if (content_len >= 0) { + if ((size_t)content_len > N4_RESP_MAX) { + free(b.data); + n4_disconnect(h); + n4_err(err, err_sz, "Neo4j response too large%s", NULL); + return -1; + } + while (b.len - body_off < (size_t)content_len) { + if (n4_recv_more(h, &b) <= 0) { + free(b.data); + n4_disconnect(h); + n4_err(err, err_sz, "truncated response from Neo4j%s", NULL); + return -1; + } + } + out_len = (size_t)content_len; + consumed = body_off + out_len; + } else { + /* Body delimited by connection close. */ + while (n4_recv_more(h, &b) > 0) {} + out_len = b.len - body_off; + consumed = b.len; + conn_close = true; + } + + /* Keep any surplus bytes for the next response (rare; no pipelining). */ + if (consumed < b.len) { + h->carry_len = b.len - consumed; + h->carry = malloc(h->carry_len); + if (h->carry) { + memcpy(h->carry, b.data + consumed, h->carry_len); + } else { + h->carry_len = 0; + } + } + + char *out = malloc(out_len + 1); + if (!out) { + free(b.data); + n4_disconnect(h); + return -1; + } + memcpy(out, b.data + body_off, out_len); + out[out_len] = '\0'; + free(b.data); + if (conn_close) { + n4_disconnect(h); + } + *status = code; + *resp = out; + if (resp_len) { + *resp_len = out_len; + } + return 0; +} + +/* ── WinHTTP transport (https on Windows) ───────────────────────── */ + +#ifdef _WIN32 +static wchar_t *n4_widen(const char *s) { + int n = MultiByteToWideChar(CP_UTF8, 0, s, -1, NULL, 0); + if (n <= 0) { + return NULL; + } + wchar_t *w = malloc((size_t)n * sizeof(wchar_t)); + if (w) { + (void)MultiByteToWideChar(CP_UTF8, 0, s, -1, w, n); + } + return w; +} + +static int n4_winhttp_post(cbm_neo4j_http_t *h, const char *full_path, const char *body, + size_t body_len, int *status, char **resp, size_t *resp_len, char *err, + size_t err_sz) { + if (!h->session) { + h->session = WinHttpOpen(L"codebase-memory-mcp", WINHTTP_ACCESS_TYPE_DEFAULT_PROXY, + WINHTTP_NO_PROXY_NAME, WINHTTP_NO_PROXY_BYPASS, 0); + if (!h->session) { + n4_err(err, err_sz, "WinHttpOpen failed%s", NULL); + return -1; + } + (void)WinHttpSetTimeouts(h->session, N4_IO_TIMEOUT_MS, N4_IO_TIMEOUT_MS, N4_IO_TIMEOUT_MS, + N4_IO_TIMEOUT_MS); + wchar_t *whost = n4_widen(h->url.host); + if (!whost) { + return -1; + } + h->connect = WinHttpConnect(h->session, whost, (INTERNET_PORT)h->url.port, 0); + free(whost); + if (!h->connect) { + n4_err(err, err_sz, "cannot connect to Neo4j host '%s'", h->url.host); + return -1; + } + } + wchar_t *wpath = n4_widen(full_path); + if (!wpath) { + return -1; + } + HINTERNET req = WinHttpOpenRequest(h->connect, L"POST", wpath, NULL, WINHTTP_NO_REFERER, + WINHTTP_DEFAULT_ACCEPT_TYPES, WINHTTP_FLAG_SECURE); + free(wpath); + if (!req) { + n4_err(err, err_sz, "WinHttpOpenRequest failed%s", NULL); + return -1; + } + char hdrs[1024]; + (void)snprintf(hdrs, sizeof(hdrs), + "Content-Type: application/json\r\nAccept: application/json\r\n%s%s%s", + h->auth_header ? "Authorization: " : "", h->auth_header ? h->auth_header : "", + h->auth_header ? "\r\n" : ""); + wchar_t *whdrs = n4_widen(hdrs); + BOOL ok = whdrs && WinHttpSendRequest(req, whdrs, (DWORD)-1L, (LPVOID)body, (DWORD)body_len, + (DWORD)body_len, 0); + free(whdrs); + if (ok) { + ok = WinHttpReceiveResponse(req, NULL); + } + if (!ok) { + char code[32]; + (void)snprintf(code, sizeof(code), "%lu", (unsigned long)GetLastError()); + n4_err(err, err_sz, "HTTPS request to Neo4j failed (WinHTTP error %s)", code); + WinHttpCloseHandle(req); + return -1; + } + DWORD sc = 0; + DWORD sc_len = sizeof(sc); + (void)WinHttpQueryHeaders(req, WINHTTP_QUERY_STATUS_CODE | WINHTTP_QUERY_FLAG_NUMBER, + WINHTTP_HEADER_NAME_BY_INDEX, &sc, &sc_len, WINHTTP_NO_HEADER_INDEX); + n4_buf_t b = {0}; + for (;;) { + DWORD avail = 0; + if (!WinHttpQueryDataAvailable(req, &avail) || avail == 0) { + break; + } + if (n4_buf_reserve(&b, avail) != 0) { + free(b.data); + WinHttpCloseHandle(req); + n4_err(err, err_sz, "Neo4j response too large%s", NULL); + return -1; + } + DWORD got = 0; + if (!WinHttpReadData(req, b.data + b.len, avail, &got) || got == 0) { + break; + } + b.len += got; + b.data[b.len] = '\0'; + } + WinHttpCloseHandle(req); + if (!b.data) { + b.data = calloc(1, 1); + } + *status = (int)sc; + *resp = b.data; + if (resp_len) { + *resp_len = b.len; + } + return 0; +} +#endif + +/* ── Public API ─────────────────────────────────────────────────── */ + +cbm_neo4j_http_t *cbm_neo4j_http_open(const cbm_neo4j_url_t *url, const char *user, + const char *password, char *err, size_t err_sz) { +#ifndef _WIN32 + if (url->tls) { + n4_err(err, err_sz, + "https:// graph_db URLs are only supported on Windows builds; use http:// " + "(e.g. through a local TLS-terminating proxy or SSH tunnel) on this platform%s", + NULL); + return NULL; + } +#endif + cbm_neo4j_http_t *h = calloc(1, sizeof(*h)); + if (!h) { + return NULL; + } + h->url = *url; + h->sock = N4_SOCK_BAD; + if (user && *user) { + size_t ul = strlen(user); + size_t pl = password ? strlen(password) : 0; + char *cred = malloc(ul + pl + 2); + if (!cred) { + free(h); + return NULL; + } + memcpy(cred, user, ul); + cred[ul] = ':'; + if (pl) { + memcpy(cred + ul + 1, password, pl); + } + cred[ul + pl + 1] = '\0'; + char *b64 = cbm_neo4j_base64(cred, ul + pl + 1); + /* Wipe the plaintext credential before freeing. */ + memset(cred, 0, ul + pl + 1); + free(cred); + if (!b64) { + free(h); + return NULL; + } + size_t hl = strlen(b64) + sizeof("Basic "); + h->auth_header = malloc(hl); + if (!h->auth_header) { + free(b64); + free(h); + return NULL; + } + (void)snprintf(h->auth_header, hl, "Basic %s", b64); + free(b64); + } + return h; +} + +int cbm_neo4j_http_post(cbm_neo4j_http_t *h, const char *path, const char *body, size_t body_len, + int *status, char **resp, size_t *resp_len, char *err, size_t err_sz) { + if (!h || !path || !status || !resp) { + return -1; + } + *resp = NULL; + char full_path[CBM_NEO4J_PATH_MAX * 2]; + (void)snprintf(full_path, sizeof(full_path), "%s%s", h->url.base_path, path); +#ifdef _WIN32 + if (h->url.tls) { + return n4_winhttp_post(h, full_path, body, body_len, status, resp, resp_len, err, err_sz); + } +#endif + size_t req_cap = strlen(full_path) + strlen(h->url.host) + + (h->auth_header ? strlen(h->auth_header) : 0) + 256; + char *req = malloc(req_cap); + if (!req) { + return -1; + } + int req_len = snprintf(req, req_cap, + "POST %s HTTP/1.1\r\n" + "Host: %s:%d\r\n" + "Content-Type: application/json\r\n" + "Accept: application/json;charset=UTF-8\r\n" + "Content-Length: %zu\r\n" + "Connection: keep-alive\r\n" + "%s%s%s" + "\r\n", + full_path, h->url.host, h->url.port, body_len, + h->auth_header ? "Authorization: " : "", + h->auth_header ? h->auth_header : "", h->auth_header ? "\r\n" : ""); + if (req_len < 0 || (size_t)req_len >= req_cap) { + free(req); + return -1; + } + bool retryable = false; + int rc = n4_socket_exchange(h, req, (size_t)req_len, body, body_len, status, resp, resp_len, + &retryable, err, err_sz); + if (rc != 0 && retryable) { + /* The server closed an idle keep-alive connection: reconnect once. */ + rc = n4_socket_exchange(h, req, (size_t)req_len, body, body_len, status, resp, resp_len, + &retryable, err, err_sz); + } + memset(req, 0, (size_t)req_len); /* carries the auth header */ + free(req); + return rc; +} + +void cbm_neo4j_http_close(cbm_neo4j_http_t *h) { + if (!h) { + return; + } + n4_disconnect(h); +#ifdef _WIN32 + if (h->connect) { + WinHttpCloseHandle(h->connect); + } + if (h->session) { + WinHttpCloseHandle(h->session); + } +#endif + if (h->auth_header) { + memset(h->auth_header, 0, strlen(h->auth_header)); + free(h->auth_header); + } + free(h); +} diff --git a/src/neo4j/neo4j_http.h b/src/neo4j/neo4j_http.h new file mode 100644 index 0000000000..3433a536fd --- /dev/null +++ b/src/neo4j/neo4j_http.h @@ -0,0 +1,48 @@ +/* + * neo4j_http.h — Minimal HTTP/1.1 client for the Neo4j HTTP API. + * + * One persistent (keep-alive) connection per handle, POST-only, Basic auth. + * Plain http:// works on every platform over raw sockets. https:// goes + * through WinHTTP on Windows; on POSIX builds it is refused with a clear + * error (no TLS library is vendored). + */ +#ifndef CBM_NEO4J_HTTP_H +#define CBM_NEO4J_HTTP_H + +#include +#include + +#define CBM_NEO4J_HOST_MAX 256 +#define CBM_NEO4J_PATH_MAX 512 + +typedef struct { + bool tls; + char host[CBM_NEO4J_HOST_MAX]; + int port; + char base_path[CBM_NEO4J_PATH_MAX]; /* "" or "/prefix" without trailing slash */ +} cbm_neo4j_url_t; + +/* Parse "http[s]://host[:port][/prefix]". bolt://, neo4j:// and their +s + * variants are rejected with an error that names the HTTP port to use. + * Returns 0 on success, -1 with err filled otherwise. */ +int cbm_neo4j_url_parse(const char *url, cbm_neo4j_url_t *out, char *err, size_t err_sz); + +/* Standard base64 (with padding). Returns a malloc'd string. */ +char *cbm_neo4j_base64(const char *data, size_t len); + +typedef struct cbm_neo4j_http cbm_neo4j_http_t; + +/* Open a client handle; the connection itself is made lazily on the first + * request and re-made once if a kept-alive socket turns out to be dead. */ +cbm_neo4j_http_t *cbm_neo4j_http_open(const cbm_neo4j_url_t *url, const char *user, + const char *password, char *err, size_t err_sz); + +/* POST a JSON body to base_path + path. On success returns 0, sets *status to + * the HTTP status and *resp to a malloc'd NUL-terminated body (caller frees). + * Transport failures return -1 with err filled. */ +int cbm_neo4j_http_post(cbm_neo4j_http_t *h, const char *path, const char *body, size_t body_len, + int *status, char **resp, size_t *resp_len, char *err, size_t err_sz); + +void cbm_neo4j_http_close(cbm_neo4j_http_t *h); + +#endif /* CBM_NEO4J_HTTP_H */ diff --git a/src/neo4j/neo4j_store.c b/src/neo4j/neo4j_store.c new file mode 100644 index 0000000000..58fb7e2c1c --- /dev/null +++ b/src/neo4j/neo4j_store.c @@ -0,0 +1,1044 @@ +/* + * neo4j_store.c — Answer a store's graph reads from Neo4j. + * + * Every query is scoped to one mirrored graph: {db: , gen: }, read once when the store is attached + * so all reads of one request see one consistent generation. + */ +#include "neo4j/neo4j_store.h" +#include "neo4j/neo4j_client.h" +#include "neo4j/neo4j_sync.h" + +#include "foundation/log.h" + +#include +#include +#include +#include +#include +#include + +#define N4S_GEN_MAX 128 +#define N4S_ID_CHUNK 5000 + +/* Node columns, in cbm_node_t order. */ +#define N4S_NODE_COLS(v) \ + v ".id, " v ".project, " v ".label, " v ".name, " v ".qualified_name, " v ".file_path, " v \ + ".start_line, " v ".end_line, " v ".properties_json" +#define N4S_NODE_NCOLS 9 + +typedef struct { + cbm_neo4j_t *c; + char db[CBM_NEO4J_URI_MAX]; + int64_t gen; + char err[CBM_NEO4J_ERR_MAX]; +} n4s_ctx_t; + +/* ── Query plumbing ─────────────────────────────────────────────── */ + +typedef struct { + cbm_neo4j_batch_t b; + yyjson_mut_val *params; +} n4s_q_t; + +static int q_begin(n4s_ctx_t *x, n4s_q_t *q, const char *cypher) { + if (cbm_neo4j_batch_init(&q->b) != 0) { + return -1; + } + q->params = cbm_neo4j_batch_add(&q->b, cypher); + yyjson_mut_obj_add_strcpy(q->b.doc, q->params, "db", x->db); + yyjson_mut_obj_add_int(q->b.doc, q->params, "gen", x->gen); + return 0; +} + +static void q_str(n4s_q_t *q, const char *key, const char *val) { + if (val) { + yyjson_mut_obj_add_strcpy(q->b.doc, q->params, key, val); + } else { + yyjson_mut_obj_add_null(q->b.doc, q->params, key); + } +} + +static void q_int(n4s_q_t *q, const char *key, int64_t val) { + yyjson_mut_obj_add_int(q->b.doc, q->params, key, val); +} + +static void q_ids(n4s_q_t *q, const char *key, const int64_t *ids, int n) { + yyjson_mut_val *arr = yyjson_mut_arr(q->b.doc); + for (int i = 0; i < n; i++) { + yyjson_mut_arr_add_int(q->b.doc, arr, ids[i]); + } + yyjson_mut_obj_add_val(q->b.doc, q->params, key, arr); +} + +/* Edge types: an empty list means CALLS, as in the SQLite BFS. */ +static void q_types(n4s_q_t *q, const char *key, const char **types, int n) { + yyjson_mut_val *arr = yyjson_mut_arr(q->b.doc); + if (n <= 0) { + yyjson_mut_arr_add_str(q->b.doc, arr, "CALLS"); + } + for (int i = 0; i < n; i++) { + yyjson_mut_arr_add_strcpy(q->b.doc, arr, types[i]); + } + yyjson_mut_obj_add_val(q->b.doc, q->params, key, arr); +} + +/* Run and keep the result; on failure record the error in the ctx. */ +static int q_run(n4s_ctx_t *x, n4s_q_t *q, cbm_neo4j_result_t *r) { + int rc = cbm_neo4j_batch_run(x->c, &q->b, r); + cbm_neo4j_batch_free(&q->b); + if (rc != 0) { + (void)snprintf(x->err, sizeof(x->err), "%s", cbm_neo4j_error(x->c)); + return CBM_STORE_ERR; + } + return CBM_STORE_OK; +} + +static void node_from_row(yyjson_val *entry, size_t off, cbm_node_t *n) { + memset(n, 0, sizeof(*n)); + n->id = cbm_neo4j_val_int(cbm_neo4j_row_col(entry, off + 0), 0); + n->project = cbm_neo4j_val_strdup(cbm_neo4j_row_col(entry, off + 1), ""); + n->label = cbm_neo4j_val_strdup(cbm_neo4j_row_col(entry, off + 2), ""); + n->name = cbm_neo4j_val_strdup(cbm_neo4j_row_col(entry, off + 3), ""); + n->qualified_name = cbm_neo4j_val_strdup(cbm_neo4j_row_col(entry, off + 4), ""); + n->file_path = cbm_neo4j_val_strdup(cbm_neo4j_row_col(entry, off + 5), ""); + n->start_line = (int)cbm_neo4j_val_int(cbm_neo4j_row_col(entry, off + 6), 0); + n->end_line = (int)cbm_neo4j_val_int(cbm_neo4j_row_col(entry, off + 7), 0); + n->properties_json = cbm_neo4j_val_strdup(cbm_neo4j_row_col(entry, off + 8), "{}"); +} + +static void node_free(cbm_node_t *n) { + free((void *)n->project); + free((void *)n->label); + free((void *)n->name); + free((void *)n->qualified_name); + free((void *)n->file_path); + free((void *)n->properties_json); + memset(n, 0, sizeof(*n)); +} + +static int collect_nodes(n4s_ctx_t *x, n4s_q_t *q, cbm_node_t **out, int *count) { + *out = NULL; + *count = 0; + cbm_neo4j_result_t r; + if (q_run(x, q, &r) != CBM_STORE_OK) { + return CBM_STORE_ERR; + } + yyjson_val *data = cbm_neo4j_result_data(&r, 0); + size_t n = yyjson_arr_size(data); + if (n > 0) { + cbm_node_t *arr = calloc(n, sizeof(cbm_node_t)); + if (!arr) { + cbm_neo4j_result_free(&r); + return CBM_STORE_ERR; + } + for (size_t i = 0; i < n; i++) { + node_from_row(yyjson_arr_get(data, i), 0, &arr[i]); + } + *out = arr; + *count = (int)n; + } + cbm_neo4j_result_free(&r); + return CBM_STORE_OK; +} + +/* ── Ops ────────────────────────────────────────────────────────── */ + +static void op_close(void *ctx) { + n4s_ctx_t *x = ctx; + if (x) { + cbm_neo4j_close(x->c); + free(x); + } +} + +static const char *op_error(void *ctx) { + return ((n4s_ctx_t *)ctx)->err; +} + +static int one_node(n4s_ctx_t *x, n4s_q_t *q, cbm_node_t *out) { + cbm_node_t *nodes = NULL; + int count = 0; + if (collect_nodes(x, q, &nodes, &count) != CBM_STORE_OK) { + return CBM_STORE_ERR; + } + if (count == 0) { + return CBM_STORE_NOT_FOUND; + } + *out = nodes[0]; + for (int i = 1; i < count; i++) { + node_free(&nodes[i]); + } + free(nodes); + return CBM_STORE_OK; +} + +static int op_find_node_by_id(void *ctx, int64_t id, cbm_node_t *out) { + n4s_ctx_t *x = ctx; + n4s_q_t q; + if (q_begin(x, &q, + "MATCH (n:CbmNode {db: $db, gen: $gen, id: $id}) RETURN " N4S_NODE_COLS( + "n") " LIMIT 1") != 0) { + return CBM_STORE_ERR; + } + q_int(&q, "id", id); + return one_node(x, &q, out); +} + +static int op_find_node_by_qn(void *ctx, const char *project, const char *qn, cbm_node_t *out) { + n4s_ctx_t *x = ctx; + n4s_q_t q; + if (q_begin(x, &q, + "MATCH (n:CbmNode {db: $db, gen: $gen, qualified_name: $qn}) " + "WHERE $project IS NULL OR n.project = $project RETURN " N4S_NODE_COLS( + "n") " ORDER BY n.id LIMIT 1") != 0) { + return CBM_STORE_ERR; + } + q_str(&q, "qn", qn); + q_str(&q, "project", project); + return one_node(x, &q, out); +} + +static int op_find_nodes(void *ctx, cbm_remote_node_query_t kind, const char *project, + const char *arg, cbm_node_t **out, int *count) { + n4s_ctx_t *x = ctx; + const char *cypher = NULL; + switch (kind) { + case CBM_REMOTE_NODES_ALL: + cypher = "MATCH (n:CbmNode {db: $db, gen: $gen}) " + "WHERE $project IS NULL OR n.project = $project RETURN " N4S_NODE_COLS( + "n") " ORDER BY n.file_path, n.id"; + break; + case CBM_REMOTE_NODES_BY_NAME: + cypher = "MATCH (n:CbmNode {db: $db, gen: $gen, name: $arg}) " + "WHERE $project IS NULL OR n.project = $project RETURN " N4S_NODE_COLS( + "n") " ORDER BY n.id"; + break; + case CBM_REMOTE_NODES_BY_LABEL: + cypher = "MATCH (n:CbmNode {db: $db, gen: $gen, label: $arg}) " + "WHERE $project IS NULL OR n.project = $project RETURN " N4S_NODE_COLS( + "n") " ORDER BY n.id"; + break; + case CBM_REMOTE_NODES_BY_FILE: + cypher = "MATCH (n:CbmNode {db: $db, gen: $gen, file_path: $arg}) " + "WHERE $project IS NULL OR n.project = $project RETURN " N4S_NODE_COLS( + "n") " ORDER BY n.id"; + break; + default: + return CBM_STORE_ERR; + } + n4s_q_t q; + if (q_begin(x, &q, cypher) != 0) { + return CBM_STORE_ERR; + } + q_str(&q, "project", project); + q_str(&q, "arg", arg ? arg : ""); + return collect_nodes(x, &q, out, count); +} + +static int collect_edges(n4s_ctx_t *x, n4s_q_t *q, cbm_edge_t **out, int *count) { + *out = NULL; + *count = 0; + cbm_neo4j_result_t r; + if (q_run(x, q, &r) != CBM_STORE_OK) { + return CBM_STORE_ERR; + } + yyjson_val *data = cbm_neo4j_result_data(&r, 0); + size_t n = yyjson_arr_size(data); + if (n > 0) { + cbm_edge_t *arr = calloc(n, sizeof(cbm_edge_t)); + if (!arr) { + cbm_neo4j_result_free(&r); + return CBM_STORE_ERR; + } + for (size_t i = 0; i < n; i++) { + yyjson_val *e = yyjson_arr_get(data, i); + arr[i].id = cbm_neo4j_val_int(cbm_neo4j_row_col(e, 0), 0); + arr[i].project = cbm_neo4j_val_strdup(cbm_neo4j_row_col(e, 1), ""); + arr[i].source_id = cbm_neo4j_val_int(cbm_neo4j_row_col(e, 2), 0); + arr[i].target_id = cbm_neo4j_val_int(cbm_neo4j_row_col(e, 3), 0); + arr[i].type = cbm_neo4j_val_strdup(cbm_neo4j_row_col(e, 4), ""); + arr[i].properties_json = cbm_neo4j_val_strdup(cbm_neo4j_row_col(e, 5), "{}"); + } + *out = arr; + *count = (int)n; + } + cbm_neo4j_result_free(&r); + return CBM_STORE_OK; +} + +#define N4S_EDGE_RET \ + " RETURN r.id, a.project, a.id, b.id, coalesce(r.type, type(r)), r.properties_json" + +static int op_find_edges(void *ctx, int64_t node_id, bool by_target, const char *type, + cbm_edge_t **out, int *count) { + n4s_ctx_t *x = ctx; + const char *cypher = + by_target + ? "MATCH (a:CbmNode)-[r]->(b:CbmNode {db: $db, gen: $gen, id: $id}) " + "WHERE a.db = $db AND a.gen = $gen AND ($type IS NULL OR r.type = $type)" N4S_EDGE_RET + " ORDER BY r.id" + : "MATCH (a:CbmNode {db: $db, gen: $gen, id: $id})-[r]->(b:CbmNode) " + "WHERE b.db = $db AND b.gen = $gen AND ($type IS NULL OR r.type = $type)" N4S_EDGE_RET + " ORDER BY r.id"; + n4s_q_t q; + if (q_begin(x, &q, cypher) != 0) { + return CBM_STORE_ERR; + } + q_int(&q, "id", node_id); + q_str(&q, "type", type); + return collect_edges(x, &q, out, count); +} + +static int op_find_edges_by_type(void *ctx, const char *project, const char *type, cbm_edge_t **out, + int *count) { + n4s_ctx_t *x = ctx; + n4s_q_t q; + if (q_begin(x, &q, + "MATCH (a:CbmNode {db: $db, gen: $gen})-[r]->(b:CbmNode) " + "WHERE r.type = $type AND ($project IS NULL OR a.project = $project)" N4S_EDGE_RET + " ORDER BY r.id") != 0) { + return CBM_STORE_ERR; + } + q_str(&q, "type", type ? type : ""); + q_str(&q, "project", project); + return collect_edges(x, &q, out, count); +} + +static int op_count(void *ctx, cbm_remote_count_t kind, const char *project, const char *arg) { + n4s_ctx_t *x = ctx; + const char *cypher = NULL; + switch (kind) { + case CBM_REMOTE_COUNT_NODES: + cypher = "MATCH (n:CbmNode {db: $db, gen: $gen}) " + "WHERE $project IS NULL OR n.project = $project RETURN count(n)"; + break; + case CBM_REMOTE_COUNT_EDGES: + cypher = "MATCH (a:CbmNode {db: $db, gen: $gen})-[r]->() " + "WHERE $project IS NULL OR a.project = $project RETURN count(r)"; + break; + case CBM_REMOTE_COUNT_EDGES_BY_TYPE: + cypher = "MATCH (a:CbmNode {db: $db, gen: $gen})-[r]->() " + "WHERE r.type = $arg AND ($project IS NULL OR a.project = $project) " + "RETURN count(r)"; + break; + default: + return CBM_STORE_ERR; + } + n4s_q_t q; + if (q_begin(x, &q, cypher) != 0) { + return CBM_STORE_ERR; + } + q_str(&q, "project", project); + q_str(&q, "arg", arg ? arg : ""); + cbm_neo4j_result_t r; + if (q_run(x, &q, &r) != CBM_STORE_OK) { + return CBM_STORE_ERR; + } + int64_t n = cbm_neo4j_val_int( + cbm_neo4j_row_col(yyjson_arr_get_first(cbm_neo4j_result_data(&r, 0)), 0), 0); + cbm_neo4j_result_free(&r); + return n > INT_MAX ? INT_MAX : (int)n; +} + +static int op_degrees(void *ctx, const int64_t *ids, int id_count, const char *type, int *in, + int *out) { + n4s_ctx_t *x = ctx; + const char *t = (type && type[0]) ? type : NULL; + for (int base = 0; base < id_count; base += N4S_ID_CHUNK) { + int n = id_count - base < N4S_ID_CHUNK ? id_count - base : N4S_ID_CHUNK; + n4s_q_t q; + if (q_begin(x, &q, + "UNWIND range(0, size($ids) - 1) AS i " + "MATCH (n:CbmNode {db: $db, gen: $gen, id: $ids[i]}) " + "RETURN i, size([(n)<-[r]-() WHERE $type IS NULL OR r.type = $type | 1]), " + "size([(n)-[r]->() WHERE $type IS NULL OR r.type = $type | 1])") != 0) { + return CBM_STORE_ERR; + } + q_ids(&q, "ids", ids + base, n); + q_str(&q, "type", t); + cbm_neo4j_result_t r; + if (q_run(x, &q, &r) != CBM_STORE_OK) { + return CBM_STORE_ERR; + } + yyjson_val *data = cbm_neo4j_result_data(&r, 0); + size_t idx; + size_t max; + yyjson_val *e; + yyjson_arr_foreach(data, idx, max, e) { + int64_t i = cbm_neo4j_val_int(cbm_neo4j_row_col(e, 0), -1); + if (i >= 0 && i < n) { + in[base + i] = (int)cbm_neo4j_val_int(cbm_neo4j_row_col(e, 1), 0); + out[base + i] = (int)cbm_neo4j_val_int(cbm_neo4j_row_col(e, 2), 0); + } + } + cbm_neo4j_result_free(&r); + } + return CBM_STORE_OK; +} + +/* ── Traversal ──────────────────────────────────────────────────── */ + +typedef struct { + int64_t *v; + int n; + int cap; +} n4s_ids_t; + +static int ids_push(n4s_ids_t *a, int64_t id) { + if (a->n >= a->cap) { + int cap = a->cap ? a->cap * 2 : 64; + int64_t *v = realloc(a->v, (size_t)cap * sizeof(int64_t)); + if (!v) { + return -1; + } + a->v = v; + a->cap = cap; + } + a->v[a->n++] = id; + return 0; +} + +static int cmp_i64(const void *a, const void *b) { + int64_t x = *(const int64_t *)a; + int64_t y = *(const int64_t *)b; + return (x > y) - (x < y); +} + +/* Open-addressing set of node ids. */ +typedef struct { + int64_t *slots; + unsigned char *used; + size_t cap; + size_t n; +} n4s_set_t; + +static size_t set_hash(int64_t id, size_t cap) { + uint64_t h = (uint64_t)id * 0x9E3779B97F4A7C15ULL; + return (size_t)(h >> 17) & (cap - 1); +} + +static int set_add(n4s_set_t *s, int64_t id); /* fwd */ + +static int set_grow(n4s_set_t *s) { + size_t cap = s->cap ? s->cap * 2 : 256; + n4s_set_t ns = {calloc(cap, sizeof(int64_t)), calloc(cap, 1), cap, 0}; + if (!ns.slots || !ns.used) { + free(ns.slots); + free(ns.used); + return -1; + } + for (size_t i = 0; i < s->cap; i++) { + if (s->used[i]) { + (void)set_add(&ns, s->slots[i]); + } + } + free(s->slots); + free(s->used); + *s = ns; + return 0; +} + +static bool set_has(const n4s_set_t *s, int64_t id) { + if (!s->cap) { + return false; + } + for (size_t i = set_hash(id, s->cap);; i = (i + 1) & (s->cap - 1)) { + if (!s->used[i]) { + return false; + } + if (s->slots[i] == id) { + return true; + } + } +} + +/* 1 when added, 0 when already present, -1 on OOM. */ +static int set_add(n4s_set_t *s, int64_t id) { + if ((s->n + 1) * 2 > s->cap && set_grow(s) != 0) { + return -1; + } + for (size_t i = set_hash(id, s->cap);; i = (i + 1) & (s->cap - 1)) { + if (!s->used[i]) { + s->used[i] = 1; + s->slots[i] = id; + s->n++; + return 1; + } + if (s->slots[i] == id) { + return 0; + } + } +} + +static void set_free(n4s_set_t *s) { + free(s->slots); + free(s->used); + memset(s, 0, sizeof(*s)); +} + +/* Next BFS layer: distinct neighbour ids of `frontier` over the edge types. */ +static int bfs_expand(n4s_ctx_t *x, const n4s_ids_t *frontier, bool inbound, const char **types, + int type_count, n4s_ids_t *next) { + const char *cypher = + inbound ? "UNWIND $ids AS fid MATCH (a:CbmNode {db: $db, gen: $gen, id: fid})<-[r]-(b) " + "WHERE r.type IN $types AND b.db = $db AND b.gen = $gen RETURN DISTINCT b.id" + : "UNWIND $ids AS fid MATCH (a:CbmNode {db: $db, gen: $gen, id: fid})-[r]->(b) " + "WHERE r.type IN $types AND b.db = $db AND b.gen = $gen RETURN DISTINCT b.id"; + for (int base = 0; base < frontier->n; base += N4S_ID_CHUNK) { + int n = frontier->n - base < N4S_ID_CHUNK ? frontier->n - base : N4S_ID_CHUNK; + n4s_q_t q; + if (q_begin(x, &q, cypher) != 0) { + return CBM_STORE_ERR; + } + q_ids(&q, "ids", frontier->v + base, n); + q_types(&q, "types", types, type_count); + cbm_neo4j_result_t r; + if (q_run(x, &q, &r) != CBM_STORE_OK) { + return CBM_STORE_ERR; + } + size_t idx; + size_t max; + yyjson_val *e; + yyjson_arr_foreach(cbm_neo4j_result_data(&r, 0), idx, max, e) { + if (ids_push(next, cbm_neo4j_val_int(cbm_neo4j_row_col(e, 0), 0)) != 0) { + cbm_neo4j_result_free(&r); + return CBM_STORE_ERR; + } + } + cbm_neo4j_result_free(&r); + } + return CBM_STORE_OK; +} + +/* Fetch node rows for ids and place them in visited[] in the given order. */ +static int fill_hops(n4s_ctx_t *x, const int64_t *ids, const int *hops, int n, cbm_node_hop_t **out, + int *out_count) { + *out = NULL; + *out_count = 0; + cbm_node_hop_t *visited = calloc(n > 0 ? (size_t)n : 1, sizeof(cbm_node_hop_t)); + if (!visited) { + return CBM_STORE_ERR; + } + int written = 0; + for (int base = 0; base < n; base += N4S_ID_CHUNK) { + int m = n - base < N4S_ID_CHUNK ? n - base : N4S_ID_CHUNK; + n4s_q_t q; + if (q_begin(x, &q, + "UNWIND range(0, size($ids) - 1) AS i " + "MATCH (n:CbmNode {db: $db, gen: $gen, id: $ids[i]}) " + "RETURN i, " N4S_NODE_COLS("n") " ORDER BY i") != 0) { + free(visited); + return CBM_STORE_ERR; + } + q_ids(&q, "ids", ids + base, m); + cbm_neo4j_result_t r; + if (q_run(x, &q, &r) != CBM_STORE_OK) { + for (int k = 0; k < written; k++) { + node_free(&visited[k].node); + } + free(visited); + return CBM_STORE_ERR; + } + size_t idx; + size_t max; + yyjson_val *e; + yyjson_arr_foreach(cbm_neo4j_result_data(&r, 0), idx, max, e) { + int64_t i = cbm_neo4j_val_int(cbm_neo4j_row_col(e, 0), -1); + if (i < 0 || i >= m) { + continue; + } + /* An id with no node row is skipped, as the SQL JOIN skips it. */ + node_from_row(e, 1, &visited[written].node); + visited[written].hop = hops[base + i]; + written++; + } + cbm_neo4j_result_free(&r); + } + *out = visited; + *out_count = written; + return CBM_STORE_OK; +} + +static int collect_bfs_edges(n4s_ctx_t *x, int64_t root_id, const cbm_node_hop_t *visited, int n, + bool inbound, const char **types, int type_count, int max_edges, + cbm_traverse_result_t *out) { + (void)inbound; + n4s_ids_t ids = {0}; + if (ids_push(&ids, root_id) != 0) { + return CBM_STORE_ERR; + } + for (int i = 0; i < n; i++) { + if (ids_push(&ids, visited[i].node.id) != 0) { + free(ids.v); + return CBM_STORE_ERR; + } + } + char cypher[1024]; + char limit[48] = ""; + if (max_edges > 0) { + (void)snprintf(limit, sizeof(limit), " LIMIT %d", + max_edges < INT_MAX ? max_edges + 1 : max_edges); + } + (void)snprintf(cypher, sizeof(cypher), + "MATCH (a:CbmNode {db: $db, gen: $gen})-[r]->(b:CbmNode {db: $db, gen: $gen}) " + "WHERE a.id IN $ids AND b.id IN $ids AND r.type IN $types " + "RETURN a.name, b.name, r.type, a.id, b.id, r.properties_json " + "ORDER BY a.id, b.id, r.type%s", + limit); + n4s_q_t q; + if (q_begin(x, &q, cypher) != 0) { + free(ids.v); + return CBM_STORE_ERR; + } + q_ids(&q, "ids", ids.v, ids.n); + free(ids.v); + q_types(&q, "types", types, type_count); + cbm_neo4j_result_t r; + if (q_run(x, &q, &r) != CBM_STORE_OK) { + return CBM_STORE_ERR; + } + yyjson_val *data = cbm_neo4j_result_data(&r, 0); + int en = (int)yyjson_arr_size(data); + if (max_edges > 0 && en > max_edges) { + en = max_edges; + out->edges_truncated = true; + } + if (en > 0) { + out->edges = calloc((size_t)en, sizeof(cbm_edge_info_t)); + if (!out->edges) { + cbm_neo4j_result_free(&r); + return CBM_STORE_ERR; + } + for (int i = 0; i < en; i++) { + yyjson_val *e = yyjson_arr_get(data, (size_t)i); + cbm_edge_info_t *ei = &out->edges[i]; + ei->from_name = cbm_neo4j_val_strdup(cbm_neo4j_row_col(e, 0), ""); + ei->to_name = cbm_neo4j_val_strdup(cbm_neo4j_row_col(e, 1), ""); + ei->type = cbm_neo4j_val_strdup(cbm_neo4j_row_col(e, 2), ""); + ei->confidence = 1.0; + ei->source_id = cbm_neo4j_val_int(cbm_neo4j_row_col(e, 3), 0); + ei->target_id = cbm_neo4j_val_int(cbm_neo4j_row_col(e, 4), 0); + ei->properties_json = cbm_neo4j_val_strdup(cbm_neo4j_row_col(e, 5), "{}"); + } + out->edge_count = en; + } + cbm_neo4j_result_free(&r); + return CBM_STORE_OK; +} + +/* Relationship-unique (trail) expansion: Neo4j's own variable-length match + * semantics, distinct (node, hop) pairs in (hop, id) order. */ +static int bfs_trail_ids(n4s_ctx_t *x, int64_t start, bool inbound, const char **types, + int type_count, int max_depth, int probe, n4s_ids_t *ids, + n4s_ids_t *hops) { + char cypher[1024]; + (void)snprintf(cypher, sizeof(cypher), + "MATCH p = (s:CbmNode {db: $db, gen: $gen, id: $id})%s[rs*1..%d]%s(n:CbmNode) " + "WHERE all(x IN rs WHERE x.type IN $types) " + "RETURN DISTINCT n.id AS nid, length(p) AS hop ORDER BY hop, nid LIMIT %d", + inbound ? "<-" : "-", max_depth, inbound ? "-" : "->", probe); + n4s_q_t q; + if (q_begin(x, &q, cypher) != 0) { + return CBM_STORE_ERR; + } + q_int(&q, "id", start); + q_types(&q, "types", types, type_count); + cbm_neo4j_result_t r; + if (q_run(x, &q, &r) != CBM_STORE_OK) { + return CBM_STORE_ERR; + } + size_t idx; + size_t max; + yyjson_val *e; + yyjson_arr_foreach(cbm_neo4j_result_data(&r, 0), idx, max, e) { + if (ids_push(ids, cbm_neo4j_val_int(cbm_neo4j_row_col(e, 0), 0)) != 0 || + ids_push(hops, cbm_neo4j_val_int(cbm_neo4j_row_col(e, 1), 0)) != 0) { + cbm_neo4j_result_free(&r); + return CBM_STORE_ERR; + } + } + cbm_neo4j_result_free(&r); + return CBM_STORE_OK; +} + +/* Shortest-path layers, client side: each node once at its minimal hop, + * layers in ascending id order, stopping once probe rows are known. Seeds + * are excluded in multi mode; a single root reached again by a cycle is + * reported at that hop, exactly like the SQL CTE (which filters hop > 0 + * before taking MIN(hop)). */ +static int bfs_shortest_ids(n4s_ctx_t *x, const int64_t *seeds, int seed_count, bool multi, + bool inbound, const char **types, int type_count, int max_depth, + int probe, n4s_ids_t *ids, n4s_ids_t *hops) { + n4s_set_t reached = {0}; + n4s_set_t seedset = {0}; + n4s_ids_t frontier = {0}; + int rc = CBM_STORE_OK; + for (int i = 0; i < seed_count; i++) { + if (set_add(&seedset, seeds[i]) == 1 && ids_push(&frontier, seeds[i]) != 0) { + rc = CBM_STORE_ERR; + goto done; + } + } + for (int hop = 1; hop <= max_depth && frontier.n > 0 && ids->n < probe; hop++) { + n4s_ids_t next = {0}; + rc = bfs_expand(x, &frontier, inbound, types, type_count, &next); + if (rc != CBM_STORE_OK) { + free(next.v); + goto done; + } + qsort(next.v, (size_t)next.n, sizeof(int64_t), cmp_i64); + frontier.n = 0; + for (int i = 0; i < next.n; i++) { + int64_t id = next.v[i]; + if (i > 0 && next.v[i - 1] == id) { + continue; + } + bool is_seed = set_has(&seedset, id); + if (multi && is_seed) { + continue; + } + int added = set_add(&reached, id); + if (added < 0) { + rc = CBM_STORE_ERR; + break; + } + if (added == 0) { + continue; + } + if (ids->n < probe && (ids_push(ids, id) != 0 || ids_push(hops, hop) != 0)) { + rc = CBM_STORE_ERR; + break; + } + /* A seed re-reached is already expanded from hop 0. */ + if (!is_seed && ids_push(&frontier, id) != 0) { + rc = CBM_STORE_ERR; + break; + } + } + free(next.v); + if (rc != CBM_STORE_OK) { + goto done; + } + } +done: + free(frontier.v); + set_free(&reached); + set_free(&seedset); + return rc; +} + +static int op_bfs(void *ctx, const int64_t *seeds, int seed_count, bool multi, + const char *direction, const char **edge_types, int edge_type_count, + int max_depth, int max_results, bool trail, int max_edges, + cbm_traverse_result_t *out, bool *truncated) { + n4s_ctx_t *x = ctx; + memset(out, 0, sizeof(*out)); + if (truncated) { + *truncated = false; + } + if (max_results < 0) { + max_results = 0; + } + bool inbound = direction && strcmp(direction, "inbound") == 0; + if (!multi) { + int rc = op_find_node_by_id(x, seeds[0], &out->root); + if (rc != CBM_STORE_OK) { + return rc; + } + } + int probe = max_results < INT_MAX ? max_results + 1 : max_results; + n4s_ids_t ids = {0}; + n4s_ids_t hops = {0}; + int rc = CBM_STORE_OK; + if (max_depth > 0 && probe > 0) { + rc = (trail && !multi) ? bfs_trail_ids(x, seeds[0], inbound, edge_types, edge_type_count, + max_depth, probe, &ids, &hops) + : bfs_shortest_ids(x, seeds, seed_count, multi, inbound, edge_types, + edge_type_count, max_depth, probe, &ids, &hops); + } + if (rc == CBM_STORE_OK && ids.n > 0) { + int *hop_ints = malloc((size_t)ids.n * sizeof(int)); + if (!hop_ints) { + rc = CBM_STORE_ERR; + } else { + for (int i = 0; i < ids.n; i++) { + hop_ints[i] = (int)hops.v[i]; + } + rc = fill_hops(x, ids.v, hop_ints, ids.n, &out->visited, &out->visited_count); + free(hop_ints); + } + } + free(ids.v); + free(hops.v); + if (rc != CBM_STORE_OK) { + return rc; + } + if (out->visited_count > max_results) { + /* The probe row fired: drop it and report the ceiling. */ + node_free(&out->visited[max_results].node); + out->visited_count = max_results; + out->truncated = true; + if (truncated) { + *truncated = true; + } + } + if (!multi && out->visited_count > 0 && max_edges != 0) { + rc = collect_bfs_edges(x, seeds[0], out->visited, out->visited_count, inbound, edge_types, + edge_type_count, max_edges, out); + } + return rc; +} + +static const cbm_store_remote_ops_t k_neo4j_ops = { + .close = op_close, + .error = op_error, + .find_node_by_id = op_find_node_by_id, + .find_node_by_qn = op_find_node_by_qn, + .find_nodes = op_find_nodes, + .find_edges = op_find_edges, + .find_edges_by_type = op_find_edges_by_type, + .count = op_count, + .degrees = op_degrees, + .bfs = op_bfs, +}; + +/* ── Attach ─────────────────────────────────────────────────────── */ + +/* Connect to the project's Neo4j graph and read its published generation. */ +static n4s_ctx_t *ctx_open(const char *db_path, const char *project, char *err, size_t err_sz, + int *status) { + cbm_neo4j_config_t cfg; + char synced[N4S_GEN_MAX]; + int rc = cbm_neo4j_sidecar_read(db_path, &cfg, synced, sizeof(synced), err, err_sz); + if (rc == 1) { + *status = CBM_NEO4J_ATTACH_NONE; + return NULL; + } + *status = CBM_NEO4J_ATTACH_FALLBACK; + if (rc != 0) { + return NULL; + } + if (!cfg.password[0]) { + /* Don't spend a failed login (Neo4j rate-limits them) on a request + * that cannot authenticate. */ + (void)snprintf(err, err_sz, + "no Neo4j password for %s: set NEO4J_PASSWORD in the MCP server's " + "environment, or re-run index_repository with --graph-password", + cfg.uri); + return NULL; + } + n4s_ctx_t *x = calloc(1, sizeof(*x)); + if (!x) { + cbm_neo4j_config_wipe(&cfg); + return NULL; + } + x->c = cbm_neo4j_connect(&cfg, err, err_sz); + cbm_neo4j_config_wipe(&cfg); + if (!x->c) { + free(x); + return NULL; + } + (void)snprintf(x->db, sizeof(x->db), "%s", project); + cbm_neo4j_batch_t b; + cbm_neo4j_result_t r; + if (cbm_neo4j_batch_init(&b) != 0) { + op_close(x); + return NULL; + } + yyjson_mut_val *params = + cbm_neo4j_batch_add(&b, "MATCH (p:CbmProject {name: $db}) RETURN p.gen, p.generation"); + yyjson_mut_obj_add_strcpy(b.doc, params, "db", project); + int run_rc = cbm_neo4j_batch_run(x->c, &b, &r); + cbm_neo4j_batch_free(&b); + if (run_rc != 0) { + (void)snprintf(err, err_sz, "%s", cbm_neo4j_error(x->c)); + op_close(x); + return NULL; + } + yyjson_val *row = yyjson_arr_get_first(cbm_neo4j_result_data(&r, 0)); + if (!row) { + cbm_neo4j_result_free(&r); + (void)snprintf(err, err_sz, "project '%s' has not been synced to Neo4j yet", project); + op_close(x); + return NULL; + } + x->gen = cbm_neo4j_val_int(cbm_neo4j_row_col(row, 0), 0); + char *remote_generation = cbm_neo4j_val_strdup(cbm_neo4j_row_col(row, 1), ""); + cbm_neo4j_result_free(&r); + + /* The local SQLite generation must be the one Neo4j mirrors; otherwise + * ids from the two would not agree. */ + char local[N4S_GEN_MAX] = ""; + cbm_store_t *probe = cbm_store_open_path_query(db_path); + if (probe) { + (void)cbm_store_generation(probe, local, sizeof(local)); + cbm_store_close(probe); + } + bool same = remote_generation && local[0] && strcmp(remote_generation, local) == 0; + free(remote_generation); + if (!same) { + (void)snprintf(err, err_sz, + "Neo4j holds an older generation of '%s' than the local index (last sync " + "failed?); re-run index_repository with graph_db to resync", + project); + op_close(x); + return NULL; + } + *status = CBM_NEO4J_ATTACHED; + return x; +} + +cbm_neo4j_attach_t cbm_neo4j_store_attach(cbm_store_t *s, const char *db_path, const char *project, + char *err, size_t err_sz) { + if (err && err_sz) { + err[0] = '\0'; + } + if (!s || !db_path || !project) { + return CBM_NEO4J_ATTACH_NONE; + } + int status = CBM_NEO4J_ATTACH_NONE; + n4s_ctx_t *x = ctx_open(db_path, project, err, err_sz, &status); + if (!x) { + if (status == CBM_NEO4J_ATTACH_FALLBACK) { + cbm_log_warn("neo4j.attach", "project", project, "result", "sqlite_fallback", "err", + err ? err : ""); + } + return (cbm_neo4j_attach_t)status; + } + cbm_store_attach_remote(s, &k_neo4j_ops, x); + return CBM_NEO4J_ATTACHED; +} + +/* ── Native Cypher ──────────────────────────────────────────────── */ + +static char *n4s_strdup(const char *s) { + size_t n = strlen(s); + char *d = malloc(n + 1); + if (d) { + memcpy(d, s, n + 1); + } + return d; +} + +static void native_fail(cbm_cypher_result_t *out, const char *msg) { + free(out->error); + out->error = n4s_strdup(msg ? msg : "Neo4j query failed"); +} + +/* query_graph is a read tool: refuse Cypher that could write. Keywords are + * matched as whole words outside string literals, comments and backticks. */ +static const char *native_write_keyword(const char *q) { + static const char *const kw[] = {"CREATE", "MERGE", "DELETE", "DETACH", "SET", "REMOVE", + "DROP", "FOREACH", "LOAD", "CALL", NULL}; + for (const char *p = q; *p;) { + if (*p == '\'' || *p == '"' || *p == '`') { + char quote = *p++; + while (*p && *p != quote) { + p += (*p == '\\' && p[1]) ? 2 : 1; + } + p += *p ? 1 : 0; + continue; + } + if (p[0] == '/' && p[1] == '/') { + while (*p && *p != '\n') { + p++; + } + continue; + } + if (isalpha((unsigned char)*p) || *p == '_') { + const char *w = p; + while (isalnum((unsigned char)*p) || *p == '_') { + p++; + } + size_t len = (size_t)(p - w); + for (int i = 0; kw[i]; i++) { + if (strlen(kw[i]) == len) { + size_t k = 0; + while (k < len && toupper((unsigned char)w[k]) == kw[i][k]) { + k++; + } + if (k == len) { + return kw[i]; + } + } + } + continue; + } + p++; + } + return NULL; +} + +int cbm_neo4j_native_query(const char *db_path, const char *project, const char *query, + int max_rows, cbm_cypher_result_t *out) { + memset(out, 0, sizeof(*out)); + const char *kw = native_write_keyword(query); + if (kw) { + char msg[160]; + (void)snprintf(msg, sizeof(msg), "native query_graph is read-only: '%s' is not allowed", + kw); + native_fail(out, msg); + return -1; + } + char err[CBM_NEO4J_ERR_MAX] = ""; + int status = 0; + n4s_ctx_t *x = ctx_open(db_path, project, err, sizeof(err), &status); + if (!x) { + native_fail(out, status == CBM_NEO4J_ATTACH_NONE + ? "project is not Neo4j-backed (index it with graph_db first)" + : err); + return -1; + } + n4s_q_t q; + if (q_begin(x, &q, query) != 0) { + op_close(x); + native_fail(out, "out of memory"); + return -1; + } + q_str(&q, "project", project); + cbm_neo4j_result_t r; + if (q_run(x, &q, &r) != CBM_STORE_OK) { + native_fail(out, x->err); + op_close(x); + return -1; + } + yyjson_val *res0 = yyjson_arr_get_first(r.results); + yyjson_val *cols = yyjson_obj_get(res0, "columns"); + yyjson_val *data = yyjson_obj_get(res0, "data"); + int ncols = (int)yyjson_arr_size(cols); + int nrows = (int)yyjson_arr_size(data); + int cap = max_rows > 0 ? max_rows : 100000; + if (nrows > cap) { + nrows = cap; + out->truncated = true; + } + out->col_count = ncols; + out->columns = calloc(ncols > 0 ? (size_t)ncols : 1, sizeof(char *)); + out->rows = calloc(nrows > 0 ? (size_t)nrows : 1, sizeof(char **)); + if (!out->columns || !out->rows) { + cbm_neo4j_result_free(&r); + op_close(x); + native_fail(out, "out of memory"); + return -1; + } + for (int j = 0; j < ncols; j++) { + out->columns[j] = cbm_neo4j_val_strdup(yyjson_arr_get(cols, (size_t)j), ""); + } + for (int i = 0; i < nrows; i++) { + yyjson_val *e = yyjson_arr_get(data, (size_t)i); + const char **row = calloc(ncols > 0 ? (size_t)ncols : 1, sizeof(char *)); + if (!row) { + break; + } + for (int j = 0; j < ncols; j++) { + yyjson_val *v = cbm_neo4j_row_col(e, (size_t)j); + row[j] = cbm_neo4j_val_strdup(v, yyjson_is_null(v) ? "null" : ""); + } + out->rows[i] = row; + out->row_count = i + 1; + } + cbm_neo4j_result_free(&r); + op_close(x); + return 0; +} diff --git a/src/neo4j/neo4j_store.h b/src/neo4j/neo4j_store.h new file mode 100644 index 0000000000..4c5971c83c --- /dev/null +++ b/src/neo4j/neo4j_store.h @@ -0,0 +1,34 @@ +/* + * neo4j_store.h — Answer a store's graph reads from Neo4j. + */ +#ifndef CBM_NEO4J_STORE_H +#define CBM_NEO4J_STORE_H + +#include "cypher/cypher.h" +#include "store/store.h" + +#include + +typedef enum { + CBM_NEO4J_ATTACH_NONE = 0, /* project is plain SQLite */ + CBM_NEO4J_ATTACHED = 1, /* graph reads now go to Neo4j */ + CBM_NEO4J_ATTACH_FALLBACK = + 2, /* Neo4j-backed, but unusable now: SQLite serves (err says why) */ +} cbm_neo4j_attach_t; + +/* If the project database at db_path is marked Neo4j-backed, connect and + * route s's graph reads to the Neo4j generation that mirrors exactly this + * SQLite generation. When Neo4j is unreachable, rejects the credentials, or + * holds a different generation (a failed sync), the store keeps reading the + * identical local copy and err explains the fallback. */ +cbm_neo4j_attach_t cbm_neo4j_store_attach(cbm_store_t *s, const char *db_path, const char *project, + char *err, size_t err_sz); + +/* Run a Cypher query natively in Neo4j for a Neo4j-backed project, with + * $project bound to the project name and $db / $gen bound to its current + * graph. Results are converted to strings like cbm_cypher_execute's. + * Returns 0, or -1 with out->error set. */ +int cbm_neo4j_native_query(const char *db_path, const char *project, const char *query, + int max_rows, cbm_cypher_result_t *out); + +#endif /* CBM_NEO4J_STORE_H */ diff --git a/src/neo4j/neo4j_sync.c b/src/neo4j/neo4j_sync.c new file mode 100644 index 0000000000..ad6bfa8f5d --- /dev/null +++ b/src/neo4j/neo4j_sync.c @@ -0,0 +1,693 @@ +/* + * neo4j_sync.c — Mirror a published project database into Neo4j. + */ +#include "neo4j/neo4j_sync.h" + +#include "foundation/compat_fs.h" +#include "foundation/log.h" +#include "store/store.h" + +#include "sqlite3.h" + +#include +#include +#include + +#define N4_SYNC_BATCH_ROWS 2000 +#define N4_DELETE_BATCH 10000 +#define N4_SIDECAR_SUFFIX ".graph.json" +#define N4_SIDECAR_MAX (64 * 1024) +#define N4_GEN_MAX 128 + +/* Columns written explicitly; a flattened property never overwrites them. */ +static const char *const k_reserved_node_keys[] = { + "db", "gen", "project", "id", + "label", "name", "qualified_name", "file_path", + "start_line", "end_line", "properties_json", NULL}; +static const char *const k_reserved_edge_keys[] = {"id", "type", "properties_json", NULL}; + +static void sync_err(char *err, size_t err_sz, const char *what, const char *detail) { + if (err && err_sz) { + (void)snprintf(err, err_sz, "%s%s%s", what, detail ? ": " : "", detail ? detail : ""); + } +} + +/* ── Sidecar marker ─────────────────────────────────────────────── */ + +int cbm_neo4j_sidecar_path(const char *db_path, char *out, size_t out_sz) { + if (!db_path || !*db_path) { + return -1; + } + int n = snprintf(out, out_sz, "%s%s", db_path, N4_SIDECAR_SUFFIX); + return (n > 0 && (size_t)n < out_sz) ? 0 : -1; +} + +int cbm_neo4j_sidecar_read(const char *db_path, cbm_neo4j_config_t *cfg, char *synced_generation, + size_t synced_sz, char *err, size_t err_sz) { + char path[4096]; + if (synced_generation && synced_sz) { + synced_generation[0] = '\0'; + } + if (cbm_neo4j_sidecar_path(db_path, path, sizeof(path)) != 0) { + return 1; + } + FILE *f = cbm_fopen(path, "rb"); + if (!f) { + return 1; + } + char *buf = malloc(N4_SIDECAR_MAX + 1); + if (!buf) { + (void)fclose(f); + return -1; + } + size_t n = fread(buf, 1, N4_SIDECAR_MAX, f); + (void)fclose(f); + buf[n] = '\0'; + int rc = cbm_neo4j_config_from_meta(buf, cfg, err, err_sz); + if (rc == 0 && synced_generation && synced_sz) { + yyjson_doc *d = yyjson_read(buf, n, 0); + const char *g = yyjson_get_str(yyjson_obj_get(yyjson_doc_get_root(d), "synced_generation")); + (void)snprintf(synced_generation, synced_sz, "%s", g ? g : ""); + yyjson_doc_free(d); + } + free(buf); + return rc == 0 ? 0 : -1; +} + +int cbm_neo4j_sidecar_field(const char *db_path, const char *key, char *out, size_t out_sz) { + char path[4096]; + out[0] = '\0'; + if (cbm_neo4j_sidecar_path(db_path, path, sizeof(path)) != 0) { + return -1; + } + FILE *f = cbm_fopen(path, "rb"); + if (!f) { + return -1; + } + char *buf = malloc(N4_SIDECAR_MAX + 1); + if (!buf) { + (void)fclose(f); + return -1; + } + size_t n = fread(buf, 1, N4_SIDECAR_MAX, f); + (void)fclose(f); + yyjson_doc *d = yyjson_read(buf, n, 0); + free(buf); + const char *v = yyjson_get_str(yyjson_obj_get(yyjson_doc_get_root(d), key)); + (void)snprintf(out, out_sz, "%s", v ? v : ""); + yyjson_doc_free(d); + return 0; +} + +int cbm_neo4j_sidecar_write(const char *db_path, const cbm_neo4j_config_t *cfg, + const char *synced_generation) { + return cbm_neo4j_sidecar_write_status(db_path, cfg, synced_generation, NULL); +} + +int cbm_neo4j_sidecar_write_status(const char *db_path, const cbm_neo4j_config_t *cfg, + const char *synced_generation, const char *last_sync_error) { + char path[4096]; + char tmp[4096 + 8]; + if (cbm_neo4j_sidecar_path(db_path, path, sizeof(path)) != 0) { + return -1; + } + (void)snprintf(tmp, sizeof(tmp), "%s.tmp", path); + char *meta = cbm_neo4j_config_to_meta(cfg); + if (!meta) { + return -1; + } + yyjson_doc *src = yyjson_read(meta, strlen(meta), 0); + free(meta); + yyjson_mut_doc *d = yyjson_doc_mut_copy(src, NULL); + yyjson_doc_free(src); + if (!d) { + return -1; + } + yyjson_mut_obj_add_strcpy(d, yyjson_mut_doc_get_root(d), "synced_generation", + synced_generation ? synced_generation : ""); + if (last_sync_error && last_sync_error[0]) { + yyjson_mut_obj_add_strcpy(d, yyjson_mut_doc_get_root(d), "last_sync_error", + last_sync_error); + } + size_t len = 0; + char *json = yyjson_mut_write(d, YYJSON_WRITE_PRETTY, &len); + yyjson_mut_doc_free(d); + if (!json) { + return -1; + } + FILE *f = cbm_fopen(tmp, "wb"); + if (!f) { + free(json); + return -1; + } + bool ok = fwrite(json, 1, len, f) == len; + ok = (fclose(f) == 0) && ok; + free(json); + if (!ok || cbm_rename_replace(tmp, path) != 0) { + (void)cbm_unlink(tmp); + return -1; + } + return 0; +} + +int cbm_neo4j_sidecar_remove(const char *db_path) { + char path[4096]; + if (cbm_neo4j_sidecar_path(db_path, path, sizeof(path)) != 0) { + return -1; + } + (void)cbm_unlink(path); + return 0; +} + +/* ── Statement helpers ──────────────────────────────────────────── */ + +static int run_simple(cbm_neo4j_t *c, const char *cypher, const char *db, int64_t gen, + cbm_neo4j_result_t *out) { + cbm_neo4j_batch_t b; + if (cbm_neo4j_batch_init(&b) != 0) { + return -1; + } + yyjson_mut_val *params = cbm_neo4j_batch_add(&b, cypher); + if (db) { + yyjson_mut_obj_add_strcpy(b.doc, params, "db", db); + } + yyjson_mut_obj_add_int(b.doc, params, "gen", gen); + int rc = cbm_neo4j_batch_run(c, &b, out); + cbm_neo4j_batch_free(&b); + return rc; +} + +int cbm_neo4j_ensure_schema(cbm_neo4j_t *c) { + static const char *const stmts[] = { + "CREATE CONSTRAINT cbm_project_name IF NOT EXISTS FOR (p:CbmProject) REQUIRE p.name IS " + "UNIQUE", + "CREATE INDEX cbm_node_db IF NOT EXISTS FOR (n:CbmNode) ON (n.db)", + "CREATE INDEX cbm_node_id IF NOT EXISTS FOR (n:CbmNode) ON (n.db, n.gen, n.id)", + "CREATE INDEX cbm_node_qn IF NOT EXISTS FOR (n:CbmNode) ON (n.db, n.gen, n.qualified_name)", + "CREATE INDEX cbm_node_name IF NOT EXISTS FOR (n:CbmNode) ON (n.db, n.gen, n.name)", + "CREATE INDEX cbm_node_file IF NOT EXISTS FOR (n:CbmNode) ON (n.db, n.gen, n.file_path)", + "CREATE INDEX cbm_node_label IF NOT EXISTS FOR (n:CbmNode) ON (n.db, n.gen, n.label)", + NULL}; + /* Schema statements cannot share a transaction with each other in every + * Neo4j version, so send them one per request. */ + for (int i = 0; stmts[i]; i++) { + cbm_neo4j_batch_t b; + if (cbm_neo4j_batch_init(&b) != 0) { + return -1; + } + (void)cbm_neo4j_batch_add(&b, stmts[i]); + int rc = cbm_neo4j_batch_run(c, &b, NULL); + cbm_neo4j_batch_free(&b); + if (rc != 0) { + return -1; + } + } + /* Best effort: wait for freshly created indexes before bulk MATCHes. */ + cbm_neo4j_batch_t b; + if (cbm_neo4j_batch_init(&b) == 0) { + (void)cbm_neo4j_batch_add(&b, "CALL db.awaitIndexes(300)"); + (void)cbm_neo4j_batch_run(c, &b, NULL); + cbm_neo4j_batch_free(&b); + } + return 0; +} + +static bool key_reserved(const char *k, const char *const *reserved) { + for (int i = 0; reserved[i]; i++) { + if (strcmp(k, reserved[i]) == 0) { + return true; + } + } + return false; +} + +/* Copy top-level scalar members of properties_json into obj. Arrays of + * scalars are kept too (Neo4j stores homogeneous lists); nested objects stay + * only inside properties_json. */ +static void flatten_props(yyjson_mut_doc *d, yyjson_mut_val *obj, const char *json, + const char *const *reserved) { + if (!json || !*json || strcmp(json, "{}") == 0) { + return; + } + yyjson_doc *pd = yyjson_read(json, strlen(json), 0); + yyjson_val *root = pd ? yyjson_doc_get_root(pd) : NULL; + if (!yyjson_is_obj(root)) { + yyjson_doc_free(pd); + return; + } + size_t idx; + size_t max; + yyjson_val *k; + yyjson_val *v; + yyjson_obj_foreach(root, idx, max, k, v) { + const char *key = yyjson_get_str(k); + if (!key || key_reserved(key, reserved)) { + continue; + } + if (yyjson_is_str(v) || yyjson_is_num(v) || yyjson_is_bool(v)) { + yyjson_mut_obj_add(obj, yyjson_mut_strcpy(d, key), yyjson_val_mut_copy(d, v)); + } else if (yyjson_is_arr(v) && yyjson_arr_size(v) > 0) { + /* Only homogeneous string or number lists are valid Neo4j values. */ + yyjson_val *first = yyjson_arr_get_first(v); + bool all_str = true; + bool all_num = true; + size_t ai; + size_t amax; + yyjson_val *av; + yyjson_arr_foreach(v, ai, amax, av) { + all_str = all_str && yyjson_is_str(av); + all_num = all_num && yyjson_is_num(av); + } + if ((all_str || all_num) && first) { + yyjson_mut_obj_add(obj, yyjson_mut_strcpy(d, key), yyjson_val_mut_copy(d, v)); + } + } + } + yyjson_doc_free(pd); +} + +static const char *col_text(sqlite3_stmt *st, int i) { + const char *s = (const char *)sqlite3_column_text(st, i); + return s ? s : ""; +} + +/* ── Node / edge streaming ──────────────────────────────────────── */ + +typedef struct { + cbm_neo4j_t *c; + const char *db; + int64_t gen; + cbm_neo4j_batch_t batch; + yyjson_mut_val *rows; + int row_count; + char group[256]; /* label or edge type of the open batch */ + bool is_edge; + int64_t total; +} n4_stream_t; + +static int stream_open_batch(n4_stream_t *s, const char *group) { + if (cbm_neo4j_batch_init(&s->batch) != 0) { + return -1; + } + (void)snprintf(s->group, sizeof(s->group), "%s", group); + char cypher[768]; + bool ident = cbm_neo4j_ident_ok(group); + if (!s->is_edge) { + if (ident) { + (void)snprintf(cypher, sizeof(cypher), + "UNWIND $rows AS r CREATE (n:CbmNode:`%s`) SET n = r", group); + } else { + (void)snprintf(cypher, sizeof(cypher), + "UNWIND $rows AS r CREATE (n:CbmNode) SET n = r"); + } + } else { + /* Types that are not plain identifiers land on CBM_EDGE with the real + * type kept in the `type` property. */ + (void)snprintf(cypher, sizeof(cypher), + "UNWIND $rows AS r " + "MATCH (a:CbmNode {db: $db, gen: $gen, id: r.s}) " + "MATCH (b:CbmNode {db: $db, gen: $gen, id: r.t}) " + "CREATE (a)-[e:`%s`]->(b) SET e = r.p", + ident ? group : "CBM_EDGE"); + } + yyjson_mut_val *params = cbm_neo4j_batch_add(&s->batch, cypher); + yyjson_mut_obj_add_strcpy(s->batch.doc, params, "db", s->db); + yyjson_mut_obj_add_int(s->batch.doc, params, "gen", s->gen); + s->rows = yyjson_mut_arr(s->batch.doc); + yyjson_mut_obj_add_val(s->batch.doc, params, "rows", s->rows); + s->row_count = 0; + return 0; +} + +static int stream_flush(n4_stream_t *s) { + if (!s->batch.doc) { + return 0; + } + int rc = s->row_count > 0 ? cbm_neo4j_batch_run(s->c, &s->batch, NULL) : 0; + s->total += s->row_count; + cbm_neo4j_batch_free(&s->batch); + s->rows = NULL; + s->row_count = 0; + s->group[0] = '\0'; + return rc; +} + +/* Returns the row object to fill, flushing/reopening on group change or when + * the batch is full. NULL on failure. */ +static yyjson_mut_val *stream_row(n4_stream_t *s, const char *group) { + if (s->batch.doc && (strcmp(s->group, group) != 0 || s->row_count >= N4_SYNC_BATCH_ROWS)) { + if (stream_flush(s) != 0) { + return NULL; + } + } + if (!s->batch.doc && stream_open_batch(s, group) != 0) { + return NULL; + } + yyjson_mut_val *row = yyjson_mut_obj(s->batch.doc); + yyjson_mut_arr_append(s->rows, row); + s->row_count++; + return row; +} + +static int sync_nodes(n4_stream_t *s, sqlite3 *db, char *err, size_t err_sz) { + sqlite3_stmt *st = NULL; + if (sqlite3_prepare_v2( + db, + "SELECT id, project, label, name, qualified_name, file_path, start_line, " + "end_line, properties FROM nodes ORDER BY label, id", + -1, &st, NULL) != SQLITE_OK) { + sync_err(err, err_sz, "read nodes", sqlite3_errmsg(db)); + return -1; + } + int rc = 0; + while (sqlite3_step(st) == SQLITE_ROW) { + const char *label = col_text(st, 2); + yyjson_mut_val *row = stream_row(s, label); + if (!row) { + sync_err(err, err_sz, "write nodes to Neo4j", cbm_neo4j_error(s->c)); + rc = -1; + break; + } + yyjson_mut_doc *d = s->batch.doc; + const char *props = col_text(st, 8); + /* Flattened properties first so explicit columns win on collision. */ + flatten_props(d, row, props, k_reserved_node_keys); + yyjson_mut_obj_add_strcpy(d, row, "db", s->db); + yyjson_mut_obj_add_int(d, row, "gen", s->gen); + yyjson_mut_obj_add_int(d, row, "id", sqlite3_column_int64(st, 0)); + yyjson_mut_obj_add_strcpy(d, row, "project", col_text(st, 1)); + yyjson_mut_obj_add_strcpy(d, row, "label", label); + yyjson_mut_obj_add_strcpy(d, row, "name", col_text(st, 3)); + yyjson_mut_obj_add_strcpy(d, row, "qualified_name", col_text(st, 4)); + yyjson_mut_obj_add_strcpy(d, row, "file_path", col_text(st, 5)); + yyjson_mut_obj_add_int(d, row, "start_line", sqlite3_column_int(st, 6)); + yyjson_mut_obj_add_int(d, row, "end_line", sqlite3_column_int(st, 7)); + yyjson_mut_obj_add_strcpy(d, row, "properties_json", props[0] ? props : "{}"); + } + sqlite3_finalize(st); + if (rc == 0 && stream_flush(s) != 0) { + sync_err(err, err_sz, "write nodes to Neo4j", cbm_neo4j_error(s->c)); + rc = -1; + } + cbm_neo4j_batch_free(&s->batch); + return rc; +} + +static int sync_edges(n4_stream_t *s, sqlite3 *db, char *err, size_t err_sz) { + sqlite3_stmt *st = NULL; + if (sqlite3_prepare_v2(db, + "SELECT id, source_id, target_id, type, properties FROM edges " + "ORDER BY type, id", + -1, &st, NULL) != SQLITE_OK) { + sync_err(err, err_sz, "read edges", sqlite3_errmsg(db)); + return -1; + } + int rc = 0; + while (sqlite3_step(st) == SQLITE_ROW) { + const char *type = col_text(st, 3); + yyjson_mut_val *row = stream_row(s, type); + if (!row) { + sync_err(err, err_sz, "write edges to Neo4j", cbm_neo4j_error(s->c)); + rc = -1; + break; + } + yyjson_mut_doc *d = s->batch.doc; + yyjson_mut_obj_add_int(d, row, "s", sqlite3_column_int64(st, 1)); + yyjson_mut_obj_add_int(d, row, "t", sqlite3_column_int64(st, 2)); + yyjson_mut_val *p = yyjson_mut_obj(d); + const char *props = col_text(st, 4); + flatten_props(d, p, props, k_reserved_edge_keys); + yyjson_mut_obj_add_int(d, p, "id", sqlite3_column_int64(st, 0)); + yyjson_mut_obj_add_strcpy(d, p, "type", type); + yyjson_mut_obj_add_strcpy(d, p, "properties_json", props[0] ? props : "{}"); + yyjson_mut_obj_add_val(d, row, "p", p); + } + sqlite3_finalize(st); + if (rc == 0 && stream_flush(s) != 0) { + sync_err(err, err_sz, "write edges to Neo4j", cbm_neo4j_error(s->c)); + rc = -1; + } + cbm_neo4j_batch_free(&s->batch); + return rc; +} + +/* Delete nodes of db matching `where` (a predicate on n using $gen) in + * bounded transactions. */ +static int delete_loop(cbm_neo4j_t *c, const char *db, const char *where, int64_t gen) { + char cypher[512]; + (void)snprintf(cypher, sizeof(cypher), + "MATCH (n:CbmNode {db: $db}) WHERE %s WITH n LIMIT %d DETACH DELETE n " + "RETURN count(*)", + where, N4_DELETE_BATCH); + for (;;) { + cbm_neo4j_result_t r; + if (run_simple(c, cypher, db, gen, &r) != 0) { + return -1; + } + yyjson_val *data = cbm_neo4j_result_data(&r, 0); + int64_t n = cbm_neo4j_val_int(cbm_neo4j_row_col(yyjson_arr_get_first(data), 0), 0); + cbm_neo4j_result_free(&r); + if (n <= 0) { + return 0; + } + } +} + +int cbm_neo4j_sync_db(cbm_neo4j_t *c, const char *db_path, const char *project, + char *generation_out, size_t generation_sz, char *err, size_t err_sz) { + if (cbm_neo4j_ensure_schema(c) != 0) { + sync_err(err, err_sz, "create Neo4j indexes", cbm_neo4j_error(c)); + return -1; + } + cbm_store_t *store = cbm_store_open_path_query(db_path); + if (!store) { + sync_err(err, err_sz, "open project database", db_path); + return -1; + } + char generation[N4_GEN_MAX] = ""; + (void)cbm_store_generation(store, generation, sizeof(generation)); + cbm_project_t proj = {0}; + bool have_proj = cbm_store_get_project(store, project, &proj) == CBM_STORE_OK; + + /* Current generation; a CbmProject is created on first sync. */ + cbm_neo4j_result_t r; + if (run_simple(c, "MERGE (p:CbmProject {name: $db}) RETURN coalesce(p.gen, 0)", project, 0, + &r) != 0) { + sync_err(err, err_sz, "read Neo4j project state", cbm_neo4j_error(c)); + goto fail; + } + /* The cleanup below deletes every generation except old_gen, so old_gen + * must be the published one: exactly one integer row, never a default. */ + yyjson_val *gen_rows = cbm_neo4j_result_data(&r, 0); + yyjson_val *gen_val = cbm_neo4j_row_col(yyjson_arr_get_first(gen_rows), 0); + if (yyjson_arr_size(gen_rows) != 1 || !yyjson_is_int(gen_val)) { + cbm_neo4j_result_free(&r); + sync_err(err, err_sz, "read Neo4j project state", "unexpected generation reply"); + goto fail; + } + int64_t old_gen = yyjson_get_sint(gen_val); + cbm_neo4j_result_free(&r); + int64_t new_gen = old_gen + 1; + + /* Leftovers of an interrupted earlier sync. */ + if (delete_loop(c, project, "n.gen <> $gen", old_gen) != 0) { + sync_err(err, err_sz, "clean up stale Neo4j generation", cbm_neo4j_error(c)); + goto fail; + } + + n4_stream_t s; + memset(&s, 0, sizeof(s)); + s.c = c; + s.db = project; + s.gen = new_gen; + sqlite3 *db = cbm_store_get_db(store); + if (sync_nodes(&s, db, err, err_sz) != 0) { + goto fail_partial; + } + int64_t node_total = s.total; + s.total = 0; + s.is_edge = true; + if (sync_edges(&s, db, err, err_sz) != 0) { + goto fail_partial; + } + int64_t edge_total = s.total; + + /* Flip readers to the new generation. */ + cbm_neo4j_batch_t b; + if (cbm_neo4j_batch_init(&b) != 0) { + goto fail_partial; + } + yyjson_mut_val *params = + cbm_neo4j_batch_add(&b, "MATCH (p:CbmProject {name: $db}) SET p.gen = $gen, " + "p.generation = $generation, p.root_path = $root, " + "p.indexed_at = $indexed_at, p.node_count = $nodes, " + "p.edge_count = $edges"); + yyjson_mut_obj_add_strcpy(b.doc, params, "db", project); + yyjson_mut_obj_add_int(b.doc, params, "gen", new_gen); + yyjson_mut_obj_add_strcpy(b.doc, params, "generation", generation); + yyjson_mut_obj_add_strcpy(b.doc, params, "root", + have_proj && proj.root_path ? proj.root_path : ""); + yyjson_mut_obj_add_strcpy(b.doc, params, "indexed_at", + have_proj && proj.indexed_at ? proj.indexed_at : ""); + yyjson_mut_obj_add_int(b.doc, params, "nodes", node_total); + yyjson_mut_obj_add_int(b.doc, params, "edges", edge_total); + int flip_rc = cbm_neo4j_batch_run(c, &b, NULL); + cbm_neo4j_batch_free(&b); + if (flip_rc != 0) { + sync_err(err, err_sz, "publish Neo4j generation", cbm_neo4j_error(c)); + goto fail_partial; + } + + /* Old generation is now unreachable for readers; reclaim it. A failure + * here leaves garbage that the next sync removes. */ + if (delete_loop(c, project, "n.gen <> $gen", new_gen) != 0) { + cbm_log_warn("neo4j.sync", "phase", "delete_old_generation", "err", cbm_neo4j_error(c)); + } + cbm_log_info("neo4j.sync", "project", project, "uri", cbm_neo4j_uri(c), "status", "ok"); + if (generation_out && generation_sz) { + (void)snprintf(generation_out, generation_sz, "%s", generation); + } + if (have_proj) { + cbm_project_free_fields(&proj); + } + cbm_store_close(store); + return 0; + +fail_partial: + /* Drop what was written for the new generation; readers never saw it. */ + (void)delete_loop(c, project, "n.gen = $gen", new_gen); +fail: + if (have_proj) { + cbm_project_free_fields(&proj); + } + cbm_store_close(store); + return -1; +} + +/* The SQLite generation Neo4j currently publishes for project ("" when the + * project is absent). 0, or -1 on a transport/Cypher failure. */ +static int remote_generation(cbm_neo4j_t *c, const char *project, char *out, size_t out_sz) { + out[0] = '\0'; + cbm_neo4j_result_t r; + if (run_simple(c, "MATCH (p:CbmProject {name: $db}) RETURN p.generation", project, 0, &r) != + 0) { + return -1; + } + yyjson_val *v = cbm_neo4j_row_col(yyjson_arr_get_first(cbm_neo4j_result_data(&r, 0)), 0); + (void)snprintf(out, out_sz, "%s", yyjson_is_str(v) ? yyjson_get_str(v) : ""); + cbm_neo4j_result_free(&r); + return 0; +} + +int cbm_neo4j_drop_project(cbm_neo4j_t *c, const char *project, char *err, size_t err_sz) { + if (delete_loop(c, project, "true", 0) != 0 || + run_simple(c, "MATCH (p:CbmProject {name: $db}) DELETE p", project, 0, NULL) != 0) { + sync_err(err, err_sz, "delete project from Neo4j", cbm_neo4j_error(c)); + return -1; + } + return 0; +} + +int cbm_neo4j_after_publish(const char *db_path, const char *project, + const cbm_neo4j_config_t *explicit_cfg, bool disable, char *err, + size_t err_sz) { + if (err && err_sz) { + err[0] = '\0'; + } + if (disable) { + (void)cbm_neo4j_sidecar_remove(db_path); + return 0; + } + cbm_neo4j_config_t cfg; + char synced[N4_GEN_MAX] = ""; + bool explicit_req = explicit_cfg != NULL; + /* True when a marker for this same target already exists: a failed sync + * is then recorded in it, so it stays visible after this run. */ + bool marked_target = !explicit_req; + if (explicit_req) { + cfg = *explicit_cfg; + /* Same target as last time: keep the recorded generation so an + * unchanged re-index skips the upload. */ + cbm_neo4j_config_t prev; + char perr[256]; + if (cbm_neo4j_sidecar_read(db_path, &prev, synced, sizeof(synced), perr, sizeof(perr)) != + 0 || + strcmp(prev.uri, cfg.uri) != 0 || strcmp(prev.database, cfg.database) != 0) { + synced[0] = '\0'; + } else { + marked_target = true; + } + cbm_neo4j_config_wipe(&prev); + } else { + int rc = cbm_neo4j_sidecar_read(db_path, &cfg, synced, sizeof(synced), err, err_sz); + if (rc == 1) { + return 0; /* plain SQLite project */ + } + if (rc != 0) { + cbm_log_error("neo4j.sync", "project", project, "err", err); + return 0; + } + } + + /* Skip the upload when Neo4j already holds this exact generation. */ + char current[N4_GEN_MAX] = ""; + cbm_store_t *store = cbm_store_open_path_query(db_path); + if (store) { + (void)cbm_store_generation(store, current, sizeof(current)); + cbm_store_close(store); + } + + int result = 0; + cbm_neo4j_t *c = cbm_neo4j_connect(&cfg, err, err_sz); + if (!c) { + result = -1; + } else { + /* The marker only records what was last sent. Skip the upload only + * when Neo4j itself still publishes this exact generation: the graph + * may have been deleted or overwritten there since (a cleared + * database must be refilled by the next run, not reported current). + * The lookup also proves reachability and credentials. */ + bool up_to_date = false; + if (current[0] && strcmp(current, synced) == 0) { + char remote[N4_GEN_MAX] = ""; + if (remote_generation(c, project, remote, sizeof(remote)) != 0) { + sync_err(err, err_sz, "connect to Neo4j", cbm_neo4j_error(c)); + result = -1; + } else { + up_to_date = strcmp(remote, current) == 0; + } + } + if (result == 0 && !up_to_date) { + char done_gen[N4_GEN_MAX] = ""; + if (cbm_neo4j_sync_db(c, db_path, project, done_gen, sizeof(done_gen), err, err_sz) != + 0) { + result = -1; + } else { + (void)snprintf(synced, sizeof(synced), "%s", done_gen); + } + } + } + cbm_neo4j_close(c); + + if (result == 0) { + if (explicit_req) { + cbm_neo4j_config_remember(&cfg); + } + if (cbm_neo4j_sidecar_write(db_path, &cfg, synced) != 0) { + sync_err(err, err_sz, "write graph backend marker", db_path); + result = -1; + } + } else { + cbm_log_error("neo4j.sync", "project", project, "err", err ? err : ""); + /* Keep the last good synced_generation and record why this sync + * failed; the next successful sync rewrites the marker without it. */ + if (marked_target) { + (void)cbm_neo4j_sidecar_write_status(db_path, &cfg, synced, + err && err[0] ? err : "Neo4j sync failed"); + } + } + cbm_neo4j_config_wipe(&cfg); + /* An inherited (watcher / plain re-index) sync failure must not fail the + * SQLite index itself; it is logged and reported through err. */ + return explicit_req ? result : 0; +} diff --git a/src/neo4j/neo4j_sync.h b/src/neo4j/neo4j_sync.h new file mode 100644 index 0000000000..f3b0725f73 --- /dev/null +++ b/src/neo4j/neo4j_sync.h @@ -0,0 +1,64 @@ +/* + * neo4j_sync.h — Mirror a published project database into Neo4j and record + * which projects are Neo4j-backed. + * + * Neo4j model (one "graph" per project database file, keyed by db = the + * project name the .db belongs to): + * (:CbmProject {name, gen, generation, root_path, indexed_at, node_count, edge_count}) + * (:CbmNode: