diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..b33edafad1 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +.git +build +graph-ui/node_modules +**/node_modules diff --git a/.github/workflows/fork-release.yml b/.github/workflows/fork-release.yml new file mode 100644 index 0000000000..2890cb2ae7 --- /dev/null +++ b/.github/workflows/fork-release.yml @@ -0,0 +1,168 @@ +# Fork release: build every platform, smoke-test, publish a GitHub release. +# +# release.yml is upstream's pipeline and publishes through accounts a fork does +# not have (VirusTotal, npm, PyPI, MCP Registry). This workflow keeps exactly +# what the installers need: the eight platform archives (+ MCPB bundles) and +# checksums.txt on a NON-prerelease "latest" release, which is what +# install.sh / install.ps1 -> https://github.com//releases/latest/download/... +# resolve. GitHub never treats a prerelease as "latest", so every release made +# here is a full release, including versions like v0.12.1-neo4j.1. +# +# Run: Actions -> Fork Release -> Run workflow (on main). +name: Fork Release + +on: + workflow_dispatch: + inputs: + version: + description: "Release version, v-prefixed (e.g. v0.12.1-neo4j.1)" + required: true + type: string + release_notes: + description: "Release notes (optional; generated from commits when empty)" + required: false + type: string + virustotal: + description: "VirusTotal-scan the binaries (needs the VIRUS_TOTAL_SCANNER_API_KEY secret)" + required: false + type: boolean + default: false + skip_smoke: + description: "Skip the smoke tests of the built binaries" + required: false + type: boolean + default: false + replace: + description: "Replace an existing release with the same version" + required: false + type: boolean + default: false + +permissions: + contents: read + +concurrency: + group: fork-release + cancel-in-progress: false + +jobs: + preflight: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + # The installers fetch releases/latest/download/; the tag is the + # version verbatim, so a bare "0.12.1" would publish an unreachable tag. + - name: Enforce v-prefixed semver version input + env: + VERSION: ${{ inputs.version }} + run: | + if [[ "$VERSION" =~ ^v[0-9]+\.[0-9]+\.[0-9]+([-.][0-9A-Za-z.]+)?$ ]]; then + echo "version OK: $VERSION" + else + echo "::error::version must be v-prefixed semver (vX.Y.Z[-suffix]), got '$VERSION'" + exit 1 + fi + + build: + needs: [preflight] + permissions: + contents: read + id-token: write + attestations: write + uses: ./.github/workflows/_build.yml + with: + version: ${{ inputs.version }} + attest: false + scan_candidates: ${{ inputs.virustotal }} + secrets: inherit + + smoke: + needs: [build] + if: ${{ !cancelled() && needs.build.result == 'success' && !inputs.skip_smoke }} + uses: ./.github/workflows/_smoke.yml + with: + broad_platforms: false + + publish: + needs: [build, smoke] + # A skipped smoke is accepted only when it was skipped on request. + if: >- + ${{ !cancelled() + && needs.build.result == 'success' + && (needs.smoke.result == 'success' + || (inputs.skip_smoke && needs.smoke.result == 'skipped')) }} + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Download the packaged containers + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: binaries-* + merge-multiple: true + path: dist + + - name: List artifacts and require every platform + working-directory: dist + run: | + ls -la + for target in linux-amd64 linux-arm64 linux-amd64-portable linux-arm64-portable \ + darwin-amd64 darwin-arm64; do + test -f "codebase-memory-mcp-$target.tar.gz" || { echo "::error::missing $target archive"; exit 1; } + done + for target in windows-amd64 windows-arm64; do + test -f "codebase-memory-mcp-$target.zip" || { echo "::error::missing $target archive"; exit 1; } + done + + # install.sh / install.ps1 refuse any archive whose SHA-256 is not listed + # here, so this file is what makes the release installable. + - name: Generate checksums + working-directory: dist + run: | + shopt -s nullglob + sha256sum *.tar.gz *.zip *.mcpb > checksums.txt + cat checksums.txt + + - name: Delete existing release + if: ${{ inputs.replace }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VERSION: ${{ inputs.version }} + run: gh release delete "$VERSION" --yes --cleanup-tag --repo "$GITHUB_REPOSITORY" || true + + # Tag the dispatched commit (the one the binaries were built from), not + # whatever main points at by now. + - name: Publish the release + working-directory: dist + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VERSION: ${{ inputs.version }} + NOTES: ${{ inputs.release_notes }} + run: | + shopt -s nullglob + notes_args=(--generate-notes) + if [ -n "$NOTES" ]; then notes_args=(--notes "$NOTES"); fi + gh release create "$VERSION" \ + --repo "$GITHUB_REPOSITORY" \ + --target "$GITHUB_SHA" \ + --title "$VERSION" \ + --latest \ + "${notes_args[@]}" \ + *.tar.gz *.zip *.mcpb checksums.txt + { + echo "### Released $VERSION" + echo + echo '```bash' + echo "curl -fsSL https://raw.githubusercontent.com/$GITHUB_REPOSITORY/main/install.sh | bash" + echo '```' + echo '```powershell' + echo "Invoke-WebRequest -Uri https://raw.githubusercontent.com/$GITHUB_REPOSITORY/main/install.ps1 -OutFile install.ps1" + echo 'Unblock-File .\install.ps1' + echo 'PowerShell -ExecutionPolicy Bypass -File .\install.ps1' + echo '```' + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 707c699246..3cea781fed 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -44,6 +44,8 @@ permissions: jobs: # ── Security (starts immediately, blocks only verify) ─────────── security: + # Upstream-only: forks release through fork-release.yml (see preflight). + if: ${{ github.repository == 'DeusData/codebase-memory-mcp' }} uses: ./.github/workflows/_security.yml # No job in _security.yml reads code scanning or the Actions API any more # (the CodeQL gate moved to codeql.yml), so the token stays read-only. @@ -57,6 +59,7 @@ jobs: # a called workflow cannot request more than its caller grants, so # withholding either permission fails the whole run at startup. codeql: + if: ${{ github.repository == 'DeusData/codebase-memory-mcp' }} uses: ./.github/workflows/codeql.yml permissions: contents: read @@ -72,6 +75,15 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 steps: + # This pipeline publishes through upstream's VirusTotal, npm, PyPI and + # MCP Registry accounts, which a fork does not have. Stop at once with a + # pointer instead of failing hours later. + - name: Refuse to run outside the upstream repository + if: ${{ !(github.repository == 'DeusData/codebase-memory-mcp') }} + run: | + echo "::error::release.yml is upstream's pipeline. On this fork run the 'Fork Release' workflow (.github/workflows/fork-release.yml) instead." + exit 1 + - name: Enforce v-prefixed semver version input env: VERSION: ${{ inputs.version }} @@ -160,6 +172,7 @@ jobs: # release-draft's needs: the report is evidence for the release notes, never # a publication gate. memwaste: + if: ${{ github.repository == 'DeusData/codebase-memory-mcp' }} uses: ./.github/workflows/_memwaste.yml with: mode: full diff --git a/Makefile.cbm b/Makefile.cbm index 3b93146f1f..0614650390 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -184,7 +184,7 @@ ifeq ($(IS_MINGW),yes) # two Windows builds of identical source are never byte-identical and every # release is a brand-new file to a reputation system. Same motivation as the # mimalloc __DATE__ patch; ELF and Mach-O have no equivalent field to clear. -WIN32_LIBS := -lws2_32 -lpsapi -lshell32 -ladvapi32 -lbcrypt -Wl,--allow-multiple-definition -Wl,--stack,8388608 -Wl,--no-insert-timestamp -static +WIN32_LIBS := -lws2_32 -lwinhttp -lpsapi -lshell32 -ladvapi32 -lbcrypt -Wl,--allow-multiple-definition -Wl,--stack,8388608 -Wl,--no-insert-timestamp -static endif # STATIC=1 produces a fully static binary (for Alpine/musl portable builds) @@ -454,6 +454,9 @@ STORE_SRCS = src/store/store.c src/store/graph_mem.c src/store/architecture_proj # Cypher module (new) CYPHER_SRCS = src/cypher/cypher.c +# Neo4j graph backend (HTTP API client, sync, remote store reads) +NEO4J_SRCS = src/neo4j/neo4j_http.c src/neo4j/neo4j_client.c src/neo4j/neo4j_sync.c src/neo4j/neo4j_store.c + # MCP server module (new) MCP_SRCS = src/mcp/mcp.c src/mcp/index_supervisor.c src/mcp/compact_out.c src/mcp/architecture_jobs.c \ src/mcp/test_impact.c src/mcp/test_impact_diff.c src/mcp/test_impact_classify.c src/mcp/test_impact_source.c src/mcp/test_impact_seed.c src/mcp/test_impact_engine.c src/mcp/test_impact_engine_snapshot.c src/mcp/test_impact_engine_reach.c src/mcp/test_impact_engine_artifact.c src/mcp/test_impact_cli.c src/mcp/test_impact_artifact.c src/mcp/test_impact_changes.c src/mcp/test_impact_origins.c src/mcp/test_impact_coverage.c src/mcp/test_impact_config.c src/mcp/test_impact_select.c src/mcp/test_impact_policy.c src/mcp/test_impact_git.c src/mcp/test_impact_inventory.c src/mcp/test_impact_inventory_input.c src/mcp/test_impact_tree.c src/mcp/test_impact_tree_audit.c src/mcp/test_impact_tree_read.c src/mcp/test_impact_tree_batch.c src/mcp/test_impact_tree_digest.c src/mcp/test_impact_tree_names.c src/mcp/test_impact_tree_plan.c src/mcp/test_impact_tree_posix.c src/mcp/test_impact_tree_posix_acl.c src/mcp/test_impact_tree_posix_io.c src/mcp/test_impact_tree_util.c src/mcp/test_impact_tree_windows.c src/mcp/test_impact_tree_windows_acl.c src/mcp/test_impact_tree_windows_io.c \ @@ -689,7 +692,7 @@ TRE_CFLAGS = -std=c11 -g -O1 -w -Ivendored/tre YYJSON_SRC = vendored/yyjson/yyjson.c # All production sources -PROD_SRCS = $(MEMWASTE_SRCS) $(FOUNDATION_SRCS) $(STORE_SRCS) $(CYPHER_SRCS) $(MCP_SRCS) $(DAEMON_SRCS) $(DISCOVER_SRCS) $(GRAPH_BUFFER_SRCS) $(PIPELINE_SRCS) $(SIMHASH_SRCS) $(SEMANTIC_SRCS) $(TRACES_SRCS) $(WATCHER_SRCS) $(GIT_SRCS) $(CLI_SRCS) $(UI_SRCS) $(YYJSON_SRC) +PROD_SRCS = $(MEMWASTE_SRCS) $(FOUNDATION_SRCS) $(STORE_SRCS) $(CYPHER_SRCS) $(NEO4J_SRCS) $(MCP_SRCS) $(DAEMON_SRCS) $(DISCOVER_SRCS) $(GRAPH_BUFFER_SRCS) $(PIPELINE_SRCS) $(SIMHASH_SRCS) $(SEMANTIC_SRCS) $(TRACES_SRCS) $(WATCHER_SRCS) $(GIT_SRCS) $(CLI_SRCS) $(UI_SRCS) $(YYJSON_SRC) EXISTING_C_SRCS = $(EXTRACTION_SRCS) $(LSP_SRCS) $(TS_RUNTIME_SRC) \ $(GRAMMAR_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) @@ -754,7 +757,8 @@ TEST_STORE_SRCS = \ tests/test_store_bulk.c \ tests/test_store_pragmas.c \ tests/test_store_checkpoint.c \ - tests/test_dump_verify_io.c + tests/test_dump_verify_io.c \ + tests/test_neo4j.c TEST_CYPHER_SRCS = \ tests/test_cypher.c @@ -1392,7 +1396,7 @@ SYSROOT = $(shell xcrun --show-sdk-path 2>/dev/null) SYSROOT_FLAG = $(if $(SYSROOT),-isysroot $(SYSROOT),) # Our source files (excluding vendored, grammars, tree-sitter runtime) -LINT_SRCS = $(FOUNDATION_SRCS) $(STORE_SRCS) $(CYPHER_SRCS) $(MCP_SRCS) $(DAEMON_SRCS) \ +LINT_SRCS = $(FOUNDATION_SRCS) $(STORE_SRCS) $(CYPHER_SRCS) $(NEO4J_SRCS) $(MCP_SRCS) $(DAEMON_SRCS) \ $(DISCOVER_SRCS) $(GRAPH_BUFFER_SRCS) $(PIPELINE_SRCS) $(SIMHASH_SRCS) $(SEMANTIC_SRCS) \ $(TRACES_SRCS) $(WATCHER_SRCS) $(CLI_SRCS) $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) \ $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) $(MAIN_SRC) diff --git a/README.md b/README.md index 7cdac83bf1..c9b5ff6be9 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,13 @@ # codebase-memory-mcp: local code intelligence for AI agents via MCP +> **Fork notice.** This is [haiderRizvii/codebase-memory-mcp](https://github.com/haiderRizvii/codebase-memory-mcp), a fork of +> [DeusData/codebase-memory-mcp](https://github.com/DeusData/codebase-memory-mcp) that adds a +> [Neo4j graph backend](#neo4j-graph-backend) and [Docker-based graph builds](docker/graph/README.md). +> The install commands below install **this fork**, and `codebase-memory-mcp update` keeps +> updating from it. Releases are built by `.github/workflows/fork-release.yml`; unlike +> upstream releases they are not VirusTotal-scanned unless that workflow's `virustotal` +> option is used. The badges below describe the upstream project. + [![GitHub Release](https://img.shields.io/github/v/release/DeusData/codebase-memory-mcp?style=flat&color=blue)](https://github.com/DeusData/codebase-memory-mcp/releases/latest) [![License](https://img.shields.io/badge/license-MIT-green)](LICENSE) [![CI](https://img.shields.io/github/actions/workflow/status/DeusData/codebase-memory-mcp/release.yml?branch=main&label=main%20CI)](https://github.com/DeusData/codebase-memory-mcp/actions/workflows/release.yml) @@ -70,7 +78,7 @@ The high-resolution GIFs show CBM Lens Beta on 20 October 2026; the still images **Install the latest release** (macOS / Linux): ```bash -curl -fsSL https://raw.githubusercontent.com/DeusData/codebase-memory-mcp/main/install.sh | bash +curl -fsSL https://raw.githubusercontent.com/haiderRizvii/codebase-memory-mcp/main/install.sh | bash ``` The standard native release includes CBM Lens. Start it after installation with `codebase-memory-mcp daemon start --open --port=9749`. The [Nix packages](#nix-flake) and [source builds](#build-from-source) also offer builds without the UI. @@ -78,7 +86,7 @@ The standard native release includes CBM Lens. Start it after installation with **Windows** (PowerShell): ```powershell # 1. Download the installer -Invoke-WebRequest -Uri https://raw.githubusercontent.com/DeusData/codebase-memory-mcp/main/install.ps1 -OutFile install.ps1 +Invoke-WebRequest -Uri https://raw.githubusercontent.com/haiderRizvii/codebase-memory-mcp/main/install.ps1 -OutFile install.ps1 # 2. (Optional but recommended) Inspect the script notepad install.ps1 @@ -108,7 +116,7 @@ Restart your coding agent. Say **"Index this project"**: done.
Manual install -1. **Download** the archive for your platform from the [latest release](https://github.com/DeusData/codebase-memory-mcp/releases/latest): +1. **Download** the archive for your platform from the [latest release](https://github.com/haiderRizvii/codebase-memory-mcp/releases/latest): - `codebase-memory-mcp--.tar.gz` (macOS/Linux) or `.zip` (Windows) 2. **Extract and install** (each archive includes `install.sh` or `install.ps1`): @@ -438,6 +446,48 @@ Share a compressed graph snapshot so teammates can reuse an indexed baseline. The artifact provides a shared baseline while each developer keeps a local query database. +## Neo4j Graph Backend + +Store a project's graph in Neo4j and have the graph tools read it from there. + +```bash +codebase-memory-mcp cli index_repository --repo-path /path/to/repo \ + --graph-db http://neo4j-host:7474 --graph-user neo4j --graph-password '***' \ + [--graph-database neo4j] +``` + +- **Transport**: Neo4j's HTTP API (`http://host:7474`, or `https://host:7473` on Windows builds). `bolt://` and `neo4j://` URLs are refused with a hint to use the HTTP port. Neo4j 4.4 and 5.x, Community or Enterprise. +- **Credentials**: flags win; otherwise `NEO4J_USER` (or `NEO4J_USERNAME`), `NEO4J_PASSWORD` and `NEO4J_DATABASE` are used, defaulting to user `neo4j` and database `neo4j`. Indexing fails fast, before any work starts, when Neo4j is unreachable or rejects the login. +- **Password handling**: the password is never written to disk and is never put on the index worker's command line. The daemon that ran the index keeps it in memory for later reads. After a daemon restart, reads need `NEO4J_PASSWORD` in the environment of the MCP server (or of whichever session starts the daemon); see the note under Environment Variables. +- **What lives where**: indexing still builds the local SQLite index, which serves incremental re-indexing, coverage, ADRs, full-text and semantic search. After every successful publish (a full index, an incremental update, or the watcher's auto-refresh), the graph is mirrored into Neo4j. The marker `/.db.graph.json` (URL, user and database, no password) keeps later re-indexes syncing to the same target with no flags. +- **Reads**: graph data comes from Neo4j in every tool: + - Node lookups by id, qualified name, QN suffix, overload base, name, label and file. + - Edge lookups, degrees, caller and callee names, node and edge counts, the indexed file list, and label, type and property schema. + - Every BFS: shortest-path, multi-seed, and the trail expansion behind variable-length Cypher. + - `search_graph`'s structured search (name/QN regex, label, file glob, degree, relationship and entry-point filters). Neo4j narrows the candidates, and the same regex and glob code as the local search applies the exact filter, so results are identical. + + That covers `search_graph`, `query_graph`, `trace_path`, `get_code_snippet`, `get_graph_schema`, `search_code`, `index_status` and most of `detect_changes`. A few reads have no Neo4j equivalent and use the local copy, which holds exactly the same generation: + - BM25 text ranking (`search_graph` `query=`) and `semantic_query` + - `get_architecture`'s aggregates + - the impact walk in `detect_changes` + - file outlines and the UI +- **Which backend answered**: every reply for a Neo4j-backed project names its source in the text and in `structuredContent.graph_backend`: + - `served_by: neo4j`: all graph data came from Neo4j. + - `neo4j+local`: the reply also lists what came from the local copy. + - `local` + - `sqlite_fallback`: with the reason. + + The daemon log (`/logs/cbm-daemon.log`) records the same per call (`graph.backend tool=… served_by=… remote_queries=N`). With `CBM_LOG_LEVEL=debug` it also logs each Cypher round trip (`neo4j.query`). +- **Consistency**: each sync writes a new generation, flips `(:CbmProject {name}).gen`, then deletes the old one, so readers never see a half-written graph. If Neo4j is down, rejects the login, or holds an older generation (a failed sync), reads fall back to the identical local copy and the reply says so. `index_status` reports `graph_backend.status` as `serving` or as `sqlite_fallback` with the reason, plus `in_sync` and `last_sync_error` for background syncs. +- **Login safety**: an indexed project first uses the login it was last synced with, then `NEO4J_PASSWORD`. After Neo4j rejects a login, the same credentials are not retried for 60 s, because every rejected attempt counts toward Neo4j's lockout, which would then refuse the correct password too. +- **Renamed databases**: a project's database found under another file name still uses Neo4j. Its marker is also looked up by project name. +- **Native Cypher**: `query_graph` with `native: true` runs the query in Neo4j itself (read-only; write clauses and `CALL` are refused), with `$project`, `$db` and `$gen` bound: + ```cypher + MATCH (f:Function {db:$db, gen:$gen})-[:CALLS]->(g) RETURN f.name, count(g) AS fan_out ORDER BY fan_out DESC LIMIT 10 + ``` +- **Model**: nodes are `(:CbmNode:
@@ -991,6 +1041,7 @@ is preserved. See [Index resource limits](docs/INDEX_RESOURCE_LIMITS.md). | `CBM_LOG_LEVEL` | role-aware | Set the minimum log level. Thin MCP/CLI/hook frontends default to `warn`; the detached daemon and its supervised index workers default to `info` so lifecycle and liveness records remain available. Accepted values (case-insensitive): `debug`, `info`, `warn`, `error`, `none`; or their numeric equivalents `0` to `4`. A physical worker retains INFO liveness records even under a stricter override because its private log drives the supervisor's no-progress timeout. Frontend messages go to that session's stderr; detached daemon events go to `${CBM_CACHE_DIR}/logs/cbm-daemon.log`. Stdout is reserved for MCP JSON-RPC. | | `CBM_WORKERS` | *(detected)* | Override the parallel-indexing worker count returned by `cbm_default_worker_count`. Useful inside containers where `sysconf(_SC_NPROCESSORS_ONLN)` reports host CPUs rather than the cgroup's effective quota. Range 1 to 256; invalid values are ignored with a warning. | | `CBM_MEM_BUDGET_MB` | *(detected)* | Override the in-memory graph budget with an explicit cap in MiB, taking precedence over the derived default (`ram_fraction × total_RAM`, adjusted for available memory and daemon worker caps). Useful on bare-metal hosts without a cgroup limit, or to pin a budget *below* the cgroup limit so headroom is left for sibling processes. Must be a positive integer; it is clamped to detected total RAM (logged as `mem.budget.clamped`), and non-numeric or non-positive values are ignored with a warning (`mem.budget.env.invalid`). | +| `NEO4J_USER` / `NEO4J_PASSWORD` / `NEO4J_DATABASE` | `neo4j` / *(unset)* / `neo4j` | Fallback credentials for the [Neo4j graph backend](#neo4j-graph-backend) when `index_repository` is called without `graph_user` / `graph_password` / `graph_database`. `NEO4J_PASSWORD` is also what lets a restarted daemon read a Neo4j-backed project. | | `CBM_DUMP_VERIFY_MIN_RATIO` | `0.5` | After indexing, compare persisted SQLite node count to the in-memory dump count. When persisted nodes fall below this fraction of committed nodes (and committed > 50), `index_repository` returns `status:"degraded"` instead of silent `indexed`. Range 0 to 1; set `0` to disable. Invalid values are ignored with a warning. | Environment used by daemon-owned components (such as diagnostics, daemon logging, and process-wide indexing resource limits) is captured from the first daemon-backed session that starts the daemon. Later sessions join that process and cannot replace those values. To change them, close all daemon-backed sessions, update the relevant agent configurations consistently, and restart a session. `CBM_ALLOWED_ROOT` remains session-specific, a conflicting `CBM_CACHE_DIR` is rejected, and a one-shot CLI command is not exempt from the rule above: it connects to the coordination daemon like any other session, starting one if none is running. Its own environment becomes the captured daemon-owned environment only when its invocation is the one that starts the daemon; joining an already-running daemon, it inherits that daemon's already-captured values instead. diff --git a/docker/graph/Dockerfile b/docker/graph/Dockerfile new file mode 100644 index 0000000000..1dda804819 --- /dev/null +++ b/docker/graph/Dockerfile @@ -0,0 +1,33 @@ +# cbm-graph: build a repository's code graph into Neo4j from a container. +# +# docker build -f docker/graph/Dockerfile -t cbm-graph:local . +# +# See docker/graph/README.md for running it (cbm-graph.ps1 on Windows). + +# ── Build ──────────────────────────────────────────────────────────── +FROM gcc:14 AS build +RUN apt-get update -qq \ + && apt-get install -y -qq --no-install-recommends zlib1g-dev \ + && rm -rf /var/lib/apt/lists/* +WORKDIR /src +COPY . . +RUN make -f Makefile.cbm cbm BUILD_DIR=/build -j"$(nproc)" \ + && /build/codebase-memory-mcp --version + +# ── Runtime ────────────────────────────────────────────────────────── +# gcc:14 is Debian 13; the binary needs its newer libstdc++/libgcc_s. +FROM debian:trixie-slim +RUN apt-get update -qq \ + && apt-get install -y -qq --no-install-recommends git rsync zlib1g ca-certificates curl \ + && rm -rf /var/lib/apt/lists/* +COPY --from=build /usr/local/lib64/libstdc++.so.6* /usr/local/lib64/libgcc_s.so.1 /usr/local/lib64/ +RUN echo /usr/local/lib64 > /etc/ld.so.conf.d/gcc14.conf && ldconfig +COPY --from=build /build/codebase-memory-mcp /usr/local/bin/codebase-memory-mcp +COPY docker/graph/cbm-graph.sh /usr/local/bin/cbm-graph +RUN sed -i 's/\r$//' /usr/local/bin/cbm-graph && chmod 0755 /usr/local/bin/cbm-graph + +# Index cache (SQLite sidecars + working copies) lives in /data: mount a +# named volume there so re-runs are incremental. +ENV HOME=/data/home +VOLUME ["/data"] +ENTRYPOINT ["cbm-graph"] diff --git a/docker/graph/README.md b/docker/graph/README.md new file mode 100644 index 0000000000..da38f59e9c --- /dev/null +++ b/docker/graph/README.md @@ -0,0 +1,38 @@ +# cbm-graph: code graphs into Neo4j from Docker + +Builds a repository's code graph and stores it in Neo4j. No local C toolchain needed. + +## One-time setup + +```powershell +# 1. Build the image (from the fork root, ~10-15 min) +cd D:\ai-skills\codebase-memory-mcp-fork +docker build -f docker/graph/Dockerfile -t cbm-graph:local . + +# 2. Store the Neo4j password for new terminals +setx NEO4J_PASSWORD "ChangeMe_Fleet123" +``` + +## Graph a repository + +```powershell +D:\ai-skills\codebase-memory-mcp-fork\docker\graph\cbm-graph.ps1 -RepoPath "D:\path\to\repo" +``` + +| Option | Default | Meaning | +|---|---|---| +| `-RepoPath` | required | Repository folder. It is mounted read-only and never modified. | +| `-Name` | folder name | Project name in Neo4j (`CbmProject.name`, `CbmNode.db`). | +| `-Fresh` | off | Drop the project's graph and rebuild from scratch. Without it, re-runs are incremental. | +| `-IgnoreFile` | `D:\cbm-fleet\.cbmignore` | Extra ignore rules, merged with the repo's own `.cbmignore`. Pass `''` to skip. | +| `-Neo4jUri` | `http://host.docker.internal:7474` | Neo4j HTTP endpoint (not Bolt). | +| `-Neo4jUser` / `-Neo4jDatabase` | `neo4j` / `neo4j` | | + +The script prints node and edge counts plus a Cypher query to paste into Neo4j Browser (http://localhost:7474). + +## How it works + +- The container syncs the repo into the `cbm-graph-data` volume. Paths matched by the ignore rules are skipped. It then writes the merged `.cbmignore` at the copy's root and indexes the copy. Your repo is never touched, and repeat runs only copy what changed. +- The graph is published to Neo4j generation by generation, so a re-run never leaves a half-written graph. +- To remove a project's graph, run in Neo4j Browser: + `MATCH (n:CbmNode {db:''}) DETACH DELETE n` and `MATCH (p:CbmProject {name:''}) DELETE p`. diff --git a/docker/graph/cbm-graph.cmd b/docker/graph/cbm-graph.cmd new file mode 100644 index 0000000000..686468d4b6 --- /dev/null +++ b/docker/graph/cbm-graph.cmd @@ -0,0 +1,6 @@ +@echo off +rem Launcher for cbm-graph.ps1 that works from cmd.exe and PowerShell alike +rem (a .ps1 double-clicked or typed in cmd opens in an editor instead of running). +rem Usage: cbm-graph -RepoPath "D:\path\to\repo" [-Name name] [-Fresh] +powershell.exe -NoProfile -ExecutionPolicy Bypass -File "%~dp0cbm-graph.ps1" %* +exit /b %ERRORLEVEL% diff --git a/docker/graph/cbm-graph.ps1 b/docker/graph/cbm-graph.ps1 new file mode 100644 index 0000000000..be5a12bf7f --- /dev/null +++ b/docker/graph/cbm-graph.ps1 @@ -0,0 +1,69 @@ +<# +.SYNOPSIS + Build a repository's code graph into Neo4j using the cbm-graph Docker image. + +.EXAMPLE + .\cbm-graph.ps1 -RepoPath "D:\batch applications\alias-directory-sync-service" + +.EXAMPLE + .\cbm-graph.ps1 -RepoPath "D:\repos\my-service" -Name my-service -Fresh + +.NOTES + The Neo4j password is read from $env:NEO4J_PASSWORD (set it once with + setx NEO4J_PASSWORD "..." and open a new terminal). It is handed to the + container through the environment, never on a command line. +#> +param( + [Parameter(Mandatory = $true)][string]$RepoPath, + [string]$Name, + [switch]$Fresh, + [string]$IgnoreFile = "D:\cbm-fleet\.cbmignore", + [string]$Neo4jUri = "http://host.docker.internal:7474", + [string]$Neo4jUser = "neo4j", + [string]$Neo4jDatabase = "neo4j", + [string]$Image = "cbm-graph:local", + [string]$CacheVolume = "cbm-graph-data" +) +$ErrorActionPreference = "Stop" + +if (-not (Test-Path -LiteralPath $RepoPath -PathType Container)) { + throw "Repository folder not found: $RepoPath" +} +$RepoPath = (Resolve-Path -LiteralPath $RepoPath).Path +if (-not $env:NEO4J_PASSWORD) { + throw 'Set the Neo4j password first: $env:NEO4J_PASSWORD = "..." (or setx NEO4J_PASSWORD "..." for every new terminal)' +} +if (-not $Name) { + $Name = (Split-Path -Leaf $RepoPath).ToLowerInvariant() -replace '[^a-z0-9._-]+', '-' + $Name = $Name.Trim('-') +} +if ($Name -notmatch '^[A-Za-z0-9._-]+$') { throw "Invalid -Name '$Name'" } + +docker image inspect $Image *> $null +if ($LASTEXITCODE -ne 0) { + throw "Docker image '$Image' not found. Build it from the fork root: docker build -f docker/graph/Dockerfile -t $Image ." +} + +$dockerArgs = @( + "run", "--rm", + "--add-host=host.docker.internal:host-gateway", + "-e", "CBM_NAME=$Name", + "-e", "CBM_FRESH=$(if ($Fresh) { '1' } else { '0' })", + "-e", "NEO4J_URI=$Neo4jUri", + "-e", "NEO4J_USER=$Neo4jUser", + "-e", "NEO4J_DATABASE=$Neo4jDatabase", + "-e", "NEO4J_PASSWORD", + "-v", "${RepoPath}:/input/repo:ro", + "-v", "${CacheVolume}:/data" +) +if ($IgnoreFile) { + if (-not (Test-Path -LiteralPath $IgnoreFile -PathType Leaf)) { + throw "Ignore file not found: $IgnoreFile (pass -IgnoreFile '' to skip it)" + } + $dockerArgs += @("-v", "$((Resolve-Path -LiteralPath $IgnoreFile).Path):/input/cbmignore:ro") +} +$dockerArgs += $Image + +Write-Host "Graphing '$RepoPath' as project '$Name' -> $Neo4jUri" -ForegroundColor Cyan +& docker @dockerArgs +exit $LASTEXITCODE diff --git a/docker/graph/cbm-graph.sh b/docker/graph/cbm-graph.sh new file mode 100644 index 0000000000..d514c34d4c --- /dev/null +++ b/docker/graph/cbm-graph.sh @@ -0,0 +1,85 @@ +#!/bin/bash +# cbm-graph — index the repository mounted at /input/repo into Neo4j. +# +# Mounts: +# /input/repo the repository (read-only is fine; it is never modified) +# /input/cbmignore optional extra .cbmignore (e.g. a fleet-wide one) +# /data named volume: index cache + working copies (incremental) +# Environment: +# CBM_NAME project name (required) +# CBM_FRESH=1 drop the project first and rebuild from scratch +# NEO4J_URI default http://host.docker.internal:7474 +# NEO4J_USER default neo4j +# NEO4J_PASSWORD required +# NEO4J_DATABASE default neo4j +set -euo pipefail + +die() { echo "cbm-graph: $*" >&2; exit 2; } + +NAME=${CBM_NAME:-} +[ -n "$NAME" ] || die "CBM_NAME is required" +[[ "$NAME" =~ ^[A-Za-z0-9._-]+$ ]] || die "CBM_NAME may contain only letters, digits, '.', '_' and '-'" +[ -d /input/repo ] || die "mount the repository at /input/repo" +[ -n "${NEO4J_PASSWORD:-}" ] || die "NEO4J_PASSWORD is required" +NEO4J_URI=${NEO4J_URI:-http://host.docker.internal:7474} +NEO4J_USER=${NEO4J_USER:-neo4j} +NEO4J_DATABASE=${NEO4J_DATABASE:-neo4j} + +WORK=/data/work/$NAME +mkdir -p "$HOME" "$WORK" + +# ── Working copy ───────────────────────────────────────────────────── +# The indexer reads .cbmignore only from the repo root, and the source must +# stay untouched, so index a synced copy with the extra rules merged in. +# Paths an ignore rule excludes are also skipped by the copy (a jar-heavy +# repo is not copied for nothing) -- unless a rule uses '!' negation, whose +# gitignore meaning rsync cannot reproduce; then everything is copied and +# only the indexer applies the rules. +IGNORE_RULES=$(mktemp) +[ -f /input/repo/.cbmignore ] && cat /input/repo/.cbmignore >> "$IGNORE_RULES" +if [ -f /input/cbmignore ]; then + printf '\n# --- merged from the mounted cbmignore ---\n' >> "$IGNORE_RULES" + tr -d '\r' < /input/cbmignore >> "$IGNORE_RULES" +fi +RSYNC_ARGS=(-a --delete --delete-excluded --include='/.git/***' --exclude='/.cbmignore') +if ! grep -qE '^[[:space:]]*!' "$IGNORE_RULES"; then + # gitignore trims trailing blanks ('target ' means 'target'); rsync would + # take them literally. + RSYNC_RULES=$(mktemp) + sed 's/[[:space:]]*$//' "$IGNORE_RULES" > "$RSYNC_RULES" + RSYNC_ARGS+=(--exclude='node_modules/' --exclude-from="$RSYNC_RULES") +fi +echo "== syncing working copy" +rsync "${RSYNC_ARGS[@]}" /input/repo/ "$WORK/" +cp "$IGNORE_RULES" "$WORK/.cbmignore" +rm -f "$IGNORE_RULES" +if [ -d "$WORK/.git" ]; then + git config --global --add safe.directory "$WORK" + grep -qx '.cbmignore' "$WORK/.git/info/exclude" 2>/dev/null || + { mkdir -p "$WORK/.git/info"; echo '.cbmignore' >> "$WORK/.git/info/exclude"; } +fi + +# ── Index ──────────────────────────────────────────────────────────── +if [ "${CBM_FRESH:-0}" = 1 ]; then + echo "== fresh: dropping previous graph of $NAME" + codebase-memory-mcp cli delete_project --project "$NAME" || true +fi + +echo "== indexing $NAME into $NEO4J_URI (database $NEO4J_DATABASE)" +OUT=$(codebase-memory-mcp cli index_repository --repo-path "$WORK" --name "$NAME" \ + --graph-db "$NEO4J_URI" --graph-user "$NEO4J_USER" --graph-database "$NEO4J_DATABASE" 2>&1) || true +STATUS=$(printf '%s' "$OUT" | grep -o '"status":"[^"]*"' | tail -1 | cut -d'"' -f4) +if [ "$STATUS" != "indexed" ] && [ "$STATUS" != "degraded" ]; then + printf '%s\n' "$OUT" >&2 + die "indexing failed (status: ${STATUS:-unknown})" +fi + +echo "== done: $STATUS" +codebase-memory-mcp cli index_status --project "$NAME" 2>&1 | + awk '/^[a-z_]+:/ { in_gb = ($1 == "graph_backend:") } + /^(project|nodes|edges|status|indexed_at):/ || in_gb' | head -14 +cat <(m) RETURN n,r,m LIMIT 300 +EOF diff --git a/install.ps1 b/install.ps1 index abd203fb25..4321a725f8 100644 --- a/install.ps1 +++ b/install.ps1 @@ -25,7 +25,7 @@ if ([Environment]::OSVersion.Version.Build -ge 20348 -and [Net.ServicePointManager]::SecurityProtocol = $CbmProtocols Add-Type -AssemblyName System.Net.Http -$Repo = "DeusData/codebase-memory-mcp" +$Repo = "haiderRizvii/codebase-memory-mcp" $InstallDir = "$env:LOCALAPPDATA\Programs\codebase-memory-mcp" $BinName = "codebase-memory-mcp.exe" $WindowsArchiveNames = @( diff --git a/install.sh b/install.sh index 9f4b4ec999..e440d35298 100755 --- a/install.sh +++ b/install.sh @@ -4,7 +4,7 @@ set -euo pipefail # install.sh — One-line installer for codebase-memory-mcp. # # Usage: -# curl -fsSL https://raw.githubusercontent.com/DeusData/codebase-memory-mcp/main/install.sh | bash +# curl -fsSL https://raw.githubusercontent.com/haiderRizvii/codebase-memory-mcp/main/install.sh | bash # curl -fsSL ... | bash -s -- --dir /path # Custom install directory # # Environment: @@ -16,7 +16,7 @@ set -euo pipefail # called because the final line hasn't arrived yet. main() { -REPO="DeusData/codebase-memory-mcp" +REPO="haiderRizvii/codebase-memory-mcp" INSTALL_DIR="$HOME/.local/bin" SKIP_CONFIG=false CLIENTS_SET=false diff --git a/scripts/security-allowlist.txt b/scripts/security-allowlist.txt index 720c602112..414324af7a 100644 --- a/scripts/security-allowlist.txt +++ b/scripts/security-allowlist.txt @@ -25,6 +25,14 @@ NETWORK:src/daemon/ipc.c:connect:1:unix:local_socket_connect:single sockaddr_un- NETWORK:src/main.c:socket:1:loopback-v4:main_daemon_ctl_ui_endpoint_ready:single AF_INET stream client for authenticated daemon UI readiness NETWORK:src/main.c:connect:1:loopback-v4:main_daemon_ctl_socket_connected:single sockaddr_in connect whose address is hardcoded to 127.0.0.1 and whose response must prove the daemon generation HMAC +# ── Neo4j graph backend: the one outbound TCP client (opt-in) ────────────── +# Connects only when the user passes index_repository graph_db (or a project +# was indexed with it), to that URL's host:port. configured-tcp is pinned by +# the source audit to src/neo4j/neo4j_http.c and to a getaddrinfo() walk over +# the parsed graph_db host with a stream-socket hint. +NETWORK:src/neo4j/neo4j_http.c:socket:1:configured-tcp:n4_connect:stream socket for one getaddrinfo() result of the user-configured graph_db host +NETWORK:src/neo4j/neo4j_http.c:connect:1:configured-tcp:n4_connect:connect to that same resolved graph_db address (Neo4j HTTP API) + # ── Reviewed count-bounded file writes: private or caller-selected paths ─── # Format: WRITE:file:expected-count:justification WRITE:src/main.c:3:supervised worker response path; cache-scoped hook notice marker; test-seam-only browser-open marker at the caller-supplied test path @@ -38,6 +46,7 @@ WRITE:src/mcp/test_impact_artifact.c:1:receipt staging file in the caller-select WRITE:src/mcp/test_impact_engine.c:1:merge-base configuration staged inside the engine's private working directory WRITE:src/mcp/test_impact_engine_artifact.c:1:reviewed coverage bytes copied into the newly-created output bundle WRITE:src/mcp/test_impact_git.c:1:bounded Git batch stdin staged in the private capture directory +WRITE:src/neo4j/neo4j_sync.c:1:graph-backend marker .db.graph.json (URL, user, database; never the password) staged as .tmp beside the project database in the cache directory, then atomically renamed # ── CLI: update command (user-initiated, interactive) ────────────────────── src/cli/cli.c:cbm_popen:sha256 checksum verification (update cmd) @@ -84,4 +93,5 @@ URL:https://github.com/DeusData/codebase-memory-mcp/releases/latest:version chec URL:http://127.0.0.1:UI server binding (localhost only) URL:https://www.sqlite.org/c3ref/c_checkpoint_full.html:sqlite WAL checkpoint API doc reference (comment only, not a network call) URL:https://github.com/DeusData/codebase-memory-mcp:project repository self-reference in update/star notice (src/mcp/mcp.c) +URL:https://github.com/haiderRizvii/codebase-memory-mcp/issues/new:fork issue link in the UI about panel (src/ui/http_server.c) URL:https://%s:GitHub blob-URL construction in src/ui/http_server.c cbm_ui_git_web_base — https-forces the repo web base for frontend deep-links (not a network call) diff --git a/scripts/security-network-source-audit.py b/scripts/security-network-source-audit.py index 5955cc622a..53783c28ec 100644 --- a/scripts/security-network-source-audit.py +++ b/scripts/security-network-source-audit.py @@ -189,6 +189,36 @@ def _fullmatch(pattern: str, call: Call) -> bool: ) +# configured-tcp: the one outbound TCP client, the Neo4j graph backend +# (src/neo4j/neo4j_http.c). It connects only when the user configures a +# graph_db URL, to exactly that URL's host and port, so the allowance is +# pinned to that file and to a getaddrinfo() walk over the parsed URL host: +# both calls must take their arguments from that result list, and the hints +# must ask for a stream socket. +CONFIGURED_TCP_FILES = ("src/neo4j/neo4j_http.c",) +SOCKET_CONFIGURED_TCP = ( + r"socket\s*\(\s*ai\s*->\s*ai_family\s*,\s*ai\s*->\s*ai_socktype\s*," + r"\s*ai\s*->\s*ai_protocol\s*\)" +) +CONNECT_CONFIGURED_TCP = ( + r"connect\s*\(\s*s\s*,\s*ai\s*->\s*ai_addr\s*,\s*\(\s*int\s*\)\s*" + r"ai\s*->\s*ai_addrlen\s*\)" +) + + +def _configured_tcp_witness(masked: str, call: Call, scope: FunctionRange) -> bool: + before = masked[scope.body_start + 1 : call.start] + witness = re.compile( + r"\bhints\s*\.\s*ai_socktype\s*=\s*SOCK_STREAM\s*;.*?" + r"\bgetaddrinfo\s*\(\s*h\s*->\s*url\s*\.\s*host\s*,\s*port_s\s*,\s*" + r"&\s*hints\s*,\s*&\s*res\s*\).*?" + r"\bfor\s*\(\s*struct\s+addrinfo\s*\*\s*ai\s*=\s*res\s*;\s*ai\s*;\s*" + r"ai\s*=\s*ai\s*->\s*ai_next\s*\)", + re.DOTALL, + ) + return witness.search(before) is not None + + def _loopback_connect_witness(masked: str, call: Call, scope: FunctionRange) -> bool: body = masked[scope.body_start + 1 : scope.body_end - 1] call_offset = call.start - scope.body_start - 1 @@ -226,6 +256,17 @@ def _semantic_ok(masked: str, calls: Sequence[Call], allowance: Allowance) -> bo return ( re.search(r"const\s+struct\s+sockaddr_un\s*\*\s*address", header) is not None ) + if allowance.transport == "configured-tcp": + if allowance.file not in CONFIGURED_TCP_FILES or len(calls) != 1: + return False + pattern = {"socket": SOCKET_CONFIGURED_TCP, "connect": CONNECT_CONFIGURED_TCP}.get( + allowance.function + ) + return ( + pattern is not None + and _fullmatch(pattern, calls[0]) + and _configured_tcp_witness(masked, calls[0], scopes[0]) + ) if allowance.transport == "loopback-v4" and allowance.function == "socket": return all(_fullmatch(SOCKET_LOOPBACK, call) for call in calls) if allowance.transport == "loopback-v4" and allowance.function == "connect": @@ -424,9 +465,59 @@ def _self_test() -> None: ) +def _self_test_configured_tcp() -> None: + client = r''' +static int n4_connect(cbm_neo4j_http_t *h, char *err, size_t err_sz) { + struct addrinfo hints; + memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + struct addrinfo *res = NULL; + if (getaddrinfo(h->url.host, port_s, &hints, &res) != 0 || !res) { + return -1; + } + for (struct addrinfo *ai = res; ai; ai = ai->ai_next) { + s = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol); + if (connect(s, ai->ai_addr, (int)ai->ai_addrlen) == 0) { + break; + } + } + return 0; +} +''' + + def allowance(function: str, file: str = "src/neo4j/neo4j_http.c") -> Allowance: + return Allowance(file, function, 1, "configured-tcp", "n4_connect", "self-test") + + def ok(source: str, function: str, file: str = "src/neo4j/neo4j_http.c") -> bool: + masked = _mask_non_code(source) + calls = [call for call in _scan_calls(masked) if call.name == function] + return _semantic_ok(masked, calls, allowance(function, file)) + + _require(ok(client, "socket"), "configured-tcp socket witness was rejected") + _require(ok(client, "connect"), "configured-tcp connect witness was rejected") + _require( + not ok(client, "connect", "src/mcp/mcp.c"), + "configured-tcp was accepted outside the Neo4j client", + ) + _require( + not ok(client.replace("getaddrinfo(h->url.host", "getaddrinfo(other_host"), "connect"), + "configured-tcp accepted a host that is not the configured URL host", + ) + _require( + not ok(client.replace("SOCK_STREAM", "SOCK_DGRAM"), "socket"), + "configured-tcp accepted a non-stream socket", + ) + _require( + not ok(client.replace("connect(s, ai->ai_addr", "connect(s, other_addr"), "connect"), + "configured-tcp accepted an address outside the resolved list", + ) + + def main(argv: Sequence[str]) -> int: try: _self_test() + _self_test_configured_tcp() except SelfTestFailure as error: print(f"BLOCKED: network source audit self-test failed: {error}", file=sys.stderr) return 1 diff --git a/scripts/security-strings.sh b/scripts/security-strings.sh index 9d6587e5b6..22c362be06 100755 --- a/scripts/security-strings.sh +++ b/scripts/security-strings.sh @@ -115,6 +115,8 @@ echo "--- URL audit ---" ALLOWED_URLS=( "https://api.github.com/repos/DeusData/codebase-memory-mcp" "https://github.com/DeusData/codebase-memory-mcp" + # This fork's own repository (installers, update handoff, UI issue link). + "https://github.com/haiderRizvii/codebase-memory-mcp" # Our own org root and documentation site: the .mcpb bundle manifest carries # them as homepage/documentation fields, and the graph UI links the docs. "https://github.com/DeusData" diff --git a/scripts/setup-windows.ps1 b/scripts/setup-windows.ps1 index 4a836f95e6..270ec2c210 100644 --- a/scripts/setup-windows.ps1 +++ b/scripts/setup-windows.ps1 @@ -9,7 +9,7 @@ param( $ErrorActionPreference = "Stop" -$Repo = "DeusData/codebase-memory-mcp" +$Repo = "haiderRizvii/codebase-memory-mcp" $BinaryName = "codebase-memory-mcp" $InstallDir = Join-Path $env:LOCALAPPDATA "codebase-memory-mcp" diff --git a/scripts/setup.sh b/scripts/setup.sh index ee1a4d4247..dbef73e6d1 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -5,7 +5,7 @@ set -euo pipefail # Default: install the latest pre-built binary through install.sh # --from-source: build from source (requires Go + C compiler) -REPO="DeusData/codebase-memory-mcp" +REPO="haiderRizvii/codebase-memory-mcp" INSTALL_DIR="$HOME/.local/bin" BINARY_NAME="codebase-memory-mcp" SOURCE_DIR="$HOME/.local/share/codebase-memory-mcp" diff --git a/src/mcp/index_supervisor.c b/src/mcp/index_supervisor.c index 223945d3c3..b6643b4342 100644 --- a/src/mcp/index_supervisor.c +++ b/src/mcp/index_supervisor.c @@ -10,7 +10,9 @@ #include "foundation/log.h" #include "foundation/platform.h" /* cbm_safe_getenv, path normalization */ #include "foundation/profile.h" /* cbm_profile_active (keep worker log under CBM_PROFILE) */ +#include "neo4j/neo4j_client.h" /* graph_password hand-off */ #include "ui/http_server.h" /* cbm_http_server_resolve_binary_path */ +#include "yyjson/yyjson.h" #include #include @@ -739,6 +741,60 @@ size_t cbm_index_worker_job_memory_limit(size_t memory_budget_bytes) { return memory_budget_bytes > SIZE_MAX - headroom ? SIZE_MAX : memory_budget_bytes + headroom; } +/* Move index_repository's graph_password out of the worker's argv (visible in + * the process list) into a CBM_NEO4J_PASSWORD=... environment entry for that + * child only. The spawning process (the daemon serving later reads) also + * remembers the credential so it can open the project's Neo4j graph. Returns + * the args to pass (args_json itself when there is no password, otherwise a + * malloc'd copy in *owned_args) and fills env_entry (malloc'd, or NULL). */ +static const char *worker_split_graph_password(const char *args_json, char **owned_args, + char **env_entry) { + *owned_args = NULL; + *env_entry = NULL; + yyjson_doc *doc = yyjson_read(args_json, strlen(args_json), 0); + yyjson_val *root = doc ? yyjson_doc_get_root(doc) : NULL; + const char *pw = yyjson_get_str(yyjson_obj_get(root, "graph_password")); + if (!pw) { + yyjson_doc_free(doc); + return args_json; + } + const char *uri = yyjson_get_str(yyjson_obj_get(root, "graph_db")); + if (uri && uri[0]) { + cbm_neo4j_config_t cfg; + char err[CBM_SZ_256]; + if (cbm_neo4j_config_resolve(uri, yyjson_get_str(yyjson_obj_get(root, "graph_user")), pw, + yyjson_get_str(yyjson_obj_get(root, "graph_database")), &cfg, + err, sizeof(err)) == 0) { + cbm_neo4j_config_remember(&cfg); + } + cbm_neo4j_config_wipe(&cfg); + } + size_t env_len = strlen(CBM_NEO4J_PASSWORD_ENV) + strlen(pw) + 2; + *env_entry = malloc(env_len); + if (*env_entry) { + (void)snprintf(*env_entry, env_len, "%s=%s", CBM_NEO4J_PASSWORD_ENV, pw); + } + yyjson_mut_doc *copy = yyjson_doc_mut_copy(doc, NULL); + yyjson_doc_free(doc); + yyjson_mut_val *croot = copy ? yyjson_mut_doc_get_root(copy) : NULL; + while (croot && yyjson_mut_obj_get(croot, "graph_password")) { + (void)yyjson_mut_obj_remove_key(croot, "graph_password"); + } + *owned_args = croot ? yyjson_mut_write(copy, 0, NULL) : NULL; + yyjson_mut_doc_free(copy); + return *owned_args ? *owned_args : args_json; +} + +static void worker_free_secret(char *s) { + if (s) { + volatile char *p = s; + while (*p) { + *p++ = 0; + } + free(s); + } +} + int cbm_index_worker_start_with_log(const char *args_json, size_t memory_budget_bytes, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, @@ -799,7 +855,9 @@ int cbm_index_worker_start_with_log(const char *args_json, size_t memory_budget_ argv[argc++] = CBM_INDEX_WORKER_BUILD_ARG; argv[argc++] = build_fingerprint; argv[argc++] = "index_repository"; - argv[argc++] = args_json; + char *owned_args = NULL; + char *password_env = NULL; + argv[argc++] = worker_split_graph_password(args_json, &owned_args, &password_env); argv[argc++] = "--response-out"; argv[argc++] = handle->response_path; char memory_budget_text[CBM_SZ_32]; @@ -833,7 +891,14 @@ int cbm_index_worker_start_with_log(const char *args_json, size_t memory_budget_ options.quiet_timeout_ms = worker_quiet_timeout_ms(); options.memory_limit_bytes = cbm_index_worker_job_memory_limit(memory_budget_bytes); options.delete_log_on_exit = false; - if (cbm_subprocess_spawn(&options, &handle->process) != 0) { + const char *env_overrides[2] = {password_env, NULL}; + if (password_env) { + options.env_overrides = env_overrides; + } + int spawn_rc = cbm_subprocess_spawn(&options, &handle->process); + free(owned_args); + worker_free_secret(password_env); + if (spawn_rc != 0) { (void)cbm_unlink(handle->response_path); (void)cbm_unlink(handle->log_path); free(handle); diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 5bd02e4e38..dc3459ab6d 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -65,6 +65,9 @@ enum { #include #include "cypher/cypher.h" #include "discover/discover.h" +#include "neo4j/neo4j_client.h" +#include "neo4j/neo4j_store.h" +#include "neo4j/neo4j_sync.h" #include "pipeline/pipeline.h" #include "callable_sig.h" /* cbm_qn_callable_base_len */ #include "pipeline/doc_links.h" /* cbm_doclink_reason_name: the reasons the layer writes */ @@ -559,7 +562,12 @@ static const tool_def_t TOOLS[] = { "one-shot cli call that is the only client of a temporary daemon (run daemon start).\"}," "\"status\":{\"type\":\"boolean\",\"default\":false,\"description\":" "\"Report the running or last index job (queued/running/succeeded/failed/cancelled) " - "instead of indexing. Pass the same repo_path (and name, if any) as the index call.\"}" + "instead of indexing. Pass the same repo_path (and name, if any) as the index call.\"}," + "\"graph_db\":{\"type\":\"string\",\"description\":" + "\"Neo4j HTTP endpoint URL (port 7474, not bolt); graph tools read it. sqlite " + "reverts.\"}," + "\"graph_user\":{\"type\":\"string\"},\"graph_password\":{\"type\":\"string\"}," + "\"graph_database\":{\"type\":\"string\"}" "},\"required\":[\"repo_path\"]}"}, {"search_graph", @@ -599,6 +607,8 @@ static const tool_def_t TOOLS[] = { "query\"},\"project\":{\"type\":\"string\"}," "\"graph\":{\"type\":\"string\",\"enum\":[\"code\",\"missed\"],\"default\":\"code\"," "\"description\":\"code graph (default) or missed coverage-gap file tree.\"}," + "\"native\":{\"type\":\"boolean\",\"description\":\"Neo4j projects: raw read-only " + "Neo4j Cypher; $db $gen $project bound.\"}," "\"max_rows\":{\"type\":\"integer\"," "\"description\":" "\"Visible rows (default 200; max 99998); 0 uses the legacy maximum. Evaluation is " @@ -1867,6 +1877,10 @@ struct cbm_mcp_server { * "project not found". */ bool readonly_resolve_hit_corrupt; char readonly_corrupt_project[CBM_SZ_1K]; + /* Set when this request's project is Neo4j-backed but its graph reads + * fell back to the local index; cbm_mcp_handle_tool attaches it to the + * reply so a fallback is never silent. Cleared at request start. */ + char graph_fallback_reason[CBM_SZ_512]; /* Session + auto-index state */ char session_root[CBM_SZ_1K]; /* detected project root path */ @@ -2511,6 +2525,24 @@ typedef enum { STORE_RECOVERY_CORRUPT, } store_recovery_status_t; +/* A Neo4j-backed project answers graph reads from Neo4j; the SQLite file keeps + * serving everything else (and the graph too when Neo4j is unusable -- named + * in the reply by cbm_mcp_handle_tool). Called on both resolve paths, the + * direct .db open and the internal-name scan (#704), so a database + * found under another file name is no silent exception. */ +static void mcp_attach_graph_backend(cbm_mcp_server_t *srv, const char *db_path, + const char *project) { + if (!srv->store || !db_path) { + return; + } + char graph_err[CBM_SZ_512]; + if (cbm_neo4j_store_attach(srv->store, db_path, project, graph_err, sizeof(graph_err)) == + CBM_NEO4J_ATTACH_FALLBACK) { + (void)snprintf(srv->graph_fallback_reason, sizeof(srv->graph_fallback_reason), "%s", + graph_err[0] ? graph_err : "Neo4j unavailable"); + } +} + static cbm_store_t *resolve_store_internal(cbm_mcp_server_t *srv, const char *project, bool mutation_already_held, bool nonblocking_recovery, store_recovery_status_t *recovery_status, @@ -2658,6 +2690,7 @@ static cbm_store_t *resolve_store_internal(cbm_mcp_server_t *srv, const char *pr srv->owns_store = true; free(srv->current_project); srv->current_project = heap_strdup(project); + mcp_attach_graph_backend(srv, path, project); return srv->store; /* fast path: filename == internal name */ } /* #704: .db exists but its INTERNAL project name differs from @@ -2679,6 +2712,7 @@ static cbm_store_t *resolve_store_internal(cbm_mcp_server_t *srv, const char *pr srv->owns_store = true; free(srv->current_project); srv->current_project = heap_strdup(project); + mcp_attach_graph_backend(srv, cbm_store_db_path(scanned), project); } return srv->store; @@ -4274,6 +4308,8 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu if (!db) { return NULL; } + /* BM25 ranking (FTS5, camelCase-split tokens) has no Neo4j equivalent. */ + cbm_store_note_local_graph_read(store, "text search"); char fts_query[BM25_QUERY_BUF]; int tok_count = bm25_build_match(query, fts_query, sizeof(fts_query)); if (tok_count == 0) { @@ -6109,6 +6145,8 @@ static char *handle_query_graph(cbm_mcp_server_t *srv, const char *args) { char *graph_arg = cbm_mcp_get_string_arg(args, "graph"); bool missed_graph = graph_arg && strcmp(graph_arg, "missed") == 0; free(graph_arg); + /* native (Neo4j-backed projects): run the query in Neo4j itself. */ + bool native = cbm_mcp_get_bool_arg(args, "native"); if (!query) { free(cursor_arg); @@ -6157,8 +6195,8 @@ static char *handle_query_graph(cbm_mcp_server_t *srv, const char *args) { } bool generation_available = generation[0] != '\0'; const char *effective_project = project ? project : srv->current_project; - uint64_t params_hash = - query_graph_params_hash(effective_project, query, missed_graph ? "missed" : "code"); + uint64_t params_hash = query_graph_params_hash( + effective_project, query, missed_graph ? "missed" : (native ? "native" : "code")); query_graph_cursor_t cursor = {0}; if (cursor_arg) { if (requested_offset != 0) { @@ -6193,7 +6231,14 @@ static char *handle_query_graph(cbm_mcp_server_t *srv, const char *args) { * Executing at visible_limit+1 corrupts ORDER BY, DISTINCT, aggregation, * and SKIP semantics before serialization gets a chance to trim rows. * Zero preserves the engine's established 100k safety ceiling. */ - int rc = cbm_cypher_execute(store, query, cypher_project, 0, &result); + int rc; + if (native) { + char native_db_path[CBM_SZ_1K]; + project_db_path(effective_project, native_db_path, sizeof(native_db_path)); + rc = cbm_neo4j_native_query(native_db_path, effective_project, query, 0, &result); + } else { + rc = cbm_cypher_execute(store, query, cypher_project, 0, &result); + } if (rc < 0) { char *err_msg = result.error ? result.error : "query execution failed"; @@ -7412,6 +7457,55 @@ static void add_watch_status_json(cbm_mcp_server_t *srv, yyjson_mut_doc *doc, yy yyjson_mut_obj_add_val(doc, root, "watch", watch); } +/* graph_backend: where graph reads come from. For a Neo4j-backed project + * whose Neo4j copy cannot serve right now, status says sqlite_fallback and + * names why (reads stay correct: the local copy is the same graph). */ +static void add_graph_backend_report(yyjson_mut_doc *doc, yyjson_mut_val *root, cbm_store_t *store, + const char *project) { + char path[CBM_SZ_1K]; + project_db_path(project, path, sizeof(path)); + cbm_neo4j_config_t cfg; + char err[CBM_SZ_512] = ""; + char synced[CBM_SZ_128] = ""; + int rc = cbm_neo4j_sidecar_read(path, &cfg, synced, sizeof(synced), err, sizeof(err)); + yyjson_mut_val *gb = yyjson_mut_obj(doc); + if (rc == 1) { + yyjson_mut_obj_add_str(doc, gb, "kind", "sqlite"); + } else { + yyjson_mut_obj_add_str(doc, gb, "kind", "neo4j"); + if (rc == 0) { + yyjson_mut_obj_add_strcpy(doc, gb, "uri", cfg.uri); + yyjson_mut_obj_add_strcpy(doc, gb, "database", cfg.database); + yyjson_mut_obj_add_strcpy(doc, gb, "synced_generation", synced); + cbm_neo4j_config_wipe(&cfg); + /* in_sync: the last generation sent to Neo4j is the one the local + * index publishes now. A background (watcher) re-index whose sync + * failed leaves it false, with last_sync_error naming why. */ + char local[CBM_SZ_128] = ""; + (void)cbm_store_generation(store, local, sizeof(local)); + yyjson_mut_obj_add_strcpy(doc, gb, "local_generation", local); + yyjson_mut_obj_add_bool(doc, gb, "in_sync", local[0] && strcmp(local, synced) == 0); + char last_error[CBM_SZ_512] = ""; + (void)cbm_neo4j_sidecar_field(path, "last_sync_error", last_error, sizeof(last_error)); + if (last_error[0]) { + yyjson_mut_obj_add_strcpy(doc, gb, "last_sync_error", last_error); + } + } + bool attached = cbm_store_is_remote(store); + yyjson_mut_obj_add_str(doc, gb, "status", attached ? "serving" : "sqlite_fallback"); + if (!attached) { + /* Re-run the attach checks on a throwaway store to name the cause. */ + cbm_store_t *probe = cbm_store_open_path_query(path); + if (probe) { + (void)cbm_neo4j_store_attach(probe, path, project, err, sizeof(err)); + cbm_store_close(probe); + } + yyjson_mut_obj_add_strcpy(doc, gb, "reason", err[0] ? err : "unknown"); + } + } + yyjson_mut_obj_add_val(doc, root, "graph_backend", gb); +} + static char *handle_index_status(cbm_mcp_server_t *srv, const char *args) { char *project = get_project_arg(args); cbm_store_t *store = resolve_store(srv, project); @@ -7465,6 +7559,7 @@ static char *handle_index_status(cbm_mcp_server_t *srv, const char *args) { add_git_context_json(doc, root, proj_info.root_path); } } + add_graph_backend_report(doc, root, store, project); add_coverage_report(doc, root, store, project, have_proj_info ? proj_info.indexed_at : NULL, coverage_samples); (void)cbm_cross_repo_add_status_json(doc, root, store, project); @@ -7547,6 +7642,23 @@ static char *handle_delete_project(cbm_mcp_server_t *srv, const char *args) { const char *error_detail = NULL; bool is_error = false; + /* A Neo4j-backed project also loses its mirrored graph. A Neo4j failure + * is reported but does not keep the local index alive. */ + char graph_status[CBM_SZ_512] = ""; + cbm_neo4j_config_t graph_cfg; + char graph_err[CBM_SZ_512] = ""; + if (cbm_neo4j_sidecar_read(path, &graph_cfg, NULL, 0, graph_err, sizeof(graph_err)) == 0) { + cbm_neo4j_t *gc = cbm_neo4j_connect(&graph_cfg, graph_err, sizeof(graph_err)); + if (gc && cbm_neo4j_drop_project(gc, name, graph_err, sizeof(graph_err)) == 0) { + (void)snprintf(graph_status, sizeof(graph_status), "deleted"); + } else { + (void)snprintf(graph_status, sizeof(graph_status), "delete_failed: %s", graph_err); + } + cbm_neo4j_close(gc); + cbm_neo4j_config_wipe(&graph_cfg); + } + (void)cbm_neo4j_sidecar_remove(path); + if (exists) { int rc = cbm_unlink(path); (void)cbm_unlink(wal); @@ -7579,6 +7691,9 @@ static char *handle_delete_project(cbm_mcp_server_t *srv, const char *args) { if (error_detail) { yyjson_mut_obj_add_str(doc, root, "error", error_detail); } + if (graph_status[0]) { + yyjson_mut_obj_add_strcpy(doc, root, "neo4j", graph_status); + } char *json = yy_doc_to_str(doc); yyjson_mut_doc_free(doc); @@ -11938,6 +12053,62 @@ static void index_doc_candidates_apply(cbm_pipeline_t *p, index_doc_candidates_t } } +static void index_free_secret(char *s) { + if (s) { + volatile char *p = s; + while (*p) { + *p++ = 0; + } + free(s); + } +} + +/* graph_db / graph_user / graph_password / graph_database: resolve the Neo4j + * target, prove it accepts the credentials before any indexing work, and hand + * it to the pipeline. graph_db "sqlite" / "none" returns the project to the + * local graph. Returns 0, or -1 with err filled. */ +static int index_graph_backend_apply(cbm_pipeline_t *p, const char *args, char *err, + size_t err_sz) { + char *uri = cbm_mcp_get_string_arg(args, "graph_db"); + if (!uri || !uri[0]) { + free(uri); + return 0; + } + if (strcmp(uri, "sqlite") == 0 || strcmp(uri, "none") == 0) { + free(uri); + cbm_pipeline_disable_graph_backend(p); + return 0; + } + char *user = cbm_mcp_get_string_arg(args, "graph_user"); + char *password = cbm_mcp_get_string_arg(args, "graph_password"); + char *database = cbm_mcp_get_string_arg(args, "graph_database"); + cbm_neo4j_config_t cfg; + int rc = cbm_neo4j_config_resolve(uri, user, password, database, &cfg, err, err_sz); + free(uri); + free(user); + index_free_secret(password); + free(database); + if (rc == 0) { + cbm_neo4j_t *c = cbm_neo4j_connect(&cfg, err, err_sz); + if (!c) { + rc = -1; + } else { + if (cbm_neo4j_ping(c) != 0) { + (void)snprintf(err, err_sz, "cannot use Neo4j at %s: %s", cfg.uri, + cbm_neo4j_error(c)); + rc = -1; + } + cbm_neo4j_close(c); + } + } + if (rc == 0) { + cbm_neo4j_config_remember(&cfg); + cbm_pipeline_set_graph_backend(p, &cfg); + } + cbm_neo4j_config_wipe(&cfg); + return rc; +} + /* Preserve every index option while replacing all caller-supplied repo_path * keys with the one canonical path that was actually authorized. A non-empty * project_name likewise replaces any caller "name" (#2134: the project key the @@ -12460,6 +12631,14 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { cbm_pipeline_set_persistence(p, persistence); cbm_pipeline_set_resource_policy(p, &resource_policy); index_doc_candidates_apply(p, doc_candidates); + char graph_err[CBM_SZ_1K]; + if (index_graph_backend_apply(p, args, graph_err, sizeof(graph_err)) != 0) { + cbm_pipeline_free(p); + mcp_project_mutation_end(srv, mutation_project); + free(mutation_project); + free(repo_path); + return cbm_mcp_text_result(graph_err, true); + } char *project_name = heap_strdup(cbm_pipeline_project_name(p)); @@ -12617,6 +12796,15 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { serving_index_was_servable && project_db_is_servable(project_name, serving_db_path)); yyjson_mut_obj_add_strcpy(doc, root, "message", message); + } else if (rc == CBM_PIPELINE_GRAPH_SYNC_FAILED) { + char hint[CBM_SZ_2K]; + (void)snprintf(hint, sizeof(hint), + "The local index was published, but mirroring it into Neo4j failed: %s. " + "Graph tools keep serving the previous Neo4j generation (if any). Fix " + "the Neo4j connection and re-run index_repository with graph_db.", + cbm_pipeline_graph_error(p)); + yyjson_mut_obj_add_str(doc, root, "status", "graph_sync_failed"); + yyjson_mut_obj_add_strcpy(doc, root, "hint", hint); } else { yyjson_mut_obj_add_str(doc, root, "status", "error"); /* #1665: a post-publish artifact export failure (read-only repo, etc.) @@ -20605,6 +20793,138 @@ static bool mcp_tool_reads_in_batch(const char *tool_name) { return false; } +/* A Neo4j-backed project whose graph reads fell back to the local index + * (Neo4j unreachable, credentials rejected, or holding another generation): + * the answer is still correct -- the local index is the newest published + * generation and Neo4j only mirrors it -- but the caller must be able to see + * that Neo4j did not serve it. Appends a text notice to content and, for a + * structured reply, a graph_backend object. Takes ownership of result; on any + * failure the original reply is returned unchanged. */ +/* Append a graph_backend report to an MCP tool result: `notice` as a text + * content item, and {kind: neo4j, status, served_by, remote_queries, local, + * reason} as structuredContent.graph_backend when the reply is structured and + * does not already carry one (index_status). Takes ownership of result; on + * any failure the original reply is returned unchanged. */ +static char *mcp_result_append_backend(char *result, const char *notice, const char *status, + const char *served_by, int64_t remote_queries, + const char *local_what, const char *reason) { + if (!result) { + return NULL; + } + yyjson_doc *in = yyjson_read(result, strlen(result), 0); + yyjson_mut_doc *doc = in ? yyjson_doc_mut_copy(in, NULL) : NULL; + yyjson_doc_free(in); + yyjson_mut_val *root = doc ? yyjson_mut_doc_get_root(doc) : NULL; + yyjson_mut_val *content = root ? yyjson_mut_obj_get(root, "content") : NULL; + if (!yyjson_mut_is_arr(content)) { + yyjson_mut_doc_free(doc); + return result; + } + yyjson_mut_val *item = yyjson_mut_obj(doc); + yyjson_mut_obj_add_str(doc, item, "type", "text"); + yyjson_mut_obj_add_strcpy(doc, item, "text", notice); + yyjson_mut_arr_append(content, item); + yyjson_mut_val *structured = yyjson_mut_obj_get(root, "structuredContent"); + if (yyjson_mut_is_obj(structured) && !yyjson_mut_obj_get(structured, "graph_backend")) { + yyjson_mut_val *gb = yyjson_mut_obj(doc); + yyjson_mut_obj_add_str(doc, gb, "kind", "neo4j"); + yyjson_mut_obj_add_strcpy(doc, gb, "status", status); + if (served_by) { + yyjson_mut_obj_add_strcpy(doc, gb, "served_by", served_by); + } + if (remote_queries >= 0) { + yyjson_mut_obj_add_int(doc, gb, "remote_queries", remote_queries); + } + if (local_what && local_what[0]) { + yyjson_mut_obj_add_strcpy(doc, gb, "local", local_what); + } + if (reason) { + yyjson_mut_obj_add_strcpy(doc, gb, "reason", reason); + } + yyjson_mut_obj_add_val(doc, structured, "graph_backend", gb); + } + char *out = yy_doc_to_str(doc); + yyjson_mut_doc_free(doc); + if (!out) { + return result; + } + free(result); + return out; +} + +char *cbm_mcp_result_add_graph_fallback(char *result, const char *reason) { + char notice[CBM_SZ_1K]; + (void)snprintf(notice, sizeof(notice), + "graph_backend: sqlite_fallback -- this Neo4j-backed project was answered from " + "the local index, not Neo4j (%s). The answer is current; Neo4j is not. Re-run " + "index_repository with graph_db to resync, and see index_status.", + reason); + return mcp_result_append_backend(result, notice, "sqlite_fallback", "local", -1, NULL, reason); +} + +const char *cbm_mcp_graph_served_by(uint64_t remote_queries, int local_reads) { + if (remote_queries > 0) { + return local_reads > 0 ? "neo4j+local" : "neo4j"; + } + return local_reads > 0 ? "local" : NULL; +} + +char *cbm_mcp_result_add_graph_served(char *result, uint64_t remote_queries, int local_reads, + const char *local_what) { + const char *served_by = cbm_mcp_graph_served_by(remote_queries, local_reads); + if (!served_by) { + return result; /* the tool read no graph data */ + } + const char *what = local_what && local_what[0] ? local_what : "some graph data"; + char notice[CBM_SZ_1K]; + if (strcmp(served_by, "neo4j") == 0) { + (void)snprintf(notice, sizeof(notice), + "graph_backend: neo4j -- the graph data in this answer came from Neo4j " + "(%llu queries).", + (unsigned long long)remote_queries); + } else if (strcmp(served_by, "neo4j+local") == 0) { + (void)snprintf(notice, sizeof(notice), + "graph_backend: neo4j+local -- graph lookups came from Neo4j (%llu " + "queries); %s came from the local index, which holds the same generation " + "(no Neo4j equivalent).", + (unsigned long long)remote_queries, what); + } else { + (void)snprintf(notice, sizeof(notice), + "graph_backend: local -- %s came from the local index of this Neo4j-backed " + "project, which holds the same generation as Neo4j (no Neo4j equivalent).", + what); + } + return mcp_result_append_backend(result, notice, "serving", served_by, + (int64_t)remote_queries, local_reads > 0 ? what : NULL, NULL); +} + +/* Name the graph backend in every reply of a Neo4j-backed project, and log it + * (info, so the daemon log answers "did this come from Neo4j?" afterwards). */ +static char *mcp_report_graph_backend(cbm_mcp_server_t *srv, const char *tool_name, + char *result) { + const char *project = srv->current_project ? srv->current_project : ""; + if (srv->graph_fallback_reason[0]) { + cbm_log_info("graph.backend", "tool", tool_name, "project", project, "served_by", + "sqlite_fallback", "reason", srv->graph_fallback_reason); + return cbm_mcp_result_add_graph_fallback(result, srv->graph_fallback_reason); + } + if (!srv->store || !cbm_store_is_remote(srv->store)) { + return result; + } + cbm_store_backend_usage_t usage; + cbm_store_backend_usage(srv->store, &usage); + const char *served_by = cbm_mcp_graph_served_by(usage.remote_queries, usage.local_reads); + if (!served_by) { + return result; + } + char queries[CBM_SZ_32]; + (void)snprintf(queries, sizeof(queries), "%llu", (unsigned long long)usage.remote_queries); + cbm_log_info("graph.backend", "tool", tool_name, "project", project, "served_by", served_by, + "remote_queries", queries, "local", usage.local_what); + return cbm_mcp_result_add_graph_served(result, usage.remote_queries, usage.local_reads, + usage.local_what); +} + /* File-backed query stores are request-scoped. Keeping one open between MCP * calls pins an old database generation after another process atomically * replaces the project DB. On Windows it can also prevent that replacement @@ -20642,8 +20962,13 @@ char *cbm_mcp_handle_tool(cbm_mcp_server_t *srv, const char *tool_name, const ch cbm_mem_phase_mark("request.dispatch_tool"); if (srv) { srv->read_batch_tool = mcp_tool_reads_in_batch(tool_name); + srv->graph_fallback_reason[0] = '\0'; + cbm_store_backend_usage_reset(srv->store); } char *result = dispatch_tool(srv, tool_name, args_json); + if (srv) { + result = mcp_report_graph_backend(srv, tool_name, result); + } if (srv) { /* An embedded store outlives the request: end its batch here. */ srv->read_batch_tool = false; diff --git a/src/mcp/mcp.h b/src/mcp/mcp.h index 3a5fce5ff8..d970fd9913 100644 --- a/src/mcp/mcp.h +++ b/src/mcp/mcp.h @@ -56,6 +56,22 @@ char *cbm_jsonrpc_format_error(int64_t id, int code, const char *message); /* Format an MCP tool result with text content. Returns heap-allocated JSON. */ char *cbm_mcp_text_result(const char *text, bool is_error); +/* Mark a tool reply as answered from the local index although its project is + * Neo4j-backed (graph_backend sqlite_fallback): appends a text notice to + * content and a graph_backend object to structuredContent. Takes ownership of + * result; returns it unchanged when it is not an MCP tool-result object. */ +char *cbm_mcp_result_add_graph_fallback(char *result, const char *reason); + +/* "neo4j", "neo4j+local" or "local" for a request that read graph data with + * that mix of remote queries / local graph reads; NULL when it read none. */ +const char *cbm_mcp_graph_served_by(uint64_t remote_queries, int local_reads); + +/* Mark a reply of a Neo4j-backed project with where its graph data came from + * (graph_backend served_by neo4j / neo4j+local / local, plus a text notice). + * Unchanged when the tool read no graph data. Takes ownership of result. */ +char *cbm_mcp_result_add_graph_served(char *result, uint64_t remote_queries, int local_reads, + const char *local_what); + /* Attach a notice to a complete MCP tool result: a JSON-object payload gains a * "notice" key (content text and structuredContent stay in step), a text * payload gains a trailing paragraph. Takes diff --git a/src/neo4j/neo4j_client.c b/src/neo4j/neo4j_client.c new file mode 100644 index 0000000000..d4bedc2980 --- /dev/null +++ b/src/neo4j/neo4j_client.c @@ -0,0 +1,623 @@ +/* + * neo4j_client.c — Neo4j config, credential cache and Cypher execution. + */ +#include "neo4j/neo4j_client.h" +#include "neo4j/neo4j_http.h" + +#include "foundation/compat_thread.h" +#include "foundation/log.h" +#include "foundation/platform.h" + +#include +#include +#include +#include +#include + +#define N4_DEFAULT_USER "neo4j" +#define N4_DEFAULT_DATABASE "neo4j" +#define N4_CRED_CACHE_MAX 16 + +struct cbm_neo4j { + cbm_neo4j_http_t *http; + char uri[CBM_NEO4J_URI_MAX]; + char commit_path[CBM_NEO4J_DATABASE_MAX + 32]; + char err[CBM_NEO4J_ERR_MAX]; + uint64_t cred_key; /* hash of uri + user + password, for the login backoff */ +}; + +/* After Neo4j rejects a login, the same credentials are not offered again for + * this long: every rejected attempt counts toward Neo4j's lockout + * (AuthenticationRateLimit), which then refuses the CORRECT password too. */ +#define N4_AUTH_BACKOFF_MS 60000ULL +#define N4_AUTH_BACKOFF_SLOTS 16 + +/* ── Credential cache ───────────────────────────────────────────── */ + +typedef struct { + char uri[CBM_NEO4J_URI_MAX]; + char user[CBM_NEO4J_USER_MAX]; + char password[CBM_NEO4J_PASSWORD_MAX]; +} n4_cred_t; + +static cbm_mutex_t g_cred_mu; +static atomic_int g_cred_mu_state; /* 0 = uninitialized, 1 = initializing, 2 = ready */ +static n4_cred_t g_creds[N4_CRED_CACHE_MAX]; +static int g_cred_count; + +static void cred_lock(void) { + /* Daemon request threads can race to the first use. */ + int expected = 0; + if (atomic_compare_exchange_strong(&g_cred_mu_state, &expected, 1)) { + cbm_mutex_init(&g_cred_mu); + atomic_store(&g_cred_mu_state, 2); + } else { + while (atomic_load(&g_cred_mu_state) != 2) { + /* another thread is finishing the one-time init */ + } + } + cbm_mutex_lock(&g_cred_mu); +} + +static void copy_str(char *dst, size_t dst_sz, const char *src) { + (void)snprintf(dst, dst_sz, "%s", src ? src : ""); +} + +static bool cred_lookup(const char *uri, const char *user, char *out, size_t out_sz) { + bool found = false; + cred_lock(); + for (int i = 0; i < g_cred_count; i++) { + if (strcmp(g_creds[i].uri, uri) == 0 && strcmp(g_creds[i].user, user) == 0) { + copy_str(out, out_sz, g_creds[i].password); + found = true; + break; + } + } + cbm_mutex_unlock(&g_cred_mu); + return found; +} + +void cbm_neo4j_config_remember(const cbm_neo4j_config_t *cfg) { + if (!cfg || !cfg->uri[0] || !cfg->password[0]) { + return; + } + cred_lock(); + int slot = -1; + for (int i = 0; i < g_cred_count; i++) { + if (strcmp(g_creds[i].uri, cfg->uri) == 0 && strcmp(g_creds[i].user, cfg->user) == 0) { + slot = i; + break; + } + } + if (slot < 0) { + slot = g_cred_count < N4_CRED_CACHE_MAX ? g_cred_count++ : 0; + } + copy_str(g_creds[slot].uri, sizeof(g_creds[slot].uri), cfg->uri); + copy_str(g_creds[slot].user, sizeof(g_creds[slot].user), cfg->user); + copy_str(g_creds[slot].password, sizeof(g_creds[slot].password), cfg->password); + cbm_mutex_unlock(&g_cred_mu); +} + +void cbm_neo4j_config_wipe(cbm_neo4j_config_t *cfg) { + if (cfg) { + volatile char *p = cfg->password; + for (size_t i = 0; i < sizeof(cfg->password); i++) { + p[i] = 0; + } + } +} + +/* ── Config ─────────────────────────────────────────────────────── */ + +static bool n4_nonempty(const char *s) { + return s && *s; +} + +/* CBM_NEO4J_PASSWORD is only ever set by the index supervisor for one worker + * and carries that request's --graph-password, so it outranks everything but + * an explicit value. For an already-indexed project (prefer_cache) the login + * this process last used successfully for exactly that uri + user comes next: + * the ambient NEO4J_PASSWORD may belong to another Neo4j altogether, and + * trying it first is how a wrong global password locked a project's server + * out. */ +static void resolve_password(cbm_neo4j_config_t *out, const char *explicit_pw, bool prefer_cache) { + char buf[CBM_NEO4J_PASSWORD_MAX]; + if (n4_nonempty(explicit_pw)) { + copy_str(out->password, sizeof(out->password), explicit_pw); + } else if (cbm_safe_getenv(CBM_NEO4J_PASSWORD_ENV, buf, sizeof(buf), NULL) && buf[0]) { + copy_str(out->password, sizeof(out->password), buf); + } else if (prefer_cache && + cred_lookup(out->uri, out->user, out->password, sizeof(out->password))) { + /* the project's own last good login */ + } else if (cbm_safe_getenv("NEO4J_PASSWORD", buf, sizeof(buf), NULL) && buf[0]) { + copy_str(out->password, sizeof(out->password), buf); + } else { + (void)cred_lookup(out->uri, out->user, out->password, sizeof(out->password)); + } + memset(buf, 0, sizeof(buf)); +} + +static void resolve_from_env(char *dst, size_t dst_sz, const char *explicit_val, + const char *env_name, const char *dflt) { + char buf[CBM_NEO4J_URI_MAX]; + if (n4_nonempty(explicit_val)) { + copy_str(dst, dst_sz, explicit_val); + } else if (cbm_safe_getenv(env_name, buf, sizeof(buf), NULL) && buf[0]) { + copy_str(dst, dst_sz, buf); + } else { + copy_str(dst, dst_sz, dflt); + } +} + +int cbm_neo4j_config_resolve(const char *uri, const char *user, const char *password, + const char *database, cbm_neo4j_config_t *out, char *err, + size_t err_sz) { + memset(out, 0, sizeof(*out)); + char uri_buf[CBM_NEO4J_URI_MAX]; + resolve_from_env(uri_buf, sizeof(uri_buf), uri, "NEO4J_URI", ""); + if (!uri_buf[0]) { + (void)snprintf(err, err_sz, "graph_db URL is required"); + return -1; + } + if (strlen(uri_buf) >= sizeof(out->uri)) { + (void)snprintf(err, err_sz, "graph_db URL too long"); + return -1; + } + cbm_neo4j_url_t parsed; + if (cbm_neo4j_url_parse(uri_buf, &parsed, err, err_sz) != 0) { + return -1; + } + copy_str(out->uri, sizeof(out->uri), uri_buf); + resolve_from_env(out->user, sizeof(out->user), user, "NEO4J_USER", ""); + if (!out->user[0]) { + /* NEO4J_USERNAME is what the official drivers and Docker image use. */ + resolve_from_env(out->user, sizeof(out->user), NULL, "NEO4J_USERNAME", N4_DEFAULT_USER); + } + resolve_from_env(out->database, sizeof(out->database), database, "NEO4J_DATABASE", + N4_DEFAULT_DATABASE); + for (const char *p = out->database; *p; p++) { + if (!isalnum((unsigned char)*p) && *p != '.' && *p != '-' && *p != '_') { + (void)snprintf(err, err_sz, "invalid graph_database name '%s'", out->database); + return -1; + } + } + resolve_password(out, password, false); + return 0; +} + +char *cbm_neo4j_config_to_meta(const cbm_neo4j_config_t *cfg) { + yyjson_mut_doc *d = yyjson_mut_doc_new(NULL); + if (!d) { + return NULL; + } + yyjson_mut_val *o = yyjson_mut_obj(d); + yyjson_mut_doc_set_root(d, o); + yyjson_mut_obj_add_str(d, o, "kind", "neo4j"); + yyjson_mut_obj_add_strcpy(d, o, "uri", cfg->uri); + yyjson_mut_obj_add_strcpy(d, o, "user", cfg->user); + yyjson_mut_obj_add_strcpy(d, o, "database", cfg->database); + char *json = yyjson_mut_write(d, 0, NULL); + yyjson_mut_doc_free(d); + return json; +} + +int cbm_neo4j_config_from_meta(const char *meta_json, cbm_neo4j_config_t *out, char *err, + size_t err_sz) { + if (!meta_json || !*meta_json) { + (void)snprintf(err, err_sz, "project has no graph backend metadata"); + return -1; + } + yyjson_doc *d = yyjson_read(meta_json, strlen(meta_json), 0); + yyjson_val *o = d ? yyjson_doc_get_root(d) : NULL; + const char *kind = yyjson_get_str(yyjson_obj_get(o, "kind")); + if (!o || !kind || strcmp(kind, "neo4j") != 0) { + yyjson_doc_free(d); + (void)snprintf(err, err_sz, "unsupported graph backend metadata"); + return -1; + } + /* Use stored uri/user/database verbatim — env overrides apply only to the + * password, so a stale NEO4J_URI can never redirect an indexed project. */ + memset(out, 0, sizeof(*out)); + copy_str(out->uri, sizeof(out->uri), yyjson_get_str(yyjson_obj_get(o, "uri"))); + copy_str(out->user, sizeof(out->user), yyjson_get_str(yyjson_obj_get(o, "user"))); + copy_str(out->database, sizeof(out->database), yyjson_get_str(yyjson_obj_get(o, "database"))); + yyjson_doc_free(d); + if (!out->uri[0]) { + (void)snprintf(err, err_sz, "graph backend metadata has no uri"); + return -1; + } + if (!out->database[0]) { + copy_str(out->database, sizeof(out->database), N4_DEFAULT_DATABASE); + } + resolve_password(out, NULL, true); + return 0; +} + +/* ── Client ─────────────────────────────────────────────────────── */ + +/* ── Login backoff ──────────────────────────────────────────────── */ + +typedef struct { + uint64_t key; + uint64_t failed_at_ms; +} n4_auth_failure_t; + +static n4_auth_failure_t g_auth_failures[N4_AUTH_BACKOFF_SLOTS]; + +static uint64_t n4_fnv1a(uint64_t h, const char *s) { + for (; s && *s; s++) { + h ^= (unsigned char)*s; + h *= 1099511628211ULL; + } + return h * 1099511628211ULL; /* separator, so "ab"+"c" != "a"+"bc" */ +} + +static uint64_t n4_cred_key(const cbm_neo4j_config_t *cfg) { + uint64_t h = 14695981039346656037ULL; + h = n4_fnv1a(h, cfg->uri); + h = n4_fnv1a(h, cfg->user); + return n4_fnv1a(h, cfg->password); +} + +/* Milliseconds left in this key's backoff (0 = may try). */ +static uint64_t n4_auth_backoff_left(uint64_t key) { + uint64_t left = 0; + uint64_t now = cbm_now_ms(); + cred_lock(); + for (int i = 0; i < N4_AUTH_BACKOFF_SLOTS; i++) { + if (g_auth_failures[i].key == key && g_auth_failures[i].failed_at_ms && + now - g_auth_failures[i].failed_at_ms < N4_AUTH_BACKOFF_MS) { + left = N4_AUTH_BACKOFF_MS - (now - g_auth_failures[i].failed_at_ms); + } + } + cbm_mutex_unlock(&g_cred_mu); + return left; +} + +/* failed: start (or restart) the backoff; !failed: the login worked. */ +static void n4_auth_note(uint64_t key, bool failed) { + uint64_t now = cbm_now_ms(); + cred_lock(); + int slot = -1; + int oldest = 0; + for (int i = 0; i < N4_AUTH_BACKOFF_SLOTS; i++) { + if (g_auth_failures[i].key == key) { + slot = i; + } + if (g_auth_failures[i].failed_at_ms < g_auth_failures[oldest].failed_at_ms) { + oldest = i; + } + } + if (failed) { + if (slot < 0) { + slot = oldest; + } + g_auth_failures[slot].key = key; + g_auth_failures[slot].failed_at_ms = now ? now : 1; + } else if (slot >= 0) { + memset(&g_auth_failures[slot], 0, sizeof(g_auth_failures[slot])); + } + cbm_mutex_unlock(&g_cred_mu); +} + +void cbm_neo4j_test_reset_auth_backoff(void) { + cred_lock(); + memset(g_auth_failures, 0, sizeof(g_auth_failures)); + cbm_mutex_unlock(&g_cred_mu); +} + +cbm_neo4j_t *cbm_neo4j_connect(const cbm_neo4j_config_t *cfg, char *err, size_t err_sz) { + cbm_neo4j_url_t url; + if (cbm_neo4j_url_parse(cfg->uri, &url, err, err_sz) != 0) { + return NULL; + } + cbm_neo4j_t *c = calloc(1, sizeof(*c)); + if (!c) { + return NULL; + } + c->http = cbm_neo4j_http_open(&url, cfg->user, cfg->password, err, err_sz); + if (!c->http) { + free(c); + return NULL; + } + copy_str(c->uri, sizeof(c->uri), cfg->uri); + (void)snprintf(c->commit_path, sizeof(c->commit_path), "/db/%s/tx/commit", + cfg->database[0] ? cfg->database : N4_DEFAULT_DATABASE); + c->cred_key = n4_cred_key(cfg); + return c; +} + +void cbm_neo4j_close(cbm_neo4j_t *c) { + if (!c) { + return; + } + cbm_neo4j_http_close(c->http); + free(c); +} + +const char *cbm_neo4j_error(const cbm_neo4j_t *c) { + return c ? c->err : "no Neo4j client"; +} + +const char *cbm_neo4j_uri(const cbm_neo4j_t *c) { + return c ? c->uri : ""; +} + +int cbm_neo4j_batch_init(cbm_neo4j_batch_t *b) { + memset(b, 0, sizeof(*b)); + b->doc = yyjson_mut_doc_new(NULL); + if (!b->doc) { + return -1; + } + yyjson_mut_val *root = yyjson_mut_obj(b->doc); + yyjson_mut_doc_set_root(b->doc, root); + b->statements = yyjson_mut_arr(b->doc); + yyjson_mut_obj_add_val(b->doc, root, "statements", b->statements); + return 0; +} + +yyjson_mut_val *cbm_neo4j_batch_add(cbm_neo4j_batch_t *b, const char *cypher) { + yyjson_mut_val *st = yyjson_mut_obj(b->doc); + yyjson_mut_obj_add_strcpy(b->doc, st, "statement", cypher); + yyjson_mut_val *params = yyjson_mut_obj(b->doc); + yyjson_mut_obj_add_val(b->doc, st, "parameters", params); + yyjson_mut_val *contents = yyjson_mut_arr(b->doc); + yyjson_mut_arr_add_str(b->doc, contents, "row"); + yyjson_mut_obj_add_val(b->doc, st, "resultDataContents", contents); + yyjson_mut_arr_append(b->statements, st); + b->count++; + return params; +} + +void cbm_neo4j_batch_free(cbm_neo4j_batch_t *b) { + if (b && b->doc) { + yyjson_mut_doc_free(b->doc); + b->doc = NULL; + } +} + +static void set_err(cbm_neo4j_t *c, const char *fmt, const char *a, const char *b2) { + (void)snprintf(c->err, sizeof(c->err), fmt, a ? a : "", b2 ? b2 : ""); +} + +/* NULL when results holds exactly one well-formed entry per statement sent: + * {"columns": [...], "data": [{"row": [one value per column]}, ...]}. + * Otherwise a short description of the first defect. */ +const char *cbm_neo4j_result_shape_error(yyjson_val *results, int statements) { + if (!yyjson_is_arr(results)) { + return "no results array"; + } + if (yyjson_arr_size(results) != (size_t)statements) { + return "result count does not match the statements sent"; + } + size_t idx; + size_t max; + yyjson_val *res; + yyjson_arr_foreach(results, idx, max, res) { + yyjson_val *columns = yyjson_obj_get(res, "columns"); + yyjson_val *data = yyjson_obj_get(res, "data"); + if (!yyjson_is_arr(columns) || !yyjson_is_arr(data)) { + return "result without columns/data arrays"; + } + size_t ncols = yyjson_arr_size(columns); + size_t ri; + size_t rmax; + yyjson_val *entry; + yyjson_arr_foreach(data, ri, rmax, entry) { + yyjson_val *row = yyjson_obj_get(entry, "row"); + if (!yyjson_is_arr(row) || yyjson_arr_size(row) != ncols) { + return "row does not match its columns"; + } + } + } + return NULL; +} + +/* One debug line per round trip (CBM_LOG_LEVEL=debug): which Neo4j, how many + * statements, the first one's Cypher, HTTP status and latency. */ +static void n4_log_request(const cbm_neo4j_t *c, cbm_neo4j_batch_t *b, int status, + uint64_t elapsed_ms) { + if (cbm_log_get_level() > CBM_LOG_DEBUG) { + return; + } + yyjson_mut_val *first = yyjson_mut_arr_get_first(b->statements); + const char *cypher = yyjson_mut_get_str(yyjson_mut_obj_get(first, "statement")); + char head[200]; + (void)snprintf(head, sizeof(head), "%s", cypher ? cypher : ""); + char stmts[16]; + char st[16]; + char ms[24]; + (void)snprintf(stmts, sizeof(stmts), "%d", b->count); + (void)snprintf(st, sizeof(st), "%d", status); + (void)snprintf(ms, sizeof(ms), "%llu", (unsigned long long)elapsed_ms); + cbm_log_debug("neo4j.query", "uri", c->uri, "statements", stmts, "status", st, "ms", ms, + "cypher", head); +} + +int cbm_neo4j_batch_run(cbm_neo4j_t *c, cbm_neo4j_batch_t *b, cbm_neo4j_result_t *out) { + if (out) { + memset(out, 0, sizeof(*out)); + } + if (!c || !b || !b->doc) { + return -1; + } + c->err[0] = '\0'; + uint64_t backoff = n4_auth_backoff_left(c->cred_key); + if (backoff > 0) { + char secs[16]; + (void)snprintf(secs, sizeof(secs), "%llu", (unsigned long long)((backoff + 999) / 1000)); + set_err(c, + "Neo4j rejected these credentials for %s; not retrying them for another %s s " + "(each rejected login counts toward Neo4j's lockout). Fix the password and retry " + "after that.", + c->uri, secs); + return -1; + } + uint64_t started_ms = cbm_now_ms(); + size_t body_len = 0; + char *body = yyjson_mut_write(b->doc, 0, &body_len); + if (!body) { + set_err(c, "failed to serialize Neo4j request%s%s", NULL, NULL); + return -1; + } + int status = 0; + char *resp = NULL; + size_t resp_len = 0; + char terr[CBM_NEO4J_ERR_MAX]; + terr[0] = '\0'; + int rc = cbm_neo4j_http_post(c->http, c->commit_path, body, body_len, &status, &resp, &resp_len, + terr, sizeof(terr)); + free(body); + n4_log_request(c, b, rc == 0 ? status : -1, cbm_now_ms() - started_ms); + if (rc != 0) { + set_err(c, "%s%s", terr[0] ? terr : "Neo4j request failed", NULL); + free(resp); + return -1; + } + if (status == 401 || status == 403) { + n4_auth_note(c->cred_key, true); + set_err(c, + "Neo4j rejected the credentials for %s (HTTP 401/403); check " + "--graph-user/--graph-password or NEO4J_USER/NEO4J_PASSWORD%s", + c->uri, NULL); + free(resp); + return -1; + } + if (status == 404) { + set_err(c, + "Neo4j endpoint %s not found (HTTP 404): wrong database name or not a " + "Neo4j HTTP port%s", + c->commit_path, NULL); + free(resp); + return -1; + } + yyjson_doc *d = yyjson_read(resp, resp_len, 0); + free(resp); + yyjson_val *root = d ? yyjson_doc_get_root(d) : NULL; + if (!root) { + char sc[16]; + (void)snprintf(sc, sizeof(sc), "%d", status); + set_err(c, "unparseable Neo4j response (HTTP %s)%s", sc, NULL); + yyjson_doc_free(d); + return -1; + } + yyjson_val *errors = yyjson_obj_get(root, "errors"); + if (errors && yyjson_arr_size(errors) > 0) { + yyjson_val *e0 = yyjson_arr_get_first(errors); + const char *code = yyjson_get_str(yyjson_obj_get(e0, "code")); + /* A rejected login, or Neo4j's lockout after too many: either way the + * same credentials must not be offered again right away. */ + if (code && (strstr(code, "Security.Unauthorized") || + strstr(code, "Security.AuthenticationRateLimit"))) { + n4_auth_note(c->cred_key, true); + } + set_err(c, "Neo4j error %s: %s", code, yyjson_get_str(yyjson_obj_get(e0, "message"))); + yyjson_doc_free(d); + return -1; + } + if (status < 200 || status >= 300) { + char sc[16]; + (void)snprintf(sc, sizeof(sc), "%d", status); + set_err(c, "Neo4j returned HTTP %s%s", sc, NULL); + yyjson_doc_free(d); + return -1; + } + /* A 2xx body must still be a complete transactional result. Anything + * else (a proxy or another service answering 200, a truncated reply) + * would read as "no rows" -- an empty graph to every reader, and a + * generation of 0 to the sync -- so it is an error, never a success. */ + const char *shape_err = cbm_neo4j_result_shape_error(yyjson_obj_get(root, "results"), b->count); + if (shape_err) { + set_err(c, "malformed Neo4j response from %s: %s", c->uri, shape_err); + yyjson_doc_free(d); + return -1; + } + n4_auth_note(c->cred_key, false); + if (out) { + out->doc = d; + out->results = yyjson_obj_get(root, "results"); + } else { + yyjson_doc_free(d); + } + return 0; +} + +int cbm_neo4j_ping(cbm_neo4j_t *c) { + cbm_neo4j_batch_t b; + if (cbm_neo4j_batch_init(&b) != 0) { + return -1; + } + (void)cbm_neo4j_batch_add(&b, "RETURN 1"); + int rc = cbm_neo4j_batch_run(c, &b, NULL); + cbm_neo4j_batch_free(&b); + return rc; +} + +yyjson_val *cbm_neo4j_result_data(const cbm_neo4j_result_t *r, size_t stmt_index) { + if (!r || !r->results) { + return NULL; + } + yyjson_val *res = yyjson_arr_get(r->results, stmt_index); + return res ? yyjson_obj_get(res, "data") : NULL; +} + +yyjson_val *cbm_neo4j_row_col(yyjson_val *data_entry, size_t col) { + return yyjson_arr_get(yyjson_obj_get(data_entry, "row"), col); +} + +void cbm_neo4j_result_free(cbm_neo4j_result_t *r) { + if (r && r->doc) { + yyjson_doc_free(r->doc); + r->doc = NULL; + r->results = NULL; + } +} + +int64_t cbm_neo4j_val_int(yyjson_val *v, int64_t fallback) { + if (yyjson_is_int(v)) { + return yyjson_get_sint(v); + } + if (yyjson_is_real(v)) { + return (int64_t)yyjson_get_real(v); + } + if (yyjson_is_str(v)) { + return strtoll(yyjson_get_str(v), NULL, 10); + } + return fallback; +} + +char *cbm_neo4j_val_strdup(yyjson_val *v, const char *fallback) { + const char *s = NULL; + if (yyjson_is_str(v)) { + s = yyjson_get_str(v); + } else if (v && !yyjson_is_null(v)) { + /* Non-string values (numbers, lists, maps) come back as their JSON. */ + return yyjson_val_write(v, 0, NULL); + } else { + s = fallback; + } + if (!s) { + return NULL; + } + size_t n = strlen(s); + char *d = malloc(n + 1); + if (d) { + memcpy(d, s, n + 1); + } + return d; +} + +bool cbm_neo4j_ident_ok(const char *s) { + if (!s || !*s || strlen(s) > 128) { + return false; + } + if (!isalpha((unsigned char)s[0]) && s[0] != '_') { + return false; + } + for (const char *p = s; *p; p++) { + if (!isalnum((unsigned char)*p) && *p != '_') { + return false; + } + } + return true; +} diff --git a/src/neo4j/neo4j_client.h b/src/neo4j/neo4j_client.h new file mode 100644 index 0000000000..2478a61daa --- /dev/null +++ b/src/neo4j/neo4j_client.h @@ -0,0 +1,122 @@ +/* + * neo4j_client.h — Neo4j connection config + Cypher execution over the + * transactional HTTP endpoint (POST /db//tx/commit). + * + * A request carries one or more statements that Neo4j runs in a single + * transaction; results come back per statement as rows of JSON values. + */ +#ifndef CBM_NEO4J_CLIENT_H +#define CBM_NEO4J_CLIENT_H + +#include +#include +#include + +#include "yyjson/yyjson.h" + +#define CBM_NEO4J_URI_MAX 1024 +#define CBM_NEO4J_USER_MAX 256 +#define CBM_NEO4J_PASSWORD_MAX 512 +#define CBM_NEO4J_DATABASE_MAX 128 +#define CBM_NEO4J_ERR_MAX 1024 + +/* Worker hand-off: the daemon passes the password to the index worker in + * this environment variable instead of on the worker's command line. */ +#define CBM_NEO4J_PASSWORD_ENV "CBM_NEO4J_PASSWORD" + +/* store_meta key holding the project's graph backend (no password). */ +#define CBM_NEO4J_META_KEY "graph_backend" + +typedef struct cbm_neo4j_config { + char uri[CBM_NEO4J_URI_MAX]; + char user[CBM_NEO4J_USER_MAX]; + char password[CBM_NEO4J_PASSWORD_MAX]; + char database[CBM_NEO4J_DATABASE_MAX]; +} cbm_neo4j_config_t; + +/* Build a config from explicit values (index_repository's graph_* args). + * uri is required. Missing user/password/database fall back to NEO4J_USER / + * NEO4J_PASSWORD (then CBM_NEO4J_PASSWORD, then the in-process credential + * cache) / NEO4J_DATABASE, then to "neo4j" / "" / "neo4j". + * Returns 0, or -1 with err filled. */ +int cbm_neo4j_config_resolve(const char *uri, const char *user, const char *password, + const char *database, cbm_neo4j_config_t *out, char *err, + size_t err_sz); + +/* Serialize the config for store_meta (password omitted). malloc'd. */ +char *cbm_neo4j_config_to_meta(const cbm_neo4j_config_t *cfg); + +/* Parse a store_meta value back into a config, resolving the password the + * same way cbm_neo4j_config_resolve does. Returns 0, or -1 with err. */ +int cbm_neo4j_config_from_meta(const char *meta_json, cbm_neo4j_config_t *out, char *err, + size_t err_sz); + +/* Remember the password for (uri, user) for the life of the process so read + * tools served by this daemon can reconnect after an index call. */ +void cbm_neo4j_config_remember(const cbm_neo4j_config_t *cfg); + +/* Overwrite the password in memory. */ +void cbm_neo4j_config_wipe(cbm_neo4j_config_t *cfg); + +typedef struct cbm_neo4j cbm_neo4j_t; + +/* Create a client (no network I/O yet). Returns NULL with err on bad URL. */ +cbm_neo4j_t *cbm_neo4j_connect(const cbm_neo4j_config_t *cfg, char *err, size_t err_sz); +void cbm_neo4j_close(cbm_neo4j_t *c); +const char *cbm_neo4j_error(const cbm_neo4j_t *c); +const char *cbm_neo4j_uri(const cbm_neo4j_t *c); + +/* Tests: forget every login rejection (the 60 s per-credential backoff that + * keeps repeated wrong passwords from tripping Neo4j's lockout). */ +void cbm_neo4j_test_reset_auth_backoff(void); + +/* Round-trip "RETURN 1" to verify reachability, credentials and database. */ +int cbm_neo4j_ping(cbm_neo4j_t *c); + +/* ── Statement batches ─────────────────────────────────────────── */ + +typedef struct { + yyjson_mut_doc *doc; + yyjson_mut_val *statements; + int count; +} cbm_neo4j_batch_t; + +typedef struct { + yyjson_doc *doc; + yyjson_val *results; /* array, one entry per statement */ +} cbm_neo4j_result_t; + +int cbm_neo4j_batch_init(cbm_neo4j_batch_t *b); +/* Append a statement; returns its parameters object (owned by b->doc). */ +yyjson_mut_val *cbm_neo4j_batch_add(cbm_neo4j_batch_t *b, const char *cypher); +void cbm_neo4j_batch_free(cbm_neo4j_batch_t *b); + +/* Run all statements in one transaction. out may be NULL. Returns 0, or -1 + * with cbm_neo4j_error() describing the transport or Cypher failure. */ +int cbm_neo4j_batch_run(cbm_neo4j_t *c, cbm_neo4j_batch_t *b, cbm_neo4j_result_t *out); + +/* Convenience: one statement. params may be NULL; it must belong to b's doc, + * so build it via cbm_neo4j_batch_add and run with cbm_neo4j_batch_run. */ + +/* NULL when a 2xx body's "results" holds exactly one well-formed entry per + * statement ({"columns":[...], "data":[{"row":[one value per column]}]}); + * otherwise a short description of the defect. cbm_neo4j_batch_run refuses + * any response this rejects, so a malformed reply is never "no rows". */ +const char *cbm_neo4j_result_shape_error(yyjson_val *results, int statements); + +/* Rows of statement i: array of {"row":[...]} objects, or NULL. */ +yyjson_val *cbm_neo4j_result_data(const cbm_neo4j_result_t *r, size_t stmt_index); +/* Column j of a data entry. */ +yyjson_val *cbm_neo4j_row_col(yyjson_val *data_entry, size_t col); +void cbm_neo4j_result_free(cbm_neo4j_result_t *r); + +/* Value helpers tolerant of null / wrong type. */ +int64_t cbm_neo4j_val_int(yyjson_val *v, int64_t fallback); +/* malloc'd copy of a string value; non-strings are serialized as JSON; + * null → NULL (or strdup(fallback) when fallback != NULL). */ +char *cbm_neo4j_val_strdup(yyjson_val *v, const char *fallback); + +/* True when s is a safe bare identifier for a label / relationship type. */ +bool cbm_neo4j_ident_ok(const char *s); + +#endif /* CBM_NEO4J_CLIENT_H */ diff --git a/src/neo4j/neo4j_http.c b/src/neo4j/neo4j_http.c new file mode 100644 index 0000000000..6360ef1fc8 --- /dev/null +++ b/src/neo4j/neo4j_http.c @@ -0,0 +1,791 @@ +/* + * neo4j_http.c — Minimal HTTP/1.1 client for the Neo4j HTTP API. + */ +#include "neo4j/neo4j_http.h" + +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#include +#include +#include +typedef SOCKET cbm_n4_sock_t; +#define N4_SOCK_BAD INVALID_SOCKET +#define n4_sock_close closesocket +#else +#include +#include +#include +#include +#include +#include +#include +typedef int cbm_n4_sock_t; +#define N4_SOCK_BAD (-1) +#define n4_sock_close close +#endif + +#ifdef MSG_NOSIGNAL +#define N4_SEND_FLAGS MSG_NOSIGNAL +#else +#define N4_SEND_FLAGS 0 +#endif + +#define N4_DEFAULT_HTTP_PORT 7474 +#define N4_DEFAULT_HTTPS_PORT 7473 +#define N4_IO_TIMEOUT_MS 120000 +#define N4_HDR_MAX (64U * 1024U) +#define N4_RESP_MAX ((size_t)1 << 30) /* 1 GiB: refuse anything larger */ + +struct cbm_neo4j_http { + cbm_neo4j_url_t url; + char *auth_header; /* "Basic xxx" or NULL */ + cbm_n4_sock_t sock; + /* Bytes read past the end of the previous response (keep-alive). */ + char *carry; + size_t carry_len; +#ifdef _WIN32 + HINTERNET session; + HINTERNET connect; +#endif +}; + +static void n4_err(char *err, size_t err_sz, const char *fmt, const char *arg) { + if (err && err_sz > 0) { + (void)snprintf(err, err_sz, fmt, arg ? arg : ""); + } +} + +/* ── URL / base64 ───────────────────────────────────────────────── */ + +static bool n4_has_prefix_ci(const char *s, const char *p) { + for (; *p; s++, p++) { + if (tolower((unsigned char)*s) != tolower((unsigned char)*p)) { + return false; + } + } + return true; +} + +int cbm_neo4j_url_parse(const char *url, cbm_neo4j_url_t *out, char *err, size_t err_sz) { + if (!url || !out || !*url) { + n4_err(err, err_sz, "graph_db URL is empty%s", NULL); + return -1; + } + memset(out, 0, sizeof(*out)); + const char *p = NULL; + if (n4_has_prefix_ci(url, "http://")) { + p = url + strlen("http://"); + out->port = N4_DEFAULT_HTTP_PORT; + } else if (n4_has_prefix_ci(url, "https://")) { + p = url + strlen("https://"); + out->tls = true; + out->port = N4_DEFAULT_HTTPS_PORT; + } else if (n4_has_prefix_ci(url, "bolt") || n4_has_prefix_ci(url, "neo4j")) { + n4_err(err, err_sz, + "unsupported graph_db scheme in '%s': codebase-memory-mcp talks to Neo4j over its " + "HTTP API; use http://:7474 (or https://:7473) instead of bolt/neo4j", + url); + return -1; + } else { + n4_err(err, err_sz, "graph_db URL must start with http:// or https:// (got '%s')", url); + return -1; + } + + const char *host_end = p; + if (*p == '[') { /* IPv6 literal */ + host_end = strchr(p, ']'); + if (!host_end) { + n4_err(err, err_sz, "malformed IPv6 host in '%s'", url); + return -1; + } + size_t hl = (size_t)(host_end - (p + 1)); + if (hl == 0 || hl >= sizeof(out->host)) { + n4_err(err, err_sz, "bad host in '%s'", url); + return -1; + } + memcpy(out->host, p + 1, hl); + host_end++; + } else { + while (*host_end && *host_end != ':' && *host_end != '/') { + host_end++; + } + size_t hl = (size_t)(host_end - p); + if (hl == 0 || hl >= sizeof(out->host)) { + n4_err(err, err_sz, "bad host in '%s'", url); + return -1; + } + memcpy(out->host, p, hl); + } + p = host_end; + if (*p == ':') { + p++; + char *end = NULL; + long port = strtol(p, &end, 10); + if (end == p || port <= 0 || port > 65535) { + n4_err(err, err_sz, "bad port in '%s'", url); + return -1; + } + out->port = (int)port; + p = end; + } + if (*p && *p != '/') { + n4_err(err, err_sz, "malformed graph_db URL '%s'", url); + return -1; + } + size_t pl = strlen(p); + while (pl > 0 && p[pl - 1] == '/') { + pl--; + } + if (pl >= sizeof(out->base_path)) { + n4_err(err, err_sz, "URL path too long in '%s'", url); + return -1; + } + memcpy(out->base_path, p, pl); + out->base_path[pl] = '\0'; + return 0; +} + +char *cbm_neo4j_base64(const char *data, size_t len) { + static const char tbl[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + size_t out_len = ((len + 2) / 3) * 4; + char *out = malloc(out_len + 1); + if (!out) { + return NULL; + } + const unsigned char *in = (const unsigned char *)data; + size_t o = 0; + size_t i = 0; + for (; i + 2 < len; i += 3) { + uint32_t v = ((uint32_t)in[i] << 16) | ((uint32_t)in[i + 1] << 8) | in[i + 2]; + out[o++] = tbl[(v >> 18) & 63]; + out[o++] = tbl[(v >> 12) & 63]; + out[o++] = tbl[(v >> 6) & 63]; + out[o++] = tbl[v & 63]; + } + if (i < len) { + uint32_t v = (uint32_t)in[i] << 16; + if (i + 1 < len) { + v |= (uint32_t)in[i + 1] << 8; + } + out[o++] = tbl[(v >> 18) & 63]; + out[o++] = tbl[(v >> 12) & 63]; + out[o++] = (i + 1 < len) ? tbl[(v >> 6) & 63] : '='; + out[o++] = '='; + } + out[o] = '\0'; + return out; +} + +/* ── Plain-socket transport ─────────────────────────────────────── */ + +static void n4_net_init(void) { +#ifdef _WIN32 + static volatile LONG done = 0; + if (InterlockedCompareExchange(&done, 1, 0) == 0) { + WSADATA wsa; + (void)WSAStartup(MAKEWORD(2, 2), &wsa); + } +#endif +} + +static void n4_set_timeouts(cbm_n4_sock_t s) { +#ifdef _WIN32 + DWORD tv = N4_IO_TIMEOUT_MS; + (void)setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, (const char *)&tv, sizeof(tv)); + (void)setsockopt(s, SOL_SOCKET, SO_SNDTIMEO, (const char *)&tv, sizeof(tv)); +#else + struct timeval tv; + tv.tv_sec = N4_IO_TIMEOUT_MS / 1000; + tv.tv_usec = 0; + (void)setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); + (void)setsockopt(s, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv)); +#endif + int one = 1; + (void)setsockopt(s, IPPROTO_TCP, TCP_NODELAY, (const char *)&one, sizeof(one)); +} + +static void n4_disconnect(cbm_neo4j_http_t *h) { + if (h->sock != N4_SOCK_BAD) { + n4_sock_close(h->sock); + h->sock = N4_SOCK_BAD; + } + free(h->carry); + h->carry = NULL; + h->carry_len = 0; +} + +static int n4_connect(cbm_neo4j_http_t *h, char *err, size_t err_sz) { + n4_net_init(); + char port_s[16]; + (void)snprintf(port_s, sizeof(port_s), "%d", h->url.port); + struct addrinfo hints; + memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + struct addrinfo *res = NULL; + if (getaddrinfo(h->url.host, port_s, &hints, &res) != 0 || !res) { + n4_err(err, err_sz, "cannot resolve Neo4j host '%s'", h->url.host); + return -1; + } + cbm_n4_sock_t s = N4_SOCK_BAD; + for (struct addrinfo *ai = res; ai; ai = ai->ai_next) { + s = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol); + if (s == N4_SOCK_BAD) { + continue; + } + if (connect(s, ai->ai_addr, (int)ai->ai_addrlen) == 0) { + break; + } + n4_sock_close(s); + s = N4_SOCK_BAD; + } + freeaddrinfo(res); + if (s == N4_SOCK_BAD) { + char buf[CBM_NEO4J_HOST_MAX + 32]; + (void)snprintf(buf, sizeof(buf), "%s:%d", h->url.host, h->url.port); + n4_err(err, err_sz, "cannot connect to Neo4j at %s (is the server running?)", buf); + return -1; + } + n4_set_timeouts(s); + h->sock = s; + return 0; +} + +static int n4_send_all(cbm_n4_sock_t s, const char *buf, size_t len) { + while (len > 0) { + int chunk = len > (size_t)(1 << 30) ? (1 << 30) : (int)len; + int n = (int)send(s, buf, chunk, N4_SEND_FLAGS); + if (n <= 0) { + return -1; + } + buf += n; + len -= (size_t)n; + } + return 0; +} + +typedef struct { + char *data; + size_t len; + size_t cap; +} n4_buf_t; + +static int n4_buf_reserve(n4_buf_t *b, size_t extra) { + if (b->len + extra + 1 <= b->cap) { + return 0; + } + size_t cap = b->cap ? b->cap : 8192; + while (cap < b->len + extra + 1) { + cap *= 2; + } + if (cap > N4_RESP_MAX + N4_HDR_MAX) { + return -1; + } + char *d = realloc(b->data, cap); + if (!d) { + return -1; + } + b->data = d; + b->cap = cap; + return 0; +} + +/* Read more bytes from the socket into b. Returns bytes read, 0 on EOF, -1 on error. */ +static int n4_recv_more(cbm_neo4j_http_t *h, n4_buf_t *b) { + if (n4_buf_reserve(b, 16384) != 0) { + return -1; + } + int n = (int)recv(h->sock, b->data + b->len, 16384, 0); + if (n > 0) { + b->len += (size_t)n; + b->data[b->len] = '\0'; + } + return n; +} + +static const char *n4_find_hdr_end(const char *d, size_t len) { + for (size_t i = 0; i + 3 < len; i++) { + if (d[i] == '\r' && d[i + 1] == '\n' && d[i + 2] == '\r' && d[i + 3] == '\n') { + return d + i; + } + } + return NULL; +} + +/* Case-insensitive header lookup within [hdrs, end). Copies value into out. */ +static bool n4_header(const char *hdrs, const char *end, const char *name, char *out, + size_t out_sz) { + size_t nl = strlen(name); + const char *line = hdrs; + while (line < end) { + const char *eol = line; + while (eol < end && *eol != '\r') { + eol++; + } + if ((size_t)(eol - line) > nl && line[nl] == ':' && n4_has_prefix_ci(line, name)) { + const char *v = line + nl + 1; + while (v < eol && (*v == ' ' || *v == '\t')) { + v++; + } + size_t vl = (size_t)(eol - v); + if (vl >= out_sz) { + vl = out_sz - 1; + } + memcpy(out, v, vl); + out[vl] = '\0'; + return true; + } + line = eol + 2; + } + return false; +} + +/* Decode a chunked body in place. Returns decoded length, or -1 when the + * buffer does not yet hold the complete body (*need_more set) or is malformed. */ +static long long n4_dechunk(char *body, size_t len, size_t *consumed, bool *need_more) { + size_t r = 0; + size_t w = 0; + *need_more = false; + for (;;) { + const char *crlf = NULL; + for (size_t i = r; i + 1 < len; i++) { + if (body[i] == '\r' && body[i + 1] == '\n') { + crlf = body + i; + break; + } + } + if (!crlf) { + *need_more = true; + return -1; + } + char *endp = NULL; + unsigned long long sz = strtoull(body + r, &endp, 16); + if (endp == body + r) { + return -1; + } + size_t data_start = (size_t)(crlf - body) + 2; + if (sz == 0) { + /* Trailer section ends with an empty line. */ + size_t t = data_start; + for (;;) { + if (t + 1 >= len) { + *need_more = true; + return -1; + } + if (body[t] == '\r' && body[t + 1] == '\n') { + *consumed = t + 2; + return (long long)w; + } + while (t + 1 < len && !(body[t] == '\r' && body[t + 1] == '\n')) { + t++; + } + if (t + 1 >= len) { + *need_more = true; + return -1; + } + t += 2; + } + } + if (data_start + sz + 2 > len) { + *need_more = true; + return -1; + } + memmove(body + w, body + data_start, (size_t)sz); + w += (size_t)sz; + r = data_start + (size_t)sz + 2; + } +} + +/* One request/response exchange over the plain socket. Returns 0 on success, + * -1 on transport failure; *retryable is set when the failure happened on a + * reused keep-alive connection before any response byte arrived. */ +static int n4_socket_exchange(cbm_neo4j_http_t *h, const char *req, size_t req_len, + const char *body, size_t body_len, int *status, char **resp, + size_t *resp_len, bool *retryable, char *err, size_t err_sz) { + *retryable = false; + bool reused = h->sock != N4_SOCK_BAD; + if (!reused && n4_connect(h, err, err_sz) != 0) { + return -1; + } + if (n4_send_all(h->sock, req, req_len) != 0 || + (body_len > 0 && n4_send_all(h->sock, body, body_len) != 0)) { + n4_disconnect(h); + *retryable = reused; + n4_err(err, err_sz, "send to Neo4j failed%s", NULL); + return -1; + } + + n4_buf_t b = {0}; + if (h->carry_len > 0) { + if (n4_buf_reserve(&b, h->carry_len) != 0) { + n4_disconnect(h); + return -1; + } + memcpy(b.data, h->carry, h->carry_len); + b.len = h->carry_len; + b.data[b.len] = '\0'; + free(h->carry); + h->carry = NULL; + h->carry_len = 0; + } + const char *hdr_end = NULL; + while (!(hdr_end = n4_find_hdr_end(b.data ? b.data : "", b.len))) { + if (b.len > N4_HDR_MAX) { + free(b.data); + n4_disconnect(h); + n4_err(err, err_sz, "Neo4j response headers too large%s", NULL); + return -1; + } + int n = n4_recv_more(h, &b); + if (n <= 0) { + *retryable = reused && b.len == 0; + free(b.data); + n4_disconnect(h); + n4_err(err, err_sz, "no response from Neo4j (connection closed or timed out)%s", NULL); + return -1; + } + } + size_t hdr_len = (size_t)(hdr_end - b.data); + int code = 0; + if (sscanf(b.data, "HTTP/%*d.%*d %d", &code) != 1) { + free(b.data); + n4_disconnect(h); + n4_err(err, err_sz, "malformed HTTP status line from Neo4j%s", NULL); + return -1; + } + char val[128]; + bool chunked = n4_header(b.data, hdr_end + 2, "Transfer-Encoding", val, sizeof(val)) && + strstr(val, "chunked") != NULL; + bool conn_close = n4_header(b.data, hdr_end + 2, "Connection", val, sizeof(val)) && + n4_has_prefix_ci(val, "close"); + long long content_len = -1; + if (!chunked && n4_header(b.data, hdr_end + 2, "Content-Length", val, sizeof(val))) { + content_len = strtoll(val, NULL, 10); + } + size_t body_off = hdr_len + 4; + + size_t out_len = 0; + size_t consumed = 0; + if (chunked) { + for (;;) { + /* Work on a scratch copy so a partial decode never corrupts b. */ + size_t avail = b.len - body_off; + char *tmp = malloc(avail + 1); + if (!tmp) { + free(b.data); + n4_disconnect(h); + return -1; + } + memcpy(tmp, b.data + body_off, avail); + bool need_more = false; + long long dl = n4_dechunk(tmp, avail, &consumed, &need_more); + if (dl >= 0) { + memcpy(b.data + body_off, tmp, (size_t)dl); + free(tmp); + out_len = (size_t)dl; + /* Decoded bytes sit before `consumed`, so any surplus after it + * is still intact for the carry below. */ + consumed += body_off; + break; + } + free(tmp); + if (!need_more) { + free(b.data); + n4_disconnect(h); + n4_err(err, err_sz, "malformed chunked response from Neo4j%s", NULL); + return -1; + } + if (n4_recv_more(h, &b) <= 0) { + free(b.data); + n4_disconnect(h); + n4_err(err, err_sz, "truncated response from Neo4j%s", NULL); + return -1; + } + } + } else if (content_len >= 0) { + if ((size_t)content_len > N4_RESP_MAX) { + free(b.data); + n4_disconnect(h); + n4_err(err, err_sz, "Neo4j response too large%s", NULL); + return -1; + } + while (b.len - body_off < (size_t)content_len) { + if (n4_recv_more(h, &b) <= 0) { + free(b.data); + n4_disconnect(h); + n4_err(err, err_sz, "truncated response from Neo4j%s", NULL); + return -1; + } + } + out_len = (size_t)content_len; + consumed = body_off + out_len; + } else { + /* Body delimited by connection close. */ + while (n4_recv_more(h, &b) > 0) {} + out_len = b.len - body_off; + consumed = b.len; + conn_close = true; + } + + /* Keep any surplus bytes for the next response (rare; no pipelining). */ + if (consumed < b.len) { + h->carry_len = b.len - consumed; + h->carry = malloc(h->carry_len); + if (h->carry) { + memcpy(h->carry, b.data + consumed, h->carry_len); + } else { + h->carry_len = 0; + } + } + + char *out = malloc(out_len + 1); + if (!out) { + free(b.data); + n4_disconnect(h); + return -1; + } + memcpy(out, b.data + body_off, out_len); + out[out_len] = '\0'; + free(b.data); + if (conn_close) { + n4_disconnect(h); + } + *status = code; + *resp = out; + if (resp_len) { + *resp_len = out_len; + } + return 0; +} + +/* ── WinHTTP transport (https on Windows) ───────────────────────── */ + +#ifdef _WIN32 +static wchar_t *n4_widen(const char *s) { + int n = MultiByteToWideChar(CP_UTF8, 0, s, -1, NULL, 0); + if (n <= 0) { + return NULL; + } + wchar_t *w = malloc((size_t)n * sizeof(wchar_t)); + if (w) { + (void)MultiByteToWideChar(CP_UTF8, 0, s, -1, w, n); + } + return w; +} + +static int n4_winhttp_post(cbm_neo4j_http_t *h, const char *full_path, const char *body, + size_t body_len, int *status, char **resp, size_t *resp_len, char *err, + size_t err_sz) { + if (!h->session) { + h->session = WinHttpOpen(L"codebase-memory-mcp", WINHTTP_ACCESS_TYPE_DEFAULT_PROXY, + WINHTTP_NO_PROXY_NAME, WINHTTP_NO_PROXY_BYPASS, 0); + if (!h->session) { + n4_err(err, err_sz, "WinHttpOpen failed%s", NULL); + return -1; + } + (void)WinHttpSetTimeouts(h->session, N4_IO_TIMEOUT_MS, N4_IO_TIMEOUT_MS, N4_IO_TIMEOUT_MS, + N4_IO_TIMEOUT_MS); + wchar_t *whost = n4_widen(h->url.host); + if (!whost) { + return -1; + } + h->connect = WinHttpConnect(h->session, whost, (INTERNET_PORT)h->url.port, 0); + free(whost); + if (!h->connect) { + n4_err(err, err_sz, "cannot connect to Neo4j host '%s'", h->url.host); + return -1; + } + } + wchar_t *wpath = n4_widen(full_path); + if (!wpath) { + return -1; + } + HINTERNET req = WinHttpOpenRequest(h->connect, L"POST", wpath, NULL, WINHTTP_NO_REFERER, + WINHTTP_DEFAULT_ACCEPT_TYPES, WINHTTP_FLAG_SECURE); + free(wpath); + if (!req) { + n4_err(err, err_sz, "WinHttpOpenRequest failed%s", NULL); + return -1; + } + char hdrs[1024]; + (void)snprintf(hdrs, sizeof(hdrs), + "Content-Type: application/json\r\nAccept: application/json\r\n%s%s%s", + h->auth_header ? "Authorization: " : "", h->auth_header ? h->auth_header : "", + h->auth_header ? "\r\n" : ""); + wchar_t *whdrs = n4_widen(hdrs); + BOOL ok = whdrs && WinHttpSendRequest(req, whdrs, (DWORD)-1L, (LPVOID)body, (DWORD)body_len, + (DWORD)body_len, 0); + free(whdrs); + if (ok) { + ok = WinHttpReceiveResponse(req, NULL); + } + if (!ok) { + char code[32]; + (void)snprintf(code, sizeof(code), "%lu", (unsigned long)GetLastError()); + n4_err(err, err_sz, "HTTPS request to Neo4j failed (WinHTTP error %s)", code); + WinHttpCloseHandle(req); + return -1; + } + DWORD sc = 0; + DWORD sc_len = sizeof(sc); + (void)WinHttpQueryHeaders(req, WINHTTP_QUERY_STATUS_CODE | WINHTTP_QUERY_FLAG_NUMBER, + WINHTTP_HEADER_NAME_BY_INDEX, &sc, &sc_len, WINHTTP_NO_HEADER_INDEX); + n4_buf_t b = {0}; + for (;;) { + DWORD avail = 0; + if (!WinHttpQueryDataAvailable(req, &avail) || avail == 0) { + break; + } + if (n4_buf_reserve(&b, avail) != 0) { + free(b.data); + WinHttpCloseHandle(req); + n4_err(err, err_sz, "Neo4j response too large%s", NULL); + return -1; + } + DWORD got = 0; + if (!WinHttpReadData(req, b.data + b.len, avail, &got) || got == 0) { + break; + } + b.len += got; + b.data[b.len] = '\0'; + } + WinHttpCloseHandle(req); + if (!b.data) { + b.data = calloc(1, 1); + } + *status = (int)sc; + *resp = b.data; + if (resp_len) { + *resp_len = b.len; + } + return 0; +} +#endif + +/* ── Public API ─────────────────────────────────────────────────── */ + +cbm_neo4j_http_t *cbm_neo4j_http_open(const cbm_neo4j_url_t *url, const char *user, + const char *password, char *err, size_t err_sz) { +#ifndef _WIN32 + if (url->tls) { + n4_err(err, err_sz, + "https:// graph_db URLs are only supported on Windows builds; use http:// " + "(e.g. through a local TLS-terminating proxy or SSH tunnel) on this platform%s", + NULL); + return NULL; + } +#endif + cbm_neo4j_http_t *h = calloc(1, sizeof(*h)); + if (!h) { + return NULL; + } + h->url = *url; + h->sock = N4_SOCK_BAD; + if (user && *user) { + size_t ul = strlen(user); + size_t pl = password ? strlen(password) : 0; + char *cred = malloc(ul + pl + 2); + if (!cred) { + free(h); + return NULL; + } + memcpy(cred, user, ul); + cred[ul] = ':'; + if (pl) { + memcpy(cred + ul + 1, password, pl); + } + cred[ul + pl + 1] = '\0'; + char *b64 = cbm_neo4j_base64(cred, ul + pl + 1); + /* Wipe the plaintext credential before freeing. */ + memset(cred, 0, ul + pl + 1); + free(cred); + if (!b64) { + free(h); + return NULL; + } + size_t hl = strlen(b64) + sizeof("Basic "); + h->auth_header = malloc(hl); + if (!h->auth_header) { + free(b64); + free(h); + return NULL; + } + (void)snprintf(h->auth_header, hl, "Basic %s", b64); + free(b64); + } + return h; +} + +int cbm_neo4j_http_post(cbm_neo4j_http_t *h, const char *path, const char *body, size_t body_len, + int *status, char **resp, size_t *resp_len, char *err, size_t err_sz) { + if (!h || !path || !status || !resp) { + return -1; + } + *resp = NULL; + char full_path[CBM_NEO4J_PATH_MAX * 2]; + (void)snprintf(full_path, sizeof(full_path), "%s%s", h->url.base_path, path); +#ifdef _WIN32 + if (h->url.tls) { + return n4_winhttp_post(h, full_path, body, body_len, status, resp, resp_len, err, err_sz); + } +#endif + size_t req_cap = strlen(full_path) + strlen(h->url.host) + + (h->auth_header ? strlen(h->auth_header) : 0) + 256; + char *req = malloc(req_cap); + if (!req) { + return -1; + } + int req_len = snprintf(req, req_cap, + "POST %s HTTP/1.1\r\n" + "Host: %s:%d\r\n" + "Content-Type: application/json\r\n" + "Accept: application/json;charset=UTF-8\r\n" + "Content-Length: %zu\r\n" + "Connection: keep-alive\r\n" + "%s%s%s" + "\r\n", + full_path, h->url.host, h->url.port, body_len, + h->auth_header ? "Authorization: " : "", + h->auth_header ? h->auth_header : "", h->auth_header ? "\r\n" : ""); + if (req_len < 0 || (size_t)req_len >= req_cap) { + free(req); + return -1; + } + bool retryable = false; + int rc = n4_socket_exchange(h, req, (size_t)req_len, body, body_len, status, resp, resp_len, + &retryable, err, err_sz); + if (rc != 0 && retryable) { + /* The server closed an idle keep-alive connection: reconnect once. */ + rc = n4_socket_exchange(h, req, (size_t)req_len, body, body_len, status, resp, resp_len, + &retryable, err, err_sz); + } + memset(req, 0, (size_t)req_len); /* carries the auth header */ + free(req); + return rc; +} + +void cbm_neo4j_http_close(cbm_neo4j_http_t *h) { + if (!h) { + return; + } + n4_disconnect(h); +#ifdef _WIN32 + if (h->connect) { + WinHttpCloseHandle(h->connect); + } + if (h->session) { + WinHttpCloseHandle(h->session); + } +#endif + if (h->auth_header) { + memset(h->auth_header, 0, strlen(h->auth_header)); + free(h->auth_header); + } + free(h); +} diff --git a/src/neo4j/neo4j_http.h b/src/neo4j/neo4j_http.h new file mode 100644 index 0000000000..3433a536fd --- /dev/null +++ b/src/neo4j/neo4j_http.h @@ -0,0 +1,48 @@ +/* + * neo4j_http.h — Minimal HTTP/1.1 client for the Neo4j HTTP API. + * + * One persistent (keep-alive) connection per handle, POST-only, Basic auth. + * Plain http:// works on every platform over raw sockets. https:// goes + * through WinHTTP on Windows; on POSIX builds it is refused with a clear + * error (no TLS library is vendored). + */ +#ifndef CBM_NEO4J_HTTP_H +#define CBM_NEO4J_HTTP_H + +#include +#include + +#define CBM_NEO4J_HOST_MAX 256 +#define CBM_NEO4J_PATH_MAX 512 + +typedef struct { + bool tls; + char host[CBM_NEO4J_HOST_MAX]; + int port; + char base_path[CBM_NEO4J_PATH_MAX]; /* "" or "/prefix" without trailing slash */ +} cbm_neo4j_url_t; + +/* Parse "http[s]://host[:port][/prefix]". bolt://, neo4j:// and their +s + * variants are rejected with an error that names the HTTP port to use. + * Returns 0 on success, -1 with err filled otherwise. */ +int cbm_neo4j_url_parse(const char *url, cbm_neo4j_url_t *out, char *err, size_t err_sz); + +/* Standard base64 (with padding). Returns a malloc'd string. */ +char *cbm_neo4j_base64(const char *data, size_t len); + +typedef struct cbm_neo4j_http cbm_neo4j_http_t; + +/* Open a client handle; the connection itself is made lazily on the first + * request and re-made once if a kept-alive socket turns out to be dead. */ +cbm_neo4j_http_t *cbm_neo4j_http_open(const cbm_neo4j_url_t *url, const char *user, + const char *password, char *err, size_t err_sz); + +/* POST a JSON body to base_path + path. On success returns 0, sets *status to + * the HTTP status and *resp to a malloc'd NUL-terminated body (caller frees). + * Transport failures return -1 with err filled. */ +int cbm_neo4j_http_post(cbm_neo4j_http_t *h, const char *path, const char *body, size_t body_len, + int *status, char **resp, size_t *resp_len, char *err, size_t err_sz); + +void cbm_neo4j_http_close(cbm_neo4j_http_t *h); + +#endif /* CBM_NEO4J_HTTP_H */ diff --git a/src/neo4j/neo4j_store.c b/src/neo4j/neo4j_store.c new file mode 100644 index 0000000000..9432375888 --- /dev/null +++ b/src/neo4j/neo4j_store.c @@ -0,0 +1,1412 @@ +/* + * neo4j_store.c — Answer a store's graph reads from Neo4j. + * + * Every query is scoped to one mirrored graph: {db: , gen: }, read once when the store is attached + * so all reads of one request see one consistent generation. + */ +#include "neo4j/neo4j_store.h" +#include "neo4j/neo4j_client.h" +#include "neo4j/neo4j_sync.h" + +#include "foundation/log.h" + +#include +#include +#include +#include +#include +#include + +#define N4S_GEN_MAX 128 +#define N4S_ID_CHUNK 5000 + +/* Node columns, in cbm_node_t order. */ +#define N4S_NODE_COLS(v) \ + v ".id, " v ".project, " v ".label, " v ".name, " v ".qualified_name, " v ".file_path, " v \ + ".start_line, " v ".end_line, " v ".properties_json" +#define N4S_NODE_NCOLS 9 + +typedef struct { + cbm_neo4j_t *c; + char db[CBM_NEO4J_URI_MAX]; + int64_t gen; + char err[CBM_NEO4J_ERR_MAX]; + uint64_t queries; /* round trips issued (per-request backend report) */ +} n4s_ctx_t; + +/* ── Query plumbing ─────────────────────────────────────────────── */ + +typedef struct { + cbm_neo4j_batch_t b; + yyjson_mut_val *params; +} n4s_q_t; + +static int q_begin(n4s_ctx_t *x, n4s_q_t *q, const char *cypher) { + if (cbm_neo4j_batch_init(&q->b) != 0) { + return -1; + } + q->params = cbm_neo4j_batch_add(&q->b, cypher); + yyjson_mut_obj_add_strcpy(q->b.doc, q->params, "db", x->db); + yyjson_mut_obj_add_int(q->b.doc, q->params, "gen", x->gen); + return 0; +} + +static void q_str(n4s_q_t *q, const char *key, const char *val) { + if (val) { + yyjson_mut_obj_add_strcpy(q->b.doc, q->params, key, val); + } else { + yyjson_mut_obj_add_null(q->b.doc, q->params, key); + } +} + +static void q_int(n4s_q_t *q, const char *key, int64_t val) { + yyjson_mut_obj_add_int(q->b.doc, q->params, key, val); +} + +static void q_ids(n4s_q_t *q, const char *key, const int64_t *ids, int n) { + yyjson_mut_val *arr = yyjson_mut_arr(q->b.doc); + for (int i = 0; i < n; i++) { + yyjson_mut_arr_add_int(q->b.doc, arr, ids[i]); + } + yyjson_mut_obj_add_val(q->b.doc, q->params, key, arr); +} + +/* Edge types: an empty list means CALLS, as in the SQLite BFS. */ +static void q_types(n4s_q_t *q, const char *key, const char **types, int n) { + yyjson_mut_val *arr = yyjson_mut_arr(q->b.doc); + if (n <= 0) { + yyjson_mut_arr_add_str(q->b.doc, arr, "CALLS"); + } + for (int i = 0; i < n; i++) { + yyjson_mut_arr_add_strcpy(q->b.doc, arr, types[i]); + } + yyjson_mut_obj_add_val(q->b.doc, q->params, key, arr); +} + +/* Run and keep the result; on failure record the error in the ctx. */ +static int q_run(n4s_ctx_t *x, n4s_q_t *q, cbm_neo4j_result_t *r) { + x->queries++; + int rc = cbm_neo4j_batch_run(x->c, &q->b, r); + cbm_neo4j_batch_free(&q->b); + if (rc != 0) { + (void)snprintf(x->err, sizeof(x->err), "%s", cbm_neo4j_error(x->c)); + return CBM_STORE_ERR; + } + return CBM_STORE_OK; +} + +static void node_from_row(yyjson_val *entry, size_t off, cbm_node_t *n) { + memset(n, 0, sizeof(*n)); + n->id = cbm_neo4j_val_int(cbm_neo4j_row_col(entry, off + 0), 0); + n->project = cbm_neo4j_val_strdup(cbm_neo4j_row_col(entry, off + 1), ""); + n->label = cbm_neo4j_val_strdup(cbm_neo4j_row_col(entry, off + 2), ""); + n->name = cbm_neo4j_val_strdup(cbm_neo4j_row_col(entry, off + 3), ""); + n->qualified_name = cbm_neo4j_val_strdup(cbm_neo4j_row_col(entry, off + 4), ""); + n->file_path = cbm_neo4j_val_strdup(cbm_neo4j_row_col(entry, off + 5), ""); + n->start_line = (int)cbm_neo4j_val_int(cbm_neo4j_row_col(entry, off + 6), 0); + n->end_line = (int)cbm_neo4j_val_int(cbm_neo4j_row_col(entry, off + 7), 0); + n->properties_json = cbm_neo4j_val_strdup(cbm_neo4j_row_col(entry, off + 8), "{}"); +} + +static void node_free(cbm_node_t *n) { + free((void *)n->project); + free((void *)n->label); + free((void *)n->name); + free((void *)n->qualified_name); + free((void *)n->file_path); + free((void *)n->properties_json); + memset(n, 0, sizeof(*n)); +} + +static int collect_nodes(n4s_ctx_t *x, n4s_q_t *q, cbm_node_t **out, int *count) { + *out = NULL; + *count = 0; + cbm_neo4j_result_t r; + if (q_run(x, q, &r) != CBM_STORE_OK) { + return CBM_STORE_ERR; + } + yyjson_val *data = cbm_neo4j_result_data(&r, 0); + size_t n = yyjson_arr_size(data); + if (n > 0) { + cbm_node_t *arr = calloc(n, sizeof(cbm_node_t)); + if (!arr) { + cbm_neo4j_result_free(&r); + return CBM_STORE_ERR; + } + for (size_t i = 0; i < n; i++) { + node_from_row(yyjson_arr_get(data, i), 0, &arr[i]); + } + *out = arr; + *count = (int)n; + } + cbm_neo4j_result_free(&r); + return CBM_STORE_OK; +} + +/* ── Ops ────────────────────────────────────────────────────────── */ + +static void op_close(void *ctx) { + n4s_ctx_t *x = ctx; + if (x) { + cbm_neo4j_close(x->c); + free(x); + } +} + +static const char *op_error(void *ctx) { + return ((n4s_ctx_t *)ctx)->err; +} + +static int one_node(n4s_ctx_t *x, n4s_q_t *q, cbm_node_t *out) { + cbm_node_t *nodes = NULL; + int count = 0; + if (collect_nodes(x, q, &nodes, &count) != CBM_STORE_OK) { + return CBM_STORE_ERR; + } + if (count == 0) { + return CBM_STORE_NOT_FOUND; + } + *out = nodes[0]; + for (int i = 1; i < count; i++) { + node_free(&nodes[i]); + } + free(nodes); + return CBM_STORE_OK; +} + +static int op_find_node_by_id(void *ctx, int64_t id, cbm_node_t *out) { + n4s_ctx_t *x = ctx; + n4s_q_t q; + if (q_begin(x, &q, + "MATCH (n:CbmNode {db: $db, gen: $gen, id: $id}) RETURN " N4S_NODE_COLS( + "n") " LIMIT 1") != 0) { + return CBM_STORE_ERR; + } + q_int(&q, "id", id); + return one_node(x, &q, out); +} + +static int op_find_node_by_qn(void *ctx, const char *project, const char *qn, cbm_node_t *out) { + n4s_ctx_t *x = ctx; + n4s_q_t q; + if (q_begin(x, &q, + "MATCH (n:CbmNode {db: $db, gen: $gen, qualified_name: $qn}) " + "WHERE $project IS NULL OR n.project = $project RETURN " N4S_NODE_COLS( + "n") " ORDER BY n.id LIMIT 1") != 0) { + return CBM_STORE_ERR; + } + q_str(&q, "qn", qn); + q_str(&q, "project", project); + return one_node(x, &q, out); +} + +static int op_find_nodes(void *ctx, cbm_remote_node_query_t kind, const char *project, + const char *arg, cbm_node_t **out, int *count) { + n4s_ctx_t *x = ctx; + const char *cypher = NULL; + switch (kind) { + case CBM_REMOTE_NODES_ALL: + cypher = "MATCH (n:CbmNode {db: $db, gen: $gen}) " + "WHERE $project IS NULL OR n.project = $project RETURN " N4S_NODE_COLS( + "n") " ORDER BY n.file_path, n.id"; + break; + case CBM_REMOTE_NODES_BY_NAME: + cypher = "MATCH (n:CbmNode {db: $db, gen: $gen, name: $arg}) " + "WHERE $project IS NULL OR n.project = $project RETURN " N4S_NODE_COLS( + "n") " ORDER BY n.id"; + break; + case CBM_REMOTE_NODES_BY_LABEL: + cypher = "MATCH (n:CbmNode {db: $db, gen: $gen, label: $arg}) " + "WHERE $project IS NULL OR n.project = $project RETURN " N4S_NODE_COLS( + "n") " ORDER BY n.id"; + break; + case CBM_REMOTE_NODES_BY_FILE: + cypher = "MATCH (n:CbmNode {db: $db, gen: $gen, file_path: $arg}) " + "WHERE $project IS NULL OR n.project = $project RETURN " N4S_NODE_COLS( + "n") " ORDER BY n.id"; + break; + /* Candidate supersets: the caller re-applies the exact predicate. */ + case CBM_REMOTE_NODES_QN_ENDS_CI: + cypher = "MATCH (n:CbmNode {db: $db, gen: $gen}) " + "WHERE ($project IS NULL OR n.project = $project) AND " + "(n.qualified_name = $arg OR " + "toLower(n.qualified_name) ENDS WITH toLower('.' + $arg)) RETURN " N4S_NODE_COLS( + "n") " ORDER BY n.id"; + break; + case CBM_REMOTE_NODES_QN_CONTAINS_CI: + cypher = "MATCH (n:CbmNode {db: $db, gen: $gen}) " + "WHERE ($project IS NULL OR n.project = $project) AND " + "toLower(n.qualified_name) CONTAINS toLower($arg) RETURN " N4S_NODE_COLS( + "n") " ORDER BY n.id"; + break; + default: + return CBM_STORE_ERR; + } + n4s_q_t q; + if (q_begin(x, &q, cypher) != 0) { + return CBM_STORE_ERR; + } + q_str(&q, "project", project); + q_str(&q, "arg", arg ? arg : ""); + return collect_nodes(x, &q, out, count); +} + +static int collect_edges(n4s_ctx_t *x, n4s_q_t *q, cbm_edge_t **out, int *count) { + *out = NULL; + *count = 0; + cbm_neo4j_result_t r; + if (q_run(x, q, &r) != CBM_STORE_OK) { + return CBM_STORE_ERR; + } + yyjson_val *data = cbm_neo4j_result_data(&r, 0); + size_t n = yyjson_arr_size(data); + if (n > 0) { + cbm_edge_t *arr = calloc(n, sizeof(cbm_edge_t)); + if (!arr) { + cbm_neo4j_result_free(&r); + return CBM_STORE_ERR; + } + for (size_t i = 0; i < n; i++) { + yyjson_val *e = yyjson_arr_get(data, i); + arr[i].id = cbm_neo4j_val_int(cbm_neo4j_row_col(e, 0), 0); + arr[i].project = cbm_neo4j_val_strdup(cbm_neo4j_row_col(e, 1), ""); + arr[i].source_id = cbm_neo4j_val_int(cbm_neo4j_row_col(e, 2), 0); + arr[i].target_id = cbm_neo4j_val_int(cbm_neo4j_row_col(e, 3), 0); + arr[i].type = cbm_neo4j_val_strdup(cbm_neo4j_row_col(e, 4), ""); + arr[i].properties_json = cbm_neo4j_val_strdup(cbm_neo4j_row_col(e, 5), "{}"); + } + *out = arr; + *count = (int)n; + } + cbm_neo4j_result_free(&r); + return CBM_STORE_OK; +} + +#define N4S_EDGE_RET \ + " RETURN r.id, a.project, a.id, b.id, coalesce(r.type, type(r)), r.properties_json" + +static int op_find_edges(void *ctx, int64_t node_id, bool by_target, const char *type, + cbm_edge_t **out, int *count) { + n4s_ctx_t *x = ctx; + const char *cypher = + by_target + ? "MATCH (a:CbmNode)-[r]->(b:CbmNode {db: $db, gen: $gen, id: $id}) " + "WHERE a.db = $db AND a.gen = $gen AND ($type IS NULL OR r.type = $type)" N4S_EDGE_RET + " ORDER BY r.id" + : "MATCH (a:CbmNode {db: $db, gen: $gen, id: $id})-[r]->(b:CbmNode) " + "WHERE b.db = $db AND b.gen = $gen AND ($type IS NULL OR r.type = $type)" N4S_EDGE_RET + " ORDER BY r.id"; + n4s_q_t q; + if (q_begin(x, &q, cypher) != 0) { + return CBM_STORE_ERR; + } + q_int(&q, "id", node_id); + q_str(&q, "type", type); + return collect_edges(x, &q, out, count); +} + +static int op_find_edges_by_type(void *ctx, const char *project, const char *type, cbm_edge_t **out, + int *count) { + n4s_ctx_t *x = ctx; + n4s_q_t q; + if (q_begin(x, &q, + "MATCH (a:CbmNode {db: $db, gen: $gen})-[r]->(b:CbmNode) " + "WHERE r.type = $type AND ($project IS NULL OR a.project = $project)" N4S_EDGE_RET + " ORDER BY r.id") != 0) { + return CBM_STORE_ERR; + } + q_str(&q, "type", type ? type : ""); + q_str(&q, "project", project); + return collect_edges(x, &q, out, count); +} + +static int op_count(void *ctx, cbm_remote_count_t kind, const char *project, const char *arg) { + n4s_ctx_t *x = ctx; + const char *cypher = NULL; + switch (kind) { + case CBM_REMOTE_COUNT_NODES: + cypher = "MATCH (n:CbmNode {db: $db, gen: $gen}) " + "WHERE $project IS NULL OR n.project = $project RETURN count(n)"; + break; + case CBM_REMOTE_COUNT_EDGES: + cypher = "MATCH (a:CbmNode {db: $db, gen: $gen})-[r]->() " + "WHERE $project IS NULL OR a.project = $project RETURN count(r)"; + break; + case CBM_REMOTE_COUNT_EDGES_BY_TYPE: + cypher = "MATCH (a:CbmNode {db: $db, gen: $gen})-[r]->() " + "WHERE r.type = $arg AND ($project IS NULL OR a.project = $project) " + "RETURN count(r)"; + break; + default: + return CBM_STORE_ERR; + } + n4s_q_t q; + if (q_begin(x, &q, cypher) != 0) { + return CBM_STORE_ERR; + } + q_str(&q, "project", project); + q_str(&q, "arg", arg ? arg : ""); + cbm_neo4j_result_t r; + if (q_run(x, &q, &r) != CBM_STORE_OK) { + return CBM_STORE_ERR; + } + int64_t n = cbm_neo4j_val_int( + cbm_neo4j_row_col(yyjson_arr_get_first(cbm_neo4j_result_data(&r, 0)), 0), 0); + cbm_neo4j_result_free(&r); + return n > INT_MAX ? INT_MAX : (int)n; +} + +static int op_degrees(void *ctx, const int64_t *ids, int id_count, const char *type, int *in, + int *out) { + n4s_ctx_t *x = ctx; + const char *t = (type && type[0]) ? type : NULL; + for (int base = 0; base < id_count; base += N4S_ID_CHUNK) { + int n = id_count - base < N4S_ID_CHUNK ? id_count - base : N4S_ID_CHUNK; + n4s_q_t q; + if (q_begin(x, &q, + "UNWIND range(0, size($ids) - 1) AS i " + "MATCH (n:CbmNode {db: $db, gen: $gen, id: $ids[i]}) " + "RETURN i, size([(n)<-[r]-() WHERE $type IS NULL OR r.type = $type | 1]), " + "size([(n)-[r]->() WHERE $type IS NULL OR r.type = $type | 1])") != 0) { + return CBM_STORE_ERR; + } + q_ids(&q, "ids", ids + base, n); + q_str(&q, "type", t); + cbm_neo4j_result_t r; + if (q_run(x, &q, &r) != CBM_STORE_OK) { + return CBM_STORE_ERR; + } + yyjson_val *data = cbm_neo4j_result_data(&r, 0); + size_t idx; + size_t max; + yyjson_val *e; + yyjson_arr_foreach(data, idx, max, e) { + int64_t i = cbm_neo4j_val_int(cbm_neo4j_row_col(e, 0), -1); + if (i >= 0 && i < n) { + in[base + i] = (int)cbm_neo4j_val_int(cbm_neo4j_row_col(e, 1), 0); + out[base + i] = (int)cbm_neo4j_val_int(cbm_neo4j_row_col(e, 2), 0); + } + } + cbm_neo4j_result_free(&r); + } + return CBM_STORE_OK; +} + +/* ── Traversal ──────────────────────────────────────────────────── */ + +typedef struct { + int64_t *v; + int n; + int cap; +} n4s_ids_t; + +static int ids_push(n4s_ids_t *a, int64_t id) { + if (a->n >= a->cap) { + int cap = a->cap ? a->cap * 2 : 64; + int64_t *v = realloc(a->v, (size_t)cap * sizeof(int64_t)); + if (!v) { + return -1; + } + a->v = v; + a->cap = cap; + } + a->v[a->n++] = id; + return 0; +} + +static int cmp_i64(const void *a, const void *b) { + int64_t x = *(const int64_t *)a; + int64_t y = *(const int64_t *)b; + return (x > y) - (x < y); +} + +/* Open-addressing set of node ids. */ +typedef struct { + int64_t *slots; + unsigned char *used; + size_t cap; + size_t n; +} n4s_set_t; + +static size_t set_hash(int64_t id, size_t cap) { + uint64_t h = (uint64_t)id * 0x9E3779B97F4A7C15ULL; + return (size_t)(h >> 17) & (cap - 1); +} + +static int set_add(n4s_set_t *s, int64_t id); /* fwd */ + +static int set_grow(n4s_set_t *s) { + size_t cap = s->cap ? s->cap * 2 : 256; + n4s_set_t ns = {calloc(cap, sizeof(int64_t)), calloc(cap, 1), cap, 0}; + if (!ns.slots || !ns.used) { + free(ns.slots); + free(ns.used); + return -1; + } + for (size_t i = 0; i < s->cap; i++) { + if (s->used[i]) { + (void)set_add(&ns, s->slots[i]); + } + } + free(s->slots); + free(s->used); + *s = ns; + return 0; +} + +static bool set_has(const n4s_set_t *s, int64_t id) { + if (!s->cap) { + return false; + } + for (size_t i = set_hash(id, s->cap);; i = (i + 1) & (s->cap - 1)) { + if (!s->used[i]) { + return false; + } + if (s->slots[i] == id) { + return true; + } + } +} + +/* 1 when added, 0 when already present, -1 on OOM. */ +static int set_add(n4s_set_t *s, int64_t id) { + if ((s->n + 1) * 2 > s->cap && set_grow(s) != 0) { + return -1; + } + for (size_t i = set_hash(id, s->cap);; i = (i + 1) & (s->cap - 1)) { + if (!s->used[i]) { + s->used[i] = 1; + s->slots[i] = id; + s->n++; + return 1; + } + if (s->slots[i] == id) { + return 0; + } + } +} + +static void set_free(n4s_set_t *s) { + free(s->slots); + free(s->used); + memset(s, 0, sizeof(*s)); +} + +/* Next BFS layer: distinct neighbour ids of `frontier` over the edge types. */ +static int bfs_expand(n4s_ctx_t *x, const n4s_ids_t *frontier, bool inbound, const char **types, + int type_count, n4s_ids_t *next) { + const char *cypher = + inbound ? "UNWIND $ids AS fid MATCH (a:CbmNode {db: $db, gen: $gen, id: fid})<-[r]-(b) " + "WHERE r.type IN $types AND b.db = $db AND b.gen = $gen RETURN DISTINCT b.id" + : "UNWIND $ids AS fid MATCH (a:CbmNode {db: $db, gen: $gen, id: fid})-[r]->(b) " + "WHERE r.type IN $types AND b.db = $db AND b.gen = $gen RETURN DISTINCT b.id"; + for (int base = 0; base < frontier->n; base += N4S_ID_CHUNK) { + int n = frontier->n - base < N4S_ID_CHUNK ? frontier->n - base : N4S_ID_CHUNK; + n4s_q_t q; + if (q_begin(x, &q, cypher) != 0) { + return CBM_STORE_ERR; + } + q_ids(&q, "ids", frontier->v + base, n); + q_types(&q, "types", types, type_count); + cbm_neo4j_result_t r; + if (q_run(x, &q, &r) != CBM_STORE_OK) { + return CBM_STORE_ERR; + } + size_t idx; + size_t max; + yyjson_val *e; + yyjson_arr_foreach(cbm_neo4j_result_data(&r, 0), idx, max, e) { + if (ids_push(next, cbm_neo4j_val_int(cbm_neo4j_row_col(e, 0), 0)) != 0) { + cbm_neo4j_result_free(&r); + return CBM_STORE_ERR; + } + } + cbm_neo4j_result_free(&r); + } + return CBM_STORE_OK; +} + +/* Fetch node rows for ids and place them in visited[] in the given order. */ +static int fill_hops(n4s_ctx_t *x, const int64_t *ids, const int *hops, int n, cbm_node_hop_t **out, + int *out_count) { + *out = NULL; + *out_count = 0; + cbm_node_hop_t *visited = calloc(n > 0 ? (size_t)n : 1, sizeof(cbm_node_hop_t)); + if (!visited) { + return CBM_STORE_ERR; + } + int written = 0; + for (int base = 0; base < n; base += N4S_ID_CHUNK) { + int m = n - base < N4S_ID_CHUNK ? n - base : N4S_ID_CHUNK; + n4s_q_t q; + if (q_begin(x, &q, + "UNWIND range(0, size($ids) - 1) AS i " + "MATCH (n:CbmNode {db: $db, gen: $gen, id: $ids[i]}) " + "RETURN i, " N4S_NODE_COLS("n") " ORDER BY i") != 0) { + free(visited); + return CBM_STORE_ERR; + } + q_ids(&q, "ids", ids + base, m); + cbm_neo4j_result_t r; + if (q_run(x, &q, &r) != CBM_STORE_OK) { + for (int k = 0; k < written; k++) { + node_free(&visited[k].node); + } + free(visited); + return CBM_STORE_ERR; + } + size_t idx; + size_t max; + yyjson_val *e; + yyjson_arr_foreach(cbm_neo4j_result_data(&r, 0), idx, max, e) { + int64_t i = cbm_neo4j_val_int(cbm_neo4j_row_col(e, 0), -1); + if (i < 0 || i >= m) { + continue; + } + /* An id with no node row is skipped, as the SQL JOIN skips it. */ + node_from_row(e, 1, &visited[written].node); + visited[written].hop = hops[base + i]; + written++; + } + cbm_neo4j_result_free(&r); + } + *out = visited; + *out_count = written; + return CBM_STORE_OK; +} + +static int collect_bfs_edges(n4s_ctx_t *x, int64_t root_id, const cbm_node_hop_t *visited, int n, + bool inbound, const char **types, int type_count, int max_edges, + cbm_traverse_result_t *out) { + (void)inbound; + n4s_ids_t ids = {0}; + if (ids_push(&ids, root_id) != 0) { + return CBM_STORE_ERR; + } + for (int i = 0; i < n; i++) { + if (ids_push(&ids, visited[i].node.id) != 0) { + free(ids.v); + return CBM_STORE_ERR; + } + } + char cypher[1024]; + char limit[48] = ""; + if (max_edges > 0) { + (void)snprintf(limit, sizeof(limit), " LIMIT %d", + max_edges < INT_MAX ? max_edges + 1 : max_edges); + } + (void)snprintf(cypher, sizeof(cypher), + "MATCH (a:CbmNode {db: $db, gen: $gen})-[r]->(b:CbmNode {db: $db, gen: $gen}) " + "WHERE a.id IN $ids AND b.id IN $ids AND r.type IN $types " + "RETURN a.name, b.name, r.type, a.id, b.id, r.properties_json " + "ORDER BY a.id, b.id, r.type%s", + limit); + n4s_q_t q; + if (q_begin(x, &q, cypher) != 0) { + free(ids.v); + return CBM_STORE_ERR; + } + q_ids(&q, "ids", ids.v, ids.n); + free(ids.v); + q_types(&q, "types", types, type_count); + cbm_neo4j_result_t r; + if (q_run(x, &q, &r) != CBM_STORE_OK) { + return CBM_STORE_ERR; + } + yyjson_val *data = cbm_neo4j_result_data(&r, 0); + int en = (int)yyjson_arr_size(data); + if (max_edges > 0 && en > max_edges) { + en = max_edges; + out->edges_truncated = true; + } + if (en > 0) { + out->edges = calloc((size_t)en, sizeof(cbm_edge_info_t)); + if (!out->edges) { + cbm_neo4j_result_free(&r); + return CBM_STORE_ERR; + } + for (int i = 0; i < en; i++) { + yyjson_val *e = yyjson_arr_get(data, (size_t)i); + cbm_edge_info_t *ei = &out->edges[i]; + ei->from_name = cbm_neo4j_val_strdup(cbm_neo4j_row_col(e, 0), ""); + ei->to_name = cbm_neo4j_val_strdup(cbm_neo4j_row_col(e, 1), ""); + ei->type = cbm_neo4j_val_strdup(cbm_neo4j_row_col(e, 2), ""); + ei->confidence = 1.0; + ei->source_id = cbm_neo4j_val_int(cbm_neo4j_row_col(e, 3), 0); + ei->target_id = cbm_neo4j_val_int(cbm_neo4j_row_col(e, 4), 0); + ei->properties_json = cbm_neo4j_val_strdup(cbm_neo4j_row_col(e, 5), "{}"); + } + out->edge_count = en; + } + cbm_neo4j_result_free(&r); + return CBM_STORE_OK; +} + +/* Relationship-unique (trail) expansion: Neo4j's own variable-length match + * semantics, distinct (node, hop) pairs in (hop, id) order. */ +static int bfs_trail_ids(n4s_ctx_t *x, int64_t start, bool inbound, const char **types, + int type_count, int max_depth, int probe, n4s_ids_t *ids, + n4s_ids_t *hops) { + char cypher[1024]; + (void)snprintf(cypher, sizeof(cypher), + "MATCH p = (s:CbmNode {db: $db, gen: $gen, id: $id})%s[rs*1..%d]%s(n:CbmNode) " + "WHERE all(x IN rs WHERE x.type IN $types) " + "RETURN DISTINCT n.id AS nid, length(p) AS hop ORDER BY hop, nid LIMIT %d", + inbound ? "<-" : "-", max_depth, inbound ? "-" : "->", probe); + n4s_q_t q; + if (q_begin(x, &q, cypher) != 0) { + return CBM_STORE_ERR; + } + q_int(&q, "id", start); + q_types(&q, "types", types, type_count); + cbm_neo4j_result_t r; + if (q_run(x, &q, &r) != CBM_STORE_OK) { + return CBM_STORE_ERR; + } + size_t idx; + size_t max; + yyjson_val *e; + yyjson_arr_foreach(cbm_neo4j_result_data(&r, 0), idx, max, e) { + if (ids_push(ids, cbm_neo4j_val_int(cbm_neo4j_row_col(e, 0), 0)) != 0 || + ids_push(hops, cbm_neo4j_val_int(cbm_neo4j_row_col(e, 1), 0)) != 0) { + cbm_neo4j_result_free(&r); + return CBM_STORE_ERR; + } + } + cbm_neo4j_result_free(&r); + return CBM_STORE_OK; +} + +/* Shortest-path layers, client side: each node once at its minimal hop, + * layers in ascending id order, stopping once probe rows are known. Seeds + * are excluded in multi mode; a single root reached again by a cycle is + * reported at that hop, exactly like the SQL CTE (which filters hop > 0 + * before taking MIN(hop)). */ +static int bfs_shortest_ids(n4s_ctx_t *x, const int64_t *seeds, int seed_count, bool multi, + bool inbound, const char **types, int type_count, int max_depth, + int probe, n4s_ids_t *ids, n4s_ids_t *hops) { + n4s_set_t reached = {0}; + n4s_set_t seedset = {0}; + n4s_ids_t frontier = {0}; + int rc = CBM_STORE_OK; + for (int i = 0; i < seed_count; i++) { + if (set_add(&seedset, seeds[i]) == 1 && ids_push(&frontier, seeds[i]) != 0) { + rc = CBM_STORE_ERR; + goto done; + } + } + for (int hop = 1; hop <= max_depth && frontier.n > 0 && ids->n < probe; hop++) { + n4s_ids_t next = {0}; + rc = bfs_expand(x, &frontier, inbound, types, type_count, &next); + if (rc != CBM_STORE_OK) { + free(next.v); + goto done; + } + qsort(next.v, (size_t)next.n, sizeof(int64_t), cmp_i64); + frontier.n = 0; + for (int i = 0; i < next.n; i++) { + int64_t id = next.v[i]; + if (i > 0 && next.v[i - 1] == id) { + continue; + } + bool is_seed = set_has(&seedset, id); + if (multi && is_seed) { + continue; + } + int added = set_add(&reached, id); + if (added < 0) { + rc = CBM_STORE_ERR; + break; + } + if (added == 0) { + continue; + } + if (ids->n < probe && (ids_push(ids, id) != 0 || ids_push(hops, hop) != 0)) { + rc = CBM_STORE_ERR; + break; + } + /* A seed re-reached is already expanded from hop 0. */ + if (!is_seed && ids_push(&frontier, id) != 0) { + rc = CBM_STORE_ERR; + break; + } + } + free(next.v); + if (rc != CBM_STORE_OK) { + goto done; + } + } +done: + free(frontier.v); + set_free(&reached); + set_free(&seedset); + return rc; +} + +static int op_bfs(void *ctx, const int64_t *seeds, int seed_count, bool multi, + const char *direction, const char **edge_types, int edge_type_count, + int max_depth, int max_results, bool trail, int max_edges, + cbm_traverse_result_t *out, bool *truncated) { + n4s_ctx_t *x = ctx; + memset(out, 0, sizeof(*out)); + if (truncated) { + *truncated = false; + } + if (max_results < 0) { + max_results = 0; + } + bool inbound = direction && strcmp(direction, "inbound") == 0; + if (!multi) { + int rc = op_find_node_by_id(x, seeds[0], &out->root); + if (rc != CBM_STORE_OK) { + return rc; + } + } + int probe = max_results < INT_MAX ? max_results + 1 : max_results; + n4s_ids_t ids = {0}; + n4s_ids_t hops = {0}; + int rc = CBM_STORE_OK; + if (max_depth > 0 && probe > 0) { + rc = (trail && !multi) ? bfs_trail_ids(x, seeds[0], inbound, edge_types, edge_type_count, + max_depth, probe, &ids, &hops) + : bfs_shortest_ids(x, seeds, seed_count, multi, inbound, edge_types, + edge_type_count, max_depth, probe, &ids, &hops); + } + if (rc == CBM_STORE_OK && ids.n > 0) { + int *hop_ints = malloc((size_t)ids.n * sizeof(int)); + if (!hop_ints) { + rc = CBM_STORE_ERR; + } else { + for (int i = 0; i < ids.n; i++) { + hop_ints[i] = (int)hops.v[i]; + } + rc = fill_hops(x, ids.v, hop_ints, ids.n, &out->visited, &out->visited_count); + free(hop_ints); + } + } + free(ids.v); + free(hops.v); + if (rc != CBM_STORE_OK) { + return rc; + } + if (out->visited_count > max_results) { + /* The probe row fired: drop it and report the ceiling. */ + node_free(&out->visited[max_results].node); + out->visited_count = max_results; + out->truncated = true; + if (truncated) { + *truncated = true; + } + } + if (!multi && out->visited_count > 0 && max_edges != 0) { + rc = collect_bfs_edges(x, seeds[0], out->visited, out->visited_count, inbound, edge_types, + edge_type_count, max_edges, out); + } + return rc; +} + +/* ── Search, names, files, schema ───────────────────────────────── */ + +static void q_str_list(n4s_q_t *q, const char *key, const char *const *items) { + yyjson_mut_val *arr = yyjson_mut_arr(q->b.doc); + for (int i = 0; items && items[i]; i++) { + yyjson_mut_arr_add_strcpy(q->b.doc, arr, items[i]); + } + yyjson_mut_obj_add_val(q->b.doc, q->params, key, arr); +} + +static void q_opt_str(n4s_q_t *q, const char *key, const char *val) { + q_str(q, key, (val && val[0]) ? val : NULL); +} + +/* The edge types search_graph's in/out degree counts (store.c search). */ +static const char *const k_search_degree_types[] = {"CALLS", "USAGE", "CALL_REFERENCE", "INHERITS", + "IMPLEMENTS", NULL}; + +/* Everything after MATCH (n): the filters of cbm_store_search's WHERE, then + * the degree columns its degree filter and rows need. */ +static const char k_search_where[] = + "MATCH (n:CbmNode {db: $db, gen: $gen}) " + "WHERE ($project IS NULL OR n.project = $project) " + "AND ($label IS NULL OR n.label = $label) " + "AND NOT n.label IN $exclude_labels " + "AND all(h IN $name_hints WHERE toLower(n.name) CONTAINS toLower(h)) " + "AND all(h IN $qn_hints WHERE toLower(n.qualified_name) CONTAINS toLower(h)) " + "AND all(h IN $file_hints WHERE toLower(coalesce(n.file_path, '')) CONTAINS toLower(h)) " + "AND ($kind IS NULL OR n.kind = $kind) " + "AND ($ns IS NULL OR (size(split(coalesce(n.obj_key, ''), '/')) >= 4 " + " AND toLower(split(n.obj_key, '/')[2]) = toLower($ns))) " + "AND ($rel IS NULL OR size([(n)-[r]-() WHERE r.type = $rel | 1]) > 0) " + "AND (NOT $exclude_entry_points OR NOT (" + " size([(n)<-[c]-() WHERE c.type = 'CALLS' | 1]) = 0 AND " + " size([(n)-[c]->() WHERE c.type = 'CALLS' | 1]) > 0)) " + "WITH n, size([(n)<-[r]-() WHERE r.type IN $degree_types | 1]) AS in_deg, " + " size([(n)-[r]->() WHERE r.type IN $degree_types | 1]) AS out_deg " + "WHERE ($min_degree < 0 OR in_deg + out_deg >= $min_degree) " + "AND ($max_degree < 0 OR in_deg + out_deg <= $max_degree) "; + +static void q_search_params(n4s_q_t *q, const cbm_remote_search_t *s) { + q_opt_str(q, "project", s->project); + q_opt_str(q, "label", s->label); + q_str_list(q, "exclude_labels", s->exclude_labels); + q_str_list(q, "name_hints", s->name_contains_ci); + q_str_list(q, "qn_hints", s->qn_contains_ci); + q_str_list(q, "file_hints", s->file_contains_ci); + q_opt_str(q, "kind", s->kind); + q_opt_str(q, "ns", s->namespace_name); + q_opt_str(q, "rel", s->relationship); + yyjson_mut_obj_add_bool(q->b.doc, q->params, "exclude_entry_points", s->exclude_entry_points); + q_str_list(q, "degree_types", k_search_degree_types); + q_int(q, "min_degree", s->min_degree); + q_int(q, "max_degree", s->max_degree); +} + +static int op_search(void *ctx, const cbm_remote_search_t *s, cbm_search_result_t **out, + int *count, int *total) { + n4s_ctx_t *x = ctx; + *out = NULL; + *count = 0; + *total = 0; + bool paged = s->limit >= 0; + char rows_cypher[sizeof(k_search_where) + 256]; + (void)snprintf(rows_cypher, sizeof(rows_cypher), + "%sRETURN " N4S_NODE_COLS("n") ", in_deg, out_deg ORDER BY n.name, n.id%s", + k_search_where, paged ? " SKIP $offset LIMIT $limit" : ""); + cbm_neo4j_batch_t b; + if (cbm_neo4j_batch_init(&b) != 0) { + return CBM_STORE_ERR; + } + n4s_q_t q = {.b = b}; + q.params = cbm_neo4j_batch_add(&q.b, rows_cypher); + yyjson_mut_obj_add_strcpy(q.b.doc, q.params, "db", x->db); + yyjson_mut_obj_add_int(q.b.doc, q.params, "gen", x->gen); + q_search_params(&q, s); + if (paged) { + q_int(&q, "offset", s->offset); + q_int(&q, "limit", s->limit); + /* Same filters, counted, in the same transaction. */ + char count_cypher[sizeof(k_search_where) + 32]; + (void)snprintf(count_cypher, sizeof(count_cypher), "%sRETURN count(n)", k_search_where); + yyjson_mut_val *saved = q.params; + q.params = cbm_neo4j_batch_add(&q.b, count_cypher); + yyjson_mut_obj_add_strcpy(q.b.doc, q.params, "db", x->db); + yyjson_mut_obj_add_int(q.b.doc, q.params, "gen", x->gen); + q_search_params(&q, s); + q.params = saved; + } + cbm_neo4j_result_t r; + if (q_run(x, &q, &r) != CBM_STORE_OK) { + return CBM_STORE_ERR; + } + yyjson_val *data = cbm_neo4j_result_data(&r, 0); + size_t n = yyjson_arr_size(data); + cbm_search_result_t *rows = n ? calloc(n, sizeof(cbm_search_result_t)) : NULL; + if (n && !rows) { + cbm_neo4j_result_free(&r); + return CBM_STORE_ERR; + } + for (size_t i = 0; i < n; i++) { + yyjson_val *e = yyjson_arr_get(data, i); + node_from_row(e, 0, &rows[i].node); + rows[i].in_degree = (int)cbm_neo4j_val_int(cbm_neo4j_row_col(e, N4S_NODE_NCOLS), 0); + rows[i].out_degree = (int)cbm_neo4j_val_int(cbm_neo4j_row_col(e, N4S_NODE_NCOLS + 1), 0); + } + *out = rows; + *count = (int)n; + *total = paged ? (int)cbm_neo4j_val_int( + cbm_neo4j_row_col(yyjson_arr_get_first(cbm_neo4j_result_data(&r, 1)), 0), + 0) + : (int)n; + cbm_neo4j_result_free(&r); + return CBM_STORE_OK; +} + +/* Column 0 of statement `stmt`, as a malloc'd string array. */ +static int collect_strings(cbm_neo4j_result_t *r, size_t stmt, char ***out, int *count) { + yyjson_val *data = cbm_neo4j_result_data(r, stmt); + size_t n = yyjson_arr_size(data); + *out = NULL; + *count = 0; + if (n == 0) { + return CBM_STORE_OK; + } + char **arr = calloc(n, sizeof(char *)); + if (!arr) { + return CBM_STORE_ERR; + } + for (size_t i = 0; i < n; i++) { + arr[i] = cbm_neo4j_val_strdup(cbm_neo4j_row_col(yyjson_arr_get(data, i), 0), ""); + } + *out = arr; + *count = (int)n; + return CBM_STORE_OK; +} + +static int op_neighbor_names(void *ctx, int64_t node_id, int limit, char ***callers, + int *caller_count, char ***callees, int *callee_count) { + n4s_ctx_t *x = ctx; + static const char *const directions[] = { + "MATCH (t:CbmNode {db: $db, gen: $gen, id: $id})<-[r]-(n:CbmNode) " + "WHERE r.type IN ['CALLS', 'HTTP_CALLS', 'ASYNC_CALLS'] " + "RETURN DISTINCT n.name AS name ORDER BY name LIMIT $limit", + "MATCH (t:CbmNode {db: $db, gen: $gen, id: $id})-[r]->(n:CbmNode) " + "WHERE r.type IN ['CALLS', 'HTTP_CALLS', 'ASYNC_CALLS'] " + "RETURN DISTINCT n.name AS name ORDER BY name LIMIT $limit"}; + cbm_neo4j_batch_t b; + if (cbm_neo4j_batch_init(&b) != 0) { + return CBM_STORE_ERR; + } + for (int i = 0; i < 2; i++) { + yyjson_mut_val *p = cbm_neo4j_batch_add(&b, directions[i]); + yyjson_mut_obj_add_strcpy(b.doc, p, "db", x->db); + yyjson_mut_obj_add_int(b.doc, p, "gen", x->gen); + yyjson_mut_obj_add_int(b.doc, p, "id", node_id); + yyjson_mut_obj_add_int(b.doc, p, "limit", limit > 0 ? limit : 0); + } + n4s_q_t q = {.b = b}; + cbm_neo4j_result_t r; + if (q_run(x, &q, &r) != CBM_STORE_OK) { + return CBM_STORE_ERR; + } + int rc = collect_strings(&r, 0, callers, caller_count); + if (rc == CBM_STORE_OK) { + rc = collect_strings(&r, 1, callees, callee_count); + } + cbm_neo4j_result_free(&r); + return rc; +} + +static int op_list_files(void *ctx, const char *project, char ***out, int *count) { + n4s_ctx_t *x = ctx; + /* cbm_store_list_files: canonical File nodes, or -- for a graph without + * any -- every non-Folder node path. */ + n4s_q_t q; + if (q_begin(x, &q, + "OPTIONAL MATCH (f:CbmNode {db: $db, gen: $gen, label: 'File'}) " + "WHERE f.project = $project " + "WITH count(f) > 0 AS has_files " + "MATCH (n:CbmNode {db: $db, gen: $gen}) " + "WHERE n.project = $project AND n.label <> 'Folder' " + "AND n.file_path IS NOT NULL AND n.file_path <> '' " + "AND (n.label = 'File' OR NOT has_files) " + "RETURN DISTINCT n.file_path AS path ORDER BY path") != 0) { + return CBM_STORE_ERR; + } + q_str(&q, "project", project); + cbm_neo4j_result_t r; + if (q_run(x, &q, &r) != CBM_STORE_OK) { + return CBM_STORE_ERR; + } + int rc = collect_strings(&r, 0, out, count); + cbm_neo4j_result_free(&r); + return rc; +} + +/* Property keys the sync stores itself; never reported as graph properties. */ +static const char *const k_internal_node_keys[] = { + "db", "gen", "project", "id", "label", "name", "qualified_name", + "file_path", "start_line", "end_line", "properties_json", "prop_keys", NULL}; +static const char *const k_internal_edge_keys[] = {"id", "type", "properties_json", "prop_keys", + NULL}; + +/* base columns + up to 50 JSON keys, exactly like schema_discover_props. */ +enum { N4S_SCHEMA_MAX_KEYS = 50 }; + +static char **schema_props(const char *const *base, int base_count, yyjson_val *keys, int *count) { + size_t nkeys = yyjson_arr_size(keys); + if (nkeys > N4S_SCHEMA_MAX_KEYS) { + nkeys = N4S_SCHEMA_MAX_KEYS; + } + char **props = calloc((size_t)base_count + nkeys + 1, sizeof(char *)); + int n = 0; + if (!props) { + *count = 0; + return NULL; + } + for (int i = 0; i < base_count; i++) { + props[n++] = cbm_neo4j_val_strdup(NULL, base[i]); + } + for (size_t i = 0; i < nkeys; i++) { + props[n++] = cbm_neo4j_val_strdup(yyjson_arr_get(keys, i), ""); + } + *count = n; + return props; +} + +static int op_schema(void *ctx, const char *project, bool with_props, cbm_schema_info_t *out) { + n4s_ctx_t *x = ctx; + static const char *const node_base[] = {"name", "qualified_name", "file_path", "start_line", + "end_line"}; + static const char *const edge_base[] = {"source_id", "target_id"}; + /* prop_keys (every top-level properties key, stored by the sync) is the + * exact json_each key set; graphs synced before it existed fall back to + * the flattened keys. */ + static const char *const statements[] = { + "MATCH (n:CbmNode {db: $db, gen: $gen}) WHERE n.project = $project " + "RETURN n.label AS l, count(*) AS c ORDER BY c DESC, l", + "MATCH (a:CbmNode {db: $db, gen: $gen})-[r]->() WHERE a.project = $project " + "RETURN r.type AS t, count(*) AS c ORDER BY c DESC, t", + "MATCH (n:CbmNode {db: $db, gen: $gen}) WHERE n.project = $project " + "UNWIND coalesce(n.prop_keys, [k IN keys(n) WHERE NOT k IN $node_internal]) AS k " + "WITH n.label AS l, k ORDER BY l, k " + "RETURN l, collect(DISTINCT k)", + "MATCH (a:CbmNode {db: $db, gen: $gen})-[r]->() WHERE a.project = $project " + "UNWIND coalesce(r.prop_keys, [k IN keys(r) WHERE NOT k IN $edge_internal]) AS k " + "WITH r.type AS t, k ORDER BY t, k " + "RETURN t, collect(DISTINCT k)"}; + int nstmt = with_props ? 4 : 2; + cbm_neo4j_batch_t b; + if (cbm_neo4j_batch_init(&b) != 0) { + return CBM_STORE_ERR; + } + for (int i = 0; i < nstmt; i++) { + yyjson_mut_val *p = cbm_neo4j_batch_add(&b, statements[i]); + yyjson_mut_obj_add_strcpy(b.doc, p, "db", x->db); + yyjson_mut_obj_add_int(b.doc, p, "gen", x->gen); + yyjson_mut_obj_add_strcpy(b.doc, p, "project", project ? project : ""); + if (i == 2) { + yyjson_mut_val *arr = yyjson_mut_arr(b.doc); + for (int k = 0; k_internal_node_keys[k]; k++) { + yyjson_mut_arr_add_str(b.doc, arr, k_internal_node_keys[k]); + } + yyjson_mut_obj_add_val(b.doc, p, "node_internal", arr); + } else if (i == 3) { + yyjson_mut_val *arr = yyjson_mut_arr(b.doc); + for (int k = 0; k_internal_edge_keys[k]; k++) { + yyjson_mut_arr_add_str(b.doc, arr, k_internal_edge_keys[k]); + } + yyjson_mut_obj_add_val(b.doc, p, "edge_internal", arr); + } + } + n4s_q_t q = {.b = b}; + cbm_neo4j_result_t r; + if (q_run(x, &q, &r) != CBM_STORE_OK) { + return CBM_STORE_ERR; + } + yyjson_val *labels = cbm_neo4j_result_data(&r, 0); + yyjson_val *types = cbm_neo4j_result_data(&r, 1); + size_t nl = yyjson_arr_size(labels); + size_t nt = yyjson_arr_size(types); + out->node_labels = nl ? calloc(nl, sizeof(cbm_label_count_t)) : NULL; + out->edge_types = nt ? calloc(nt, sizeof(cbm_type_count_t)) : NULL; + if ((nl && !out->node_labels) || (nt && !out->edge_types)) { + cbm_neo4j_result_free(&r); + return CBM_STORE_ERR; + } + for (size_t i = 0; i < nl; i++) { + yyjson_val *e = yyjson_arr_get(labels, i); + out->node_labels[i].label = cbm_neo4j_val_strdup(cbm_neo4j_row_col(e, 0), ""); + out->node_labels[i].count = (int)cbm_neo4j_val_int(cbm_neo4j_row_col(e, 1), 0); + } + out->node_label_count = (int)nl; + for (size_t i = 0; i < nt; i++) { + yyjson_val *e = yyjson_arr_get(types, i); + out->edge_types[i].type = cbm_neo4j_val_strdup(cbm_neo4j_row_col(e, 0), ""); + out->edge_types[i].count = (int)cbm_neo4j_val_int(cbm_neo4j_row_col(e, 1), 0); + } + out->edge_type_count = (int)nt; + if (with_props) { + yyjson_val *label_keys = cbm_neo4j_result_data(&r, 2); + yyjson_val *type_keys = cbm_neo4j_result_data(&r, 3); + for (size_t i = 0; i < nl; i++) { + yyjson_val *keys = NULL; + size_t idx; + size_t max; + yyjson_val *e; + yyjson_arr_foreach(label_keys, idx, max, e) { + const char *l = yyjson_get_str(cbm_neo4j_row_col(e, 0)); + if (l && strcmp(l, out->node_labels[i].label) == 0) { + keys = cbm_neo4j_row_col(e, 1); + break; + } + } + out->node_labels[i].properties = + schema_props(node_base, 5, keys, &out->node_labels[i].property_count); + } + for (size_t i = 0; i < nt; i++) { + yyjson_val *keys = NULL; + size_t idx; + size_t max; + yyjson_val *e; + yyjson_arr_foreach(type_keys, idx, max, e) { + const char *t = yyjson_get_str(cbm_neo4j_row_col(e, 0)); + if (t && strcmp(t, out->edge_types[i].type) == 0) { + keys = cbm_neo4j_row_col(e, 1); + break; + } + } + out->edge_types[i].properties = + schema_props(edge_base, 2, keys, &out->edge_types[i].property_count); + } + } + cbm_neo4j_result_free(&r); + return CBM_STORE_OK; +} + +static uint64_t op_queries(void *ctx) { + return ((n4s_ctx_t *)ctx)->queries; +} + +static const cbm_store_remote_ops_t k_neo4j_ops = { + .close = op_close, + .error = op_error, + .find_node_by_id = op_find_node_by_id, + .find_node_by_qn = op_find_node_by_qn, + .find_nodes = op_find_nodes, + .find_edges = op_find_edges, + .find_edges_by_type = op_find_edges_by_type, + .count = op_count, + .degrees = op_degrees, + .bfs = op_bfs, + .search = op_search, + .neighbor_names = op_neighbor_names, + .list_files = op_list_files, + .schema = op_schema, + .queries = op_queries, +}; + +/* ── Attach ─────────────────────────────────────────────────────── */ + +/* Connect to the project's Neo4j graph and read its published generation. */ +static n4s_ctx_t *ctx_open(const char *db_path, const char *project, char *err, size_t err_sz, + int *status) { + cbm_neo4j_config_t cfg; + char synced[N4S_GEN_MAX]; + char marker_db[4096]; + if (cbm_neo4j_sidecar_locate(db_path, project, marker_db, sizeof(marker_db)) != 0) { + *status = CBM_NEO4J_ATTACH_NONE; + return NULL; + } + int rc = cbm_neo4j_sidecar_read(marker_db, &cfg, synced, sizeof(synced), err, err_sz); + if (rc == 1) { + *status = CBM_NEO4J_ATTACH_NONE; + return NULL; + } + *status = CBM_NEO4J_ATTACH_FALLBACK; + if (rc != 0) { + return NULL; + } + if (!cfg.password[0]) { + /* Don't spend a failed login (Neo4j rate-limits them) on a request + * that cannot authenticate. */ + (void)snprintf(err, err_sz, + "no Neo4j password for %s: set NEO4J_PASSWORD in the MCP server's " + "environment, or re-run index_repository with --graph-password", + cfg.uri); + return NULL; + } + n4s_ctx_t *x = calloc(1, sizeof(*x)); + if (!x) { + cbm_neo4j_config_wipe(&cfg); + return NULL; + } + x->c = cbm_neo4j_connect(&cfg, err, err_sz); + cbm_neo4j_config_wipe(&cfg); + if (!x->c) { + free(x); + return NULL; + } + (void)snprintf(x->db, sizeof(x->db), "%s", project); + cbm_neo4j_batch_t b; + cbm_neo4j_result_t r; + if (cbm_neo4j_batch_init(&b) != 0) { + op_close(x); + return NULL; + } + yyjson_mut_val *params = + cbm_neo4j_batch_add(&b, "MATCH (p:CbmProject {name: $db}) RETURN p.gen, p.generation"); + yyjson_mut_obj_add_strcpy(b.doc, params, "db", project); + int run_rc = cbm_neo4j_batch_run(x->c, &b, &r); + cbm_neo4j_batch_free(&b); + if (run_rc != 0) { + (void)snprintf(err, err_sz, "%s", cbm_neo4j_error(x->c)); + op_close(x); + return NULL; + } + yyjson_val *row = yyjson_arr_get_first(cbm_neo4j_result_data(&r, 0)); + if (!row) { + cbm_neo4j_result_free(&r); + (void)snprintf(err, err_sz, "project '%s' has not been synced to Neo4j yet", project); + op_close(x); + return NULL; + } + x->gen = cbm_neo4j_val_int(cbm_neo4j_row_col(row, 0), 0); + char *remote_generation = cbm_neo4j_val_strdup(cbm_neo4j_row_col(row, 1), ""); + cbm_neo4j_result_free(&r); + + /* The local SQLite generation must be the one Neo4j mirrors; otherwise + * ids from the two would not agree. */ + char local[N4S_GEN_MAX] = ""; + cbm_store_t *probe = cbm_store_open_path_query(db_path); + if (probe) { + (void)cbm_store_generation(probe, local, sizeof(local)); + cbm_store_close(probe); + } + bool same = remote_generation && local[0] && strcmp(remote_generation, local) == 0; + free(remote_generation); + if (!same) { + (void)snprintf(err, err_sz, + "Neo4j holds an older generation of '%s' than the local index (last sync " + "failed?); re-run index_repository with graph_db to resync", + project); + op_close(x); + return NULL; + } + *status = CBM_NEO4J_ATTACHED; + return x; +} + +cbm_neo4j_attach_t cbm_neo4j_store_attach(cbm_store_t *s, const char *db_path, const char *project, + char *err, size_t err_sz) { + if (err && err_sz) { + err[0] = '\0'; + } + if (!s || !db_path || !project) { + return CBM_NEO4J_ATTACH_NONE; + } + int status = CBM_NEO4J_ATTACH_NONE; + n4s_ctx_t *x = ctx_open(db_path, project, err, err_sz, &status); + if (!x) { + if (status == CBM_NEO4J_ATTACH_FALLBACK) { + cbm_log_warn("neo4j.attach", "project", project, "result", "sqlite_fallback", "err", + err ? err : ""); + } + return (cbm_neo4j_attach_t)status; + } + cbm_store_attach_remote(s, &k_neo4j_ops, x); + return CBM_NEO4J_ATTACHED; +} + +/* ── Native Cypher ──────────────────────────────────────────────── */ + +static char *n4s_strdup(const char *s) { + size_t n = strlen(s); + char *d = malloc(n + 1); + if (d) { + memcpy(d, s, n + 1); + } + return d; +} + +static void native_fail(cbm_cypher_result_t *out, const char *msg) { + free(out->error); + out->error = n4s_strdup(msg ? msg : "Neo4j query failed"); +} + +/* query_graph is a read tool: refuse Cypher that could write. Keywords are + * matched as whole words outside string literals, comments and backticks. */ +static const char *native_write_keyword(const char *q) { + static const char *const kw[] = {"CREATE", "MERGE", "DELETE", "DETACH", "SET", "REMOVE", + "DROP", "FOREACH", "LOAD", "CALL", NULL}; + for (const char *p = q; *p;) { + if (*p == '\'' || *p == '"' || *p == '`') { + char quote = *p++; + while (*p && *p != quote) { + p += (*p == '\\' && p[1]) ? 2 : 1; + } + p += *p ? 1 : 0; + continue; + } + if (p[0] == '/' && p[1] == '/') { + while (*p && *p != '\n') { + p++; + } + continue; + } + if (isalpha((unsigned char)*p) || *p == '_') { + const char *w = p; + while (isalnum((unsigned char)*p) || *p == '_') { + p++; + } + size_t len = (size_t)(p - w); + for (int i = 0; kw[i]; i++) { + if (strlen(kw[i]) == len) { + size_t k = 0; + while (k < len && toupper((unsigned char)w[k]) == kw[i][k]) { + k++; + } + if (k == len) { + return kw[i]; + } + } + } + continue; + } + p++; + } + return NULL; +} + +int cbm_neo4j_native_query(const char *db_path, const char *project, const char *query, + int max_rows, cbm_cypher_result_t *out) { + memset(out, 0, sizeof(*out)); + const char *kw = native_write_keyword(query); + if (kw) { + char msg[160]; + (void)snprintf(msg, sizeof(msg), "native query_graph is read-only: '%s' is not allowed", + kw); + native_fail(out, msg); + return -1; + } + char err[CBM_NEO4J_ERR_MAX] = ""; + int status = 0; + n4s_ctx_t *x = ctx_open(db_path, project, err, sizeof(err), &status); + if (!x) { + native_fail(out, status == CBM_NEO4J_ATTACH_NONE + ? "project is not Neo4j-backed (index it with graph_db first)" + : err); + return -1; + } + n4s_q_t q; + if (q_begin(x, &q, query) != 0) { + op_close(x); + native_fail(out, "out of memory"); + return -1; + } + q_str(&q, "project", project); + cbm_neo4j_result_t r; + if (q_run(x, &q, &r) != CBM_STORE_OK) { + native_fail(out, x->err); + op_close(x); + return -1; + } + yyjson_val *res0 = yyjson_arr_get_first(r.results); + yyjson_val *cols = yyjson_obj_get(res0, "columns"); + yyjson_val *data = yyjson_obj_get(res0, "data"); + int ncols = (int)yyjson_arr_size(cols); + int nrows = (int)yyjson_arr_size(data); + int cap = max_rows > 0 ? max_rows : 100000; + if (nrows > cap) { + nrows = cap; + out->truncated = true; + } + out->col_count = ncols; + out->columns = calloc(ncols > 0 ? (size_t)ncols : 1, sizeof(char *)); + out->rows = calloc(nrows > 0 ? (size_t)nrows : 1, sizeof(char **)); + if (!out->columns || !out->rows) { + cbm_neo4j_result_free(&r); + op_close(x); + native_fail(out, "out of memory"); + return -1; + } + for (int j = 0; j < ncols; j++) { + out->columns[j] = cbm_neo4j_val_strdup(yyjson_arr_get(cols, (size_t)j), ""); + } + for (int i = 0; i < nrows; i++) { + yyjson_val *e = yyjson_arr_get(data, (size_t)i); + const char **row = calloc(ncols > 0 ? (size_t)ncols : 1, sizeof(char *)); + if (!row) { + break; + } + for (int j = 0; j < ncols; j++) { + yyjson_val *v = cbm_neo4j_row_col(e, (size_t)j); + row[j] = cbm_neo4j_val_strdup(v, yyjson_is_null(v) ? "null" : ""); + } + out->rows[i] = row; + out->row_count = i + 1; + } + cbm_neo4j_result_free(&r); + op_close(x); + return 0; +} diff --git a/src/neo4j/neo4j_store.h b/src/neo4j/neo4j_store.h new file mode 100644 index 0000000000..4c5971c83c --- /dev/null +++ b/src/neo4j/neo4j_store.h @@ -0,0 +1,34 @@ +/* + * neo4j_store.h — Answer a store's graph reads from Neo4j. + */ +#ifndef CBM_NEO4J_STORE_H +#define CBM_NEO4J_STORE_H + +#include "cypher/cypher.h" +#include "store/store.h" + +#include + +typedef enum { + CBM_NEO4J_ATTACH_NONE = 0, /* project is plain SQLite */ + CBM_NEO4J_ATTACHED = 1, /* graph reads now go to Neo4j */ + CBM_NEO4J_ATTACH_FALLBACK = + 2, /* Neo4j-backed, but unusable now: SQLite serves (err says why) */ +} cbm_neo4j_attach_t; + +/* If the project database at db_path is marked Neo4j-backed, connect and + * route s's graph reads to the Neo4j generation that mirrors exactly this + * SQLite generation. When Neo4j is unreachable, rejects the credentials, or + * holds a different generation (a failed sync), the store keeps reading the + * identical local copy and err explains the fallback. */ +cbm_neo4j_attach_t cbm_neo4j_store_attach(cbm_store_t *s, const char *db_path, const char *project, + char *err, size_t err_sz); + +/* Run a Cypher query natively in Neo4j for a Neo4j-backed project, with + * $project bound to the project name and $db / $gen bound to its current + * graph. Results are converted to strings like cbm_cypher_execute's. + * Returns 0, or -1 with out->error set. */ +int cbm_neo4j_native_query(const char *db_path, const char *project, const char *query, + int max_rows, cbm_cypher_result_t *out); + +#endif /* CBM_NEO4J_STORE_H */ diff --git a/src/neo4j/neo4j_sync.c b/src/neo4j/neo4j_sync.c new file mode 100644 index 0000000000..9a1c4807af --- /dev/null +++ b/src/neo4j/neo4j_sync.c @@ -0,0 +1,747 @@ +/* + * neo4j_sync.c — Mirror a published project database into Neo4j. + */ +#include "neo4j/neo4j_sync.h" + +#include "foundation/compat_fs.h" +#include "foundation/log.h" +#include "store/store.h" + +#include "sqlite3.h" + +#include +#include +#include + +#define N4_SYNC_BATCH_ROWS 2000 +#define N4_DELETE_BATCH 10000 +#define N4_SIDECAR_SUFFIX ".graph.json" +#define N4_SIDECAR_MAX (64 * 1024) +#define N4_GEN_MAX 128 + +/* Columns written explicitly; a flattened property never overwrites them. */ +static const char *const k_reserved_node_keys[] = { + "db", "gen", "project", "id", "label", + "name", "qualified_name", "file_path", "start_line", "end_line", + "properties_json", "prop_keys", NULL}; +static const char *const k_reserved_edge_keys[] = {"id", "type", "properties_json", "prop_keys", + NULL}; + +/* Layout version of a synced graph (CbmProject.sync_format). Bump when the + * stored shape changes; an older graph is re-uploaded on the next sync even + * if the SQLite generation is unchanged. 2: prop_keys on nodes and edges. */ +#define N4_SYNC_FORMAT 2 + +static void sync_err(char *err, size_t err_sz, const char *what, const char *detail) { + if (err && err_sz) { + (void)snprintf(err, err_sz, "%s%s%s", what, detail ? ": " : "", detail ? detail : ""); + } +} + +/* ── Sidecar marker ─────────────────────────────────────────────── */ + +int cbm_neo4j_sidecar_path(const char *db_path, char *out, size_t out_sz) { + if (!db_path || !*db_path) { + return -1; + } + int n = snprintf(out, out_sz, "%s%s", db_path, N4_SIDECAR_SUFFIX); + return (n > 0 && (size_t)n < out_sz) ? 0 : -1; +} + +int cbm_neo4j_sidecar_read(const char *db_path, cbm_neo4j_config_t *cfg, char *synced_generation, + size_t synced_sz, char *err, size_t err_sz) { + char path[4096]; + if (synced_generation && synced_sz) { + synced_generation[0] = '\0'; + } + if (cbm_neo4j_sidecar_path(db_path, path, sizeof(path)) != 0) { + return 1; + } + FILE *f = cbm_fopen(path, "rb"); + if (!f) { + return 1; + } + char *buf = malloc(N4_SIDECAR_MAX + 1); + if (!buf) { + (void)fclose(f); + return -1; + } + size_t n = fread(buf, 1, N4_SIDECAR_MAX, f); + (void)fclose(f); + buf[n] = '\0'; + int rc = cbm_neo4j_config_from_meta(buf, cfg, err, err_sz); + if (rc == 0 && synced_generation && synced_sz) { + yyjson_doc *d = yyjson_read(buf, n, 0); + const char *g = yyjson_get_str(yyjson_obj_get(yyjson_doc_get_root(d), "synced_generation")); + (void)snprintf(synced_generation, synced_sz, "%s", g ? g : ""); + yyjson_doc_free(d); + } + free(buf); + return rc == 0 ? 0 : -1; +} + +int cbm_neo4j_sidecar_locate(const char *db_path, const char *project, char *out, + size_t out_sz) { + char path[4096]; + if (cbm_neo4j_sidecar_path(db_path, path, sizeof(path)) == 0) { + FILE *f = cbm_fopen(path, "rb"); + if (f) { + (void)fclose(f); + (void)snprintf(out, out_sz, "%s", db_path); + return 0; + } + } + if (!project || !project[0]) { + return 1; + } + /* The database was found under another file name (the #704 internal-name + * scan): its marker still carries the project's canonical name. */ + size_t dir_len = 0; + for (const char *p = db_path; *p; p++) { + if (*p == '/' || *p == '\\') { + dir_len = (size_t)(p - db_path) + 1; + } + } + char canonical[4096]; + int n = snprintf(canonical, sizeof(canonical), "%.*s%s.db", (int)dir_len, db_path, project); + if (n <= 0 || (size_t)n >= sizeof(canonical) || + cbm_neo4j_sidecar_path(canonical, path, sizeof(path)) != 0) { + return 1; + } + FILE *f = cbm_fopen(path, "rb"); + if (!f) { + return 1; + } + (void)fclose(f); + (void)snprintf(out, out_sz, "%s", canonical); + return 0; +} + +int cbm_neo4j_sidecar_field(const char *db_path, const char *key, char *out, size_t out_sz) { + char path[4096]; + out[0] = '\0'; + if (cbm_neo4j_sidecar_path(db_path, path, sizeof(path)) != 0) { + return -1; + } + FILE *f = cbm_fopen(path, "rb"); + if (!f) { + return -1; + } + char *buf = malloc(N4_SIDECAR_MAX + 1); + if (!buf) { + (void)fclose(f); + return -1; + } + size_t n = fread(buf, 1, N4_SIDECAR_MAX, f); + (void)fclose(f); + yyjson_doc *d = yyjson_read(buf, n, 0); + free(buf); + const char *v = yyjson_get_str(yyjson_obj_get(yyjson_doc_get_root(d), key)); + (void)snprintf(out, out_sz, "%s", v ? v : ""); + yyjson_doc_free(d); + return 0; +} + +int cbm_neo4j_sidecar_write(const char *db_path, const cbm_neo4j_config_t *cfg, + const char *synced_generation) { + return cbm_neo4j_sidecar_write_status(db_path, cfg, synced_generation, NULL); +} + +int cbm_neo4j_sidecar_write_status(const char *db_path, const cbm_neo4j_config_t *cfg, + const char *synced_generation, const char *last_sync_error) { + char path[4096]; + char tmp[4096 + 8]; + if (cbm_neo4j_sidecar_path(db_path, path, sizeof(path)) != 0) { + return -1; + } + (void)snprintf(tmp, sizeof(tmp), "%s.tmp", path); + char *meta = cbm_neo4j_config_to_meta(cfg); + if (!meta) { + return -1; + } + yyjson_doc *src = yyjson_read(meta, strlen(meta), 0); + free(meta); + yyjson_mut_doc *d = yyjson_doc_mut_copy(src, NULL); + yyjson_doc_free(src); + if (!d) { + return -1; + } + yyjson_mut_obj_add_strcpy(d, yyjson_mut_doc_get_root(d), "synced_generation", + synced_generation ? synced_generation : ""); + if (last_sync_error && last_sync_error[0]) { + yyjson_mut_obj_add_strcpy(d, yyjson_mut_doc_get_root(d), "last_sync_error", + last_sync_error); + } + size_t len = 0; + char *json = yyjson_mut_write(d, YYJSON_WRITE_PRETTY, &len); + yyjson_mut_doc_free(d); + if (!json) { + return -1; + } + FILE *f = cbm_fopen(tmp, "wb"); + if (!f) { + free(json); + return -1; + } + bool ok = fwrite(json, 1, len, f) == len; + ok = (fclose(f) == 0) && ok; + free(json); + if (!ok || cbm_rename_replace(tmp, path) != 0) { + (void)cbm_unlink(tmp); + return -1; + } + return 0; +} + +int cbm_neo4j_sidecar_remove(const char *db_path) { + char path[4096]; + if (cbm_neo4j_sidecar_path(db_path, path, sizeof(path)) != 0) { + return -1; + } + (void)cbm_unlink(path); + return 0; +} + +/* ── Statement helpers ──────────────────────────────────────────── */ + +static int run_simple(cbm_neo4j_t *c, const char *cypher, const char *db, int64_t gen, + cbm_neo4j_result_t *out) { + cbm_neo4j_batch_t b; + if (cbm_neo4j_batch_init(&b) != 0) { + return -1; + } + yyjson_mut_val *params = cbm_neo4j_batch_add(&b, cypher); + if (db) { + yyjson_mut_obj_add_strcpy(b.doc, params, "db", db); + } + yyjson_mut_obj_add_int(b.doc, params, "gen", gen); + int rc = cbm_neo4j_batch_run(c, &b, out); + cbm_neo4j_batch_free(&b); + return rc; +} + +int cbm_neo4j_ensure_schema(cbm_neo4j_t *c) { + static const char *const stmts[] = { + "CREATE CONSTRAINT cbm_project_name IF NOT EXISTS FOR (p:CbmProject) REQUIRE p.name IS " + "UNIQUE", + "CREATE INDEX cbm_node_db IF NOT EXISTS FOR (n:CbmNode) ON (n.db)", + "CREATE INDEX cbm_node_id IF NOT EXISTS FOR (n:CbmNode) ON (n.db, n.gen, n.id)", + "CREATE INDEX cbm_node_qn IF NOT EXISTS FOR (n:CbmNode) ON (n.db, n.gen, n.qualified_name)", + "CREATE INDEX cbm_node_name IF NOT EXISTS FOR (n:CbmNode) ON (n.db, n.gen, n.name)", + "CREATE INDEX cbm_node_file IF NOT EXISTS FOR (n:CbmNode) ON (n.db, n.gen, n.file_path)", + "CREATE INDEX cbm_node_label IF NOT EXISTS FOR (n:CbmNode) ON (n.db, n.gen, n.label)", + NULL}; + /* Schema statements cannot share a transaction with each other in every + * Neo4j version, so send them one per request. */ + for (int i = 0; stmts[i]; i++) { + cbm_neo4j_batch_t b; + if (cbm_neo4j_batch_init(&b) != 0) { + return -1; + } + (void)cbm_neo4j_batch_add(&b, stmts[i]); + int rc = cbm_neo4j_batch_run(c, &b, NULL); + cbm_neo4j_batch_free(&b); + if (rc != 0) { + return -1; + } + } + /* Best effort: wait for freshly created indexes before bulk MATCHes. */ + cbm_neo4j_batch_t b; + if (cbm_neo4j_batch_init(&b) == 0) { + (void)cbm_neo4j_batch_add(&b, "CALL db.awaitIndexes(300)"); + (void)cbm_neo4j_batch_run(c, &b, NULL); + cbm_neo4j_batch_free(&b); + } + return 0; +} + +static bool key_reserved(const char *k, const char *const *reserved) { + for (int i = 0; reserved[i]; i++) { + if (strcmp(k, reserved[i]) == 0) { + return true; + } + } + return false; +} + +/* Copy top-level scalar members of properties_json into obj. Arrays of + * scalars are kept too (Neo4j stores homogeneous lists); nested objects stay + * only inside properties_json. prop_keys lists every top-level key -- the + * set SQLite's json_each reports for get_graph_schema -- flattened or not. */ +static void flatten_props(yyjson_mut_doc *d, yyjson_mut_val *obj, const char *json, + const char *const *reserved) { + yyjson_mut_val *prop_keys = yyjson_mut_arr(d); + yyjson_mut_obj_add_val(d, obj, "prop_keys", prop_keys); + if (!json || !*json || strcmp(json, "{}") == 0) { + return; + } + yyjson_doc *pd = yyjson_read(json, strlen(json), 0); + yyjson_val *root = pd ? yyjson_doc_get_root(pd) : NULL; + if (!yyjson_is_obj(root)) { + yyjson_doc_free(pd); + return; + } + size_t idx; + size_t max; + yyjson_val *k; + yyjson_val *v; + yyjson_obj_foreach(root, idx, max, k, v) { + const char *key = yyjson_get_str(k); + if (key) { + yyjson_mut_arr_add_strcpy(d, prop_keys, key); + } + if (!key || key_reserved(key, reserved)) { + continue; + } + if (yyjson_is_str(v) || yyjson_is_num(v) || yyjson_is_bool(v)) { + yyjson_mut_obj_add(obj, yyjson_mut_strcpy(d, key), yyjson_val_mut_copy(d, v)); + } else if (yyjson_is_arr(v) && yyjson_arr_size(v) > 0) { + /* Only homogeneous string or number lists are valid Neo4j values. */ + yyjson_val *first = yyjson_arr_get_first(v); + bool all_str = true; + bool all_num = true; + size_t ai; + size_t amax; + yyjson_val *av; + yyjson_arr_foreach(v, ai, amax, av) { + all_str = all_str && yyjson_is_str(av); + all_num = all_num && yyjson_is_num(av); + } + if ((all_str || all_num) && first) { + yyjson_mut_obj_add(obj, yyjson_mut_strcpy(d, key), yyjson_val_mut_copy(d, v)); + } + } + } + yyjson_doc_free(pd); +} + +static const char *col_text(sqlite3_stmt *st, int i) { + const char *s = (const char *)sqlite3_column_text(st, i); + return s ? s : ""; +} + +/* ── Node / edge streaming ──────────────────────────────────────── */ + +typedef struct { + cbm_neo4j_t *c; + const char *db; + int64_t gen; + cbm_neo4j_batch_t batch; + yyjson_mut_val *rows; + int row_count; + char group[256]; /* label or edge type of the open batch */ + bool is_edge; + int64_t total; +} n4_stream_t; + +static int stream_open_batch(n4_stream_t *s, const char *group) { + if (cbm_neo4j_batch_init(&s->batch) != 0) { + return -1; + } + (void)snprintf(s->group, sizeof(s->group), "%s", group); + char cypher[768]; + bool ident = cbm_neo4j_ident_ok(group); + if (!s->is_edge) { + if (ident) { + (void)snprintf(cypher, sizeof(cypher), + "UNWIND $rows AS r CREATE (n:CbmNode:`%s`) SET n = r", group); + } else { + (void)snprintf(cypher, sizeof(cypher), + "UNWIND $rows AS r CREATE (n:CbmNode) SET n = r"); + } + } else { + /* Types that are not plain identifiers land on CBM_EDGE with the real + * type kept in the `type` property. */ + (void)snprintf(cypher, sizeof(cypher), + "UNWIND $rows AS r " + "MATCH (a:CbmNode {db: $db, gen: $gen, id: r.s}) " + "MATCH (b:CbmNode {db: $db, gen: $gen, id: r.t}) " + "CREATE (a)-[e:`%s`]->(b) SET e = r.p", + ident ? group : "CBM_EDGE"); + } + yyjson_mut_val *params = cbm_neo4j_batch_add(&s->batch, cypher); + yyjson_mut_obj_add_strcpy(s->batch.doc, params, "db", s->db); + yyjson_mut_obj_add_int(s->batch.doc, params, "gen", s->gen); + s->rows = yyjson_mut_arr(s->batch.doc); + yyjson_mut_obj_add_val(s->batch.doc, params, "rows", s->rows); + s->row_count = 0; + return 0; +} + +static int stream_flush(n4_stream_t *s) { + if (!s->batch.doc) { + return 0; + } + int rc = s->row_count > 0 ? cbm_neo4j_batch_run(s->c, &s->batch, NULL) : 0; + s->total += s->row_count; + cbm_neo4j_batch_free(&s->batch); + s->rows = NULL; + s->row_count = 0; + s->group[0] = '\0'; + return rc; +} + +/* Returns the row object to fill, flushing/reopening on group change or when + * the batch is full. NULL on failure. */ +static yyjson_mut_val *stream_row(n4_stream_t *s, const char *group) { + if (s->batch.doc && (strcmp(s->group, group) != 0 || s->row_count >= N4_SYNC_BATCH_ROWS)) { + if (stream_flush(s) != 0) { + return NULL; + } + } + if (!s->batch.doc && stream_open_batch(s, group) != 0) { + return NULL; + } + yyjson_mut_val *row = yyjson_mut_obj(s->batch.doc); + yyjson_mut_arr_append(s->rows, row); + s->row_count++; + return row; +} + +static int sync_nodes(n4_stream_t *s, sqlite3 *db, char *err, size_t err_sz) { + sqlite3_stmt *st = NULL; + if (sqlite3_prepare_v2( + db, + "SELECT id, project, label, name, qualified_name, file_path, start_line, " + "end_line, properties FROM nodes ORDER BY label, id", + -1, &st, NULL) != SQLITE_OK) { + sync_err(err, err_sz, "read nodes", sqlite3_errmsg(db)); + return -1; + } + int rc = 0; + while (sqlite3_step(st) == SQLITE_ROW) { + const char *label = col_text(st, 2); + yyjson_mut_val *row = stream_row(s, label); + if (!row) { + sync_err(err, err_sz, "write nodes to Neo4j", cbm_neo4j_error(s->c)); + rc = -1; + break; + } + yyjson_mut_doc *d = s->batch.doc; + const char *props = col_text(st, 8); + /* Flattened properties first so explicit columns win on collision. */ + flatten_props(d, row, props, k_reserved_node_keys); + yyjson_mut_obj_add_strcpy(d, row, "db", s->db); + yyjson_mut_obj_add_int(d, row, "gen", s->gen); + yyjson_mut_obj_add_int(d, row, "id", sqlite3_column_int64(st, 0)); + yyjson_mut_obj_add_strcpy(d, row, "project", col_text(st, 1)); + yyjson_mut_obj_add_strcpy(d, row, "label", label); + yyjson_mut_obj_add_strcpy(d, row, "name", col_text(st, 3)); + yyjson_mut_obj_add_strcpy(d, row, "qualified_name", col_text(st, 4)); + yyjson_mut_obj_add_strcpy(d, row, "file_path", col_text(st, 5)); + yyjson_mut_obj_add_int(d, row, "start_line", sqlite3_column_int(st, 6)); + yyjson_mut_obj_add_int(d, row, "end_line", sqlite3_column_int(st, 7)); + yyjson_mut_obj_add_strcpy(d, row, "properties_json", props[0] ? props : "{}"); + } + sqlite3_finalize(st); + if (rc == 0 && stream_flush(s) != 0) { + sync_err(err, err_sz, "write nodes to Neo4j", cbm_neo4j_error(s->c)); + rc = -1; + } + cbm_neo4j_batch_free(&s->batch); + return rc; +} + +static int sync_edges(n4_stream_t *s, sqlite3 *db, char *err, size_t err_sz) { + sqlite3_stmt *st = NULL; + if (sqlite3_prepare_v2(db, + "SELECT id, source_id, target_id, type, properties FROM edges " + "ORDER BY type, id", + -1, &st, NULL) != SQLITE_OK) { + sync_err(err, err_sz, "read edges", sqlite3_errmsg(db)); + return -1; + } + int rc = 0; + while (sqlite3_step(st) == SQLITE_ROW) { + const char *type = col_text(st, 3); + yyjson_mut_val *row = stream_row(s, type); + if (!row) { + sync_err(err, err_sz, "write edges to Neo4j", cbm_neo4j_error(s->c)); + rc = -1; + break; + } + yyjson_mut_doc *d = s->batch.doc; + yyjson_mut_obj_add_int(d, row, "s", sqlite3_column_int64(st, 1)); + yyjson_mut_obj_add_int(d, row, "t", sqlite3_column_int64(st, 2)); + yyjson_mut_val *p = yyjson_mut_obj(d); + const char *props = col_text(st, 4); + flatten_props(d, p, props, k_reserved_edge_keys); + yyjson_mut_obj_add_int(d, p, "id", sqlite3_column_int64(st, 0)); + yyjson_mut_obj_add_strcpy(d, p, "type", type); + yyjson_mut_obj_add_strcpy(d, p, "properties_json", props[0] ? props : "{}"); + yyjson_mut_obj_add_val(d, row, "p", p); + } + sqlite3_finalize(st); + if (rc == 0 && stream_flush(s) != 0) { + sync_err(err, err_sz, "write edges to Neo4j", cbm_neo4j_error(s->c)); + rc = -1; + } + cbm_neo4j_batch_free(&s->batch); + return rc; +} + +/* Delete nodes of db matching `where` (a predicate on n using $gen) in + * bounded transactions. */ +static int delete_loop(cbm_neo4j_t *c, const char *db, const char *where, int64_t gen) { + char cypher[512]; + (void)snprintf(cypher, sizeof(cypher), + "MATCH (n:CbmNode {db: $db}) WHERE %s WITH n LIMIT %d DETACH DELETE n " + "RETURN count(*)", + where, N4_DELETE_BATCH); + for (;;) { + cbm_neo4j_result_t r; + if (run_simple(c, cypher, db, gen, &r) != 0) { + return -1; + } + yyjson_val *data = cbm_neo4j_result_data(&r, 0); + int64_t n = cbm_neo4j_val_int(cbm_neo4j_row_col(yyjson_arr_get_first(data), 0), 0); + cbm_neo4j_result_free(&r); + if (n <= 0) { + return 0; + } + } +} + +int cbm_neo4j_sync_db(cbm_neo4j_t *c, const char *db_path, const char *project, + char *generation_out, size_t generation_sz, char *err, size_t err_sz) { + if (cbm_neo4j_ensure_schema(c) != 0) { + sync_err(err, err_sz, "create Neo4j indexes", cbm_neo4j_error(c)); + return -1; + } + cbm_store_t *store = cbm_store_open_path_query(db_path); + if (!store) { + sync_err(err, err_sz, "open project database", db_path); + return -1; + } + char generation[N4_GEN_MAX] = ""; + (void)cbm_store_generation(store, generation, sizeof(generation)); + cbm_project_t proj = {0}; + bool have_proj = cbm_store_get_project(store, project, &proj) == CBM_STORE_OK; + + /* Current generation; a CbmProject is created on first sync. */ + cbm_neo4j_result_t r; + if (run_simple(c, "MERGE (p:CbmProject {name: $db}) RETURN coalesce(p.gen, 0)", project, 0, + &r) != 0) { + sync_err(err, err_sz, "read Neo4j project state", cbm_neo4j_error(c)); + goto fail; + } + /* The cleanup below deletes every generation except old_gen, so old_gen + * must be the published one: exactly one integer row, never a default. */ + yyjson_val *gen_rows = cbm_neo4j_result_data(&r, 0); + yyjson_val *gen_val = cbm_neo4j_row_col(yyjson_arr_get_first(gen_rows), 0); + if (yyjson_arr_size(gen_rows) != 1 || !yyjson_is_int(gen_val)) { + cbm_neo4j_result_free(&r); + sync_err(err, err_sz, "read Neo4j project state", "unexpected generation reply"); + goto fail; + } + int64_t old_gen = yyjson_get_sint(gen_val); + cbm_neo4j_result_free(&r); + int64_t new_gen = old_gen + 1; + + /* Leftovers of an interrupted earlier sync. */ + if (delete_loop(c, project, "n.gen <> $gen", old_gen) != 0) { + sync_err(err, err_sz, "clean up stale Neo4j generation", cbm_neo4j_error(c)); + goto fail; + } + + n4_stream_t s; + memset(&s, 0, sizeof(s)); + s.c = c; + s.db = project; + s.gen = new_gen; + sqlite3 *db = cbm_store_get_db(store); + if (sync_nodes(&s, db, err, err_sz) != 0) { + goto fail_partial; + } + int64_t node_total = s.total; + s.total = 0; + s.is_edge = true; + if (sync_edges(&s, db, err, err_sz) != 0) { + goto fail_partial; + } + int64_t edge_total = s.total; + + /* Flip readers to the new generation. */ + cbm_neo4j_batch_t b; + if (cbm_neo4j_batch_init(&b) != 0) { + goto fail_partial; + } + yyjson_mut_val *params = + cbm_neo4j_batch_add(&b, "MATCH (p:CbmProject {name: $db}) SET p.gen = $gen, " + "p.generation = $generation, p.root_path = $root, " + "p.indexed_at = $indexed_at, p.node_count = $nodes, " + "p.edge_count = $edges, p.sync_format = $format"); + yyjson_mut_obj_add_int(b.doc, params, "format", N4_SYNC_FORMAT); + yyjson_mut_obj_add_strcpy(b.doc, params, "db", project); + yyjson_mut_obj_add_int(b.doc, params, "gen", new_gen); + yyjson_mut_obj_add_strcpy(b.doc, params, "generation", generation); + yyjson_mut_obj_add_strcpy(b.doc, params, "root", + have_proj && proj.root_path ? proj.root_path : ""); + yyjson_mut_obj_add_strcpy(b.doc, params, "indexed_at", + have_proj && proj.indexed_at ? proj.indexed_at : ""); + yyjson_mut_obj_add_int(b.doc, params, "nodes", node_total); + yyjson_mut_obj_add_int(b.doc, params, "edges", edge_total); + int flip_rc = cbm_neo4j_batch_run(c, &b, NULL); + cbm_neo4j_batch_free(&b); + if (flip_rc != 0) { + sync_err(err, err_sz, "publish Neo4j generation", cbm_neo4j_error(c)); + goto fail_partial; + } + + /* Old generation is now unreachable for readers; reclaim it. A failure + * here leaves garbage that the next sync removes. */ + if (delete_loop(c, project, "n.gen <> $gen", new_gen) != 0) { + cbm_log_warn("neo4j.sync", "phase", "delete_old_generation", "err", cbm_neo4j_error(c)); + } + cbm_log_info("neo4j.sync", "project", project, "uri", cbm_neo4j_uri(c), "status", "ok"); + if (generation_out && generation_sz) { + (void)snprintf(generation_out, generation_sz, "%s", generation); + } + if (have_proj) { + cbm_project_free_fields(&proj); + } + cbm_store_close(store); + return 0; + +fail_partial: + /* Drop what was written for the new generation; readers never saw it. */ + (void)delete_loop(c, project, "n.gen = $gen", new_gen); +fail: + if (have_proj) { + cbm_project_free_fields(&proj); + } + cbm_store_close(store); + return -1; +} + +/* The SQLite generation Neo4j currently publishes for project ("" when the + * project is absent, or was synced in an older layout and must be re-sent). + * 0, or -1 on a transport/Cypher failure. */ +static int remote_generation(cbm_neo4j_t *c, const char *project, char *out, size_t out_sz) { + out[0] = '\0'; + cbm_neo4j_result_t r; + if (run_simple(c, + "MATCH (p:CbmProject {name: $db}) RETURN p.generation, coalesce(p.sync_format, 1)", + project, 0, &r) != 0) { + return -1; + } + yyjson_val *row = yyjson_arr_get_first(cbm_neo4j_result_data(&r, 0)); + yyjson_val *v = cbm_neo4j_row_col(row, 0); + bool current_format = cbm_neo4j_val_int(cbm_neo4j_row_col(row, 1), 1) >= N4_SYNC_FORMAT; + (void)snprintf(out, out_sz, "%s", current_format && yyjson_is_str(v) ? yyjson_get_str(v) : ""); + cbm_neo4j_result_free(&r); + return 0; +} + +int cbm_neo4j_drop_project(cbm_neo4j_t *c, const char *project, char *err, size_t err_sz) { + if (delete_loop(c, project, "true", 0) != 0 || + run_simple(c, "MATCH (p:CbmProject {name: $db}) DELETE p", project, 0, NULL) != 0) { + sync_err(err, err_sz, "delete project from Neo4j", cbm_neo4j_error(c)); + return -1; + } + return 0; +} + +int cbm_neo4j_after_publish(const char *db_path, const char *project, + const cbm_neo4j_config_t *explicit_cfg, bool disable, char *err, + size_t err_sz) { + if (err && err_sz) { + err[0] = '\0'; + } + if (disable) { + (void)cbm_neo4j_sidecar_remove(db_path); + return 0; + } + cbm_neo4j_config_t cfg; + char synced[N4_GEN_MAX] = ""; + bool explicit_req = explicit_cfg != NULL; + /* True when a marker for this same target already exists: a failed sync + * is then recorded in it, so it stays visible after this run. */ + bool marked_target = !explicit_req; + if (explicit_req) { + cfg = *explicit_cfg; + /* Same target as last time: keep the recorded generation so an + * unchanged re-index skips the upload. */ + cbm_neo4j_config_t prev; + char perr[256]; + if (cbm_neo4j_sidecar_read(db_path, &prev, synced, sizeof(synced), perr, sizeof(perr)) != + 0 || + strcmp(prev.uri, cfg.uri) != 0 || strcmp(prev.database, cfg.database) != 0) { + synced[0] = '\0'; + } else { + marked_target = true; + } + cbm_neo4j_config_wipe(&prev); + } else { + int rc = cbm_neo4j_sidecar_read(db_path, &cfg, synced, sizeof(synced), err, err_sz); + if (rc == 1) { + return 0; /* plain SQLite project */ + } + if (rc != 0) { + cbm_log_error("neo4j.sync", "project", project, "err", err); + return 0; + } + } + + /* Skip the upload when Neo4j already holds this exact generation. */ + char current[N4_GEN_MAX] = ""; + cbm_store_t *store = cbm_store_open_path_query(db_path); + if (store) { + (void)cbm_store_generation(store, current, sizeof(current)); + cbm_store_close(store); + } + + int result = 0; + cbm_neo4j_t *c = cbm_neo4j_connect(&cfg, err, err_sz); + if (!c) { + result = -1; + } else { + /* The marker only records what was last sent. Skip the upload only + * when Neo4j itself still publishes this exact generation: the graph + * may have been deleted or overwritten there since (a cleared + * database must be refilled by the next run, not reported current). + * The lookup also proves reachability and credentials. */ + bool up_to_date = false; + if (current[0] && strcmp(current, synced) == 0) { + char remote[N4_GEN_MAX] = ""; + if (remote_generation(c, project, remote, sizeof(remote)) != 0) { + sync_err(err, err_sz, "connect to Neo4j", cbm_neo4j_error(c)); + result = -1; + } else { + up_to_date = strcmp(remote, current) == 0; + } + } + if (result == 0 && !up_to_date) { + char done_gen[N4_GEN_MAX] = ""; + if (cbm_neo4j_sync_db(c, db_path, project, done_gen, sizeof(done_gen), err, err_sz) != + 0) { + result = -1; + } else { + (void)snprintf(synced, sizeof(synced), "%s", done_gen); + } + } + } + cbm_neo4j_close(c); + + if (result == 0) { + if (explicit_req) { + cbm_neo4j_config_remember(&cfg); + } + if (cbm_neo4j_sidecar_write(db_path, &cfg, synced) != 0) { + sync_err(err, err_sz, "write graph backend marker", db_path); + result = -1; + } + } else { + cbm_log_error("neo4j.sync", "project", project, "err", err ? err : ""); + /* Keep the last good synced_generation and record why this sync + * failed; the next successful sync rewrites the marker without it. */ + if (marked_target) { + (void)cbm_neo4j_sidecar_write_status(db_path, &cfg, synced, + err && err[0] ? err : "Neo4j sync failed"); + } + } + cbm_neo4j_config_wipe(&cfg); + /* An inherited (watcher / plain re-index) sync failure must not fail the + * SQLite index itself; it is logged and reported through err. */ + return explicit_req ? result : 0; +} diff --git a/src/neo4j/neo4j_sync.h b/src/neo4j/neo4j_sync.h new file mode 100644 index 0000000000..3f0dc79e96 --- /dev/null +++ b/src/neo4j/neo4j_sync.h @@ -0,0 +1,70 @@ +/* + * neo4j_sync.h — Mirror a published project database into Neo4j and record + * which projects are Neo4j-backed. + * + * Neo4j model (one "graph" per project database file, keyed by db = the + * project name the .db belongs to): + * (:CbmProject {name, gen, generation, root_path, indexed_at, node_count, edge_count}) + * (:CbmNode: