From ac0189a5d7ff1d74fd0f630eacfaa950d0da7935 Mon Sep 17 00:00:00 2001 From: aamijar Date: Fri, 23 Oct 2026 17:36:16 +0000 Subject: [PATCH 1/2] faiss_cpu_hnsw_cagra --- cpp/bench/ann/CMakeLists.txt | 9 ++ .../ann/src/faiss/faiss_cpu_benchmark.cpp | 17 +++- cpp/bench/ann/src/faiss/faiss_cpu_wrapper.h | 77 +++++++++++++++- fern/pages/cuvs_bench/param_tuning.md | 12 +++ fern/pages/cuvs_bench/running.md | 76 +++++++++++++++- .../cuvs_bench/cuvs_bench/backends/_utils.py | 30 ++++++- .../cuvs_bench/config/algorithms.yaml | 3 + .../config/algos/faiss_cpu_hnsw_cagra.yaml | 14 +++ .../cuvs_bench/orchestrator/config_loaders.py | 20 +++-- python/cuvs_bench/cuvs_bench/plot/__main__.py | 23 +++-- .../cuvs_bench/cuvs_bench/run/data_export.py | 8 +- python/cuvs_bench/cuvs_bench/run/run.py | 6 +- .../cuvs_bench/tests/test_data_export.py | 89 ++++++++++++++++++- .../cuvs_bench/cuvs_bench/tests/test_run.py | 10 +-- .../cuvs_bench/cuvs_bench/tests/test_utils.py | 64 ++++++++++++- 15 files changed, 419 insertions(+), 39 deletions(-) create mode 100644 python/cuvs_bench/cuvs_bench/config/algos/faiss_cpu_hnsw_cagra.yaml diff --git a/cpp/bench/ann/CMakeLists.txt b/cpp/bench/ann/CMakeLists.txt index 9e3b69fdbe..2f959da3c1 100644 --- a/cpp/bench/ann/CMakeLists.txt +++ b/cpp/bench/ann/CMakeLists.txt @@ -28,6 +28,9 @@ option(CUVS_ANN_BENCH_USE_FAISS_CPU_IVF_RABITQ ) option(CUVS_ANN_BENCH_USE_FAISS_CPU_IVF_SQ "Include faiss' cpu ivf sq algorithm in benchmark" ON) option(CUVS_ANN_BENCH_USE_FAISS_CPU_HNSW_FLAT "Include faiss' hnsw algorithm in benchmark" ON) +option(CUVS_ANN_BENCH_USE_FAISS_CPU_HNSW_CAGRA + "Include faiss' cpu search of hnsw indexes exported from cagra in benchmark" ON +) option(CUVS_ANN_BENCH_USE_CUVS_IVF_FLAT "Include cuVS ivf flat algorithm in benchmark" ON) option(CUVS_ANN_BENCH_USE_CUVS_IVF_SQ "Include cuVS ivf sq algorithm in benchmark" ON) option(CUVS_ANN_BENCH_USE_CUVS_IVF_PQ "Include cuVS ivf pq algorithm in benchmark" ON) @@ -351,6 +354,12 @@ if(CUVS_ANN_BENCH_USE_FAISS_CPU_HNSW_FLAT) ) endif() +if(CUVS_ANN_BENCH_USE_FAISS_CPU_HNSW_CAGRA) + ConfigureAnnBench( + NAME FAISS_CPU_HNSW_CAGRA PATH src/faiss/faiss_cpu_benchmark.cpp LINKS ${CUVS_FAISS_TARGETS} + ) +endif() + if(CUVS_ANN_BENCH_USE_FAISS_GPU_IVF_FLAT AND CUVS_FAISS_ENABLE_GPU) ConfigureAnnBench( NAME FAISS_GPU_IVF_FLAT PATH src/faiss/faiss_gpu_benchmark.cu LINKS ${CUVS_FAISS_TARGETS} diff --git a/cpp/bench/ann/src/faiss/faiss_cpu_benchmark.cpp b/cpp/bench/ann/src/faiss/faiss_cpu_benchmark.cpp index 1a99e56028..9451072d93 100644 --- a/cpp/bench/ann/src/faiss/faiss_cpu_benchmark.cpp +++ b/cpp/bench/ann/src/faiss/faiss_cpu_benchmark.cpp @@ -1,18 +1,16 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #include "../common/ann_types.hpp" #include "faiss_cpu_wrapper.h" -#include #include #include #include #include #include -#include namespace cuvs::bench { @@ -98,6 +96,13 @@ void parse_search_param(const nlohmann::json& conf, param.p = p; } +template +void parse_search_param(const nlohmann::json& conf, + typename cuvs::bench::faiss_cpu_hnsw_cagra::search_param& param) +{ + if (conf.contains("efSearch")) { param.p.efSearch = conf.at("efSearch"); } +} + template class Algo> auto make_algo(cuvs::bench::Metric metric, int dim, const nlohmann::json& conf) -> std::unique_ptr> @@ -129,6 +134,8 @@ auto create_algo(const std::string& algo_name, a = std::make_unique>(metric, dim); } else if (algo_name == "faiss_cpu_hnsw_flat") { a = make_algo(metric, dim, conf); + } else if (algo_name == "faiss_cpu_hnsw_cagra") { + a = std::make_unique>(metric, dim); } } @@ -155,6 +162,10 @@ auto create_search_param(const std::string& algo_name, const nlohmann::json& con auto param = std::make_unique::search_param>(); parse_search_param(conf, *param); return param; + } else if (algo_name == "faiss_cpu_hnsw_cagra") { + auto param = std::make_unique::search_param>(); + parse_search_param(conf, *param); + return param; } // else throw std::runtime_error("invalid algo: '" + algo_name + "'"); diff --git a/cpp/bench/ann/src/faiss/faiss_cpu_wrapper.h b/cpp/bench/ann/src/faiss/faiss_cpu_wrapper.h index 006f8703fa..078ac77837 100644 --- a/cpp/bench/ann/src/faiss/faiss_cpu_wrapper.h +++ b/cpp/bench/ann/src/faiss/faiss_cpu_wrapper.h @@ -21,6 +21,7 @@ #include #include #include +#include namespace { @@ -71,7 +72,7 @@ class faiss_cpu : public algo { static_assert(std::is_same_v, "faiss support only float type"); } - void build(const T* dataset, size_t nrow) final; + void build(const T* dataset, size_t nrow) override; void set_search_param(const search_param_base& param, const void* filter_bitset) override; @@ -185,7 +186,14 @@ template template void faiss_cpu::load_(const std::string& file) { - index_ = std::shared_ptr(dynamic_cast(faiss::read_index(file.c_str()))); + std::unique_ptr loaded{faiss::read_index(file.c_str())}; + auto* typed = dynamic_cast(loaded.get()); + if (typed == nullptr) { + throw std::runtime_error("'" + file + "' does not contain a " + typeid(Index).name() + + " (found " + typeid(*loaded).name() + ")"); + } + loaded.release(); + index_.reset(typed); } template @@ -267,8 +275,6 @@ class faiss_cpu_ivfrabitq : public faiss_cpu { } }; -// TODO(snanditale): Enable this in cmake -// ref: https://github.com/NVIDIA/raft/issues/1876 template class faiss_cpu_ivfsq : public faiss_cpu { public: @@ -396,4 +402,67 @@ class faiss_cpu_hnsw_flat : public faiss_cpu { std::shared_ptr search_params_; }; +/** + * Search-only wrapper for a faiss::IndexHNSWCagra exported by `faiss_gpu_cagra_hnsw`. + * + * The index is built and converted on a GPU host; this wrapper only loads and searches it, so it + * can run in a CPU-only build. + */ +template +class faiss_cpu_hnsw_cagra : public faiss_cpu { + public: + struct search_param : public faiss_cpu::search_param { + faiss::SearchParametersHNSW p; + }; + faiss_cpu_hnsw_cagra(Metric metric, int dim) + : faiss_cpu(metric, dim, typename faiss_cpu::build_param{}) + { + } + + void build(const T*, size_t) override + { + throw std::runtime_error( + "faiss_cpu_hnsw_cagra does not build indexes; build them with faiss_gpu_cagra_hnsw"); + } + + void set_search_param(const typename algo::search_param& param, + const void* filter_bitset) override + { + if (filter_bitset != nullptr) { throw std::runtime_error("Filtering is not supported yet."); } + auto sp = static_cast::search_param&>(param); + this->search_params_ = std::make_shared(sp.p); + } + + void save(const std::string& file) const override + { + this->template save_(file); + } + void load(const std::string& file) override { this->template load_(file); } + + std::unique_ptr> copy() override + { + return std::make_unique>(*this); // use copy constructor + } + + void search(const T* queries, + int batch_size, + int k, + algo_base::index_type* neighbors, + float* distances) const override + { + static_assert(sizeof(size_t) == sizeof(faiss::idx_t), + "sizes of size_t and faiss::idx_t are different"); + + this->index_->search(batch_size, + queries, + k, + distances, + reinterpret_cast(neighbors), + search_params_.get()); + } + + private: + std::shared_ptr search_params_; +}; + } // namespace cuvs::bench diff --git a/fern/pages/cuvs_bench/param_tuning.md b/fern/pages/cuvs_bench/param_tuning.md index a25b66378e..c048d48790 100644 --- a/fern/pages/cuvs_bench/param_tuning.md +++ b/fern/pages/cuvs_bench/param_tuning.md @@ -208,6 +208,18 @@ Use FAISS IVF-PQ index on CPU | `refine_ratio` | `search` | N | Positive number >=1 | 1 | `refine_ratio * k` nearest neighbors are queried from the index initially and an additional refinement step improves recall by selecting only the best `k` neighbors. | | `numThreads` | `search` | N | Positive integer >0 | 1 | Number of threads to use for queries. | +### faiss_cpu_hnsw_cagra + +Search, on the CPU, a FAISS `IndexHNSWCagra` that was built and exported by `faiss_gpu_cagra_hnsw`. This algorithm does not build indexes; its build parameters only identify which exported index to load and must match the `faiss_gpu_cagra_hnsw` build. See [Building on a GPU host and searching on a CPU-only host](/user-guide/benchmarking-guide/cu-vs-bench-tool/usage#building-on-a-gpu-host-and-searching-on-a-cpu-only-host). + +| Parameter | Type | Required | Data Type | Default | Description | +| --- | --- | --- | --- | --- | --- | +| `graph_degree` | `build` | Y | Positive integer >0 | | `graph_degree` of the `faiss_gpu_cagra_hnsw` build. | +| `intermediate_graph_degree` | `build` | Y | Positive integer >0 | | `intermediate_graph_degree` of the `faiss_gpu_cagra_hnsw` build. | +| `cagra_build_algo` | `build` | Y | [`IVF_PQ`, `NN_DESCENT`] | | `cagra_build_algo` of the `faiss_gpu_cagra_hnsw` build. | +| `base_level_only` | `build` | Y | Boolean | | `base_level_only` of the `faiss_gpu_cagra_hnsw` build. When `true`, the index has no upper HNSW levels and search starts from random entry points in the base level. | +| `efSearch` | `search` | N | Positive integer >0 | 16 | Size of the candidate list during search. Larger values improve recall but increase latency. | + ## HNSW ### cuvs_hnsw diff --git a/fern/pages/cuvs_bench/running.md b/fern/pages/cuvs_bench/running.md index 94606bb214..94da9b2c30 100644 --- a/fern/pages/cuvs_bench/running.md +++ b/fern/pages/cuvs_bench/running.md @@ -58,8 +58,8 @@ Create a custom YAML file with a `base` group to override the default benchmark | Library | Algorithms | | --- | --- | -| FAISS_GPU | `faiss_gpu_flat`, `faiss_gpu_ivf_flat`, `faiss_gpu_ivf_pq`, `faiss_gpu_cagra` | -| FAISS_CPU | `faiss_cpu_flat`, `faiss_cpu_ivf_flat`, `faiss_cpu_ivf_pq`, `faiss_cpu_hnsw_flat` | +| FAISS_GPU | `faiss_gpu_flat`, `faiss_gpu_ivf_flat`, `faiss_gpu_ivf_pq`, `faiss_gpu_cagra`, `faiss_gpu_cagra_hnsw` | +| FAISS_CPU | `faiss_cpu_flat`, `faiss_cpu_ivf_flat`, `faiss_cpu_ivf_pq`, `faiss_cpu_hnsw_flat`, `faiss_cpu_hnsw_cagra` | | GGNN | `ggnn` | | HNSWLIB | `hnswlib` | | DiskANN | `diskann_memory`, `diskann_ssd` | @@ -125,6 +125,78 @@ python -m cuvs_bench.plot --dataset deep-1B Use `python -m cuvs_bench.split_groundtruth --help` to see the full CLI help for the ground-truth split command. +## Building on a GPU host and searching on a CPU-only host + +Build a CAGRA index on a GPU host, export it as a Faiss `IndexHNSWCagra`, and measure search on a CPU-only host: + +| Host | Algorithm | Measures | +| --- | --- | --- | +| GPU | `faiss_gpu_cagra_hnsw` | Build time, including conversion to `IndexHNSWCagra` | +| CPU-only | `faiss_cpu_hnsw_cagra` | Recall, latency, and throughput of the exported indexes. Search only; it does not build. | + +The CPU-only host only needs the CPU package (`cuvs-bench-cpu`). + +### (1) Build on the GPU host + +```bash +export DATASET_PATH=/path/to/datasets + +python -m cuvs_bench.get_dataset --dataset sift-128-euclidean --dataset-path $DATASET_PATH +python -m cuvs_bench.run --dataset sift-128-euclidean --dataset-path $DATASET_PATH \ + --algorithms faiss_gpu_cagra_hnsw --groups base --search-mode latency \ + --build -k 10 --batch-size 10 +``` + +This writes one index file per build configuration to `index/` and the build times to `result/build/faiss_gpu_cagra_hnsw,base.csv`. `-k`, `--batch-size`, and `--search-mode` do not affect the build; they are passed only so `cuvs_bench.run` does not prompt for them. + +### (2) Copy the files to the CPU-only host + +Keep the same layout under the dataset directory. `base.fbin` is not needed, because each index file contains the vectors. + +```text +sift-128-euclidean/ +├── query.fbin +├── groundtruth.neighbors.ibin +├── index/ +└── result/build/ +``` + +### (3) Search on the CPU-only host + +```bash +export DATASET_PATH=/path/to/datasets + +OMP_NUM_THREADS=1 python -m cuvs_bench.run --dataset sift-128-euclidean --dataset-path $DATASET_PATH \ + --algorithms faiss_cpu_hnsw_cagra --groups base --search-mode latency \ + --search -k 10 --batch-size 10 +python -m cuvs_bench.plot --dataset sift-128-euclidean --dataset-path $DATASET_PATH \ + --algorithms faiss_cpu_hnsw_cagra -k 10 --batch-size 10 +``` + +Each row of `result/search/faiss_cpu_hnsw_cagra,base,k10,bs10,raw.csv` includes the build time and build parameters from the GPU host. + +- `OMP_NUM_THREADS` sets the search thread count. Without it, Faiss uses all cores. The thread count is not recorded in the results. +- Pass `cuvs_bench.plot` the same `-k` and `--batch-size` as the search. + +### Matching configurations on both hosts + +The CPU-only host finds each index by its name, which is built from the group and build parameters. Both default configurations use the same `base` build parameters. If you build with a custom `--configuration`, use a `faiss_cpu_hnsw_cagra` configuration with the same group and build parameters on the CPU-only host. Its search parameters can differ: + +```yaml +name: faiss_cpu_hnsw_cagra +constraints: + build: cuvs_bench.config.algos.constraints.cuvs_cagra_build +groups: + large: + build: + graph_degree: [64] + intermediate_graph_degree: [128] + cagra_build_algo: ["NN_DESCENT"] + base_level_only: [true] + search: + efSearch: [32, 64, 128, 256] +``` + ## Running with Docker containers Docker images can run the full workflow or open a shell for manual commands. See [Installation](/user-guide/benchmarking-guide/cu-vs-bench-tool/installation#docker) for image and tag guidance. diff --git a/python/cuvs_bench/cuvs_bench/backends/_utils.py b/python/cuvs_bench/cuvs_bench/backends/_utils.py index 250e2d2f50..fa5be90a3f 100644 --- a/python/cuvs_bench/cuvs_bench/backends/_utils.py +++ b/python/cuvs_bench/cuvs_bench/backends/_utils.py @@ -1,5 +1,5 @@ # -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # @@ -21,6 +21,7 @@ formats (.fbin, .i8bin, .u8bin). """ +import hashlib import itertools import os from typing import Any, Dict, List, Optional @@ -153,6 +154,33 @@ def expand_param_grid( ] +def get_index_name_prefix(algo: str) -> str: + """ + Return the prefix of the names of the indexes ``algo`` searches. + + The prefix is the name of the algorithm that built the indexes, which is + also the name its build results are stored under. ``faiss_cpu_hnsw_cagra`` + only searches indexes built by ``faiss_gpu_cagra_hnsw``; every other + algorithm builds its own indexes. + """ + if algo == "faiss_cpu_hnsw_cagra": + return "faiss_gpu_cagra_hnsw" + return algo + + +def get_index_filename(index_name: str) -> str: + """ + Return the file name an index named ``index_name`` is saved under. + + Names of 128 characters or more are replaced by their SHA-256 digest. The + digest is the same in every process, so separate build and search runs, + including runs on different hosts, resolve the same file. + """ + if len(index_name) < 128: + return index_name + return hashlib.sha256(index_name.encode()).hexdigest() + + def compute_recall( neighbors: np.ndarray, groundtruth: np.ndarray, k: int ) -> float: diff --git a/python/cuvs_bench/cuvs_bench/config/algorithms.yaml b/python/cuvs_bench/cuvs_bench/config/algorithms.yaml index 5f77c43623..f161cd6fad 100644 --- a/python/cuvs_bench/cuvs_bench/config/algorithms.yaml +++ b/python/cuvs_bench/cuvs_bench/config/algorithms.yaml @@ -34,6 +34,9 @@ faiss_cpu_ivf_sq: faiss_cpu_hnsw_flat: executable: FAISS_CPU_HNSW_FLAT_ANN_BENCH requires_gpu: false +faiss_cpu_hnsw_cagra: + executable: FAISS_CPU_HNSW_CAGRA_ANN_BENCH + requires_gpu: false cuvs_ivf_flat: executable: CUVS_IVF_FLAT_ANN_BENCH requires_gpu: true diff --git a/python/cuvs_bench/cuvs_bench/config/algos/faiss_cpu_hnsw_cagra.yaml b/python/cuvs_bench/cuvs_bench/config/algos/faiss_cpu_hnsw_cagra.yaml new file mode 100644 index 0000000000..91c6b78134 --- /dev/null +++ b/python/cuvs_bench/cuvs_bench/config/algos/faiss_cpu_hnsw_cagra.yaml @@ -0,0 +1,14 @@ +name: faiss_cpu_hnsw_cagra +constraints: + build: cuvs_bench.config.algos.constraints.cuvs_cagra_build +groups: + # Build parameters and constraints must match faiss_gpu_cagra_hnsw.yaml so the index names + # resolve to the files built by faiss_gpu_cagra_hnsw. + base: + build: + graph_degree: [32, 64] + intermediate_graph_degree: [64, 128] + cagra_build_algo: ["NN_DESCENT"] + base_level_only: [false, true] + search: + efSearch: [16, 32, 64, 128, 256, 512, 768, 1024] diff --git a/python/cuvs_bench/cuvs_bench/orchestrator/config_loaders.py b/python/cuvs_bench/cuvs_bench/orchestrator/config_loaders.py index e3af2fe58e..55973a65d8 100644 --- a/python/cuvs_bench/cuvs_bench/orchestrator/config_loaders.py +++ b/python/cuvs_bench/cuvs_bench/orchestrator/config_loaders.py @@ -1,5 +1,5 @@ # -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # @@ -20,7 +20,11 @@ import yaml -from ..backends._utils import expand_param_grid +from ..backends._utils import ( + expand_param_grid, + get_index_filename, + get_index_name_prefix, +) @dataclass @@ -839,11 +843,16 @@ def prepare_indexes( A list of index configurations. """ indexes = [] + name_prefix = get_index_name_prefix(algo) for build_param in all_build_params: index = {"algo": algo, "build_param": build_param} - index_name = f"{algo}_{group}" if group != "base" else f"{algo}" + index_name = ( + f"{name_prefix}_{group}" + if group != "base" + else f"{name_prefix}" + ) for name, val in build_param.items(): index_name += f".{name}{val}" @@ -861,12 +870,9 @@ def prepare_indexes( ): continue - index_filename = ( - index_name if len(index_name) < 128 else str(hash(index_name)) - ) index["name"] = index_name index["file"] = os.path.join( - dataset_path, dataset, "index", index_filename + dataset_path, dataset, "index", get_index_filename(index_name) ) # Validate and filter search params (skipped in tune mode diff --git a/python/cuvs_bench/cuvs_bench/plot/__main__.py b/python/cuvs_bench/cuvs_bench/plot/__main__.py index bea9afe6a2..db29771a0d 100644 --- a/python/cuvs_bench/cuvs_bench/plot/__main__.py +++ b/python/cuvs_bench/cuvs_bench/plot/__main__.py @@ -18,6 +18,8 @@ import numpy as np import pandas as pd +from ..backends._utils import get_index_name_prefix + mpl.use("Agg") metrics = { @@ -227,20 +229,24 @@ def mean_y(algo): xs = points[:, 2] ls = points[:, 0] idxs = points[:, 1] + build_algos = {name: get_index_name_prefix(name) for name in set(ls)} + + def build_time(i): + return build_results[(build_algos[ls[i]], idxs[i])][0][2] len_80, len_90, len_95, len_99 = 0, 0, 0, 0 for i in range(len(xs)): if xs[i] >= 0.80 and xs[i] < 0.90: - bt_80[pos] = bt_80[pos] + build_results[(ls[i], idxs[i])][0][2] + bt_80[pos] = bt_80[pos] + build_time(i) len_80 = len_80 + 1 elif xs[i] >= 0.9 and xs[i] < 0.95: - bt_90[pos] = bt_90[pos] + build_results[(ls[i], idxs[i])][0][2] + bt_90[pos] = bt_90[pos] + build_time(i) len_90 = len_90 + 1 elif xs[i] >= 0.95 and xs[i] < 0.99: - bt_95[pos] = bt_95[pos] + build_results[(ls[i], idxs[i])][0][2] + bt_95[pos] = bt_95[pos] + build_time(i) len_95 = len_95 + 1 elif xs[i] >= 0.99: - bt_99[pos] = bt_99[pos] + build_results[(ls[i], idxs[i])][0][2] + bt_99[pos] = bt_99[pos] + build_time(i) len_99 = len_99 + 1 if len_80 > 0: bt_80[pos] = bt_80[pos] / len_80 @@ -578,11 +584,16 @@ def main( args["x_start"], ) if build: + build_algorithms = [get_index_name_prefix(algo) for algo in algorithms] + build_algo_groups = [ + ".".join([get_index_name_prefix(algo), *group]) + for algo, *group in (ag.split(".") for ag in algo_groups) + ] build_results = load_all_results( os.path.join(args["dataset_path"], args["dataset"]), - algorithms, + build_algorithms, groups, - algo_groups, + build_algo_groups, k, batch_size, "build", diff --git a/python/cuvs_bench/cuvs_bench/run/data_export.py b/python/cuvs_bench/cuvs_bench/run/data_export.py index 50afa7a5ea..860e438887 100644 --- a/python/cuvs_bench/cuvs_bench/run/data_export.py +++ b/python/cuvs_bench/cuvs_bench/run/data_export.py @@ -10,6 +10,7 @@ import pandas as pd +from ..backends._utils import get_index_name_prefix from ..backends.base import BuildResult, SearchResult skip_build_cols = set( @@ -179,7 +180,7 @@ def _write_search_results( dataset, "result", "build", - f"{algorithm},{group}.csv", + f"{get_index_name_prefix(algorithm)},{group}.csv", ) if os.path.exists(build_file): build = pd.read_csv(build_file).drop_duplicates( @@ -344,7 +345,10 @@ def convert_json_to_csv_search(dataset, dataset_path): dataset, "result", "build", - f"{','.join(algo_name)}.csv", + ",".join( + (get_index_name_prefix(algo_name[0]),) + algo_name[1:] + ) + + ".csv", ) algo_name = clean_algo_name(algo_name) df["name"] = df["name"].str.split("/").str[0] diff --git a/python/cuvs_bench/cuvs_bench/run/run.py b/python/cuvs_bench/cuvs_bench/run/run.py index 37cc035a32..c21d344f57 100644 --- a/python/cuvs_bench/cuvs_bench/run/run.py +++ b/python/cuvs_bench/cuvs_bench/run/run.py @@ -11,6 +11,7 @@ import yaml +from ..backends._utils import get_index_filename from .runners import cuvs_bench_cpp @@ -440,12 +441,9 @@ def prepare_indexes( ): continue - index_filename = ( - index_name if len(index_name) < 128 else str(hash(index_name)) - ) index["name"] = index_name index["file"] = os.path.join( - dataset_path, dataset, "index", index_filename + dataset_path, dataset, "index", get_index_filename(index_name) ) index["search_params"] = validate_search_params( itertools.product(*search_param_lists), diff --git a/python/cuvs_bench/cuvs_bench/tests/test_data_export.py b/python/cuvs_bench/cuvs_bench/tests/test_data_export.py index 721d03c5ac..652340c377 100644 --- a/python/cuvs_bench/cuvs_bench/tests/test_data_export.py +++ b/python/cuvs_bench/cuvs_bench/tests/test_data_export.py @@ -3,6 +3,8 @@ # SPDX-License-Identifier: Apache-2.0 # +import json + import numpy as np import pandas as pd @@ -13,8 +15,91 @@ IndexConfig, ) from cuvs_bench.orchestrator.orchestrator import BenchmarkOrchestrator -from cuvs_bench.plot.__main__ import load_all_results -from cuvs_bench.run.data_export import write_results_to_csv +from cuvs_bench.plot.__main__ import create_plot_build, load_all_results +from cuvs_bench.backends._utils import get_index_name_prefix +from cuvs_bench.run.data_export import ( + convert_json_to_csv_search, + write_results_to_csv, +) + + +def test_get_index_name_prefix(): + assert ( + get_index_name_prefix("faiss_cpu_hnsw_cagra") == "faiss_gpu_cagra_hnsw" + ) + assert get_index_name_prefix("cuvs_cagra") == "cuvs_cagra" + + +def test_cpu_search_csv_includes_gpu_build_results(tmp_path): + dataset = "test-dataset" + index_name = "faiss_gpu_cagra_hnsw.graph_degree32" + result_dir = tmp_path / dataset / "result" + (result_dir / "build").mkdir(parents=True) + (result_dir / "search").mkdir(parents=True) + pd.DataFrame( + { + "algo_name": ["faiss_gpu_cagra_hnsw"], + "index_name": [index_name], + "time": [20.5], + "threads": [1], + "cpu_time": [90.0], + "graph_degree": [32], + } + ).to_csv( + result_dir / "build" / "faiss_gpu_cagra_hnsw,base.csv", index=False + ) + search_json = { + "benchmarks": [ + { + "name": f"{index_name}/0/process_time/real_time", + "Recall": 0.95, + "items_per_second": 1000.0, + "Latency": 0.001, + "efSearch": 64, + } + ] + } + with open( + result_dir / "search" / "faiss_cpu_hnsw_cagra,base,k10,bs10.json", "w" + ) as f: + json.dump(search_json, f) + + convert_json_to_csv_search(dataset, str(tmp_path)) + + raw = pd.read_csv( + result_dir / "search" / "faiss_cpu_hnsw_cagra,base,k10,bs10,raw.csv" + ) + assert raw.loc[0, "algo_name"] == "faiss_cpu_hnsw_cagra" + assert raw.loc[0, "index_name"] == index_name + assert raw.loc[0, "build time"] == 20.5 + assert raw.loc[0, "graph_degree"] == 32 + + +def test_cpu_build_plot_uses_gpu_build_results(tmp_path): + index_name = "faiss_gpu_cagra_hnsw.graph_degree32" + build_results = { + ("faiss_gpu_cagra_hnsw", index_name): [ + ["faiss_gpu_cagra_hnsw", index_name, 20.5] + ] + } + search_results = { + "faiss_cpu_hnsw_cagra": [ + ["faiss_cpu_hnsw_cagra", index_name, 0.95, 1000.0] + ] + } + fn_out = tmp_path / "build.png" + + create_plot_build( + build_results, + search_results, + {"faiss_cpu_hnsw_cagra": ("C0", "-", 1, "o")}, + str(fn_out), + "test-dataset", + 10, + 10, + ) + + assert fn_out.exists() def test_python_backend_csv_is_plot_compatible(tmp_path): diff --git a/python/cuvs_bench/cuvs_bench/tests/test_run.py b/python/cuvs_bench/cuvs_bench/tests/test_run.py index 93c70bc5de..f9ef888edd 100644 --- a/python/cuvs_bench/cuvs_bench/tests/test_run.py +++ b/python/cuvs_bench/cuvs_bench/tests/test_run.py @@ -1,5 +1,5 @@ # -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # @@ -9,6 +9,7 @@ from unittest.mock import MagicMock, mock_open, patch import pytest +from cuvs_bench.backends._utils import get_index_filename from cuvs_bench.run.run import ( find_executable, gather_algorithm_configs, @@ -179,13 +180,8 @@ def test_prepare_indexes_valid( for index in indexes: assert index["algo"] == algo - expected_filename = ( - index["name"] - if len(index["name"]) < 128 - else str(hash(index["name"])) - ) expected_file = os.path.join( - dataset_path, dataset, "index", expected_filename + dataset_path, dataset, "index", get_index_filename(index["name"]) ) assert index["file"] == expected_file # Verify that our dummy search parameters were set. diff --git a/python/cuvs_bench/cuvs_bench/tests/test_utils.py b/python/cuvs_bench/cuvs_bench/tests/test_utils.py index 01dd803a07..dbdbc96978 100644 --- a/python/cuvs_bench/cuvs_bench/tests/test_utils.py +++ b/python/cuvs_bench/cuvs_bench/tests/test_utils.py @@ -1,12 +1,15 @@ # -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 """ Unit tests for shared backend utilities and Dataset transparent loading. """ +import os import struct +import subprocess +import sys import numpy as np import pytest @@ -24,6 +27,7 @@ compute_recall, dtype_from_filename, expand_param_grid, + get_index_filename, load_vectors, ) from cuvs_bench.generate_groundtruth.utils import ( @@ -669,6 +673,27 @@ def test_get_dataset_configuration(self): assert result["name"] == "glove-100" assert result["dims"] == 100 + def test_prepare_indexes_uses_gpu_index_names(self, tmp_path): + """faiss_cpu_hnsw_cagra index names resolve to GPU-built files.""" + loader = CppGBenchConfigLoader() + indexes = loader.prepare_indexes( + [{"graph_degree": 32, "base_level_only": True}], + [{"efSearch": 64}], + "faiss_cpu_hnsw_cagra", + "base", + {"dataset": {"dims": 128}}, + {"faiss_cpu_hnsw_cagra": {"groups": {}, "constraints": {}}}, + str(tmp_path), + "test", + 10, + 10, + ) + name = "faiss_gpu_cagra_hnsw.graph_degree32.base_level_onlyTrue" + assert len(indexes) == 1 + assert indexes[0]["algo"] == "faiss_cpu_hnsw_cagra" + assert indexes[0]["name"] == name + assert indexes[0]["file"] == str(tmp_path / "test" / "index" / name) + def test_get_dataset_configuration_not_found(self): """Test that missing datasets raise ValueError.""" loader = CppGBenchConfigLoader() @@ -715,6 +740,43 @@ def test_three_params(self): assert {"a": 2, "b": 3, "c": 5} in result +class TestGetIndexFilename: + """Tests for get_index_filename.""" + + LONG_NAME = "faiss_gpu_cagra_hnsw" + ".param_with_a_long_name12345" * 5 + + def test_short_name_is_unchanged(self): + """Names under 128 characters are used as file names.""" + name = "faiss_gpu_cagra_hnsw.graph_degree32" + assert get_index_filename(name) == name + + def test_long_name_is_shortened(self): + """Long names are replaced by a 64-character hex digest.""" + filename = get_index_filename(self.LONG_NAME) + assert len(self.LONG_NAME) >= 128 + assert len(filename) == 64 + assert filename != get_index_filename(self.LONG_NAME + "x") + + def test_long_name_is_stable_across_processes(self): + """Separate processes, as in build and search runs, agree.""" + code = ( + "import sys;" + "from cuvs_bench.backends._utils import get_index_filename;" + "print(get_index_filename(sys.argv[1]))" + ) + filenames = { + subprocess.run( + [sys.executable, "-c", code, self.LONG_NAME], + env={**os.environ, "PYTHONHASHSEED": seed}, + capture_output=True, + text=True, + check=True, + ).stdout.splitlines()[-1] + for seed in ("1", "2") + } + assert filenames == {get_index_filename(self.LONG_NAME)} + + class TestComputeRecall: """Tests for compute_recall.""" From 731a0b25263a5a585425260a625743a40a4cea44 Mon Sep 17 00:00:00 2001 From: aamijar Date: Fri, 23 Oct 2026 19:05:36 +0000 Subject: [PATCH 2/2] faiss_cpu_hnsw_cagra --- cpp/bench/ann/src/faiss/faiss_cpu_wrapper.h | 15 +++++++++++ .../cuvs_bench/cuvs_bench/backends/_utils.py | 9 +++++-- .../cuvs_bench/tests/test_data_export.py | 26 +++++++++---------- 3 files changed, 35 insertions(+), 15 deletions(-) diff --git a/cpp/bench/ann/src/faiss/faiss_cpu_wrapper.h b/cpp/bench/ann/src/faiss/faiss_cpu_wrapper.h index 078ac77837..65576da944 100644 --- a/cpp/bench/ann/src/faiss/faiss_cpu_wrapper.h +++ b/cpp/bench/ann/src/faiss/faiss_cpu_wrapper.h @@ -35,6 +35,13 @@ auto parse_metric_type(cuvs::bench::Metric metric) -> faiss::MetricType throw std::runtime_error("faiss supports only metric type of inner product and L2"); } } + +auto metric_type_name(faiss::MetricType metric_type) -> std::string +{ + if (metric_type == faiss::METRIC_INNER_PRODUCT) { return "inner product"; } + if (metric_type == faiss::METRIC_L2) { return "L2"; } + return std::to_string(metric_type); +} } // namespace namespace cuvs::bench { @@ -192,6 +199,14 @@ void faiss_cpu::load_(const std::string& file) throw std::runtime_error("'" + file + "' does not contain a " + typeid(Index).name() + " (found " + typeid(*loaded).name() + ")"); } + if (typed->d != this->dim_) { + throw std::runtime_error("'" + file + "' has dimension " + std::to_string(typed->d) + + ", but the dataset has dimension " + std::to_string(this->dim_)); + } + if (typed->metric_type != metric_type_) { + throw std::runtime_error("'" + file + "' uses the " + metric_type_name(typed->metric_type) + + " metric, but the dataset uses " + metric_type_name(metric_type_)); + } loaded.release(); index_.reset(typed); } diff --git a/python/cuvs_bench/cuvs_bench/backends/_utils.py b/python/cuvs_bench/cuvs_bench/backends/_utils.py index fa5be90a3f..85345b2dda 100644 --- a/python/cuvs_bench/cuvs_bench/backends/_utils.py +++ b/python/cuvs_bench/cuvs_bench/backends/_utils.py @@ -162,9 +162,14 @@ def get_index_name_prefix(algo: str) -> str: also the name its build results are stored under. ``faiss_cpu_hnsw_cagra`` only searches indexes built by ``faiss_gpu_cagra_hnsw``; every other algorithm builds its own indexes. + + ``algo`` may include a group suffix, as in result CSVs for groups other + than ``base``: ``faiss_cpu_hnsw_cagra_large`` maps to + ``faiss_gpu_cagra_hnsw_large``. """ - if algo == "faiss_cpu_hnsw_cagra": - return "faiss_gpu_cagra_hnsw" + search_algo, build_algo = "faiss_cpu_hnsw_cagra", "faiss_gpu_cagra_hnsw" + if algo == search_algo or algo.startswith(f"{search_algo}_"): + return build_algo + algo[len(search_algo) :] return algo diff --git a/python/cuvs_bench/cuvs_bench/tests/test_data_export.py b/python/cuvs_bench/cuvs_bench/tests/test_data_export.py index 652340c377..4a35b338a4 100644 --- a/python/cuvs_bench/cuvs_bench/tests/test_data_export.py +++ b/python/cuvs_bench/cuvs_bench/tests/test_data_export.py @@ -7,6 +7,7 @@ import numpy as np import pandas as pd +import pytest from cuvs_bench.backends.base import BuildResult, SearchResult from cuvs_bench.orchestrator.config_loaders import ( @@ -28,6 +29,10 @@ def test_get_index_name_prefix(): get_index_name_prefix("faiss_cpu_hnsw_cagra") == "faiss_gpu_cagra_hnsw" ) assert get_index_name_prefix("cuvs_cagra") == "cuvs_cagra" + assert ( + get_index_name_prefix("faiss_cpu_hnsw_cagra_large") + == "faiss_gpu_cagra_hnsw_large" + ) def test_cpu_search_csv_includes_gpu_build_results(tmp_path): @@ -75,24 +80,19 @@ def test_cpu_search_csv_includes_gpu_build_results(tmp_path): assert raw.loc[0, "graph_degree"] == 32 -def test_cpu_build_plot_uses_gpu_build_results(tmp_path): - index_name = "faiss_gpu_cagra_hnsw.graph_degree32" - build_results = { - ("faiss_gpu_cagra_hnsw", index_name): [ - ["faiss_gpu_cagra_hnsw", index_name, 20.5] - ] - } - search_results = { - "faiss_cpu_hnsw_cagra": [ - ["faiss_cpu_hnsw_cagra", index_name, 0.95, 1000.0] - ] - } +@pytest.mark.parametrize("suffix", ["", "_large"]) +def test_cpu_build_plot_uses_gpu_build_results(tmp_path, suffix): + gpu_algo = f"faiss_gpu_cagra_hnsw{suffix}" + cpu_algo = f"faiss_cpu_hnsw_cagra{suffix}" + index_name = f"{gpu_algo}.graph_degree32" + build_results = {(gpu_algo, index_name): [[gpu_algo, index_name, 20.5]]} + search_results = {cpu_algo: [[cpu_algo, index_name, 0.95, 1000.0]]} fn_out = tmp_path / "build.png" create_plot_build( build_results, search_results, - {"faiss_cpu_hnsw_cagra": ("C0", "-", 1, "o")}, + {cpu_algo: ("C0", "-", 1, "o")}, str(fn_out), "test-dataset", 10,