From d99e6a03e43c6763eb5616677e519ef2178f4766 Mon Sep 17 00:00:00 2001 From: Max Buckley Date: Sat, 23 May 2026 14:56:21 +0200 Subject: [PATCH 1/7] Promote filtering_rate to base search_params; honor it in brute_force MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `filtering_rate` (a hint of the fraction of items filtered out by the sample filter) previously lived on `cagra::search_params` only. CAGRA used it to size `itopk_size` and, when the user left it at its default of `-1.0`, called `bitset_view::count(res)` on every search — a GPU popcount reduction + host sync that adds measurable latency. `brute_force` filtered search also called `bitset_view::count(res)` on every search to compute `sparsity` (used to choose between the dense tiled GEMM path and the sparse CSR path), but had no user-facing knob to skip the auto-detection. This change: - Moves `float filtering_rate = -1.0` from `cagra::search_params` to the base `cuvs::neighbors::search_params`. `cagra::search_params` inherits it; existing code that accesses `params.filtering_rate` is unaffected. - Plumbs `brute_force::search_params` through `detail::search` and `brute_force_search_filtered`. When `params.filtering_rate >= 0`, the hint is used directly as `sparsity` and the per-search popcount is skipped on the dense path. The CSR path still needs an exact non-zero count to size the matrix, so popcount runs lazily there. - Algorithms that don't use the hint (`ivf_flat`, `ivf_pq`, `hnsw`) ignore the new base field; their `search_params` inherit it but nothing reads it. Tests: extends `brute_force_prefiltered.cu` with `ResultWithFilteringRateHint` cases for both bitmap and bitset fixtures (float and half), reusing the existing parameter matrix and asserting that results match the auto-detect path. Closes #1960. Co-Authored-By: Claude Opus 4.7 (1M context) --- cpp/include/cuvs/neighbors/cagra.hpp | 14 +-- cpp/include/cuvs/neighbors/common.hpp | 18 ++- cpp/src/neighbors/brute_force.cu | 8 +- cpp/src/neighbors/detail/knn_brute_force.cuh | 43 ++++++-- .../neighbors/brute_force_prefiltered.cu | 104 ++++++++++++++++++ 5 files changed, 165 insertions(+), 22 deletions(-) diff --git a/cpp/include/cuvs/neighbors/cagra.hpp b/cpp/include/cuvs/neighbors/cagra.hpp index d1937cba27..87e2834239 100644 --- a/cpp/include/cuvs/neighbors/cagra.hpp +++ b/cpp/include/cuvs/neighbors/cagra.hpp @@ -346,15 +346,11 @@ struct search_params : cuvs::neighbors::search_params { */ float persistent_device_usage = 1.0; - /** - * A parameter indicating the rate of nodes to be filtered-out, when filtering is used. - * The value must be equal to or greater than 0.0 and less than 1.0. Default value is - * negative, in which case the filtering rate is automatically calculated when possible. - * For `filtering::udf_filter`, CAGRA uses `udf_filter::filtering_rate` when this value is - * negative. If both values are negative, CAGRA assumes 0.0 because a UDF's selectivity cannot be - * inferred from the source string. - */ - float filtering_rate = -1.0; + // `filtering_rate` is inherited from `cuvs::neighbors::search_params`. CAGRA uses it to size + // `itopk_size`; supplying a non-negative value avoids a per-search popcount + host sync on the + // `bitset_filter` path. For `filtering::udf_filter`, CAGRA falls back to + // `udf_filter::filtering_rate` when this value is negative. If both are negative, CAGRA assumes + // 0.0 because a UDF's selectivity cannot be inferred from the source string. /** Data type of the query vector and codebook table on shared memory. Currently, only VPQ * supports FP8. **/ diff --git a/cpp/include/cuvs/neighbors/common.hpp b/cpp/include/cuvs/neighbors/common.hpp index 2fd804f115..d99d0fe6c5 100644 --- a/cpp/include/cuvs/neighbors/common.hpp +++ b/cpp/include/cuvs/neighbors/common.hpp @@ -118,7 +118,23 @@ struct index_params { float metric_arg = 2.0f; }; -struct search_params {}; +struct search_params { + /** + * A hint indicating the rate at which the sample filter is expected to filter out items + * (i.e. `(n_dataset - n_set_bits) / n_dataset` for a `bitset_filter`). + * + * Algorithms that benefit from knowing the filter's selectivity may use this hint to tune + * internal parameters or skip a hidden popcount kernel + host sync that would otherwise be + * required to derive it from the filter on every search call. + * + * - Negative (default): the algorithm auto-detects the rate from the filter when needed. + * This launches a GPU popcount reduction and synchronizes the stream per search call. + * - In `[0.0, 1.0)`: the algorithm trusts the supplied value and skips the auto-detection. + * + * Algorithms that do not use this hint (e.g. `ivf_flat`, `ivf_pq`) ignore it. + */ + float filtering_rate = -1.0; +}; /** * @brief Strategy for merging indices. diff --git a/cpp/src/neighbors/brute_force.cu b/cpp/src/neighbors/brute_force.cu index 2f9000acf7..c4d9b002f3 100644 --- a/cpp/src/neighbors/brute_force.cu +++ b/cpp/src/neighbors/brute_force.cu @@ -199,7 +199,7 @@ void index::update_dataset( const cuvs::neighbors::filtering::base_filter& sample_filter) \ { \ detail::search( \ - res, idx, queries, neighbors, distances, sample_filter); \ + res, params, idx, queries, neighbors, distances, sample_filter); \ } \ void search(raft::resources const& res, \ const cuvs::neighbors::brute_force::index& idx, \ @@ -209,7 +209,7 @@ void index::update_dataset( const cuvs::neighbors::filtering::base_filter& sample_filter) \ { \ detail::search( \ - res, idx, queries, neighbors, distances, sample_filter); \ + res, {}, idx, queries, neighbors, distances, sample_filter); \ } \ void search(raft::resources const& res, \ const cuvs::neighbors::brute_force::search_params& params, \ @@ -220,7 +220,7 @@ void index::update_dataset( const cuvs::neighbors::filtering::base_filter& sample_filter) \ { \ detail::search( \ - res, idx, queries, neighbors, distances, sample_filter); \ + res, params, idx, queries, neighbors, distances, sample_filter); \ } \ void search(raft::resources const& res, \ const cuvs::neighbors::brute_force::index& idx, \ @@ -230,7 +230,7 @@ void index::update_dataset( const cuvs::neighbors::filtering::base_filter& sample_filter) \ { \ detail::search( \ - res, idx, queries, neighbors, distances, sample_filter); \ + res, {}, idx, queries, neighbors, distances, sample_filter); \ } CUVS_INST_BFKNN(float, float); diff --git a/cpp/src/neighbors/detail/knn_brute_force.cuh b/cpp/src/neighbors/detail/knn_brute_force.cuh index 34ef8dd937..63d6c0210e 100644 --- a/cpp/src/neighbors/detail/knn_brute_force.cuh +++ b/cpp/src/neighbors/detail/knn_brute_force.cuh @@ -584,6 +584,7 @@ void brute_force_search( template void brute_force_search_filtered( raft::resources const& res, + const cuvs::neighbors::brute_force::search_params& params, const cuvs::neighbors::brute_force::index& idx, raft::device_matrix_view queries, const cuvs::neighbors::filtering::base_filter* filter, @@ -619,8 +620,10 @@ void brute_force_search_filtered( const cuvs::core::bitset_view>> filter_view; - IdxT nnz_h = 0; - float sparsity = 0.0f; + IdxT nnz_h = 0; + float sparsity = 0.0f; + bool nnz_h_is_set = false; + const bool use_hint = params.filtering_rate >= 0.0f; const BitsT* filter_data = nullptr; @@ -628,14 +631,24 @@ void brute_force_search_filtered( auto actual_filter = dynamic_cast*>(filter); filter_view.emplace(actual_filter->view()); - nnz_h = actual_filter->view().count(res); - sparsity = 1.0 - nnz_h / (1.0 * n_queries * n_dataset); + if (use_hint) { + sparsity = params.filtering_rate; + } else { + nnz_h = actual_filter->view().count(res); + sparsity = 1.0 - nnz_h / (1.0 * n_queries * n_dataset); + nnz_h_is_set = true; + } } else if (filter_type == cuvs::neighbors::filtering::FilterType::Bitset) { auto actual_filter = dynamic_cast*>(filter); filter_view.emplace(actual_filter->view()); - nnz_h = n_queries * actual_filter->view().count(res); - sparsity = 1.0 - nnz_h / (1.0 * n_queries * n_dataset); + if (use_hint) { + sparsity = params.filtering_rate; + } else { + nnz_h = n_queries * actual_filter->view().count(res); + sparsity = 1.0 - nnz_h / (1.0 * n_queries * n_dataset); + nnz_h_is_set = true; + } } else { RAFT_FAIL("Unsupported sample filter type"); } @@ -666,6 +679,19 @@ void brute_force_search_filtered( raft::identity_op(), filter_type); } else { + // The CSR path needs an exact non-zero count to size the matrix. If the hint was used, + // popcount lazily here — we still save the kernel + sync on the (more common) dense path. + if (!nnz_h_is_set) { + if (filter_type == cuvs::neighbors::filtering::FilterType::Bitmap) { + auto actual_filter = + dynamic_cast*>(filter); + nnz_h = actual_filter->view().count(res); + } else { + auto actual_filter = + dynamic_cast*>(filter); + nnz_h = n_queries * actual_filter->view().count(res); + } + } auto csr = raft::make_device_csr_matrix(res, n_queries, n_dataset, nnz_h); std::visit([&](const auto& actual_view) { actual_view.to_csr(res, csr); }, *filter_view); @@ -739,6 +765,7 @@ void brute_force_search_filtered( template void search(raft::resources const& res, + const cuvs::neighbors::brute_force::search_params& params, const cuvs::neighbors::brute_force::index& idx, raft::device_matrix_view queries, raft::device_matrix_view neighbors, @@ -759,7 +786,7 @@ void search(raft::resources const& res, dynamic_cast&>( sample_filter_ref); return brute_force_search_filtered( - res, idx, queries, &sample_filter, neighbors, distances); + res, params, idx, queries, &sample_filter, neighbors, distances); } catch (const std::bad_cast&) { } @@ -768,7 +795,7 @@ void search(raft::resources const& res, dynamic_cast&>( sample_filter_ref); return brute_force_search_filtered( - res, idx, queries, &sample_filter, neighbors, distances); + res, params, idx, queries, &sample_filter, neighbors, distances); } catch (const std::bad_cast&) { RAFT_FAIL("Unsupported sample filter type"); } diff --git a/cpp/tests/neighbors/brute_force_prefiltered.cu b/cpp/tests/neighbors/brute_force_prefiltered.cu index c15f08363f..7553490acc 100644 --- a/cpp/tests/neighbors/brute_force_prefiltered.cu +++ b/cpp/tests/neighbors/brute_force_prefiltered.cu @@ -520,6 +520,50 @@ class PrefilteredBruteForceOnBitmapTest true)); } + // Same as Run(), but passes the true sparsity as a filtering_rate hint to the + // params-taking search overload. Confirms results match auto-detection. + void RunWithFilteringRateHint() + { + auto dataset_raw = raft::make_device_matrix_view( + (const value_t*)dataset_d.data(), params.n_dataset, params.dim); + + auto queries = raft::make_device_matrix_view( + (const value_t*)queries_d.data(), params.n_queries, params.dim); + + auto dataset = brute_force::build(handle, dataset_raw, params.metric); + + auto filter = cuvs::core::bitmap_view( + (bitmap_t*)filter_d.data(), params.n_queries, params.n_dataset); + + auto out_val = raft::make_device_matrix_view( + out_val_d.data(), params.n_queries, params.top_k); + auto out_idx = raft::make_device_matrix_view( + out_idx_d.data(), params.n_queries, params.top_k); + + cuvs::neighbors::brute_force::search_params search_params; + search_params.filtering_rate = params.sparsity; + + brute_force::search(handle, + search_params, + dataset, + queries, + out_idx, + out_val, + cuvs::neighbors::filtering::bitmap_filter(filter)); + + raft::resource::sync_stream(handle); + + ASSERT_TRUE(cuvs::neighbors::devArrMatchKnnPair(out_idx_expected_d.data(), + out_idx.data_handle(), + out_val_expected_d.data(), + out_val.data_handle(), + params.n_queries, + params.top_k, + 0.001f, + stream, + true)); + } + protected: raft::resources handle; cudaStream_t stream; @@ -941,6 +985,50 @@ class PrefilteredBruteForceOnBitsetTest true)); } + // Same as Run(), but passes the true sparsity as a filtering_rate hint to the + // params-taking search overload. Confirms results match auto-detection. + void RunWithFilteringRateHint() + { + auto dataset_raw = raft::make_device_matrix_view( + (const value_t*)dataset_d.data(), params.n_dataset, params.dim); + + auto queries = raft::make_device_matrix_view( + (const value_t*)queries_d.data(), params.n_queries, params.dim); + + auto dataset = brute_force::build(handle, dataset_raw, params.metric); + + auto filter = + cuvs::core::bitset_view((bitset_t*)filter_d.data(), params.n_dataset); + + auto out_val = raft::make_device_matrix_view( + out_val_d.data(), params.n_queries, params.top_k); + auto out_idx = raft::make_device_matrix_view( + out_idx_d.data(), params.n_queries, params.top_k); + + cuvs::neighbors::brute_force::search_params search_params; + search_params.filtering_rate = params.sparsity; + + brute_force::search(handle, + search_params, + dataset, + queries, + out_idx, + out_val, + cuvs::neighbors::filtering::bitset_filter(filter)); + + raft::resource::sync_stream(handle); + + ASSERT_TRUE(cuvs::neighbors::devArrMatchKnnPair(out_idx_expected_d.data(), + out_idx.data_handle(), + out_val_expected_d.data(), + out_val.data_handle(), + params.n_queries, + params.top_k, + 0.001f, + stream, + true)); + } + protected: raft::resources handle; cudaStream_t stream; @@ -963,18 +1051,34 @@ class PrefilteredBruteForceOnBitsetTest using PrefilteredBruteForceTestOnBitmap_float_int64 = PrefilteredBruteForceOnBitmapTest; TEST_P(PrefilteredBruteForceTestOnBitmap_float_int64, Result) { Run(); } +TEST_P(PrefilteredBruteForceTestOnBitmap_float_int64, ResultWithFilteringRateHint) +{ + RunWithFilteringRateHint(); +} using PrefilteredBruteForceTestOnBitmap_half_int64 = PrefilteredBruteForceOnBitmapTest; TEST_P(PrefilteredBruteForceTestOnBitmap_half_int64, Result) { Run(); } +TEST_P(PrefilteredBruteForceTestOnBitmap_half_int64, ResultWithFilteringRateHint) +{ + RunWithFilteringRateHint(); +} using PrefilteredBruteForceTestOnBitset_float_int64 = PrefilteredBruteForceOnBitsetTest; TEST_P(PrefilteredBruteForceTestOnBitset_float_int64, Result) { Run(); } +TEST_P(PrefilteredBruteForceTestOnBitset_float_int64, ResultWithFilteringRateHint) +{ + RunWithFilteringRateHint(); +} using PrefilteredBruteForceTestOnBitset_half_int64 = PrefilteredBruteForceOnBitsetTest; TEST_P(PrefilteredBruteForceTestOnBitset_half_int64, Result) { Run(); } +TEST_P(PrefilteredBruteForceTestOnBitset_half_int64, ResultWithFilteringRateHint) +{ + RunWithFilteringRateHint(); +} template const std::vector> selectk_inputs = { From 693f2160267ad6e61e701c43a712378b0e35a9d2 Mon Sep 17 00:00:00 2001 From: Max Buckley Date: Tue, 14 Jul 2026 23:18:25 +0200 Subject: [PATCH 2/7] brute_force: validate filtering_rate before using it as a hint The base search_params docstring already defines the contract: negative means "auto-detect from the filter", and [0.0, 1.0) is trusted as a hint. The code only checked `>= 0.0f`, so an out-of-contract value (e.g. a caller passing a percentage like 50.0, or 1.5) was silently accepted as `sparsity`, forcing the sparse CSR path plus its popcount -- slower than not setting the hint at all, with no diagnostic. Enforce the documented range with RAFT_EXPECTS. NaN is rejected too, since it compares false against both bounds. Behavior for in-contract values, including the negative default, is unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --- cpp/src/neighbors/detail/knn_brute_force.cuh | 16 ++++++-- .../neighbors/brute_force_prefiltered.cu | 39 +++++++++++++++++++ 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/cpp/src/neighbors/detail/knn_brute_force.cuh b/cpp/src/neighbors/detail/knn_brute_force.cuh index 63d6c0210e..a2ea7ed38f 100644 --- a/cpp/src/neighbors/detail/knn_brute_force.cuh +++ b/cpp/src/neighbors/detail/knn_brute_force.cuh @@ -608,6 +608,15 @@ void brute_force_search_filtered( metric == cuvs::distance::DistanceType::CosineExpanded), "Index must has norms when using Euclidean, IP, and Cosine!"); + // Negative means "auto-detect from the filter"; [0.0, 1.0) is trusted as-is. Anything else + // (including NaN, which compares false against both bounds) is out of contract. + const bool auto_filtering_rate = params.filtering_rate < 0.0f; + const bool use_hint = params.filtering_rate >= 0.0f && params.filtering_rate < 1.0f; + RAFT_EXPECTS(auto_filtering_rate || use_hint, + "search_params::filtering_rate must be negative (auto-detect) or in [0.0, 1.0), " + "got %f", + params.filtering_rate); + IdxT n_queries = queries.extent(0); IdxT n_dataset = idx.dataset().extent(0); IdxT dim = idx.dataset().extent(1); @@ -620,10 +629,9 @@ void brute_force_search_filtered( const cuvs::core::bitset_view>> filter_view; - IdxT nnz_h = 0; - float sparsity = 0.0f; - bool nnz_h_is_set = false; - const bool use_hint = params.filtering_rate >= 0.0f; + IdxT nnz_h = 0; + float sparsity = 0.0f; + bool nnz_h_is_set = false; const BitsT* filter_data = nullptr; diff --git a/cpp/tests/neighbors/brute_force_prefiltered.cu b/cpp/tests/neighbors/brute_force_prefiltered.cu index 7553490acc..485127c2c6 100644 --- a/cpp/tests/neighbors/brute_force_prefiltered.cu +++ b/cpp/tests/neighbors/brute_force_prefiltered.cu @@ -564,6 +564,41 @@ class PrefilteredBruteForceOnBitmapTest true)); } + // A filtering_rate that is neither the negative auto-detect sentinel nor in [0.0, 1.0) is + // out of contract and must be rejected rather than silently steering path selection. + void RunWithInvalidFilteringRate() + { + auto dataset_raw = raft::make_device_matrix_view( + (const value_t*)dataset_d.data(), params.n_dataset, params.dim); + + auto queries = raft::make_device_matrix_view( + (const value_t*)queries_d.data(), params.n_queries, params.dim); + + auto dataset = brute_force::build(handle, dataset_raw, params.metric); + + auto filter = cuvs::core::bitmap_view( + (bitmap_t*)filter_d.data(), params.n_queries, params.n_dataset); + + auto out_val = raft::make_device_matrix_view( + out_val_d.data(), params.n_queries, params.top_k); + auto out_idx = raft::make_device_matrix_view( + out_idx_d.data(), params.n_queries, params.top_k); + + for (float invalid_rate : {1.0f, 1.5f, 50.0f}) { + cuvs::neighbors::brute_force::search_params search_params; + search_params.filtering_rate = invalid_rate; + + EXPECT_THROW(brute_force::search(handle, + search_params, + dataset, + queries, + out_idx, + out_val, + cuvs::neighbors::filtering::bitmap_filter(filter)), + raft::logic_error); + } + } + protected: raft::resources handle; cudaStream_t stream; @@ -1055,6 +1090,10 @@ TEST_P(PrefilteredBruteForceTestOnBitmap_float_int64, ResultWithFilteringRateHin { RunWithFilteringRateHint(); } +TEST_P(PrefilteredBruteForceTestOnBitmap_float_int64, InvalidFilteringRateThrows) +{ + RunWithInvalidFilteringRate(); +} using PrefilteredBruteForceTestOnBitmap_half_int64 = PrefilteredBruteForceOnBitmapTest; From ce922ff118611cb7f27626e86e33d3b2081c16aa Mon Sep 17 00:00:00 2001 From: Max Buckley Date: Thu, 23 Jul 2026 10:54:28 +0200 Subject: [PATCH 3/7] test(brute_force): pass the real sparsity as the filtering_rate hint PrefilteredBruteForceInputs::sparsity is the density of kept entries (n_edges = m * n * sparsity), while search_params::filtering_rate is 1 - nnz / (n_queries * n_dataset). RunWithFilteringRateHint() passed the former as the latter, so the hint was the complement of the true sparsity and every hinted run took the dense path -- the lazy-nnz CSR branch was never exercised. Co-Authored-By: Claude Opus 4.8 (1M context) --- cpp/tests/neighbors/brute_force_prefiltered.cu | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cpp/tests/neighbors/brute_force_prefiltered.cu b/cpp/tests/neighbors/brute_force_prefiltered.cu index 485127c2c6..ff595d290b 100644 --- a/cpp/tests/neighbors/brute_force_prefiltered.cu +++ b/cpp/tests/neighbors/brute_force_prefiltered.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ @@ -522,6 +522,7 @@ class PrefilteredBruteForceOnBitmapTest // Same as Run(), but passes the true sparsity as a filtering_rate hint to the // params-taking search overload. Confirms results match auto-detection. + // `params.sparsity` is the density of kept entries, so the hint is its complement. void RunWithFilteringRateHint() { auto dataset_raw = raft::make_device_matrix_view( @@ -541,7 +542,7 @@ class PrefilteredBruteForceOnBitmapTest out_idx_d.data(), params.n_queries, params.top_k); cuvs::neighbors::brute_force::search_params search_params; - search_params.filtering_rate = params.sparsity; + search_params.filtering_rate = 1.0f - params.sparsity; brute_force::search(handle, search_params, @@ -1022,6 +1023,7 @@ class PrefilteredBruteForceOnBitsetTest // Same as Run(), but passes the true sparsity as a filtering_rate hint to the // params-taking search overload. Confirms results match auto-detection. + // `params.sparsity` is the density of kept entries, so the hint is its complement. void RunWithFilteringRateHint() { auto dataset_raw = raft::make_device_matrix_view( @@ -1041,7 +1043,7 @@ class PrefilteredBruteForceOnBitsetTest out_idx_d.data(), params.n_queries, params.top_k); cuvs::neighbors::brute_force::search_params search_params; - search_params.filtering_rate = params.sparsity; + search_params.filtering_rate = 1.0f - params.sparsity; brute_force::search(handle, search_params, From a98bbd8540de27b4b86e7652f56f61f39c331595 Mon Sep 17 00:00:00 2001 From: Max Buckley Date: Thu, 3 Sep 2026 10:18:20 +0200 Subject: [PATCH 4/7] Use the canonical NVIDIA copyright notice Applied by `pre-commit run --all-files`; the verify-copyright hook requires the "& AFFILIATES. All rights reserved." form. --- cpp/src/neighbors/brute_force.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/neighbors/brute_force.cu b/cpp/src/neighbors/brute_force.cu index c4d9b002f3..9712a360e7 100644 --- a/cpp/src/neighbors/brute_force.cu +++ b/cpp/src/neighbors/brute_force.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ From d74acd977f42cd136c2250252fcdc6fa459bb042 Mon Sep 17 00:00:00 2001 From: Max Buckley Date: Thu, 8 Oct 2026 01:11:09 +0200 Subject: [PATCH 5/7] Move filtering_rate from the base search_params to brute_force::search_params Hoisting the field into cuvs::neighbors::search_params broke source compatibility for CAGRA callers: a designated initializer cannot name an inherited member, so `cagra::search_params{.filtering_rate = x}` stopped compiling. It also changed the layout of every algorithm's search_params. Restore common.hpp and cagra.hpp to main exactly and declare the hint on brute_force::search_params, which was empty. CAGRA's API is untouched; the only struct that changes is the one gaining the knob. Semantics are unchanged: negative auto-detects, [0, 1) is trusted, anything else throws. The invalid-rate test now also covers NaN. Co-Authored-By: Claude Opus 5.5 --- cpp/include/cuvs/neighbors/brute_force.hpp | 16 +++++++++++++++- cpp/include/cuvs/neighbors/cagra.hpp | 13 +++++++++---- cpp/include/cuvs/neighbors/common.hpp | 19 +------------------ .../neighbors/brute_force_prefiltered.cu | 2 +- 4 files changed, 26 insertions(+), 24 deletions(-) diff --git a/cpp/include/cuvs/neighbors/brute_force.hpp b/cpp/include/cuvs/neighbors/brute_force.hpp index d30ea21320..098aec30dd 100644 --- a/cpp/include/cuvs/neighbors/brute_force.hpp +++ b/cpp/include/cuvs/neighbors/brute_force.hpp @@ -21,7 +21,21 @@ namespace brute_force { struct index_params : cuvs::neighbors::index_params {}; -struct search_params : cuvs::neighbors::search_params {}; +struct search_params : cuvs::neighbors::search_params { + /** + * A hint for the fraction of entries the filter removes, i.e. + * `(n_queries * n_dataset - n_set_bits) / (n_queries * n_dataset)`; for a `bitset_filter` this is + * `(n_dataset - n_set_bits) / n_dataset`. + * + * - Negative (default): the rate is computed from the filter, which runs a popcount kernel and + * synchronizes the stream on every filtered search. + * - In `[0.0, 1.0)`: the value is trusted and the popcount is skipped when the dense path is + * selected. The sparse and gather paths still count the filter, because they need the exact + * number of passing entries. + * - Any other value (including NaN) is rejected. + */ + float filtering_rate = -1.0; +}; /** * @defgroup bruteforce_cpp_index Bruteforce index diff --git a/cpp/include/cuvs/neighbors/cagra.hpp b/cpp/include/cuvs/neighbors/cagra.hpp index b276b7e216..418db9d88f 100644 --- a/cpp/include/cuvs/neighbors/cagra.hpp +++ b/cpp/include/cuvs/neighbors/cagra.hpp @@ -120,10 +120,15 @@ struct search_params : cuvs::neighbors::search_params { */ float persistent_device_usage = 1.0; - // `filtering_rate` is inherited from `cuvs::neighbors::search_params`. CAGRA uses it to size - // `itopk_size`. When it is negative, CAGRA derives the rate from the filter where possible; for - // `filtering::udf_filter` it uses `udf_filter::filtering_rate`, and if both values are negative - // it assumes 0.0 because a UDF's selectivity cannot be inferred from the source string. + /** + * A parameter indicating the rate of nodes to be filtered-out, when filtering is used. + * The value must be equal to or greater than 0.0 and less than 1.0. Default value is + * negative, in which case the filtering rate is automatically calculated when possible. + * For `filtering::udf_filter`, CAGRA uses `udf_filter::filtering_rate` when this value is + * negative. If both values are negative, CAGRA assumes 0.0 because a UDF's selectivity cannot be + * inferred from the source string. + */ + float filtering_rate = -1.0; /** Data type of the query vector and codebook table on shared memory. Currently, only VPQ * supports FP8. **/ diff --git a/cpp/include/cuvs/neighbors/common.hpp b/cpp/include/cuvs/neighbors/common.hpp index bf9ed65892..baa5b44da2 100644 --- a/cpp/include/cuvs/neighbors/common.hpp +++ b/cpp/include/cuvs/neighbors/common.hpp @@ -132,24 +132,7 @@ struct index_params { float metric_arg = 2.0f; }; -struct search_params { - /** - * A hint indicating the rate at which the sample filter is expected to filter out items - * (i.e. `(n_dataset - n_set_bits) / n_dataset` for a `bitset_filter`). - * - * Algorithms that benefit from knowing the filter's selectivity may use this hint to tune - * internal parameters or skip a hidden popcount kernel + host sync that would otherwise be - * required to derive it from the filter on every search call. - * - * - Negative (default): the algorithm auto-detects the rate from the filter when needed. - * For a `bitset_filter` this launches a GPU popcount reduction and synchronizes the stream - * per search call. - * - In `[0.0, 1.0)`: the algorithm trusts the supplied value and skips the auto-detection. - * - * Algorithms that do not use this hint (e.g. `ivf_flat`, `ivf_pq`) ignore it. - */ - float filtering_rate = -1.0; -}; +struct search_params {}; /** * @brief Strategy for merging indices. diff --git a/cpp/tests/neighbors/brute_force_prefiltered.cu b/cpp/tests/neighbors/brute_force_prefiltered.cu index ac9ba6ae0f..6371824d01 100644 --- a/cpp/tests/neighbors/brute_force_prefiltered.cu +++ b/cpp/tests/neighbors/brute_force_prefiltered.cu @@ -632,7 +632,7 @@ class PrefilteredBruteForceOnBitmapTest auto out_idx = raft::make_device_matrix_view( out_idx_d.data(), params.n_queries, params.top_k); - for (float invalid_rate : {1.0f, 1.5f, 50.0f}) { + for (float invalid_rate : {1.0f, 1.5f, 50.0f, std::numeric_limits::quiet_NaN()}) { cuvs::neighbors::brute_force::search_params search_params; search_params.filtering_rate = invalid_rate; From 91c3c719043fe1617454c3a8d5d1d01cf3f2bf44 Mon Sep 17 00:00:00 2001 From: Max Buckley Date: Thu, 8 Oct 2026 01:14:38 +0200 Subject: [PATCH 6/7] docs(brute_force): document the filtering_rate search parameter Co-Authored-By: Claude Opus 5.5 --- fern/pages/neighbors/bruteforce.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fern/pages/neighbors/bruteforce.md b/fern/pages/neighbors/bruteforce.md index 1219288576..f76c96ff2e 100644 --- a/fern/pages/neighbors/bruteforce.md +++ b/fern/pages/neighbors/bruteforce.md @@ -542,7 +542,9 @@ SearchResults results = index.search(query); ### Search parameters -The C++ `brute_force::search_params` struct currently has no tunable fields. +| Name | Default | Description | +| --- | --- | --- | +| `filtering_rate` | `-1.0` | C++ only. Expected fraction of query-vector pairs removed by the filter. Negative values make filtered search count the filter on every call, which launches a GPU reduction and synchronizes the stream. A value in `[0.0, 1.0)` skips that count when the dense path is selected. Other values are rejected. | Filters are passed as search function arguments in bindings that expose filtered search, not as fields in `brute_force::search_params`. From 73d346f80e84c8eb36d8153276529b299b35ab8a Mon Sep 17 00:00:00 2001 From: Max Buckley Date: Thu, 8 Oct 2026 09:32:59 +0200 Subject: [PATCH 7/7] Derive the test hint from the measured nnz; document brute_force::search_params RMAT can draw the same edge more than once, so the filter may set fewer bits than `params.sparsity` implies. Compute the filtering_rate hint from the popcounted `nnz` the fixture already has, so the hinted search sees the filter's true rate. Co-Authored-By: Claude Opus 5.5 --- cpp/include/cuvs/neighbors/brute_force.hpp | 1 + .../neighbors/brute_force_prefiltered.cu | 20 +++++++++++-------- .../cpp_api/cpp-api-neighbors-brute-force.md | 19 ++++++++++++++++++ 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/cpp/include/cuvs/neighbors/brute_force.hpp b/cpp/include/cuvs/neighbors/brute_force.hpp index 098aec30dd..b4addb22dd 100644 --- a/cpp/include/cuvs/neighbors/brute_force.hpp +++ b/cpp/include/cuvs/neighbors/brute_force.hpp @@ -21,6 +21,7 @@ namespace brute_force { struct index_params : cuvs::neighbors::index_params {}; +/** @brief Parameters for brute-force search. */ struct search_params : cuvs::neighbors::search_params { /** * A hint for the fraction of entries the filter removes, i.e. diff --git a/cpp/tests/neighbors/brute_force_prefiltered.cu b/cpp/tests/neighbors/brute_force_prefiltered.cu index 6371824d01..f02b0d9e5c 100644 --- a/cpp/tests/neighbors/brute_force_prefiltered.cu +++ b/cpp/tests/neighbors/brute_force_prefiltered.cu @@ -567,9 +567,9 @@ class PrefilteredBruteForceOnBitmapTest true)); } - // Same as Run(), but passes the true sparsity as a filtering_rate hint to the - // params-taking search overload. Confirms results match auto-detection. - // `params.sparsity` is the density of kept entries, so the hint is its complement. + // Same as Run(), but passes the filter's true filtering_rate as a hint to the params-taking + // search overload. Confirms results match auto-detection. The rate comes from the measured + // `nnz`, not `params.sparsity`: RMAT can draw the same edge twice, so fewer bits may be set. void RunWithFilteringRateHint() { auto dataset_raw = raft::make_device_matrix_view( @@ -589,7 +589,9 @@ class PrefilteredBruteForceOnBitmapTest out_idx_d.data(), params.n_queries, params.top_k); cuvs::neighbors::brute_force::search_params search_params; - search_params.filtering_rate = 1.0f - params.sparsity; + search_params.filtering_rate = + static_cast(1.0 - static_cast(nnz) / (static_cast(params.n_queries) * + static_cast(params.n_dataset))); brute_force::search(handle, search_params, @@ -1028,9 +1030,9 @@ class PrefilteredBruteForceOnBitsetTest true)); } - // Same as Run(), but passes the true sparsity as a filtering_rate hint to the - // params-taking search overload. Confirms results match auto-detection. - // `params.sparsity` is the density of kept entries, so the hint is its complement. + // Same as Run(), but passes the filter's true filtering_rate as a hint to the params-taking + // search overload. Confirms results match auto-detection. The rate comes from the measured + // `nnz`, not `params.sparsity`: RMAT can draw the same edge twice, so fewer bits may be set. void RunWithFilteringRateHint() { auto dataset_raw = raft::make_device_matrix_view( @@ -1050,7 +1052,9 @@ class PrefilteredBruteForceOnBitsetTest out_idx_d.data(), params.n_queries, params.top_k); cuvs::neighbors::brute_force::search_params search_params; - search_params.filtering_rate = 1.0f - params.sparsity; + search_params.filtering_rate = + static_cast(1.0 - static_cast(nnz) / (static_cast(params.n_queries) * + static_cast(params.n_dataset))); brute_force::search(handle, search_params, diff --git a/fern/pages/cpp_api/cpp-api-neighbors-brute-force.md b/fern/pages/cpp_api/cpp-api-neighbors-brute-force.md index b5bdd6cc78..fdce39b85b 100644 --- a/fern/pages/cpp_api/cpp-api-neighbors-brute-force.md +++ b/fern/pages/cpp_api/cpp-api-neighbors-brute-force.md @@ -6,6 +6,25 @@ slug: api-reference/cpp-api-neighbors-brute-force _Source header: `cuvs/neighbors/brute_force.hpp`_ +## Types + + +### neighbors::brute_force::search_params + +Parameters for brute-force search. + +```cpp +struct search_params : cuvs::neighbors::search_params { + float filtering_rate; +}; +``` + +**Fields** + +| Name | Type | Description | +| --- | --- | --- | +| `filtering_rate` | `float` | A hint for the fraction of entries the filter removes, i.e. `(n_queries * n_dataset - n_set_bits) / (n_queries * n_dataset)`; for a `bitset_filter` this is `(n_dataset - n_set_bits) / n_dataset`.

- Negative (default): the rate is computed from the filter, which runs a popcount kernel and synchronizes the stream on every filtered search.
- In `[0.0, 1.0)`: the value is trusted and the popcount is skipped when the dense path is selected. The sparse and gather paths still count the filter, because they need the exact number of passing entries.
- Any other value (including NaN) is rejected. | + ## Bruteforce index