Prepare for supporting multiple Lucene versions

.github/workflows/build.yaml

@@ -147,7 +147,7 @@ jobs:
       matrix_filter: map(select(.ARCH == "amd64"))
       script: "ci/build_lucene.sh"
       artifact-name-prefix: "cuvs-lucene"
-      file_to_upload: "java/cuvs-lucene/target/"
+      file_to_upload: "java/cuvs-lucene/*/target/"
   python-build:
     needs: [build-details, cpp-build]
     permissions:

.github/workflows/pr.yaml

@@ -27,6 +27,7 @@ jobs:
       - conda-java-tests
       - conda-lucene-build
       - conda-lucene-tests
+      - conda-lucene-tests-jdk21
       - rust-build
       - go-build
       - docs-build
@@ -240,6 +241,7 @@ jobs:
           - '!ci/test_cpp.sh'
           - '!ci/test_java.sh'
           - '!ci/test_lucene.sh'
+          - '!ci/test_lucene_jdk21.sh'
           - '!ci/test_standalone_c.sh'
           - '!ci/test_wheel_cuvs.sh'
           - '!ci/validate_wheel.sh'
@@ -296,6 +298,7 @@ jobs:
           - '!ci/test_cpp.sh'
           - '!ci/test_java.sh'
           - '!ci/test_lucene.sh'
+          - '!ci/test_lucene_jdk21.sh'
           - '!ci/test_python.sh'
           - '!ci/test_standalone_c.sh'
           - '!ci/validate_wheel.sh'
@@ -356,6 +359,7 @@ jobs:
           - '!ci/test_cpp.sh'
           - '!ci/test_java.sh'
           - '!ci/test_lucene.sh'
+          - '!ci/test_lucene_jdk21.sh'
           - '!ci/test_python.sh'
           - '!ci/test_wheel_cuvs.sh'
           - '!ci/validate_wheel.sh'
@@ -409,6 +413,7 @@ jobs:
           - '!ci/test_cpp.sh'
           - '!ci/test_java.sh'
           - '!ci/test_lucene.sh'
+          - '!ci/test_lucene_jdk21.sh'
           - '!ci/test_python.sh'
           - '!ci/test_standalone_c.sh'
           - '!ci/test_wheel_cuvs.sh'
@@ -646,7 +651,7 @@ jobs:
       matrix_filter: map(select(.ARCH == "amd64"))
       script: "ci/build_lucene.sh"
       artifact-name-prefix: "cuvs-lucene"
-      file_to_upload: "java/cuvs-lucene/target/"
+      file_to_upload: "java/cuvs-lucene/*/target/"
   conda-lucene-tests:
     needs: conda-lucene-build
     permissions:
@@ -659,6 +664,27 @@ jobs:
     with:
       build_type: pull-request
       script: "ci/test_lucene.sh"
+  conda-lucene-tests-jdk21:
+    # Verifies that cuvs-lucene, compiled with --release 21, loads and works without the GPU on
+    # JDK 21, the oldest JDK it supports, by running its test suite against the amd64-built
+    # classes without recompiling -- see ci/test_lucene_jdk21.sh. It runs once, not per CUDA
+    # version: it uses no GPU, so the artifacts of any amd64 entry of the build matrix will do.
+    # This matters because many applications built on Lucene 10 support JDK 21 as their minimum.
+    # cuvs-lucene only uses the GPU on JDK 22+, but it can't stop such an application from starting
+    # on JDK 21, so there it has to load and fall back to the CPU.
+    needs: conda-lucene-build
+    permissions:
+      actions: read
+      contents: read
+      id-token: write
+      packages: read
+      pull-requests: read
+    uses: ./.github/workflows/matrix-job.yaml
+    with:
+      build_type: pull-request
+      node_type: cpu8
+      matrix_filter: '[map(select(.ARCH == "amd64")) | max_by(.CUDA_VER | split(".") | map(tonumber))]'
+      script: "ci/test_lucene_jdk21.sh"
   rust-build:
     needs: [conda-cpp-build, changed-files]
     permissions:

.pre-commit-config.yaml

@@ -121,7 +121,7 @@ repos:
         name: spotless-fmt
         entry: ci/checks/run_spotless.sh
         pass_filenames: false
-        files: ^(java|examples/java)/(cuvs-java|cuvs-lucene)/([^/]+/)?src/.*\.java$
+        files: ^(java|examples/java)/(cuvs-java|cuvs-lucene)/([^/]+/){0,2}src/.*\.java$
         exclude: .*/panama/.*
         language: script
         verbose: true

build.sh

@@ -81,15 +81,15 @@ RUST_BUILD_DIR=${REPODIR}/rust/target
 JAVA_INTERNAL_BUILD_DIR=${REPODIR}/java/internal/build
 JAVA_BUILD_DIR=${REPODIR}/java/cuvs-java/target
 JAVA_PANAMA_DIR=${REPODIR}/java/cuvs-java/src/main/java22/com/nvidia/cuvs/internal/panama
-LUCENE_BUILD_DIR=${REPODIR}/java/cuvs-lucene/target
+LUCENE_BUILD_DIRS=("${REPODIR}"/java/cuvs-lucene/target "${REPODIR}"/java/cuvs-lucene/*/target)
 BUILD_DIRS=(
     "${LIBCUVS_BUILD_DIR}"
     "${PYTHON_BUILD_DIR}"
     "${RUST_BUILD_DIR}"
     "${JAVA_INTERNAL_BUILD_DIR}"
     "${JAVA_BUILD_DIR}"
     "${JAVA_PANAMA_DIR}"
-    "${LUCENE_BUILD_DIR}"
+    "${LUCENE_BUILD_DIRS[@]}"
 )
 
 # Set defaults for vars modified by flags to this script

ci/checks/run_spotless.sh

@@ -15,7 +15,7 @@ set -euo pipefail
 
 # Keep these in sync with the spotless-fmt hook's 'files'/'exclude' entries in
 # .pre-commit-config.yaml.
-JAVA_SRC_PATTERN='^(java|examples/java)/(cuvs-java|cuvs-lucene)/([^/]+/)?src/.*\.java$'
+JAVA_SRC_PATTERN='^(java|examples/java)/(cuvs-java|cuvs-lucene)/([^/]+/){0,2}src/.*\.java$'
 JAVA_SRC_EXCLUDE='.*/panama/.*'
 
 java_sources_modified() {

ci/release/update-version.sh

@@ -172,7 +172,7 @@ done
 NEXT_FULL_JAVA_TAG="${NEXT_SHORT_TAG}.${PATCH_PEP440}"
 sed_runner "s/VERSION=\".*\"/VERSION=\"${NEXT_FULL_JAVA_TAG}\"/g" java/build.sh
 sed_runner "s/VERSION=\".*\"/VERSION=\"${NEXT_FULL_JAVA_TAG}\"/g" java/cuvs-lucene/build.sh
-for FILE in java/*/pom.xml java/cuvs-lucene/bench/pom.xml examples/java/cuvs-java/pom.xml examples/java/cuvs-lucene/pom.xml; do
+for FILE in java/*/pom.xml java/cuvs-lucene/lucene-*/pom.xml java/cuvs-lucene/relocation/pom.xml java/cuvs-lucene/bench/pom.xml examples/java/cuvs-java/pom.xml examples/java/cuvs-lucene/pom.xml; do
   sed_runner "/<!--CUVS_JAVA#VERSION_UPDATE_MARKER_START-->.*<!--CUVS_JAVA#VERSION_UPDATE_MARKER_END-->/s//<!--CUVS_JAVA#VERSION_UPDATE_MARKER_START--><version>${NEXT_FULL_JAVA_TAG}<\/version><!--CUVS_JAVA#VERSION_UPDATE_MARKER_END-->/g" "${FILE}"
 done
 

ci/test_lucene.sh

@@ -79,11 +79,11 @@ pushd java/cuvs-lucene
 mvn --batch-mode install:install-file -Dfile="${CUVS_JAVA_JARS[0]}" -DpomFile="${CUVS_JAVA_POM}"
 popd
 
-rapids-logger "Restore the amd64-built cuvs-lucene target/ directory so Maven can run the tests without recompiling"
+rapids-logger "Restore the amd64-built cuvs-lucene target/ directories so Maven can run the tests without recompiling"
 
-rm -rf java/cuvs-lucene/target
-mkdir -p java/cuvs-lucene/target
-cp -a "${CUVS_LUCENE_DIR}/." java/cuvs-lucene/target/
+# The artifact holds one <module>/target/ per module.
+rm -rf java/cuvs-lucene/*/target
+cp -a "${CUVS_LUCENE_DIR}/." java/cuvs-lucene/
 
 EXITCODE=0
 trap "EXITCODE=1" ERR
@@ -93,9 +93,9 @@ rapids-logger "Run cuvs-lucene tests against the amd64-built classes"
 
 # -Dskip.compile activates the pom's "skip-compile" profile, which disables all
 # compilation for this run, forcing test to use amd64-compiled jar instead of
-# local code.
+# local code. --fail-at-end tests every module even if an earlier one fails.
 pushd java/cuvs-lucene
-mvn --batch-mode test -Dskip.compile=true
+mvn --batch-mode --fail-at-end test -Dskip.compile=true
 popd
 
 rapids-logger "Test script exiting with value: $EXITCODE"

ci/test_lucene_jdk21.sh

@@ -0,0 +1,93 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+set -euo pipefail
+
+# Runs cuvs-lucene's test suite on JDK 21 against the classes from a prior amd64 build job,
+# without recompiling. cuvs-lucene is compiled with --release 21 so that applications on JDK 21
+# (Solr, for example) can load it, but cuvs-java needs JDK 22+ to reach the GPU. On JDK 21 the
+# GPU tests skip themselves, and what runs checks that the codecs still load, read existing
+# indexes and fall back to the CPU. None of that depends on the GPU or on the Lucene release, so
+# this runs on a CPU node and tests only the newest Lucene module.
+
+CUVS_JAVA_ARTIFACT="cuvs-java-cuda${RAPIDS_CUDA_VERSION}"
+CUVS_LUCENE_ARTIFACT="cuvs-lucene-cuda${RAPIDS_CUDA_VERSION}"
+
+if [ -e "/opt/conda/etc/profile.d/conda.sh" ]; then
+  . /opt/conda/etc/profile.d/conda.sh
+fi
+
+rapids-logger "Configuring conda strict channel priority"
+conda config --set channel_priority strict
+
+rapids-logger "Downloading artifacts from previous jobs"
+CUVS_JAVA_DIR=$(rapids-download-from-github "${CUVS_JAVA_ARTIFACT}")
+CUVS_LUCENE_DIR=$(rapids-download-from-github "${CUVS_LUCENE_ARTIFACT}")
+
+rapids-logger "Generate JDK 21 testing dependencies"
+
+ENV_YAML_DIR="$(mktemp -d)"
+
+rapids-dependency-file-generator \
+  --output conda \
+  --file-key java_jdk21 \
+  --matrix "" | tee "${ENV_YAML_DIR}/env.yaml"
+
+rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n java_jdk21
+
+# Temporarily allow unbound variables for conda activation.
+set +u
+conda activate java_jdk21
+set -u
+
+rapids-print-env
+
+# Maven, and the JVM surefire forks for the tests, run on the JDK of JAVA_HOME rather than the java
+# on PATH, so check the one Maven reports.
+MAVEN_JAVA_VERSION="$(cd java/cuvs-lucene && mvn --batch-mode -v | sed -n 's/^Java version: \([0-9]*\).*/\1/p')"
+if [ "${MAVEN_JAVA_VERSION}" != "21" ]; then
+  echo "Expected Maven to run on JDK 21, but it runs on JDK '${MAVEN_JAVA_VERSION}'" >&2
+  exit 1
+fi
+
+rapids-logger "Install the amd64-built cuvs-java artifact into the local Maven repository"
+
+CUVS_JAVA_POM="${CUVS_JAVA_DIR}/pom.xml"
+if [ ! -f "${CUVS_JAVA_POM}" ]; then
+  echo "Could not find pom.xml in the cuvs-java artifact at ${CUVS_JAVA_DIR}" >&2
+  exit 1
+fi
+
+mapfile -t CUVS_JAVA_JARS < <(find "${CUVS_JAVA_DIR}" -maxdepth 1 -name 'cuvs-java-*.jar' \
+  ! -name '*-sources.jar' ! -name '*-javadoc.jar' ! -name '*-tests.jar' ! -name '*-cuda*.jar')
+if [ "${#CUVS_JAVA_JARS[@]}" -ne 1 ]; then
+  echo "Expected exactly one cuvs-java jar in ${CUVS_JAVA_DIR}, found: ${CUVS_JAVA_JARS[*]:-none}" >&2
+  exit 1
+fi
+
+# cd is needed to pick up pom.xml in order to avoid rate limit of main maven repo
+pushd java/cuvs-lucene
+mvn --batch-mode install:install-file -Dfile="${CUVS_JAVA_JARS[0]}" -DpomFile="${CUVS_JAVA_POM}"
+popd
+
+rapids-logger "Restore the amd64-built cuvs-lucene target/ directories so Maven can run the tests without recompiling"
+
+rm -rf java/cuvs-lucene/*/target
+cp -a "${CUVS_LUCENE_DIR}/." java/cuvs-lucene/
+
+EXITCODE=0
+trap "EXITCODE=1" ERR
+set +e
+
+rapids-logger "Run cuvs-lucene tests on JDK 21 against the amd64-built classes"
+
+# -Dskip.compile disables all compilation, see ci/test_lucene.sh.
+pushd java/cuvs-lucene
+NEWEST_MODULE="$(find . -maxdepth 1 -type d -name 'lucene-*' -printf '%f\n' | sort -V | tail -1)"
+rapids-logger "Testing ${NEWEST_MODULE}"
+mvn --batch-mode test -Dskip.compile=true -pl "${NEWEST_MODULE}"
+popd
+
+rapids-logger "Test script exiting with value: $EXITCODE"
+exit ${EXITCODE}

dependencies.yaml

@@ -120,6 +120,10 @@ files:
       - depends_on_nccl
       - java
       - rapids_build
+  java_jdk21:
+    output: none
+    includes:
+      - java_jdk21
   py_build_libcuvs:
     output: pyproject
     pyproject_dir: python/libcuvs
@@ -658,6 +662,13 @@ dependencies:
           - libboost-devel
           - maven>=3.10.0,<3.11.0
           - *openjdk
+  java_jdk21:
+    common:
+      - output_types: conda
+        packages:
+          - maven>=3.10.0,<3.11.0
+          # The oldest JDK cuvs-lucene supports; cuvs-java needs 22+ to use the GPU.
+          - openjdk>=21,<22
   bench:
     common:
       - output_types: [conda, pyproject, requirements, constraints]

examples/java/cuvs-lucene/README.md

@@ -15,6 +15,9 @@ First build `cuvs-lucene` and install it into your local Maven repository, as de
 ./build.sh libcuvs java lucene
 ```
 
+The examples use Lucene 10.2 and the matching `cuvs-lucene-10.2` artifact, set by the `lucene.version` and
+`cuvs.lucene.artifactId` properties in `pom.xml`.
+
 Then return to this directory:
 
 ```sh

examples/java/cuvs-lucene/pom.xml

@@ -19,6 +19,8 @@
     <maven.compiler.source>22</maven.compiler.source>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+    <lucene.version>10.2.0</lucene.version>
+    <cuvs.lucene.artifactId>cuvs-lucene-10.2</cuvs.lucene.artifactId>
   </properties>
 
     <repositories>
@@ -79,18 +81,18 @@
     <dependency>
         <groupId>org.apache.lucene</groupId>
         <artifactId>lucene-core</artifactId>
-        <version>10.2.0</version>
+        <version>${lucene.version}</version>
     </dependency>
     <dependency>
         <groupId>org.apache.lucene</groupId>
         <artifactId>lucene-codecs</artifactId>
-        <version>10.2.0</version>
+        <version>${lucene.version}</version>
         <scope>test</scope>
     </dependency>
     <dependency>
         <groupId>org.apache.lucene</groupId>
         <artifactId>lucene-backward-codecs</artifactId>
-        <version>10.2.0</version>
+        <version>${lucene.version}</version>
     </dependency>
     <dependency>
         <groupId>commons-io</groupId>
@@ -104,7 +106,7 @@
     </dependency>
     <dependency>
       <groupId>com.nvidia.cuvs.lucene</groupId>
-      <artifactId>cuvs-lucene</artifactId>
+      <artifactId>${cuvs.lucene.artifactId}</artifactId>
       <!--CUVS_JAVA#VERSION_UPDATE_MARKER_START--><version>26.12.0</version><!--CUVS_JAVA#VERSION_UPDATE_MARKER_END-->
     </dependency>
   </dependencies>

examples/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/examples/AcceleratedHnswExample.java

@@ -8,7 +8,7 @@
 import static org.apache.lucene.index.VectorSimilarityFunction.EUCLIDEAN;
 
 import com.nvidia.cuvs.lucene.AcceleratedHNSWParams;
-import com.nvidia.cuvs.lucene.Lucene101AcceleratedHNSWCodec;
+import com.nvidia.cuvs.lucene.CuVSCodecs;
 import java.io.File;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -46,7 +46,7 @@ public class AcceleratedHnswExample {
   public static void main(String[] args) throws Exception {
 
     AcceleratedHNSWParams params = new AcceleratedHNSWParams.Builder().build();
-    Codec codec = new Lucene101AcceleratedHNSWCodec(params);
+    Codec codec = CuVSCodecs.acceleratedHNSW(params);
     IndexWriterConfig config = new IndexWriterConfig().setCodec(codec).setUseCompoundFile(false);
 
     random = new Random(222);

examples/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/examples/IndexAndSearchonGPUExample.java

@@ -7,7 +7,7 @@
 import static com.nvidia.cuvs.lucene.examples.Utils.generateDataset;
 import static org.apache.lucene.index.VectorSimilarityFunction.EUCLIDEAN;
 
-import com.nvidia.cuvs.lucene.CuVS2510GPUSearchCodec;
+import com.nvidia.cuvs.lucene.CuVSCodecs;
 import com.nvidia.cuvs.lucene.GPUKnnFloatVectorQuery;
 import com.nvidia.cuvs.lucene.GPUSearchParams;
 import com.nvidia.cuvs.spi.CuVSProvider;
@@ -54,7 +54,7 @@ public static void main(String[] args) throws Exception {
     CuVSProvider.provider().enableRMMAsyncMemory();
 
     GPUSearchParams params = new GPUSearchParams.Builder().build();
-    Codec codec = new CuVS2510GPUSearchCodec(params);
+    Codec codec = CuVSCodecs.gpuSearch(params);
     IndexWriterConfig config = new IndexWriterConfig().setCodec(codec).setUseCompoundFile(false);
 
     random = new Random(222);

fern/docs.yml

@@ -571,6 +571,8 @@ navigation:
             path: "./pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-acceleratedhnswutils.md"
           - page: "CagraIndexParamsFactory"
             path: "./pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-cagraindexparamsfactory.md"
+          - page: "CuVSCodecs"
+            path: "./pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-cuvscodecs.md"
           - page: "FilterBitsetCacheConfig"
             path: "./pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-filterbitsetcacheconfig.md"
           - page: "FilterCuVSServiceProvider"
@@ -579,8 +581,6 @@ navigation:
             path: "./pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-gpuindex.md"
           - page: "GPUSearchParams"
             path: "./pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-gpusearchparams.md"
-          - page: "LuceneProvider"
-            path: "./pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-luceneprovider.md"
           - page: "ThreadLocalCuVSResourcesProvider"
             path: "./pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-threadlocalcuvsresourcesprovider.md"
           - page: "Utils"