diff --git a/docs/docs/multimodal-table/global-index/manage-indexes.mdx b/docs/docs/multimodal-table/global-index/manage-indexes.mdx index f7acd7d4e102..ad5d669b30f9 100644 --- a/docs/docs/multimodal-table/global-index/manage-indexes.mdx +++ b/docs/docs/multimodal-table/global-index/manage-indexes.mdx @@ -463,6 +463,43 @@ sorting or guarantee lower elapsed time for every workload. Composite and Multivalue indexes retain their existing build paths. Evaluate CPU, network and spill costs on a representative cluster before enabling it for a workload. +## Upgrade Java Index Plugins + +Support for multiple `global-index.external-path` directories changes the Java +global index writer interface from file names to allocated full paths. Plugins +compiled against the previous interface must be upgraded together with Paimon. +This applies even when the option is unset or contains only one directory. + +Deploy the Paimon engine connector/runtime and Paimon-provided Java index plugins, +including `paimon-lumina`, from the same Paimon release. For snapshot or custom +Paimon builds, build them from the same source revision; matching `SNAPSHOT` +version strings alone do not ensure compatibility. Replace the old plugin JARs +in the engine's classpath or plugin directory and restart the processes that +load them. +Updating only the `lumina-jni` dependency does not update Paimon's Java adapter. + +Custom Java index plugins must make these changes before being rebuilt against +the target Paimon version: + +- Replace `GlobalIndexFileWriter.newFileName(prefix)` with + `newFilePath(prefix)`, which allocates and returns a full path. +- Pass that path unchanged to `newOutputStream(filePath)`. Do not prepend + another directory or select an external directory again. +- Pass the same full path as the first argument of `ResultEntry`. Its + constructor signature is unchanged, but this argument now represents a full + path rather than a file name. +- Replace `ResultEntry.fileName()` with `filePath()` where the result is read; + extract the basename only when a file name is required. + +There is no compatibility adapter for the previous Java writer interface. +Using an old plugin JAR with the new runtime can fail with `NoSuchMethodError` +for `GlobalIndexFileWriter.newFileName`. Recompiling unchanged plugin source +against the new interface is insufficient; migrate its calls first. + +This Java interface migration does not change index file contents or the +persisted `IndexFileMeta` format. It does not by itself require rebuilding +existing indexes. + ## Shared Options These table options affect global index build and read behavior: @@ -475,7 +512,7 @@ These table options affect global index build and read behavior: | `scalar-index.search-mode` | `fast` | Search mode for BTree, Bitmap, Multivalue, FM, and R-tree bbox queries. | | `vector-index.search-mode` | `fast` | Search mode for vector queries. | | `full-text-index.search-mode` | `fast` | Search mode for full-text queries. | -| `global-index.external-path` | Not set | Root directory for global index files. If not set, files are stored under the table index directory. | +| `global-index.external-path` | Not set | Comma-separated root directories for global index files, for example `s3://bucket-a/index,s3://bucket-b/index`. New files use round-robin selection independently of data-file external path settings. Each file's actual path is saved in metadata, so changing this option does not relocate existing files. If not set, files are stored under the table index directory. PyPaimon index builds support only a single external directory and reject comma-separated values in table or build options. | | `global-index.column-update-action` | `THROW_ERROR` | Update policy: `THROW_ERROR`, `DROP_PARTITION_INDEX`, or `IGNORE`. See [Update Indexed Columns](#update-indexed-columns) for refresh requirements and engine differences. | | `sorted-index.records-per-file` | `25000000` | Expected number of records per sorted global index file for BTree, Bitmap, and Multivalue builds. Falls back to `sorted-index.records-per-range`, then `btree-index.records-per-range`. | | `sorted-index.build.max-parallelism` | `4096` | Maximum Flink or Spark parallelism for building sorted global indexes. | diff --git a/docs/generated/core_configuration.html b/docs/generated/core_configuration.html index 4bbc4e082cc2..d3c1538abc61 100644 --- a/docs/generated/core_configuration.html +++ b/docs/generated/core_configuration.html @@ -942,7 +942,7 @@
global-index.external-path
(none) String - Global index root directory, if not set, the global index files will be stored under the <table-root-directory>/index. + Comma-separated global index root directories, new global index files are distributed in round-robin order. If not set, global index files are stored under <table-root-directory>/index.
global-index.filter.refine-from-data
diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java index 06d94253fcd3..872026d81686 100644 --- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java @@ -2908,7 +2908,8 @@ public String toString() { .stringType() .noDefaultValue() .withDescription( - "Global index root directory, if not set, the global index files will be stored under the /index."); + "Comma-separated global index root directories, new global index files are distributed " + + "in round-robin order. If not set, global index files are stored under /index."); public static final ConfigOption GLOBAL_INDEX_COLUMN_UPDATE_ACTION = @@ -4188,17 +4189,19 @@ public boolean indexFileInDataFileDir() { return options.get(INDEX_FILE_IN_DATA_FILE_DIR); } - public Path globalIndexExternalPath() { + public List globalIndexExternalPaths() { String pathString = options.get(GLOBAL_INDEX_EXTERNAL_PATH); if (pathString == null || pathString.isEmpty()) { - return null; + return Collections.emptyList(); } - Path path = new Path(pathString); - String scheme = path.toUri().getScheme(); - if (scheme == null) { - throw new IllegalArgumentException("scheme should not be null: " + path); + List paths = new ArrayList<>(); + for (String value : pathString.split(",")) { + Path path = new Path(value.trim()); + checkArgument(path.toUri().getScheme() != null, "scheme should not be null: %s", path); + paths.add(path); } - return path; + checkArgument(!paths.isEmpty(), "global index external paths is empty"); + return paths; } public GlobalIndexColumnUpdateAction globalIndexColumnUpdateAction() { @@ -4473,6 +4476,49 @@ public String dataFileExternalPaths() { return options.get(DATA_FILE_EXTERNAL_PATHS); } + /** Returns all configured external data paths without applying the write strategy. */ + public List dataFileExternalPathList() { + String externalPaths = dataFileExternalPaths(); + if (externalPaths == null || externalPaths.isEmpty()) { + return Collections.emptyList(); + } + List paths = new ArrayList<>(); + for (String pathString : externalPaths.split(",")) { + paths.add(new Path(pathString.trim())); + } + return paths; + } + + /** Returns external data paths eligible for writing under the configured strategy. */ + public List createExternalPaths() { + String externalPaths = dataFileExternalPaths(); + ExternalPathStrategy strategy = externalPathStrategy(); + if (externalPaths == null + || externalPaths.isEmpty() + || strategy == ExternalPathStrategy.NONE) { + return Collections.emptyList(); + } + + String specificFS = externalSpecificFS(); + List paths = new ArrayList<>(); + for (Path path : dataFileExternalPathList()) { + String scheme = path.toUri().getScheme(); + checkArgument(scheme != null, "scheme should not be null: %s", path); + if (strategy == ExternalPathStrategy.SPECIFIC_FS) { + checkArgument( + specificFS != null, + "External path specificFS should not be null when strategy is specificFS."); + if (scheme.equalsIgnoreCase(specificFS)) { + paths.add(path); + } + } else { + paths.add(path); + } + } + checkArgument(!paths.isEmpty(), "External paths should not be empty"); + return paths; + } + public ExternalPathStrategy externalPathStrategy() { return options.get(DATA_FILE_EXTERNAL_PATHS_STRATEGY); } diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/RoundRobinExternalPathProvider.java b/paimon-common/src/main/java/org/apache/paimon/fs/RoundRobinExternalPathProvider.java index e9ef9fa431ef..42e379ca5e4a 100644 --- a/paimon-common/src/main/java/org/apache/paimon/fs/RoundRobinExternalPathProvider.java +++ b/paimon-common/src/main/java/org/apache/paimon/fs/RoundRobinExternalPathProvider.java @@ -18,6 +18,8 @@ package org.apache.paimon.fs; +import javax.annotation.Nullable; + import java.util.List; import java.util.concurrent.ThreadLocalRandom; @@ -25,11 +27,12 @@ public class RoundRobinExternalPathProvider implements ExternalPathProvider { private final List externalTablePaths; - private final Path relativeBucketPath; + @Nullable private final Path relativeBucketPath; private int position; - public RoundRobinExternalPathProvider(List externalTablePaths, Path relativeBucketPath) { + public RoundRobinExternalPathProvider( + List externalTablePaths, @Nullable Path relativeBucketPath) { this.externalTablePaths = externalTablePaths; this.relativeBucketPath = relativeBucketPath; this.position = ThreadLocalRandom.current().nextInt(externalTablePaths.size()); @@ -46,6 +49,10 @@ public Path getNextExternalDataPath(String fileName) { if (position == externalTablePaths.size()) { position = 0; } - return new Path(new Path(externalTablePaths.get(position), relativeBucketPath), fileName); + Path parent = externalTablePaths.get(position); + if (relativeBucketPath != null) { + parent = new Path(parent, relativeBucketPath); + } + return new Path(parent, fileName); } } diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/CentroidVectorGlobalIndexer.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/CentroidVectorGlobalIndexer.java index fb2c845198a7..ecd5eb4fbdb1 100644 --- a/paimon-common/src/main/java/org/apache/paimon/globalindex/CentroidVectorGlobalIndexer.java +++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/CentroidVectorGlobalIndexer.java @@ -43,7 +43,7 @@ static boolean centroidSharding(String indexType, Options options) { /** Persist the trained router and return the model entry with its underlying index metadata. */ ResultEntry writeRoutingModel( - GlobalIndexFileWriter io, String name, byte[] payload, String modelId) + GlobalIndexFileWriter io, String filePath, byte[] payload, String modelId) throws IOException; /** Open an independently trained local index for the assigned centroid. */ diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/ResultEntry.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/ResultEntry.java index 0d6200330cf2..7a7ffd3aa7d5 100644 --- a/paimon-common/src/main/java/org/apache/paimon/globalindex/ResultEntry.java +++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/ResultEntry.java @@ -23,21 +23,22 @@ /** Write result meta. */ public class ResultEntry { - private final String fileName; + /** Full file path returned by the writer. */ + private final String filePath; /** Number of source rows covered by this index file, including null or skipped rows. */ private final long rowCount; @Nullable private final byte[] meta; - public ResultEntry(String fileName, long rowCount, @Nullable byte[] meta) { - this.fileName = fileName; + public ResultEntry(String filePath, long rowCount, @Nullable byte[] meta) { + this.filePath = filePath; this.rowCount = rowCount; this.meta = meta; } - public String fileName() { - return fileName; + public String filePath() { + return filePath; } public long rowCount() { diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/bitmap/BitmapGlobalIndexWriter.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/bitmap/BitmapGlobalIndexWriter.java index f13bf0055585..85f7a8ded96e 100644 --- a/paimon-common/src/main/java/org/apache/paimon/globalindex/bitmap/BitmapGlobalIndexWriter.java +++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/bitmap/BitmapGlobalIndexWriter.java @@ -51,7 +51,7 @@ public class BitmapGlobalIndexWriter implements GlobalIndexSingleColumnWriter, C private final RoaringNavigableMap64 nullRows; private final RoaringNavigableMap64 nonNullRows; - private String fileName; + private String filePath; private PositionOutputStream outputStream; private BitmapGlobalIndexFormat.StreamingWriter streamingWriter; private long rowCount; @@ -119,7 +119,7 @@ public List finish() { lastKey == null ? null : keySerializer.serialize(lastKey), !nullRows.isEmpty()) .serialize(); - return Collections.singletonList(new ResultEntry(fileName, rowCount, meta)); + return Collections.singletonList(new ResultEntry(filePath, rowCount, meta)); } @Override @@ -149,8 +149,8 @@ private void flushCurrentBitmap() { private BitmapGlobalIndexFormat.StreamingWriter streamingWriter() throws IOException { if (streamingWriter == null) { - fileName = fileWriter.newFileName(BitmapGlobalIndexerFactory.IDENTIFIER); - outputStream = fileWriter.newOutputStream(fileName); + filePath = fileWriter.newFilePath(BitmapGlobalIndexerFactory.IDENTIFIER); + outputStream = fileWriter.newOutputStream(filePath); streamingWriter = new BitmapGlobalIndexFormat.StreamingWriter( outputStream, dictionaryBlockSize, compressionFactory); diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/bitmap/MultiValueBitmapIndexWriter.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/bitmap/MultiValueBitmapIndexWriter.java index beda6cfaba4b..9d1dab7bbb13 100644 --- a/paimon-common/src/main/java/org/apache/paimon/globalindex/bitmap/MultiValueBitmapIndexWriter.java +++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/bitmap/MultiValueBitmapIndexWriter.java @@ -47,7 +47,7 @@ public class MultiValueBitmapIndexWriter implements GlobalIndexSingleColumnWrite @Nullable private final BlockCompressionFactory compressionFactory; private final RoaringNavigableMap64 currentBitmap = new RoaringNavigableMap64(); - private String fileName; + private String filePath; private PositionOutputStream outputStream; private BitmapGlobalIndexFormat.StreamingWriter streamingWriter; private BitmapGlobalIndexFormat.SerializedKey currentKey; @@ -122,7 +122,7 @@ public List finish(long sourceRowCount) { } byte[] meta = new SortedIndexFileMeta(firstKey, lastKey, false).serialize(); - return Collections.singletonList(new ResultEntry(fileName, sourceRowCount, meta)); + return Collections.singletonList(new ResultEntry(filePath, sourceRowCount, meta)); } @Override @@ -150,8 +150,8 @@ private void flushCurrentBitmap() { private BitmapGlobalIndexFormat.StreamingWriter streamingWriter() throws IOException { if (streamingWriter == null) { - fileName = fileWriter.newFileName(MultiValueGlobalIndexerFactory.IDENTIFIER); - outputStream = fileWriter.newOutputStream(fileName); + filePath = fileWriter.newFilePath(MultiValueGlobalIndexerFactory.IDENTIFIER); + outputStream = fileWriter.newOutputStream(filePath); streamingWriter = new BitmapGlobalIndexFormat.StreamingWriter( outputStream, dictionaryBlockSize, compressionFactory); diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/btree/BTreeIndexWriter.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/btree/BTreeIndexWriter.java index fecf600f56ac..dcbd6fcb854b 100644 --- a/paimon-common/src/main/java/org/apache/paimon/globalindex/btree/BTreeIndexWriter.java +++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/btree/BTreeIndexWriter.java @@ -74,7 +74,7 @@ */ public class BTreeIndexWriter implements GlobalIndexSingleColumnWriter, Closeable { - private final String fileName; + private final String filePath; private final PositionOutputStream out; private boolean closed; @@ -127,8 +127,8 @@ public BTreeIndexWriter( || fileVersion > BTreeFileFooter.MAX_SUPPORTED_VERSION) { throw new IllegalArgumentException("Unsupported BTree index version: " + fileVersion); } - this.fileName = indexFileWriter.newFileName(BTreeGlobalIndexerFactory.IDENTIFIER); - this.out = indexFileWriter.newOutputStream(this.fileName); + this.filePath = indexFileWriter.newFilePath(BTreeGlobalIndexerFactory.IDENTIFIER); + this.out = indexFileWriter.newOutputStream(this.filePath); try { this.keySerializer = keySerializer; this.comparator = keySerializer.createComparator(); @@ -230,7 +230,7 @@ public List finish() { lastKey == null ? null : keySerializer.serialize(lastKey), nullBitmap.initialized()) .serialize(); - return Collections.singletonList(new ResultEntry(fileName, rowCount, metaBytes)); + return Collections.singletonList(new ResultEntry(filePath, rowCount, metaBytes)); } @Nullable diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/fmindex/FMGlobalIndexWriter.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/fmindex/FMGlobalIndexWriter.java index 4c18b6da6ffd..0dd8b479fb91 100644 --- a/paimon-common/src/main/java/org/apache/paimon/globalindex/fmindex/FMGlobalIndexWriter.java +++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/fmindex/FMGlobalIndexWriter.java @@ -48,7 +48,7 @@ public class FMGlobalIndexWriter implements GlobalIndexSingleColumnWriter, Close private CharBuilder text = new CharBuilder(); private boolean[] nullRows = new boolean[128]; - @Nullable private String fileName; + @Nullable private String filePath; @Nullable private PositionOutputStream stream; @Nullable private DataOutputStream output; private long firstRowId; @@ -131,7 +131,7 @@ public List finish() { return Collections.emptyList(); } Preconditions.checkState( - stream != null && output != null && fileName != null, + stream != null && output != null && filePath != null, "FM index container output is missing."); byte[] indexMeta = FMIndexFile.writeIndexMeta(firstRowId, totalRowCount, partitions); FMIndexFile.BlockInfo directory = @@ -140,7 +140,7 @@ public List finish() { FMIndexFile.writeContainerFooter( output, directory, firstRowId, totalRowCount, partitions.size()); closeOutput(); - return Collections.singletonList(new ResultEntry(fileName, totalRowCount, indexMeta)); + return Collections.singletonList(new ResultEntry(filePath, totalRowCount, indexMeta)); } catch (IOException e) { closeOutputQuietly(); throw new RuntimeException("Failed to finish FM global index container.", e); @@ -290,8 +290,8 @@ private void ensureOutput() throws IOException { if (output != null) { return; } - fileName = fileWriter.newFileName("fm"); - stream = fileWriter.newOutputStream(fileName); + filePath = fileWriter.newFilePath("fm"); + stream = fileWriter.newOutputStream(filePath); output = new DataOutputStream(stream); } diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/io/GlobalIndexFileWriter.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/io/GlobalIndexFileWriter.java index cbc0dd4ea2a9..2809d955e962 100644 --- a/paimon-common/src/main/java/org/apache/paimon/globalindex/io/GlobalIndexFileWriter.java +++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/io/GlobalIndexFileWriter.java @@ -25,7 +25,9 @@ /** Global index writer io operators. */ public interface GlobalIndexFileWriter { - String newFileName(String prefix); + /** Allocates and returns the full path for a new index file. */ + String newFilePath(String prefix); - PositionOutputStream newOutputStream(String fileName) throws IOException; + /** Opens the allocated file path without selecting another location. */ + PositionOutputStream newOutputStream(String filePath) throws IOException; } diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/rtree/RTreeGlobalIndexWriter.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/rtree/RTreeGlobalIndexWriter.java index 52db12fe2f44..b15c2b24873a 100644 --- a/paimon-common/src/main/java/org/apache/paimon/globalindex/rtree/RTreeGlobalIndexWriter.java +++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/rtree/RTreeGlobalIndexWriter.java @@ -139,8 +139,8 @@ private void flush() { firstRow, lastRow, bounds); - String name = files.newFileName("rtree"); - try (PositionOutputStream output = files.newOutputStream(name)) { + String filePath = files.newFilePath("rtree"); + try (PositionOutputStream output = files.newOutputStream(filePath)) { output.write(meta.bytes); List level = entries; int depth = 0; @@ -175,7 +175,7 @@ private void flush() { finished = true; throw new UncheckedIOException("Failed to write R-tree index.", e); } - results.add(new ResultEntry(name, meta.rowCount(), meta.bytes)); + results.add(new ResultEntry(filePath, meta.rowCount(), meta.bytes)); entries.clear(); nulls.clear(); nonNulls.clear(); diff --git a/paimon-common/src/test/java/org/apache/paimon/fs/RoundRobinExternalPathProviderTest.java b/paimon-common/src/test/java/org/apache/paimon/fs/RoundRobinExternalPathProviderTest.java new file mode 100644 index 000000000000..678b873fb06e --- /dev/null +++ b/paimon-common/src/test/java/org/apache/paimon/fs/RoundRobinExternalPathProviderTest.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.fs; + +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for {@link RoundRobinExternalPathProvider}. */ +public class RoundRobinExternalPathProviderTest { + + @Test + public void testWithoutRelativeBucketPath() { + RoundRobinExternalPathProvider provider = + new RoundRobinExternalPathProvider( + Arrays.asList( + new Path("s3://bucket-a/index"), new Path("s3://bucket-b/index")), + null); + + Path first = provider.getNextExternalDataPath("index-file"); + Path second = provider.getNextExternalDataPath("index-file"); + assertThat(Arrays.asList(first, second)) + .containsExactlyInAnyOrder( + new Path("s3://bucket-a/index/index-file"), + new Path("s3://bucket-b/index/index-file")); + assertThat(provider.getNextExternalDataPath("index-file")).isEqualTo(first); + assertThat(provider.getNextExternalDataPath("index-file")).isEqualTo(second); + } + + @Test + public void testWithRelativeBucketPath() { + RoundRobinExternalPathProvider provider = + new RoundRobinExternalPathProvider( + Collections.singletonList(new Path("s3://bucket-a/data")), + new Path("dt=2026-11-18/bucket-3")); + + assertThat(provider.getNextExternalDataPath("data-file.parquet")) + .isEqualTo(new Path("s3://bucket-a/data/dt=2026-11-18/bucket-3/data-file.parquet")); + } +} diff --git a/paimon-common/src/test/java/org/apache/paimon/globalindex/SortedIndexFileMetaTest.java b/paimon-common/src/test/java/org/apache/paimon/globalindex/SortedIndexFileMetaTest.java index 9973ee4f3650..229138845b84 100644 --- a/paimon-common/src/test/java/org/apache/paimon/globalindex/SortedIndexFileMetaTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/globalindex/SortedIndexFileMetaTest.java @@ -149,15 +149,15 @@ private List writeVarCharIndex(Object... keys) throws IOException { GlobalIndexFileWriter fileWriter = new GlobalIndexFileWriter() { @Override - public String newFileName(String prefix) { - return "test-btree-" + prefix; + public String newFilePath(String prefix) { + return new Path(new Path(tempPath.toUri()), "test-btree-" + prefix) + .toString(); } @Override - public PositionOutputStream newOutputStream(String fileName) + public PositionOutputStream newOutputStream(String filePath) throws IOException { - return fileIO.newOutputStream( - new Path(new Path(tempPath.toUri()), fileName), true); + return fileIO.newOutputStream(new Path(filePath), true); } }; GlobalIndexSingleColumnWriter indexWriter = diff --git a/paimon-common/src/test/java/org/apache/paimon/globalindex/bitmap/LazyFilteredBitmapIndexReaderTest.java b/paimon-common/src/test/java/org/apache/paimon/globalindex/bitmap/LazyFilteredBitmapIndexReaderTest.java index 389e1a7d7204..67877c0174bd 100644 --- a/paimon-common/src/test/java/org/apache/paimon/globalindex/bitmap/LazyFilteredBitmapIndexReaderTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/globalindex/bitmap/LazyFilteredBitmapIndexReaderTest.java @@ -83,14 +83,15 @@ public void setUp() { fileWriter = new GlobalIndexFileWriter() { @Override - public String newFileName(String prefix) { - return prefix + "-" + UUID.randomUUID() + ".index"; + public String newFilePath(String prefix) { + return new Path(basePath, prefix + "-" + UUID.randomUUID() + ".index") + .toString(); } @Override - public PositionOutputStream newOutputStream(String fileName) + public PositionOutputStream newOutputStream(String filePath) throws IOException { - return fileIO.newOutputStream(new Path(basePath, fileName), true); + return fileIO.newOutputStream(new Path(filePath), true); } }; fileReader = meta -> fileIO.newInputStream(meta.filePath()); @@ -187,12 +188,12 @@ public void testFlushesCompletedBitmapBeforeFinish() throws Exception { GlobalIndexFileWriter streamingFileWriter = new GlobalIndexFileWriter() { @Override - public String newFileName(String prefix) { - return prefix + ".index"; + public String newFilePath(String prefix) { + return "memory://test/" + prefix + ".index"; } @Override - public PositionOutputStream newOutputStream(String fileName) { + public PositionOutputStream newOutputStream(String filePath) { ByteArrayPositionOutputStream stream = new ByteArrayPositionOutputStream(); output.set(stream); return stream; @@ -214,12 +215,12 @@ public void testClosesWriterAfterWriteFailure() throws Exception { GlobalIndexFileWriter streamingFileWriter = new GlobalIndexFileWriter() { @Override - public String newFileName(String prefix) { - return prefix + ".index"; + public String newFilePath(String prefix) { + return "memory://test/" + prefix + ".index"; } @Override - public PositionOutputStream newOutputStream(String fileName) { + public PositionOutputStream newOutputStream(String filePath) { ByteArrayPositionOutputStream stream = new ByteArrayPositionOutputStream(); output.set(stream); return stream; @@ -263,7 +264,7 @@ public void testLogicalOrderForNumericKeys() throws Exception { } writer.write(null, keys.length); ResultEntry result = writer.finish().get(0); - Path filePath = new Path(basePath, result.fileName()); + Path filePath = new Path(result.filePath()); GlobalIndexIOMeta meta = new GlobalIndexIOMeta(filePath, fileIO.getFileSize(filePath), result.meta()); @@ -661,7 +662,7 @@ private GlobalIndexIOMeta writeData(List> data) throws List results = writer.finish(); assertThat(results).hasSize(1); ResultEntry resultEntry = results.get(0); - Path filePath = new Path(basePath, resultEntry.fileName()); + Path filePath = new Path(resultEntry.filePath()); return new GlobalIndexIOMeta(filePath, fileIO.getFileSize(filePath), resultEntry.meta()); } diff --git a/paimon-common/src/test/java/org/apache/paimon/globalindex/bitmap/MultiValueBitmapIndexReaderTest.java b/paimon-common/src/test/java/org/apache/paimon/globalindex/bitmap/MultiValueBitmapIndexReaderTest.java index e008c5602a9a..bd1cbcae785d 100644 --- a/paimon-common/src/test/java/org/apache/paimon/globalindex/bitmap/MultiValueBitmapIndexReaderTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/globalindex/bitmap/MultiValueBitmapIndexReaderTest.java @@ -80,14 +80,15 @@ void setUp() { fileWriter = new GlobalIndexFileWriter() { @Override - public String newFileName(String prefix) { - return prefix + "-" + UUID.randomUUID() + ".index"; + public String newFilePath(String prefix) { + return new Path(basePath, prefix + "-" + UUID.randomUUID() + ".index") + .toString(); } @Override - public PositionOutputStream newOutputStream(String fileName) + public PositionOutputStream newOutputStream(String filePath) throws IOException { - return fileIO.newOutputStream(new Path(basePath, fileName), true); + return fileIO.newOutputStream(new Path(filePath), true); } }; fileReader = meta -> fileIO.newInputStream(meta.filePath()); @@ -106,7 +107,7 @@ void testArrayContainsAndSafeFallback() throws Exception { ResultEntry result = writer.finish(5).get(0); assertThat(result.rowCount()).isEqualTo(5); - Path path = new Path(basePath, result.fileName()); + Path path = new Path(result.filePath()); GlobalIndexIOMeta meta = new GlobalIndexIOMeta(path, fileIO.getFileSize(path), result.meta()); assertThat(result.meta()) @@ -164,7 +165,7 @@ void testRowsWithoutIndexableElementsStillProduceAnIndex() throws Exception { ResultEntry result = writer.finish(3).get(0); assertThat(result.rowCount()).isEqualTo(3); - Path path = new Path(basePath, result.fileName()); + Path path = new Path(result.filePath()); GlobalIndexIOMeta meta = new GlobalIndexIOMeta(path, fileIO.getFileSize(path), result.meta()); try (GlobalIndexReader reader = @@ -191,7 +192,7 @@ void testNumericKeysUseLogicalOrder() throws Exception { writer.write(1, 1); ResultEntry result = writer.finish(2).get(0); - Path path = new Path(basePath, result.fileName()); + Path path = new Path(result.filePath()); GlobalIndexIOMeta meta = new GlobalIndexIOMeta(path, fileIO.getFileSize(path), result.meta()); try (GlobalIndexReader reader = diff --git a/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/AbstractIndexReaderTest.java b/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/AbstractIndexReaderTest.java index c46472305c04..d1b4849f5e1c 100644 --- a/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/AbstractIndexReaderTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/AbstractIndexReaderTest.java @@ -126,21 +126,20 @@ public void setUp() throws Exception { fileWriter = new GlobalIndexFileWriter() { @Override - public String newFileName(String prefix) { - return "test-btree-" + UUID.randomUUID() + prefix; + public String newFilePath(String prefix) { + return new Path( + new Path(tempPath.toUri()), + "test-btree-" + UUID.randomUUID() + prefix) + .toString(); } @Override - public PositionOutputStream newOutputStream(String fileName) + public PositionOutputStream newOutputStream(String filePath) throws IOException { - return fileIO.newOutputStream( - new Path(new Path(tempPath.toUri()), fileName), true); + return fileIO.newOutputStream(new Path(filePath), true); } }; - fileReader = - meta -> - fileIO.newInputStream( - new Path(new Path(tempPath.toUri()), meta.filePath())); + fileReader = meta -> fileIO.newInputStream(meta.filePath()); options = new Options(); options.set(BTreeIndexOptions.BTREE_INDEX_CACHE_SIZE, MemorySize.ofMebiBytes(8)); globalIndexer = @@ -350,12 +349,9 @@ protected GlobalIndexIOMeta writeData( Assertions.assertEquals(1, results.size()); ResultEntry resultEntry = results.get(0); - String fileName = resultEntry.fileName(); + Path filePath = new Path(resultEntry.filePath()); return new GlobalIndexIOMeta( - new Path(new Path(tempPath.toUri()), fileName), - fileIO.getFileSize(new Path(new Path(tempPath.toUri()), fileName)), - resultEntry.rowCount(), - resultEntry.meta()); + filePath, fileIO.getFileSize(filePath), resultEntry.rowCount(), resultEntry.meta()); } protected List filter(Predicate filter) { diff --git a/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/BTreeBloomFilterTest.java b/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/BTreeBloomFilterTest.java index dd72b0ea22e2..1c92bf321d4c 100644 --- a/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/BTreeBloomFilterTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/BTreeBloomFilterTest.java @@ -99,7 +99,7 @@ void testCompositeNaNPointLookupWithBloom() throws Exception { byte[] bytes = fileWriter.bytes(); GlobalIndexIOMeta meta = new GlobalIndexIOMeta( - new Path(entry.fileName()), bytes.length, entry.rowCount(), entry.meta()); + new Path(entry.filePath()), bytes.length, entry.rowCount(), entry.meta()); try (CacheManager cacheManager = new CacheManager(MemorySize.ofMebiBytes(1), 0.5); BTreeIndexReader reader = new BTreeIndexReader( @@ -177,7 +177,7 @@ private Fixture writeIndex(Options options) throws IOException { .toInput()); GlobalIndexIOMeta meta = new GlobalIndexIOMeta( - new Path(entry.fileName()), bytes.length, entry.rowCount(), entry.meta()); + new Path(entry.filePath()), bytes.length, entry.rowCount(), entry.meta()); return new Fixture(bytes, footer, meta); } @@ -214,12 +214,12 @@ private static class ByteArrayGlobalIndexFileWriter implements GlobalIndexFileWr private final ByteArrayPositionOutputStream output = new ByteArrayPositionOutputStream(); @Override - public String newFileName(String prefix) { - return prefix + "-test"; + public String newFilePath(String prefix) { + return "memory://test/" + prefix + "-test"; } @Override - public PositionOutputStream newOutputStream(String fileName) { + public PositionOutputStream newOutputStream(String filePath) { return output; } diff --git a/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/BTreeIndexReaderCloseTest.java b/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/BTreeIndexReaderCloseTest.java index 11a389bd2f14..25e97eda7175 100644 --- a/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/BTreeIndexReaderCloseTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/BTreeIndexReaderCloseTest.java @@ -71,14 +71,14 @@ public void setUp() throws Exception { GlobalIndexFileWriter fileWriter = new GlobalIndexFileWriter() { @Override - public String newFileName(String prefix) { - return "test-btree-" + UUID.randomUUID() + prefix; + public String newFilePath(String prefix) { + return path("test-btree-" + UUID.randomUUID() + prefix).toString(); } @Override - public PositionOutputStream newOutputStream(String fileName) + public PositionOutputStream newOutputStream(String filePath) throws IOException { - return fileIO.newOutputStream(path(fileName), true); + return fileIO.newOutputStream(new Path(filePath), true); } }; @@ -94,7 +94,7 @@ public PositionOutputStream newOutputStream(String fileName) assertThat(results).hasSize(1); ResultEntry entry = results.get(0); - Path filePath = path(entry.fileName()); + Path filePath = new Path(entry.filePath()); meta = new GlobalIndexIOMeta(filePath, fileIO.getFileSize(filePath), results.get(0).meta()); } diff --git a/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/BTreeIndexWriterCloseTest.java b/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/BTreeIndexWriterCloseTest.java index b351db26b8b2..7f7f33f5c722 100644 --- a/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/BTreeIndexWriterCloseTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/BTreeIndexWriterCloseTest.java @@ -156,12 +156,12 @@ private static GlobalIndexFileWriter failingWriter(AtomicInteger closed) { return new GlobalIndexFileWriter() { @Override - public String newFileName(String prefix) { - return "test-btree" + prefix; + public String newFilePath(String prefix) { + return "memory://test/test-btree" + prefix; } @Override - public PositionOutputStream newOutputStream(String fileName) { + public PositionOutputStream newOutputStream(String filePath) { return new PositionOutputStream() { private long pos; diff --git a/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/BTreeThreadSafetyTest.java b/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/BTreeThreadSafetyTest.java index 9e6edfbd667b..96a699dcb8e9 100644 --- a/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/BTreeThreadSafetyTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/BTreeThreadSafetyTest.java @@ -89,21 +89,20 @@ void setUp() { fileWriter = new GlobalIndexFileWriter() { @Override - public String newFileName(String prefix) { - return "test-btree-" + UUID.randomUUID() + prefix; + public String newFilePath(String prefix) { + return new Path( + new Path(tempPath.toUri()), + "test-btree-" + UUID.randomUUID() + prefix) + .toString(); } @Override - public PositionOutputStream newOutputStream(String fileName) + public PositionOutputStream newOutputStream(String filePath) throws IOException { - return fileIO.newOutputStream( - new Path(new Path(tempPath.toUri()), fileName), true); + return fileIO.newOutputStream(new Path(filePath), true); } }; - fileReader = - meta -> - fileIO.newInputStream( - new Path(new Path(tempPath.toUri()), meta.filePath())); + fileReader = meta -> fileIO.newInputStream(meta.filePath()); Options options = new Options(); options.set(BTreeIndexOptions.BTREE_INDEX_CACHE_SIZE, MemorySize.ofMebiBytes(8)); @@ -274,8 +273,7 @@ void testLazyCreationOnlyOncePerFile() throws Exception { GlobalIndexFileReader countingFileReader = meta -> { creationCount.incrementAndGet(); - return fileIO.newInputStream( - new Path(new Path(tempPath.toUri()), meta.filePath())); + return fileIO.newInputStream(meta.filePath()); }; try (GlobalIndexReader reader = @@ -326,8 +324,7 @@ void testSelectorPrunesCorrectly() throws Exception { synchronized (openedFiles) { openedFiles.add(meta.filePath().getName()); } - return fileIO.newInputStream( - new Path(new Path(tempPath.toUri()), meta.filePath())); + return fileIO.newInputStream(meta.filePath()); }; try (GlobalIndexReader reader = @@ -366,11 +363,8 @@ private GlobalIndexIOMeta writeData(List> subData) throws IOE List results = indexWriter.finish(); Assertions.assertEquals(1, results.size()); ResultEntry resultEntry = results.get(0); - String fileName = resultEntry.fileName(); - return new GlobalIndexIOMeta( - new Path(new Path(tempPath.toUri()), fileName), - fileIO.getFileSize(new Path(new Path(tempPath.toUri()), fileName)), - resultEntry.meta()); + Path filePath = new Path(resultEntry.filePath()); + return new GlobalIndexIOMeta(filePath, fileIO.getFileSize(filePath), resultEntry.meta()); } private List filter(java.util.function.Predicate predicate) { diff --git a/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/CompositeBTreeIndexTest.java b/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/CompositeBTreeIndexTest.java index 6733b3af88d2..b2b67edb55f0 100644 --- a/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/CompositeBTreeIndexTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/CompositeBTreeIndexTest.java @@ -93,14 +93,14 @@ void testDeserializedTupleKeysPostingListsAndLocalRanges(int version) throws Exc GlobalIndexFileWriter files = new GlobalIndexFileWriter() { @Override - public String newFileName(String prefix) { - return prefix + UUID.randomUUID(); + public String newFilePath(String prefix) { + return new Path(directory, prefix + UUID.randomUUID()).toString(); } @Override - public PositionOutputStream newOutputStream(String name) + public PositionOutputStream newOutputStream(String filePath) throws java.io.IOException { - return io.newOutputStream(new Path(directory, name), false); + return io.newOutputStream(new Path(filePath), false); } }; GenericRow reused = row("category-a", -1, ""); @@ -126,7 +126,7 @@ public PositionOutputStream newOutputStream(String name) .isEqualTo(row("category-a", -1, "")); assertThat(serializer.deserialize(MemorySlice.wrap(indexMeta.getLastKey()))) .isEqualTo(row("category-b", 7, "tag")); - Path path = new Path(directory, result.fileName()); + Path path = new Path(result.filePath()); GlobalIndexIOMeta meta = new GlobalIndexIOMeta(path, io.getFileSize(path), result.rowCount(), result.meta()); AtomicInteger deserializations = new AtomicInteger(); diff --git a/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/CompositeBTreeRangeTest.java b/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/CompositeBTreeRangeTest.java index ce8e5644469e..037c04fc746a 100644 --- a/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/CompositeBTreeRangeTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/CompositeBTreeRangeTest.java @@ -291,13 +291,14 @@ private List write(Options options) throws Exception { GlobalIndexFileWriter files = new GlobalIndexFileWriter() { @Override - public String newFileName(String prefix) { - return prefix + UUID.randomUUID(); + public String newFilePath(String prefix) { + return new Path(directory, prefix + UUID.randomUUID()).toString(); } @Override - public PositionOutputStream newOutputStream(String name) throws IOException { - return io.newOutputStream(new Path(directory, name), false); + public PositionOutputStream newOutputStream(String filePath) + throws IOException { + return io.newOutputStream(new Path(filePath), false); } }; GlobalIndexSingleColumnWriter writer = @@ -309,7 +310,7 @@ public PositionOutputStream newOutputStream(String name) throws IOException { } List result = new ArrayList<>(); for (ResultEntry entry : writer.finish()) { - Path path = new Path(directory, entry.fileName()); + Path path = new Path(entry.filePath()); result.add( new GlobalIndexIOMeta( path, io.getFileSize(path), entry.rowCount(), entry.meta())); diff --git a/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/LazyFilteredBTreeIndexReaderTest.java b/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/LazyFilteredBTreeIndexReaderTest.java index 658d74cafbcb..620a8b1a31f2 100644 --- a/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/LazyFilteredBTreeIndexReaderTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/LazyFilteredBTreeIndexReaderTest.java @@ -598,12 +598,9 @@ private GlobalIndexIOMeta writeDataWithIndexer( List results = indexWriter.finish(); Assertions.assertEquals(1, results.size()); ResultEntry resultEntry = results.get(0); - String fileName = resultEntry.fileName(); + Path filePath = new Path(resultEntry.filePath()); return new GlobalIndexIOMeta( - new Path(new Path(tempPath.toUri()), fileName), - fileIO.getFileSize(new Path(new Path(tempPath.toUri()), fileName)), - resultEntry.rowCount(), - resultEntry.meta()); + filePath, fileIO.getFileSize(filePath), resultEntry.rowCount(), resultEntry.meta()); } private List> singletonData(Object key, long rowId) { diff --git a/paimon-common/src/test/java/org/apache/paimon/globalindex/fmindex/FMGlobalIndexTest.java b/paimon-common/src/test/java/org/apache/paimon/globalindex/fmindex/FMGlobalIndexTest.java index b5a4013c23ab..c807a8a33508 100644 --- a/paimon-common/src/test/java/org/apache/paimon/globalindex/fmindex/FMGlobalIndexTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/globalindex/fmindex/FMGlobalIndexTest.java @@ -94,14 +94,15 @@ public void setUp() { fileWriter = new GlobalIndexFileWriter() { @Override - public String newFileName(String prefix) { - return prefix + "-" + UUID.randomUUID() + ".index"; + public String newFilePath(String prefix) { + return new Path(basePath, prefix + "-" + UUID.randomUUID() + ".index") + .toString(); } @Override - public PositionOutputStream newOutputStream(String fileName) + public PositionOutputStream newOutputStream(String filePath) throws IOException { - return fileIO.newOutputStream(new Path(basePath, fileName), true); + return fileIO.newOutputStream(new Path(filePath), true); } }; fileReader = meta -> fileIO.newInputStream(meta.filePath()); @@ -908,7 +909,7 @@ private List writeData(List values, long firstR private List toIOMetas(List entries) throws IOException { List result = new ArrayList<>(); for (ResultEntry entry : entries) { - Path path = new Path(basePath, entry.fileName()); + Path path = new Path(entry.filePath()); result.add(new GlobalIndexIOMeta(path, fileIO.getFileSize(path), entry.meta())); } return result; diff --git a/paimon-common/src/test/java/org/apache/paimon/globalindex/rtree/RTreeGlobalIndexTest.java b/paimon-common/src/test/java/org/apache/paimon/globalindex/rtree/RTreeGlobalIndexTest.java index e96d8c801b03..3cd9636f0972 100644 --- a/paimon-common/src/test/java/org/apache/paimon/globalindex/rtree/RTreeGlobalIndexTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/globalindex/rtree/RTreeGlobalIndexTest.java @@ -101,14 +101,15 @@ void setUp() { output = new GlobalIndexFileWriter() { @Override - public String newFileName(String prefix) { - return prefix + "-" + UUID.randomUUID() + ".index"; + public String newFilePath(String prefix) { + return new Path(base, prefix + "-" + UUID.randomUUID() + ".index") + .toString(); } @Override - public PositionOutputStream newOutputStream(String fileName) + public PositionOutputStream newOutputStream(String filePath) throws IOException { - return io.newOutputStream(new Path(base, fileName), false); + return io.newOutputStream(new Path(filePath), false); } }; input = file -> io.newInputStream(file.filePath()); @@ -1001,12 +1002,13 @@ void testWriteFailurePreservesCause() { GlobalIndexFileWriter failing = new GlobalIndexFileWriter() { @Override - public String newFileName(String prefix) { - return "failed.index"; + public String newFilePath(String prefix) { + return new Path(base, "failed.index").toString(); } @Override - public PositionOutputStream newOutputStream(String name) throws IOException { + public PositionOutputStream newOutputStream(String filePath) + throws IOException { throw new IOException("injected write failure"); } }; @@ -1224,7 +1226,7 @@ private List write(RTreeGlobalIndexer indexer, List v private List metas(List entries) throws IOException { List files = new ArrayList<>(); for (ResultEntry entry : entries) { - Path path = new Path(base, entry.fileName()); + Path path = new Path(entry.filePath()); files.add( new GlobalIndexIOMeta( path, io.getFileSize(path), entry.rowCount(), entry.meta())); diff --git a/paimon-common/src/test/java/org/apache/paimon/globalindex/testfulltext/TestFullTextGlobalIndexWriter.java b/paimon-common/src/test/java/org/apache/paimon/globalindex/testfulltext/TestFullTextGlobalIndexWriter.java index 71ce5a096bee..37a920385c58 100644 --- a/paimon-common/src/test/java/org/apache/paimon/globalindex/testfulltext/TestFullTextGlobalIndexWriter.java +++ b/paimon-common/src/test/java/org/apache/paimon/globalindex/testfulltext/TestFullTextGlobalIndexWriter.java @@ -85,8 +85,8 @@ public List finish() { } try { - String fileName = fileWriter.newFileName(FILE_NAME_PREFIX); - try (PositionOutputStream out = fileWriter.newOutputStream(fileName)) { + String filePath = fileWriter.newFilePath(FILE_NAME_PREFIX); + try (PositionOutputStream out = fileWriter.newOutputStream(filePath)) { // Header: count ByteBuffer header = ByteBuffer.allocate(4); header.order(ByteOrder.LITTLE_ENDIAN); @@ -106,7 +106,7 @@ public List finish() { out.flush(); } - return Collections.singletonList(new ResultEntry(fileName, rowCount, null)); + return Collections.singletonList(new ResultEntry(filePath, rowCount, null)); } catch (IOException e) { throw new RuntimeException("Failed to write test full-text index", e); } diff --git a/paimon-common/src/test/java/org/apache/paimon/globalindex/testvector/TestVectorGlobalIndexWriter.java b/paimon-common/src/test/java/org/apache/paimon/globalindex/testvector/TestVectorGlobalIndexWriter.java index 3bb3dde42f61..4a5ccc11d678 100644 --- a/paimon-common/src/test/java/org/apache/paimon/globalindex/testvector/TestVectorGlobalIndexWriter.java +++ b/paimon-common/src/test/java/org/apache/paimon/globalindex/testvector/TestVectorGlobalIndexWriter.java @@ -107,8 +107,8 @@ public List finish() { int count = vectors.size(); try { - String fileName = fileWriter.newFileName(FILE_NAME_PREFIX); - try (PositionOutputStream out = fileWriter.newOutputStream(fileName)) { + String filePath = fileWriter.newFilePath(FILE_NAME_PREFIX); + try (PositionOutputStream out = fileWriter.newOutputStream(filePath)) { // Header: dimension + count ByteBuffer header = ByteBuffer.allocate(8); header.order(ByteOrder.LITTLE_ENDIAN); @@ -134,7 +134,7 @@ public List finish() { out.flush(); } - return Collections.singletonList(new ResultEntry(fileName, rowCount, null)); + return Collections.singletonList(new ResultEntry(filePath, rowCount, null)); } catch (IOException e) { throw new RuntimeException("Failed to write test vector index", e); } diff --git a/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java b/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java index 866394577870..a3a4013728bf 100644 --- a/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java +++ b/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java @@ -18,7 +18,6 @@ package org.apache.paimon; -import org.apache.paimon.CoreOptions.ExternalPathStrategy; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.catalog.SnapshotCommit; import org.apache.paimon.catalog.TableRollback; @@ -82,7 +81,6 @@ import java.time.Duration; import java.util.ArrayList; -import java.util.Collections; import java.util.Comparator; import java.util.List; @@ -142,45 +140,11 @@ protected FileStorePathFactory pathFactory(CoreOptions options, String format) { options.fileSuffixIncludeCompression(), options.fileCompression(), options.dataFilePathDirectory(), - createExternalPaths(), + options.createExternalPaths(), options.externalPathStrategy(), options.externalPathWeights(), options.indexFileInDataFileDir(), - options.globalIndexExternalPath()); - } - - private List createExternalPaths() { - String externalPaths = options.dataFileExternalPaths(); - ExternalPathStrategy strategy = options.externalPathStrategy(); - if (externalPaths == null - || externalPaths.isEmpty() - || strategy == ExternalPathStrategy.NONE) { - return Collections.emptyList(); - } - - String specificFS = options.externalSpecificFS(); - - List paths = new ArrayList<>(); - for (String pathString : externalPaths.split(",")) { - Path path = new Path(pathString.trim()); - String scheme = path.toUri().getScheme(); - if (scheme == null) { - throw new IllegalArgumentException("scheme should not be null: " + path); - } - - if (strategy == ExternalPathStrategy.SPECIFIC_FS) { - checkArgument( - specificFS != null, - "External path specificFS should not be null when strategy is specificFS."); - if (scheme.equalsIgnoreCase(specificFS)) { - paths.add(path); - } - } else { - paths.add(path); - } - } - checkArgument(!paths.isEmpty(), "External paths should not be empty"); - return paths; + options.globalIndexExternalPaths()); } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java index f0e3064693f2..ff4f9f39a345 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java @@ -59,7 +59,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -70,8 +69,6 @@ import java.util.Set; import java.util.stream.Collectors; -import static org.apache.paimon.CoreOptions.DATA_FILE_EXTERNAL_PATHS; -import static org.apache.paimon.CoreOptions.GLOBAL_INDEX_EXTERNAL_PATH; import static org.apache.paimon.CoreOptions.PATH; import static org.apache.paimon.CoreOptions.TYPE; import static org.apache.paimon.catalog.CatalogUtils.checkNotBranch; @@ -414,16 +411,11 @@ private List getSchemaExternalPaths(List schemas) { return schemas.stream() .flatMap( schema -> { - Map options = schema.toSchema().options(); - return Arrays.stream( - new String[] { - options.get(DATA_FILE_EXTERNAL_PATHS.key()), - options.get(GLOBAL_INDEX_EXTERNAL_PATH.key()) - }); + CoreOptions options = new CoreOptions(schema.options()); + List paths = new ArrayList<>(options.globalIndexExternalPaths()); + paths.addAll(options.dataFileExternalPathList()); + return paths.stream(); }) - .filter(Objects::nonNull) - .flatMap(externalPath -> Arrays.stream(externalPath.split(","))) - .map(Path::new) .distinct() .collect(Collectors.toList()); } diff --git a/paimon-core/src/main/java/org/apache/paimon/globalindex/CentroidIndexMeta.java b/paimon-core/src/main/java/org/apache/paimon/globalindex/CentroidIndexMeta.java index e02f72f51ed8..fcac1222037d 100644 --- a/paimon-core/src/main/java/org/apache/paimon/globalindex/CentroidIndexMeta.java +++ b/paimon-core/src/main/java/org/apache/paimon/globalindex/CentroidIndexMeta.java @@ -95,7 +95,7 @@ private static ResultEntry wrap(ResultEntry file, String modelId, byte role, int if (payload != null) { out.put(payload); } - return new ResultEntry(file.fileName(), file.rowCount(), out.array()); + return new ResultEntry(file.filePath(), file.rowCount(), out.array()); } public static GlobalIndexIOMeta file(IndexFileMeta file, IndexPathFactory paths) { diff --git a/paimon-core/src/main/java/org/apache/paimon/globalindex/GlobalIndexBuilderUtils.java b/paimon-core/src/main/java/org/apache/paimon/globalindex/GlobalIndexBuilderUtils.java index a5914c5c0320..ff3ad7a2a7cf 100644 --- a/paimon-core/src/main/java/org/apache/paimon/globalindex/GlobalIndexBuilderUtils.java +++ b/paimon-core/src/main/java/org/apache/paimon/globalindex/GlobalIndexBuilderUtils.java @@ -67,7 +67,6 @@ public class GlobalIndexBuilderUtils { public static List toIndexFileMetas( FileIO fileIO, - IndexPathFactory indexPathFactory, CoreOptions options, Range range, int indexFieldId, @@ -75,15 +74,7 @@ public static List toIndexFileMetas( List entries) throws IOException { return toIndexFileMetas( - fileIO, - indexPathFactory, - options, - range, - indexFieldId, - null, - indexType, - entries, - null); + fileIO, options, range, indexFieldId, null, indexType, entries, null); } /** @@ -91,10 +82,11 @@ public static List toIndexFileMetas( * index column (e.g. the first column in {@code CREATE ... INDEX ON (a, b, c)}) and is stored * as {@code indexFieldId}; the remaining columns go into {@code extraFieldIds}. Callers must * pass {@code fields} in the intended column order. + * + *

The entries already contain their allocated full paths. */ public static List toIndexFileMetas( FileIO fileIO, - IndexPathFactory indexPathFactory, CoreOptions options, Range range, List fields, @@ -104,7 +96,6 @@ public static List toIndexFileMetas( throws IOException { return toIndexFileMetas( fileIO, - indexPathFactory, options, range, fields.get(0).id(), @@ -571,7 +562,6 @@ private static void addIndexedSplitForFileGroup( private static List toIndexFileMetas( FileIO fileIO, - IndexPathFactory indexPathFactory, CoreOptions options, Range range, int indexFieldId, @@ -580,10 +570,12 @@ private static List toIndexFileMetas( List entries, @Nullable byte[] sourceMeta) throws IOException { + boolean isExternalPath = !options.globalIndexExternalPaths().isEmpty(); List results = new ArrayList<>(); for (ResultEntry entry : entries) { - String fileName = entry.fileName(); - long fileSize = fileIO.getFileSize(indexPathFactory.toPath(fileName)); + Path filePath = new Path(entry.filePath()); + String fileName = filePath.getName(); + long fileSize = fileIO.getFileSize(filePath); GlobalIndexMeta globalIndexMeta = new GlobalIndexMeta( range.from, @@ -593,12 +585,7 @@ private static List toIndexFileMetas( entry.meta(), sourceMeta); - Path externalPathDir = options.globalIndexExternalPath(); - String externalPathString = null; - if (externalPathDir != null) { - Path externalPath = new Path(externalPathDir, fileName); - externalPathString = externalPath.toString(); - } + String externalPathString = isExternalPath ? filePath.toString() : null; IndexFileMeta indexFileMeta = new IndexFileMeta( indexType, @@ -615,11 +602,21 @@ private static List toIndexFileMetas( public static GlobalIndexWriter createIndexWriter( FileStoreTable table, String indexType, List indexFields, Options options) throws IOException { + return createIndexWriter( + createGlobalIndexFileReadWrite(table), indexType, indexFields, options); + } + + public static GlobalIndexWriter createIndexWriter( + GlobalIndexFileReadWrite fileReadWrite, + String indexType, + List indexFields, + Options options) + throws IOException { GlobalIndexer globalIndexer = GlobalIndexer.create(indexType, indexFields, options); - return globalIndexer.createWriter(createGlobalIndexFileReadWrite(table)); + return globalIndexer.createWriter(fileReadWrite); } - private static GlobalIndexFileReadWrite createGlobalIndexFileReadWrite(FileStoreTable table) { + public static GlobalIndexFileReadWrite createGlobalIndexFileReadWrite(FileStoreTable table) { IndexPathFactory indexPathFactory = table.store().pathFactory().globalIndexFileFactory(); return new GlobalIndexFileReadWrite(table.fileIO(), indexPathFactory); } diff --git a/paimon-core/src/main/java/org/apache/paimon/globalindex/GlobalIndexFileReadWrite.java b/paimon-core/src/main/java/org/apache/paimon/globalindex/GlobalIndexFileReadWrite.java index 1df217135436..821d1c26c9d0 100644 --- a/paimon-core/src/main/java/org/apache/paimon/globalindex/GlobalIndexFileReadWrite.java +++ b/paimon-core/src/main/java/org/apache/paimon/globalindex/GlobalIndexFileReadWrite.java @@ -19,6 +19,7 @@ package org.apache.paimon.globalindex; import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; import org.apache.paimon.fs.PositionOutputStream; import org.apache.paimon.fs.SeekableInputStream; import org.apache.paimon.globalindex.io.GlobalIndexFileReader; @@ -39,18 +40,22 @@ public GlobalIndexFileReadWrite(FileIO fileIO, IndexPathFactory indexPathFactory this.indexPathFactory = indexPathFactory; } - public String newFileName(String prefix) { - return prefix + "-" + "global-index-" + UUID.randomUUID() + ".index"; + @Override + public String newFilePath(String prefix) { + String fileName = prefix + "-" + "global-index-" + UUID.randomUUID() + ".index"; + return indexPathFactory.newPathFromName(fileName).toString(); } - public long fileSize(String fileName) throws IOException { - return fileIO.getFileSize(indexPathFactory.toPath(fileName)); + public long fileSize(String filePath) throws IOException { + return fileIO.getFileSize(new Path(filePath)); } - public PositionOutputStream newOutputStream(String fileName) throws IOException { - return fileIO.newOutputStream(indexPathFactory.toPath(fileName), true); + @Override + public PositionOutputStream newOutputStream(String filePath) throws IOException { + return fileIO.newOutputStream(new Path(filePath), true); } + @Override public SeekableInputStream getInputStream(GlobalIndexIOMeta meta) throws IOException { return fileIO.newInputStream(meta.filePath()); } diff --git a/paimon-core/src/main/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexWriter.java b/paimon-core/src/main/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexWriter.java index 19a8ce457557..f51cf5dd30c5 100644 --- a/paimon-core/src/main/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexWriter.java +++ b/paimon-core/src/main/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexWriter.java @@ -22,6 +22,7 @@ import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.data.InternalRow.FieldGetter; +import org.apache.paimon.globalindex.GlobalIndexFileReadWrite; import org.apache.paimon.globalindex.GlobalIndexKeyExtractor; import org.apache.paimon.globalindex.GlobalIndexSingleColumnWriter; import org.apache.paimon.globalindex.GlobalIndexWriter; @@ -47,6 +48,7 @@ import java.util.Iterator; import java.util.List; +import static org.apache.paimon.globalindex.GlobalIndexBuilderUtils.createGlobalIndexFileReadWrite; import static org.apache.paimon.globalindex.GlobalIndexBuilderUtils.createIndexWriter; import static org.apache.paimon.globalindex.GlobalIndexBuilderUtils.toIndexFileMetas; import static org.apache.paimon.utils.Preconditions.checkArgument; @@ -65,6 +67,7 @@ public class SortedGlobalIndexWriter implements Serializable { private List indexFields; private GlobalIndexKeyExtractor keyExtractor; + private transient GlobalIndexFileReadWrite fileReadWrite; public SortedGlobalIndexWriter(Table table, String indexType) { this(table, indexType, ((FileStoreTable) table).coreOptions().toConfiguration()); @@ -141,8 +144,11 @@ public SortedSingleColumnIndexWriter createTaskWriter(Range rowRange) throws IOE } public GlobalIndexSingleColumnWriter createWriter() throws IOException { + if (fileReadWrite == null) { + fileReadWrite = createGlobalIndexFileReadWrite(table); + } GlobalIndexWriter indexWriter = - createIndexWriter(table, indexType, indexFields, indexOptions); + createIndexWriter(fileReadWrite, indexType, indexFields, indexOptions); if (!(indexWriter instanceof GlobalIndexSingleColumnWriter)) { throw new RuntimeException( "Unexpected implementation, the index writer of " @@ -163,7 +169,6 @@ public CommitMessage flushIndex( List indexFileMetas = toIndexFileMetas( table.fileIO(), - table.store().pathFactory().globalIndexFileFactory(), table.coreOptions(), rowRange, indexFields, diff --git a/paimon-core/src/main/java/org/apache/paimon/index/IndexInDataFileDirPathFactory.java b/paimon-core/src/main/java/org/apache/paimon/index/IndexInDataFileDirPathFactory.java index f664802164ca..386fca829c23 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/IndexInDataFileDirPathFactory.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/IndexInDataFileDirPathFactory.java @@ -43,14 +43,14 @@ public IndexInDataFileDirPathFactory( } @Override - public Path toPath(String fileName) { + public Path newPathFromName(String fileName) { return dataFilePathFactory.newPathFromName(fileName); } @Override public Path newPath() { String name = INDEX_PREFIX + uuid + "-" + indexFileCount.getAndIncrement(); - return dataFilePathFactory.newPathFromName(name); + return newPathFromName(name); } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/index/IndexPathFactory.java b/paimon-core/src/main/java/org/apache/paimon/index/IndexPathFactory.java index 3180bb2c44fa..2ea74c647f46 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/IndexPathFactory.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/IndexPathFactory.java @@ -25,13 +25,14 @@ /** Path factory to create an index path. */ public interface IndexPathFactory { - Path toPath(String fileName); - + /** Generates a file name and allocates a path for a new index file. */ Path newPath(); - default Path toPath(String fileName, @Nullable String externalPath) { - return toPath(fileName); - } + /** Allocates a path for a new index file with the given file name. */ + Path newPathFromName(String fileName); + + /** Resolves an existing file without allocating an external location. */ + Path toPath(String fileName, @Nullable String externalPath); default Path toPath(IndexFileMeta file) { return toPath(file.fileName(), file.externalPath()); diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkfulltext/PkFullTextIndexFile.java b/paimon-core/src/main/java/org/apache/paimon/index/pkfulltext/PkFullTextIndexFile.java index 4a6e02e062bc..0e45d7b34af4 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pkfulltext/PkFullTextIndexFile.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkfulltext/PkFullTextIndexFile.java @@ -160,12 +160,12 @@ IndexFileMeta build(List sources, DataField textField, GlobalIndexer ind "Full-text archive row count %s does not match source row count %s.", result.rowCount(), totalRowCount); - Path archivePath = fileWriter.path(result.fileName()); + Path archivePath = fileWriter.path(result.filePath()); byte[] archiveMetadata = result.meta() == null ? new byte[0] : result.meta(); IndexFileMeta archive = new IndexFileMeta( INDEX_TYPE, - result.fileName(), + archivePath.getName(), fileIO.getFileSize(archivePath), result.rowCount(), new GlobalIndexMeta( @@ -219,20 +219,20 @@ private class ArchiveFileWriter implements GlobalIndexFileWriter { private final Map createdFiles = new HashMap<>(); @Override - public String newFileName(String prefix) { + public String newFilePath(String prefix) { Path path = pathFactory.newPath(); - createdFiles.put(path.getName(), path); - return path.getName(); + createdFiles.put(path.toString(), path); + return path.toString(); } @Override - public PositionOutputStream newOutputStream(String fileName) throws IOException { - return fileIO.newOutputStream(path(fileName), false); + public PositionOutputStream newOutputStream(String filePath) throws IOException { + return fileIO.newOutputStream(path(filePath), false); } - private Path path(String fileName) { - Path path = createdFiles.get(fileName); - checkArgument(path != null, "Full-text archive %s was not allocated.", fileName); + private Path path(String filePath) { + Path path = createdFiles.get(filePath); + checkArgument(path != null, "Full-text archive %s was not allocated.", filePath); return path; } diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pksorted/PkSortedIndexFile.java b/paimon-core/src/main/java/org/apache/paimon/index/pksorted/PkSortedIndexFile.java index 456f7878f487..72919d056f38 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pksorted/PkSortedIndexFile.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pksorted/PkSortedIndexFile.java @@ -98,11 +98,11 @@ public IndexFileMeta build( result.rowCount(), sourceRowCount); byte[] sourceMeta = new PrimaryKeyIndexSourceMeta(dataLevel, sourceFiles).serialize(); - Path payloadPath = fileWriter.path(result.fileName()); + Path payloadPath = fileWriter.path(result.filePath()); IndexFileMeta payload = new IndexFileMeta( indexType, - result.fileName(), + payloadPath.getName(), fileIO.getFileSize(payloadPath), result.rowCount(), new GlobalIndexMeta( @@ -168,20 +168,20 @@ private final class TrackingFileWriter implements GlobalIndexFileWriter { private final Map createdFiles = new LinkedHashMap<>(); @Override - public String newFileName(String prefix) { + public String newFilePath(String prefix) { Path path = pathFactory.newPath(); - createdFiles.put(path.getName(), path); - return path.getName(); + createdFiles.put(path.toString(), path); + return path.toString(); } @Override - public PositionOutputStream newOutputStream(String fileName) throws IOException { - return fileIO.newOutputStream(path(fileName), false); + public PositionOutputStream newOutputStream(String filePath) throws IOException { + return fileIO.newOutputStream(path(filePath), false); } - private Path path(String fileName) { - Path path = createdFiles.get(fileName); - checkArgument(path != null, "Sorted payload file %s was not allocated.", fileName); + private Path path(String filePath) { + Path path = createdFiles.get(filePath); + checkArgument(path != null, "Sorted payload file %s was not allocated.", filePath); return path; } diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java index 93ac807aff4c..9683127b5953 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java @@ -160,12 +160,12 @@ public IndexFileMeta build( "ANN segment build must produce exactly one payload file, but produced %s.", results.size()); ResultEntry result = results.get(0); - Path payloadPath = fileWriter.path(result.fileName()); + Path payloadPath = fileWriter.path(result.filePath()); byte[] payloadMetadata = result.meta() == null ? new byte[0] : result.meta(); IndexFileMeta segment = new IndexFileMeta( indexType, - result.fileName(), + payloadPath.getName(), fileIO.getFileSize(payloadPath), liveRowCount, new GlobalIndexMeta( @@ -202,31 +202,31 @@ private class SegmentFileWriter implements GlobalIndexFileWriter { private final Map createdFiles = new HashMap<>(); @Override - public String newFileName(String prefix) { + public String newFilePath(String prefix) { Path path = pathFactory.newPath(); - createdFiles.put(path.getName(), path); - return path.getName(); + createdFiles.put(path.toString(), path); + return path.toString(); } @Override - public PositionOutputStream newOutputStream(String fileName) throws IOException { - return fileIO.newOutputStream(path(fileName), false); + public PositionOutputStream newOutputStream(String filePath) throws IOException { + return fileIO.newOutputStream(path(filePath), false); } - private Path path(String fileName) { - Path path = createdFiles.get(fileName); - checkArgument(path != null, "ANN payload file %s was not allocated.", fileName); + private Path path(String filePath) { + Path path = createdFiles.get(filePath); + checkArgument(path != null, "ANN payload file %s was not allocated.", filePath); return path; } private ResultEntry emptyResult() throws IOException { deleteCreatedFiles(); createdFiles.clear(); - String fileName = newFileName("empty-vector"); - try (PositionOutputStream ignored = newOutputStream(fileName)) { + String filePath = newFilePath("empty-vector"); + try (PositionOutputStream ignored = newOutputStream(filePath)) { // The searcher does not open payloads for segments without live rows. } - return new ResultEntry(fileName, 0, null); + return new ResultEntry(filePath, 0, null); } private void deleteCreatedFiles() { diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/RawFullTextReadImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/source/RawFullTextReadImpl.java index 91271ab95d6e..cde9fc179d07 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/RawFullTextReadImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/RawFullTextReadImpl.java @@ -362,18 +362,14 @@ private IndexFullTextSearchSplit finish() { List indexFiles = new ArrayList<>(resultEntries.size()); for (ResultEntry entry : resultEntries) { - byte[] bytes = fileWriter.fileBytes(entry.fileName()); + String fileName = new Path(entry.filePath()).getName(); + byte[] bytes = fileWriter.fileBytes(fileName); GlobalIndexMeta meta = new GlobalIndexMeta( rowRangeStart, rowRangeEnd, fieldId, null, entry.meta()); indexFiles.add( new IndexFileMeta( - indexType, - entry.fileName(), - bytes.length, - entry.rowCount(), - meta, - null)); + indexType, fileName, bytes.length, entry.rowCount(), meta, null)); } return new IndexFullTextSearchSplit(column, rowRangeStart, rowRangeEnd, indexFiles); } @@ -409,12 +405,14 @@ private RawFullTextIndexFileWriter(String column) { } @Override - public String newFileName(String prefix) { - return "raw-" + column + "-" + prefix + "-" + id + "-" + files.size() + ".index"; + public String newFilePath(String prefix) { + return path("raw-" + column + "-" + prefix + "-" + id + "-" + files.size() + ".index") + .toString(); } @Override - public PositionOutputStream newOutputStream(String fileName) { + public PositionOutputStream newOutputStream(String filePath) { + String fileName = new Path(filePath).getName(); return new MemoryPositionOutputStream(bytes -> files.put(fileName, bytes)); } @@ -440,7 +438,7 @@ private RawFullTextIndexPathFactory(Map rawIndexes) { } @Override - public Path toPath(String fileName) { + public Path toPath(String fileName, @Nullable String externalPath) { RawFullTextIndex rawIndex = rawIndex(fileName); return rawIndex.path(fileName); } @@ -450,6 +448,11 @@ public Path newPath() { throw new UnsupportedOperationException(); } + @Override + public Path newPathFromName(String fileName) { + throw new UnsupportedOperationException(); + } + @Override public boolean isExternalPath() { return false; diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/FileStorePathFactory.java b/paimon-core/src/main/java/org/apache/paimon/utils/FileStorePathFactory.java index 981b3a24da8d..c4dc894d8b93 100644 --- a/paimon-core/src/main/java/org/apache/paimon/utils/FileStorePathFactory.java +++ b/paimon-core/src/main/java/org/apache/paimon/utils/FileStorePathFactory.java @@ -24,6 +24,7 @@ import org.apache.paimon.data.BinaryRow; import org.apache.paimon.fs.ExternalPathProvider; import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.RoundRobinExternalPathProvider; import org.apache.paimon.index.IndexInDataFileDirPathFactory; import org.apache.paimon.index.IndexPathFactory; import org.apache.paimon.io.ChainReadContext; @@ -83,7 +84,7 @@ public class FileStorePathFactory { private final List externalPaths; private final ExternalPathStrategy strategy; @Nullable private final int[] externalPathWeights; - @Nullable private final Path globalIndexExternalRootDir; + @Nullable private final List globalIndexExternalPaths; public FileStorePathFactory( Path root, @@ -100,7 +101,7 @@ public FileStorePathFactory( ExternalPathStrategy strategy, @Nullable int[] externalPathWeights, boolean indexFileInDataFileDir, - @Nullable Path globalIndexExternalRootDir) { + @Nullable List globalIndexExternalPaths) { this.root = root; this.dataFilePathDirectory = dataFilePathDirectory; this.indexFileInDataFileDir = indexFileInDataFileDir; @@ -122,7 +123,7 @@ public FileStorePathFactory( this.externalPaths = externalPaths; this.strategy = strategy; this.externalPathWeights = externalPathWeights; - this.globalIndexExternalRootDir = globalIndexExternalRootDir; + this.globalIndexExternalPaths = globalIndexExternalPaths; } public Path root() { @@ -137,10 +138,6 @@ public Path indexPath() { return new Path(root, INDEX_PATH); } - public Path globalIndexRootDir() { - return globalIndexExternalRootDir != null ? globalIndexExternalRootDir : indexPath(); - } - public Path statisticsPath() { return new Path(root, STATISTICS_PATH); } @@ -329,15 +326,22 @@ public IndexPathFactory indexFileFactory(BinaryRow partition, int bucket) { } public IndexPathFactory globalIndexFileFactory() { + ExternalPathProvider provider = + globalIndexExternalPaths == null || globalIndexExternalPaths.isEmpty() + ? null + : new RoundRobinExternalPathProvider(globalIndexExternalPaths, null); return new IndexPathFactory() { @Override - public Path toPath(String fileName) { - return new Path(globalIndexRootDir(), fileName); + public synchronized Path newPathFromName(String fileName) { + return provider == null + ? new Path(indexPath(), fileName) + : provider.getNextExternalDataPath(fileName); } @Override public Path newPath() { - return toPath(INDEX_PREFIX + uuid + "-" + indexFileCount.getAndIncrement()); + return newPathFromName( + INDEX_PREFIX + uuid + "-" + indexFileCount.getAndIncrement()); } @Override @@ -351,7 +355,7 @@ public Path toPath(String fileName, @Nullable String externalPath) { @Override public boolean isExternalPath() { - return globalIndexExternalRootDir != null; + return provider != null; } }; } diff --git a/paimon-core/src/test/java/org/apache/paimon/CoreOptionsTest.java b/paimon-core/src/test/java/org/apache/paimon/CoreOptionsTest.java index f6931d44a3ce..9a2b783c2025 100644 --- a/paimon-core/src/test/java/org/apache/paimon/CoreOptionsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/CoreOptionsTest.java @@ -18,10 +18,13 @@ package org.apache.paimon; +import org.apache.paimon.fs.Path; import org.apache.paimon.options.MemorySize; import org.apache.paimon.options.Options; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -29,6 +32,102 @@ /** Tests for {@link org.apache.paimon.CoreOptions}. */ public class CoreOptionsTest { + @Test + void testDataFileExternalPathList() { + Options options = new Options(); + CoreOptions core = new CoreOptions(options); + assertThat(core.dataFileExternalPaths()).isNull(); + assertThat(core.dataFileExternalPathList()).isEmpty(); + options.set(CoreOptions.DATA_FILE_EXTERNAL_PATHS, ""); + assertThat(core.dataFileExternalPathList()).isEmpty(); + for (String value : + new String[] {"file:///a", " file:///a/ ", "file:///a,", "file:///a,,"}) { + options.set(CoreOptions.DATA_FILE_EXTERNAL_PATHS, value); + assertThat(core.dataFileExternalPaths()).isEqualTo(value); + assertThat(core.dataFileExternalPathList()).containsExactly(new Path("file:///a")); + } + for (String value : + new String[] { + "file:///a,s3://bucket/b", + " file:///a , s3://bucket/b ", + "file:///a,s3://bucket/b,", + "file:///a,s3://bucket/b,," + }) { + options.set(CoreOptions.DATA_FILE_EXTERNAL_PATHS, value); + assertThat(core.dataFileExternalPaths()).isEqualTo(value); + assertThat(core.dataFileExternalPathList()) + .containsExactly(new Path("file:///a"), new Path("s3://bucket/b")); + } + } + + @ParameterizedTest + @EnumSource(CoreOptions.ExternalPathStrategy.class) + void testCreateExternalPaths(CoreOptions.ExternalPathStrategy strategy) { + Options options = new Options(); + options.set(CoreOptions.DATA_FILE_EXTERNAL_PATHS_STRATEGY, strategy); + CoreOptions core = new CoreOptions(options); + assertThat(core.createExternalPaths()).isEmpty(); + options.set(CoreOptions.DATA_FILE_EXTERNAL_PATHS, ""); + assertThat(core.createExternalPaths()).isEmpty(); + + options.set(CoreOptions.DATA_FILE_EXTERNAL_PATHS, " file:///a , s3://bucket/b ,file:///c,"); + options.set(CoreOptions.DATA_FILE_EXTERNAL_PATHS_SPECIFIC_FS, "FILE"); + assertThat(core.dataFileExternalPathList()) + .containsExactly( + new Path("file:///a"), new Path("s3://bucket/b"), new Path("file:///c")); + if (strategy == CoreOptions.ExternalPathStrategy.NONE) { + assertThat(core.createExternalPaths()).isEmpty(); + } else if (strategy == CoreOptions.ExternalPathStrategy.SPECIFIC_FS) { + assertThat(core.createExternalPaths()) + .containsExactly(new Path("file:///a"), new Path("file:///c")); + } else { + assertThat(core.createExternalPaths()).isEqualTo(core.dataFileExternalPathList()); + } + } + + @Test + void testGlobalIndexExternalPaths() { + Options options = new Options(); + CoreOptions core = new CoreOptions(options); + assertThat(core.globalIndexExternalPaths()).isEmpty(); + options.set(CoreOptions.GLOBAL_INDEX_EXTERNAL_PATH, ""); + assertThat(core.globalIndexExternalPaths()).isEmpty(); + for (String value : + new String[] {"file:///a", " file:///a/ ", "file:///a,", "file:///a,,"}) { + options.set(CoreOptions.GLOBAL_INDEX_EXTERNAL_PATH, value); + assertThat(core.globalIndexExternalPaths()) + .as("global-index.external-path=%s", value) + .containsExactly(new Path("file:///a")); + } + for (String value : + new String[] { + "file:///a, file:///b", "file:///a,file:///b,", "file:///a,file:///b,," + }) { + options.set(CoreOptions.GLOBAL_INDEX_EXTERNAL_PATH, value); + assertThat(core.globalIndexExternalPaths()) + .as("global-index.external-path=%s", value) + .containsExactly(new Path("file:///a"), new Path("file:///b")); + } + for (String value : + new String[] { + ",", + ",,", + " ", + "a", + ",file:///a", + "file:///a,,file:///b", + ",file:///a,,file:///b,", + "file:///a, ", + "file:///a,/b", + "file:///a, ,file:///b" + }) { + options.set(CoreOptions.GLOBAL_INDEX_EXTERNAL_PATH, value); + assertThatThrownBy(core::globalIndexExternalPaths) + .as("global-index.external-path=%s", value) + .isInstanceOf(IllegalArgumentException.class); + } + } + @Test void testCommitLastSafeSnapshotAndStrictMode() { Options options = new Options(); diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java index f119c84e5837..193345e18c04 100644 --- a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java +++ b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java @@ -27,6 +27,7 @@ import org.apache.paimon.data.serializer.InternalRowSerializer; import org.apache.paimon.format.HadoopCompressionType; import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; import org.apache.paimon.fs.ResolvingFileIO; import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.ConfigOption; @@ -1086,21 +1087,28 @@ public void testDropTable() throws Exception { assertThatCode(() -> catalog.dropTable(nonExistingTable, true)).doesNotThrowAnyException(); } - @Test - public void testDropTableWithExternalPaths() throws Exception { + @ParameterizedTest + @ValueSource(strings = {"%s,%s", " %s , %s ", "%s,%s,", "%s,%s,,"}) + public void testDropTableWithExternalPaths(String pathFormat) throws Exception { catalog.createDatabase("test_db", false); Identifier identifier = Identifier.create("test_db", "table_with_external_paths"); java.nio.file.Path dataExternalPath = tempFile.resolve("data-external-path"); + java.nio.file.Path secondDataPath = tempFile.resolve("data-external-path-2"); java.nio.file.Path globalIndexExternalPath = tempFile.resolve("global-index-external-path"); + java.nio.file.Path secondIndexPath = tempFile.resolve("global-index-external-path-2"); Files.createDirectories(dataExternalPath.resolve("data")); + Files.createDirectories(secondDataPath.resolve("data")); Files.createDirectories(globalIndexExternalPath.resolve("index")); + Files.createDirectories(secondIndexPath.resolve("index")); Map options = new HashMap<>(); options.put( - CoreOptions.DATA_FILE_EXTERNAL_PATHS.key(), dataExternalPath.toUri().toString()); + CoreOptions.DATA_FILE_EXTERNAL_PATHS.key(), + String.format(pathFormat, dataExternalPath.toUri(), secondDataPath.toUri())); options.put( CoreOptions.GLOBAL_INDEX_EXTERNAL_PATH.key(), - globalIndexExternalPath.toUri().toString()); + String.format( + pathFormat, globalIndexExternalPath.toUri(), secondIndexPath.toUri())); Schema schema = new Schema( Lists.newArrayList( @@ -1112,10 +1120,21 @@ public void testDropTableWithExternalPaths() throws Exception { ""); catalog.createTable(identifier, schema, false); + FileStoreTable table = (FileStoreTable) catalog.getTable(identifier); + assertThat(table.coreOptions().dataFileExternalPathList()) + .containsExactly( + new Path(dataExternalPath.toUri().toString()), + new Path(secondDataPath.toUri().toString())); + assertThat(table.coreOptions().globalIndexExternalPaths()) + .containsExactly( + new Path(globalIndexExternalPath.toUri().toString()), + new Path(secondIndexPath.toUri().toString())); catalog.dropTable(identifier, false); assertThat(Files.exists(dataExternalPath)).isFalse(); + assertThat(Files.exists(secondDataPath)).isFalse(); assertThat(Files.exists(globalIndexExternalPath)).isFalse(); + assertThat(Files.exists(secondIndexPath)).isFalse(); } @Test diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/FileSystemCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/catalog/FileSystemCatalogTest.java index 0e180938a160..0876c2c1b665 100644 --- a/paimon-core/src/test/java/org/apache/paimon/catalog/FileSystemCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/catalog/FileSystemCatalogTest.java @@ -34,6 +34,7 @@ import org.apache.paimon.table.sink.BatchWriteBuilder; import org.apache.paimon.types.DataTypes; import org.apache.paimon.utils.SnapshotManager; +import org.apache.paimon.utils.TraceableFileIO; import org.apache.paimon.shade.guava30.com.google.common.collect.Lists; @@ -308,6 +309,62 @@ public void testDropTableWithInvalidOptions(boolean cacheEnabled) throws Excepti catalog.dropTable(identifier, true); } + @ParameterizedTest + @EnumSource( + value = CoreOptions.ExternalPathStrategy.class, + names = {"NONE", "SPECIFIC_FS"}) + public void testDropTableWithDynamicExternalPathStrategy( + CoreOptions.ExternalPathStrategy strategy) throws Exception { + Identifier identifier = Identifier.create("test_db", "dynamic_external_paths"); + catalog.createDatabase(identifier.getDatabaseName(), false); + Path firstPath = new Path(tempFile.resolve("external-file").toUri().toString()); + Path secondPath = + new Path(TraceableFileIO.SCHEME + "://" + tempFile.resolve("external-traceable")); + catalog.createTable( + identifier, + Schema.newBuilder() + .column("id", DataTypes.INT()) + .option(CoreOptions.BUCKET.key(), "-1") + .option( + CoreOptions.DATA_FILE_EXTERNAL_PATHS.key(), + " " + firstPath + " , " + secondPath + " ") + .option( + CoreOptions.DATA_FILE_EXTERNAL_PATHS_STRATEGY.key(), + strategy.toString()) + .option(CoreOptions.DATA_FILE_EXTERNAL_PATHS_SPECIFIC_FS.key(), "file") + .build(), + false); + FileStoreTable table = (FileStoreTable) catalog.getTable(identifier); + if (strategy == CoreOptions.ExternalPathStrategy.NONE) { + assertThat(table.store().pathFactory().getExternalPaths()).isEmpty(); + } else { + assertThat(table.store().pathFactory().getExternalPaths()).containsExactly(firstPath); + } + + Map dynamicOptions = new HashMap<>(); + dynamicOptions.put(CoreOptions.DATA_FILE_EXTERNAL_PATHS_STRATEGY.key(), "specific-fs"); + dynamicOptions.put( + CoreOptions.DATA_FILE_EXTERNAL_PATHS_SPECIFIC_FS.key(), TraceableFileIO.SCHEME); + FileStoreTable writerTable = table.copy(dynamicOptions); + assertThat(writerTable.store().pathFactory().getExternalPaths()) + .containsExactly(secondPath); + BatchWriteBuilder writeBuilder = writerTable.newBatchWriteBuilder(); + try (BatchTableWrite write = writeBuilder.newWrite(); + BatchTableCommit commit = writeBuilder.newCommit()) { + write.write(GenericRow.of(1)); + commit.commit(write.prepareCommit()); + } + assertThat(fileIO.exists(secondPath)).isTrue(); + FileSystemSchemaManager schemaManager = + new FileSystemSchemaManager(fileIO, table.location()); + assertThat(schemaManager.listAll()).hasSize(1); + assertThat(schemaManager.latest().get().options()) + .containsEntry(CoreOptions.DATA_FILE_EXTERNAL_PATHS_SPECIFIC_FS.key(), "file"); + + catalog.dropTable(identifier, false); + assertThat(fileIO.exists(secondPath)).isFalse(); + } + @ParameterizedTest @ValueSource(booleans = {false, true}) public void testDropTableWithInvalidOptionsAndExternalPaths(boolean cacheEnabled) diff --git a/paimon-core/src/test/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFileTest.java b/paimon-core/src/test/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFileTest.java index c1a8e2a0fed2..23f27769b38c 100644 --- a/paimon-core/src/test/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFileTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFileTest.java @@ -31,6 +31,8 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import javax.annotation.Nullable; + import java.io.DataInputStream; import java.io.IOException; import java.io.InputStream; @@ -448,7 +450,7 @@ private IndexPathFactory getPathFactory() { Path dir = new Path(tempPath.toUri()); return new IndexPathFactory() { @Override - public Path toPath(String fileName) { + public Path newPathFromName(String fileName) { return new Path(dir, fileName); } @@ -461,6 +463,11 @@ public Path newPath() { public boolean isExternalPath() { return false; } + + @Override + public Path toPath(String fileName, @Nullable String externalPath) { + return externalPath == null ? newPathFromName(fileName) : new Path(externalPath); + } }; } } diff --git a/paimon-core/src/test/java/org/apache/paimon/globalindex/GlobalIndexBuilderUtilsTest.java b/paimon-core/src/test/java/org/apache/paimon/globalindex/GlobalIndexBuilderUtilsTest.java index b4f42dff78ed..27953c1fe6b5 100644 --- a/paimon-core/src/test/java/org/apache/paimon/globalindex/GlobalIndexBuilderUtilsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/globalindex/GlobalIndexBuilderUtilsTest.java @@ -22,6 +22,8 @@ import org.apache.paimon.data.BinaryRow; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.PositionOutputStream; +import org.apache.paimon.fs.SeekableInputStream; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.index.DataEvolutionIndexSourceMeta; import org.apache.paimon.index.IndexFileMeta; @@ -45,8 +47,13 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import javax.annotation.Nullable; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.LinkedHashMap; @@ -54,6 +61,7 @@ import java.util.Map; import java.util.UUID; +import static org.apache.paimon.utils.FileStorePathFactoryTest.createNonPartFactory; import static org.assertj.core.api.Assertions.assertThat; /** Tests for {@link GlobalIndexBuilderUtils}. */ @@ -65,6 +73,69 @@ class GlobalIndexBuilderUtilsTest { private IndexPathFactory indexPathFactory; private CoreOptions coreOptions; + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testFileLocationsWithDifferentReaderOptions(boolean external) throws Exception { + Path root = new Path(tempDir.toUri()); + List roots = Arrays.asList(new Path(root, "a"), new Path(root, "b")); + if (external) { + coreOptions + .toConfiguration() + .set(CoreOptions.GLOBAL_INDEX_EXTERNAL_PATH, roots.get(0) + "," + roots.get(1)); + } + IndexPathFactory writerPaths = + createNonPartFactory(root, external ? roots : Collections.emptyList()) + .globalIndexFileFactory(); + GlobalIndexFileReadWrite writer = new GlobalIndexFileReadWrite(fileIO, writerPaths); + List entries = new ArrayList<>(); + List parents = new ArrayList<>(); + for (int i = 0; i < 6; i++) { + String file = writer.newFilePath("test"); + assertThat(new Path(file).toUri().getScheme()).isEqualTo("file"); + assertThat(new Path(file).getName()) + .startsWith("test-global-index-") + .endsWith(".index"); + try (PositionOutputStream output = writer.newOutputStream(file)) { + output.write(i); + } + assertThat(writer.fileSize(file)).isEqualTo(1); + parents.add(new Path(file).getParent()); + entries.add(new ResultEntry(file, 1, null)); + } + if (external) { + for (Path externalRoot : roots) { + assertThat(Collections.frequency(parents, externalRoot)).isEqualTo(3); + } + } else { + assertThat(parents).containsOnly(new Path(root, "index")); + } + + List metas = + GlobalIndexBuilderUtils.toIndexFileMetas( + fileIO, coreOptions, new Range(0, 5), 0, "test", entries); + + // Reading committed files must not depend on the current external path configuration. + IndexPathFactory readerPaths = + createNonPartFactory(root, external ? Collections.emptyList() : roots) + .globalIndexFileFactory(); + GlobalIndexFileReadWrite reader = new GlobalIndexFileReadWrite(fileIO, readerPaths); + for (int i = 0; i < metas.size(); i++) { + IndexFileMeta meta = metas.get(i); + Path path = new Path(entries.get(i).filePath()); + assertThat(meta.fileName()).isEqualTo(path.getName()); + assertThat(meta.externalPath()).isEqualTo(external ? path.toString() : null); + assertThat(readerPaths.toPath(meta)).isEqualTo(path); + try (SeekableInputStream input = + reader.getInputStream( + new GlobalIndexIOMeta( + readerPaths.toPath(meta), meta.fileSize(), null))) { + assertThat(input.read()).isEqualTo(i); + } + fileIO.delete(readerPaths.toPath(meta), false); + assertThat(fileIO.exists(path)).isFalse(); + } + } + @BeforeEach void setUp() { fileIO = new LocalFileIO(); @@ -72,7 +143,7 @@ void setUp() { indexPathFactory = new IndexPathFactory() { @Override - public Path toPath(String fileName) { + public Path newPathFromName(String fileName) { return new Path(dir, fileName); } @@ -85,6 +156,13 @@ public Path newPath() { public boolean isExternalPath() { return false; } + + @Override + public Path toPath(String fileName, @Nullable String externalPath) { + return externalPath == null + ? newPathFromName(fileName) + : new Path(externalPath); + } }; coreOptions = new CoreOptions(new Options().toMap()); } @@ -101,14 +179,7 @@ void testToIndexFileMetasMultiColumn() throws IOException { List metas = GlobalIndexBuilderUtils.toIndexFileMetas( - fileIO, - indexPathFactory, - coreOptions, - range, - fields, - "test-type", - entries, - null); + fileIO, coreOptions, range, fields, "test-type", entries, null); assertThat(metas).hasSize(1); assertThat(metas.get(0).globalIndexMeta().indexFieldId()).isEqualTo(1); @@ -129,14 +200,7 @@ void testToIndexFileMetasSingleColumn() throws IOException { List metas = GlobalIndexBuilderUtils.toIndexFileMetas( - fileIO, - indexPathFactory, - coreOptions, - range, - fields, - "test-type", - entries, - null); + fileIO, coreOptions, range, fields, "test-type", entries, null); assertThat(metas).hasSize(1); assertThat(metas.get(0).globalIndexMeta().indexFieldId()).isEqualTo(1); @@ -147,19 +211,16 @@ void testToIndexFileMetasSingleColumn() throws IOException { void testRowCountPerIndexFile() throws IOException { List entries = Arrays.asList( - new ResultEntry("first", 2, null), new ResultEntry("second", 1, null)); + new ResultEntry( + indexPathFactory.newPathFromName("first").toString(), 2, null), + new ResultEntry( + indexPathFactory.newPathFromName("second").toString(), 1, null)); for (ResultEntry entry : entries) { - fileIO.newOutputStream(indexPathFactory.toPath(entry.fileName()), false).close(); + fileIO.newOutputStream(new Path(entry.filePath()), false).close(); } List metas = GlobalIndexBuilderUtils.toIndexFileMetas( - fileIO, - indexPathFactory, - coreOptions, - new Range(100, 102), - 1, - "test-type", - entries); + fileIO, coreOptions, new Range(100, 102), 1, "test-type", entries); assertThat(metas).extracting(IndexFileMeta::rowCount).containsExactly(2L, 1L); assertThat(metas) .extracting(file -> file.globalIndexMeta().rowRange()) @@ -174,7 +235,6 @@ void testToIndexFileMetasWithSourceMeta() throws IOException { List metas = GlobalIndexBuilderUtils.toIndexFileMetas( fileIO, - indexPathFactory, coreOptions, new Range(0, 9), Collections.singletonList(field), @@ -199,14 +259,7 @@ void testToIndexFileMetasThreeColumns() throws IOException { List metas = GlobalIndexBuilderUtils.toIndexFileMetas( - fileIO, - indexPathFactory, - coreOptions, - range, - fields, - "test-type", - entries, - null); + fileIO, coreOptions, range, fields, "test-type", entries, null); assertThat(metas).hasSize(1); assertThat(metas.get(0).globalIndexMeta().indexFieldId()).isEqualTo(1); @@ -327,9 +380,9 @@ void testShardSplitsUsesStableGlobalRowIdWindows() { private List createDummyResultEntries() throws IOException { String fileName = "test-index-" + UUID.randomUUID(); - Path filePath = indexPathFactory.toPath(fileName); + Path filePath = indexPathFactory.newPathFromName(fileName); fileIO.newOutputStream(filePath, false).close(); - return Collections.singletonList(new ResultEntry(fileName, 100, null)); + return Collections.singletonList(new ResultEntry(filePath.toString(), 100, null)); } private ManifestEntry createEntry(Long firstRowId, long rowCount) { diff --git a/paimon-core/src/test/java/org/apache/paimon/globalindex/GlobalIndexQueryTest.java b/paimon-core/src/test/java/org/apache/paimon/globalindex/GlobalIndexQueryTest.java index 23bf51366c1f..1101acb05a38 100644 --- a/paimon-core/src/test/java/org/apache/paimon/globalindex/GlobalIndexQueryTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/globalindex/GlobalIndexQueryTest.java @@ -56,6 +56,8 @@ import org.junit.jupiter.params.provider.ValueSource; import org.mockito.MockedStatic; +import javax.annotation.Nullable; + import java.io.IOException; import java.math.BigDecimal; import java.nio.charset.StandardCharsets; @@ -132,7 +134,7 @@ void workerReceivesRoutedFilesAfterSplitSerialization() throws Exception { files.add( new IndexFileMeta( "ivf-pq", - file.fileName(), + new Path(file.filePath()).getName(), 100, file.rowCount(), new GlobalIndexMeta(0, 2, 0, null, file.meta()), @@ -304,7 +306,7 @@ void testNegativePredicatesRetainNullFilesAndOriginalRowCount(String indexType) IndexPathFactory paths = new IndexPathFactory() { @Override - public Path toPath(String name) { + public Path newPathFromName(String name) { return new Path(tempDir.resolve(name).toString()); } @@ -317,6 +319,13 @@ public Path newPath() { public boolean isExternalPath() { return false; } + + @Override + public Path toPath(String fileName, @Nullable String externalPath) { + return externalPath == null + ? newPathFromName(fileName) + : new Path(externalPath); + } }; GlobalIndexFileReadWrite io = new GlobalIndexFileReadWrite(fileIO, paths); List files = new ArrayList<>(); @@ -338,8 +347,8 @@ public boolean isExternalPath() { files.add( new IndexFileMeta( indexType, - entry.fileName(), - io.fileSize(entry.fileName()), + new Path(entry.filePath()).getName(), + io.fileSize(entry.filePath()), entry.rowCount(), new GlobalIndexMeta(100, 102, 0, null, entry.meta()), null)); @@ -900,7 +909,7 @@ private GlobalIndexQuery create( IndexPathFactory paths = new IndexPathFactory() { @Override - public Path toPath(String name) { + public Path newPathFromName(String name) { return new Path("index/" + name); } @@ -913,6 +922,13 @@ public Path newPath() { public boolean isExternalPath() { return false; } + + @Override + public Path toPath(String fileName, @Nullable String externalPath) { + return externalPath == null + ? newPathFromName(fileName) + : new Path(externalPath); + } }; return GlobalIndexQuery.create( rowType, predicate, Collections.singletonList(file), paths, new Options()); diff --git a/paimon-core/src/test/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexWriterTest.java b/paimon-core/src/test/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexWriterTest.java index 817eeb02105a..ac40a193b19b 100644 --- a/paimon-core/src/test/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexWriterTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexWriterTest.java @@ -20,18 +20,26 @@ import org.apache.paimon.data.GenericRow; import org.apache.paimon.data.InternalRow; +import org.apache.paimon.fs.Path; import org.apache.paimon.globalindex.GlobalIndexSingleColumnWriter; import org.apache.paimon.globalindex.ResultEntry; +import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.options.Options; import org.apache.paimon.schema.Schema; +import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.TableTestBase; +import org.apache.paimon.table.sink.CommitMessage; +import org.apache.paimon.table.sink.CommitMessageImpl; import org.apache.paimon.types.DataTypes; import org.apache.paimon.utils.Range; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.io.Closeable; import java.util.ArrayDeque; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -46,6 +54,54 @@ /** Tests for {@link SortedGlobalIndexWriter}. */ public class SortedGlobalIndexWriterTest extends TableTestBase { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testBuildPreservesAllocatedPathsInCommit(boolean external) throws Exception { + createTableDefault(); + FileStoreTable table = getTableDefault(); + Path firstRoot = new Path(table.location(), "external-a"); + Path secondRoot = new Path(table.location(), "external-b"); + if (external) { + table = + table.copy( + Collections.singletonMap( + "global-index.external-path", firstRoot + "," + secondRoot)); + } + Options options = new Options(); + options.setString("btree-index.records-per-range", "1"); + SortedGlobalIndexWriter writer = + new SortedGlobalIndexWriter(table, "btree", options) + .withIndexFields(Collections.singletonList("f0")); + List commits = + writer.buildForSinglePartition( + new Range(0, 1), + null, + Arrays.asList(GenericRow.of(10, 0L), GenericRow.of(20, 1L)) + .iterator(), + 1L); + assertThat(commits).hasSize(2); + List parents = new ArrayList<>(); + for (CommitMessage commit : commits) { + List files = + ((CommitMessageImpl) commit).newFilesIncrement().newIndexFiles(); + assertThat(files).hasSize(1); + IndexFileMeta file = files.get(0); + Path path = table.store().pathFactory().globalIndexFileFactory().toPath(file); + assertThat(table.fileIO().exists(path)).isTrue(); + assertThat(file.fileName()) + .isEqualTo(path.getName()) + .startsWith("btree-global-index-") + .endsWith(".index"); + assertThat(file.externalPath()).isEqualTo(external ? path.toString() : null); + parents.add(path.getParent()); + } + if (external) { + assertThat(parents).containsExactlyInAnyOrder(firstRoot, secondRoot); + } else { + assertThat(parents).containsOnly(new Path(table.location(), "index")); + } + } + @Test public void testFieldOptionsControlFileRotationAndBuildSettings() throws Exception { createTableDefault(); @@ -72,9 +128,13 @@ public void testSingleColumnWriterRotationPreservesResultGroups() throws Excepti GlobalIndexSingleColumnWriter first = mock(GlobalIndexSingleColumnWriter.class); GlobalIndexSingleColumnWriter second = mock(GlobalIndexSingleColumnWriter.class); when(first.finish(2)) - .thenReturn(Collections.singletonList(new ResultEntry("index-1", 2, null))); + .thenReturn( + Collections.singletonList( + new ResultEntry("memory://test/index-1", 2, null))); when(second.finish(1)) - .thenReturn(Collections.singletonList(new ResultEntry("index-2", 1, null))); + .thenReturn( + Collections.singletonList( + new ResultEntry("memory://test/index-2", 1, null))); Queue writers = new ArrayDeque<>(Arrays.asList(first, second)); SortedSingleColumnIndexWriter rotatingWriter = @@ -89,15 +149,20 @@ public void testSingleColumnWriterRotationPreservesResultGroups() throws Excepti verify(first).write(20, 1); verify(second).write(30, 2); assertThat(results).hasSize(2); - assertThat(results.get(0)).extracting(ResultEntry::fileName).containsExactly("index-1"); - assertThat(results.get(1)).extracting(ResultEntry::fileName).containsExactly("index-2"); + assertThat(results.get(0)) + .extracting(ResultEntry::filePath) + .containsExactly("memory://test/index-1"); + assertThat(results.get(1)) + .extracting(ResultEntry::filePath) + .containsExactly("memory://test/index-2"); } @Test public void testSingleColumnWriterPreservesSourceRowCount() throws Exception { GlobalIndexSingleColumnWriter writer = mock(GlobalIndexSingleColumnWriter.class); when(writer.finish(5)) - .thenReturn(Collections.singletonList(new ResultEntry("index", 5, null))); + .thenReturn( + Collections.singletonList(new ResultEntry("memory://test/index", 5, null))); SortedSingleColumnIndexWriter taskWriter = SortedSingleColumnIndexWriter.forSourceRowCount(5, writer); @@ -114,7 +179,8 @@ public void testSingleColumnWriterPreservesSourceRowCount() throws Exception { public void testSingleColumnWriterPreservesSourceRangeWithoutEntries() { GlobalIndexSingleColumnWriter writer = mock(GlobalIndexSingleColumnWriter.class); when(writer.finish(3)) - .thenReturn(Collections.singletonList(new ResultEntry("index", 3, null))); + .thenReturn( + Collections.singletonList(new ResultEntry("memory://test/index", 3, null))); SortedSingleColumnIndexWriter taskWriter = SortedSingleColumnIndexWriter.forSourceRowCount(3, writer); diff --git a/paimon-core/src/test/java/org/apache/paimon/index/HashIndexFileTest.java b/paimon-core/src/test/java/org/apache/paimon/index/HashIndexFileTest.java index 974a518e9ae3..7fbcdba397bf 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/HashIndexFileTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/HashIndexFileTest.java @@ -25,6 +25,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import javax.annotation.Nullable; + import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -49,7 +51,7 @@ public Path newPath() { } @Override - public Path toPath(String fileName) { + public Path newPathFromName(String fileName) { return new Path(dir, fileName); } @@ -57,6 +59,13 @@ public Path toPath(String fileName) { public boolean isExternalPath() { return false; } + + @Override + public Path toPath(String fileName, @Nullable String externalPath) { + return externalPath == null + ? newPathFromName(fileName) + : new Path(externalPath); + } }; HashIndexFile file = new HashIndexFile(LocalFileIO.create(), pathFactory); diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pk/BucketedPrimaryKeyIndexMaintainerTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pk/BucketedPrimaryKeyIndexMaintainerTest.java index d4df19ce1d16..250b595f21d4 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pk/BucketedPrimaryKeyIndexMaintainerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pk/BucketedPrimaryKeyIndexMaintainerTest.java @@ -52,6 +52,8 @@ import org.junit.jupiter.params.provider.ValueSource; import org.mockito.InOrder; +import javax.annotation.Nullable; + import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -871,7 +873,7 @@ private IndexPathFactory pathFactory() { Path directory = new Path(tempPath.toUri()); return new IndexPathFactory() { @Override - public Path toPath(String fileName) { + public Path newPathFromName(String fileName) { return new Path(directory, fileName); } @@ -884,6 +886,11 @@ public Path newPath() { public boolean isExternalPath() { return false; } + + @Override + public Path toPath(String fileName, @Nullable String externalPath) { + return externalPath == null ? newPathFromName(fileName) : new Path(externalPath); + } }; } } diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkfulltext/PkFullTextIndexFileTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkfulltext/PkFullTextIndexFileTest.java index bb7f19fe6147..ef9d726b9b00 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pkfulltext/PkFullTextIndexFileTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pkfulltext/PkFullTextIndexFileTest.java @@ -45,6 +45,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import javax.annotation.Nullable; + import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -224,7 +226,7 @@ private IndexPathFactory pathFactory() { Path directory = new Path(tempPath.toUri()); return new IndexPathFactory() { @Override - public Path toPath(String fileName) { + public Path newPathFromName(String fileName) { return new Path(directory, fileName); } @@ -237,6 +239,11 @@ public Path newPath() { public boolean isExternalPath() { return false; } + + @Override + public Path toPath(String fileName, @Nullable String externalPath) { + return externalPath == null ? newPathFromName(fileName) : new Path(externalPath); + } }; } @@ -257,12 +264,12 @@ public void write(Object key, long relativeRowId) { @Override public List finish() { try { - String fileName = fileWriter.newFileName("full-text"); - try (PositionOutputStream out = fileWriter.newOutputStream(fileName)) { + String filePath = fileWriter.newFilePath("full-text"); + try (PositionOutputStream out = fileWriter.newOutputStream(filePath)) { out.write(1); } return Collections.singletonList( - new ResultEntry(fileName, rowIds.size(), new byte[] {2})); + new ResultEntry(filePath, rowIds.size(), new byte[] {2})); } catch (IOException e) { throw new RuntimeException(e); } diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pksorted/BucketedSortedIndexMaintainerTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pksorted/BucketedSortedIndexMaintainerTest.java index 1e15994e08af..2e80ca7dd627 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pksorted/BucketedSortedIndexMaintainerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pksorted/BucketedSortedIndexMaintainerTest.java @@ -36,6 +36,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import javax.annotation.Nullable; + import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -798,7 +800,7 @@ private IndexPathFactory pathFactory() { Path directory = new Path(tempPath.toUri()); return new IndexPathFactory() { @Override - public Path toPath(String fileName) { + public Path newPathFromName(String fileName) { return new Path(directory, fileName); } @@ -811,6 +813,11 @@ public Path newPath() { public boolean isExternalPath() { return false; } + + @Override + public Path toPath(String fileName, @Nullable String externalPath) { + return externalPath == null ? newPathFromName(fileName) : new Path(externalPath); + } }; } diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PkSequentialIndexBuilderTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PkSequentialIndexBuilderTest.java index da9f1670a574..90dd839e707f 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PkSequentialIndexBuilderTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PkSequentialIndexBuilderTest.java @@ -177,7 +177,7 @@ private IndexPathFactory pathFactory() { Path root = new Path(tempPath.toUri()); return new IndexPathFactory() { @Override - public Path toPath(String fileName) { + public Path newPathFromName(String fileName) { return new Path(root, fileName); } @@ -190,6 +190,11 @@ public Path newPath() { public boolean isExternalPath() { return false; } + + @Override + public Path toPath(String fileName, @Nullable String externalPath) { + return externalPath == null ? newPathFromName(fileName) : new Path(externalPath); + } }; } diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PkSortedIndexBuilderTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PkSortedIndexBuilderTest.java index 5af2f8b00f9b..3d168eeb64d3 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PkSortedIndexBuilderTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PkSortedIndexBuilderTest.java @@ -47,6 +47,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import javax.annotation.Nullable; + import java.io.IOException; import java.lang.reflect.Method; import java.util.ArrayList; @@ -137,7 +139,7 @@ void testBuildsQueryableMultiValueFromUnsortedArrayRows() throws Exception { List ioMetas = Collections.singletonList( new GlobalIndexIOMeta( - pathFactory.toPath(payload.fileName()), + pathFactory.toPath(payload), payload.fileSize(), payload.globalIndexMeta().indexMeta())); ExecutorService executor = newDirectExecutorService(); @@ -386,7 +388,7 @@ private static void assertQuery( List ioMetas = Collections.singletonList( new GlobalIndexIOMeta( - pathFactory.toPath(payload.fileName()), + pathFactory.toPath(payload), payload.fileSize(), payload.globalIndexMeta().indexMeta())); ExecutorService executor = newDirectExecutorService(); @@ -448,7 +450,7 @@ private static IndexPathFactory pathFactory(java.nio.file.Path directory) { Path path = new Path(directory.toUri()); return new IndexPathFactory() { @Override - public Path toPath(String fileName) { + public Path newPathFromName(String fileName) { return new Path(path, fileName); } @@ -461,6 +463,11 @@ public Path newPath() { public boolean isExternalPath() { return false; } + + @Override + public Path toPath(String fileName, @Nullable String externalPath) { + return externalPath == null ? newPathFromName(fileName) : new Path(externalPath); + } }; } diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PkSortedIndexFileTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PkSortedIndexFileTest.java index 0e0baf60b462..de492b752d9d 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PkSortedIndexFileTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PkSortedIndexFileTest.java @@ -36,6 +36,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import javax.annotation.Nullable; + import java.io.IOException; import java.lang.reflect.Method; import java.nio.file.Files; @@ -77,12 +79,12 @@ void testUsesCloseableStreamingBitmapWriter() throws Exception { options(), new GlobalIndexFileWriter() { @Override - public String newFileName(String prefix) { - return prefix + ".index"; + public String newFilePath(String prefix) { + return "memory://test/" + prefix + ".index"; } @Override - public PositionOutputStream newOutputStream(String fileName) { + public PositionOutputStream newOutputStream(String filePath) { throw new AssertionError("Writer must open output lazily."); } }); @@ -206,16 +208,16 @@ public void write(Object key, long relativeRowId) { public List finish() { List results = new ArrayList<>(); for (int i = 0; i < 2; i++) { - String fileName = fileWriter.newFileName("test"); + String filePath = fileWriter.newFilePath("test"); try (PositionOutputStream output = - fileWriter.newOutputStream(fileName)) { + fileWriter.newOutputStream(filePath)) { output.write(new byte[] {1}); } catch (IOException e) { throw new RuntimeException(e); } results.add( new ResultEntry( - fileName, rowCount / 2, new byte[] {2})); + filePath, rowCount / 2, new byte[] {2})); } return results; } @@ -302,7 +304,7 @@ private static IndexPathFactory pathFactory(java.nio.file.Path directory) { Path path = new Path(directory.toUri()); return new IndexPathFactory() { @Override - public Path toPath(String fileName) { + public Path newPathFromName(String fileName) { return new Path(path, fileName); } @@ -315,6 +317,11 @@ public Path newPath() { public boolean isExternalPath() { return false; } + + @Override + public Path toPath(String fileName, @Nullable String externalPath) { + return externalPath == null ? newPathFromName(fileName) : new Path(externalPath); + } }; } } diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/BucketedVectorIndexMaintainerTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/BucketedVectorIndexMaintainerTest.java index e6b1986a070e..a91f6e13684e 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/BucketedVectorIndexMaintainerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/BucketedVectorIndexMaintainerTest.java @@ -41,6 +41,8 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import javax.annotation.Nullable; + import java.io.IOException; import java.io.UncheckedIOException; import java.util.ArrayList; @@ -869,7 +871,7 @@ private IndexPathFactory pathFactory() { Path directory = new Path(tempPath.toUri()); return new IndexPathFactory() { @Override - public Path toPath(String fileName) { + public Path newPathFromName(String fileName) { return new Path(directory, fileName); } @@ -882,6 +884,11 @@ public Path newPath() { public boolean isExternalPath() { return false; } + + @Override + public Path toPath(String fileName, @Nullable String externalPath) { + return externalPath == null ? newPathFromName(fileName) : new Path(externalPath); + } }; } diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java index fd972d40c48e..599b0e469d82 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java @@ -36,6 +36,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import javax.annotation.Nullable; + import java.io.IOException; import java.util.Arrays; import java.util.Collections; @@ -330,7 +332,7 @@ private IndexPathFactory pathFactory() { Path directory = new Path(tempPath.toUri()); return new IndexPathFactory() { @Override - public Path toPath(String fileName) { + public Path newPathFromName(String fileName) { return new Path(directory, fileName); } @@ -343,6 +345,11 @@ public Path newPath() { public boolean isExternalPath() { return false; } + + @Override + public Path toPath(String fileName, @Nullable String externalPath) { + return externalPath == null ? newPathFromName(fileName) : new Path(externalPath); + } }; } diff --git a/paimon-core/src/test/java/org/apache/paimon/table/BitmapGlobalIndexTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/BitmapGlobalIndexTableTest.java index a50eb2d0375e..ca40cf68b4fc 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/BitmapGlobalIndexTableTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/BitmapGlobalIndexTableTest.java @@ -258,7 +258,6 @@ private CommitMessage buildIndex( List indexFileMetas = GlobalIndexBuilderUtils.toIndexFileMetas( table.fileIO(), - table.store().pathFactory().globalIndexFileFactory(), table.coreOptions(), rowRange, indexField.id(), diff --git a/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionTableTest.java index 1a5347278b78..ddf030cf6fda 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionTableTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionTableTest.java @@ -1899,7 +1899,7 @@ public void testIndexPath() throws Exception { FileStoreTable table = getTableDefault(); IndexPathFactory indexPathFactory = table.store().pathFactory().globalIndexFileFactory(); - Path path0 = indexPathFactory.toPath("test-file"); + Path path0 = indexPathFactory.newPathFromName("test-file"); assertThat(path0.toString().contains(warehouse.toString())).isTrue(); String testExternalpath = "file:/external/path/test-file-dir"; Path path1 = @@ -1920,7 +1920,7 @@ public void testIndexPath() throws Exception { CoreOptions.GLOBAL_INDEX_EXTERNAL_PATH.key(), testExternalpath)); indexPathFactory = table.store().pathFactory().globalIndexFileFactory(); - Path path3 = indexPathFactory.toPath("test-file"); + Path path3 = indexPathFactory.newPathFromName("test-file"); assertThat(path3.toString()).isEqualTo(testExternalpath + "/test-file"); String testExternalpath2 = "file:/external/path2/test-file"; diff --git a/paimon-core/src/test/java/org/apache/paimon/table/IndexQuerySplitTest.java b/paimon-core/src/test/java/org/apache/paimon/table/IndexQuerySplitTest.java index 3e1612003f25..197688f49623 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/IndexQuerySplitTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/IndexQuerySplitTest.java @@ -932,8 +932,8 @@ public void testFMPartitionPruningAfterSplitRestore() throws Exception { indexes.add( new IndexFileMeta( "fm", - entry.fileName(), - io.fileSize(entry.fileName()), + new Path(entry.filePath()).getName(), + io.fileSize(entry.filePath()), entry.rowCount(), new GlobalIndexMeta( 100, @@ -1063,8 +1063,8 @@ private void createFMIndex(FileStoreTable table) throws Exception { indexes.add( new IndexFileMeta( "fm", - entry.fileName(), - io.fileSize(entry.fileName()), + new Path(entry.filePath()).getName(), + io.fileSize(entry.filePath()), entry.rowCount(), new GlobalIndexMeta( 0, 99, table.rowType().getField("f1").id(), null, entry.meta()), diff --git a/paimon-core/src/test/java/org/apache/paimon/table/RTreeGlobalIndexTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/RTreeGlobalIndexTableTest.java index cd08d1be7ffa..517f9c03e659 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/RTreeGlobalIndexTableTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/RTreeGlobalIndexTableTest.java @@ -411,7 +411,6 @@ private void buildIndex( List metas = GlobalIndexBuilderUtils.toIndexFileMetas( table.fileIO(), - table.store().pathFactory().globalIndexFileFactory(), table.coreOptions(), range, Collections.singletonList(field), diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java index f43b7c590a44..ff741a11c99d 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java @@ -188,7 +188,6 @@ public void testPartialCompositeKeepsScalarFilterCoverage(String mode, boolean s List files = GlobalIndexBuilderUtils.toIndexFileMetas( table.fileIO(), - table.store().pathFactory().globalIndexFileFactory(), table.coreOptions(), new Range(0, 0), fields, @@ -1733,7 +1732,6 @@ private void buildAndCommitIndexWithFields( List indexFiles = GlobalIndexBuilderUtils.toIndexFileMetas( table.fileIO(), - table.store().pathFactory().globalIndexFileFactory(), table.coreOptions(), rowRange, indexFields, @@ -1772,7 +1770,6 @@ private void buildAndCommitSourceBackedIndex(FileStoreTable table, String[] docu List indexFiles = GlobalIndexBuilderUtils.toIndexFileMetas( table.fileIO(), - table.store().pathFactory().globalIndexFileFactory(), table.coreOptions(), new Range(0, documents.length - 1), Collections.singletonList(textField), @@ -1844,7 +1841,6 @@ private void buildAndCommitIndexRange( List indexFiles = GlobalIndexBuilderUtils.toIndexFileMetas( table.fileIO(), - table.store().pathFactory().globalIndexFileFactory(), table.coreOptions(), rowRange, indexFields, @@ -1933,7 +1929,6 @@ private void buildAndCommitIndexForColumn( List indexFiles = GlobalIndexBuilderUtils.toIndexFileMetas( table.fileIO(), - table.store().pathFactory().globalIndexFileFactory(), table.coreOptions(), rowRange, textField.id(), @@ -1974,7 +1969,6 @@ private void buildAndCommitIdBTreeIndexRange(FileStoreTable table, Range rowRang List indexFiles = GlobalIndexBuilderUtils.toIndexFileMetas( table.fileIO(), - table.store().pathFactory().globalIndexFileFactory(), table.coreOptions(), rowRange, idField.id(), @@ -2016,7 +2010,6 @@ private void buildAndCommitBTreeIndex( List indexFiles = GlobalIndexBuilderUtils.toIndexFileMetas( table.fileIO(), - table.store().pathFactory().globalIndexFileFactory(), table.coreOptions(), rowRange, textField.id(), @@ -2068,7 +2061,6 @@ private void buildAndCommitMultipleIndexFiles(FileStoreTable table, String[] doc List indexFiles1 = GlobalIndexBuilderUtils.toIndexFileMetas( table.fileIO(), - table.store().pathFactory().globalIndexFileFactory(), table.coreOptions(), rowRange1, textField.id(), @@ -2091,7 +2083,6 @@ private void buildAndCommitMultipleIndexFiles(FileStoreTable table, String[] doc List indexFiles2 = GlobalIndexBuilderUtils.toIndexFileMetas( table.fileIO(), - table.store().pathFactory().globalIndexFileFactory(), table.coreOptions(), rowRange2, textField.id(), diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/RTreeRankedFilterTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/RTreeRankedFilterTest.java index ead7613f004b..09847ba5c84a 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/RTreeRankedFilterTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/RTreeRankedFilterTest.java @@ -178,7 +178,6 @@ private static void buildIndex( List files = GlobalIndexBuilderUtils.toIndexFileMetas( table.fileIO(), - table.store().pathFactory().globalIndexFileFactory(), table.coreOptions(), new Range(0, values.length - 1), field.id(), diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java index 30900d54a4d1..d2781a928655 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java @@ -728,7 +728,6 @@ private void buildAndCommitMultiValueIndex(FileStoreTable table, int rowCount) List indexFiles = GlobalIndexBuilderUtils.toIndexFileMetas( table.fileIO(), - table.store().pathFactory().globalIndexFileFactory(), table.coreOptions(), new Range(0, rowCount - 1), tagsField.id(), @@ -2150,7 +2149,6 @@ private void buildAndCommitIndex(FileStoreTable table, String fieldName, float[] List indexFiles = GlobalIndexBuilderUtils.toIndexFileMetas( table.fileIO(), - table.store().pathFactory().globalIndexFileFactory(), table.coreOptions(), rowRange, vectorField.id(), @@ -2192,7 +2190,6 @@ private void buildAndCommitMultipleIndexFiles(FileStoreTable table, float[][] ve List indexFiles1 = GlobalIndexBuilderUtils.toIndexFileMetas( table.fileIO(), - table.store().pathFactory().globalIndexFileFactory(), table.coreOptions(), rowRange1, vectorField.id(), @@ -2215,7 +2212,6 @@ private void buildAndCommitMultipleIndexFiles(FileStoreTable table, float[][] ve List indexFiles2 = GlobalIndexBuilderUtils.toIndexFileMetas( table.fileIO(), - table.store().pathFactory().globalIndexFileFactory(), table.coreOptions(), rowRange2, vectorField.id(), @@ -2387,7 +2383,6 @@ private void buildAndCommitVectorIndexWithFields( List indexFiles = GlobalIndexBuilderUtils.toIndexFileMetas( table.fileIO(), - table.store().pathFactory().globalIndexFileFactory(), table.coreOptions(), rowRange, indexFields, @@ -2429,7 +2424,6 @@ private void buildAndCommitBTreeIndex(FileStoreTable table, int[] ids, Range row List indexFiles = GlobalIndexBuilderUtils.toIndexFileMetas( table.fileIO(), - table.store().pathFactory().globalIndexFileFactory(), table.coreOptions(), rowRange, idField.id(), @@ -2470,7 +2464,6 @@ private void buildAndCommitPartitionedIndex( List indexFiles = GlobalIndexBuilderUtils.toIndexFileMetas( table.fileIO(), - table.store().pathFactory().globalIndexFileFactory(), table.coreOptions(), rowRange, vectorField.id(), diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchRowFilterExactnessTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchRowFilterExactnessTest.java index 48d525287600..ec2d5a05126b 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchRowFilterExactnessTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchRowFilterExactnessTest.java @@ -522,7 +522,6 @@ public void testPartialCompositeKeepsScalarFilterCoverage(String mode, boolean i table, GlobalIndexBuilderUtils.toIndexFileMetas( table.fileIO(), - table.store().pathFactory().globalIndexFileFactory(), table.coreOptions(), new Range(0, 1), fields, @@ -884,7 +883,6 @@ private void buildAndCommitVectorIndex(FileStoreTable table, float[][] vectors, table, GlobalIndexBuilderUtils.toIndexFileMetas( table.fileIO(), - table.store().pathFactory().globalIndexFileFactory(), table.coreOptions(), rowRange, vectorField.id(), @@ -921,7 +919,6 @@ private void buildAndCommitNameBTreeIndex(FileStoreTable table, String[] names, table, GlobalIndexBuilderUtils.toIndexFileMetas( table.fileIO(), - table.store().pathFactory().globalIndexFileFactory(), table.coreOptions(), rowRange, nameField.id(), @@ -950,7 +947,6 @@ private void buildAndCommitIdBTreeIndex(FileStoreTable table, Range rowRange) th table, GlobalIndexBuilderUtils.toIndexFileMetas( table.fileIO(), - table.store().pathFactory().globalIndexFileFactory(), table.coreOptions(), rowRange, idField.id(), diff --git a/paimon-core/src/test/java/org/apache/paimon/utils/FileStorePathFactoryTest.java b/paimon-core/src/test/java/org/apache/paimon/utils/FileStorePathFactoryTest.java index 4f067163fc6b..994088152042 100644 --- a/paimon-core/src/test/java/org/apache/paimon/utils/FileStorePathFactoryTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/utils/FileStorePathFactoryTest.java @@ -23,6 +23,7 @@ import org.apache.paimon.data.BinaryRowWriter; import org.apache.paimon.data.BinaryString; import org.apache.paimon.fs.Path; +import org.apache.paimon.index.IndexPathFactory; import org.apache.paimon.io.DataFilePathFactory; import org.apache.paimon.types.DataType; import org.apache.paimon.types.IntType; @@ -32,6 +33,16 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + import static org.apache.paimon.CoreOptions.PARTITION_DEFAULT_NAME; import static org.assertj.core.api.Assertions.assertThat; @@ -40,6 +51,86 @@ public class FileStorePathFactoryTest { @TempDir java.nio.file.Path tempDir; + @Test + public void testGlobalIndexExternalPaths() { + Path root = new Path(tempDir.toUri()); + List roots = + Arrays.asList(new Path(root, "external-a"), new Path(root, "external-b")); + FileStorePathFactory factory = createNonPartFactory(root, roots); + IndexPathFactory indexes = factory.globalIndexFileFactory(); + assertThat(indexes.isExternalPath()).isTrue(); + List parents = new ArrayList<>(); + for (int i = 0; i < 6; i++) { + Path path = indexes.newPath(); + parents.add(path.getParent()); + assertThat(path.getName()).isEqualTo("index-" + factory.uuid() + "-" + i); + assertThat(indexes.toPath(path.getName(), path.toString())).isEqualTo(path); + assertThat(indexes.toPath(path.getName(), path.toString())).isEqualTo(path); + } + for (Path externalRoot : roots) { + assertThat(Collections.frequency(parents, externalRoot)).isEqualTo(3); + } + assertThat(indexes.toPath("old-index", null)).isEqualTo(new Path(root, "index/old-index")); + assertThat(indexes.toPath("old-index", "file:///old-root/old-index")) + .isEqualTo(new Path("file:///old-root/old-index")); + assertThat(factory.indexFileFactory(BinaryRow.EMPTY_ROW, 0).isExternalPath()).isTrue(); + Path namedPath = indexes.newPathFromName("btree-global-index-custom.index"); + assertThat(namedPath.getName()).isEqualTo("btree-global-index-custom.index"); + assertThat(namedPath.getParent()).isEqualTo(parents.get(0)); + } + + @Test + public void testConcurrentGlobalIndexPathAllocation() throws Exception { + Path root = new Path(tempDir.toUri()); + List roots = + Arrays.asList(new Path(root, "external-a"), new Path(root, "external-b")); + IndexPathFactory indexes = createNonPartFactory(root, roots).globalIndexFileFactory(); + int parallelism = 8; + int filesPerThread = 1000; + ExecutorService executor = Executors.newFixedThreadPool(parallelism); + CountDownLatch start = new CountDownLatch(1); + List>> futures = new ArrayList<>(); + try { + for (int thread = 0; thread < parallelism; thread++) { + int threadId = thread; + futures.add( + executor.submit( + () -> { + start.await(); + List paths = new ArrayList<>(); + for (int i = 0; i < filesPerThread; i++) { + paths.add( + i % 2 == 0 + ? indexes.newPath() + : indexes.newPathFromName( + "btree-global-index-" + + threadId + + "-" + + i + + ".index")); + } + return paths; + })); + } + start.countDown(); + List paths = new ArrayList<>(); + for (Future> future : futures) { + paths.addAll(future.get(30, TimeUnit.SECONDS)); + } + assertThat(paths).hasSize(parallelism * filesPerThread).doesNotHaveDuplicates(); + assertThat(paths).extracting(Path::getName).doesNotHaveDuplicates(); + for (Path externalRoot : roots) { + assertThat( + paths.stream() + .filter(path -> path.getParent().equals(externalRoot)) + .count()) + .isEqualTo(parallelism * filesPerThread / roots.size()); + } + } finally { + executor.shutdownNow(); + } + } + @Test public void testManifestPaths() { FileStorePathFactory pathFactory = createNonPartFactory(new Path(tempDir.toString())); @@ -129,6 +220,11 @@ private void assertPartition( } public static FileStorePathFactory createNonPartFactory(Path root) { + return createNonPartFactory(root, Collections.emptyList()); + } + + public static FileStorePathFactory createNonPartFactory( + Path root, List globalIndexPaths) { return new FileStorePathFactory( root, RowType.builder().build(), @@ -144,6 +240,6 @@ public static FileStorePathFactory createNonPartFactory(Path root) { CoreOptions.ExternalPathStrategy.NONE, null, false, - null); + globalIndexPaths); } } diff --git a/paimon-eslib/src/main/java/org/apache/paimon/eslib/index/ESIndexGlobalIndexWriter.java b/paimon-eslib/src/main/java/org/apache/paimon/eslib/index/ESIndexGlobalIndexWriter.java index 1de6847e437c..0e807f56e65b 100644 --- a/paimon-eslib/src/main/java/org/apache/paimon/eslib/index/ESIndexGlobalIndexWriter.java +++ b/paimon-eslib/src/main/java/org/apache/paimon/eslib/index/ESIndexGlobalIndexWriter.java @@ -464,17 +464,17 @@ public List finish() { byte[] meta = buildMeta(segFiles); - String fileName = fileWriter.newFileName(FILE_NAME_PREFIX); + String filePath = fileWriter.newFilePath(FILE_NAME_PREFIX); // Stream the archive directly to the output stream — DO NOT materialise the entire // archive in memory. For real DiskBBQ output (1M × 768-dim vectors ≈ 3 GB per shard) // a ByteArrayOutputStream-backed approach hits both heap and DirectByteBuffer limits // (java.nio.Bits.reserveMemory OOM at Files.readAllBytes). - try (PositionOutputStream out = fileWriter.newOutputStream(fileName)) { + try (PositionOutputStream out = fileWriter.newOutputStream(filePath)) { packDirectoryStream(segFiles, out); out.flush(); } - return Collections.singletonList(new ResultEntry(fileName, docCount, meta)); + return Collections.singletonList(new ResultEntry(filePath, docCount, meta)); } catch (IOException e) { RuntimeException wrapped = new RuntimeException("Failed to finish ES index build", e); failure = wrapped; diff --git a/paimon-eslib/src/test/java/org/apache/paimon/eslib/index/ESIndexGlobalIndexE2ETest.java b/paimon-eslib/src/test/java/org/apache/paimon/eslib/index/ESIndexGlobalIndexE2ETest.java index 464d1ac8a47e..a02153ae4f69 100644 --- a/paimon-eslib/src/test/java/org/apache/paimon/eslib/index/ESIndexGlobalIndexE2ETest.java +++ b/paimon-eslib/src/test/java/org/apache/paimon/eslib/index/ESIndexGlobalIndexE2ETest.java @@ -49,6 +49,7 @@ import java.io.IOException; import java.nio.file.Files; +import java.nio.file.Paths; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; @@ -94,14 +95,15 @@ private static final class LocalDirWriter implements GlobalIndexFileWriter { } @Override - public String newFileName(String prefix) { - return prefix + "-" + UUID.randomUUID() + ".index"; + public String newFilePath(String prefix) { + return new org.apache.paimon.fs.Path( + dir.resolve(prefix + "-" + UUID.randomUUID() + ".index").toString()) + .toString(); } @Override - public PositionOutputStream newOutputStream(String fileName) throws IOException { - return fio.newOutputStream( - new org.apache.paimon.fs.Path(dir.resolve(fileName).toString()), true); + public PositionOutputStream newOutputStream(String filePath) throws IOException { + return fio.newOutputStream(new org.apache.paimon.fs.Path(filePath), true); } } @@ -323,9 +325,8 @@ void writeArchiveThenReadAndSearch(@TempDir java.nio.file.Path tmp) throws IOExc assertThrows(IllegalStateException.class, () -> writer.write(new float[] {1, 1, 1, 1}, 20)); // --- reconstruct reader over the archive --- - org.apache.paimon.fs.Path filePath = - new org.apache.paimon.fs.Path(archiveDir.resolve(entry.fileName()).toString()); - long fileSize = Files.size(archiveDir.resolve(entry.fileName())); + org.apache.paimon.fs.Path filePath = new org.apache.paimon.fs.Path(entry.filePath()); + long fileSize = Files.size(Paths.get(entry.filePath())); GlobalIndexIOMeta ioMeta = new GlobalIndexIOMeta(filePath, fileSize, entry.meta()); ESIndexGlobalIndexReader reader = @@ -545,10 +546,7 @@ void readsIndexAfterIndexedColumnsAreRenamed(@TempDir java.nio.file.Path tmp) new DataField(3, "cost", DataTypes.INT())); ESIndexGlobalIndexReader reader = new ESIndexGlobalIndexReader( - new LocalFileReader(), - List.of(ioMeta(archiveDir, entry)), - currentFields, - buildOptions); + new LocalFileReader(), List.of(ioMeta(entry)), currentFields, buildOptions); try { Optional vector = reader.visitVectorSearch(new VectorSearch(new float[] {0, 0}, 1, "vector")) @@ -584,10 +582,7 @@ void readsIndexAfterIndexedColumnsAreRenamed(@TempDir java.nio.file.Path tmp) new DataField(3, "cost", DataTypes.BIGINT())); ESIndexGlobalIndexReader widenedReader = new ESIndexGlobalIndexReader( - new LocalFileReader(), - List.of(ioMeta(archiveDir, entry)), - widenedFields, - buildOptions); + new LocalFileReader(), List.of(ioMeta(entry)), widenedFields, buildOptions); try { Optional widenedFallback = widenedReader @@ -612,7 +607,7 @@ void readsIndexAfterIndexedColumnsAreRenamed(@TempDir java.nio.file.Path tmp) ESIndexGlobalIndexReader resizedVectorReader = new ESIndexGlobalIndexReader( new LocalFileReader(), - List.of(ioMeta(archiveDir, entry)), + List.of(ioMeta(entry)), resizedVectorFields, buildOptions); try { @@ -659,13 +654,13 @@ void mixedSchemaShardsDoNotDropRowsFromAnOlderScalarType(@TempDir java.nio.file. ESIndexGlobalIndexReader oldReader = new ESIndexGlobalIndexReader( new LocalFileReader(), - List.of(ioMeta(oldDir, oldEntry)), + List.of(ioMeta(oldEntry)), currentFields, currentOptions); ESIndexGlobalIndexReader currentReader = new ESIndexGlobalIndexReader( new LocalFileReader(), - List.of(ioMeta(currentDir, currentEntry)), + List.of(ioMeta(currentEntry)), currentFields, currentOptions); UnionGlobalIndexReader union = @@ -710,8 +705,8 @@ void textMultiFieldServesBothMatchAndExact(@TempDir java.nio.file.Path tmp) thro ResultEntry entry = w.finish().get(0); GlobalIndexIOMeta ioMeta = new GlobalIndexIOMeta( - new org.apache.paimon.fs.Path(dir.resolve(entry.fileName()).toString()), - Files.size(dir.resolve(entry.fileName())), + new org.apache.paimon.fs.Path(entry.filePath()), + Files.size(Paths.get(entry.filePath())), entry.meta()); ESIndexGlobalIndexReader reader = new ESIndexGlobalIndexReader( @@ -754,8 +749,8 @@ void fullTextSearchHonorsIncludeRowIdsBeforeTopK(@TempDir java.nio.file.Path tmp ResultEntry entry = writer.finish().get(0); GlobalIndexIOMeta ioMeta = new GlobalIndexIOMeta( - new org.apache.paimon.fs.Path(dir.resolve(entry.fileName()).toString()), - Files.size(dir.resolve(entry.fileName())), + new org.apache.paimon.fs.Path(entry.filePath()), + Files.size(Paths.get(entry.filePath())), entry.meta()); ESIndexGlobalIndexReader reader = @@ -802,8 +797,8 @@ void nullLiteralsFollowSqlThreeValuedLogicAndKeywordRangeFallsBack( ResultEntry entry = writer.finish().get(0); GlobalIndexIOMeta ioMeta = new GlobalIndexIOMeta( - new org.apache.paimon.fs.Path(dir.resolve(entry.fileName()).toString()), - Files.size(dir.resolve(entry.fileName())), + new org.apache.paimon.fs.Path(entry.filePath()), + Files.size(Paths.get(entry.filePath())), entry.meta()); ESIndexGlobalIndexReader reader = @@ -876,9 +871,8 @@ void arrayLongLabelsRoundTripAsMultiValueScalar(@TempDir java.nio.file.Path tmp) ResultEntry entry = entries.get(0); assertEquals(3L, entry.rowCount(), "row count"); - org.apache.paimon.fs.Path filePath = - new org.apache.paimon.fs.Path(archiveDir.resolve(entry.fileName()).toString()); - long fileSize = Files.size(archiveDir.resolve(entry.fileName())); + org.apache.paimon.fs.Path filePath = new org.apache.paimon.fs.Path(entry.filePath()); + long fileSize = Files.size(Paths.get(entry.filePath())); GlobalIndexIOMeta ioMeta = new GlobalIndexIOMeta(filePath, fileSize, entry.meta()); ESIndexGlobalIndexReader reader = new ESIndexGlobalIndexReader( @@ -936,9 +930,8 @@ void nullValuesKeepDocIdRowIdAligned(@TempDir java.nio.file.Path tmp) throws IOE ResultEntry entry = entries.get(0); assertEquals((long) n, entry.rowCount(), "row count includes the null row"); - org.apache.paimon.fs.Path filePath = - new org.apache.paimon.fs.Path(archiveDir.resolve(entry.fileName()).toString()); - long fileSize = Files.size(archiveDir.resolve(entry.fileName())); + org.apache.paimon.fs.Path filePath = new org.apache.paimon.fs.Path(entry.filePath()); + long fileSize = Files.size(Paths.get(entry.filePath())); GlobalIndexIOMeta ioMeta = new GlobalIndexIOMeta(filePath, fileSize, entry.meta()); ESIndexGlobalIndexReader reader = new ESIndexGlobalIndexReader( @@ -980,7 +973,7 @@ void skippedRowIdsAreCountedAndRemainSearchableAsNull(@TempDir java.nio.file.Pat ESIndexGlobalIndexReader reader = new ESIndexGlobalIndexReader( - new LocalFileReader(), List.of(ioMeta(archiveDir, entry)), fields, options); + new LocalFileReader(), List.of(ioMeta(entry)), fields, options); try { Optional nullRows = reader.visitIsNull(new FieldRef(0, "k", DataTypes.STRING())).join(); @@ -1019,9 +1012,8 @@ void likeHonorsSqlEscapeAndLuceneLiterals(@TempDir java.nio.file.Path tmp) throw List entries = writer.finish(); ResultEntry entry = entries.get(0); - org.apache.paimon.fs.Path filePath = - new org.apache.paimon.fs.Path(archiveDir.resolve(entry.fileName()).toString()); - long fileSize = Files.size(archiveDir.resolve(entry.fileName())); + org.apache.paimon.fs.Path filePath = new org.apache.paimon.fs.Path(entry.filePath()); + long fileSize = Files.size(Paths.get(entry.filePath())); GlobalIndexIOMeta ioMeta = new GlobalIndexIOMeta(filePath, fileSize, entry.meta()); ESIndexGlobalIndexReader reader = new ESIndexGlobalIndexReader( @@ -1091,11 +1083,10 @@ void closeWaitsForInFlightLoadAndClosesEveryStream(@TempDir java.nio.file.Path t writer.write(BinaryString.fromString("admin_001"), 0); ResultEntry entry = writer.finish().get(0); - org.apache.paimon.fs.Path filePath = - new org.apache.paimon.fs.Path(archiveDir.resolve(entry.fileName()).toString()); + org.apache.paimon.fs.Path filePath = new org.apache.paimon.fs.Path(entry.filePath()); GlobalIndexIOMeta ioMeta = new GlobalIndexIOMeta( - filePath, Files.size(archiveDir.resolve(entry.fileName())), entry.meta()); + filePath, Files.size(Paths.get(entry.filePath())), entry.meta()); BlockingCountingFileReader fileReader = new BlockingCountingFileReader(); ExecutorService queryExecutor = Executors.newSingleThreadExecutor(); ExecutorService closeExecutor = Executors.newSingleThreadExecutor(); @@ -1153,7 +1144,7 @@ void failedLazyLoadClosesStreamsBeforeRetry(@TempDir java.nio.file.Path tmp) thr new ESIndexGlobalIndexWriter(new LocalDirWriter(dir), fields, options); writer.write(BinaryString.fromString("x"), 0); ResultEntry entry = writer.finish().get(0); - java.nio.file.Path badArchive = dir.resolve(entry.fileName()); + java.nio.file.Path badArchive = Paths.get(entry.filePath()); long declaredArchiveSize = Files.size(badArchive); // Keep the valid offset metadata, but truncate the archive so failure happens after the // root provider (and potentially Lucene forks) have actually been opened. @@ -1215,9 +1206,8 @@ void isNullReturnsNullRowsViaIndex(@TempDir java.nio.file.Path tmp) throws IOExc List entries = writer.finish(); ResultEntry entry = entries.get(0); - org.apache.paimon.fs.Path filePath = - new org.apache.paimon.fs.Path(archiveDir.resolve(entry.fileName()).toString()); - long fileSize = Files.size(archiveDir.resolve(entry.fileName())); + org.apache.paimon.fs.Path filePath = new org.apache.paimon.fs.Path(entry.filePath()); + long fileSize = Files.size(Paths.get(entry.filePath())); GlobalIndexIOMeta ioMeta = new GlobalIndexIOMeta(filePath, fileSize, entry.meta()); ESIndexGlobalIndexReader reader = new ESIndexGlobalIndexReader( @@ -1293,11 +1283,10 @@ void exclusiveNumericExtremaDoNotOverflow(@TempDir java.nio.file.Path tmp) throw Double.POSITIVE_INFINITY)); ResultEntry entry = writer.finish().get(0); - org.apache.paimon.fs.Path filePath = - new org.apache.paimon.fs.Path(archiveDir.resolve(entry.fileName()).toString()); + org.apache.paimon.fs.Path filePath = new org.apache.paimon.fs.Path(entry.filePath()); GlobalIndexIOMeta ioMeta = new GlobalIndexIOMeta( - filePath, Files.size(archiveDir.resolve(entry.fileName())), entry.meta()); + filePath, Files.size(Paths.get(entry.filePath())), entry.meta()); ESIndexGlobalIndexReader reader = new ESIndexGlobalIndexReader( new LocalFileReader(), Arrays.asList(ioMeta), fields, options); @@ -1365,11 +1354,10 @@ void temporalPredicatesUseLongScalarPath(@TempDir java.nio.file.Path tmp) throws writer.write(3, GenericRow.of(null, null)); ResultEntry entry = writer.finish().get(0); - org.apache.paimon.fs.Path filePath = - new org.apache.paimon.fs.Path(archiveDir.resolve(entry.fileName()).toString()); + org.apache.paimon.fs.Path filePath = new org.apache.paimon.fs.Path(entry.filePath()); GlobalIndexIOMeta ioMeta = new GlobalIndexIOMeta( - filePath, Files.size(archiveDir.resolve(entry.fileName())), entry.meta()); + filePath, Files.size(Paths.get(entry.filePath())), entry.meta()); ESIndexGlobalIndexReader reader = new ESIndexGlobalIndexReader( new LocalFileReader(), List.of(ioMeta), fields, options); @@ -1443,7 +1431,7 @@ void buildTimeFieldConfigurationIsReadFromIndexMetadata(@TempDir java.nio.file.P writer.write(BinaryString.fromString("viewer"), 1); ResultEntry entry = writer.finish().get(0); - GlobalIndexIOMeta ioMeta = ioMeta(archiveDir, entry); + GlobalIndexIOMeta ioMeta = ioMeta(entry); // Deliberately use the current/default FULLTEXT options. The reader must instead use the // KEYWORD-primary layout persisted when this particular index was built. ESIndexOptions currentTableOptions = new ESIndexOptions(fields, new Options()); @@ -1484,7 +1472,7 @@ void tinySmallAndEmptyArrayNullSemanticsAreIndexed(@TempDir java.nio.file.Path t ESIndexGlobalIndexReader reader = new ESIndexGlobalIndexReader( - new LocalFileReader(), List.of(ioMeta(archiveDir, entry)), fields, options); + new LocalFileReader(), List.of(ioMeta(entry)), fields, options); try { Optional tiny = reader.visitEqual(new FieldRef(0, "tiny", DataTypes.TINYINT()), (byte) 1) @@ -1538,8 +1526,7 @@ void closeCanRetryTransientStreamFailures(@TempDir java.nio.file.Path tmp) throw TransientCloseFailureFileReader fileReader = new TransientCloseFailureFileReader(); ESIndexGlobalIndexReader reader = - new ESIndexGlobalIndexReader( - fileReader, List.of(ioMeta(archiveDir, entry)), fields, options); + new ESIndexGlobalIndexReader(fileReader, List.of(ioMeta(entry)), fields, options); assertTrue(reader.visitEqual(new FieldRef(0, "id", DataTypes.INT()), 1).join().isPresent()); fileReader.setCloseFailures(true); @@ -1564,7 +1551,7 @@ void highPrecisionTimestampPredicatesFallBackToRawScan(@TempDir java.nio.file.Pa ESIndexGlobalIndexReader reader = new ESIndexGlobalIndexReader( - new LocalFileReader(), List.of(ioMeta(archiveDir, entry)), fields, options); + new LocalFileReader(), List.of(ioMeta(entry)), fields, options); try { FieldRef ts = new FieldRef(0, "ts", DataTypes.TIMESTAMP(6)); Optional fallback = @@ -1596,7 +1583,7 @@ void standardBooleanDslAndCamelCaseFuzzyParamsAreApplied(@TempDir java.nio.file. ESIndexGlobalIndexReader reader = new ESIndexGlobalIndexReader( - new LocalFileReader(), List.of(ioMeta(archiveDir, entry)), fields, options); + new LocalFileReader(), List.of(ioMeta(entry)), fields, options); String query = "{\"boolean\":{\"must\":[{\"match\":{\"query\":\"administrater\"," + "\"fuzziness\":\"AUTO\",\"maxExpansions\":20,\"prefixLength\":0}}]," @@ -1628,11 +1615,10 @@ void standardBooleanDslAndCamelCaseFuzzyParamsAreApplied(@TempDir java.nio.file. } } - private static GlobalIndexIOMeta ioMeta(java.nio.file.Path archiveDir, ResultEntry entry) - throws IOException { + private static GlobalIndexIOMeta ioMeta(ResultEntry entry) throws IOException { return new GlobalIndexIOMeta( - new org.apache.paimon.fs.Path(archiveDir.resolve(entry.fileName()).toString()), - Files.size(archiveDir.resolve(entry.fileName())), + new org.apache.paimon.fs.Path(entry.filePath()), + Files.size(Paths.get(entry.filePath())), entry.meta()); } @@ -1656,7 +1642,7 @@ void vectorSearchReturnsPaimonCosineScoreScale(@TempDir java.nio.file.Path tmp) ESIndexGlobalIndexReader reader = new ESIndexGlobalIndexReader( - new LocalFileReader(), List.of(ioMeta(archiveDir, entry)), fields, options); + new LocalFileReader(), List.of(ioMeta(entry)), fields, options); try { ScoredGlobalIndexResult result = reader.visitVectorSearch( @@ -1756,9 +1742,8 @@ void writeDiskBBQArchiveThenReadAndSearch(@TempDir java.nio.file.Path tmp) throw "archive must not contain HNSW-specific files (.vex/.vem), got: " + exts); // --- reconstruct reader over the archive --- - org.apache.paimon.fs.Path filePath = - new org.apache.paimon.fs.Path(archiveDir.resolve(entry.fileName()).toString()); - long fileSize = Files.size(archiveDir.resolve(entry.fileName())); + org.apache.paimon.fs.Path filePath = new org.apache.paimon.fs.Path(entry.filePath()); + long fileSize = Files.size(Paths.get(entry.filePath())); GlobalIndexIOMeta ioMeta = new GlobalIndexIOMeta(filePath, fileSize, entry.meta()); BlockingCountingFileReader fileReader = new BlockingCountingFileReader(); diff --git a/paimon-eslib/src/test/java/org/apache/paimon/eslib/index/ESIndexVectorMetricTest.java b/paimon-eslib/src/test/java/org/apache/paimon/eslib/index/ESIndexVectorMetricTest.java index 4f92ab06fb0c..23a472d3366a 100644 --- a/paimon-eslib/src/test/java/org/apache/paimon/eslib/index/ESIndexVectorMetricTest.java +++ b/paimon-eslib/src/test/java/org/apache/paimon/eslib/index/ESIndexVectorMetricTest.java @@ -43,6 +43,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; @@ -167,7 +168,7 @@ void readerDoesNotRequireCurrentOptionsToDescribeBuild(@TempDir Path tempDir) th writer.write(new float[] {0f, 1f}, 1); writer.write(new float[] {-1f, 0f}, 2); ResultEntry entry = writer.finish().get(0); - Path archive = archiveDir.resolve(entry.fileName()); + Path archive = Paths.get(entry.filePath()); GlobalIndexIOMeta ioMeta = new GlobalIndexIOMeta( new org.apache.paimon.fs.Path(archive.toString()), @@ -278,14 +279,15 @@ private static final class LocalDirWriter implements GlobalIndexFileWriter { } @Override - public String newFileName(String prefix) { - return prefix + "-" + UUID.randomUUID() + ".index"; + public String newFilePath(String prefix) { + return new org.apache.paimon.fs.Path( + dir.resolve(prefix + "-" + UUID.randomUUID() + ".index").toString()) + .toString(); } @Override - public PositionOutputStream newOutputStream(String fileName) throws IOException { - return fio.newOutputStream( - new org.apache.paimon.fs.Path(dir.resolve(fileName).toString()), true); + public PositionOutputStream newOutputStream(String filePath) throws IOException { + return fio.newOutputStream(new org.apache.paimon.fs.Path(filePath), true); } } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java index 352bc76942d8..42c07a1fa0b7 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java @@ -19,6 +19,7 @@ package org.apache.paimon.flink.globalindex; import org.apache.paimon.CoreOptions; +import org.apache.paimon.annotation.VisibleForTesting; import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.flink.sink.Committable; @@ -29,6 +30,7 @@ import org.apache.paimon.flink.utils.BoundedOneInputOperator; import org.apache.paimon.flink.utils.JavaTypeInfo; import org.apache.paimon.flink.utils.StreamExecutionEnvironmentUtils; +import org.apache.paimon.globalindex.GlobalIndexFileReadWrite; import org.apache.paimon.globalindex.GlobalIndexMultiColumnWriter; import org.apache.paimon.globalindex.GlobalIndexSingleColumnWriter; import org.apache.paimon.globalindex.GlobalIndexWriter; @@ -75,6 +77,7 @@ import java.util.function.Supplier; import java.util.stream.Collectors; +import static org.apache.paimon.globalindex.GlobalIndexBuilderUtils.createGlobalIndexFileReadWrite; import static org.apache.paimon.globalindex.GlobalIndexBuilderUtils.createIndexWriter; import static org.apache.paimon.globalindex.GlobalIndexBuilderUtils.createShardIndexedSplits; import static org.apache.paimon.globalindex.GlobalIndexBuilderUtils.toIndexFileMetas; @@ -355,8 +358,8 @@ private static void commit( * and emits a {@link Committable}. Each shard's split contains only the files relevant to this * shard, so no redundant I/O occurs. */ - private static class BuildIndexOperator - extends BoundedOneInputOperator { + @VisibleForTesting + static class BuildIndexOperator extends BoundedOneInputOperator { private static final long serialVersionUID = 1L; @@ -370,6 +373,7 @@ private static class BuildIndexOperator private final byte[] sourceMeta; private transient TableRead tableRead; + private transient GlobalIndexFileReadWrite fileReadWrite; private transient List indexedFields; private transient InternalRow.FieldGetter[] indexFieldGetters; private transient int rowIdFieldIndex; @@ -399,6 +403,7 @@ private static class BuildIndexOperator public void open() throws Exception { super.open(); this.tableRead = readBuilder.newRead(); + this.fileReadWrite = createGlobalIndexFileReadWrite(table); // The primary column followed by the extra columns, in index order. Field getters and // the writer projection both need the full ordered list. this.indexedFields = new ArrayList<>(1 + extraFields.size()); @@ -441,7 +446,7 @@ public void processElement(StreamRecord element) throws Exception long startTime = System.currentTimeMillis(); GlobalIndexWriter indexWriter = - createIndexWriter(table, indexType, indexedFields, mergedOptions); + createIndexWriter(fileReadWrite, indexType, indexedFields, mergedOptions); try { long rowsSeen = 0; @@ -542,7 +547,6 @@ private static CommitMessage flushIndex( List indexFileMetas = toIndexFileMetas( table.fileIO(), - table.store().pathFactory().globalIndexFileFactory(), table.coreOptions(), rowRange, indexFields, diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilderTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilderTest.java index 343c74419440..d5d7816384ff 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilderTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilderTest.java @@ -18,37 +18,60 @@ package org.apache.paimon.flink.globalindex; +import org.apache.paimon.CoreOptions; import org.apache.paimon.FileStore; import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.BinaryRowWriter; import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.flink.sink.Committable; +import org.apache.paimon.flink.sink.CommittableTypeInfo; import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.globalindex.IndexedSplit; import org.apache.paimon.globalindex.generic.GenericGlobalIndexScanner; +import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.io.PojoDataFileMeta; import org.apache.paimon.manifest.FileKind; import org.apache.paimon.manifest.ManifestEntry; import org.apache.paimon.options.Options; import org.apache.paimon.stats.SimpleStats; import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.SpecialFields; +import org.apache.paimon.table.sink.CommitMessageImpl; +import org.apache.paimon.table.source.ReadBuilder; +import org.apache.paimon.table.source.TableRead; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.RowType; import org.apache.paimon.utils.FileStorePathFactory; +import org.apache.paimon.utils.IteratorRecordReader; import org.apache.paimon.utils.Range; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.UUID; +import static org.apache.paimon.utils.FileStorePathFactoryTest.createNonPartFactory; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; /** Tests for shard computation in {@link GenericIndexTopoBuilder}. */ @@ -233,6 +256,78 @@ void testShardRangeClampedToFileRange() throws IOException { assertThat(tasks.get(1).rowRanges().get(0)).isEqualTo(new Range(100, 149)); } + @Test + void testRoundRobinAcrossShards(@TempDir java.nio.file.Path directory) throws Exception { + Path root = new Path(directory.toUri()); + List roots = Arrays.asList(new Path(root, "a"), new Path(root, "b")); + CoreOptions options = + new CoreOptions( + Collections.singletonMap( + CoreOptions.GLOBAL_INDEX_EXTERNAL_PATH.key(), + roots.get(0) + "," + roots.get(1))); + FileStorePathFactory paths = table.store().pathFactory(); + when(paths.globalIndexFileFactory()) + .thenAnswer(ignored -> createNonPartFactory(root, roots).globalIndexFileFactory()); + when(table.fileIO()).thenReturn(new LocalFileIO()); + when(table.coreOptions()).thenReturn(options); + DataField field = new DataField(0, "id", DataTypes.INT()); + RowType readType = + SpecialFields.rowTypeWithRowId(new RowType(Collections.singletonList(field))); + ReadBuilder readBuilder = mock(ReadBuilder.class); + TableRead read = mock(TableRead.class); + when(readBuilder.newRead()).thenReturn(read); + List commits; + try (OneInputStreamOperatorTestHarness harness = + new OneInputStreamOperatorTestHarness<>( + new GenericIndexTopoBuilder.BuildIndexOperator( + readBuilder, + table, + "bitmap", + field, + Collections.emptyList(), + readType, + options.toConfiguration(), + null))) { + harness.setup(new CommittableTypeInfo().createSerializer(new ExecutionConfig())); + harness.open(); + for (int i = 0; i < 6; i++) { + IndexedSplit split = + GenericIndexTopoBuilder.computeShardTasks( + table, + Collections.singletonList( + createEntry(BinaryRow.EMPTY_ROW, (long) i, 1)), + 1) + .get(0); + when(read.createReader(split.dataSplit())) + .thenReturn( + new IteratorRecordReader<>( + Collections.singletonList( + GenericRow.of(i, (long) i)) + .iterator())); + harness.processElement(new StreamRecord<>(split)); + } + commits = harness.extractOutputValues(); + } + assertThat(commits).hasSize(6); + List parents = new ArrayList<>(); + for (Committable commit : commits) { + List files = + ((CommitMessageImpl) commit.commitMessage()) + .newFilesIncrement() + .newIndexFiles(); + assertThat(files).hasSize(1); + Path file = new Path(files.get(0).externalPath()); + assertThat(table.fileIO().exists(file)).isTrue(); + assertThat(files.get(0).fileName()).isEqualTo(file.getName()); + parents.add(file.getParent()); + } + assertThat(parents).containsOnlyElementsOf(roots); + for (int i = 1; i < parents.size(); i++) { + assertThat(parents.get(i)).isNotEqualTo(parents.get(i - 1)); + } + verify(paths, times(1)).globalIndexFileFactory(); + } + // -- Helpers -- private static ManifestEntry createEntry(BinaryRow partition, Long firstRowId, long rowCount) { diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/VectorSearchProcedureITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/VectorSearchProcedureITCase.java index a874a5030fc6..aa0e99738354 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/VectorSearchProcedureITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/VectorSearchProcedureITCase.java @@ -1159,7 +1159,6 @@ private void buildAndCommitVectorIndex(FileStoreTable table, float[][] vectors, List indexFiles = GlobalIndexBuilderUtils.toIndexFileMetas( table.fileIO(), - table.store().pathFactory().globalIndexFileFactory(), table.coreOptions(), rowRange, vectorField.id(), @@ -1200,7 +1199,6 @@ private void buildAndCommitMultiFieldVectorIndex( List indexFiles = GlobalIndexBuilderUtils.toIndexFileMetas( table.fileIO(), - table.store().pathFactory().globalIndexFileFactory(), table.coreOptions(), rowRange, Arrays.asList(vectorField, idField), diff --git a/paimon-full-text/src/main/java/org/apache/paimon/fulltext/index/NativeFullTextGlobalIndexWriter.java b/paimon-full-text/src/main/java/org/apache/paimon/fulltext/index/NativeFullTextGlobalIndexWriter.java index 13f03ba46088..28a53fcbb04b 100644 --- a/paimon-full-text/src/main/java/org/apache/paimon/fulltext/index/NativeFullTextGlobalIndexWriter.java +++ b/paimon-full-text/src/main/java/org/apache/paimon/fulltext/index/NativeFullTextGlobalIndexWriter.java @@ -104,8 +104,8 @@ public List finish() { } private ResultEntry writeIndex() throws IOException { - String fileName = fileWriter.newFileName(FILE_NAME_PREFIX); - try (PositionOutputStream out = fileWriter.newOutputStream(fileName)) { + String filePath = fileWriter.newFilePath(FILE_NAME_PREFIX); + try (PositionOutputStream out = fileWriter.newOutputStream(filePath)) { writer.writeIndex( new FullTextIndexOutput() { @Override @@ -121,7 +121,7 @@ public void flush() throws IOException { }); out.flush(); } - return new ResultEntry(fileName, rowCount, indexOptions.serialize()); + return new ResultEntry(filePath, rowCount, indexOptions.serialize()); } @Override diff --git a/paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/JavaPyNativeFullTextE2ETest.java b/paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/JavaPyNativeFullTextE2ETest.java index 3ac0c4a3b5e8..de864a34de48 100644 --- a/paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/JavaPyNativeFullTextE2ETest.java +++ b/paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/JavaPyNativeFullTextE2ETest.java @@ -220,7 +220,6 @@ private void writeTableWithNativeFullTextIndex( List indexFiles = GlobalIndexBuilderUtils.toIndexFileMetas( table.fileIO(), - table.store().pathFactory().globalIndexFileFactory(), table.coreOptions(), rowRange, contentField.id(), diff --git a/paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/NativeFullTextGlobalIndexTest.java b/paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/NativeFullTextGlobalIndexTest.java index 7f11936b4b3c..de2df5c4dcea 100644 --- a/paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/NativeFullTextGlobalIndexTest.java +++ b/paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/NativeFullTextGlobalIndexTest.java @@ -75,13 +75,13 @@ public void cleanup() throws IOException { private GlobalIndexFileWriter createFileWriter(Path path) { return new GlobalIndexFileWriter() { @Override - public String newFileName(String prefix) { - return prefix + "-" + UUID.randomUUID(); + public String newFilePath(String prefix) { + return new Path(path, prefix + "-" + UUID.randomUUID()).toString(); } @Override - public PositionOutputStream newOutputStream(String fileName) throws IOException { - return fileIO.newOutputStream(new Path(path, fileName), false); + public PositionOutputStream newOutputStream(String filePath) throws IOException { + return fileIO.newOutputStream(new Path(filePath), false); } }; } @@ -90,11 +90,10 @@ private GlobalIndexFileReader createFileReader() { return meta -> fileIO.newInputStream(meta.filePath()); } - private List toIOMetas(List results, Path path) - throws IOException { + private List toIOMetas(List results) throws IOException { assertThat(results).hasSize(1); ResultEntry result = results.get(0); - Path filePath = new Path(path, result.fileName()); + Path filePath = new Path(result.filePath()); return Collections.singletonList( new GlobalIndexIOMeta(filePath, fileIO.getFileSize(filePath), result.meta())); } @@ -128,7 +127,7 @@ public void testEndToEnd() throws IOException { assertThat(results).hasSize(1); assertThat(results.get(0).rowCount()).isEqualTo(3); - List metas = toIOMetas(results, indexPath); + List metas = toIOMetas(results); GlobalIndexFileReader fileReader = createFileReader(); try (NativeFullTextGlobalIndexReader reader = createReader(fileReader, metas)) { @@ -189,7 +188,7 @@ public void testJiebaTokenizerFindsChineseWord() throws IOException { List results = writer.finish(); assertThat(serializedOptions(results.get(0).meta())).containsEntry("tokenizer", "jieba"); - List metas = toIOMetas(results, indexPath); + List metas = toIOMetas(results); GlobalIndexFileReader fileReader = createFileReader(); try (NativeFullTextGlobalIndexReader reader = createReader(fileReader, metas)) { @@ -227,7 +226,7 @@ public void testIcuSplitTokenizerFindsChineseWordsAndIdentifierParts() throws IO .containsEntry("tokenizer", "icu/split"); try (NativeFullTextGlobalIndexReader reader = - createReader(createFileReader(), toIOMetas(results, indexPath))) { + createReader(createFileReader(), toIOMetas(results))) { for (String query : new String[] {"中文", "RUST", "world", "beta", "getusername"}) { Optional searchResult = reader.visitFullTextSearch( @@ -255,7 +254,7 @@ public void testSearchNoResults() throws IOException { writer.write(BinaryString.fromString("Foo bar baz"), 1); List results = writer.finish(); - List metas = toIOMetas(results, indexPath); + List metas = toIOMetas(results); GlobalIndexFileReader fileReader = createFileReader(); try (NativeFullTextGlobalIndexReader reader = createReader(fileReader, metas)) { @@ -281,7 +280,7 @@ public void testNullFieldSkipped() throws IOException { List results = writer.finish(); assertThat(results.get(0).rowCount()).isEqualTo(3); - List metas = toIOMetas(results, indexPath); + List metas = toIOMetas(results); GlobalIndexFileReader fileReader = createFileReader(); try (NativeFullTextGlobalIndexReader reader = createReader(fileReader, metas)) { @@ -326,7 +325,7 @@ public void testLargeDataset() throws IOException { List results = writer.finish(); assertThat(results.get(0).rowCount()).isEqualTo(numDocs); - List metas = toIOMetas(results, indexPath); + List metas = toIOMetas(results); GlobalIndexFileReader fileReader = createFileReader(); try (NativeFullTextGlobalIndexReader reader = createReader(fileReader, metas)) { @@ -356,7 +355,7 @@ public void testLimitRespected() throws IOException { } List results = writer.finish(); - List metas = toIOMetas(results, indexPath); + List metas = toIOMetas(results); GlobalIndexFileReader fileReader = createFileReader(); try (NativeFullTextGlobalIndexReader reader = createReader(fileReader, metas)) { @@ -381,7 +380,7 @@ public void testIncludeRowIdsSearchesFullShardBeforeTopK() throws IOException { writer.write(BinaryString.fromString("paimon"), 2); List results = writer.finish(); - List metas = toIOMetas(results, indexPath); + List metas = toIOMetas(results); GlobalIndexFileReader fileReader = createFileReader(); RoaringNavigableMap64 includeRowIds = new RoaringNavigableMap64(); @@ -409,7 +408,7 @@ public void testRepeatedReadersReturnConsistentResults() throws IOException { writer.write(BinaryString.fromString("Native full-text search"), 1); List results = writer.finish(); - List metas = toIOMetas(results, indexPath); + List metas = toIOMetas(results); GlobalIndexFileReader fileReader = createFileReader(); FullTextSearch search = new FullTextSearch("text", matchQuery("paimon"), 10); @@ -439,7 +438,7 @@ public void testViaIndexer() throws IOException { List results = writer.finish(); assertThat(results).hasSize(1); - List metas = toIOMetas(results, indexPath); + List metas = toIOMetas(results); GlobalIndexFileReader fileReader = createFileReader(); try (NativeFullTextGlobalIndexReader reader = @@ -464,7 +463,7 @@ public void testStructuredBoostQueryDemotesNegativeMatches() throws IOException writer.write(BinaryString.fromString("paimon vector"), 1); List results = writer.finish(); - List metas = toIOMetas(results, indexPath); + List metas = toIOMetas(results); GlobalIndexFileReader fileReader = createFileReader(); try (NativeFullTextGlobalIndexReader reader = createReader(fileReader, metas)) { diff --git a/paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/NativeFullTextRowFilterTest.java b/paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/NativeFullTextRowFilterTest.java index a659b1860a7d..cfbe20e3dd4a 100644 --- a/paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/NativeFullTextRowFilterTest.java +++ b/paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/NativeFullTextRowFilterTest.java @@ -459,7 +459,6 @@ private Dataset writeDataset( new ArrayList<>( GlobalIndexBuilderUtils.toIndexFileMetas( table.fileIO(), - table.store().pathFactory().globalIndexFileFactory(), table.coreOptions(), new Range(0, fullTextIndexedRows - 1), contentField.id(), @@ -491,7 +490,6 @@ private Dataset writeDataset( indexFiles.addAll( GlobalIndexBuilderUtils.toIndexFileMetas( table.fileIO(), - table.store().pathFactory().globalIndexFileFactory(), table.coreOptions(), rowRange, categoryField.id(), diff --git a/paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/NativePrimaryKeyFullTextIndexTest.java b/paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/NativePrimaryKeyFullTextIndexTest.java index db0c26b7cc62..5f1656964ab5 100644 --- a/paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/NativePrimaryKeyFullTextIndexTest.java +++ b/paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/NativePrimaryKeyFullTextIndexTest.java @@ -180,10 +180,10 @@ private IndexFileMeta buildArchive(List texts, Options options) th List results = writer.finish(); assertThat(results).hasSize(1); ResultEntry result = results.get(0); - Path path = new Path(indexPath, result.fileName()); + Path path = new Path(result.filePath()); return new IndexFileMeta( "full-text", - result.fileName(), + new Path(result.filePath()).getName(), fileIO.getFileSize(path), result.rowCount(), new GlobalIndexMeta( @@ -201,13 +201,13 @@ private IndexFileMeta buildArchive(List texts, Options options) th private GlobalIndexFileWriter fileWriter() { return new GlobalIndexFileWriter() { @Override - public String newFileName(String prefix) { - return prefix + "-" + UUID.randomUUID(); + public String newFilePath(String prefix) { + return new Path(indexPath, prefix + "-" + UUID.randomUUID()).toString(); } @Override - public PositionOutputStream newOutputStream(String fileName) throws IOException { - return fileIO.newOutputStream(new Path(indexPath, fileName), false); + public PositionOutputStream newOutputStream(String filePath) throws IOException { + return fileIO.newOutputStream(new Path(filePath), false); } }; } diff --git a/paimon-lumina/README.md b/paimon-lumina/README.md index feef158c3f64..8ac06c305186 100644 --- a/paimon-lumina/README.md +++ b/paimon-lumina/README.md @@ -8,6 +8,16 @@ Alibaba Storage Service Team. It is accessed via JNI through the `lumina-jni` ar **Note:** Lumina native library only supports **x86_64 (AMD64)** architecture. It is not available on ARM (e.g., Apple Silicon, aarch64). +### Upgrading with Paimon + +Deploy `paimon-lumina` and the Paimon engine connector/runtime from the same Paimon +release. For snapshot or custom builds, build both from the same source revision. +The Java writer interface for multiple external index paths requires an updated +Lumina plugin, even when only one external directory is configured or the option +is unset. Replace the old plugin JAR when upgrading Paimon; updating only +`lumina-jni` is insufficient. See [Upgrade Java Index Plugins](../docs/docs/multimodal-table/global-index/manage-indexes.mdx#upgrade-java-index-plugins) +for deployment requirements and interface migration details. + ### Supported Index Types | Index Type | Description | diff --git a/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexWriter.java b/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexWriter.java index ac52e418d5ad..5eede90d77ec 100644 --- a/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexWriter.java +++ b/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexWriter.java @@ -279,8 +279,8 @@ private ResultEntry buildIndex() throws IOException { phaseStart = System.currentTimeMillis(); LOG.info("Lumina dump phase started"); - String fileName = fileWriter.newFileName(FILE_NAME_PREFIX); - try (PositionOutputStream out = fileWriter.newOutputStream(fileName)) { + String filePath = fileWriter.newFilePath(FILE_NAME_PREFIX); + try (PositionOutputStream out = fileWriter.newOutputStream(filePath)) { index.dump(new OutputStreamFileOutput(out)); out.flush(); } @@ -292,7 +292,7 @@ private ResultEntry buildIndex() throws IOException { LuminaIndexMeta meta = new LuminaIndexMeta(luminaOptions); // rowCount = logical rows including nulls (not just indexed vectors) - return new ResultEntry(fileName, rowCount, meta.serialize()); + return new ResultEntry(filePath, rowCount, meta.serialize()); } } diff --git a/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/JavaPyLuminaE2ETest.java b/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/JavaPyLuminaE2ETest.java index 1f14493b282b..eedb01e238d4 100644 --- a/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/JavaPyLuminaE2ETest.java +++ b/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/JavaPyLuminaE2ETest.java @@ -189,7 +189,6 @@ public void testLuminaVectorIndexWrite(String fileFormat) throws Exception { List indexFiles = GlobalIndexBuilderUtils.toIndexFileMetas( table.fileIO(), - table.store().pathFactory().globalIndexFileFactory(), table.coreOptions(), rowRange, embeddingField.id(), @@ -293,7 +292,6 @@ public void testLuminaVectorWithBTreeIndexWrite() throws Exception { List vectorIndexFiles = GlobalIndexBuilderUtils.toIndexFileMetas( table.fileIO(), - table.store().pathFactory().globalIndexFileFactory(), table.coreOptions(), rowRange, embeddingField.id(), @@ -316,7 +314,6 @@ public void testLuminaVectorWithBTreeIndexWrite() throws Exception { List idIndexFiles = GlobalIndexBuilderUtils.toIndexFileMetas( table.fileIO(), - table.store().pathFactory().globalIndexFileFactory(), table.coreOptions(), rowRange, idField.id(), diff --git a/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorBenchmark.java b/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorBenchmark.java index 3511fba4a33a..63c73451937a 100644 --- a/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorBenchmark.java +++ b/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorBenchmark.java @@ -359,21 +359,20 @@ public void benchmarkBuild() throws Exception { buildDir = indexDir; } - String indexFileName; try { final Path finalBuildDir = buildDir; GlobalIndexFileWriter gFileWriter = new GlobalIndexFileWriter() { @Override - public String newFileName(String prefix) { - return prefix + "-" + UUID.randomUUID(); + public String newFilePath(String prefix) { + return new Path(finalBuildDir, prefix + "-" + UUID.randomUUID()) + .toString(); } @Override - public PositionOutputStream newOutputStream(String fileName) + public PositionOutputStream newOutputStream(String filePath) throws IOException { - return buildFileIO.newOutputStream( - new Path(finalBuildDir, fileName), false); + return buildFileIO.newOutputStream(new Path(filePath), false); } }; @@ -404,18 +403,19 @@ public PositionOutputStream newOutputStream(String fileName) "Index build done in %.2f s%n", (finishEnd - finishStart) / 1000.0); } - indexFileName = results.get(0).fileName(); + Path indexFilePath = new Path(results.get(0).filePath()); // Upload to OSS if needed if (isRemote) { - Path localFile = new Path(buildDir, indexFileName); - Path remoteFile = new Path(indexDir, indexFileName); + Path localFile = indexFilePath; + Path remoteFile = new Path(indexDir, localFile.getName()); long localSize = localFileIO.getFileSize(localFile); System.out.printf( "Uploading to OSS: %s (%,d bytes, %.2f GB)...%n", remoteFile, localSize, localSize / (1024.0 * 1024 * 1024)); long uploadStart = System.currentTimeMillis(); uploadFile(localFileIO, localFile, fileIO, remoteFile); + indexFilePath = remoteFile; long uploadEnd = System.currentTimeMillis(); double uploadSec = (uploadEnd - uploadStart) / 1000.0; System.out.printf( @@ -423,7 +423,6 @@ public PositionOutputStream newOutputStream(String fileName) uploadSec, localSize / (1024.0 * 1024) / uploadSec); } - Path indexFilePath = new Path(indexDir, indexFileName); long buildFileSize = fileIO.getFileSize(indexFilePath); long buildEndTime = System.currentTimeMillis(); @@ -517,7 +516,6 @@ public void benchmarkQuery() throws Exception { List ioMetas = Collections.singletonList(ioMeta); final FileIO benchFileIO = fileIO; - final Path benchIndexDir = indexDir; System.out.printf( "Running %,d queries (top-%d), each with fresh index load...%n", numQueries, topK); @@ -540,9 +538,7 @@ public void benchmarkQuery() throws Exception { long queryStart = System.nanoTime(); GlobalIndexFileReader gFileReader = - ioMetaArg -> - benchFileIO.newInputStream( - new Path(benchIndexDir, ioMetaArg.filePath().getName())); + ioMetaArg -> benchFileIO.newInputStream(ioMetaArg.filePath()); ExecutorService executor = Executors.newCachedThreadPool(); try (LuminaVectorGlobalIndexReader reader = new LuminaVectorGlobalIndexReader( diff --git a/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexScanTest.java b/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexScanTest.java index cd7919096555..0ca4e079c415 100644 --- a/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexScanTest.java +++ b/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexScanTest.java @@ -199,14 +199,14 @@ public void testVectorIndexScanWithDifferentMetrics() throws Exception { GlobalIndexFileWriter fileWriter = new GlobalIndexFileWriter() { @Override - public String newFileName(String prefix) { - return prefix + "-" + UUID.randomUUID(); + public String newFilePath(String prefix) { + return new Path(indexDir, prefix + "-" + UUID.randomUUID()).toString(); } @Override - public PositionOutputStream newOutputStream(String fileName) + public PositionOutputStream newOutputStream(String filePath) throws IOException { - return fileIO.newOutputStream(new Path(indexDir, fileName), false); + return fileIO.newOutputStream(new Path(filePath), false); } }; @@ -222,14 +222,14 @@ public PositionOutputStream newOutputStream(String fileName) int fieldId = ipTable.rowType().getFieldIndex(vectorFieldName); for (ResultEntry entry : entries) { - long fileSize = fileIO.getFileSize(new Path(indexDir, entry.fileName())); + long fileSize = fileIO.getFileSize(new Path(entry.filePath())); GlobalIndexMeta globalMeta = new GlobalIndexMeta(0, vectors.length - 1, fieldId, null, entry.meta()); metas.add( new IndexFileMeta( LuminaVectorGlobalIndexerFactory.IDENTIFIER, - entry.fileName(), + new Path(entry.filePath()).getName(), fileSize, entry.rowCount(), globalMeta, @@ -289,14 +289,14 @@ private List buildIndexManually(float[][] vectors) throws Excepti GlobalIndexFileWriter fileWriter = new GlobalIndexFileWriter() { @Override - public String newFileName(String prefix) { - return prefix + "-" + UUID.randomUUID(); + public String newFilePath(String prefix) { + return new Path(indexDir, prefix + "-" + UUID.randomUUID()).toString(); } @Override - public PositionOutputStream newOutputStream(String fileName) + public PositionOutputStream newOutputStream(String filePath) throws IOException { - return fileIO.newOutputStream(new Path(indexDir, fileName), false); + return fileIO.newOutputStream(new Path(filePath), false); } }; @@ -313,14 +313,14 @@ public PositionOutputStream newOutputStream(String fileName) int fieldId = rowType.getFieldIndex(vectorFieldName); for (ResultEntry entry : entries) { - long fileSize = fileIO.getFileSize(new Path(indexDir, entry.fileName())); + long fileSize = fileIO.getFileSize(new Path(entry.filePath())); GlobalIndexMeta globalMeta = new GlobalIndexMeta(0, vectors.length - 1, fieldId, null, entry.meta()); metas.add( new IndexFileMeta( LuminaVectorGlobalIndexerFactory.IDENTIFIER, - entry.fileName(), + new Path(entry.filePath()).getName(), fileSize, entry.rowCount(), globalMeta, diff --git a/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexTest.java b/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexTest.java index c9dd07fddbe4..540889e62e57 100644 --- a/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexTest.java +++ b/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexTest.java @@ -108,26 +108,25 @@ public void cleanup() throws IOException { private GlobalIndexFileWriter createFileWriter(Path path) { return new GlobalIndexFileWriter() { @Override - public String newFileName(String prefix) { - return prefix + "-" + UUID.randomUUID(); + public String newFilePath(String prefix) { + return new Path(path, prefix + "-" + UUID.randomUUID()).toString(); } @Override - public PositionOutputStream newOutputStream(String fileName) throws IOException { - return fileIO.newOutputStream(new Path(path, fileName), false); + public PositionOutputStream newOutputStream(String filePath) throws IOException { + return fileIO.newOutputStream(new Path(filePath), false); } }; } - private GlobalIndexFileReader createFileReader(Path path) { - return meta -> fileIO.newInputStream(new Path(path, meta.filePath())); + private GlobalIndexFileReader createFileReader() { + return meta -> fileIO.newInputStream(meta.filePath()); } - private List toIOMetas(List results, Path path) - throws IOException { + private List toIOMetas(List results) throws IOException { assertThat(results).hasSize(1); ResultEntry result = results.get(0); - Path filePath = new Path(path, result.fileName()); + Path filePath = new Path(result.filePath()); return Collections.singletonList( new GlobalIndexIOMeta(filePath, fileIO.getFileSize(filePath), result.meta())); } @@ -156,9 +155,9 @@ public void testDifferentMetrics() throws IOException { writeVectors(writer, testVectors); List results = writer.finish(); - List metas = toIOMetas(results, metricIndexPath); + List metas = toIOMetas(results); - GlobalIndexFileReader fileReader = createFileReader(metricIndexPath); + GlobalIndexFileReader fileReader = createFileReader(); try (LuminaVectorGlobalIndexReader reader = new LuminaVectorGlobalIndexReader( fileReader, metas, vectorType, indexOptions, executor)) { @@ -191,9 +190,9 @@ public void testDifferentDimensions() throws IOException { writeVectors(writer, testVectors); List results = writer.finish(); - List metas = toIOMetas(results, dimIndexPath); + List metas = toIOMetas(results); - GlobalIndexFileReader fileReader = createFileReader(dimIndexPath); + GlobalIndexFileReader fileReader = createFileReader(); try (LuminaVectorGlobalIndexReader reader = new LuminaVectorGlobalIndexReader( fileReader, metas, vectorType, indexOptions, executor)) { @@ -242,9 +241,9 @@ public void testFloatVectorIndexEndToEnd() throws IOException { writeVectors(writer, vectors); List results = writer.finish(); - List metas = toIOMetas(results, indexPath); + List metas = toIOMetas(results); - GlobalIndexFileReader fileReader = createFileReader(indexPath); + GlobalIndexFileReader fileReader = createFileReader(); try (LuminaVectorGlobalIndexReader reader = new LuminaVectorGlobalIndexReader( fileReader, metas, vectorType, indexOptions, executor)) { @@ -304,9 +303,9 @@ public void testSearchWithFilter() throws IOException { new LuminaVectorGlobalIndexWriter(fileWriter, vectorType, indexOptions); writeVectors(writer, vectors); List results = writer.finish(); - List metas = toIOMetas(results, indexPath); + List metas = toIOMetas(results); - GlobalIndexFileReader fileReader = createFileReader(indexPath); + GlobalIndexFileReader fileReader = createFileReader(); try (LuminaVectorGlobalIndexReader reader = new LuminaVectorGlobalIndexReader( fileReader, metas, vectorType, indexOptions, executor)) { @@ -372,13 +371,13 @@ public void testLargeVectorSet() throws IOException { writeVectors(writer, testVectors); List results = writer.finish(); - List metas = toIOMetas(results, indexPath); + List metas = toIOMetas(results); - Path filePath = new Path(indexPath, results.get(0).fileName()); + Path filePath = new Path(results.get(0).filePath()); assertThat(fileIO.exists(filePath)).isTrue(); assertThat(fileIO.getFileSize(filePath)).isGreaterThan(0); - GlobalIndexFileReader fileReader = createFileReader(indexPath); + GlobalIndexFileReader fileReader = createFileReader(); try (LuminaVectorGlobalIndexReader reader = new LuminaVectorGlobalIndexReader( fileReader, metas, vectorType, indexOptions, executor)) { @@ -422,7 +421,7 @@ public void testReaderMetaOptionsOverrideDefaultOptions() throws IOException { new LuminaVectorGlobalIndexWriter(fileWriter, vectorType, writeIndexOptions); writeVectors(writer, vectors); List results = writer.finish(); - List metas = toIOMetas(results, indexPath); + List metas = toIOMetas(results); // Read with default options (dimension=128, the default) — simulates // the case where table options do not contain lumina.index.dimension. @@ -434,7 +433,7 @@ public void testReaderMetaOptionsOverrideDefaultOptions() throws IOException { LuminaVectorIndexOptions readIndexOptions = new LuminaVectorIndexOptions(readOptions); assertThat(readIndexOptions.dimension()).isEqualTo(128); - GlobalIndexFileReader fileReader = createFileReader(indexPath); + GlobalIndexFileReader fileReader = createFileReader(); try (LuminaVectorGlobalIndexReader reader = new LuminaVectorGlobalIndexReader( fileReader, metas, vectorType, readIndexOptions, executor)) { @@ -475,9 +474,9 @@ public void testVectorTypeEndToEnd() throws IOException { } List results = writer.finish(); - List metas = toIOMetas(results, vecIndexPath); + List metas = toIOMetas(results); - GlobalIndexFileReader fileReader = createFileReader(vecIndexPath); + GlobalIndexFileReader fileReader = createFileReader(); try (LuminaVectorGlobalIndexReader reader = new LuminaVectorGlobalIndexReader( fileReader, metas, vecFieldType, indexOptions, executor)) { @@ -562,8 +561,8 @@ public void testNullVectorSkipWithCorrectIds() throws IOException { assertThat(results).hasSize(1); assertThat(results.get(0).rowCount()).isEqualTo(6); - List metas = toIOMetas(results, indexPath); - GlobalIndexFileReader fileReader = createFileReader(indexPath); + List metas = toIOMetas(results); + GlobalIndexFileReader fileReader = createFileReader(); try (LuminaVectorGlobalIndexReader reader = new LuminaVectorGlobalIndexReader( fileReader, metas, vectorType, indexOptions, executor)) { @@ -629,9 +628,9 @@ public void testNullGapWithPreFilter() throws IOException { writer.write(vectors[3], 5); // row 5 List results = writer.finish(); - List metas = toIOMetas(results, indexPath); + List metas = toIOMetas(results); - GlobalIndexFileReader fileReader = createFileReader(indexPath); + GlobalIndexFileReader fileReader = createFileReader(); try (LuminaVectorGlobalIndexReader reader = new LuminaVectorGlobalIndexReader( fileReader, metas, vectorType, indexOptions, executor)) { @@ -689,8 +688,8 @@ public void testNullAtStart() throws IOException { assertThat(results).hasSize(1); assertThat(results.get(0).rowCount()).isEqualTo(2); - List metas = toIOMetas(results, indexPath); - GlobalIndexFileReader fileReader = createFileReader(indexPath); + List metas = toIOMetas(results); + GlobalIndexFileReader fileReader = createFileReader(); try (LuminaVectorGlobalIndexReader reader = new LuminaVectorGlobalIndexReader( fileReader, metas, vectorType, indexOptions, executor)) { @@ -719,8 +718,8 @@ public void testNullAtEnd() throws IOException { assertThat(results).hasSize(1); assertThat(results.get(0).rowCount()).isEqualTo(2); - List metas = toIOMetas(results, indexPath); - GlobalIndexFileReader fileReader = createFileReader(indexPath); + List metas = toIOMetas(results); + GlobalIndexFileReader fileReader = createFileReader(); try (LuminaVectorGlobalIndexReader reader = new LuminaVectorGlobalIndexReader( fileReader, metas, vectorType, indexOptions, executor)) { @@ -795,9 +794,9 @@ public void testBatchVectorSearch() throws IOException { writeVectors(writer, vectors); List results = writer.finish(); - List metas = toIOMetas(results, indexPath); + List metas = toIOMetas(results); - GlobalIndexFileReader fileReader = createFileReader(indexPath); + GlobalIndexFileReader fileReader = createFileReader(); try (LuminaVectorGlobalIndexReader reader = new LuminaVectorGlobalIndexReader( fileReader, metas, vectorType, indexOptions, executor)) { @@ -846,9 +845,9 @@ public void testBatchVectorSearchWithFilter() throws IOException { writeVectors(writer, vectors); List results = writer.finish(); - List metas = toIOMetas(results, indexPath); + List metas = toIOMetas(results); - GlobalIndexFileReader fileReader = createFileReader(indexPath); + GlobalIndexFileReader fileReader = createFileReader(); try (LuminaVectorGlobalIndexReader reader = new LuminaVectorGlobalIndexReader( fileReader, metas, vectorType, indexOptions, executor)) { @@ -889,9 +888,9 @@ public void testBatchConsistentWithSingle() throws IOException { writeVectors(writer, testVectors); List results = writer.finish(); - List metas = toIOMetas(results, indexPath); + List metas = toIOMetas(results); - GlobalIndexFileReader fileReader = createFileReader(indexPath); + GlobalIndexFileReader fileReader = createFileReader(); try (LuminaVectorGlobalIndexReader reader = new LuminaVectorGlobalIndexReader( fileReader, metas, vectorType, indexOptions, executor)) { @@ -935,9 +934,9 @@ public void testBatchVectorSearchAppliesQueryOptions() throws IOException { writeVectors(writer, testVectors); List results = writer.finish(); - List metas = toIOMetas(results, indexPath); + List metas = toIOMetas(results); - GlobalIndexFileReader fileReader = createFileReader(indexPath); + GlobalIndexFileReader fileReader = createFileReader(); try (LuminaVectorGlobalIndexReader reader = new LuminaVectorGlobalIndexReader( fileReader, metas, vectorType, indexOptions, executor)) { diff --git a/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexWriterTest.java b/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexWriterTest.java index 87cd6b56efc2..5a6625b2fd7e 100644 --- a/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexWriterTest.java +++ b/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexWriterTest.java @@ -135,12 +135,12 @@ private static Options createDefaultOptions(int dimension) { private static GlobalIndexFileWriter createNoopFileWriter() { return new GlobalIndexFileWriter() { @Override - public String newFileName(String prefix) { - return prefix; + public String newFilePath(String prefix) { + return "memory://test/" + prefix; } @Override - public PositionOutputStream newOutputStream(String fileName) { + public PositionOutputStream newOutputStream(String filePath) { throw new UnsupportedOperationException(); } }; diff --git a/paimon-python/pypaimon/globalindex/create_global_index.py b/paimon-python/pypaimon/globalindex/create_global_index.py index 10b88a2eb8f7..f407c598c891 100644 --- a/paimon-python/pypaimon/globalindex/create_global_index.py +++ b/paimon-python/pypaimon/globalindex/create_global_index.py @@ -115,6 +115,7 @@ def __init__( if CoreOptions._is_vector_index_type(self._index_type) else self._core_options.global_index_options(self._index_type, self._index_columns[0])) self._core_options = CoreOptions(self._options) + self._validate_external_path() if self._index_type in VINDEX_IDENTIFIERS: key = self._index_type + ".sharding" layout = self._options.to_map().get(key, "range") @@ -152,6 +153,18 @@ def build(self, *, execution="local", concurrency=None, ray_remote_args=None) -> self, partition_filter, execution=execution, concurrency=concurrency, ray_remote_args=ray_remote_args) + def _validate_external_path(self) -> None: + # Python placement uses table options; native builds also receive build + # overrides. Validate both before selecting a backend or creating files. + for options in (self._table.options, self._core_options): + path = options.global_index_external_path() + if path is not None and ',' in path: + raise ValueError( + "PyPaimon global index build only supports a single " + "'global-index.external-path' directory; comma-separated " + "paths are not supported." + ) + def _validate_unaware_bucket_table(self) -> None: # Global index files live in bucket 0 and are keyed by the table-wide # _ROW_ID, so every index type needs an unaware-bucket table. Check the diff --git a/paimon-python/pypaimon/tests/global_index_build_test.py b/paimon-python/pypaimon/tests/global_index_build_test.py index dfdac43dbf13..dee016112bfd 100644 --- a/paimon-python/pypaimon/tests/global_index_build_test.py +++ b/paimon-python/pypaimon/tests/global_index_build_test.py @@ -739,6 +739,61 @@ def test_create_global_index_uses_external_path(self): self.assertTrue(external_path.startswith(external_root + '/')) self.assertTrue(table.file_io.exists(external_path)) + def test_create_global_index_rejects_multiple_external_paths_before_build(self): + from pypaimon.globalindex.index_dispatch import build_global_index + + table = self._create_table() + self._write_arrow(table, pa.table( + {'id': [1], 'name': ['a'], 'age': [10], 'city': ['x']}, + schema=self.pa_schema, + )) + option = 'global-index.external-path' + root_a = 'file://%s' % os.path.join(self.tempdir, 'external-a') + root_b = 'file://%s' % os.path.join(self.tempdir, 'external-b') + snapshot = table.snapshot_manager().get_latest_snapshot() + files_before = { + os.path.join(root, name) + for root, dirs, files in os.walk(self.tempdir) + for name in dirs + files + } + + for paths in (root_a + ',' + root_b, root_a + ', ' + root_b + ','): + for native in ('false', 'true'): + for source in ('table', 'build', 'table-with-build-override'): + with self.subTest(paths=paths, native=native, source=source): + table_options = {'write.native.enabled': native} + build_options = {} + if source == 'build': + build_options[option] = paths + else: + table_options[option] = paths + if source == 'table-with-build-override': + build_options[option] = root_a + configured = table.copy(table_options) + with patch( + 'pypaimon.globalindex.index_dispatch.build_global_index', + wraps=build_global_index, + ) as build: + with patch.object( + configured, 'new_batch_write_builder', + wraps=configured.new_batch_write_builder, + ) as write_builder: + with self.assertRaisesRegex( + ValueError, + 'global-index.external-path.*comma-separated', + ): + configured.create_global_index('id', options=build_options) + build.assert_not_called() + write_builder.assert_not_called() + self.assertEqual( + snapshot.id, table.snapshot_manager().get_latest_snapshot().id) + self.assertEqual([], IndexFileHandler(table).scan(snapshot)) + self.assertEqual(files_before, { + os.path.join(root, name) + for root, dirs, files in os.walk(self.tempdir) + for name in dirs + files + }) + def test_create_global_index_skips_existing_ranges(self): table = self._create_table() self._write_arrow(table, pa.table( diff --git a/paimon-python/pypaimon/tests/resources/btree/BTreeFixtureGenerator.java b/paimon-python/pypaimon/tests/resources/btree/BTreeFixtureGenerator.java index ae0c087b1e36..e7b9992ff590 100644 --- a/paimon-python/pypaimon/tests/resources/btree/BTreeFixtureGenerator.java +++ b/paimon-python/pypaimon/tests/resources/btree/BTreeFixtureGenerator.java @@ -106,12 +106,12 @@ public static void main(String[] args) throws Exception { GlobalIndexFileWriter fileWriter = new GlobalIndexFileWriter() { @Override - public String newFileName(String prefix) { - return name; + public String newFilePath(String prefix) { + return directory.resolve(name).toAbsolutePath().toString(); } @Override - public PositionOutputStream newOutputStream(String fileName) { + public PositionOutputStream newOutputStream(String filePath) { return new PositionOutputStream() { @Override public long getPos() { @@ -176,12 +176,12 @@ private static void writeComposite(Path directory, String name, boolean reversed GlobalIndexFileWriter files = new GlobalIndexFileWriter() { @Override - public String newFileName(String prefix) { - return name; + public String newFilePath(String prefix) { + return directory.resolve(name).toAbsolutePath().toString(); } @Override - public PositionOutputStream newOutputStream(String fileName) { + public PositionOutputStream newOutputStream(String filePath) { return new PositionOutputStream() { @Override public long getPos() { diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/copy/CopyFilesUtil.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/copy/CopyFilesUtil.java index 5ec69b75011e..c9466a9951c0 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/copy/CopyFilesUtil.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/copy/CopyFilesUtil.java @@ -118,18 +118,15 @@ public static DataFileMeta toNewDataFileMeta( null); } - public static IndexFileMeta toNewIndexFileMeta(IndexFileMeta oldFileMeta, String newFileName) { - String newExternalPath = - externalPathDir(oldFileMeta.externalPath()) - .map(dir -> dir + "/" + newFileName) - .orElse(null); + public static IndexFileMeta toNewIndexFileMeta( + IndexFileMeta oldFileMeta, Path newFilePath, boolean isExternalPath) { return new IndexFileMeta( oldFileMeta.indexType(), - newFileName, + newFilePath.getName(), oldFileMeta.fileSize(), oldFileMeta.rowCount(), oldFileMeta.dvRanges(), - newExternalPath); + isExternalPath ? newFilePath.toString() : null); } public static Optional externalPathDir(String externalPath) { diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/copy/ListIndexFilesOperator.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/copy/ListIndexFilesOperator.java index 4e1d6c456261..288476ffb856 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/copy/ListIndexFilesOperator.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/copy/ListIndexFilesOperator.java @@ -25,6 +25,7 @@ import org.apache.paimon.index.IndexFileHandler; import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.index.IndexFileMetaSerializer; +import org.apache.paimon.index.IndexPathFactory; import org.apache.paimon.manifest.IndexManifestEntry; import org.apache.paimon.partition.PartitionPredicate; import org.apache.paimon.table.FileStoreTable; @@ -87,14 +88,14 @@ private CopyFileInfo pickIndexFiles( FileStorePathFactory targetFileStorePathFactory) throws IOException { Path indexFilePath = sourceIndexFileHandler.filePath(indexManifestEntry); - Path targetIndexFilePath = - targetFileStorePathFactory - .indexFileFactory( - indexManifestEntry.partition(), indexManifestEntry.bucket()) - .newPath(); + IndexPathFactory targetIndexPathFactory = + targetFileStorePathFactory.indexFileFactory( + indexManifestEntry.partition(), indexManifestEntry.bucket()); + Path targetIndexFilePath = targetIndexPathFactory.newPath(); IndexFileMeta fileMeta = indexManifestEntry.indexFile(); IndexFileMeta targetFileMeta = - CopyFilesUtil.toNewIndexFileMeta(fileMeta, targetIndexFilePath.getName()); + CopyFilesUtil.toNewIndexFileMeta( + fileMeta, targetIndexFilePath, targetIndexPathFactory.isExternalPath()); return new CopyFileInfo( indexFilePath.toString(), targetIndexFilePath.toString(), diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/CentroidGlobalIndexTopoBuilder.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/CentroidGlobalIndexTopoBuilder.java index 4bbe847699c2..262b5527a02e 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/CentroidGlobalIndexTopoBuilder.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/CentroidGlobalIndexTopoBuilder.java @@ -296,16 +296,22 @@ private static CommitMessage writeRouter( int nlist) throws IOException { GlobalIndexFileReadWrite io = context.io(); - String modelFile = + String modelFileName = "vector-" + context.indexType + "-global-index-training-model-" + modelId + ".index"; + String modelFilePath = + context.table + .store() + .pathFactory() + .globalIndexFileFactory() + .newPathFromName(modelFileName) + .toString(); IndexFileMeta modelMeta = GlobalIndexBuilderUtils.toIndexFileMetas( context.table.fileIO(), - context.table.store().pathFactory().globalIndexFileFactory(), context.table.coreOptions(), context.rowRange, Collections.singletonList(context.field), @@ -313,7 +319,7 @@ private static CommitMessage writeRouter( Collections.singletonList( CentroidIndexMeta.router( indexer.writeRoutingModel( - io, modelFile, payload, modelId), + io, modelFilePath, payload, modelId), modelId, nlist)), context.sourceMeta()) @@ -581,7 +587,6 @@ byte[] finish(GlobalIndexSingleColumnWriter writer, String modelId, int centroid List files = GlobalIndexBuilderUtils.toIndexFileMetas( table.fileIO(), - table.store().pathFactory().globalIndexFileFactory(), table.coreOptions(), rowRange, Collections.singletonList(field), diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/DefaultGlobalIndexBuilder.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/DefaultGlobalIndexBuilder.java index a78ad1d7e19a..02a221042aef 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/DefaultGlobalIndexBuilder.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/DefaultGlobalIndexBuilder.java @@ -20,6 +20,7 @@ import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.InternalRow; +import org.apache.paimon.globalindex.GlobalIndexFileReadWrite; import org.apache.paimon.globalindex.GlobalIndexMultiColumnWriter; import org.apache.paimon.globalindex.GlobalIndexSingleColumnWriter; import org.apache.paimon.globalindex.GlobalIndexWriter; @@ -47,6 +48,7 @@ import java.util.Collections; import java.util.List; +import static org.apache.paimon.globalindex.GlobalIndexBuilderUtils.createGlobalIndexFileReadWrite; import static org.apache.paimon.globalindex.GlobalIndexBuilderUtils.createIndexWriter; import static org.apache.paimon.globalindex.GlobalIndexBuilderUtils.toIndexFileMetas; @@ -148,12 +150,16 @@ public RowType readType() { } public CommitMessage build(CloseableIterator data) throws IOException { + return build(data, createGlobalIndexFileReadWrite(table)); + } + + CommitMessage build(CloseableIterator data, GlobalIndexFileReadWrite fileReadWrite) + throws IOException { LongCounter rowCounter = new LongCounter(0); - List resultEntries = writePaimonRows(data, rowCounter); + List resultEntries = writePaimonRows(data, rowCounter, fileReadWrite); List indexFileMetas = toIndexFileMetas( table.fileIO(), - table.store().pathFactory().globalIndexFileFactory(), table.coreOptions(), rowRange, indexedFields(), @@ -166,9 +172,12 @@ public CommitMessage build(CloseableIterator data) throws IOExcepti } private List writePaimonRows( - CloseableIterator rows, LongCounter rowCounter) throws IOException { + CloseableIterator rows, + LongCounter rowCounter, + GlobalIndexFileReadWrite fileReadWrite) + throws IOException { GlobalIndexWriter indexWriter = - createIndexWriter(table, indexType, indexedFields(), options); + createIndexWriter(fileReadWrite, indexType, indexedFields(), options); boolean multiColumn = !extraFields.isEmpty(); try { diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/DefaultGlobalIndexTopoBuilder.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/DefaultGlobalIndexTopoBuilder.java index 7e5a401df302..d5c85d6c2249 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/DefaultGlobalIndexTopoBuilder.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/DefaultGlobalIndexTopoBuilder.java @@ -19,11 +19,13 @@ package org.apache.paimon.spark.globalindex; import org.apache.paimon.Snapshot; +import org.apache.paimon.annotation.VisibleForTesting; import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.fs.Path; import org.apache.paimon.globalindex.DataEvolutionGlobalIndexRefreshPlanner; import org.apache.paimon.globalindex.GlobalIndexBuilderUtils; +import org.apache.paimon.globalindex.GlobalIndexFileReadWrite; import org.apache.paimon.globalindex.IndexedSplit; import org.apache.paimon.index.DataEvolutionIndexSourceMeta; import org.apache.paimon.io.CompactIncrement; @@ -54,6 +56,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.function.BiFunction; @@ -63,6 +66,7 @@ import static org.apache.paimon.CoreOptions.GLOBAL_INDEX_COLUMN_UPDATE_ACTION; import static org.apache.paimon.CoreOptions.GLOBAL_INDEX_ROW_COUNT_PER_SHARD; import static org.apache.paimon.CoreOptions.GlobalIndexColumnUpdateAction.IGNORE; +import static org.apache.paimon.globalindex.GlobalIndexBuilderUtils.createGlobalIndexFileReadWrite; import static org.apache.paimon.utils.Preconditions.checkArgument; /** Default topology builder. */ @@ -178,7 +182,7 @@ public List buildIndex( List commitMessageBytes = javaSparkContext .parallelize(taskList, parallelism) - .map(DefaultGlobalIndexTopoBuilder::buildIndex) + .mapPartitions(DefaultGlobalIndexTopoBuilder::buildIndexes) .collect(); commitMessages.addAll(CommitMessageSerializer.deserializeAll(commitMessageBytes)); } @@ -211,20 +215,30 @@ static int parallelism(int taskCount, Options options) { return Math.min(taskCount, maxParallelism); } - private static byte[] buildIndex(Pair builderAndSplits) throws Exception { + @VisibleForTesting + static Iterator buildIndexes(Iterator> tasks) throws Exception { ClassLoader classLoader = DefaultGlobalIndexBuilder.class.getClassLoader(); - DefaultGlobalIndexBuilder indexBuilder = - InstantiationUtil.deserializeObject(builderAndSplits.getLeft(), classLoader); - byte[] dataSplitBytes = builderAndSplits.getRight(); - IndexedSplit split = InstantiationUtil.deserializeObject(dataSplitBytes, classLoader); - ReadBuilder builder = indexBuilder.table().newReadBuilder(); - builder.withReadType(indexBuilder.readType()); + List messages = new ArrayList<>(); + // Each Spark partition owns one allocator across all of its serialized shard tasks. + GlobalIndexFileReadWrite fileReadWrite = null; + CommitMessageSerializer serializer = new CommitMessageSerializer(); + while (tasks.hasNext()) { + Pair task = tasks.next(); + DefaultGlobalIndexBuilder indexBuilder = + InstantiationUtil.deserializeObject(task.getLeft(), classLoader); + IndexedSplit split = InstantiationUtil.deserializeObject(task.getRight(), classLoader); + if (fileReadWrite == null) { + fileReadWrite = createGlobalIndexFileReadWrite(indexBuilder.table()); + } + ReadBuilder builder = indexBuilder.table().newReadBuilder(); + builder.withReadType(indexBuilder.readType()); - try (RecordReader recordReader = builder.newRead().createReader(split); - CloseableIterator data = recordReader.toCloseableIterator()) { - CommitMessage commitMessage = indexBuilder.build(data); - return new CommitMessageSerializer().serialize(commitMessage); + try (RecordReader recordReader = builder.newRead().createReader(split); + CloseableIterator data = recordReader.toCloseableIterator()) { + messages.add(serializer.serialize(indexBuilder.build(data, fileReadWrite))); + } } + return messages.iterator(); } /** diff --git a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/copy/CopyFilesUtilTest.java b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/copy/CopyFilesUtilTest.java index 67fa6873115d..bd5fb527a143 100644 --- a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/copy/CopyFilesUtilTest.java +++ b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/copy/CopyFilesUtilTest.java @@ -19,7 +19,12 @@ package org.apache.paimon.spark.copy; import org.apache.paimon.CoreOptions; +import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.index.HashIndexFile; +import org.apache.paimon.index.IndexFileMeta; +import org.apache.paimon.index.IndexPathFactory; import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.io.DataFilePathFactory; import org.apache.paimon.schema.Schema; @@ -28,15 +33,46 @@ import org.apache.paimon.types.DataTypes; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.util.Arrays; import java.util.Collections; +import static org.apache.paimon.utils.FileStorePathFactoryTest.createNonPartFactory; import static org.assertj.core.api.Assertions.assertThat; /** Tests for {@link CopyFilesUtil}. */ public class CopyFilesUtilTest { + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testCopiedHashIndexUsesAllocatedPath( + boolean externalTarget, @TempDir java.nio.file.Path tempDir) throws Exception { + FileIO fileIO = LocalFileIO.create(); + Path root = new Path(tempDir.toUri()); + IndexPathFactory sourcePaths = + createNonPartFactory(root, Arrays.asList(new Path(root, "a"), new Path(root, "b"))) + .globalIndexFileFactory(); + IndexFileMeta source = new HashIndexFile(fileIO, sourcePaths).write(new int[] {1, 2, 3}); + IndexPathFactory targetPaths = + externalTarget + ? sourcePaths + : createNonPartFactory(new Path(root, "target")).globalIndexFileFactory(); + Path sourcePath = sourcePaths.toPath(source); + Path targetPath = targetPaths.newPath(); + assertThat(targetPath.getParent()).isNotEqualTo(sourcePath.getParent()); + CopyFilesUtil.copyFiles(fileIO, fileIO, sourcePath, targetPath, false); + + IndexFileMeta copied = + CopyFilesUtil.toNewIndexFileMeta(source, targetPath, targetPaths.isExternalPath()); + assertThat(copied.fileName()).isEqualTo(targetPath.getName()); + assertThat(copied.externalPath()).isEqualTo(externalTarget ? targetPath.toString() : null); + assertThat(new HashIndexFile(fileIO, targetPaths).readList(copied)) + .containsExactly(1, 2, 3); + } + @Test void testNewDataFilePathKeepsTheKindOfTheFile() { DataFilePathFactory pathFactory = diff --git a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/globalindex/GenericIndexExternalPathsTest.java b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/globalindex/GenericIndexExternalPathsTest.java new file mode 100644 index 000000000000..8ca8333161de --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/globalindex/GenericIndexExternalPathsTest.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.globalindex; + +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.fs.Path; +import org.apache.paimon.globalindex.GlobalIndexBuilderUtils; +import org.apache.paimon.globalindex.IndexedSplit; +import org.apache.paimon.index.IndexFileMeta; +import org.apache.paimon.options.Options; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.SpecialFields; +import org.apache.paimon.table.TableTestBase; +import org.apache.paimon.table.sink.BatchTableCommit; +import org.apache.paimon.table.sink.BatchTableWrite; +import org.apache.paimon.table.sink.BatchWriteBuilder; +import org.apache.paimon.table.sink.CommitMessageImpl; +import org.apache.paimon.table.sink.CommitMessageSerializer; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.InstantiationUtil; +import org.apache.paimon.utils.Pair; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests path allocation across serialized shards in one Spark partition. */ +class GenericIndexExternalPathsTest extends TableTestBase { + + @Test + void testRoundRobinAcrossShards() throws Exception { + List roots = Arrays.asList(new Path(warehouse, "a"), new Path(warehouse, "b")); + catalog.createTable( + identifier(), + Schema.newBuilder() + .column("id", DataTypes.INT()) + .option("bucket", "-1") + .option("file.format", "avro") + .option("row-tracking.enabled", "true") + .option("data-evolution.enabled", "true") + .option("global-index.external-path", roots.get(0) + "," + roots.get(1)) + .build(), + false); + FileStoreTable table = getTableDefault(); + BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder(); + try (BatchTableWrite write = writeBuilder.newWrite(); + BatchTableCommit commit = writeBuilder.newCommit()) { + for (int i = 0; i < 20; i++) { + write.write(GenericRow.of(i)); + } + commit.commit(write.prepareCommit()); + } + List splits = + GlobalIndexBuilderUtils.createShardIndexedSplits( + table, table.store().newScan().plan().files(), 1); + assertThat(splits).hasSize(20); + RowType readType = SpecialFields.rowTypeWithRowId(table.rowType()); + List> tasks = new ArrayList<>(); + for (IndexedSplit split : splits) { + DefaultGlobalIndexBuilder builder = + new DefaultGlobalIndexBuilder( + table, + split.dataSplit().partition(), + readType, + table.rowType().getField("id"), + "bitmap", + split.rowRanges().get(0), + new Options()); + tasks.add( + Pair.of( + InstantiationUtil.serializeObject(builder), + InstantiationUtil.serializeObject(split))); + } + Iterator messages = DefaultGlobalIndexTopoBuilder.buildIndexes(tasks.iterator()); + CommitMessageSerializer serializer = new CommitMessageSerializer(); + List parents = new ArrayList<>(); + while (messages.hasNext()) { + CommitMessageImpl message = + (CommitMessageImpl) + serializer.deserialize(serializer.getVersion(), messages.next()); + List files = message.newFilesIncrement().newIndexFiles(); + assertThat(files).hasSize(1); + Path file = new Path(files.get(0).externalPath()); + assertThat(table.fileIO().exists(file)).isTrue(); + assertThat(files.get(0).fileName()).isEqualTo(file.getName()); + parents.add(file.getParent()); + } + assertThat(parents).hasSize(20).containsOnlyElementsOf(roots); + for (int i = 1; i < parents.size(); i++) { + assertThat(parents.get(i)).isNotEqualTo(parents.get(i - 1)); + } + } +} diff --git a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/globalindex/sorted/NullBitmapIndexComparisonTest.java b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/globalindex/sorted/NullBitmapIndexComparisonTest.java index 7a3c89c07425..d736fb9728c5 100644 --- a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/globalindex/sorted/NullBitmapIndexComparisonTest.java +++ b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/globalindex/sorted/NullBitmapIndexComparisonTest.java @@ -251,13 +251,14 @@ private List write(String type, String directory, List result = new ArrayList<>(); @@ -272,7 +273,7 @@ public PositionOutputStream newOutputStream(String name) throws IOException { for (List group : writer.finish()) { assertThat(group).hasSize(1); ResultEntry entry = group.get(0); - Path path = new Path(base, entry.fileName()); + Path path = new Path(entry.filePath()); result.add( new GlobalIndexIOMeta( path, io.getFileSize(path), entry.rowCount(), entry.meta())); diff --git a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeCentroidVectorGlobalIndexer.java b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeCentroidVectorGlobalIndexer.java index 9505d9c7c1d9..a61d11a7b739 100644 --- a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeCentroidVectorGlobalIndexer.java +++ b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeCentroidVectorGlobalIndexer.java @@ -62,12 +62,12 @@ public GlobalIndexWriter createWriter(GlobalIndexFileWriter fileWriter) { @Override public ResultEntry writeRoutingModel( - GlobalIndexFileWriter io, String name, byte[] payload, String modelId) + GlobalIndexFileWriter io, String filePath, byte[] payload, String modelId) throws IOException { - try (OutputStream out = io.newOutputStream(name)) { + try (OutputStream out = io.newOutputStream(filePath)) { VectorTrainingModelIO.write(out, payload, modelId); } - return new ResultEntry(name, 0, new VectorIndexMeta(null, metric().trim()).serialize()); + return new ResultEntry(filePath, 0, new VectorIndexMeta(null, metric().trim()).serialize()); } static String storedMetric(GlobalIndexFileReader reader, List files) { diff --git a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexWriter.java b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexWriter.java index 6531054af0ac..f3774ed5f5e7 100644 --- a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexWriter.java +++ b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexWriter.java @@ -276,11 +276,11 @@ public List finish() { if (!centroidShard) { return Collections.emptyList(); } - String name = fileWriter.newFileName(fileNamePrefix()); - try (PositionOutputStream out = fileWriter.newOutputStream(name)) { + String filePath = fileWriter.newFilePath(fileNamePrefix()); + try (PositionOutputStream out = fileWriter.newOutputStream(filePath)) { out.write(new byte[] {'P', 'C', 'E', 'M', 'P', 'T', 'Y', '1'}); } - return Collections.singletonList(new ResultEntry(name, 0, null)); + return Collections.singletonList(new ResultEntry(filePath, 0, null)); } flushWriteBuffer(); writeChannel.close(); @@ -319,14 +319,14 @@ private ResultEntry buildIndex() throws IOException { // Phase 3: Write index phaseStart = System.currentTimeMillis(); LOG.info("{} write phase started", identifier); - String fileName = fileWriter.newFileName(fileNamePrefix()); + String filePath = fileWriter.newFilePath(fileNamePrefix()); // Native IVF-FLAT and IVF-PQ readers obtain their geometry from the 64-byte header. // Capture it as it is written so automatic local nlist is resolved by native itself. byte[] header = centroidShard && ("ivf-flat".equals(identifier) || "ivf-pq".equals(identifier)) ? new byte[64] : null; - try (PositionOutputStream out = fileWriter.newOutputStream(fileName)) { + try (PositionOutputStream out = fileWriter.newOutputStream(filePath)) { writer.writeIndex( new PositionOutputStreamWrapper(out) { private long bytesWritten; @@ -377,7 +377,7 @@ public void write(byte[] b) throws IOException { } } return new ResultEntry( - fileName, !centroidShard ? rowCount : count, descriptor.serialize()); + filePath, !centroidShard ? rowCount : count, descriptor.serialize()); } } diff --git a/paimon-vector/src/test/java/org/apache/paimon/JavaPyE2ETest.java b/paimon-vector/src/test/java/org/apache/paimon/JavaPyE2ETest.java index 7975fc76fb64..e8ce14ccfcc6 100644 --- a/paimon-vector/src/test/java/org/apache/paimon/JavaPyE2ETest.java +++ b/paimon-vector/src/test/java/org/apache/paimon/JavaPyE2ETest.java @@ -172,7 +172,6 @@ public void testVindexVectorIndexWrite() throws Exception { List indexFiles = GlobalIndexBuilderUtils.toIndexFileMetas( table.fileIO(), - table.store().pathFactory().globalIndexFileFactory(), table.coreOptions(), rowRange, embeddingField.id(), @@ -282,7 +281,6 @@ public void testVindexVectorRawFallbackWrite() throws Exception { List indexFiles = GlobalIndexBuilderUtils.toIndexFileMetas( table.fileIO(), - table.store().pathFactory().globalIndexFileFactory(), table.coreOptions(), rowRange, embeddingField.id(), diff --git a/paimon-vector/src/test/java/org/apache/paimon/vector/index/CentroidVectorGlobalIndexTest.java b/paimon-vector/src/test/java/org/apache/paimon/vector/index/CentroidVectorGlobalIndexTest.java index 118d361e9e54..9db083db6ce4 100644 --- a/paimon-vector/src/test/java/org/apache/paimon/vector/index/CentroidVectorGlobalIndexTest.java +++ b/paimon-vector/src/test/java/org/apache/paimon/vector/index/CentroidVectorGlobalIndexTest.java @@ -292,11 +292,11 @@ void testRoutingPrunesFilesAndAppliesRangeOffset(String type) throws Exception { .isInstanceOf(IllegalStateException.class) .hasMessageContaining("finished"); String id = "test-model"; - String name = files.newFileName("routing"); - try (PositionOutputStream out = files.newOutputStream(name)) { + String filePath = files.newFilePath("routing"); + try (PositionOutputStream out = files.newOutputStream(filePath)) { VectorTrainingModelIO.write(out, model.serialize(), id); } - Path modelPath = new Path(directory, name); + Path modelPath = new Path(filePath); metas.add( new CentroidIndexFile( modelPath, @@ -313,7 +313,7 @@ void testRoutingPrunesFilesAndAppliesRangeOffset(String type) throws Exception { .extracting("tempVectorFile", "spillBuffer") .containsOnlyNulls(); ResultEntry result = empty.finish().get(0); - Path path = new Path(directory, result.fileName()); + Path path = new Path(result.filePath()); emptyMeta = toIOMeta(io, directory, CentroidIndexMeta.shard(result, id, 0)); assertThat(io.getFileSize(path)).isEqualTo(8); } finally { @@ -331,7 +331,7 @@ void testRoutingPrunesFilesAndAppliesRangeOffset(String type) throws Exception { } } ResultEntry result = writer.finish().get(0); - Path path = new Path(directory, result.fileName()); + Path path = new Path(result.filePath()); if (result.rowCount() > 0) { try (SeekableInputStream in = io.newInputStream(path); VectorIndexReader local = @@ -837,7 +837,9 @@ private List buildMixedShards( try (VectorTrainingModel model = indexer.trainGlobalModel( 64, Collections.singletonList(new TrainingBatch(sample, 64)).iterator())) { - ResultEntry routing = indexer.writeRoutingModel(files, "router", model.serialize(), id); + ResultEntry routing = + indexer.writeRoutingModel( + files, files.newFilePath("router"), model.serialize(), id); metas.add( toIOMeta(io, directory, CentroidIndexMeta.router(routing, id, model.nlist()))); NativeCentroidVectorGlobalIndexer smallIndexer = jointIndexer(type, 0); @@ -883,20 +885,21 @@ private static void assertSameScores( private static GlobalIndexIOMeta toIOMeta(FileIO io, Path directory, ResultEntry entry) throws java.io.IOException { - Path path = new Path(directory, entry.fileName()); + Path path = new Path(entry.filePath()); return CentroidIndexMeta.file(path, io.getFileSize(path), entry.rowCount(), entry.meta()); } private static GlobalIndexFileWriter fileWriter(FileIO io, Path directory) { return new GlobalIndexFileWriter() { @Override - public String newFileName(String prefix) { - return prefix + "-" + UUID.randomUUID() + ".index"; + public String newFilePath(String prefix) { + return new Path(directory, prefix + "-" + UUID.randomUUID() + ".index").toString(); } @Override - public PositionOutputStream newOutputStream(String name) throws java.io.IOException { - return io.newOutputStream(new Path(directory, name), false); + public PositionOutputStream newOutputStream(String filePath) + throws java.io.IOException { + return io.newOutputStream(new Path(filePath), false); } }; } diff --git a/paimon-vector/src/test/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexTest.java b/paimon-vector/src/test/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexTest.java index c876e3167919..ad08d154c3de 100644 --- a/paimon-vector/src/test/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexTest.java +++ b/paimon-vector/src/test/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexTest.java @@ -375,9 +375,9 @@ public void testFloatVectorEndToEnd() throws IOException { writer.write(vectors[i], i); } List results = writer.finish(); - List metas = toIOMetas(results, indexPath); + List metas = toIOMetas(results); - GlobalIndexFileReader fileReader = createFileReader(indexPath); + GlobalIndexFileReader fileReader = createFileReader(); try (NativeVectorGlobalIndexReader reader = new NativeVectorGlobalIndexReader(fileReader, metas, vectorType, executor)) { VectorSearch vectorSearch = new VectorSearch(vectors[0], 3, fieldName); @@ -426,11 +426,11 @@ public void testNewIndexTypesEndToEnd() throws IOException { vectorType, options, identifier, fieldName), identifier); writeVectors(writer, vectors); - List metas = toIOMetas(writer.finish(), indexPath); + List metas = toIOMetas(writer.finish()); try (NativeVectorGlobalIndexReader reader = new NativeVectorGlobalIndexReader( - createFileReader(indexPath), metas, vectorType, executor)) { + createFileReader(), metas, vectorType, executor)) { ScoredGlobalIndexResult result = reader.visitVectorSearch(new VectorSearch(vectors[0], 2, fieldName)) .join() @@ -465,9 +465,9 @@ public void testSearchWithRoaringFilter() throws IOException { writer.write(vectors[i], i); } List results = writer.finish(); - List metas = toIOMetas(results, indexPath); + List metas = toIOMetas(results); - GlobalIndexFileReader fileReader = createFileReader(indexPath); + GlobalIndexFileReader fileReader = createFileReader(); try (NativeVectorGlobalIndexReader reader = new NativeVectorGlobalIndexReader(fileReader, metas, vectorType, executor)) { @@ -514,8 +514,8 @@ public void testNullVectorSkipWithCorrectIds() throws IOException { assertThat(results).hasSize(1); assertThat(results.get(0).rowCount()).isEqualTo(6); - List metas = toIOMetas(results, indexPath); - GlobalIndexFileReader fileReader = createFileReader(indexPath); + List metas = toIOMetas(results); + GlobalIndexFileReader fileReader = createFileReader(); try (NativeVectorGlobalIndexReader reader = new NativeVectorGlobalIndexReader(fileReader, metas, vectorType, executor)) { VectorSearch vectorSearch = new VectorSearch(vectors[0], 3, fieldName); @@ -560,9 +560,9 @@ public void testViaIndexer() throws IOException { writer.write(vectors[i], i); } List results = writer.finish(); - List metas = toIOMetas(results, indexPath); + List metas = toIOMetas(results); - GlobalIndexFileReader fileReader = createFileReader(indexPath); + GlobalIndexFileReader fileReader = createFileReader(); try (NativeVectorGlobalIndexReader reader = (NativeVectorGlobalIndexReader) indexer.createReader(fileReader, metas, vectors.length, null, executor)) { @@ -596,9 +596,9 @@ public void testBatchVectorSearch() throws IOException { NativeVectorGlobalIndexWriter writer = createIvfPqWriter(fileWriter, vectorType, options); writeVectors(writer, vectors); List results = writer.finish(); - List metas = toIOMetas(results, indexPath); + List metas = toIOMetas(results); - GlobalIndexFileReader fileReader = createFileReader(indexPath); + GlobalIndexFileReader fileReader = createFileReader(); try (NativeVectorGlobalIndexReader reader = new NativeVectorGlobalIndexReader(fileReader, metas, vectorType, executor)) { float[][] queryVectors = @@ -648,9 +648,9 @@ public void testBatchVectorSearchWithFilter() throws IOException { NativeVectorGlobalIndexWriter writer = createIvfPqWriter(fileWriter, vectorType, options); writeVectors(writer, vectors); List results = writer.finish(); - List metas = toIOMetas(results, indexPath); + List metas = toIOMetas(results); - GlobalIndexFileReader fileReader = createFileReader(indexPath); + GlobalIndexFileReader fileReader = createFileReader(); try (NativeVectorGlobalIndexReader reader = new NativeVectorGlobalIndexReader(fileReader, metas, vectorType, executor)) { float[][] queryVectors = @@ -703,9 +703,9 @@ public void testBatchConsistentWithSingle() throws IOException { NativeVectorGlobalIndexWriter writer = createIvfPqWriter(fileWriter, vectorType, options); writeVectors(writer, vectors); List results = writer.finish(); - List metas = toIOMetas(results, indexPath); + List metas = toIOMetas(results); - GlobalIndexFileReader fileReader = createFileReader(indexPath); + GlobalIndexFileReader fileReader = createFileReader(); try (NativeVectorGlobalIndexReader reader = new NativeVectorGlobalIndexReader(fileReader, metas, vectorType, executor)) { float[][] queryVectors = @@ -767,26 +767,25 @@ private void writeVectors(NativeVectorGlobalIndexWriter writer, float[][] vector private GlobalIndexFileWriter createFileWriter(Path path) { return new GlobalIndexFileWriter() { @Override - public String newFileName(String prefix) { - return prefix + "-" + UUID.randomUUID(); + public String newFilePath(String prefix) { + return new Path(path, prefix + "-" + UUID.randomUUID()).toString(); } @Override - public PositionOutputStream newOutputStream(String fileName) throws IOException { - return fileIO.newOutputStream(new Path(path, fileName), false); + public PositionOutputStream newOutputStream(String filePath) throws IOException { + return fileIO.newOutputStream(new Path(filePath), false); } }; } - private GlobalIndexFileReader createFileReader(Path path) { - return meta -> fileIO.newInputStream(new Path(path, meta.filePath())); + private GlobalIndexFileReader createFileReader() { + return meta -> fileIO.newInputStream(meta.filePath()); } - private List toIOMetas(List results, Path path) - throws IOException { + private List toIOMetas(List results) throws IOException { assertThat(results).hasSize(1); ResultEntry result = results.get(0); - Path filePath = new Path(path, result.fileName()); + Path filePath = new Path(result.filePath()); return Collections.singletonList( new GlobalIndexIOMeta(filePath, fileIO.getFileSize(filePath), result.meta())); }