toIndexFileMetas(
* index column (e.g. the first column in {@code CREATE ... INDEX ON (a, b, c)}) and is stored
* as {@code indexFieldId}; the remaining columns go into {@code extraFieldIds}. Callers must
* pass {@code fields} in the intended column order.
+ *
+ * The entries already contain their allocated full paths.
*/
public static List toIndexFileMetas(
FileIO fileIO,
- IndexPathFactory indexPathFactory,
CoreOptions options,
Range range,
List fields,
@@ -104,7 +96,6 @@ public static List toIndexFileMetas(
throws IOException {
return toIndexFileMetas(
fileIO,
- indexPathFactory,
options,
range,
fields.get(0).id(),
@@ -571,7 +562,6 @@ private static void addIndexedSplitForFileGroup(
private static List toIndexFileMetas(
FileIO fileIO,
- IndexPathFactory indexPathFactory,
CoreOptions options,
Range range,
int indexFieldId,
@@ -580,10 +570,12 @@ private static List toIndexFileMetas(
List entries,
@Nullable byte[] sourceMeta)
throws IOException {
+ boolean isExternalPath = !options.globalIndexExternalPaths().isEmpty();
List results = new ArrayList<>();
for (ResultEntry entry : entries) {
- String fileName = entry.fileName();
- long fileSize = fileIO.getFileSize(indexPathFactory.toPath(fileName));
+ Path filePath = new Path(entry.filePath());
+ String fileName = filePath.getName();
+ long fileSize = fileIO.getFileSize(filePath);
GlobalIndexMeta globalIndexMeta =
new GlobalIndexMeta(
range.from,
@@ -593,12 +585,7 @@ private static List toIndexFileMetas(
entry.meta(),
sourceMeta);
- Path externalPathDir = options.globalIndexExternalPath();
- String externalPathString = null;
- if (externalPathDir != null) {
- Path externalPath = new Path(externalPathDir, fileName);
- externalPathString = externalPath.toString();
- }
+ String externalPathString = isExternalPath ? filePath.toString() : null;
IndexFileMeta indexFileMeta =
new IndexFileMeta(
indexType,
@@ -615,11 +602,21 @@ private static List toIndexFileMetas(
public static GlobalIndexWriter createIndexWriter(
FileStoreTable table, String indexType, List indexFields, Options options)
throws IOException {
+ return createIndexWriter(
+ createGlobalIndexFileReadWrite(table), indexType, indexFields, options);
+ }
+
+ public static GlobalIndexWriter createIndexWriter(
+ GlobalIndexFileReadWrite fileReadWrite,
+ String indexType,
+ List indexFields,
+ Options options)
+ throws IOException {
GlobalIndexer globalIndexer = GlobalIndexer.create(indexType, indexFields, options);
- return globalIndexer.createWriter(createGlobalIndexFileReadWrite(table));
+ return globalIndexer.createWriter(fileReadWrite);
}
- private static GlobalIndexFileReadWrite createGlobalIndexFileReadWrite(FileStoreTable table) {
+ public static GlobalIndexFileReadWrite createGlobalIndexFileReadWrite(FileStoreTable table) {
IndexPathFactory indexPathFactory = table.store().pathFactory().globalIndexFileFactory();
return new GlobalIndexFileReadWrite(table.fileIO(), indexPathFactory);
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/globalindex/GlobalIndexFileReadWrite.java b/paimon-core/src/main/java/org/apache/paimon/globalindex/GlobalIndexFileReadWrite.java
index 1df217135436..821d1c26c9d0 100644
--- a/paimon-core/src/main/java/org/apache/paimon/globalindex/GlobalIndexFileReadWrite.java
+++ b/paimon-core/src/main/java/org/apache/paimon/globalindex/GlobalIndexFileReadWrite.java
@@ -19,6 +19,7 @@
package org.apache.paimon.globalindex;
import org.apache.paimon.fs.FileIO;
+import org.apache.paimon.fs.Path;
import org.apache.paimon.fs.PositionOutputStream;
import org.apache.paimon.fs.SeekableInputStream;
import org.apache.paimon.globalindex.io.GlobalIndexFileReader;
@@ -39,18 +40,22 @@ public GlobalIndexFileReadWrite(FileIO fileIO, IndexPathFactory indexPathFactory
this.indexPathFactory = indexPathFactory;
}
- public String newFileName(String prefix) {
- return prefix + "-" + "global-index-" + UUID.randomUUID() + ".index";
+ @Override
+ public String newFilePath(String prefix) {
+ String fileName = prefix + "-" + "global-index-" + UUID.randomUUID() + ".index";
+ return indexPathFactory.newPathFromName(fileName).toString();
}
- public long fileSize(String fileName) throws IOException {
- return fileIO.getFileSize(indexPathFactory.toPath(fileName));
+ public long fileSize(String filePath) throws IOException {
+ return fileIO.getFileSize(new Path(filePath));
}
- public PositionOutputStream newOutputStream(String fileName) throws IOException {
- return fileIO.newOutputStream(indexPathFactory.toPath(fileName), true);
+ @Override
+ public PositionOutputStream newOutputStream(String filePath) throws IOException {
+ return fileIO.newOutputStream(new Path(filePath), true);
}
+ @Override
public SeekableInputStream getInputStream(GlobalIndexIOMeta meta) throws IOException {
return fileIO.newInputStream(meta.filePath());
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexWriter.java b/paimon-core/src/main/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexWriter.java
index 19a8ce457557..f51cf5dd30c5 100644
--- a/paimon-core/src/main/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexWriter.java
+++ b/paimon-core/src/main/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexWriter.java
@@ -22,6 +22,7 @@
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.data.InternalRow.FieldGetter;
+import org.apache.paimon.globalindex.GlobalIndexFileReadWrite;
import org.apache.paimon.globalindex.GlobalIndexKeyExtractor;
import org.apache.paimon.globalindex.GlobalIndexSingleColumnWriter;
import org.apache.paimon.globalindex.GlobalIndexWriter;
@@ -47,6 +48,7 @@
import java.util.Iterator;
import java.util.List;
+import static org.apache.paimon.globalindex.GlobalIndexBuilderUtils.createGlobalIndexFileReadWrite;
import static org.apache.paimon.globalindex.GlobalIndexBuilderUtils.createIndexWriter;
import static org.apache.paimon.globalindex.GlobalIndexBuilderUtils.toIndexFileMetas;
import static org.apache.paimon.utils.Preconditions.checkArgument;
@@ -65,6 +67,7 @@ public class SortedGlobalIndexWriter implements Serializable {
private List indexFields;
private GlobalIndexKeyExtractor keyExtractor;
+ private transient GlobalIndexFileReadWrite fileReadWrite;
public SortedGlobalIndexWriter(Table table, String indexType) {
this(table, indexType, ((FileStoreTable) table).coreOptions().toConfiguration());
@@ -141,8 +144,11 @@ public SortedSingleColumnIndexWriter createTaskWriter(Range rowRange) throws IOE
}
public GlobalIndexSingleColumnWriter createWriter() throws IOException {
+ if (fileReadWrite == null) {
+ fileReadWrite = createGlobalIndexFileReadWrite(table);
+ }
GlobalIndexWriter indexWriter =
- createIndexWriter(table, indexType, indexFields, indexOptions);
+ createIndexWriter(fileReadWrite, indexType, indexFields, indexOptions);
if (!(indexWriter instanceof GlobalIndexSingleColumnWriter)) {
throw new RuntimeException(
"Unexpected implementation, the index writer of "
@@ -163,7 +169,6 @@ public CommitMessage flushIndex(
List indexFileMetas =
toIndexFileMetas(
table.fileIO(),
- table.store().pathFactory().globalIndexFileFactory(),
table.coreOptions(),
rowRange,
indexFields,
diff --git a/paimon-core/src/main/java/org/apache/paimon/index/IndexInDataFileDirPathFactory.java b/paimon-core/src/main/java/org/apache/paimon/index/IndexInDataFileDirPathFactory.java
index f664802164ca..386fca829c23 100644
--- a/paimon-core/src/main/java/org/apache/paimon/index/IndexInDataFileDirPathFactory.java
+++ b/paimon-core/src/main/java/org/apache/paimon/index/IndexInDataFileDirPathFactory.java
@@ -43,14 +43,14 @@ public IndexInDataFileDirPathFactory(
}
@Override
- public Path toPath(String fileName) {
+ public Path newPathFromName(String fileName) {
return dataFilePathFactory.newPathFromName(fileName);
}
@Override
public Path newPath() {
String name = INDEX_PREFIX + uuid + "-" + indexFileCount.getAndIncrement();
- return dataFilePathFactory.newPathFromName(name);
+ return newPathFromName(name);
}
@Override
diff --git a/paimon-core/src/main/java/org/apache/paimon/index/IndexPathFactory.java b/paimon-core/src/main/java/org/apache/paimon/index/IndexPathFactory.java
index 3180bb2c44fa..2ea74c647f46 100644
--- a/paimon-core/src/main/java/org/apache/paimon/index/IndexPathFactory.java
+++ b/paimon-core/src/main/java/org/apache/paimon/index/IndexPathFactory.java
@@ -25,13 +25,14 @@
/** Path factory to create an index path. */
public interface IndexPathFactory {
- Path toPath(String fileName);
-
+ /** Generates a file name and allocates a path for a new index file. */
Path newPath();
- default Path toPath(String fileName, @Nullable String externalPath) {
- return toPath(fileName);
- }
+ /** Allocates a path for a new index file with the given file name. */
+ Path newPathFromName(String fileName);
+
+ /** Resolves an existing file without allocating an external location. */
+ Path toPath(String fileName, @Nullable String externalPath);
default Path toPath(IndexFileMeta file) {
return toPath(file.fileName(), file.externalPath());
diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkfulltext/PkFullTextIndexFile.java b/paimon-core/src/main/java/org/apache/paimon/index/pkfulltext/PkFullTextIndexFile.java
index 4a6e02e062bc..0e45d7b34af4 100644
--- a/paimon-core/src/main/java/org/apache/paimon/index/pkfulltext/PkFullTextIndexFile.java
+++ b/paimon-core/src/main/java/org/apache/paimon/index/pkfulltext/PkFullTextIndexFile.java
@@ -160,12 +160,12 @@ IndexFileMeta build(List sources, DataField textField, GlobalIndexer ind
"Full-text archive row count %s does not match source row count %s.",
result.rowCount(),
totalRowCount);
- Path archivePath = fileWriter.path(result.fileName());
+ Path archivePath = fileWriter.path(result.filePath());
byte[] archiveMetadata = result.meta() == null ? new byte[0] : result.meta();
IndexFileMeta archive =
new IndexFileMeta(
INDEX_TYPE,
- result.fileName(),
+ archivePath.getName(),
fileIO.getFileSize(archivePath),
result.rowCount(),
new GlobalIndexMeta(
@@ -219,20 +219,20 @@ private class ArchiveFileWriter implements GlobalIndexFileWriter {
private final Map createdFiles = new HashMap<>();
@Override
- public String newFileName(String prefix) {
+ public String newFilePath(String prefix) {
Path path = pathFactory.newPath();
- createdFiles.put(path.getName(), path);
- return path.getName();
+ createdFiles.put(path.toString(), path);
+ return path.toString();
}
@Override
- public PositionOutputStream newOutputStream(String fileName) throws IOException {
- return fileIO.newOutputStream(path(fileName), false);
+ public PositionOutputStream newOutputStream(String filePath) throws IOException {
+ return fileIO.newOutputStream(path(filePath), false);
}
- private Path path(String fileName) {
- Path path = createdFiles.get(fileName);
- checkArgument(path != null, "Full-text archive %s was not allocated.", fileName);
+ private Path path(String filePath) {
+ Path path = createdFiles.get(filePath);
+ checkArgument(path != null, "Full-text archive %s was not allocated.", filePath);
return path;
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pksorted/PkSortedIndexFile.java b/paimon-core/src/main/java/org/apache/paimon/index/pksorted/PkSortedIndexFile.java
index 456f7878f487..72919d056f38 100644
--- a/paimon-core/src/main/java/org/apache/paimon/index/pksorted/PkSortedIndexFile.java
+++ b/paimon-core/src/main/java/org/apache/paimon/index/pksorted/PkSortedIndexFile.java
@@ -98,11 +98,11 @@ public IndexFileMeta build(
result.rowCount(),
sourceRowCount);
byte[] sourceMeta = new PrimaryKeyIndexSourceMeta(dataLevel, sourceFiles).serialize();
- Path payloadPath = fileWriter.path(result.fileName());
+ Path payloadPath = fileWriter.path(result.filePath());
IndexFileMeta payload =
new IndexFileMeta(
indexType,
- result.fileName(),
+ payloadPath.getName(),
fileIO.getFileSize(payloadPath),
result.rowCount(),
new GlobalIndexMeta(
@@ -168,20 +168,20 @@ private final class TrackingFileWriter implements GlobalIndexFileWriter {
private final Map createdFiles = new LinkedHashMap<>();
@Override
- public String newFileName(String prefix) {
+ public String newFilePath(String prefix) {
Path path = pathFactory.newPath();
- createdFiles.put(path.getName(), path);
- return path.getName();
+ createdFiles.put(path.toString(), path);
+ return path.toString();
}
@Override
- public PositionOutputStream newOutputStream(String fileName) throws IOException {
- return fileIO.newOutputStream(path(fileName), false);
+ public PositionOutputStream newOutputStream(String filePath) throws IOException {
+ return fileIO.newOutputStream(path(filePath), false);
}
- private Path path(String fileName) {
- Path path = createdFiles.get(fileName);
- checkArgument(path != null, "Sorted payload file %s was not allocated.", fileName);
+ private Path path(String filePath) {
+ Path path = createdFiles.get(filePath);
+ checkArgument(path != null, "Sorted payload file %s was not allocated.", filePath);
return path;
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java
index 93ac807aff4c..9683127b5953 100644
--- a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java
+++ b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java
@@ -160,12 +160,12 @@ public IndexFileMeta build(
"ANN segment build must produce exactly one payload file, but produced %s.",
results.size());
ResultEntry result = results.get(0);
- Path payloadPath = fileWriter.path(result.fileName());
+ Path payloadPath = fileWriter.path(result.filePath());
byte[] payloadMetadata = result.meta() == null ? new byte[0] : result.meta();
IndexFileMeta segment =
new IndexFileMeta(
indexType,
- result.fileName(),
+ payloadPath.getName(),
fileIO.getFileSize(payloadPath),
liveRowCount,
new GlobalIndexMeta(
@@ -202,31 +202,31 @@ private class SegmentFileWriter implements GlobalIndexFileWriter {
private final Map createdFiles = new HashMap<>();
@Override
- public String newFileName(String prefix) {
+ public String newFilePath(String prefix) {
Path path = pathFactory.newPath();
- createdFiles.put(path.getName(), path);
- return path.getName();
+ createdFiles.put(path.toString(), path);
+ return path.toString();
}
@Override
- public PositionOutputStream newOutputStream(String fileName) throws IOException {
- return fileIO.newOutputStream(path(fileName), false);
+ public PositionOutputStream newOutputStream(String filePath) throws IOException {
+ return fileIO.newOutputStream(path(filePath), false);
}
- private Path path(String fileName) {
- Path path = createdFiles.get(fileName);
- checkArgument(path != null, "ANN payload file %s was not allocated.", fileName);
+ private Path path(String filePath) {
+ Path path = createdFiles.get(filePath);
+ checkArgument(path != null, "ANN payload file %s was not allocated.", filePath);
return path;
}
private ResultEntry emptyResult() throws IOException {
deleteCreatedFiles();
createdFiles.clear();
- String fileName = newFileName("empty-vector");
- try (PositionOutputStream ignored = newOutputStream(fileName)) {
+ String filePath = newFilePath("empty-vector");
+ try (PositionOutputStream ignored = newOutputStream(filePath)) {
// The searcher does not open payloads for segments without live rows.
}
- return new ResultEntry(fileName, 0, null);
+ return new ResultEntry(filePath, 0, null);
}
private void deleteCreatedFiles() {
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/RawFullTextReadImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/source/RawFullTextReadImpl.java
index 91271ab95d6e..cde9fc179d07 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/source/RawFullTextReadImpl.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/source/RawFullTextReadImpl.java
@@ -362,18 +362,14 @@ private IndexFullTextSearchSplit finish() {
List indexFiles = new ArrayList<>(resultEntries.size());
for (ResultEntry entry : resultEntries) {
- byte[] bytes = fileWriter.fileBytes(entry.fileName());
+ String fileName = new Path(entry.filePath()).getName();
+ byte[] bytes = fileWriter.fileBytes(fileName);
GlobalIndexMeta meta =
new GlobalIndexMeta(
rowRangeStart, rowRangeEnd, fieldId, null, entry.meta());
indexFiles.add(
new IndexFileMeta(
- indexType,
- entry.fileName(),
- bytes.length,
- entry.rowCount(),
- meta,
- null));
+ indexType, fileName, bytes.length, entry.rowCount(), meta, null));
}
return new IndexFullTextSearchSplit(column, rowRangeStart, rowRangeEnd, indexFiles);
}
@@ -409,12 +405,14 @@ private RawFullTextIndexFileWriter(String column) {
}
@Override
- public String newFileName(String prefix) {
- return "raw-" + column + "-" + prefix + "-" + id + "-" + files.size() + ".index";
+ public String newFilePath(String prefix) {
+ return path("raw-" + column + "-" + prefix + "-" + id + "-" + files.size() + ".index")
+ .toString();
}
@Override
- public PositionOutputStream newOutputStream(String fileName) {
+ public PositionOutputStream newOutputStream(String filePath) {
+ String fileName = new Path(filePath).getName();
return new MemoryPositionOutputStream(bytes -> files.put(fileName, bytes));
}
@@ -440,7 +438,7 @@ private RawFullTextIndexPathFactory(Map rawIndexes) {
}
@Override
- public Path toPath(String fileName) {
+ public Path toPath(String fileName, @Nullable String externalPath) {
RawFullTextIndex rawIndex = rawIndex(fileName);
return rawIndex.path(fileName);
}
@@ -450,6 +448,11 @@ public Path newPath() {
throw new UnsupportedOperationException();
}
+ @Override
+ public Path newPathFromName(String fileName) {
+ throw new UnsupportedOperationException();
+ }
+
@Override
public boolean isExternalPath() {
return false;
diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/FileStorePathFactory.java b/paimon-core/src/main/java/org/apache/paimon/utils/FileStorePathFactory.java
index 981b3a24da8d..c4dc894d8b93 100644
--- a/paimon-core/src/main/java/org/apache/paimon/utils/FileStorePathFactory.java
+++ b/paimon-core/src/main/java/org/apache/paimon/utils/FileStorePathFactory.java
@@ -24,6 +24,7 @@
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.fs.ExternalPathProvider;
import org.apache.paimon.fs.Path;
+import org.apache.paimon.fs.RoundRobinExternalPathProvider;
import org.apache.paimon.index.IndexInDataFileDirPathFactory;
import org.apache.paimon.index.IndexPathFactory;
import org.apache.paimon.io.ChainReadContext;
@@ -83,7 +84,7 @@ public class FileStorePathFactory {
private final List externalPaths;
private final ExternalPathStrategy strategy;
@Nullable private final int[] externalPathWeights;
- @Nullable private final Path globalIndexExternalRootDir;
+ @Nullable private final List globalIndexExternalPaths;
public FileStorePathFactory(
Path root,
@@ -100,7 +101,7 @@ public FileStorePathFactory(
ExternalPathStrategy strategy,
@Nullable int[] externalPathWeights,
boolean indexFileInDataFileDir,
- @Nullable Path globalIndexExternalRootDir) {
+ @Nullable List globalIndexExternalPaths) {
this.root = root;
this.dataFilePathDirectory = dataFilePathDirectory;
this.indexFileInDataFileDir = indexFileInDataFileDir;
@@ -122,7 +123,7 @@ public FileStorePathFactory(
this.externalPaths = externalPaths;
this.strategy = strategy;
this.externalPathWeights = externalPathWeights;
- this.globalIndexExternalRootDir = globalIndexExternalRootDir;
+ this.globalIndexExternalPaths = globalIndexExternalPaths;
}
public Path root() {
@@ -137,10 +138,6 @@ public Path indexPath() {
return new Path(root, INDEX_PATH);
}
- public Path globalIndexRootDir() {
- return globalIndexExternalRootDir != null ? globalIndexExternalRootDir : indexPath();
- }
-
public Path statisticsPath() {
return new Path(root, STATISTICS_PATH);
}
@@ -329,15 +326,22 @@ public IndexPathFactory indexFileFactory(BinaryRow partition, int bucket) {
}
public IndexPathFactory globalIndexFileFactory() {
+ ExternalPathProvider provider =
+ globalIndexExternalPaths == null || globalIndexExternalPaths.isEmpty()
+ ? null
+ : new RoundRobinExternalPathProvider(globalIndexExternalPaths, null);
return new IndexPathFactory() {
@Override
- public Path toPath(String fileName) {
- return new Path(globalIndexRootDir(), fileName);
+ public synchronized Path newPathFromName(String fileName) {
+ return provider == null
+ ? new Path(indexPath(), fileName)
+ : provider.getNextExternalDataPath(fileName);
}
@Override
public Path newPath() {
- return toPath(INDEX_PREFIX + uuid + "-" + indexFileCount.getAndIncrement());
+ return newPathFromName(
+ INDEX_PREFIX + uuid + "-" + indexFileCount.getAndIncrement());
}
@Override
@@ -351,7 +355,7 @@ public Path toPath(String fileName, @Nullable String externalPath) {
@Override
public boolean isExternalPath() {
- return globalIndexExternalRootDir != null;
+ return provider != null;
}
};
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/CoreOptionsTest.java b/paimon-core/src/test/java/org/apache/paimon/CoreOptionsTest.java
index f6931d44a3ce..9a2b783c2025 100644
--- a/paimon-core/src/test/java/org/apache/paimon/CoreOptionsTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/CoreOptionsTest.java
@@ -18,10 +18,13 @@
package org.apache.paimon;
+import org.apache.paimon.fs.Path;
import org.apache.paimon.options.MemorySize;
import org.apache.paimon.options.Options;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -29,6 +32,102 @@
/** Tests for {@link org.apache.paimon.CoreOptions}. */
public class CoreOptionsTest {
+ @Test
+ void testDataFileExternalPathList() {
+ Options options = new Options();
+ CoreOptions core = new CoreOptions(options);
+ assertThat(core.dataFileExternalPaths()).isNull();
+ assertThat(core.dataFileExternalPathList()).isEmpty();
+ options.set(CoreOptions.DATA_FILE_EXTERNAL_PATHS, "");
+ assertThat(core.dataFileExternalPathList()).isEmpty();
+ for (String value :
+ new String[] {"file:///a", " file:///a/ ", "file:///a,", "file:///a,,"}) {
+ options.set(CoreOptions.DATA_FILE_EXTERNAL_PATHS, value);
+ assertThat(core.dataFileExternalPaths()).isEqualTo(value);
+ assertThat(core.dataFileExternalPathList()).containsExactly(new Path("file:///a"));
+ }
+ for (String value :
+ new String[] {
+ "file:///a,s3://bucket/b",
+ " file:///a , s3://bucket/b ",
+ "file:///a,s3://bucket/b,",
+ "file:///a,s3://bucket/b,,"
+ }) {
+ options.set(CoreOptions.DATA_FILE_EXTERNAL_PATHS, value);
+ assertThat(core.dataFileExternalPaths()).isEqualTo(value);
+ assertThat(core.dataFileExternalPathList())
+ .containsExactly(new Path("file:///a"), new Path("s3://bucket/b"));
+ }
+ }
+
+ @ParameterizedTest
+ @EnumSource(CoreOptions.ExternalPathStrategy.class)
+ void testCreateExternalPaths(CoreOptions.ExternalPathStrategy strategy) {
+ Options options = new Options();
+ options.set(CoreOptions.DATA_FILE_EXTERNAL_PATHS_STRATEGY, strategy);
+ CoreOptions core = new CoreOptions(options);
+ assertThat(core.createExternalPaths()).isEmpty();
+ options.set(CoreOptions.DATA_FILE_EXTERNAL_PATHS, "");
+ assertThat(core.createExternalPaths()).isEmpty();
+
+ options.set(CoreOptions.DATA_FILE_EXTERNAL_PATHS, " file:///a , s3://bucket/b ,file:///c,");
+ options.set(CoreOptions.DATA_FILE_EXTERNAL_PATHS_SPECIFIC_FS, "FILE");
+ assertThat(core.dataFileExternalPathList())
+ .containsExactly(
+ new Path("file:///a"), new Path("s3://bucket/b"), new Path("file:///c"));
+ if (strategy == CoreOptions.ExternalPathStrategy.NONE) {
+ assertThat(core.createExternalPaths()).isEmpty();
+ } else if (strategy == CoreOptions.ExternalPathStrategy.SPECIFIC_FS) {
+ assertThat(core.createExternalPaths())
+ .containsExactly(new Path("file:///a"), new Path("file:///c"));
+ } else {
+ assertThat(core.createExternalPaths()).isEqualTo(core.dataFileExternalPathList());
+ }
+ }
+
+ @Test
+ void testGlobalIndexExternalPaths() {
+ Options options = new Options();
+ CoreOptions core = new CoreOptions(options);
+ assertThat(core.globalIndexExternalPaths()).isEmpty();
+ options.set(CoreOptions.GLOBAL_INDEX_EXTERNAL_PATH, "");
+ assertThat(core.globalIndexExternalPaths()).isEmpty();
+ for (String value :
+ new String[] {"file:///a", " file:///a/ ", "file:///a,", "file:///a,,"}) {
+ options.set(CoreOptions.GLOBAL_INDEX_EXTERNAL_PATH, value);
+ assertThat(core.globalIndexExternalPaths())
+ .as("global-index.external-path=%s", value)
+ .containsExactly(new Path("file:///a"));
+ }
+ for (String value :
+ new String[] {
+ "file:///a, file:///b", "file:///a,file:///b,", "file:///a,file:///b,,"
+ }) {
+ options.set(CoreOptions.GLOBAL_INDEX_EXTERNAL_PATH, value);
+ assertThat(core.globalIndexExternalPaths())
+ .as("global-index.external-path=%s", value)
+ .containsExactly(new Path("file:///a"), new Path("file:///b"));
+ }
+ for (String value :
+ new String[] {
+ ",",
+ ",,",
+ " ",
+ "a",
+ ",file:///a",
+ "file:///a,,file:///b",
+ ",file:///a,,file:///b,",
+ "file:///a, ",
+ "file:///a,/b",
+ "file:///a, ,file:///b"
+ }) {
+ options.set(CoreOptions.GLOBAL_INDEX_EXTERNAL_PATH, value);
+ assertThatThrownBy(core::globalIndexExternalPaths)
+ .as("global-index.external-path=%s", value)
+ .isInstanceOf(IllegalArgumentException.class);
+ }
+ }
+
@Test
void testCommitLastSafeSnapshotAndStrictMode() {
Options options = new Options();
diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java
index f119c84e5837..193345e18c04 100644
--- a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java
+++ b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java
@@ -27,6 +27,7 @@
import org.apache.paimon.data.serializer.InternalRowSerializer;
import org.apache.paimon.format.HadoopCompressionType;
import org.apache.paimon.fs.FileIO;
+import org.apache.paimon.fs.Path;
import org.apache.paimon.fs.ResolvingFileIO;
import org.apache.paimon.options.CatalogOptions;
import org.apache.paimon.options.ConfigOption;
@@ -1086,21 +1087,28 @@ public void testDropTable() throws Exception {
assertThatCode(() -> catalog.dropTable(nonExistingTable, true)).doesNotThrowAnyException();
}
- @Test
- public void testDropTableWithExternalPaths() throws Exception {
+ @ParameterizedTest
+ @ValueSource(strings = {"%s,%s", " %s , %s ", "%s,%s,", "%s,%s,,"})
+ public void testDropTableWithExternalPaths(String pathFormat) throws Exception {
catalog.createDatabase("test_db", false);
Identifier identifier = Identifier.create("test_db", "table_with_external_paths");
java.nio.file.Path dataExternalPath = tempFile.resolve("data-external-path");
+ java.nio.file.Path secondDataPath = tempFile.resolve("data-external-path-2");
java.nio.file.Path globalIndexExternalPath = tempFile.resolve("global-index-external-path");
+ java.nio.file.Path secondIndexPath = tempFile.resolve("global-index-external-path-2");
Files.createDirectories(dataExternalPath.resolve("data"));
+ Files.createDirectories(secondDataPath.resolve("data"));
Files.createDirectories(globalIndexExternalPath.resolve("index"));
+ Files.createDirectories(secondIndexPath.resolve("index"));
Map options = new HashMap<>();
options.put(
- CoreOptions.DATA_FILE_EXTERNAL_PATHS.key(), dataExternalPath.toUri().toString());
+ CoreOptions.DATA_FILE_EXTERNAL_PATHS.key(),
+ String.format(pathFormat, dataExternalPath.toUri(), secondDataPath.toUri()));
options.put(
CoreOptions.GLOBAL_INDEX_EXTERNAL_PATH.key(),
- globalIndexExternalPath.toUri().toString());
+ String.format(
+ pathFormat, globalIndexExternalPath.toUri(), secondIndexPath.toUri()));
Schema schema =
new Schema(
Lists.newArrayList(
@@ -1112,10 +1120,21 @@ public void testDropTableWithExternalPaths() throws Exception {
"");
catalog.createTable(identifier, schema, false);
+ FileStoreTable table = (FileStoreTable) catalog.getTable(identifier);
+ assertThat(table.coreOptions().dataFileExternalPathList())
+ .containsExactly(
+ new Path(dataExternalPath.toUri().toString()),
+ new Path(secondDataPath.toUri().toString()));
+ assertThat(table.coreOptions().globalIndexExternalPaths())
+ .containsExactly(
+ new Path(globalIndexExternalPath.toUri().toString()),
+ new Path(secondIndexPath.toUri().toString()));
catalog.dropTable(identifier, false);
assertThat(Files.exists(dataExternalPath)).isFalse();
+ assertThat(Files.exists(secondDataPath)).isFalse();
assertThat(Files.exists(globalIndexExternalPath)).isFalse();
+ assertThat(Files.exists(secondIndexPath)).isFalse();
}
@Test
diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/FileSystemCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/catalog/FileSystemCatalogTest.java
index 0e180938a160..0876c2c1b665 100644
--- a/paimon-core/src/test/java/org/apache/paimon/catalog/FileSystemCatalogTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/catalog/FileSystemCatalogTest.java
@@ -34,6 +34,7 @@
import org.apache.paimon.table.sink.BatchWriteBuilder;
import org.apache.paimon.types.DataTypes;
import org.apache.paimon.utils.SnapshotManager;
+import org.apache.paimon.utils.TraceableFileIO;
import org.apache.paimon.shade.guava30.com.google.common.collect.Lists;
@@ -308,6 +309,62 @@ public void testDropTableWithInvalidOptions(boolean cacheEnabled) throws Excepti
catalog.dropTable(identifier, true);
}
+ @ParameterizedTest
+ @EnumSource(
+ value = CoreOptions.ExternalPathStrategy.class,
+ names = {"NONE", "SPECIFIC_FS"})
+ public void testDropTableWithDynamicExternalPathStrategy(
+ CoreOptions.ExternalPathStrategy strategy) throws Exception {
+ Identifier identifier = Identifier.create("test_db", "dynamic_external_paths");
+ catalog.createDatabase(identifier.getDatabaseName(), false);
+ Path firstPath = new Path(tempFile.resolve("external-file").toUri().toString());
+ Path secondPath =
+ new Path(TraceableFileIO.SCHEME + "://" + tempFile.resolve("external-traceable"));
+ catalog.createTable(
+ identifier,
+ Schema.newBuilder()
+ .column("id", DataTypes.INT())
+ .option(CoreOptions.BUCKET.key(), "-1")
+ .option(
+ CoreOptions.DATA_FILE_EXTERNAL_PATHS.key(),
+ " " + firstPath + " , " + secondPath + " ")
+ .option(
+ CoreOptions.DATA_FILE_EXTERNAL_PATHS_STRATEGY.key(),
+ strategy.toString())
+ .option(CoreOptions.DATA_FILE_EXTERNAL_PATHS_SPECIFIC_FS.key(), "file")
+ .build(),
+ false);
+ FileStoreTable table = (FileStoreTable) catalog.getTable(identifier);
+ if (strategy == CoreOptions.ExternalPathStrategy.NONE) {
+ assertThat(table.store().pathFactory().getExternalPaths()).isEmpty();
+ } else {
+ assertThat(table.store().pathFactory().getExternalPaths()).containsExactly(firstPath);
+ }
+
+ Map dynamicOptions = new HashMap<>();
+ dynamicOptions.put(CoreOptions.DATA_FILE_EXTERNAL_PATHS_STRATEGY.key(), "specific-fs");
+ dynamicOptions.put(
+ CoreOptions.DATA_FILE_EXTERNAL_PATHS_SPECIFIC_FS.key(), TraceableFileIO.SCHEME);
+ FileStoreTable writerTable = table.copy(dynamicOptions);
+ assertThat(writerTable.store().pathFactory().getExternalPaths())
+ .containsExactly(secondPath);
+ BatchWriteBuilder writeBuilder = writerTable.newBatchWriteBuilder();
+ try (BatchTableWrite write = writeBuilder.newWrite();
+ BatchTableCommit commit = writeBuilder.newCommit()) {
+ write.write(GenericRow.of(1));
+ commit.commit(write.prepareCommit());
+ }
+ assertThat(fileIO.exists(secondPath)).isTrue();
+ FileSystemSchemaManager schemaManager =
+ new FileSystemSchemaManager(fileIO, table.location());
+ assertThat(schemaManager.listAll()).hasSize(1);
+ assertThat(schemaManager.latest().get().options())
+ .containsEntry(CoreOptions.DATA_FILE_EXTERNAL_PATHS_SPECIFIC_FS.key(), "file");
+
+ catalog.dropTable(identifier, false);
+ assertThat(fileIO.exists(secondPath)).isFalse();
+ }
+
@ParameterizedTest
@ValueSource(booleans = {false, true})
public void testDropTableWithInvalidOptionsAndExternalPaths(boolean cacheEnabled)
diff --git a/paimon-core/src/test/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFileTest.java b/paimon-core/src/test/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFileTest.java
index c1a8e2a0fed2..23f27769b38c 100644
--- a/paimon-core/src/test/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFileTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFileTest.java
@@ -31,6 +31,8 @@
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
+import javax.annotation.Nullable;
+
import java.io.DataInputStream;
import java.io.IOException;
import java.io.InputStream;
@@ -448,7 +450,7 @@ private IndexPathFactory getPathFactory() {
Path dir = new Path(tempPath.toUri());
return new IndexPathFactory() {
@Override
- public Path toPath(String fileName) {
+ public Path newPathFromName(String fileName) {
return new Path(dir, fileName);
}
@@ -461,6 +463,11 @@ public Path newPath() {
public boolean isExternalPath() {
return false;
}
+
+ @Override
+ public Path toPath(String fileName, @Nullable String externalPath) {
+ return externalPath == null ? newPathFromName(fileName) : new Path(externalPath);
+ }
};
}
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/globalindex/GlobalIndexBuilderUtilsTest.java b/paimon-core/src/test/java/org/apache/paimon/globalindex/GlobalIndexBuilderUtilsTest.java
index b4f42dff78ed..27953c1fe6b5 100644
--- a/paimon-core/src/test/java/org/apache/paimon/globalindex/GlobalIndexBuilderUtilsTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/globalindex/GlobalIndexBuilderUtilsTest.java
@@ -22,6 +22,8 @@
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.fs.FileIO;
import org.apache.paimon.fs.Path;
+import org.apache.paimon.fs.PositionOutputStream;
+import org.apache.paimon.fs.SeekableInputStream;
import org.apache.paimon.fs.local.LocalFileIO;
import org.apache.paimon.index.DataEvolutionIndexSourceMeta;
import org.apache.paimon.index.IndexFileMeta;
@@ -45,8 +47,13 @@
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import javax.annotation.Nullable;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.LinkedHashMap;
@@ -54,6 +61,7 @@
import java.util.Map;
import java.util.UUID;
+import static org.apache.paimon.utils.FileStorePathFactoryTest.createNonPartFactory;
import static org.assertj.core.api.Assertions.assertThat;
/** Tests for {@link GlobalIndexBuilderUtils}. */
@@ -65,6 +73,69 @@ class GlobalIndexBuilderUtilsTest {
private IndexPathFactory indexPathFactory;
private CoreOptions coreOptions;
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void testFileLocationsWithDifferentReaderOptions(boolean external) throws Exception {
+ Path root = new Path(tempDir.toUri());
+ List roots = Arrays.asList(new Path(root, "a"), new Path(root, "b"));
+ if (external) {
+ coreOptions
+ .toConfiguration()
+ .set(CoreOptions.GLOBAL_INDEX_EXTERNAL_PATH, roots.get(0) + "," + roots.get(1));
+ }
+ IndexPathFactory writerPaths =
+ createNonPartFactory(root, external ? roots : Collections.emptyList())
+ .globalIndexFileFactory();
+ GlobalIndexFileReadWrite writer = new GlobalIndexFileReadWrite(fileIO, writerPaths);
+ List entries = new ArrayList<>();
+ List parents = new ArrayList<>();
+ for (int i = 0; i < 6; i++) {
+ String file = writer.newFilePath("test");
+ assertThat(new Path(file).toUri().getScheme()).isEqualTo("file");
+ assertThat(new Path(file).getName())
+ .startsWith("test-global-index-")
+ .endsWith(".index");
+ try (PositionOutputStream output = writer.newOutputStream(file)) {
+ output.write(i);
+ }
+ assertThat(writer.fileSize(file)).isEqualTo(1);
+ parents.add(new Path(file).getParent());
+ entries.add(new ResultEntry(file, 1, null));
+ }
+ if (external) {
+ for (Path externalRoot : roots) {
+ assertThat(Collections.frequency(parents, externalRoot)).isEqualTo(3);
+ }
+ } else {
+ assertThat(parents).containsOnly(new Path(root, "index"));
+ }
+
+ List metas =
+ GlobalIndexBuilderUtils.toIndexFileMetas(
+ fileIO, coreOptions, new Range(0, 5), 0, "test", entries);
+
+ // Reading committed files must not depend on the current external path configuration.
+ IndexPathFactory readerPaths =
+ createNonPartFactory(root, external ? Collections.emptyList() : roots)
+ .globalIndexFileFactory();
+ GlobalIndexFileReadWrite reader = new GlobalIndexFileReadWrite(fileIO, readerPaths);
+ for (int i = 0; i < metas.size(); i++) {
+ IndexFileMeta meta = metas.get(i);
+ Path path = new Path(entries.get(i).filePath());
+ assertThat(meta.fileName()).isEqualTo(path.getName());
+ assertThat(meta.externalPath()).isEqualTo(external ? path.toString() : null);
+ assertThat(readerPaths.toPath(meta)).isEqualTo(path);
+ try (SeekableInputStream input =
+ reader.getInputStream(
+ new GlobalIndexIOMeta(
+ readerPaths.toPath(meta), meta.fileSize(), null))) {
+ assertThat(input.read()).isEqualTo(i);
+ }
+ fileIO.delete(readerPaths.toPath(meta), false);
+ assertThat(fileIO.exists(path)).isFalse();
+ }
+ }
+
@BeforeEach
void setUp() {
fileIO = new LocalFileIO();
@@ -72,7 +143,7 @@ void setUp() {
indexPathFactory =
new IndexPathFactory() {
@Override
- public Path toPath(String fileName) {
+ public Path newPathFromName(String fileName) {
return new Path(dir, fileName);
}
@@ -85,6 +156,13 @@ public Path newPath() {
public boolean isExternalPath() {
return false;
}
+
+ @Override
+ public Path toPath(String fileName, @Nullable String externalPath) {
+ return externalPath == null
+ ? newPathFromName(fileName)
+ : new Path(externalPath);
+ }
};
coreOptions = new CoreOptions(new Options().toMap());
}
@@ -101,14 +179,7 @@ void testToIndexFileMetasMultiColumn() throws IOException {
List metas =
GlobalIndexBuilderUtils.toIndexFileMetas(
- fileIO,
- indexPathFactory,
- coreOptions,
- range,
- fields,
- "test-type",
- entries,
- null);
+ fileIO, coreOptions, range, fields, "test-type", entries, null);
assertThat(metas).hasSize(1);
assertThat(metas.get(0).globalIndexMeta().indexFieldId()).isEqualTo(1);
@@ -129,14 +200,7 @@ void testToIndexFileMetasSingleColumn() throws IOException {
List metas =
GlobalIndexBuilderUtils.toIndexFileMetas(
- fileIO,
- indexPathFactory,
- coreOptions,
- range,
- fields,
- "test-type",
- entries,
- null);
+ fileIO, coreOptions, range, fields, "test-type", entries, null);
assertThat(metas).hasSize(1);
assertThat(metas.get(0).globalIndexMeta().indexFieldId()).isEqualTo(1);
@@ -147,19 +211,16 @@ void testToIndexFileMetasSingleColumn() throws IOException {
void testRowCountPerIndexFile() throws IOException {
List entries =
Arrays.asList(
- new ResultEntry("first", 2, null), new ResultEntry("second", 1, null));
+ new ResultEntry(
+ indexPathFactory.newPathFromName("first").toString(), 2, null),
+ new ResultEntry(
+ indexPathFactory.newPathFromName("second").toString(), 1, null));
for (ResultEntry entry : entries) {
- fileIO.newOutputStream(indexPathFactory.toPath(entry.fileName()), false).close();
+ fileIO.newOutputStream(new Path(entry.filePath()), false).close();
}
List metas =
GlobalIndexBuilderUtils.toIndexFileMetas(
- fileIO,
- indexPathFactory,
- coreOptions,
- new Range(100, 102),
- 1,
- "test-type",
- entries);
+ fileIO, coreOptions, new Range(100, 102), 1, "test-type", entries);
assertThat(metas).extracting(IndexFileMeta::rowCount).containsExactly(2L, 1L);
assertThat(metas)
.extracting(file -> file.globalIndexMeta().rowRange())
@@ -174,7 +235,6 @@ void testToIndexFileMetasWithSourceMeta() throws IOException {
List metas =
GlobalIndexBuilderUtils.toIndexFileMetas(
fileIO,
- indexPathFactory,
coreOptions,
new Range(0, 9),
Collections.singletonList(field),
@@ -199,14 +259,7 @@ void testToIndexFileMetasThreeColumns() throws IOException {
List metas =
GlobalIndexBuilderUtils.toIndexFileMetas(
- fileIO,
- indexPathFactory,
- coreOptions,
- range,
- fields,
- "test-type",
- entries,
- null);
+ fileIO, coreOptions, range, fields, "test-type", entries, null);
assertThat(metas).hasSize(1);
assertThat(metas.get(0).globalIndexMeta().indexFieldId()).isEqualTo(1);
@@ -327,9 +380,9 @@ void testShardSplitsUsesStableGlobalRowIdWindows() {
private List createDummyResultEntries() throws IOException {
String fileName = "test-index-" + UUID.randomUUID();
- Path filePath = indexPathFactory.toPath(fileName);
+ Path filePath = indexPathFactory.newPathFromName(fileName);
fileIO.newOutputStream(filePath, false).close();
- return Collections.singletonList(new ResultEntry(fileName, 100, null));
+ return Collections.singletonList(new ResultEntry(filePath.toString(), 100, null));
}
private ManifestEntry createEntry(Long firstRowId, long rowCount) {
diff --git a/paimon-core/src/test/java/org/apache/paimon/globalindex/GlobalIndexQueryTest.java b/paimon-core/src/test/java/org/apache/paimon/globalindex/GlobalIndexQueryTest.java
index 23bf51366c1f..1101acb05a38 100644
--- a/paimon-core/src/test/java/org/apache/paimon/globalindex/GlobalIndexQueryTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/globalindex/GlobalIndexQueryTest.java
@@ -56,6 +56,8 @@
import org.junit.jupiter.params.provider.ValueSource;
import org.mockito.MockedStatic;
+import javax.annotation.Nullable;
+
import java.io.IOException;
import java.math.BigDecimal;
import java.nio.charset.StandardCharsets;
@@ -132,7 +134,7 @@ void workerReceivesRoutedFilesAfterSplitSerialization() throws Exception {
files.add(
new IndexFileMeta(
"ivf-pq",
- file.fileName(),
+ new Path(file.filePath()).getName(),
100,
file.rowCount(),
new GlobalIndexMeta(0, 2, 0, null, file.meta()),
@@ -304,7 +306,7 @@ void testNegativePredicatesRetainNullFilesAndOriginalRowCount(String indexType)
IndexPathFactory paths =
new IndexPathFactory() {
@Override
- public Path toPath(String name) {
+ public Path newPathFromName(String name) {
return new Path(tempDir.resolve(name).toString());
}
@@ -317,6 +319,13 @@ public Path newPath() {
public boolean isExternalPath() {
return false;
}
+
+ @Override
+ public Path toPath(String fileName, @Nullable String externalPath) {
+ return externalPath == null
+ ? newPathFromName(fileName)
+ : new Path(externalPath);
+ }
};
GlobalIndexFileReadWrite io = new GlobalIndexFileReadWrite(fileIO, paths);
List files = new ArrayList<>();
@@ -338,8 +347,8 @@ public boolean isExternalPath() {
files.add(
new IndexFileMeta(
indexType,
- entry.fileName(),
- io.fileSize(entry.fileName()),
+ new Path(entry.filePath()).getName(),
+ io.fileSize(entry.filePath()),
entry.rowCount(),
new GlobalIndexMeta(100, 102, 0, null, entry.meta()),
null));
@@ -900,7 +909,7 @@ private GlobalIndexQuery create(
IndexPathFactory paths =
new IndexPathFactory() {
@Override
- public Path toPath(String name) {
+ public Path newPathFromName(String name) {
return new Path("index/" + name);
}
@@ -913,6 +922,13 @@ public Path newPath() {
public boolean isExternalPath() {
return false;
}
+
+ @Override
+ public Path toPath(String fileName, @Nullable String externalPath) {
+ return externalPath == null
+ ? newPathFromName(fileName)
+ : new Path(externalPath);
+ }
};
return GlobalIndexQuery.create(
rowType, predicate, Collections.singletonList(file), paths, new Options());
diff --git a/paimon-core/src/test/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexWriterTest.java b/paimon-core/src/test/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexWriterTest.java
index 817eeb02105a..ac40a193b19b 100644
--- a/paimon-core/src/test/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexWriterTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexWriterTest.java
@@ -20,18 +20,26 @@
import org.apache.paimon.data.GenericRow;
import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.fs.Path;
import org.apache.paimon.globalindex.GlobalIndexSingleColumnWriter;
import org.apache.paimon.globalindex.ResultEntry;
+import org.apache.paimon.index.IndexFileMeta;
import org.apache.paimon.options.Options;
import org.apache.paimon.schema.Schema;
+import org.apache.paimon.table.FileStoreTable;
import org.apache.paimon.table.TableTestBase;
+import org.apache.paimon.table.sink.CommitMessage;
+import org.apache.paimon.table.sink.CommitMessageImpl;
import org.apache.paimon.types.DataTypes;
import org.apache.paimon.utils.Range;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
import java.io.Closeable;
import java.util.ArrayDeque;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
@@ -46,6 +54,54 @@
/** Tests for {@link SortedGlobalIndexWriter}. */
public class SortedGlobalIndexWriterTest extends TableTestBase {
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ public void testBuildPreservesAllocatedPathsInCommit(boolean external) throws Exception {
+ createTableDefault();
+ FileStoreTable table = getTableDefault();
+ Path firstRoot = new Path(table.location(), "external-a");
+ Path secondRoot = new Path(table.location(), "external-b");
+ if (external) {
+ table =
+ table.copy(
+ Collections.singletonMap(
+ "global-index.external-path", firstRoot + "," + secondRoot));
+ }
+ Options options = new Options();
+ options.setString("btree-index.records-per-range", "1");
+ SortedGlobalIndexWriter writer =
+ new SortedGlobalIndexWriter(table, "btree", options)
+ .withIndexFields(Collections.singletonList("f0"));
+ List commits =
+ writer.buildForSinglePartition(
+ new Range(0, 1),
+ null,
+ Arrays.asList(GenericRow.of(10, 0L), GenericRow.of(20, 1L))
+ .iterator(),
+ 1L);
+ assertThat(commits).hasSize(2);
+ List parents = new ArrayList<>();
+ for (CommitMessage commit : commits) {
+ List files =
+ ((CommitMessageImpl) commit).newFilesIncrement().newIndexFiles();
+ assertThat(files).hasSize(1);
+ IndexFileMeta file = files.get(0);
+ Path path = table.store().pathFactory().globalIndexFileFactory().toPath(file);
+ assertThat(table.fileIO().exists(path)).isTrue();
+ assertThat(file.fileName())
+ .isEqualTo(path.getName())
+ .startsWith("btree-global-index-")
+ .endsWith(".index");
+ assertThat(file.externalPath()).isEqualTo(external ? path.toString() : null);
+ parents.add(path.getParent());
+ }
+ if (external) {
+ assertThat(parents).containsExactlyInAnyOrder(firstRoot, secondRoot);
+ } else {
+ assertThat(parents).containsOnly(new Path(table.location(), "index"));
+ }
+ }
+
@Test
public void testFieldOptionsControlFileRotationAndBuildSettings() throws Exception {
createTableDefault();
@@ -72,9 +128,13 @@ public void testSingleColumnWriterRotationPreservesResultGroups() throws Excepti
GlobalIndexSingleColumnWriter first = mock(GlobalIndexSingleColumnWriter.class);
GlobalIndexSingleColumnWriter second = mock(GlobalIndexSingleColumnWriter.class);
when(first.finish(2))
- .thenReturn(Collections.singletonList(new ResultEntry("index-1", 2, null)));
+ .thenReturn(
+ Collections.singletonList(
+ new ResultEntry("memory://test/index-1", 2, null)));
when(second.finish(1))
- .thenReturn(Collections.singletonList(new ResultEntry("index-2", 1, null)));
+ .thenReturn(
+ Collections.singletonList(
+ new ResultEntry("memory://test/index-2", 1, null)));
Queue writers =
new ArrayDeque<>(Arrays.asList(first, second));
SortedSingleColumnIndexWriter rotatingWriter =
@@ -89,15 +149,20 @@ public void testSingleColumnWriterRotationPreservesResultGroups() throws Excepti
verify(first).write(20, 1);
verify(second).write(30, 2);
assertThat(results).hasSize(2);
- assertThat(results.get(0)).extracting(ResultEntry::fileName).containsExactly("index-1");
- assertThat(results.get(1)).extracting(ResultEntry::fileName).containsExactly("index-2");
+ assertThat(results.get(0))
+ .extracting(ResultEntry::filePath)
+ .containsExactly("memory://test/index-1");
+ assertThat(results.get(1))
+ .extracting(ResultEntry::filePath)
+ .containsExactly("memory://test/index-2");
}
@Test
public void testSingleColumnWriterPreservesSourceRowCount() throws Exception {
GlobalIndexSingleColumnWriter writer = mock(GlobalIndexSingleColumnWriter.class);
when(writer.finish(5))
- .thenReturn(Collections.singletonList(new ResultEntry("index", 5, null)));
+ .thenReturn(
+ Collections.singletonList(new ResultEntry("memory://test/index", 5, null)));
SortedSingleColumnIndexWriter taskWriter =
SortedSingleColumnIndexWriter.forSourceRowCount(5, writer);
@@ -114,7 +179,8 @@ public void testSingleColumnWriterPreservesSourceRowCount() throws Exception {
public void testSingleColumnWriterPreservesSourceRangeWithoutEntries() {
GlobalIndexSingleColumnWriter writer = mock(GlobalIndexSingleColumnWriter.class);
when(writer.finish(3))
- .thenReturn(Collections.singletonList(new ResultEntry("index", 3, null)));
+ .thenReturn(
+ Collections.singletonList(new ResultEntry("memory://test/index", 3, null)));
SortedSingleColumnIndexWriter taskWriter =
SortedSingleColumnIndexWriter.forSourceRowCount(3, writer);
diff --git a/paimon-core/src/test/java/org/apache/paimon/index/HashIndexFileTest.java b/paimon-core/src/test/java/org/apache/paimon/index/HashIndexFileTest.java
index 974a518e9ae3..7fbcdba397bf 100644
--- a/paimon-core/src/test/java/org/apache/paimon/index/HashIndexFileTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/index/HashIndexFileTest.java
@@ -25,6 +25,8 @@
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
+import javax.annotation.Nullable;
+
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@@ -49,7 +51,7 @@ public Path newPath() {
}
@Override
- public Path toPath(String fileName) {
+ public Path newPathFromName(String fileName) {
return new Path(dir, fileName);
}
@@ -57,6 +59,13 @@ public Path toPath(String fileName) {
public boolean isExternalPath() {
return false;
}
+
+ @Override
+ public Path toPath(String fileName, @Nullable String externalPath) {
+ return externalPath == null
+ ? newPathFromName(fileName)
+ : new Path(externalPath);
+ }
};
HashIndexFile file = new HashIndexFile(LocalFileIO.create(), pathFactory);
diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pk/BucketedPrimaryKeyIndexMaintainerTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pk/BucketedPrimaryKeyIndexMaintainerTest.java
index d4df19ce1d16..250b595f21d4 100644
--- a/paimon-core/src/test/java/org/apache/paimon/index/pk/BucketedPrimaryKeyIndexMaintainerTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/index/pk/BucketedPrimaryKeyIndexMaintainerTest.java
@@ -52,6 +52,8 @@
import org.junit.jupiter.params.provider.ValueSource;
import org.mockito.InOrder;
+import javax.annotation.Nullable;
+
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -871,7 +873,7 @@ private IndexPathFactory pathFactory() {
Path directory = new Path(tempPath.toUri());
return new IndexPathFactory() {
@Override
- public Path toPath(String fileName) {
+ public Path newPathFromName(String fileName) {
return new Path(directory, fileName);
}
@@ -884,6 +886,11 @@ public Path newPath() {
public boolean isExternalPath() {
return false;
}
+
+ @Override
+ public Path toPath(String fileName, @Nullable String externalPath) {
+ return externalPath == null ? newPathFromName(fileName) : new Path(externalPath);
+ }
};
}
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkfulltext/PkFullTextIndexFileTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkfulltext/PkFullTextIndexFileTest.java
index bb7f19fe6147..ef9d726b9b00 100644
--- a/paimon-core/src/test/java/org/apache/paimon/index/pkfulltext/PkFullTextIndexFileTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/index/pkfulltext/PkFullTextIndexFileTest.java
@@ -45,6 +45,8 @@
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
+import javax.annotation.Nullable;
+
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -224,7 +226,7 @@ private IndexPathFactory pathFactory() {
Path directory = new Path(tempPath.toUri());
return new IndexPathFactory() {
@Override
- public Path toPath(String fileName) {
+ public Path newPathFromName(String fileName) {
return new Path(directory, fileName);
}
@@ -237,6 +239,11 @@ public Path newPath() {
public boolean isExternalPath() {
return false;
}
+
+ @Override
+ public Path toPath(String fileName, @Nullable String externalPath) {
+ return externalPath == null ? newPathFromName(fileName) : new Path(externalPath);
+ }
};
}
@@ -257,12 +264,12 @@ public void write(Object key, long relativeRowId) {
@Override
public List finish() {
try {
- String fileName = fileWriter.newFileName("full-text");
- try (PositionOutputStream out = fileWriter.newOutputStream(fileName)) {
+ String filePath = fileWriter.newFilePath("full-text");
+ try (PositionOutputStream out = fileWriter.newOutputStream(filePath)) {
out.write(1);
}
return Collections.singletonList(
- new ResultEntry(fileName, rowIds.size(), new byte[] {2}));
+ new ResultEntry(filePath, rowIds.size(), new byte[] {2}));
} catch (IOException e) {
throw new RuntimeException(e);
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pksorted/BucketedSortedIndexMaintainerTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pksorted/BucketedSortedIndexMaintainerTest.java
index 1e15994e08af..2e80ca7dd627 100644
--- a/paimon-core/src/test/java/org/apache/paimon/index/pksorted/BucketedSortedIndexMaintainerTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/index/pksorted/BucketedSortedIndexMaintainerTest.java
@@ -36,6 +36,8 @@
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
+import javax.annotation.Nullable;
+
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
@@ -798,7 +800,7 @@ private IndexPathFactory pathFactory() {
Path directory = new Path(tempPath.toUri());
return new IndexPathFactory() {
@Override
- public Path toPath(String fileName) {
+ public Path newPathFromName(String fileName) {
return new Path(directory, fileName);
}
@@ -811,6 +813,11 @@ public Path newPath() {
public boolean isExternalPath() {
return false;
}
+
+ @Override
+ public Path toPath(String fileName, @Nullable String externalPath) {
+ return externalPath == null ? newPathFromName(fileName) : new Path(externalPath);
+ }
};
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PkSequentialIndexBuilderTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PkSequentialIndexBuilderTest.java
index da9f1670a574..90dd839e707f 100644
--- a/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PkSequentialIndexBuilderTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PkSequentialIndexBuilderTest.java
@@ -177,7 +177,7 @@ private IndexPathFactory pathFactory() {
Path root = new Path(tempPath.toUri());
return new IndexPathFactory() {
@Override
- public Path toPath(String fileName) {
+ public Path newPathFromName(String fileName) {
return new Path(root, fileName);
}
@@ -190,6 +190,11 @@ public Path newPath() {
public boolean isExternalPath() {
return false;
}
+
+ @Override
+ public Path toPath(String fileName, @Nullable String externalPath) {
+ return externalPath == null ? newPathFromName(fileName) : new Path(externalPath);
+ }
};
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PkSortedIndexBuilderTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PkSortedIndexBuilderTest.java
index 5af2f8b00f9b..3d168eeb64d3 100644
--- a/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PkSortedIndexBuilderTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PkSortedIndexBuilderTest.java
@@ -47,6 +47,8 @@
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
+import javax.annotation.Nullable;
+
import java.io.IOException;
import java.lang.reflect.Method;
import java.util.ArrayList;
@@ -137,7 +139,7 @@ void testBuildsQueryableMultiValueFromUnsortedArrayRows() throws Exception {
List ioMetas =
Collections.singletonList(
new GlobalIndexIOMeta(
- pathFactory.toPath(payload.fileName()),
+ pathFactory.toPath(payload),
payload.fileSize(),
payload.globalIndexMeta().indexMeta()));
ExecutorService executor = newDirectExecutorService();
@@ -386,7 +388,7 @@ private static void assertQuery(
List ioMetas =
Collections.singletonList(
new GlobalIndexIOMeta(
- pathFactory.toPath(payload.fileName()),
+ pathFactory.toPath(payload),
payload.fileSize(),
payload.globalIndexMeta().indexMeta()));
ExecutorService executor = newDirectExecutorService();
@@ -448,7 +450,7 @@ private static IndexPathFactory pathFactory(java.nio.file.Path directory) {
Path path = new Path(directory.toUri());
return new IndexPathFactory() {
@Override
- public Path toPath(String fileName) {
+ public Path newPathFromName(String fileName) {
return new Path(path, fileName);
}
@@ -461,6 +463,11 @@ public Path newPath() {
public boolean isExternalPath() {
return false;
}
+
+ @Override
+ public Path toPath(String fileName, @Nullable String externalPath) {
+ return externalPath == null ? newPathFromName(fileName) : new Path(externalPath);
+ }
};
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PkSortedIndexFileTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PkSortedIndexFileTest.java
index 0e0baf60b462..de492b752d9d 100644
--- a/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PkSortedIndexFileTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PkSortedIndexFileTest.java
@@ -36,6 +36,8 @@
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
+import javax.annotation.Nullable;
+
import java.io.IOException;
import java.lang.reflect.Method;
import java.nio.file.Files;
@@ -77,12 +79,12 @@ void testUsesCloseableStreamingBitmapWriter() throws Exception {
options(),
new GlobalIndexFileWriter() {
@Override
- public String newFileName(String prefix) {
- return prefix + ".index";
+ public String newFilePath(String prefix) {
+ return "memory://test/" + prefix + ".index";
}
@Override
- public PositionOutputStream newOutputStream(String fileName) {
+ public PositionOutputStream newOutputStream(String filePath) {
throw new AssertionError("Writer must open output lazily.");
}
});
@@ -206,16 +208,16 @@ public void write(Object key, long relativeRowId) {
public List finish() {
List results = new ArrayList<>();
for (int i = 0; i < 2; i++) {
- String fileName = fileWriter.newFileName("test");
+ String filePath = fileWriter.newFilePath("test");
try (PositionOutputStream output =
- fileWriter.newOutputStream(fileName)) {
+ fileWriter.newOutputStream(filePath)) {
output.write(new byte[] {1});
} catch (IOException e) {
throw new RuntimeException(e);
}
results.add(
new ResultEntry(
- fileName, rowCount / 2, new byte[] {2}));
+ filePath, rowCount / 2, new byte[] {2}));
}
return results;
}
@@ -302,7 +304,7 @@ private static IndexPathFactory pathFactory(java.nio.file.Path directory) {
Path path = new Path(directory.toUri());
return new IndexPathFactory() {
@Override
- public Path toPath(String fileName) {
+ public Path newPathFromName(String fileName) {
return new Path(path, fileName);
}
@@ -315,6 +317,11 @@ public Path newPath() {
public boolean isExternalPath() {
return false;
}
+
+ @Override
+ public Path toPath(String fileName, @Nullable String externalPath) {
+ return externalPath == null ? newPathFromName(fileName) : new Path(externalPath);
+ }
};
}
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/BucketedVectorIndexMaintainerTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/BucketedVectorIndexMaintainerTest.java
index e6b1986a070e..a91f6e13684e 100644
--- a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/BucketedVectorIndexMaintainerTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/BucketedVectorIndexMaintainerTest.java
@@ -41,6 +41,8 @@
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
+import javax.annotation.Nullable;
+
import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.ArrayList;
@@ -869,7 +871,7 @@ private IndexPathFactory pathFactory() {
Path directory = new Path(tempPath.toUri());
return new IndexPathFactory() {
@Override
- public Path toPath(String fileName) {
+ public Path newPathFromName(String fileName) {
return new Path(directory, fileName);
}
@@ -882,6 +884,11 @@ public Path newPath() {
public boolean isExternalPath() {
return false;
}
+
+ @Override
+ public Path toPath(String fileName, @Nullable String externalPath) {
+ return externalPath == null ? newPathFromName(fileName) : new Path(externalPath);
+ }
};
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java
index fd972d40c48e..599b0e469d82 100644
--- a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java
@@ -36,6 +36,8 @@
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
+import javax.annotation.Nullable;
+
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
@@ -330,7 +332,7 @@ private IndexPathFactory pathFactory() {
Path directory = new Path(tempPath.toUri());
return new IndexPathFactory() {
@Override
- public Path toPath(String fileName) {
+ public Path newPathFromName(String fileName) {
return new Path(directory, fileName);
}
@@ -343,6 +345,11 @@ public Path newPath() {
public boolean isExternalPath() {
return false;
}
+
+ @Override
+ public Path toPath(String fileName, @Nullable String externalPath) {
+ return externalPath == null ? newPathFromName(fileName) : new Path(externalPath);
+ }
};
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/BitmapGlobalIndexTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/BitmapGlobalIndexTableTest.java
index a50eb2d0375e..ca40cf68b4fc 100644
--- a/paimon-core/src/test/java/org/apache/paimon/table/BitmapGlobalIndexTableTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/table/BitmapGlobalIndexTableTest.java
@@ -258,7 +258,6 @@ private CommitMessage buildIndex(
List indexFileMetas =
GlobalIndexBuilderUtils.toIndexFileMetas(
table.fileIO(),
- table.store().pathFactory().globalIndexFileFactory(),
table.coreOptions(),
rowRange,
indexField.id(),
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionTableTest.java
index 1a5347278b78..ddf030cf6fda 100644
--- a/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionTableTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionTableTest.java
@@ -1899,7 +1899,7 @@ public void testIndexPath() throws Exception {
FileStoreTable table = getTableDefault();
IndexPathFactory indexPathFactory = table.store().pathFactory().globalIndexFileFactory();
- Path path0 = indexPathFactory.toPath("test-file");
+ Path path0 = indexPathFactory.newPathFromName("test-file");
assertThat(path0.toString().contains(warehouse.toString())).isTrue();
String testExternalpath = "file:/external/path/test-file-dir";
Path path1 =
@@ -1920,7 +1920,7 @@ public void testIndexPath() throws Exception {
CoreOptions.GLOBAL_INDEX_EXTERNAL_PATH.key(), testExternalpath));
indexPathFactory = table.store().pathFactory().globalIndexFileFactory();
- Path path3 = indexPathFactory.toPath("test-file");
+ Path path3 = indexPathFactory.newPathFromName("test-file");
assertThat(path3.toString()).isEqualTo(testExternalpath + "/test-file");
String testExternalpath2 = "file:/external/path2/test-file";
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/IndexQuerySplitTest.java b/paimon-core/src/test/java/org/apache/paimon/table/IndexQuerySplitTest.java
index 3e1612003f25..197688f49623 100644
--- a/paimon-core/src/test/java/org/apache/paimon/table/IndexQuerySplitTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/table/IndexQuerySplitTest.java
@@ -932,8 +932,8 @@ public void testFMPartitionPruningAfterSplitRestore() throws Exception {
indexes.add(
new IndexFileMeta(
"fm",
- entry.fileName(),
- io.fileSize(entry.fileName()),
+ new Path(entry.filePath()).getName(),
+ io.fileSize(entry.filePath()),
entry.rowCount(),
new GlobalIndexMeta(
100,
@@ -1063,8 +1063,8 @@ private void createFMIndex(FileStoreTable table) throws Exception {
indexes.add(
new IndexFileMeta(
"fm",
- entry.fileName(),
- io.fileSize(entry.fileName()),
+ new Path(entry.filePath()).getName(),
+ io.fileSize(entry.filePath()),
entry.rowCount(),
new GlobalIndexMeta(
0, 99, table.rowType().getField("f1").id(), null, entry.meta()),
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/RTreeGlobalIndexTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/RTreeGlobalIndexTableTest.java
index cd08d1be7ffa..517f9c03e659 100644
--- a/paimon-core/src/test/java/org/apache/paimon/table/RTreeGlobalIndexTableTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/table/RTreeGlobalIndexTableTest.java
@@ -411,7 +411,6 @@ private void buildIndex(
List metas =
GlobalIndexBuilderUtils.toIndexFileMetas(
table.fileIO(),
- table.store().pathFactory().globalIndexFileFactory(),
table.coreOptions(),
range,
Collections.singletonList(field),
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java
index f43b7c590a44..ff741a11c99d 100644
--- a/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java
@@ -188,7 +188,6 @@ public void testPartialCompositeKeepsScalarFilterCoverage(String mode, boolean s
List files =
GlobalIndexBuilderUtils.toIndexFileMetas(
table.fileIO(),
- table.store().pathFactory().globalIndexFileFactory(),
table.coreOptions(),
new Range(0, 0),
fields,
@@ -1733,7 +1732,6 @@ private void buildAndCommitIndexWithFields(
List indexFiles =
GlobalIndexBuilderUtils.toIndexFileMetas(
table.fileIO(),
- table.store().pathFactory().globalIndexFileFactory(),
table.coreOptions(),
rowRange,
indexFields,
@@ -1772,7 +1770,6 @@ private void buildAndCommitSourceBackedIndex(FileStoreTable table, String[] docu
List indexFiles =
GlobalIndexBuilderUtils.toIndexFileMetas(
table.fileIO(),
- table.store().pathFactory().globalIndexFileFactory(),
table.coreOptions(),
new Range(0, documents.length - 1),
Collections.singletonList(textField),
@@ -1844,7 +1841,6 @@ private void buildAndCommitIndexRange(
List indexFiles =
GlobalIndexBuilderUtils.toIndexFileMetas(
table.fileIO(),
- table.store().pathFactory().globalIndexFileFactory(),
table.coreOptions(),
rowRange,
indexFields,
@@ -1933,7 +1929,6 @@ private void buildAndCommitIndexForColumn(
List indexFiles =
GlobalIndexBuilderUtils.toIndexFileMetas(
table.fileIO(),
- table.store().pathFactory().globalIndexFileFactory(),
table.coreOptions(),
rowRange,
textField.id(),
@@ -1974,7 +1969,6 @@ private void buildAndCommitIdBTreeIndexRange(FileStoreTable table, Range rowRang
List indexFiles =
GlobalIndexBuilderUtils.toIndexFileMetas(
table.fileIO(),
- table.store().pathFactory().globalIndexFileFactory(),
table.coreOptions(),
rowRange,
idField.id(),
@@ -2016,7 +2010,6 @@ private void buildAndCommitBTreeIndex(
List indexFiles =
GlobalIndexBuilderUtils.toIndexFileMetas(
table.fileIO(),
- table.store().pathFactory().globalIndexFileFactory(),
table.coreOptions(),
rowRange,
textField.id(),
@@ -2068,7 +2061,6 @@ private void buildAndCommitMultipleIndexFiles(FileStoreTable table, String[] doc
List indexFiles1 =
GlobalIndexBuilderUtils.toIndexFileMetas(
table.fileIO(),
- table.store().pathFactory().globalIndexFileFactory(),
table.coreOptions(),
rowRange1,
textField.id(),
@@ -2091,7 +2083,6 @@ private void buildAndCommitMultipleIndexFiles(FileStoreTable table, String[] doc
List indexFiles2 =
GlobalIndexBuilderUtils.toIndexFileMetas(
table.fileIO(),
- table.store().pathFactory().globalIndexFileFactory(),
table.coreOptions(),
rowRange2,
textField.id(),
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/RTreeRankedFilterTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/RTreeRankedFilterTest.java
index ead7613f004b..09847ba5c84a 100644
--- a/paimon-core/src/test/java/org/apache/paimon/table/source/RTreeRankedFilterTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/table/source/RTreeRankedFilterTest.java
@@ -178,7 +178,6 @@ private static void buildIndex(
List files =
GlobalIndexBuilderUtils.toIndexFileMetas(
table.fileIO(),
- table.store().pathFactory().globalIndexFileFactory(),
table.coreOptions(),
new Range(0, values.length - 1),
field.id(),
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java
index 30900d54a4d1..d2781a928655 100644
--- a/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java
@@ -728,7 +728,6 @@ private void buildAndCommitMultiValueIndex(FileStoreTable table, int rowCount)
List indexFiles =
GlobalIndexBuilderUtils.toIndexFileMetas(
table.fileIO(),
- table.store().pathFactory().globalIndexFileFactory(),
table.coreOptions(),
new Range(0, rowCount - 1),
tagsField.id(),
@@ -2150,7 +2149,6 @@ private void buildAndCommitIndex(FileStoreTable table, String fieldName, float[]
List indexFiles =
GlobalIndexBuilderUtils.toIndexFileMetas(
table.fileIO(),
- table.store().pathFactory().globalIndexFileFactory(),
table.coreOptions(),
rowRange,
vectorField.id(),
@@ -2192,7 +2190,6 @@ private void buildAndCommitMultipleIndexFiles(FileStoreTable table, float[][] ve
List indexFiles1 =
GlobalIndexBuilderUtils.toIndexFileMetas(
table.fileIO(),
- table.store().pathFactory().globalIndexFileFactory(),
table.coreOptions(),
rowRange1,
vectorField.id(),
@@ -2215,7 +2212,6 @@ private void buildAndCommitMultipleIndexFiles(FileStoreTable table, float[][] ve
List indexFiles2 =
GlobalIndexBuilderUtils.toIndexFileMetas(
table.fileIO(),
- table.store().pathFactory().globalIndexFileFactory(),
table.coreOptions(),
rowRange2,
vectorField.id(),
@@ -2387,7 +2383,6 @@ private void buildAndCommitVectorIndexWithFields(
List indexFiles =
GlobalIndexBuilderUtils.toIndexFileMetas(
table.fileIO(),
- table.store().pathFactory().globalIndexFileFactory(),
table.coreOptions(),
rowRange,
indexFields,
@@ -2429,7 +2424,6 @@ private void buildAndCommitBTreeIndex(FileStoreTable table, int[] ids, Range row
List indexFiles =
GlobalIndexBuilderUtils.toIndexFileMetas(
table.fileIO(),
- table.store().pathFactory().globalIndexFileFactory(),
table.coreOptions(),
rowRange,
idField.id(),
@@ -2470,7 +2464,6 @@ private void buildAndCommitPartitionedIndex(
List indexFiles =
GlobalIndexBuilderUtils.toIndexFileMetas(
table.fileIO(),
- table.store().pathFactory().globalIndexFileFactory(),
table.coreOptions(),
rowRange,
vectorField.id(),
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchRowFilterExactnessTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchRowFilterExactnessTest.java
index 48d525287600..ec2d5a05126b 100644
--- a/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchRowFilterExactnessTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchRowFilterExactnessTest.java
@@ -522,7 +522,6 @@ public void testPartialCompositeKeepsScalarFilterCoverage(String mode, boolean i
table,
GlobalIndexBuilderUtils.toIndexFileMetas(
table.fileIO(),
- table.store().pathFactory().globalIndexFileFactory(),
table.coreOptions(),
new Range(0, 1),
fields,
@@ -884,7 +883,6 @@ private void buildAndCommitVectorIndex(FileStoreTable table, float[][] vectors,
table,
GlobalIndexBuilderUtils.toIndexFileMetas(
table.fileIO(),
- table.store().pathFactory().globalIndexFileFactory(),
table.coreOptions(),
rowRange,
vectorField.id(),
@@ -921,7 +919,6 @@ private void buildAndCommitNameBTreeIndex(FileStoreTable table, String[] names,
table,
GlobalIndexBuilderUtils.toIndexFileMetas(
table.fileIO(),
- table.store().pathFactory().globalIndexFileFactory(),
table.coreOptions(),
rowRange,
nameField.id(),
@@ -950,7 +947,6 @@ private void buildAndCommitIdBTreeIndex(FileStoreTable table, Range rowRange) th
table,
GlobalIndexBuilderUtils.toIndexFileMetas(
table.fileIO(),
- table.store().pathFactory().globalIndexFileFactory(),
table.coreOptions(),
rowRange,
idField.id(),
diff --git a/paimon-core/src/test/java/org/apache/paimon/utils/FileStorePathFactoryTest.java b/paimon-core/src/test/java/org/apache/paimon/utils/FileStorePathFactoryTest.java
index 4f067163fc6b..994088152042 100644
--- a/paimon-core/src/test/java/org/apache/paimon/utils/FileStorePathFactoryTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/utils/FileStorePathFactoryTest.java
@@ -23,6 +23,7 @@
import org.apache.paimon.data.BinaryRowWriter;
import org.apache.paimon.data.BinaryString;
import org.apache.paimon.fs.Path;
+import org.apache.paimon.index.IndexPathFactory;
import org.apache.paimon.io.DataFilePathFactory;
import org.apache.paimon.types.DataType;
import org.apache.paimon.types.IntType;
@@ -32,6 +33,16 @@
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
import static org.apache.paimon.CoreOptions.PARTITION_DEFAULT_NAME;
import static org.assertj.core.api.Assertions.assertThat;
@@ -40,6 +51,86 @@ public class FileStorePathFactoryTest {
@TempDir java.nio.file.Path tempDir;
+ @Test
+ public void testGlobalIndexExternalPaths() {
+ Path root = new Path(tempDir.toUri());
+ List roots =
+ Arrays.asList(new Path(root, "external-a"), new Path(root, "external-b"));
+ FileStorePathFactory factory = createNonPartFactory(root, roots);
+ IndexPathFactory indexes = factory.globalIndexFileFactory();
+ assertThat(indexes.isExternalPath()).isTrue();
+ List parents = new ArrayList<>();
+ for (int i = 0; i < 6; i++) {
+ Path path = indexes.newPath();
+ parents.add(path.getParent());
+ assertThat(path.getName()).isEqualTo("index-" + factory.uuid() + "-" + i);
+ assertThat(indexes.toPath(path.getName(), path.toString())).isEqualTo(path);
+ assertThat(indexes.toPath(path.getName(), path.toString())).isEqualTo(path);
+ }
+ for (Path externalRoot : roots) {
+ assertThat(Collections.frequency(parents, externalRoot)).isEqualTo(3);
+ }
+ assertThat(indexes.toPath("old-index", null)).isEqualTo(new Path(root, "index/old-index"));
+ assertThat(indexes.toPath("old-index", "file:///old-root/old-index"))
+ .isEqualTo(new Path("file:///old-root/old-index"));
+ assertThat(factory.indexFileFactory(BinaryRow.EMPTY_ROW, 0).isExternalPath()).isTrue();
+ Path namedPath = indexes.newPathFromName("btree-global-index-custom.index");
+ assertThat(namedPath.getName()).isEqualTo("btree-global-index-custom.index");
+ assertThat(namedPath.getParent()).isEqualTo(parents.get(0));
+ }
+
+ @Test
+ public void testConcurrentGlobalIndexPathAllocation() throws Exception {
+ Path root = new Path(tempDir.toUri());
+ List roots =
+ Arrays.asList(new Path(root, "external-a"), new Path(root, "external-b"));
+ IndexPathFactory indexes = createNonPartFactory(root, roots).globalIndexFileFactory();
+ int parallelism = 8;
+ int filesPerThread = 1000;
+ ExecutorService executor = Executors.newFixedThreadPool(parallelism);
+ CountDownLatch start = new CountDownLatch(1);
+ List>> futures = new ArrayList<>();
+ try {
+ for (int thread = 0; thread < parallelism; thread++) {
+ int threadId = thread;
+ futures.add(
+ executor.submit(
+ () -> {
+ start.await();
+ List paths = new ArrayList<>();
+ for (int i = 0; i < filesPerThread; i++) {
+ paths.add(
+ i % 2 == 0
+ ? indexes.newPath()
+ : indexes.newPathFromName(
+ "btree-global-index-"
+ + threadId
+ + "-"
+ + i
+ + ".index"));
+ }
+ return paths;
+ }));
+ }
+ start.countDown();
+ List paths = new ArrayList<>();
+ for (Future> future : futures) {
+ paths.addAll(future.get(30, TimeUnit.SECONDS));
+ }
+ assertThat(paths).hasSize(parallelism * filesPerThread).doesNotHaveDuplicates();
+ assertThat(paths).extracting(Path::getName).doesNotHaveDuplicates();
+ for (Path externalRoot : roots) {
+ assertThat(
+ paths.stream()
+ .filter(path -> path.getParent().equals(externalRoot))
+ .count())
+ .isEqualTo(parallelism * filesPerThread / roots.size());
+ }
+ } finally {
+ executor.shutdownNow();
+ }
+ }
+
@Test
public void testManifestPaths() {
FileStorePathFactory pathFactory = createNonPartFactory(new Path(tempDir.toString()));
@@ -129,6 +220,11 @@ private void assertPartition(
}
public static FileStorePathFactory createNonPartFactory(Path root) {
+ return createNonPartFactory(root, Collections.emptyList());
+ }
+
+ public static FileStorePathFactory createNonPartFactory(
+ Path root, List globalIndexPaths) {
return new FileStorePathFactory(
root,
RowType.builder().build(),
@@ -144,6 +240,6 @@ public static FileStorePathFactory createNonPartFactory(Path root) {
CoreOptions.ExternalPathStrategy.NONE,
null,
false,
- null);
+ globalIndexPaths);
}
}
diff --git a/paimon-eslib/src/main/java/org/apache/paimon/eslib/index/ESIndexGlobalIndexWriter.java b/paimon-eslib/src/main/java/org/apache/paimon/eslib/index/ESIndexGlobalIndexWriter.java
index 1de6847e437c..0e807f56e65b 100644
--- a/paimon-eslib/src/main/java/org/apache/paimon/eslib/index/ESIndexGlobalIndexWriter.java
+++ b/paimon-eslib/src/main/java/org/apache/paimon/eslib/index/ESIndexGlobalIndexWriter.java
@@ -464,17 +464,17 @@ public List finish() {
byte[] meta = buildMeta(segFiles);
- String fileName = fileWriter.newFileName(FILE_NAME_PREFIX);
+ String filePath = fileWriter.newFilePath(FILE_NAME_PREFIX);
// Stream the archive directly to the output stream — DO NOT materialise the entire
// archive in memory. For real DiskBBQ output (1M × 768-dim vectors ≈ 3 GB per shard)
// a ByteArrayOutputStream-backed approach hits both heap and DirectByteBuffer limits
// (java.nio.Bits.reserveMemory OOM at Files.readAllBytes).
- try (PositionOutputStream out = fileWriter.newOutputStream(fileName)) {
+ try (PositionOutputStream out = fileWriter.newOutputStream(filePath)) {
packDirectoryStream(segFiles, out);
out.flush();
}
- return Collections.singletonList(new ResultEntry(fileName, docCount, meta));
+ return Collections.singletonList(new ResultEntry(filePath, docCount, meta));
} catch (IOException e) {
RuntimeException wrapped = new RuntimeException("Failed to finish ES index build", e);
failure = wrapped;
diff --git a/paimon-eslib/src/test/java/org/apache/paimon/eslib/index/ESIndexGlobalIndexE2ETest.java b/paimon-eslib/src/test/java/org/apache/paimon/eslib/index/ESIndexGlobalIndexE2ETest.java
index 464d1ac8a47e..a02153ae4f69 100644
--- a/paimon-eslib/src/test/java/org/apache/paimon/eslib/index/ESIndexGlobalIndexE2ETest.java
+++ b/paimon-eslib/src/test/java/org/apache/paimon/eslib/index/ESIndexGlobalIndexE2ETest.java
@@ -49,6 +49,7 @@
import java.io.IOException;
import java.nio.file.Files;
+import java.nio.file.Paths;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
@@ -94,14 +95,15 @@ private static final class LocalDirWriter implements GlobalIndexFileWriter {
}
@Override
- public String newFileName(String prefix) {
- return prefix + "-" + UUID.randomUUID() + ".index";
+ public String newFilePath(String prefix) {
+ return new org.apache.paimon.fs.Path(
+ dir.resolve(prefix + "-" + UUID.randomUUID() + ".index").toString())
+ .toString();
}
@Override
- public PositionOutputStream newOutputStream(String fileName) throws IOException {
- return fio.newOutputStream(
- new org.apache.paimon.fs.Path(dir.resolve(fileName).toString()), true);
+ public PositionOutputStream newOutputStream(String filePath) throws IOException {
+ return fio.newOutputStream(new org.apache.paimon.fs.Path(filePath), true);
}
}
@@ -323,9 +325,8 @@ void writeArchiveThenReadAndSearch(@TempDir java.nio.file.Path tmp) throws IOExc
assertThrows(IllegalStateException.class, () -> writer.write(new float[] {1, 1, 1, 1}, 20));
// --- reconstruct reader over the archive ---
- org.apache.paimon.fs.Path filePath =
- new org.apache.paimon.fs.Path(archiveDir.resolve(entry.fileName()).toString());
- long fileSize = Files.size(archiveDir.resolve(entry.fileName()));
+ org.apache.paimon.fs.Path filePath = new org.apache.paimon.fs.Path(entry.filePath());
+ long fileSize = Files.size(Paths.get(entry.filePath()));
GlobalIndexIOMeta ioMeta = new GlobalIndexIOMeta(filePath, fileSize, entry.meta());
ESIndexGlobalIndexReader reader =
@@ -545,10 +546,7 @@ void readsIndexAfterIndexedColumnsAreRenamed(@TempDir java.nio.file.Path tmp)
new DataField(3, "cost", DataTypes.INT()));
ESIndexGlobalIndexReader reader =
new ESIndexGlobalIndexReader(
- new LocalFileReader(),
- List.of(ioMeta(archiveDir, entry)),
- currentFields,
- buildOptions);
+ new LocalFileReader(), List.of(ioMeta(entry)), currentFields, buildOptions);
try {
Optional vector =
reader.visitVectorSearch(new VectorSearch(new float[] {0, 0}, 1, "vector"))
@@ -584,10 +582,7 @@ void readsIndexAfterIndexedColumnsAreRenamed(@TempDir java.nio.file.Path tmp)
new DataField(3, "cost", DataTypes.BIGINT()));
ESIndexGlobalIndexReader widenedReader =
new ESIndexGlobalIndexReader(
- new LocalFileReader(),
- List.of(ioMeta(archiveDir, entry)),
- widenedFields,
- buildOptions);
+ new LocalFileReader(), List.of(ioMeta(entry)), widenedFields, buildOptions);
try {
Optional widenedFallback =
widenedReader
@@ -612,7 +607,7 @@ void readsIndexAfterIndexedColumnsAreRenamed(@TempDir java.nio.file.Path tmp)
ESIndexGlobalIndexReader resizedVectorReader =
new ESIndexGlobalIndexReader(
new LocalFileReader(),
- List.of(ioMeta(archiveDir, entry)),
+ List.of(ioMeta(entry)),
resizedVectorFields,
buildOptions);
try {
@@ -659,13 +654,13 @@ void mixedSchemaShardsDoNotDropRowsFromAnOlderScalarType(@TempDir java.nio.file.
ESIndexGlobalIndexReader oldReader =
new ESIndexGlobalIndexReader(
new LocalFileReader(),
- List.of(ioMeta(oldDir, oldEntry)),
+ List.of(ioMeta(oldEntry)),
currentFields,
currentOptions);
ESIndexGlobalIndexReader currentReader =
new ESIndexGlobalIndexReader(
new LocalFileReader(),
- List.of(ioMeta(currentDir, currentEntry)),
+ List.of(ioMeta(currentEntry)),
currentFields,
currentOptions);
UnionGlobalIndexReader union =
@@ -710,8 +705,8 @@ void textMultiFieldServesBothMatchAndExact(@TempDir java.nio.file.Path tmp) thro
ResultEntry entry = w.finish().get(0);
GlobalIndexIOMeta ioMeta =
new GlobalIndexIOMeta(
- new org.apache.paimon.fs.Path(dir.resolve(entry.fileName()).toString()),
- Files.size(dir.resolve(entry.fileName())),
+ new org.apache.paimon.fs.Path(entry.filePath()),
+ Files.size(Paths.get(entry.filePath())),
entry.meta());
ESIndexGlobalIndexReader reader =
new ESIndexGlobalIndexReader(
@@ -754,8 +749,8 @@ void fullTextSearchHonorsIncludeRowIdsBeforeTopK(@TempDir java.nio.file.Path tmp
ResultEntry entry = writer.finish().get(0);
GlobalIndexIOMeta ioMeta =
new GlobalIndexIOMeta(
- new org.apache.paimon.fs.Path(dir.resolve(entry.fileName()).toString()),
- Files.size(dir.resolve(entry.fileName())),
+ new org.apache.paimon.fs.Path(entry.filePath()),
+ Files.size(Paths.get(entry.filePath())),
entry.meta());
ESIndexGlobalIndexReader reader =
@@ -802,8 +797,8 @@ void nullLiteralsFollowSqlThreeValuedLogicAndKeywordRangeFallsBack(
ResultEntry entry = writer.finish().get(0);
GlobalIndexIOMeta ioMeta =
new GlobalIndexIOMeta(
- new org.apache.paimon.fs.Path(dir.resolve(entry.fileName()).toString()),
- Files.size(dir.resolve(entry.fileName())),
+ new org.apache.paimon.fs.Path(entry.filePath()),
+ Files.size(Paths.get(entry.filePath())),
entry.meta());
ESIndexGlobalIndexReader reader =
@@ -876,9 +871,8 @@ void arrayLongLabelsRoundTripAsMultiValueScalar(@TempDir java.nio.file.Path tmp)
ResultEntry entry = entries.get(0);
assertEquals(3L, entry.rowCount(), "row count");
- org.apache.paimon.fs.Path filePath =
- new org.apache.paimon.fs.Path(archiveDir.resolve(entry.fileName()).toString());
- long fileSize = Files.size(archiveDir.resolve(entry.fileName()));
+ org.apache.paimon.fs.Path filePath = new org.apache.paimon.fs.Path(entry.filePath());
+ long fileSize = Files.size(Paths.get(entry.filePath()));
GlobalIndexIOMeta ioMeta = new GlobalIndexIOMeta(filePath, fileSize, entry.meta());
ESIndexGlobalIndexReader reader =
new ESIndexGlobalIndexReader(
@@ -936,9 +930,8 @@ void nullValuesKeepDocIdRowIdAligned(@TempDir java.nio.file.Path tmp) throws IOE
ResultEntry entry = entries.get(0);
assertEquals((long) n, entry.rowCount(), "row count includes the null row");
- org.apache.paimon.fs.Path filePath =
- new org.apache.paimon.fs.Path(archiveDir.resolve(entry.fileName()).toString());
- long fileSize = Files.size(archiveDir.resolve(entry.fileName()));
+ org.apache.paimon.fs.Path filePath = new org.apache.paimon.fs.Path(entry.filePath());
+ long fileSize = Files.size(Paths.get(entry.filePath()));
GlobalIndexIOMeta ioMeta = new GlobalIndexIOMeta(filePath, fileSize, entry.meta());
ESIndexGlobalIndexReader reader =
new ESIndexGlobalIndexReader(
@@ -980,7 +973,7 @@ void skippedRowIdsAreCountedAndRemainSearchableAsNull(@TempDir java.nio.file.Pat
ESIndexGlobalIndexReader reader =
new ESIndexGlobalIndexReader(
- new LocalFileReader(), List.of(ioMeta(archiveDir, entry)), fields, options);
+ new LocalFileReader(), List.of(ioMeta(entry)), fields, options);
try {
Optional nullRows =
reader.visitIsNull(new FieldRef(0, "k", DataTypes.STRING())).join();
@@ -1019,9 +1012,8 @@ void likeHonorsSqlEscapeAndLuceneLiterals(@TempDir java.nio.file.Path tmp) throw
List entries = writer.finish();
ResultEntry entry = entries.get(0);
- org.apache.paimon.fs.Path filePath =
- new org.apache.paimon.fs.Path(archiveDir.resolve(entry.fileName()).toString());
- long fileSize = Files.size(archiveDir.resolve(entry.fileName()));
+ org.apache.paimon.fs.Path filePath = new org.apache.paimon.fs.Path(entry.filePath());
+ long fileSize = Files.size(Paths.get(entry.filePath()));
GlobalIndexIOMeta ioMeta = new GlobalIndexIOMeta(filePath, fileSize, entry.meta());
ESIndexGlobalIndexReader reader =
new ESIndexGlobalIndexReader(
@@ -1091,11 +1083,10 @@ void closeWaitsForInFlightLoadAndClosesEveryStream(@TempDir java.nio.file.Path t
writer.write(BinaryString.fromString("admin_001"), 0);
ResultEntry entry = writer.finish().get(0);
- org.apache.paimon.fs.Path filePath =
- new org.apache.paimon.fs.Path(archiveDir.resolve(entry.fileName()).toString());
+ org.apache.paimon.fs.Path filePath = new org.apache.paimon.fs.Path(entry.filePath());
GlobalIndexIOMeta ioMeta =
new GlobalIndexIOMeta(
- filePath, Files.size(archiveDir.resolve(entry.fileName())), entry.meta());
+ filePath, Files.size(Paths.get(entry.filePath())), entry.meta());
BlockingCountingFileReader fileReader = new BlockingCountingFileReader();
ExecutorService queryExecutor = Executors.newSingleThreadExecutor();
ExecutorService closeExecutor = Executors.newSingleThreadExecutor();
@@ -1153,7 +1144,7 @@ void failedLazyLoadClosesStreamsBeforeRetry(@TempDir java.nio.file.Path tmp) thr
new ESIndexGlobalIndexWriter(new LocalDirWriter(dir), fields, options);
writer.write(BinaryString.fromString("x"), 0);
ResultEntry entry = writer.finish().get(0);
- java.nio.file.Path badArchive = dir.resolve(entry.fileName());
+ java.nio.file.Path badArchive = Paths.get(entry.filePath());
long declaredArchiveSize = Files.size(badArchive);
// Keep the valid offset metadata, but truncate the archive so failure happens after the
// root provider (and potentially Lucene forks) have actually been opened.
@@ -1215,9 +1206,8 @@ void isNullReturnsNullRowsViaIndex(@TempDir java.nio.file.Path tmp) throws IOExc
List entries = writer.finish();
ResultEntry entry = entries.get(0);
- org.apache.paimon.fs.Path filePath =
- new org.apache.paimon.fs.Path(archiveDir.resolve(entry.fileName()).toString());
- long fileSize = Files.size(archiveDir.resolve(entry.fileName()));
+ org.apache.paimon.fs.Path filePath = new org.apache.paimon.fs.Path(entry.filePath());
+ long fileSize = Files.size(Paths.get(entry.filePath()));
GlobalIndexIOMeta ioMeta = new GlobalIndexIOMeta(filePath, fileSize, entry.meta());
ESIndexGlobalIndexReader reader =
new ESIndexGlobalIndexReader(
@@ -1293,11 +1283,10 @@ void exclusiveNumericExtremaDoNotOverflow(@TempDir java.nio.file.Path tmp) throw
Double.POSITIVE_INFINITY));
ResultEntry entry = writer.finish().get(0);
- org.apache.paimon.fs.Path filePath =
- new org.apache.paimon.fs.Path(archiveDir.resolve(entry.fileName()).toString());
+ org.apache.paimon.fs.Path filePath = new org.apache.paimon.fs.Path(entry.filePath());
GlobalIndexIOMeta ioMeta =
new GlobalIndexIOMeta(
- filePath, Files.size(archiveDir.resolve(entry.fileName())), entry.meta());
+ filePath, Files.size(Paths.get(entry.filePath())), entry.meta());
ESIndexGlobalIndexReader reader =
new ESIndexGlobalIndexReader(
new LocalFileReader(), Arrays.asList(ioMeta), fields, options);
@@ -1365,11 +1354,10 @@ void temporalPredicatesUseLongScalarPath(@TempDir java.nio.file.Path tmp) throws
writer.write(3, GenericRow.of(null, null));
ResultEntry entry = writer.finish().get(0);
- org.apache.paimon.fs.Path filePath =
- new org.apache.paimon.fs.Path(archiveDir.resolve(entry.fileName()).toString());
+ org.apache.paimon.fs.Path filePath = new org.apache.paimon.fs.Path(entry.filePath());
GlobalIndexIOMeta ioMeta =
new GlobalIndexIOMeta(
- filePath, Files.size(archiveDir.resolve(entry.fileName())), entry.meta());
+ filePath, Files.size(Paths.get(entry.filePath())), entry.meta());
ESIndexGlobalIndexReader reader =
new ESIndexGlobalIndexReader(
new LocalFileReader(), List.of(ioMeta), fields, options);
@@ -1443,7 +1431,7 @@ void buildTimeFieldConfigurationIsReadFromIndexMetadata(@TempDir java.nio.file.P
writer.write(BinaryString.fromString("viewer"), 1);
ResultEntry entry = writer.finish().get(0);
- GlobalIndexIOMeta ioMeta = ioMeta(archiveDir, entry);
+ GlobalIndexIOMeta ioMeta = ioMeta(entry);
// Deliberately use the current/default FULLTEXT options. The reader must instead use the
// KEYWORD-primary layout persisted when this particular index was built.
ESIndexOptions currentTableOptions = new ESIndexOptions(fields, new Options());
@@ -1484,7 +1472,7 @@ void tinySmallAndEmptyArrayNullSemanticsAreIndexed(@TempDir java.nio.file.Path t
ESIndexGlobalIndexReader reader =
new ESIndexGlobalIndexReader(
- new LocalFileReader(), List.of(ioMeta(archiveDir, entry)), fields, options);
+ new LocalFileReader(), List.of(ioMeta(entry)), fields, options);
try {
Optional tiny =
reader.visitEqual(new FieldRef(0, "tiny", DataTypes.TINYINT()), (byte) 1)
@@ -1538,8 +1526,7 @@ void closeCanRetryTransientStreamFailures(@TempDir java.nio.file.Path tmp) throw
TransientCloseFailureFileReader fileReader = new TransientCloseFailureFileReader();
ESIndexGlobalIndexReader reader =
- new ESIndexGlobalIndexReader(
- fileReader, List.of(ioMeta(archiveDir, entry)), fields, options);
+ new ESIndexGlobalIndexReader(fileReader, List.of(ioMeta(entry)), fields, options);
assertTrue(reader.visitEqual(new FieldRef(0, "id", DataTypes.INT()), 1).join().isPresent());
fileReader.setCloseFailures(true);
@@ -1564,7 +1551,7 @@ void highPrecisionTimestampPredicatesFallBackToRawScan(@TempDir java.nio.file.Pa
ESIndexGlobalIndexReader reader =
new ESIndexGlobalIndexReader(
- new LocalFileReader(), List.of(ioMeta(archiveDir, entry)), fields, options);
+ new LocalFileReader(), List.of(ioMeta(entry)), fields, options);
try {
FieldRef ts = new FieldRef(0, "ts", DataTypes.TIMESTAMP(6));
Optional fallback =
@@ -1596,7 +1583,7 @@ void standardBooleanDslAndCamelCaseFuzzyParamsAreApplied(@TempDir java.nio.file.
ESIndexGlobalIndexReader reader =
new ESIndexGlobalIndexReader(
- new LocalFileReader(), List.of(ioMeta(archiveDir, entry)), fields, options);
+ new LocalFileReader(), List.of(ioMeta(entry)), fields, options);
String query =
"{\"boolean\":{\"must\":[{\"match\":{\"query\":\"administrater\","
+ "\"fuzziness\":\"AUTO\",\"maxExpansions\":20,\"prefixLength\":0}}],"
@@ -1628,11 +1615,10 @@ void standardBooleanDslAndCamelCaseFuzzyParamsAreApplied(@TempDir java.nio.file.
}
}
- private static GlobalIndexIOMeta ioMeta(java.nio.file.Path archiveDir, ResultEntry entry)
- throws IOException {
+ private static GlobalIndexIOMeta ioMeta(ResultEntry entry) throws IOException {
return new GlobalIndexIOMeta(
- new org.apache.paimon.fs.Path(archiveDir.resolve(entry.fileName()).toString()),
- Files.size(archiveDir.resolve(entry.fileName())),
+ new org.apache.paimon.fs.Path(entry.filePath()),
+ Files.size(Paths.get(entry.filePath())),
entry.meta());
}
@@ -1656,7 +1642,7 @@ void vectorSearchReturnsPaimonCosineScoreScale(@TempDir java.nio.file.Path tmp)
ESIndexGlobalIndexReader reader =
new ESIndexGlobalIndexReader(
- new LocalFileReader(), List.of(ioMeta(archiveDir, entry)), fields, options);
+ new LocalFileReader(), List.of(ioMeta(entry)), fields, options);
try {
ScoredGlobalIndexResult result =
reader.visitVectorSearch(
@@ -1756,9 +1742,8 @@ void writeDiskBBQArchiveThenReadAndSearch(@TempDir java.nio.file.Path tmp) throw
"archive must not contain HNSW-specific files (.vex/.vem), got: " + exts);
// --- reconstruct reader over the archive ---
- org.apache.paimon.fs.Path filePath =
- new org.apache.paimon.fs.Path(archiveDir.resolve(entry.fileName()).toString());
- long fileSize = Files.size(archiveDir.resolve(entry.fileName()));
+ org.apache.paimon.fs.Path filePath = new org.apache.paimon.fs.Path(entry.filePath());
+ long fileSize = Files.size(Paths.get(entry.filePath()));
GlobalIndexIOMeta ioMeta = new GlobalIndexIOMeta(filePath, fileSize, entry.meta());
BlockingCountingFileReader fileReader = new BlockingCountingFileReader();
diff --git a/paimon-eslib/src/test/java/org/apache/paimon/eslib/index/ESIndexVectorMetricTest.java b/paimon-eslib/src/test/java/org/apache/paimon/eslib/index/ESIndexVectorMetricTest.java
index 4f92ab06fb0c..23a472d3366a 100644
--- a/paimon-eslib/src/test/java/org/apache/paimon/eslib/index/ESIndexVectorMetricTest.java
+++ b/paimon-eslib/src/test/java/org/apache/paimon/eslib/index/ESIndexVectorMetricTest.java
@@ -43,6 +43,7 @@
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
@@ -167,7 +168,7 @@ void readerDoesNotRequireCurrentOptionsToDescribeBuild(@TempDir Path tempDir) th
writer.write(new float[] {0f, 1f}, 1);
writer.write(new float[] {-1f, 0f}, 2);
ResultEntry entry = writer.finish().get(0);
- Path archive = archiveDir.resolve(entry.fileName());
+ Path archive = Paths.get(entry.filePath());
GlobalIndexIOMeta ioMeta =
new GlobalIndexIOMeta(
new org.apache.paimon.fs.Path(archive.toString()),
@@ -278,14 +279,15 @@ private static final class LocalDirWriter implements GlobalIndexFileWriter {
}
@Override
- public String newFileName(String prefix) {
- return prefix + "-" + UUID.randomUUID() + ".index";
+ public String newFilePath(String prefix) {
+ return new org.apache.paimon.fs.Path(
+ dir.resolve(prefix + "-" + UUID.randomUUID() + ".index").toString())
+ .toString();
}
@Override
- public PositionOutputStream newOutputStream(String fileName) throws IOException {
- return fio.newOutputStream(
- new org.apache.paimon.fs.Path(dir.resolve(fileName).toString()), true);
+ public PositionOutputStream newOutputStream(String filePath) throws IOException {
+ return fio.newOutputStream(new org.apache.paimon.fs.Path(filePath), true);
}
}
}
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java
index 352bc76942d8..42c07a1fa0b7 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java
@@ -19,6 +19,7 @@
package org.apache.paimon.flink.globalindex;
import org.apache.paimon.CoreOptions;
+import org.apache.paimon.annotation.VisibleForTesting;
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.flink.sink.Committable;
@@ -29,6 +30,7 @@
import org.apache.paimon.flink.utils.BoundedOneInputOperator;
import org.apache.paimon.flink.utils.JavaTypeInfo;
import org.apache.paimon.flink.utils.StreamExecutionEnvironmentUtils;
+import org.apache.paimon.globalindex.GlobalIndexFileReadWrite;
import org.apache.paimon.globalindex.GlobalIndexMultiColumnWriter;
import org.apache.paimon.globalindex.GlobalIndexSingleColumnWriter;
import org.apache.paimon.globalindex.GlobalIndexWriter;
@@ -75,6 +77,7 @@
import java.util.function.Supplier;
import java.util.stream.Collectors;
+import static org.apache.paimon.globalindex.GlobalIndexBuilderUtils.createGlobalIndexFileReadWrite;
import static org.apache.paimon.globalindex.GlobalIndexBuilderUtils.createIndexWriter;
import static org.apache.paimon.globalindex.GlobalIndexBuilderUtils.createShardIndexedSplits;
import static org.apache.paimon.globalindex.GlobalIndexBuilderUtils.toIndexFileMetas;
@@ -355,8 +358,8 @@ private static void commit(
* and emits a {@link Committable}. Each shard's split contains only the files relevant to this
* shard, so no redundant I/O occurs.
*/
- private static class BuildIndexOperator
- extends BoundedOneInputOperator {
+ @VisibleForTesting
+ static class BuildIndexOperator extends BoundedOneInputOperator {
private static final long serialVersionUID = 1L;
@@ -370,6 +373,7 @@ private static class BuildIndexOperator
private final byte[] sourceMeta;
private transient TableRead tableRead;
+ private transient GlobalIndexFileReadWrite fileReadWrite;
private transient List indexedFields;
private transient InternalRow.FieldGetter[] indexFieldGetters;
private transient int rowIdFieldIndex;
@@ -399,6 +403,7 @@ private static class BuildIndexOperator
public void open() throws Exception {
super.open();
this.tableRead = readBuilder.newRead();
+ this.fileReadWrite = createGlobalIndexFileReadWrite(table);
// The primary column followed by the extra columns, in index order. Field getters and
// the writer projection both need the full ordered list.
this.indexedFields = new ArrayList<>(1 + extraFields.size());
@@ -441,7 +446,7 @@ public void processElement(StreamRecord element) throws Exception
long startTime = System.currentTimeMillis();
GlobalIndexWriter indexWriter =
- createIndexWriter(table, indexType, indexedFields, mergedOptions);
+ createIndexWriter(fileReadWrite, indexType, indexedFields, mergedOptions);
try {
long rowsSeen = 0;
@@ -542,7 +547,6 @@ private static CommitMessage flushIndex(
List indexFileMetas =
toIndexFileMetas(
table.fileIO(),
- table.store().pathFactory().globalIndexFileFactory(),
table.coreOptions(),
rowRange,
indexFields,
diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilderTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilderTest.java
index 343c74419440..d5d7816384ff 100644
--- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilderTest.java
+++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilderTest.java
@@ -18,37 +18,60 @@
package org.apache.paimon.flink.globalindex;
+import org.apache.paimon.CoreOptions;
import org.apache.paimon.FileStore;
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.data.BinaryRowWriter;
import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.flink.sink.Committable;
+import org.apache.paimon.flink.sink.CommittableTypeInfo;
import org.apache.paimon.fs.Path;
+import org.apache.paimon.fs.local.LocalFileIO;
import org.apache.paimon.globalindex.IndexedSplit;
import org.apache.paimon.globalindex.generic.GenericGlobalIndexScanner;
+import org.apache.paimon.index.IndexFileMeta;
import org.apache.paimon.io.PojoDataFileMeta;
import org.apache.paimon.manifest.FileKind;
import org.apache.paimon.manifest.ManifestEntry;
import org.apache.paimon.options.Options;
import org.apache.paimon.stats.SimpleStats;
import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.SpecialFields;
+import org.apache.paimon.table.sink.CommitMessageImpl;
+import org.apache.paimon.table.source.ReadBuilder;
+import org.apache.paimon.table.source.TableRead;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.DataTypes;
+import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.FileStorePathFactory;
+import org.apache.paimon.utils.IteratorRecordReader;
import org.apache.paimon.utils.Range;
+import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
+import static org.apache.paimon.utils.FileStorePathFactoryTest.createNonPartFactory;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/** Tests for shard computation in {@link GenericIndexTopoBuilder}. */
@@ -233,6 +256,78 @@ void testShardRangeClampedToFileRange() throws IOException {
assertThat(tasks.get(1).rowRanges().get(0)).isEqualTo(new Range(100, 149));
}
+ @Test
+ void testRoundRobinAcrossShards(@TempDir java.nio.file.Path directory) throws Exception {
+ Path root = new Path(directory.toUri());
+ List roots = Arrays.asList(new Path(root, "a"), new Path(root, "b"));
+ CoreOptions options =
+ new CoreOptions(
+ Collections.singletonMap(
+ CoreOptions.GLOBAL_INDEX_EXTERNAL_PATH.key(),
+ roots.get(0) + "," + roots.get(1)));
+ FileStorePathFactory paths = table.store().pathFactory();
+ when(paths.globalIndexFileFactory())
+ .thenAnswer(ignored -> createNonPartFactory(root, roots).globalIndexFileFactory());
+ when(table.fileIO()).thenReturn(new LocalFileIO());
+ when(table.coreOptions()).thenReturn(options);
+ DataField field = new DataField(0, "id", DataTypes.INT());
+ RowType readType =
+ SpecialFields.rowTypeWithRowId(new RowType(Collections.singletonList(field)));
+ ReadBuilder readBuilder = mock(ReadBuilder.class);
+ TableRead read = mock(TableRead.class);
+ when(readBuilder.newRead()).thenReturn(read);
+ List commits;
+ try (OneInputStreamOperatorTestHarness harness =
+ new OneInputStreamOperatorTestHarness<>(
+ new GenericIndexTopoBuilder.BuildIndexOperator(
+ readBuilder,
+ table,
+ "bitmap",
+ field,
+ Collections.emptyList(),
+ readType,
+ options.toConfiguration(),
+ null))) {
+ harness.setup(new CommittableTypeInfo().createSerializer(new ExecutionConfig()));
+ harness.open();
+ for (int i = 0; i < 6; i++) {
+ IndexedSplit split =
+ GenericIndexTopoBuilder.computeShardTasks(
+ table,
+ Collections.singletonList(
+ createEntry(BinaryRow.EMPTY_ROW, (long) i, 1)),
+ 1)
+ .get(0);
+ when(read.createReader(split.dataSplit()))
+ .thenReturn(
+ new IteratorRecordReader<>(
+ Collections.singletonList(
+ GenericRow.of(i, (long) i))
+ .iterator()));
+ harness.processElement(new StreamRecord<>(split));
+ }
+ commits = harness.extractOutputValues();
+ }
+ assertThat(commits).hasSize(6);
+ List parents = new ArrayList<>();
+ for (Committable commit : commits) {
+ List files =
+ ((CommitMessageImpl) commit.commitMessage())
+ .newFilesIncrement()
+ .newIndexFiles();
+ assertThat(files).hasSize(1);
+ Path file = new Path(files.get(0).externalPath());
+ assertThat(table.fileIO().exists(file)).isTrue();
+ assertThat(files.get(0).fileName()).isEqualTo(file.getName());
+ parents.add(file.getParent());
+ }
+ assertThat(parents).containsOnlyElementsOf(roots);
+ for (int i = 1; i < parents.size(); i++) {
+ assertThat(parents.get(i)).isNotEqualTo(parents.get(i - 1));
+ }
+ verify(paths, times(1)).globalIndexFileFactory();
+ }
+
// -- Helpers --
private static ManifestEntry createEntry(BinaryRow partition, Long firstRowId, long rowCount) {
diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/VectorSearchProcedureITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/VectorSearchProcedureITCase.java
index a874a5030fc6..aa0e99738354 100644
--- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/VectorSearchProcedureITCase.java
+++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/VectorSearchProcedureITCase.java
@@ -1159,7 +1159,6 @@ private void buildAndCommitVectorIndex(FileStoreTable table, float[][] vectors,
List indexFiles =
GlobalIndexBuilderUtils.toIndexFileMetas(
table.fileIO(),
- table.store().pathFactory().globalIndexFileFactory(),
table.coreOptions(),
rowRange,
vectorField.id(),
@@ -1200,7 +1199,6 @@ private void buildAndCommitMultiFieldVectorIndex(
List indexFiles =
GlobalIndexBuilderUtils.toIndexFileMetas(
table.fileIO(),
- table.store().pathFactory().globalIndexFileFactory(),
table.coreOptions(),
rowRange,
Arrays.asList(vectorField, idField),
diff --git a/paimon-full-text/src/main/java/org/apache/paimon/fulltext/index/NativeFullTextGlobalIndexWriter.java b/paimon-full-text/src/main/java/org/apache/paimon/fulltext/index/NativeFullTextGlobalIndexWriter.java
index 13f03ba46088..28a53fcbb04b 100644
--- a/paimon-full-text/src/main/java/org/apache/paimon/fulltext/index/NativeFullTextGlobalIndexWriter.java
+++ b/paimon-full-text/src/main/java/org/apache/paimon/fulltext/index/NativeFullTextGlobalIndexWriter.java
@@ -104,8 +104,8 @@ public List finish() {
}
private ResultEntry writeIndex() throws IOException {
- String fileName = fileWriter.newFileName(FILE_NAME_PREFIX);
- try (PositionOutputStream out = fileWriter.newOutputStream(fileName)) {
+ String filePath = fileWriter.newFilePath(FILE_NAME_PREFIX);
+ try (PositionOutputStream out = fileWriter.newOutputStream(filePath)) {
writer.writeIndex(
new FullTextIndexOutput() {
@Override
@@ -121,7 +121,7 @@ public void flush() throws IOException {
});
out.flush();
}
- return new ResultEntry(fileName, rowCount, indexOptions.serialize());
+ return new ResultEntry(filePath, rowCount, indexOptions.serialize());
}
@Override
diff --git a/paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/JavaPyNativeFullTextE2ETest.java b/paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/JavaPyNativeFullTextE2ETest.java
index 3ac0c4a3b5e8..de864a34de48 100644
--- a/paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/JavaPyNativeFullTextE2ETest.java
+++ b/paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/JavaPyNativeFullTextE2ETest.java
@@ -220,7 +220,6 @@ private void writeTableWithNativeFullTextIndex(
List indexFiles =
GlobalIndexBuilderUtils.toIndexFileMetas(
table.fileIO(),
- table.store().pathFactory().globalIndexFileFactory(),
table.coreOptions(),
rowRange,
contentField.id(),
diff --git a/paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/NativeFullTextGlobalIndexTest.java b/paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/NativeFullTextGlobalIndexTest.java
index 7f11936b4b3c..de2df5c4dcea 100644
--- a/paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/NativeFullTextGlobalIndexTest.java
+++ b/paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/NativeFullTextGlobalIndexTest.java
@@ -75,13 +75,13 @@ public void cleanup() throws IOException {
private GlobalIndexFileWriter createFileWriter(Path path) {
return new GlobalIndexFileWriter() {
@Override
- public String newFileName(String prefix) {
- return prefix + "-" + UUID.randomUUID();
+ public String newFilePath(String prefix) {
+ return new Path(path, prefix + "-" + UUID.randomUUID()).toString();
}
@Override
- public PositionOutputStream newOutputStream(String fileName) throws IOException {
- return fileIO.newOutputStream(new Path(path, fileName), false);
+ public PositionOutputStream newOutputStream(String filePath) throws IOException {
+ return fileIO.newOutputStream(new Path(filePath), false);
}
};
}
@@ -90,11 +90,10 @@ private GlobalIndexFileReader createFileReader() {
return meta -> fileIO.newInputStream(meta.filePath());
}
- private List toIOMetas(List results, Path path)
- throws IOException {
+ private List toIOMetas(List results) throws IOException {
assertThat(results).hasSize(1);
ResultEntry result = results.get(0);
- Path filePath = new Path(path, result.fileName());
+ Path filePath = new Path(result.filePath());
return Collections.singletonList(
new GlobalIndexIOMeta(filePath, fileIO.getFileSize(filePath), result.meta()));
}
@@ -128,7 +127,7 @@ public void testEndToEnd() throws IOException {
assertThat(results).hasSize(1);
assertThat(results.get(0).rowCount()).isEqualTo(3);
- List metas = toIOMetas(results, indexPath);
+ List metas = toIOMetas(results);
GlobalIndexFileReader fileReader = createFileReader();
try (NativeFullTextGlobalIndexReader reader = createReader(fileReader, metas)) {
@@ -189,7 +188,7 @@ public void testJiebaTokenizerFindsChineseWord() throws IOException {
List results = writer.finish();
assertThat(serializedOptions(results.get(0).meta())).containsEntry("tokenizer", "jieba");
- List metas = toIOMetas(results, indexPath);
+ List metas = toIOMetas(results);
GlobalIndexFileReader fileReader = createFileReader();
try (NativeFullTextGlobalIndexReader reader = createReader(fileReader, metas)) {
@@ -227,7 +226,7 @@ public void testIcuSplitTokenizerFindsChineseWordsAndIdentifierParts() throws IO
.containsEntry("tokenizer", "icu/split");
try (NativeFullTextGlobalIndexReader reader =
- createReader(createFileReader(), toIOMetas(results, indexPath))) {
+ createReader(createFileReader(), toIOMetas(results))) {
for (String query : new String[] {"中文", "RUST", "world", "beta", "getusername"}) {
Optional searchResult =
reader.visitFullTextSearch(
@@ -255,7 +254,7 @@ public void testSearchNoResults() throws IOException {
writer.write(BinaryString.fromString("Foo bar baz"), 1);
List results = writer.finish();
- List metas = toIOMetas(results, indexPath);
+ List metas = toIOMetas(results);
GlobalIndexFileReader fileReader = createFileReader();
try (NativeFullTextGlobalIndexReader reader = createReader(fileReader, metas)) {
@@ -281,7 +280,7 @@ public void testNullFieldSkipped() throws IOException {
List results = writer.finish();
assertThat(results.get(0).rowCount()).isEqualTo(3);
- List metas = toIOMetas(results, indexPath);
+ List metas = toIOMetas(results);
GlobalIndexFileReader fileReader = createFileReader();
try (NativeFullTextGlobalIndexReader reader = createReader(fileReader, metas)) {
@@ -326,7 +325,7 @@ public void testLargeDataset() throws IOException {
List results = writer.finish();
assertThat(results.get(0).rowCount()).isEqualTo(numDocs);
- List metas = toIOMetas(results, indexPath);
+ List metas = toIOMetas(results);
GlobalIndexFileReader fileReader = createFileReader();
try (NativeFullTextGlobalIndexReader reader = createReader(fileReader, metas)) {
@@ -356,7 +355,7 @@ public void testLimitRespected() throws IOException {
}
List results = writer.finish();
- List metas = toIOMetas(results, indexPath);
+ List metas = toIOMetas(results);
GlobalIndexFileReader fileReader = createFileReader();
try (NativeFullTextGlobalIndexReader reader = createReader(fileReader, metas)) {
@@ -381,7 +380,7 @@ public void testIncludeRowIdsSearchesFullShardBeforeTopK() throws IOException {
writer.write(BinaryString.fromString("paimon"), 2);
List results = writer.finish();
- List metas = toIOMetas(results, indexPath);
+ List metas = toIOMetas(results);
GlobalIndexFileReader fileReader = createFileReader();
RoaringNavigableMap64 includeRowIds = new RoaringNavigableMap64();
@@ -409,7 +408,7 @@ public void testRepeatedReadersReturnConsistentResults() throws IOException {
writer.write(BinaryString.fromString("Native full-text search"), 1);
List results = writer.finish();
- List metas = toIOMetas(results, indexPath);
+ List metas = toIOMetas(results);
GlobalIndexFileReader fileReader = createFileReader();
FullTextSearch search = new FullTextSearch("text", matchQuery("paimon"), 10);
@@ -439,7 +438,7 @@ public void testViaIndexer() throws IOException {
List results = writer.finish();
assertThat(results).hasSize(1);
- List metas = toIOMetas(results, indexPath);
+ List metas = toIOMetas(results);
GlobalIndexFileReader fileReader = createFileReader();
try (NativeFullTextGlobalIndexReader reader =
@@ -464,7 +463,7 @@ public void testStructuredBoostQueryDemotesNegativeMatches() throws IOException
writer.write(BinaryString.fromString("paimon vector"), 1);
List results = writer.finish();
- List metas = toIOMetas(results, indexPath);
+ List metas = toIOMetas(results);
GlobalIndexFileReader fileReader = createFileReader();
try (NativeFullTextGlobalIndexReader reader = createReader(fileReader, metas)) {
diff --git a/paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/NativeFullTextRowFilterTest.java b/paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/NativeFullTextRowFilterTest.java
index a659b1860a7d..cfbe20e3dd4a 100644
--- a/paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/NativeFullTextRowFilterTest.java
+++ b/paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/NativeFullTextRowFilterTest.java
@@ -459,7 +459,6 @@ private Dataset writeDataset(
new ArrayList<>(
GlobalIndexBuilderUtils.toIndexFileMetas(
table.fileIO(),
- table.store().pathFactory().globalIndexFileFactory(),
table.coreOptions(),
new Range(0, fullTextIndexedRows - 1),
contentField.id(),
@@ -491,7 +490,6 @@ private Dataset writeDataset(
indexFiles.addAll(
GlobalIndexBuilderUtils.toIndexFileMetas(
table.fileIO(),
- table.store().pathFactory().globalIndexFileFactory(),
table.coreOptions(),
rowRange,
categoryField.id(),
diff --git a/paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/NativePrimaryKeyFullTextIndexTest.java b/paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/NativePrimaryKeyFullTextIndexTest.java
index db0c26b7cc62..5f1656964ab5 100644
--- a/paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/NativePrimaryKeyFullTextIndexTest.java
+++ b/paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/NativePrimaryKeyFullTextIndexTest.java
@@ -180,10 +180,10 @@ private IndexFileMeta buildArchive(List texts, Options options) th
List results = writer.finish();
assertThat(results).hasSize(1);
ResultEntry result = results.get(0);
- Path path = new Path(indexPath, result.fileName());
+ Path path = new Path(result.filePath());
return new IndexFileMeta(
"full-text",
- result.fileName(),
+ new Path(result.filePath()).getName(),
fileIO.getFileSize(path),
result.rowCount(),
new GlobalIndexMeta(
@@ -201,13 +201,13 @@ private IndexFileMeta buildArchive(List texts, Options options) th
private GlobalIndexFileWriter fileWriter() {
return new GlobalIndexFileWriter() {
@Override
- public String newFileName(String prefix) {
- return prefix + "-" + UUID.randomUUID();
+ public String newFilePath(String prefix) {
+ return new Path(indexPath, prefix + "-" + UUID.randomUUID()).toString();
}
@Override
- public PositionOutputStream newOutputStream(String fileName) throws IOException {
- return fileIO.newOutputStream(new Path(indexPath, fileName), false);
+ public PositionOutputStream newOutputStream(String filePath) throws IOException {
+ return fileIO.newOutputStream(new Path(filePath), false);
}
};
}
diff --git a/paimon-lumina/README.md b/paimon-lumina/README.md
index feef158c3f64..8ac06c305186 100644
--- a/paimon-lumina/README.md
+++ b/paimon-lumina/README.md
@@ -8,6 +8,16 @@ Alibaba Storage Service Team. It is accessed via JNI through the `lumina-jni` ar
**Note:** Lumina native library only supports **x86_64 (AMD64)** architecture. It is not available on ARM (e.g., Apple Silicon, aarch64).
+### Upgrading with Paimon
+
+Deploy `paimon-lumina` and the Paimon engine connector/runtime from the same Paimon
+release. For snapshot or custom builds, build both from the same source revision.
+The Java writer interface for multiple external index paths requires an updated
+Lumina plugin, even when only one external directory is configured or the option
+is unset. Replace the old plugin JAR when upgrading Paimon; updating only
+`lumina-jni` is insufficient. See [Upgrade Java Index Plugins](../docs/docs/multimodal-table/global-index/manage-indexes.mdx#upgrade-java-index-plugins)
+for deployment requirements and interface migration details.
+
### Supported Index Types
| Index Type | Description |
diff --git a/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexWriter.java b/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexWriter.java
index ac52e418d5ad..5eede90d77ec 100644
--- a/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexWriter.java
+++ b/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexWriter.java
@@ -279,8 +279,8 @@ private ResultEntry buildIndex() throws IOException {
phaseStart = System.currentTimeMillis();
LOG.info("Lumina dump phase started");
- String fileName = fileWriter.newFileName(FILE_NAME_PREFIX);
- try (PositionOutputStream out = fileWriter.newOutputStream(fileName)) {
+ String filePath = fileWriter.newFilePath(FILE_NAME_PREFIX);
+ try (PositionOutputStream out = fileWriter.newOutputStream(filePath)) {
index.dump(new OutputStreamFileOutput(out));
out.flush();
}
@@ -292,7 +292,7 @@ private ResultEntry buildIndex() throws IOException {
LuminaIndexMeta meta = new LuminaIndexMeta(luminaOptions);
// rowCount = logical rows including nulls (not just indexed vectors)
- return new ResultEntry(fileName, rowCount, meta.serialize());
+ return new ResultEntry(filePath, rowCount, meta.serialize());
}
}
diff --git a/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/JavaPyLuminaE2ETest.java b/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/JavaPyLuminaE2ETest.java
index 1f14493b282b..eedb01e238d4 100644
--- a/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/JavaPyLuminaE2ETest.java
+++ b/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/JavaPyLuminaE2ETest.java
@@ -189,7 +189,6 @@ public void testLuminaVectorIndexWrite(String fileFormat) throws Exception {
List indexFiles =
GlobalIndexBuilderUtils.toIndexFileMetas(
table.fileIO(),
- table.store().pathFactory().globalIndexFileFactory(),
table.coreOptions(),
rowRange,
embeddingField.id(),
@@ -293,7 +292,6 @@ public void testLuminaVectorWithBTreeIndexWrite() throws Exception {
List vectorIndexFiles =
GlobalIndexBuilderUtils.toIndexFileMetas(
table.fileIO(),
- table.store().pathFactory().globalIndexFileFactory(),
table.coreOptions(),
rowRange,
embeddingField.id(),
@@ -316,7 +314,6 @@ public void testLuminaVectorWithBTreeIndexWrite() throws Exception {
List idIndexFiles =
GlobalIndexBuilderUtils.toIndexFileMetas(
table.fileIO(),
- table.store().pathFactory().globalIndexFileFactory(),
table.coreOptions(),
rowRange,
idField.id(),
diff --git a/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorBenchmark.java b/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorBenchmark.java
index 3511fba4a33a..63c73451937a 100644
--- a/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorBenchmark.java
+++ b/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorBenchmark.java
@@ -359,21 +359,20 @@ public void benchmarkBuild() throws Exception {
buildDir = indexDir;
}
- String indexFileName;
try {
final Path finalBuildDir = buildDir;
GlobalIndexFileWriter gFileWriter =
new GlobalIndexFileWriter() {
@Override
- public String newFileName(String prefix) {
- return prefix + "-" + UUID.randomUUID();
+ public String newFilePath(String prefix) {
+ return new Path(finalBuildDir, prefix + "-" + UUID.randomUUID())
+ .toString();
}
@Override
- public PositionOutputStream newOutputStream(String fileName)
+ public PositionOutputStream newOutputStream(String filePath)
throws IOException {
- return buildFileIO.newOutputStream(
- new Path(finalBuildDir, fileName), false);
+ return buildFileIO.newOutputStream(new Path(filePath), false);
}
};
@@ -404,18 +403,19 @@ public PositionOutputStream newOutputStream(String fileName)
"Index build done in %.2f s%n", (finishEnd - finishStart) / 1000.0);
}
- indexFileName = results.get(0).fileName();
+ Path indexFilePath = new Path(results.get(0).filePath());
// Upload to OSS if needed
if (isRemote) {
- Path localFile = new Path(buildDir, indexFileName);
- Path remoteFile = new Path(indexDir, indexFileName);
+ Path localFile = indexFilePath;
+ Path remoteFile = new Path(indexDir, localFile.getName());
long localSize = localFileIO.getFileSize(localFile);
System.out.printf(
"Uploading to OSS: %s (%,d bytes, %.2f GB)...%n",
remoteFile, localSize, localSize / (1024.0 * 1024 * 1024));
long uploadStart = System.currentTimeMillis();
uploadFile(localFileIO, localFile, fileIO, remoteFile);
+ indexFilePath = remoteFile;
long uploadEnd = System.currentTimeMillis();
double uploadSec = (uploadEnd - uploadStart) / 1000.0;
System.out.printf(
@@ -423,7 +423,6 @@ public PositionOutputStream newOutputStream(String fileName)
uploadSec, localSize / (1024.0 * 1024) / uploadSec);
}
- Path indexFilePath = new Path(indexDir, indexFileName);
long buildFileSize = fileIO.getFileSize(indexFilePath);
long buildEndTime = System.currentTimeMillis();
@@ -517,7 +516,6 @@ public void benchmarkQuery() throws Exception {
List ioMetas = Collections.singletonList(ioMeta);
final FileIO benchFileIO = fileIO;
- final Path benchIndexDir = indexDir;
System.out.printf(
"Running %,d queries (top-%d), each with fresh index load...%n", numQueries, topK);
@@ -540,9 +538,7 @@ public void benchmarkQuery() throws Exception {
long queryStart = System.nanoTime();
GlobalIndexFileReader gFileReader =
- ioMetaArg ->
- benchFileIO.newInputStream(
- new Path(benchIndexDir, ioMetaArg.filePath().getName()));
+ ioMetaArg -> benchFileIO.newInputStream(ioMetaArg.filePath());
ExecutorService executor = Executors.newCachedThreadPool();
try (LuminaVectorGlobalIndexReader reader =
new LuminaVectorGlobalIndexReader(
diff --git a/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexScanTest.java b/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexScanTest.java
index cd7919096555..0ca4e079c415 100644
--- a/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexScanTest.java
+++ b/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexScanTest.java
@@ -199,14 +199,14 @@ public void testVectorIndexScanWithDifferentMetrics() throws Exception {
GlobalIndexFileWriter fileWriter =
new GlobalIndexFileWriter() {
@Override
- public String newFileName(String prefix) {
- return prefix + "-" + UUID.randomUUID();
+ public String newFilePath(String prefix) {
+ return new Path(indexDir, prefix + "-" + UUID.randomUUID()).toString();
}
@Override
- public PositionOutputStream newOutputStream(String fileName)
+ public PositionOutputStream newOutputStream(String filePath)
throws IOException {
- return fileIO.newOutputStream(new Path(indexDir, fileName), false);
+ return fileIO.newOutputStream(new Path(filePath), false);
}
};
@@ -222,14 +222,14 @@ public PositionOutputStream newOutputStream(String fileName)
int fieldId = ipTable.rowType().getFieldIndex(vectorFieldName);
for (ResultEntry entry : entries) {
- long fileSize = fileIO.getFileSize(new Path(indexDir, entry.fileName()));
+ long fileSize = fileIO.getFileSize(new Path(entry.filePath()));
GlobalIndexMeta globalMeta =
new GlobalIndexMeta(0, vectors.length - 1, fieldId, null, entry.meta());
metas.add(
new IndexFileMeta(
LuminaVectorGlobalIndexerFactory.IDENTIFIER,
- entry.fileName(),
+ new Path(entry.filePath()).getName(),
fileSize,
entry.rowCount(),
globalMeta,
@@ -289,14 +289,14 @@ private List buildIndexManually(float[][] vectors) throws Excepti
GlobalIndexFileWriter fileWriter =
new GlobalIndexFileWriter() {
@Override
- public String newFileName(String prefix) {
- return prefix + "-" + UUID.randomUUID();
+ public String newFilePath(String prefix) {
+ return new Path(indexDir, prefix + "-" + UUID.randomUUID()).toString();
}
@Override
- public PositionOutputStream newOutputStream(String fileName)
+ public PositionOutputStream newOutputStream(String filePath)
throws IOException {
- return fileIO.newOutputStream(new Path(indexDir, fileName), false);
+ return fileIO.newOutputStream(new Path(filePath), false);
}
};
@@ -313,14 +313,14 @@ public PositionOutputStream newOutputStream(String fileName)
int fieldId = rowType.getFieldIndex(vectorFieldName);
for (ResultEntry entry : entries) {
- long fileSize = fileIO.getFileSize(new Path(indexDir, entry.fileName()));
+ long fileSize = fileIO.getFileSize(new Path(entry.filePath()));
GlobalIndexMeta globalMeta =
new GlobalIndexMeta(0, vectors.length - 1, fieldId, null, entry.meta());
metas.add(
new IndexFileMeta(
LuminaVectorGlobalIndexerFactory.IDENTIFIER,
- entry.fileName(),
+ new Path(entry.filePath()).getName(),
fileSize,
entry.rowCount(),
globalMeta,
diff --git a/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexTest.java b/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexTest.java
index c9dd07fddbe4..540889e62e57 100644
--- a/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexTest.java
+++ b/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexTest.java
@@ -108,26 +108,25 @@ public void cleanup() throws IOException {
private GlobalIndexFileWriter createFileWriter(Path path) {
return new GlobalIndexFileWriter() {
@Override
- public String newFileName(String prefix) {
- return prefix + "-" + UUID.randomUUID();
+ public String newFilePath(String prefix) {
+ return new Path(path, prefix + "-" + UUID.randomUUID()).toString();
}
@Override
- public PositionOutputStream newOutputStream(String fileName) throws IOException {
- return fileIO.newOutputStream(new Path(path, fileName), false);
+ public PositionOutputStream newOutputStream(String filePath) throws IOException {
+ return fileIO.newOutputStream(new Path(filePath), false);
}
};
}
- private GlobalIndexFileReader createFileReader(Path path) {
- return meta -> fileIO.newInputStream(new Path(path, meta.filePath()));
+ private GlobalIndexFileReader createFileReader() {
+ return meta -> fileIO.newInputStream(meta.filePath());
}
- private List toIOMetas(List results, Path path)
- throws IOException {
+ private List toIOMetas(List results) throws IOException {
assertThat(results).hasSize(1);
ResultEntry result = results.get(0);
- Path filePath = new Path(path, result.fileName());
+ Path filePath = new Path(result.filePath());
return Collections.singletonList(
new GlobalIndexIOMeta(filePath, fileIO.getFileSize(filePath), result.meta()));
}
@@ -156,9 +155,9 @@ public void testDifferentMetrics() throws IOException {
writeVectors(writer, testVectors);
List results = writer.finish();
- List metas = toIOMetas(results, metricIndexPath);
+ List metas = toIOMetas(results);
- GlobalIndexFileReader fileReader = createFileReader(metricIndexPath);
+ GlobalIndexFileReader fileReader = createFileReader();
try (LuminaVectorGlobalIndexReader reader =
new LuminaVectorGlobalIndexReader(
fileReader, metas, vectorType, indexOptions, executor)) {
@@ -191,9 +190,9 @@ public void testDifferentDimensions() throws IOException {
writeVectors(writer, testVectors);
List results = writer.finish();
- List metas = toIOMetas(results, dimIndexPath);
+ List metas = toIOMetas(results);
- GlobalIndexFileReader fileReader = createFileReader(dimIndexPath);
+ GlobalIndexFileReader fileReader = createFileReader();
try (LuminaVectorGlobalIndexReader reader =
new LuminaVectorGlobalIndexReader(
fileReader, metas, vectorType, indexOptions, executor)) {
@@ -242,9 +241,9 @@ public void testFloatVectorIndexEndToEnd() throws IOException {
writeVectors(writer, vectors);
List results = writer.finish();
- List metas = toIOMetas(results, indexPath);
+ List metas = toIOMetas(results);
- GlobalIndexFileReader fileReader = createFileReader(indexPath);
+ GlobalIndexFileReader fileReader = createFileReader();
try (LuminaVectorGlobalIndexReader reader =
new LuminaVectorGlobalIndexReader(
fileReader, metas, vectorType, indexOptions, executor)) {
@@ -304,9 +303,9 @@ public void testSearchWithFilter() throws IOException {
new LuminaVectorGlobalIndexWriter(fileWriter, vectorType, indexOptions);
writeVectors(writer, vectors);
List results = writer.finish();
- List metas = toIOMetas(results, indexPath);
+ List metas = toIOMetas(results);
- GlobalIndexFileReader fileReader = createFileReader(indexPath);
+ GlobalIndexFileReader fileReader = createFileReader();
try (LuminaVectorGlobalIndexReader reader =
new LuminaVectorGlobalIndexReader(
fileReader, metas, vectorType, indexOptions, executor)) {
@@ -372,13 +371,13 @@ public void testLargeVectorSet() throws IOException {
writeVectors(writer, testVectors);
List results = writer.finish();
- List metas = toIOMetas(results, indexPath);
+ List metas = toIOMetas(results);
- Path filePath = new Path(indexPath, results.get(0).fileName());
+ Path filePath = new Path(results.get(0).filePath());
assertThat(fileIO.exists(filePath)).isTrue();
assertThat(fileIO.getFileSize(filePath)).isGreaterThan(0);
- GlobalIndexFileReader fileReader = createFileReader(indexPath);
+ GlobalIndexFileReader fileReader = createFileReader();
try (LuminaVectorGlobalIndexReader reader =
new LuminaVectorGlobalIndexReader(
fileReader, metas, vectorType, indexOptions, executor)) {
@@ -422,7 +421,7 @@ public void testReaderMetaOptionsOverrideDefaultOptions() throws IOException {
new LuminaVectorGlobalIndexWriter(fileWriter, vectorType, writeIndexOptions);
writeVectors(writer, vectors);
List results = writer.finish();
- List metas = toIOMetas(results, indexPath);
+ List metas = toIOMetas(results);
// Read with default options (dimension=128, the default) — simulates
// the case where table options do not contain lumina.index.dimension.
@@ -434,7 +433,7 @@ public void testReaderMetaOptionsOverrideDefaultOptions() throws IOException {
LuminaVectorIndexOptions readIndexOptions = new LuminaVectorIndexOptions(readOptions);
assertThat(readIndexOptions.dimension()).isEqualTo(128);
- GlobalIndexFileReader fileReader = createFileReader(indexPath);
+ GlobalIndexFileReader fileReader = createFileReader();
try (LuminaVectorGlobalIndexReader reader =
new LuminaVectorGlobalIndexReader(
fileReader, metas, vectorType, readIndexOptions, executor)) {
@@ -475,9 +474,9 @@ public void testVectorTypeEndToEnd() throws IOException {
}
List results = writer.finish();
- List metas = toIOMetas(results, vecIndexPath);
+ List metas = toIOMetas(results);
- GlobalIndexFileReader fileReader = createFileReader(vecIndexPath);
+ GlobalIndexFileReader fileReader = createFileReader();
try (LuminaVectorGlobalIndexReader reader =
new LuminaVectorGlobalIndexReader(
fileReader, metas, vecFieldType, indexOptions, executor)) {
@@ -562,8 +561,8 @@ public void testNullVectorSkipWithCorrectIds() throws IOException {
assertThat(results).hasSize(1);
assertThat(results.get(0).rowCount()).isEqualTo(6);
- List metas = toIOMetas(results, indexPath);
- GlobalIndexFileReader fileReader = createFileReader(indexPath);
+ List metas = toIOMetas(results);
+ GlobalIndexFileReader fileReader = createFileReader();
try (LuminaVectorGlobalIndexReader reader =
new LuminaVectorGlobalIndexReader(
fileReader, metas, vectorType, indexOptions, executor)) {
@@ -629,9 +628,9 @@ public void testNullGapWithPreFilter() throws IOException {
writer.write(vectors[3], 5); // row 5
List results = writer.finish();
- List metas = toIOMetas(results, indexPath);
+ List metas = toIOMetas(results);
- GlobalIndexFileReader fileReader = createFileReader(indexPath);
+ GlobalIndexFileReader fileReader = createFileReader();
try (LuminaVectorGlobalIndexReader reader =
new LuminaVectorGlobalIndexReader(
fileReader, metas, vectorType, indexOptions, executor)) {
@@ -689,8 +688,8 @@ public void testNullAtStart() throws IOException {
assertThat(results).hasSize(1);
assertThat(results.get(0).rowCount()).isEqualTo(2);
- List metas = toIOMetas(results, indexPath);
- GlobalIndexFileReader fileReader = createFileReader(indexPath);
+ List metas = toIOMetas(results);
+ GlobalIndexFileReader fileReader = createFileReader();
try (LuminaVectorGlobalIndexReader reader =
new LuminaVectorGlobalIndexReader(
fileReader, metas, vectorType, indexOptions, executor)) {
@@ -719,8 +718,8 @@ public void testNullAtEnd() throws IOException {
assertThat(results).hasSize(1);
assertThat(results.get(0).rowCount()).isEqualTo(2);
- List metas = toIOMetas(results, indexPath);
- GlobalIndexFileReader fileReader = createFileReader(indexPath);
+ List metas = toIOMetas(results);
+ GlobalIndexFileReader fileReader = createFileReader();
try (LuminaVectorGlobalIndexReader reader =
new LuminaVectorGlobalIndexReader(
fileReader, metas, vectorType, indexOptions, executor)) {
@@ -795,9 +794,9 @@ public void testBatchVectorSearch() throws IOException {
writeVectors(writer, vectors);
List results = writer.finish();
- List metas = toIOMetas(results, indexPath);
+ List metas = toIOMetas(results);
- GlobalIndexFileReader fileReader = createFileReader(indexPath);
+ GlobalIndexFileReader fileReader = createFileReader();
try (LuminaVectorGlobalIndexReader reader =
new LuminaVectorGlobalIndexReader(
fileReader, metas, vectorType, indexOptions, executor)) {
@@ -846,9 +845,9 @@ public void testBatchVectorSearchWithFilter() throws IOException {
writeVectors(writer, vectors);
List results = writer.finish();
- List metas = toIOMetas(results, indexPath);
+ List metas = toIOMetas(results);
- GlobalIndexFileReader fileReader = createFileReader(indexPath);
+ GlobalIndexFileReader fileReader = createFileReader();
try (LuminaVectorGlobalIndexReader reader =
new LuminaVectorGlobalIndexReader(
fileReader, metas, vectorType, indexOptions, executor)) {
@@ -889,9 +888,9 @@ public void testBatchConsistentWithSingle() throws IOException {
writeVectors(writer, testVectors);
List results = writer.finish();
- List metas = toIOMetas(results, indexPath);
+ List metas = toIOMetas(results);
- GlobalIndexFileReader fileReader = createFileReader(indexPath);
+ GlobalIndexFileReader fileReader = createFileReader();
try (LuminaVectorGlobalIndexReader reader =
new LuminaVectorGlobalIndexReader(
fileReader, metas, vectorType, indexOptions, executor)) {
@@ -935,9 +934,9 @@ public void testBatchVectorSearchAppliesQueryOptions() throws IOException {
writeVectors(writer, testVectors);
List results = writer.finish();
- List metas = toIOMetas(results, indexPath);
+ List metas = toIOMetas(results);
- GlobalIndexFileReader fileReader = createFileReader(indexPath);
+ GlobalIndexFileReader fileReader = createFileReader();
try (LuminaVectorGlobalIndexReader reader =
new LuminaVectorGlobalIndexReader(
fileReader, metas, vectorType, indexOptions, executor)) {
diff --git a/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexWriterTest.java b/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexWriterTest.java
index 87cd6b56efc2..5a6625b2fd7e 100644
--- a/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexWriterTest.java
+++ b/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexWriterTest.java
@@ -135,12 +135,12 @@ private static Options createDefaultOptions(int dimension) {
private static GlobalIndexFileWriter createNoopFileWriter() {
return new GlobalIndexFileWriter() {
@Override
- public String newFileName(String prefix) {
- return prefix;
+ public String newFilePath(String prefix) {
+ return "memory://test/" + prefix;
}
@Override
- public PositionOutputStream newOutputStream(String fileName) {
+ public PositionOutputStream newOutputStream(String filePath) {
throw new UnsupportedOperationException();
}
};
diff --git a/paimon-python/pypaimon/globalindex/create_global_index.py b/paimon-python/pypaimon/globalindex/create_global_index.py
index 10b88a2eb8f7..f407c598c891 100644
--- a/paimon-python/pypaimon/globalindex/create_global_index.py
+++ b/paimon-python/pypaimon/globalindex/create_global_index.py
@@ -115,6 +115,7 @@ def __init__(
if CoreOptions._is_vector_index_type(self._index_type) else
self._core_options.global_index_options(self._index_type, self._index_columns[0]))
self._core_options = CoreOptions(self._options)
+ self._validate_external_path()
if self._index_type in VINDEX_IDENTIFIERS:
key = self._index_type + ".sharding"
layout = self._options.to_map().get(key, "range")
@@ -152,6 +153,18 @@ def build(self, *, execution="local", concurrency=None, ray_remote_args=None) ->
self, partition_filter, execution=execution, concurrency=concurrency,
ray_remote_args=ray_remote_args)
+ def _validate_external_path(self) -> None:
+ # Python placement uses table options; native builds also receive build
+ # overrides. Validate both before selecting a backend or creating files.
+ for options in (self._table.options, self._core_options):
+ path = options.global_index_external_path()
+ if path is not None and ',' in path:
+ raise ValueError(
+ "PyPaimon global index build only supports a single "
+ "'global-index.external-path' directory; comma-separated "
+ "paths are not supported."
+ )
+
def _validate_unaware_bucket_table(self) -> None:
# Global index files live in bucket 0 and are keyed by the table-wide
# _ROW_ID, so every index type needs an unaware-bucket table. Check the
diff --git a/paimon-python/pypaimon/tests/global_index_build_test.py b/paimon-python/pypaimon/tests/global_index_build_test.py
index dfdac43dbf13..dee016112bfd 100644
--- a/paimon-python/pypaimon/tests/global_index_build_test.py
+++ b/paimon-python/pypaimon/tests/global_index_build_test.py
@@ -739,6 +739,61 @@ def test_create_global_index_uses_external_path(self):
self.assertTrue(external_path.startswith(external_root + '/'))
self.assertTrue(table.file_io.exists(external_path))
+ def test_create_global_index_rejects_multiple_external_paths_before_build(self):
+ from pypaimon.globalindex.index_dispatch import build_global_index
+
+ table = self._create_table()
+ self._write_arrow(table, pa.table(
+ {'id': [1], 'name': ['a'], 'age': [10], 'city': ['x']},
+ schema=self.pa_schema,
+ ))
+ option = 'global-index.external-path'
+ root_a = 'file://%s' % os.path.join(self.tempdir, 'external-a')
+ root_b = 'file://%s' % os.path.join(self.tempdir, 'external-b')
+ snapshot = table.snapshot_manager().get_latest_snapshot()
+ files_before = {
+ os.path.join(root, name)
+ for root, dirs, files in os.walk(self.tempdir)
+ for name in dirs + files
+ }
+
+ for paths in (root_a + ',' + root_b, root_a + ', ' + root_b + ','):
+ for native in ('false', 'true'):
+ for source in ('table', 'build', 'table-with-build-override'):
+ with self.subTest(paths=paths, native=native, source=source):
+ table_options = {'write.native.enabled': native}
+ build_options = {}
+ if source == 'build':
+ build_options[option] = paths
+ else:
+ table_options[option] = paths
+ if source == 'table-with-build-override':
+ build_options[option] = root_a
+ configured = table.copy(table_options)
+ with patch(
+ 'pypaimon.globalindex.index_dispatch.build_global_index',
+ wraps=build_global_index,
+ ) as build:
+ with patch.object(
+ configured, 'new_batch_write_builder',
+ wraps=configured.new_batch_write_builder,
+ ) as write_builder:
+ with self.assertRaisesRegex(
+ ValueError,
+ 'global-index.external-path.*comma-separated',
+ ):
+ configured.create_global_index('id', options=build_options)
+ build.assert_not_called()
+ write_builder.assert_not_called()
+ self.assertEqual(
+ snapshot.id, table.snapshot_manager().get_latest_snapshot().id)
+ self.assertEqual([], IndexFileHandler(table).scan(snapshot))
+ self.assertEqual(files_before, {
+ os.path.join(root, name)
+ for root, dirs, files in os.walk(self.tempdir)
+ for name in dirs + files
+ })
+
def test_create_global_index_skips_existing_ranges(self):
table = self._create_table()
self._write_arrow(table, pa.table(
diff --git a/paimon-python/pypaimon/tests/resources/btree/BTreeFixtureGenerator.java b/paimon-python/pypaimon/tests/resources/btree/BTreeFixtureGenerator.java
index ae0c087b1e36..e7b9992ff590 100644
--- a/paimon-python/pypaimon/tests/resources/btree/BTreeFixtureGenerator.java
+++ b/paimon-python/pypaimon/tests/resources/btree/BTreeFixtureGenerator.java
@@ -106,12 +106,12 @@ public static void main(String[] args) throws Exception {
GlobalIndexFileWriter fileWriter =
new GlobalIndexFileWriter() {
@Override
- public String newFileName(String prefix) {
- return name;
+ public String newFilePath(String prefix) {
+ return directory.resolve(name).toAbsolutePath().toString();
}
@Override
- public PositionOutputStream newOutputStream(String fileName) {
+ public PositionOutputStream newOutputStream(String filePath) {
return new PositionOutputStream() {
@Override
public long getPos() {
@@ -176,12 +176,12 @@ private static void writeComposite(Path directory, String name, boolean reversed
GlobalIndexFileWriter files =
new GlobalIndexFileWriter() {
@Override
- public String newFileName(String prefix) {
- return name;
+ public String newFilePath(String prefix) {
+ return directory.resolve(name).toAbsolutePath().toString();
}
@Override
- public PositionOutputStream newOutputStream(String fileName) {
+ public PositionOutputStream newOutputStream(String filePath) {
return new PositionOutputStream() {
@Override
public long getPos() {
diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/copy/CopyFilesUtil.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/copy/CopyFilesUtil.java
index 5ec69b75011e..c9466a9951c0 100644
--- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/copy/CopyFilesUtil.java
+++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/copy/CopyFilesUtil.java
@@ -118,18 +118,15 @@ public static DataFileMeta toNewDataFileMeta(
null);
}
- public static IndexFileMeta toNewIndexFileMeta(IndexFileMeta oldFileMeta, String newFileName) {
- String newExternalPath =
- externalPathDir(oldFileMeta.externalPath())
- .map(dir -> dir + "/" + newFileName)
- .orElse(null);
+ public static IndexFileMeta toNewIndexFileMeta(
+ IndexFileMeta oldFileMeta, Path newFilePath, boolean isExternalPath) {
return new IndexFileMeta(
oldFileMeta.indexType(),
- newFileName,
+ newFilePath.getName(),
oldFileMeta.fileSize(),
oldFileMeta.rowCount(),
oldFileMeta.dvRanges(),
- newExternalPath);
+ isExternalPath ? newFilePath.toString() : null);
}
public static Optional externalPathDir(String externalPath) {
diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/copy/ListIndexFilesOperator.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/copy/ListIndexFilesOperator.java
index 4e1d6c456261..288476ffb856 100644
--- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/copy/ListIndexFilesOperator.java
+++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/copy/ListIndexFilesOperator.java
@@ -25,6 +25,7 @@
import org.apache.paimon.index.IndexFileHandler;
import org.apache.paimon.index.IndexFileMeta;
import org.apache.paimon.index.IndexFileMetaSerializer;
+import org.apache.paimon.index.IndexPathFactory;
import org.apache.paimon.manifest.IndexManifestEntry;
import org.apache.paimon.partition.PartitionPredicate;
import org.apache.paimon.table.FileStoreTable;
@@ -87,14 +88,14 @@ private CopyFileInfo pickIndexFiles(
FileStorePathFactory targetFileStorePathFactory)
throws IOException {
Path indexFilePath = sourceIndexFileHandler.filePath(indexManifestEntry);
- Path targetIndexFilePath =
- targetFileStorePathFactory
- .indexFileFactory(
- indexManifestEntry.partition(), indexManifestEntry.bucket())
- .newPath();
+ IndexPathFactory targetIndexPathFactory =
+ targetFileStorePathFactory.indexFileFactory(
+ indexManifestEntry.partition(), indexManifestEntry.bucket());
+ Path targetIndexFilePath = targetIndexPathFactory.newPath();
IndexFileMeta fileMeta = indexManifestEntry.indexFile();
IndexFileMeta targetFileMeta =
- CopyFilesUtil.toNewIndexFileMeta(fileMeta, targetIndexFilePath.getName());
+ CopyFilesUtil.toNewIndexFileMeta(
+ fileMeta, targetIndexFilePath, targetIndexPathFactory.isExternalPath());
return new CopyFileInfo(
indexFilePath.toString(),
targetIndexFilePath.toString(),
diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/CentroidGlobalIndexTopoBuilder.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/CentroidGlobalIndexTopoBuilder.java
index 4bbe847699c2..262b5527a02e 100644
--- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/CentroidGlobalIndexTopoBuilder.java
+++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/CentroidGlobalIndexTopoBuilder.java
@@ -296,16 +296,22 @@ private static CommitMessage writeRouter(
int nlist)
throws IOException {
GlobalIndexFileReadWrite io = context.io();
- String modelFile =
+ String modelFileName =
"vector-"
+ context.indexType
+ "-global-index-training-model-"
+ modelId
+ ".index";
+ String modelFilePath =
+ context.table
+ .store()
+ .pathFactory()
+ .globalIndexFileFactory()
+ .newPathFromName(modelFileName)
+ .toString();
IndexFileMeta modelMeta =
GlobalIndexBuilderUtils.toIndexFileMetas(
context.table.fileIO(),
- context.table.store().pathFactory().globalIndexFileFactory(),
context.table.coreOptions(),
context.rowRange,
Collections.singletonList(context.field),
@@ -313,7 +319,7 @@ private static CommitMessage writeRouter(
Collections.singletonList(
CentroidIndexMeta.router(
indexer.writeRoutingModel(
- io, modelFile, payload, modelId),
+ io, modelFilePath, payload, modelId),
modelId,
nlist)),
context.sourceMeta())
@@ -581,7 +587,6 @@ byte[] finish(GlobalIndexSingleColumnWriter writer, String modelId, int centroid
List files =
GlobalIndexBuilderUtils.toIndexFileMetas(
table.fileIO(),
- table.store().pathFactory().globalIndexFileFactory(),
table.coreOptions(),
rowRange,
Collections.singletonList(field),
diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/DefaultGlobalIndexBuilder.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/DefaultGlobalIndexBuilder.java
index a78ad1d7e19a..02a221042aef 100644
--- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/DefaultGlobalIndexBuilder.java
+++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/DefaultGlobalIndexBuilder.java
@@ -20,6 +20,7 @@
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.globalindex.GlobalIndexFileReadWrite;
import org.apache.paimon.globalindex.GlobalIndexMultiColumnWriter;
import org.apache.paimon.globalindex.GlobalIndexSingleColumnWriter;
import org.apache.paimon.globalindex.GlobalIndexWriter;
@@ -47,6 +48,7 @@
import java.util.Collections;
import java.util.List;
+import static org.apache.paimon.globalindex.GlobalIndexBuilderUtils.createGlobalIndexFileReadWrite;
import static org.apache.paimon.globalindex.GlobalIndexBuilderUtils.createIndexWriter;
import static org.apache.paimon.globalindex.GlobalIndexBuilderUtils.toIndexFileMetas;
@@ -148,12 +150,16 @@ public RowType readType() {
}
public CommitMessage build(CloseableIterator data) throws IOException {
+ return build(data, createGlobalIndexFileReadWrite(table));
+ }
+
+ CommitMessage build(CloseableIterator data, GlobalIndexFileReadWrite fileReadWrite)
+ throws IOException {
LongCounter rowCounter = new LongCounter(0);
- List resultEntries = writePaimonRows(data, rowCounter);
+ List resultEntries = writePaimonRows(data, rowCounter, fileReadWrite);
List indexFileMetas =
toIndexFileMetas(
table.fileIO(),
- table.store().pathFactory().globalIndexFileFactory(),
table.coreOptions(),
rowRange,
indexedFields(),
@@ -166,9 +172,12 @@ public CommitMessage build(CloseableIterator data) throws IOExcepti
}
private List writePaimonRows(
- CloseableIterator rows, LongCounter rowCounter) throws IOException {
+ CloseableIterator rows,
+ LongCounter rowCounter,
+ GlobalIndexFileReadWrite fileReadWrite)
+ throws IOException {
GlobalIndexWriter indexWriter =
- createIndexWriter(table, indexType, indexedFields(), options);
+ createIndexWriter(fileReadWrite, indexType, indexedFields(), options);
boolean multiColumn = !extraFields.isEmpty();
try {
diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/DefaultGlobalIndexTopoBuilder.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/DefaultGlobalIndexTopoBuilder.java
index 7e5a401df302..d5c85d6c2249 100644
--- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/DefaultGlobalIndexTopoBuilder.java
+++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/DefaultGlobalIndexTopoBuilder.java
@@ -19,11 +19,13 @@
package org.apache.paimon.spark.globalindex;
import org.apache.paimon.Snapshot;
+import org.apache.paimon.annotation.VisibleForTesting;
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.fs.Path;
import org.apache.paimon.globalindex.DataEvolutionGlobalIndexRefreshPlanner;
import org.apache.paimon.globalindex.GlobalIndexBuilderUtils;
+import org.apache.paimon.globalindex.GlobalIndexFileReadWrite;
import org.apache.paimon.globalindex.IndexedSplit;
import org.apache.paimon.index.DataEvolutionIndexSourceMeta;
import org.apache.paimon.io.CompactIncrement;
@@ -54,6 +56,7 @@
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.function.BiFunction;
@@ -63,6 +66,7 @@
import static org.apache.paimon.CoreOptions.GLOBAL_INDEX_COLUMN_UPDATE_ACTION;
import static org.apache.paimon.CoreOptions.GLOBAL_INDEX_ROW_COUNT_PER_SHARD;
import static org.apache.paimon.CoreOptions.GlobalIndexColumnUpdateAction.IGNORE;
+import static org.apache.paimon.globalindex.GlobalIndexBuilderUtils.createGlobalIndexFileReadWrite;
import static org.apache.paimon.utils.Preconditions.checkArgument;
/** Default topology builder. */
@@ -178,7 +182,7 @@ public List buildIndex(
List commitMessageBytes =
javaSparkContext
.parallelize(taskList, parallelism)
- .map(DefaultGlobalIndexTopoBuilder::buildIndex)
+ .mapPartitions(DefaultGlobalIndexTopoBuilder::buildIndexes)
.collect();
commitMessages.addAll(CommitMessageSerializer.deserializeAll(commitMessageBytes));
}
@@ -211,20 +215,30 @@ static int parallelism(int taskCount, Options options) {
return Math.min(taskCount, maxParallelism);
}
- private static byte[] buildIndex(Pair builderAndSplits) throws Exception {
+ @VisibleForTesting
+ static Iterator