From 2ea74d4d063a77ca2d1eb17af23b0a9f90824366 Mon Sep 17 00:00:00 2001 From: zenotme Date: Thu, 19 Nov 2026 20:32:13 +0800 Subject: [PATCH 1/2] [Improvement] Avoid an unnecessary S3 PUT when opening a data file --- .../datafusion/src/physical_plan/sink.rs | 61 +++++++++++-------- .../datafusion/tests/native_merge.rs | 2 +- crates/paimon/src/table/data_file_writer.rs | 1 - .../paimon/src/table/kv_data_file_writer.rs | 3 - 4 files changed, 36 insertions(+), 31 deletions(-) diff --git a/crates/integrations/datafusion/src/physical_plan/sink.rs b/crates/integrations/datafusion/src/physical_plan/sink.rs index 76524da06..f1a374174 100644 --- a/crates/integrations/datafusion/src/physical_plan/sink.rs +++ b/crates/integrations/datafusion/src/physical_plan/sink.rs @@ -296,7 +296,7 @@ impl DataSink for PaimonDataSink { #[cfg(test)] mod tests { use super::*; - use datafusion::arrow::array::{Int32Array, Int64Array, StringViewArray}; + use datafusion::arrow::array::{Int32Array, Int64Array, StringArray, StringViewArray}; use datafusion::arrow::datatypes::Field; #[test] @@ -336,7 +336,7 @@ mod tests { use datafusion::prelude::SessionConfig; use paimon::catalog::Identifier; use paimon::io::FileIOBuilder; - use paimon::spec::{DataType, IntType, Schema as PaimonSchema, TableSchema}; + use paimon::spec::{DataType, Schema as PaimonSchema, TableSchema, VarCharType}; // When a writer fails while the input waits for its next batch, the // INSERT must return the writer's error instead of waiting for input @@ -346,7 +346,10 @@ mod tests { // first data file fails with a real filesystem error. std::fs::write(dir.path().join("bucket-0"), b"").unwrap(); let schema = PaimonSchema::builder() - .column("id", DataType::Int(IntType::new())) + .column("payload", DataType::VarChar(VarCharType::string_type())) + .option("file.compression", "none") + // Flush the first batch while the input stream is still pending. + .option("write.parquet-buffer-size", "4 kb") .option(SINK_PARALLELISM, "2") .build() .unwrap(); @@ -358,13 +361,15 @@ mod tests { None, ); let arrow_schema = Arc::new(Schema::new(vec![Field::new( - "id", - ArrowDataType::Int32, + "payload", + ArrowDataType::Utf8, true, )])); + // Exceed the 8 MiB output chunk so the first batch reaches the filesystem. + let large = "x".repeat(9 * 1024 * 1024); let batch = RecordBatch::try_new( arrow_schema.clone(), - vec![Arc::new(Int32Array::from(vec![1]))], + vec![Arc::new(StringArray::from(vec![large]))], ) .unwrap(); // One batch for the first writer, then the input stays pending like a @@ -390,17 +395,12 @@ mod tests { async fn test_later_writer_failure_ends_insert_while_earlier_writer_waits() { use std::time::Duration; - use datafusion::arrow::array::StringArray; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::prelude::SessionConfig; use paimon::catalog::Identifier; use paimon::io::FileIOBuilder; - use paimon::spec::{DataType, IntType, Schema as PaimonSchema, TableSchema, VarCharType}; + use paimon::spec::{DataType, Schema as PaimonSchema, TableSchema, VarCharType}; - // The second writer fails while the first writer is still alive and - // waiting for more input. The INSERT must return the second writer's - // error; waiting on the writers in creation order would block on the - // first writer and never see it. let dir = tempfile::tempdir().unwrap(); // A regular file where the bucket directory of partition p=bad // belongs: only writes to that partition fail, with a real filesystem @@ -408,10 +408,12 @@ mod tests { std::fs::create_dir_all(dir.path().join("p=bad")).unwrap(); std::fs::write(dir.path().join("p=bad").join("bucket-0"), b"").unwrap(); let schema = PaimonSchema::builder() - .column("id", DataType::Int(IntType::new())) + .column("payload", DataType::VarChar(VarCharType::string_type())) .column("p", DataType::VarChar(VarCharType::string_type())) .partition_keys(["p"]) .option(SINK_PARALLELISM, "2") + .option("file.compression", "none") + .option("write.parquet-buffer-size", "4 kb") .build() .unwrap(); let table = Table::new( @@ -422,25 +424,34 @@ mod tests { None, ); let arrow_schema = Arc::new(Schema::new(vec![ - Field::new("id", ArrowDataType::Int32, true), + Field::new("payload", ArrowDataType::Utf8, true), Field::new("p", ArrowDataType::Utf8, true), ])); - let batch = |id: i32, p: &str| { + let batch = |payload: &str, p: &str| { RecordBatch::try_new( arrow_schema.clone(), vec![ - Arc::new(Int32Array::from(vec![id])), + Arc::new(StringArray::from(vec![payload])), Arc::new(StringArray::from(vec![p])), ], ) .unwrap() }; - // Batches are dealt round-robin: the p=ok batch goes to the first - // writer, which buffers it and waits; the p=bad batch goes to the - // second writer, which fails. Then the input stays pending like a - // slow source. - let input = futures::stream::iter([Ok(batch(1, "ok")), Ok(batch(2, "bad"))]) - .chain(futures::stream::pending()); + // Batches are dealt round-robin: the first writer buffers the p=ok + // batches and waits. The second writer gets both p=bad batches. Its + // large second batch forces a Parquet flush past the 8 MiB output chunk. + // The filesystem tries to create the data file and fails because + // p=bad/bucket-0 is a regular file. + // The input stays pending like a slow source. INSERT must return the + // second writer's error even while the first writer waits for more input. + let large = "x".repeat(9 * 1024 * 1024); + let input = futures::stream::iter([ + Ok(batch("first", "ok")), + Ok(batch("first", "bad")), + Ok(batch("second", "ok")), + Ok(batch(&large, "bad")), + ]) + .chain(futures::stream::pending()); let data = Box::pin(RecordBatchStreamAdapter::new(arrow_schema.clone(), input)); let context = Arc::new( TaskContext::default() @@ -448,10 +459,8 @@ mod tests { ); let sink = PaimonDataSink::new(table, arrow_schema, false); - let result = - tokio::time::timeout(Duration::from_secs(10), sink.write_all(data, &context)).await; - - let error = result + let error = tokio::time::timeout(Duration::from_secs(10), sink.write_all(data, &context)) + .await .expect("INSERT still waits after the second writer failed") .unwrap_err(); // The error comes from opening a data file under p=bad/bucket-0, so diff --git a/crates/integrations/datafusion/tests/native_merge.rs b/crates/integrations/datafusion/tests/native_merge.rs index bb816cebf..cc8ce339b 100644 --- a/crates/integrations/datafusion/tests/native_merge.rs +++ b/crates/integrations/datafusion/tests/native_merge.rs @@ -477,7 +477,7 @@ async fn failed_merge_insert_discards_prepared_cow_files() { USING (VALUES (1, 100, 'a'), (2, 200, 'b')) AS s(id, value, pt) \ ON t.id = s.id WHEN MATCHED THEN UPDATE SET value = s.value \ WHEN NOT MATCHED THEN INSERT (id, value, pt) VALUES (s.id, s.value, s.pt)", - "Failed to create directory", + "Not a directory", ) .await; assert_eq!( diff --git a/crates/paimon/src/table/data_file_writer.rs b/crates/paimon/src/table/data_file_writer.rs index a97ebea26..de56071b5 100644 --- a/crates/paimon/src/table/data_file_writer.rs +++ b/crates/paimon/src/table/data_file_writer.rs @@ -317,7 +317,6 @@ impl DataFileWriter { .new_file_name(&self.data_file_prefix, &self.file_format); let location = self.paths.new_path(&file_name)?; let bucket_dir = location.parent(); - self.file_io.mkdirs(&format!("{bucket_dir}/")).await?; let file_path = location.path.clone(); self.created_paths.push(file_path.clone()); diff --git a/crates/paimon/src/table/kv_data_file_writer.rs b/crates/paimon/src/table/kv_data_file_writer.rs index 30044b738..42ad5ad91 100644 --- a/crates/paimon/src/table/kv_data_file_writer.rs +++ b/crates/paimon/src/table/kv_data_file_writer.rs @@ -200,9 +200,6 @@ impl<'a> KeyValueFileOptions<'a> { self.file_format ); let location = self.paths.new_path(&name)?; - self.file_io - .mkdirs(&format!("{}/", location.parent())) - .await?; let output = self.file_io.new_output(&location.path)?; let writer = match factory.create_writer(&output, self.file_compression).await { Ok(writer) => with_write_resources(writer, self.resources), From b285e5779978fb32fbcd99da3dbfac99a3e3313d Mon Sep 17 00:00:00 2001 From: zenotme Date: Sun, 22 Nov 2026 11:42:31 +0800 Subject: [PATCH 2/2] [Improvement] Keep parent directory creation for custom FileIO providers --- .../src/catalog/rest/rest_token_file_io.rs | 4 ++ crates/paimon/src/io/file_io.rs | 62 ++++++++++++++++--- crates/paimon/src/io/file_io/fuse.rs | 4 ++ crates/paimon/src/io/storage.rs | 35 +++++++++++ 4 files changed, 98 insertions(+), 7 deletions(-) diff --git a/crates/paimon/src/catalog/rest/rest_token_file_io.rs b/crates/paimon/src/catalog/rest/rest_token_file_io.rs index 2ea331714..c1e0c378a 100644 --- a/crates/paimon/src/catalog/rest/rest_token_file_io.rs +++ b/crates/paimon/src/catalog/rest/rest_token_file_io.rs @@ -199,6 +199,10 @@ impl FileIOProvider for RESTTokenFileIO { let (op, relative_path) = file_io.create_static(path)?; Ok((op, relative_path, Some(cache_namespace))) } + + fn requires_parent_dir_for_write(&self) -> bool { + false + } } #[cfg(test)] diff --git a/crates/paimon/src/io/file_io.rs b/crates/paimon/src/io/file_io.rs index edc472a10..ac8b06d5e 100644 --- a/crates/paimon/src/io/file_io.rs +++ b/crates/paimon/src/io/file_io.rs @@ -110,6 +110,17 @@ pub trait FileIOProvider: std::fmt::Debug + Send + Sync + 'static { let (op, relative_path) = self.create(path).await?; Ok((op, relative_path, None)) } + + /// Whether opening a file for writing requires calling `create_dir` on its + /// parent directory first. + /// + /// Defaults to `true` for compatibility with hierarchical backends that + /// require parent directories to exist before writing. Providers backed by + /// object stores or operators that create missing parent directories on + /// write may return `false` to avoid extra directory marker requests. + fn requires_parent_dir_for_write(&self) -> bool { + true + } } #[derive(Clone, Debug)] @@ -542,6 +553,14 @@ impl FileIO { Ok((op, relative_path, cache_path)) } + #[cfg(feature = "storage-fs")] + pub(crate) fn requires_parent_dir_for_write(&self) -> bool { + match &self.backend { + FileIOBackend::Storage(storage) => storage.requires_parent_dir_for_write(), + FileIOBackend::Provider(provider) => provider.requires_parent_dir_for_write(), + } + } + fn file_source(&self, path: &str) -> crate::Result { match &self.backend { FileIOBackend::Provider(provider) => Ok(FileSource::Provider(provider.clone())), @@ -552,15 +571,14 @@ impl FileIO { #[cfg(feature = "storage-fs")] let local_fs = matches!(storage.as_ref(), Storage::LocalFs { .. }); #[cfg(not(feature = "storage-fs"))] - let local_fs = { - let _ = storage; - false - }; + let local_fs = false; + let requires_parent_dir = storage.requires_parent_dir_for_write(); Ok(FileSource::Static { op, relative_path, cache_path, local_fs, + requires_parent_dir, }) } } @@ -1746,6 +1764,7 @@ enum FileSource { relative_path: String, cache_path: String, local_fs: bool, + requires_parent_dir: bool, }, Provider(Arc), } @@ -1768,6 +1787,35 @@ impl FileSource { } => Ok((op.clone(), relative_path.clone(), Some(cache_path.clone()))), } } + + async fn resolve_for_write( + &self, + path: &str, + ) -> crate::Result<(Operator, String, Option)> { + let (op, relative_path, cache_path) = self.resolve(path).await?; + let requires_parent_dir = match self { + Self::Static { + requires_parent_dir, + .. + } => *requires_parent_dir, + Self::Provider(provider) => provider.requires_parent_dir_for_write(), + }; + if requires_parent_dir && op.info().capability().create_dir { + let trimmed = relative_path.trim_end_matches('/'); + if let Some((parent, _)) = trimmed.rsplit_once('/') { + if !parent.trim_matches('/').is_empty() { + let dir_path = normalize_root(parent); + op.create_dir(&dir_path).await.map_err(|error| { + Error::from_opendal_with_context( + error, + format!("Failed to create parent directory for '{path}'"), + ) + })?; + } + } + } + Ok((op, relative_path, cache_path)) + } } #[derive(Debug)] @@ -1917,7 +1965,7 @@ impl OutputFile { } pub async fn writer(&self) -> crate::Result> { - let (op, relative_path, cache_path) = self.source.resolve(&self.path).await?; + let (op, relative_path, cache_path) = self.source.resolve_for_write(&self.path).await?; let writer: Box = Box::new( op.writer_with(&relative_path) .chunk(8 * 1024 * 1024) @@ -1930,7 +1978,7 @@ impl OutputFile { /// bulk output path unchanged; do not install OpenDAL's fixed-size chunk /// queue here because it cannot drain a partial chunk before close. pub(crate) async fn flushable_writer(&self) -> crate::Result> { - let (op, relative_path, cache_path) = self.source.resolve(&self.path).await?; + let (op, relative_path, cache_path) = self.source.resolve_for_write(&self.path).await?; let writer: Box = if matches!(self.source, FileSource::Static { local_fs: true, .. }) { Box::new(LocalFileWriter::new(&op, &relative_path).await?) @@ -1962,7 +2010,7 @@ impl OutputFile { /// Get an async streaming writer for format-level writes (e.g. parquet). pub(crate) async fn async_writer(&self) -> crate::Result> { - let (op, relative_path, cache_path) = self.source.resolve(&self.path).await?; + let (op, relative_path, cache_path) = self.source.resolve_for_write(&self.path).await?; let writer: Box = Box::new( op.writer_with(&relative_path) .chunk(8 * 1024 * 1024) diff --git a/crates/paimon/src/io/file_io/fuse.rs b/crates/paimon/src/io/file_io/fuse.rs index 27c039635..4c5442ce7 100644 --- a/crates/paimon/src/io/file_io/fuse.rs +++ b/crates/paimon/src/io/file_io/fuse.rs @@ -65,6 +65,10 @@ impl FileIOProvider for FuseFileIO { } self.fallback.create_with_cache_namespace(path).await } + + fn requires_parent_dir_for_write(&self) -> bool { + self.fallback.requires_parent_dir_for_write() + } } impl FuseFileIO { diff --git a/crates/paimon/src/io/storage.rs b/crates/paimon/src/io/storage.rs index 2ac36195c..c50dfcc0c 100644 --- a/crates/paimon/src/io/storage.rs +++ b/crates/paimon/src/io/storage.rs @@ -555,6 +555,41 @@ impl Storage { super::s3_config_build(config, path) }) } + + /// Whether opening a file for writing requires calling `create_dir` on its + /// parent directory first. + /// + /// Built-in storage services either accept object keys directly without + /// directory markers or create missing parent directories in their write + /// path. `CustomFs` wraps an unclassified caller-supplied operator, so keep + /// parent directory creation enabled for compatibility. + pub(crate) fn requires_parent_dir_for_write(&self) -> bool { + match self { + Storage::CustomFs { .. } => true, + #[cfg(feature = "storage-memory")] + Storage::Memory { .. } => false, + #[cfg(feature = "storage-fs")] + Storage::LocalFs { .. } => false, + #[cfg(feature = "storage-oss")] + Storage::Oss { .. } => false, + #[cfg(feature = "storage-oss-cpp")] + Storage::OssCpp { .. } => false, + #[cfg(feature = "storage-jindo")] + Storage::Jindo { .. } => false, + #[cfg(feature = "storage-s3")] + Storage::S3 { .. } => false, + #[cfg(feature = "storage-cos")] + Storage::Cos { .. } => false, + #[cfg(feature = "storage-azdls")] + Storage::Azdls { .. } => false, + #[cfg(feature = "storage-obs")] + Storage::Obs { .. } => false, + #[cfg(feature = "storage-gcs")] + Storage::Gcs { .. } => false, + #[cfg(feature = "storage-hdfs")] + Storage::Hdfs { .. } => false, + } + } } #[cfg(test)]