Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 35 additions & 26 deletions crates/integrations/datafusion/src/physical_plan/sink.rs
Original file line number Diff line number Diff line change
Expand Up @@ -296,7 +296,7 @@ impl DataSink for PaimonDataSink {
#[cfg(test)]
mod tests {
use super::*;
use datafusion::arrow::array::{Int32Array, Int64Array, StringViewArray};
use datafusion::arrow::array::{Int32Array, Int64Array, StringArray, StringViewArray};
use datafusion::arrow::datatypes::Field;

#[test]
Expand Down Expand Up @@ -336,7 +336,7 @@ mod tests {
use datafusion::prelude::SessionConfig;
use paimon::catalog::Identifier;
use paimon::io::FileIOBuilder;
use paimon::spec::{DataType, IntType, Schema as PaimonSchema, TableSchema};
use paimon::spec::{DataType, Schema as PaimonSchema, TableSchema, VarCharType};

// When a writer fails while the input waits for its next batch, the
// INSERT must return the writer's error instead of waiting for input
Expand All @@ -346,7 +346,10 @@ mod tests {
// first data file fails with a real filesystem error.
std::fs::write(dir.path().join("bucket-0"), b"").unwrap();
let schema = PaimonSchema::builder()
.column("id", DataType::Int(IntType::new()))
.column("payload", DataType::VarChar(VarCharType::string_type()))
.option("file.compression", "none")
// Flush the first batch while the input stream is still pending.
.option("write.parquet-buffer-size", "4 kb")
.option(SINK_PARALLELISM, "2")
.build()
.unwrap();
Expand All @@ -358,13 +361,15 @@ mod tests {
None,
);
let arrow_schema = Arc::new(Schema::new(vec![Field::new(
"id",
ArrowDataType::Int32,
"payload",
ArrowDataType::Utf8,
true,
)]));
// Exceed the 8 MiB output chunk so the first batch reaches the filesystem.
let large = "x".repeat(9 * 1024 * 1024);
let batch = RecordBatch::try_new(
arrow_schema.clone(),
vec![Arc::new(Int32Array::from(vec![1]))],
vec![Arc::new(StringArray::from(vec![large]))],
)
.unwrap();
// One batch for the first writer, then the input stays pending like a
Expand All @@ -390,28 +395,25 @@ mod tests {
async fn test_later_writer_failure_ends_insert_while_earlier_writer_waits() {
use std::time::Duration;

use datafusion::arrow::array::StringArray;
use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
use datafusion::prelude::SessionConfig;
use paimon::catalog::Identifier;
use paimon::io::FileIOBuilder;
use paimon::spec::{DataType, IntType, Schema as PaimonSchema, TableSchema, VarCharType};
use paimon::spec::{DataType, Schema as PaimonSchema, TableSchema, VarCharType};

// The second writer fails while the first writer is still alive and
// waiting for more input. The INSERT must return the second writer's
// error; waiting on the writers in creation order would block on the
// first writer and never see it.
let dir = tempfile::tempdir().unwrap();
// A regular file where the bucket directory of partition p=bad
// belongs: only writes to that partition fail, with a real filesystem
// error.
std::fs::create_dir_all(dir.path().join("p=bad")).unwrap();
std::fs::write(dir.path().join("p=bad").join("bucket-0"), b"").unwrap();
let schema = PaimonSchema::builder()
.column("id", DataType::Int(IntType::new()))
.column("payload", DataType::VarChar(VarCharType::string_type()))
.column("p", DataType::VarChar(VarCharType::string_type()))
.partition_keys(["p"])
.option(SINK_PARALLELISM, "2")
.option("file.compression", "none")
.option("write.parquet-buffer-size", "4 kb")
.build()
.unwrap();
let table = Table::new(
Expand All @@ -422,36 +424,43 @@ mod tests {
None,
);
let arrow_schema = Arc::new(Schema::new(vec![
Field::new("id", ArrowDataType::Int32, true),
Field::new("payload", ArrowDataType::Utf8, true),
Field::new("p", ArrowDataType::Utf8, true),
]));
let batch = |id: i32, p: &str| {
let batch = |payload: &str, p: &str| {
RecordBatch::try_new(
arrow_schema.clone(),
vec![
Arc::new(Int32Array::from(vec![id])),
Arc::new(StringArray::from(vec![payload])),
Arc::new(StringArray::from(vec![p])),
],
)
.unwrap()
};
// Batches are dealt round-robin: the p=ok batch goes to the first
// writer, which buffers it and waits; the p=bad batch goes to the
// second writer, which fails. Then the input stays pending like a
// slow source.
let input = futures::stream::iter([Ok(batch(1, "ok")), Ok(batch(2, "bad"))])
.chain(futures::stream::pending());
// Batches are dealt round-robin: the first writer buffers the p=ok
// batches and waits. The second writer gets both p=bad batches. Its
// large second batch forces a Parquet flush past the 8 MiB output chunk.
// The filesystem tries to create the data file and fails because
// p=bad/bucket-0 is a regular file.
// The input stays pending like a slow source. INSERT must return the
// second writer's error even while the first writer waits for more input.
let large = "x".repeat(9 * 1024 * 1024);
let input = futures::stream::iter([
Ok(batch("first", "ok")),
Ok(batch("first", "bad")),
Ok(batch("second", "ok")),
Ok(batch(&large, "bad")),
])
.chain(futures::stream::pending());
let data = Box::pin(RecordBatchStreamAdapter::new(arrow_schema.clone(), input));
let context = Arc::new(
TaskContext::default()
.with_session_config(SessionConfig::new().with_target_partitions(2)),
);
let sink = PaimonDataSink::new(table, arrow_schema, false);

let result =
tokio::time::timeout(Duration::from_secs(10), sink.write_all(data, &context)).await;

let error = result
let error = tokio::time::timeout(Duration::from_secs(10), sink.write_all(data, &context))
.await
.expect("INSERT still waits after the second writer failed")
.unwrap_err();
// The error comes from opening a data file under p=bad/bucket-0, so
Expand Down
2 changes: 1 addition & 1 deletion crates/integrations/datafusion/tests/native_merge.rs
Original file line number Diff line number Diff line change
Expand Up @@ -477,7 +477,7 @@ async fn failed_merge_insert_discards_prepared_cow_files() {
USING (VALUES (1, 100, 'a'), (2, 200, 'b')) AS s(id, value, pt) \
ON t.id = s.id WHEN MATCHED THEN UPDATE SET value = s.value \
WHEN NOT MATCHED THEN INSERT (id, value, pt) VALUES (s.id, s.value, s.pt)",
"Failed to create directory",
"Not a directory",
)
.await;
assert_eq!(
Expand Down
1 change: 0 additions & 1 deletion crates/paimon/src/table/data_file_writer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -317,7 +317,6 @@ impl DataFileWriter {
.new_file_name(&self.data_file_prefix, &self.file_format);
let location = self.paths.new_path(&file_name)?;
let bucket_dir = location.parent();
self.file_io.mkdirs(&format!("{bucket_dir}/")).await?;

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could we skip mkdirs() for S3 in FileIO and keep it for backends that need it? I tested a FileIOProvider backed by OpenDAL WebDAV with automatic parent creation disabled. Both append and primary-key writes to a new bucket-0 failed with HTTP 409 during prepare_commit(). The same writes passed after explicitly creating the bucket directory.

Could we also add a regression test that checks both S3 write paths produce a Parquet file without sending a directory marker PUT?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My understanding is that each operator handles parent directories as needed. E.g. FsWriter::create() calls ensure_write_abs_path(), which creates missing parent directories. The WebDAV writer does the same by default in WebdavWriter::write_once().

Is there a use case for disabling automatic directory creation when the WebDAV server requires the parent to exist, or is the concern mainly backward compatibility with providers that rely on Paimon to create it?

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Mainly backward compatibility. I don’t have a concrete use case for that WebDAV setup. I used it to reproduce what happens when a custom provider relies on Paimon to create the bucket directory.

I’m fine with leaving parent directory creation to the operator and documenting that in FileIOProvider. @JingsongLi does that match what you’d expect from FileIOProvider?


let file_path = location.path.clone();
self.created_paths.push(file_path.clone());
Expand Down
3 changes: 0 additions & 3 deletions crates/paimon/src/table/kv_data_file_writer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -200,9 +200,6 @@ impl<'a> KeyValueFileOptions<'a> {
self.file_format
);
let location = self.paths.new_path(&name)?;
self.file_io
.mkdirs(&format!("{}/", location.parent()))
.await?;
let output = self.file_io.new_output(&location.path)?;
let writer = match factory.create_writer(&output, self.file_compression).await {
Ok(writer) => with_write_resources(writer, self.resources),
Expand Down
Loading