From bd9675c66db03d0bae2bf83f623669a67867e14b Mon Sep 17 00:00:00 2001 From: Nikita Matskevich Date: Sat, 5 Sep 2026 23:40:17 +0200 Subject: [PATCH 01/32] feat: support bloom filters in native Iceberg writes --- .../user-guide/latest/iceberg-writes.md | 325 ++++++++++++++++++ .../src/execution/operators/iceberg_write.rs | 23 +- native/proto/src/proto/operator.proto | 4 + .../operator/CometIcebergNativeWrite.scala | 8 - .../IcebergWriteProtoTranslation.scala | 12 + .../comet/CometIcebergWriteActionSuite.scala | 61 ++++ .../CometIcebergWriteDetectionSuite.scala | 8 +- .../IcebergWriteProtoTranslationSuite.scala | 8 + 8 files changed, 433 insertions(+), 16 deletions(-) create mode 100644 docs/source/user-guide/latest/iceberg-writes.md diff --git a/docs/source/user-guide/latest/iceberg-writes.md b/docs/source/user-guide/latest/iceberg-writes.md new file mode 100644 index 00000000000..cdd077356d0 --- /dev/null +++ b/docs/source/user-guide/latest/iceberg-writes.md @@ -0,0 +1,325 @@ + + +# Iceberg Writes: Comet's Split-Operator Plan (Experimental) + +**This feature is experimental and disabled by default.** Enable it only after validating it +against your own workloads. + +## Overview + +Spark writes an Iceberg table through a single physical operator that combines data-file +writing with metadata writing, committing, and catalog validation. Because that operator sits +outside Spark's Adaptive Query Execution (AQE), the sub-query feeding the write — the scans, +projects, sorts, and exchanges producing the rows — cannot be re-planned at runtime. + +When `spark.comet.write.iceberg.splitOperator.enabled=true`, Comet rewrites eligible Iceberg +writes into two operators: + +1. **`IcebergWrite`** — writes the data files on the executors, exactly as iceberg-java does + today, and returns each task's serialized commit message. This operator and the sub-query + feeding it run inside AQE. +2. **`IcebergCommit`** — collects the commit messages on the driver and performs the normal + Iceberg commit (including commit-time validation), outside AQE, exactly once. + +With only the split plan enabled, data files are still written by iceberg-java; only the plan +shape changes. The split makes the write's input visible to AQE and to Comet's columnar rules, +and it is the foundation for the second toggle: when +`spark.comet.iceberg.write.enabled=true` and the write passes the eligibility check below, the +`IcebergWrite` operator's per-task Parquet write is delegated to +[iceberg-rust](https://github.com/apache/iceberg-rust) via Comet's native execution pipeline +([#5308](https://github.com/apache/datafusion-comet/issues/5308)). + +## How the native write works + +The JVM-side planner marshals everything iceberg-rust needs — the write schema and partition +spec as JSON, the data location, the resolved parquet writer settings, the writer mode +(unpartitioned / fanout / clustered, mirroring `SparkWrite`'s own choice), object-store +configuration (the table's `FileIO` properties, e.g. REST-vended credentials, merged over +`fs.s3a.*` settings translated from the session Hadoop configuration — the same translation +the native scan uses, since `HadoopFileIO` carries its S3 configuration in the Hadoop +Configuration rather than in `FileIO` properties), and per-task IDs — into the serialized +native plan. On each task, iceberg-rust writes the Parquet files and +returns its `DataFile` metadata packed as a single in-memory Iceberg V2 data manifest; the JVM +decodes those bytes with Iceberg's own `ManifestFiles.read`, re-derives each file's manifest +metrics from the written Parquet footer with Iceberg's `MetricsConfig` logic (so metrics modes, +truncation, and bounds decisions are iceberg-java's by construction), and wraps the result in +the same `TaskCommit` message the JVM writer would have produced. Everything iceberg-java does +post-write — snapshot assignment, manifest-list aggregation, commit validation and retries — +is untouched: `IcebergCommit` performs the normal `BatchWrite.commit`. + +## Configuration + +Standard Comet + Iceberg setup (see [`iceberg.md`](iceberg.md)) plus the write-side toggle: + +``` +# Standard Comet / Iceberg wiring +spark.plugins=org.apache.spark.CometPlugin +spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions +spark.sql.catalog.=org.apache.iceberg.spark.SparkCatalog +spark.sql.catalog..type=hadoop # or hive / glue / rest / ... +spark.sql.catalog..warehouse=... + +# Split-operator plan (experimental, off by default) +spark.comet.write.iceberg.splitOperator.enabled=true + +# Native-write eligibility detection (experimental, off by default; requires the split plan) +spark.comet.iceberg.write.enabled=true +``` + +## Supported operations + +The split-operator plan supports the following operations on every Spark version Comet +supports: + +- `INSERT INTO` / DataFrame `append` (`AppendData`) +- `INSERT OVERWRITE`, static and dynamic (`OverwriteByExpression`, + `OverwritePartitionsDynamic`) +- Copy-on-write `DELETE` / `UPDATE` / `MERGE` (`ReplaceData`) + +The mechanism behind row-level DML differs by Spark version: on Spark 4.0+ the analyzer emits +operation-coded rows that Comet's writer dispatches through `ReplaceData`'s projections, while +on Spark 3.4/3.5 the rewritten rows are written as a plain row stream. The supported set of +operations is the same either way. + +On Spark 4.1+ the split plan matches two further stock-Spark behaviours: MERGE metrics are +forwarded to the writer's commit (Iceberg 1.11+ records them in the snapshot summary), and +cached catalog tables are recached by name after a write so cache entries survive schema +changes. + +## When Comet falls back to Spark's write operator + +The rewrite is skipped — and the write runs through Spark's stock combined operator — when: + +- `spark.comet.write.iceberg.splitOperator.enabled` is `false` (the default); +- the write is not an Iceberg `SparkWrite` (any other V2 data source); +- the table uses merge-on-read: delta writes (Iceberg `WriteDelta`) are not intercepted; +- the statement is CTAS / RTAS on Spark 3.4, where the staged exec writes inline; on Spark + 3.5+ those statements re-plan their inner append, which is intercepted normally; +- the write requires Spark's commit coordinator, which Comet's per-task commit protocol does + not use; +- Comet cannot reflect the Iceberg internals needed to build the two-operator plan (for + example an unrecognised write class or a `ReplaceData` projection it cannot map). + +In every fallback case the write is planned as if Comet were absent; there is no correctness +trade-off, only no plan change. + +## Native Parquet write eligibility + +When `spark.comet.iceberg.write.enabled=true` +([#5308](https://github.com/apache/datafusion-comet/issues/5308)), the `IcebergWrite` operator's +per-task Parquet write is delegated to [iceberg-rust](https://github.com/apache/iceberg-rust). +The native writer must produce the same outcome as iceberg-java — the same Parquet features, +statistics, and manifest metadata — so a write is only eligible when every table property it +depends on is one the native path reproduces exactly, and additionally only when the plan +feeding the write is fully Comet-native. For a partitioned table that plan includes the hash +distribution and local sort Iceberg requests on its partition transforms; those stay native +because the transforms themselves have native implementations (see +[Iceberg system functions](iceberg.md)). Ineligible writes run through iceberg-java unchanged, +with the reason reported as a fall-back reason in Comet's extended EXPLAIN output. + +**Most Iceberg write settings are not supported.** Detection is an allowlist: a write is +eligible only when its entire effective configuration matches the table below, and anything +else — any other write-affecting property, any key added by a future Iceberg version, any +value outside the supported set, any reflection failure while inspecting the write — falls +back to iceberg-java with a reason reported in extended EXPLAIN. Checks run on the effective +configuration: table properties overlaid with `SparkWrite.writeProperties`, which is where +iceberg-java resolves per-write options and `spark.sql.iceberg.*` session overrides. + +A write is eligible only when ALL of the following hold: + +| Setting | Supported values | +| ------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| resolved write format (`write-format` option overlaid on `write.format.default`) | `parquet` | +| `format-version` | `1` or `2` | +| `write.parquet.compression-codec` / `compression-level` / `row-group-size-bytes` / `page-size-bytes` / `page-row-limit` / `dict-size-bytes` | the sizes/limit must be positive Java ints (`Integer.parseInt` semantics — no trimming, no values past `Int.MaxValue` — matching iceberg-java, whose writer fails on anything else); `compression-level` must be a Java int within the native writer's per-codec range (zstd 1–22, gzip 0–9, brotli 0–11; ignored by both writers for snappy/lz4/none) — iceberg-java never validates the level, so an out-of-range value falls back rather than becoming a native task failure | +| `write.parquet.row-group-check-min-record-count` | unset or `100` (the default) | +| `write.parquet.row-group-check-max-record-count` | unset or `10000` (the default) | +| `write.parquet.page-version` | unset or `v1` | +| `write.parquet.shred-variants` | unset or `false` (Spark 4.x / Iceberg 1.11 resolve this into every parquet write) | +| `write.parquet.variant-inference-buffer-size` | any value (only meaningful when shredding, which is gated) | +| `write.parquet.bloom-filter-enabled.column.` | `true` or `false`; bloom filters are written only for columns set to `true` | +| `write.metadata.metrics.*` | any value (manifest metrics are re-derived on the JVM with Iceberg's own logic) | +| `write.spark.fanout.enabled` | any value (the native writer implements both clustered and fanout modes) | +| `write.target-file-size-bytes` | any value (the two writers can choose different roll points; see accepted divergences) | +| data location URI scheme | `file`, `memory`, `s3`, `s3a`, `gs` | +| partition spec | any | +| column types | any except `uuid` (Spark plans it as a string; no Arrow cast reaches `fixed(16)`) | + +Within the namespaces that shape data-file bytes — `write.parquet.*` and `parquet.*` — +everything not listed above must be absent: unvetted `write.parquet.*` keys (e.g. +`bloom-filter-max-bytes`, `stats-enabled.column.*`, keys added by future Iceberg versions), +any `parquet.*` table property (including `parquet.enable.dictionary`), and any `parquet.*` +key in the session Hadoop configuration (with `HadoopFileIO`-backed output those reach +iceberg-java's writer but not the native one). Also gated explicitly: any `encryption.*` key, +`write.object-storage.enabled=true`, `write.location-provider.impl`, and `io-impl`. + +Two checks look past properties at the table's instantiated state, because both can be +configured at the catalog level (or by a custom `TableOperations`) without any table or write +property changing: `table.io()` must be a recognized `FileIO` (the same allowlist the native +scan uses, minus the `EncryptingFileIO` family — the native writer produces plaintext files, +so an encrypting `FileIO` is rejected on the write side), and `table.encryption()` must be +Iceberg's `PlaintextEncryptionManager`. Anything else falls back. + +Other `write.*` properties are intentionally not gated because they cannot make the native +writer produce different data files: distribution and ordering settings shape the Spark plan +identically on both paths, WAP / branch / snapshot properties act on the JVM committer, +`write.avro.*` / `write.orc.*` apply only to formats already excluded, and merge-on-read +settings route the write through `WriteDelta`, which the split plan never intercepts. Every +rule is pinned by `CometIcebergWriteDetectionSuite`. + +Manifest `DataFile` metrics are assembled on the JVM before commit: each written file's +metrics are re-derived from its parquet footer through the version-matched +`ParquetUtil.footerMetrics` and `MetricsConfig.forTable`, with float/double NaN counts and +bounds carried over from the native writer's tracked state. iceberg-java's metadata decisions +— metrics modes, the inferred-column cap +(`write.metadata.metrics.max-inferred-column-defaults`), bound truncation, and list/map bounds +suppression — are therefore applied by iceberg-java's own code regardless of what the native +writer reports. This costs one footer-sized ranged read per written file at write time. + +## Failure handling + +Eligibility is decided entirely at plan time. That includes the reflection surface: every +iceberg-java class, method, and constructor the executor-side commit-message assembly uses is +eagerly resolved by the eligibility gate on the driver, so an Iceberg release that moves any +of them declines the native path with a fall-back reason instead of failing tasks mid-write. +Once planned, the physical plan is fixed — there is no per-task re-decision or runtime switch +back to the JVM writer. + +When a native write fails partway through a task (an object-store error, a data-dependent cast +failure), the error propagates as an ordinary Spark task failure and Spark's task retry +re-executes it — through the native writer again. Retries cannot collide: each attempt's task +attempt id is embedded in its data file names. + +Partial results are never committed. The commit set is exactly the commit messages returned by +successful tasks — a failed task contributes none — and if the job fails, the driver-side +commit operator aborts without committing anything. A failed task attempt also deletes the +data files it created, as iceberg-java's writer abort does. The native writer records every +location it hands to a file writer, and exactly one side owns deleting them at any moment: the +native writer owns them until its output batch reaches the JVM (so it cleans up a failed write, +a task torn down before the write completed — for example because the operator feeding it threw +— and a failure encoding the manifest or building that batch), and the JVM owns them from then +on through a task failure listener. The handoff does not depend on decoding the manifest: the +native side reports the locations in the output batch alongside it, and the listener is handed +them before the manifest is decoded, so a failure in that decode still cleans up. Both +deletions are best-effort and never mask the original failure; anything they miss is invisible +to every reader, since readers resolve files through committed manifests only, and is reclaimed +by Iceberg's normal `remove_orphan_files` maintenance. + +When one task fails, the tasks that had already completed leave committed-nothing data files +too. The committer collects each task's commit message as that task finishes, so on a job +failure it aborts with the completed messages and deletes their data files through the table +`FileIO`. (Iceberg's own `SparkWrite.abort` skips cleanup unless a commit failed with a +cleanable error, so on the stock path those files are left for `remove_orphan_files`.) A +failure during the driver-side commit itself behaves exactly as on the stock path: the commit +messages carry genuine `SparkWrite$TaskCommit` objects, so Iceberg's own `SparkWrite.abort` +cleanup (which deletes the files listed in the commit messages for cleanable failures) applies +unchanged. + +## Accepted divergences behind the toggle + +Some differences between parquet-mr and the pinned parquet-rs / iceberg-rust are unconditional — +they apply to every native write and cannot be configured away. Enabling +`spark.comet.iceberg.write.enabled` accepts them. They fall into three classes with very +different blast radius: differences confined to the physical bytes of a data file (cosmetic — +no reader decision is based on them), differences visible in manifest metadata (these outlive +the write and feed later readers' pruning decisions, so each one is analyzed individually +below), and one operational path-layout caveat. + +### Physical file layout only (cosmetic) + +No Iceberg reader bases a planning or correctness decision on these; they change the bytes of +a data file but not what any reader computes from it: + +- Footer key-value metadata differs: native files carry an `ARROW:schema` entry and no + `iceberg.schema` entry; iceberg-java files are the opposite. +- The Parquet root schema element is named `arrow_schema` (iceberg-java: `table`). +- `created_by` identifies parquet-rs, not parquet-mr. +- No page CRC checksums and no page-header statistics (parquet-mr writes both by default). + Absent page-header statistics can only make a reader scan more pages, never skip pages it + should have read; page pruning uses the column index, which the native writer does produce. +- Dictionary-encoded pages are labeled `RLE_DICTIONARY` (parquet-mr v1 files: `PLAIN_DICTIONARY`). +- Fixed-length binary columns (`uuid`, `fixed`, decimals with precision > 18) are not + dictionary-encoded (parquet-mr dictionary-encodes them). +- Row-group boundaries: parquet-mr flushes by byte size at a record-count check cadence, + parquet-rs buffers by row count. File naming follows the same cadence-style difference + (iceberg-java names files `---`; iceberg-rust uses a + process-local counter). +- Partition directory names match iceberg-java 1.8+'s `PartitionSpec.partitionToPath` for every + partition type except `float` and `double`, where the value is rendered with Rust's shortest + representation instead of `Float.toString` / `Double.toString` (`f=1` where iceberg-java writes + `f=1.0`). On Iceberg 1.5.x, which the Spark 3.4 profile pins, iceberg-java itself spelled + `timestamp` and `timestamptz` directories with `LocalDateTime.toString()` / + `OffsetDateTime.toString()` (`ts=1969-12-31T23:59:58.500Z`) and left the partition field name + unescaped; Comet uses the 1.8+ spelling on every profile. Distinct partition values still get + distinct directories in all cases, and no reader parses these names — files are resolved through + committed manifests. Iceberg deprecated float and double partitioning in 1.3. +- File rolling lands on the same row grid as iceberg-java but not necessarily on the same row. + Both writers re-check the current file's size against `write.target-file-size-bytes` once + every 1000 rows of that file (iceberg-java's `RollingFileWriter.ROWS_DIVISOR`; Comet hands the + iceberg-rust writer rows in 1000-row units, per partition file, to get the same grid), so each + writer rolls only on a 1000-row boundary of its own file. + The shared grid is all that is shared. What each writer compares against the target differs — + flushed bytes plus parquet-rs's estimate of the open row group, versus parquet-mr's file + position plus its buffered size — and the two use different threshold comparisons. These are + independent size estimates, so nothing bounds how far apart the two writers' roll points are: + they may cross the target several grid steps apart, and the resulting files can differ in row + count by an arbitrary number of 1000-row blocks. Do not rely on file-layout parity between the + two writers; rely only on each file rolling on its own 1000-row boundary. +- A fanout write lists a task's data files in file-path order, where iceberg-java lists them in + its own `StructLikeMap` iteration order. Both are stable across runs, and neither is a + documented ordering, but the manifest entry order becomes the scan-task order and so the row + order of an unordered `SELECT *`. Only the sorted order is reproducible on the native path: + iceberg-rust's `FanoutWriter` closes its per-partition writers out of a `HashMap`, which under + Rust's per-process `RandomState` would otherwise give a different order on every run. Clustered + and unpartitioned writes append in creation order on both paths and are unaffected. +- Compressed page bytes are implementation-defined: the codec and any explicit level are + translated, but parquet-rs and parquet-mr embed different encoder implementations and + defaults (zstd default levels, LZ4 framing), so byte-identical output is not achievable even + for a default `zstd` table. The decompressed data is identical. For the same reason, + codec-level side channels (`zlib.compress.level`, `compression.brotli.quality`, + `io.compression.codec.zstd.level` — the last is present in every Hadoop configuration by + default) are not gated: they can only shift compressed bytes, which are already accepted as + divergent. + +### Manifest metadata visible to later readers + +Manifest metrics drive partition- and file-level pruning for every future reader of the table, +so a divergence here would outlive the write. This class is deliberately kept almost empty: +`DataFile` metrics are not taken from the native writer's manifest but re-derived on the JVM +from each written file's parquet footer through iceberg-java's own `ParquetUtil.footerMetrics` +and `MetricsConfig.forTable` (see above). Metrics modes, lower/upper bound truncation, the +null-count conventions, and list/map bounds suppression are therefore iceberg-java's code +making iceberg-java's decisions, and the parity suite compares committed manifests +byte-for-byte against JVM-written ones. Two footer-derived values can still differ from what +iceberg-java's _writer-tracked_ state would have recorded, and both are analyzed safe: + +- Float/double bounds involving zero may differ in sign: parquet-rs normalises footer + statistics to min `-0.0` / max `+0.0` (the parquet-format recommendation), while + iceberg-java's writer-tracked bounds preserve the exact sign it saw. The native path's + manifest bounds inherit the normalised values — a strictly conservative widening that cannot + change pruning decisions. +- On Iceberg 1.10+, manifest `value_counts` / `null_value_counts` for float/double columns + nested under a nullable struct count rows whose parent struct is null (they come from the + parquet footer), while iceberg-java's writer-tracked counts do not. Both counts inflate by + the same amount, so the derived null ratios and `IS NULL` / `IS NOT NULL` pruning decisions + are unaffected. + +All content not listed above — the logical data, encodings for non-FLBA columns, statistics +values, and manifest metadata — must match iceberg-java exactly, or the write falls back. diff --git a/native/core/src/execution/operators/iceberg_write.rs b/native/core/src/execution/operators/iceberg_write.rs index c949b95cf66..6e8d45af587 100644 --- a/native/core/src/execution/operators/iceberg_write.rs +++ b/native/core/src/execution/operators/iceberg_write.rs @@ -2299,7 +2299,7 @@ fn build_output_batch( /// the JVM re-derives from the footer with Iceberg's own `MetricsConfig` logic before commit. fn build_writer_properties(settings: &IcebergParquetWriteSettings) -> DFResult { let compression = compression_from_proto(settings.compression, settings.compression_level)?; - Ok(WriterProperties::builder() + let mut builder = WriterProperties::builder() .set_compression(compression) .set_created_by(settings.created_by.clone()) .set_max_row_group_bytes(Some(settings.row_group_size_bytes as usize)) @@ -2310,8 +2310,11 @@ fn build_writer_properties(settings: &IcebergParquetWriteSettings) -> DFResult) -> DFResult { @@ -2481,6 +2484,7 @@ mod tests { dict_size_bytes: 2 * 1024 * 1024, page_row_limit: 20_000, created_by: "Apache Iceberg (Comet test)".to_string(), + bloom_filter_enabled_columns: Vec::new(), } } @@ -2560,6 +2564,18 @@ mod tests { assert_eq!(props.created_by(), "Apache Iceberg 1.7.1 (Comet 0.16.0)"); } + #[test] + fn enables_bloom_filters_only_for_configured_columns() { + let mut settings = base_settings(); + settings.bloom_filter_enabled_columns = vec!["id".to_string(), "nested.value".to_string()]; + let props = build_writer_properties(&settings).unwrap(); + assert!(props.bloom_filter_properties(&"id".into()).is_some()); + assert!(props + .bloom_filter_properties(&"nested.value".into()) + .is_some()); + assert!(props.bloom_filter_properties(&"other".into()).is_none()); + } + #[test] fn rejects_unknown_codec() { let mut settings = base_settings(); @@ -3035,6 +3051,7 @@ mod tests { dict_size_bytes: 2 * 1024 * 1024, page_row_limit: 20_000, created_by: "Apache Iceberg (Comet integration test)".to_string(), + bloom_filter_enabled_columns: Vec::new(), }; Arc::new(IcebergWriteCommon { catalog_properties: HashMap::new(), diff --git a/native/proto/src/proto/operator.proto b/native/proto/src/proto/operator.proto index 8e720a8897c..3c4c6ca12eb 100644 --- a/native/proto/src/proto/operator.proto +++ b/native/proto/src/proto/operator.proto @@ -807,6 +807,10 @@ message IcebergParquetWriteSettings { // String written into parquet file metadata. JVM-side default is // `"Apache Iceberg (Comet)"`. string created_by = 7; + // Iceberg columns whose `write.parquet.bloom-filter-enabled.column.` property resolves to + // true. Entries use Iceberg's dotted column-path syntax and are translated to parquet-rs + // `ColumnPath`s by the native writer. + repeated string bloom_filter_enabled_columns = 8; } // Broadcast payload -- one of these per write, identical for every task. diff --git a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala index b385668291d..6d96fba5d07 100644 --- a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala +++ b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala @@ -417,7 +417,6 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { requireSupportedColumnTypes, requireNoVoidFieldWithDroppedSource, requireNoEncryptionPrefix, - requireNoBloomFilterColumnsEnabled, requireRowGroupCheckMinRecordCountAtDefault, requireRowGroupCheckMaxRecordCountAtDefault, requireParquetPageVersionDefault, @@ -563,13 +562,6 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { // `CometIcebergWriteExec`), so every `write.metadata.metrics.*` value behaves exactly as it // does on the iceberg-java path. - private val requireNoBloomFilterColumnsEnabled: TriggerRule = ctx => { - val prefix = PropertyKeys.BloomFilterColumnEnabledPrefix - ctx.properties - .find { case (k, v) => k.startsWith(prefix) && v.equalsIgnoreCase("true") } - .map { case (k, _) => s"$k=true: bloom filters unsupported" } - } - private val requireParquetPageVersionDefault: TriggerRule = ctx => { val key = PropertyKeys.ParquetPageVersion ctx.properties diff --git a/spark/src/main/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslation.scala b/spark/src/main/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslation.scala index 531f3c227dd..c2de086bd8d 100644 --- a/spark/src/main/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslation.scala +++ b/spark/src/main/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslation.scala @@ -57,6 +57,8 @@ object IcebergWriteProtoTranslation { IcebergReflection.tablePropertyConstant("PARQUET_PAGE_ROW_LIMIT") lazy val ParquetDictSizeBytes: String = IcebergReflection.tablePropertyConstant("PARQUET_DICT_SIZE_BYTES") + lazy val ParquetBloomFilterColumnEnabledPrefix: String = + IcebergReflection.tablePropertyConstant("PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX") } /** Iceberg's numeric defaults, pulled at runtime so they stay in lock-step with the runtime. */ @@ -112,6 +114,15 @@ object IcebergWriteProtoTranslation { parseJavaInt(props, Keys.ParquetDictSizeBytes, Defaults.DictSizeBytes.toInt).toLong val pageRowLimit = parseJavaInt(props, Keys.ParquetPageRowLimit, Defaults.PageRowLimit) val compression = resolveCompression(props) + val bloomFilterEnabledColumns = props.iterator + .collect { + case (key, value) + if key.startsWith(Keys.ParquetBloomFilterColumnEnabledPrefix) && + value.equalsIgnoreCase("true") => + key.substring(Keys.ParquetBloomFilterColumnEnabledPrefix.length) + } + .toSeq + .sorted val builder = IcebergParquetWriteSettings .newBuilder() .setCompression(compression) @@ -120,6 +131,7 @@ object IcebergWriteProtoTranslation { .setDictSizeBytes(dictSize) .setPageRowLimit(pageRowLimit) .setCreatedBy(createdBy) + .addAllBloomFilterEnabledColumns(bloomFilterEnabledColumns.asJava) resolveCompressionLevel(props, compression).foreach(builder.setCompressionLevel) diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala index 6f752267a63..bccf0826644 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala @@ -1241,6 +1241,28 @@ class CometIcebergWriteActionSuite } } + test("native acceleration: writes configured Iceberg Parquet bloom filters") { + assumeNativeAcceleration() + withIcebergCatalog { warehouseDir => + createTable( + warehouseDir, + "native_bloom", + partitionSpec = "", + properties = Some("'write.parquet.bloom-filter-enabled.column.id'='true'")) + + val expectedIds = 0 until 256 + assertNativeWriteEngages("native_bloom", expectedIds) { + spark.sql( + "INSERT INTO cat.db.native_bloom " + + "SELECT CAST(id AS INT), CONCAT('region-', CAST(id % 4 AS STRING)), " + + "CAST(id AS DOUBLE) FROM range(256)") + } + + assertParquetBloomFilters("native_bloom", enabledColumns = Set("id")) + + } + } + // What Iceberg's Spark writer stamps for `sort_order_id` on appended files changed across // releases: through 1.10 `SparkWrite$WriterFactory` never wires the table sort order (files // get 0 even on a sorted table); 1.11 added `SparkWriteConf.outputSortOrderId` and stamps the @@ -4821,6 +4843,45 @@ class CometIcebergWriteActionSuite assert(ids == expectedIds, s"expected $expectedIds, got $ids") } + /** + * Reads every current data-file footer and verifies that parquet-rs emitted bloom-filter data + * exactly for the configured columns. Checking the plan alone would not catch a translation bug + * that selected `CometIcebergWriteExec` but silently omitted the bloom writer properties. + */ + private def assertParquetBloomFilters(tableName: String, enabledColumns: Set[String]): Unit = { + val paths = spark + .sql(s"SELECT file_path FROM $catalog.$ns.$tableName.data_files") + .collect() + .map(_.getString(0)) + .toSeq + assert(paths.nonEmpty, s"expected $tableName to have at least one data file") + + val conf = spark.sparkContext.hadoopConfiguration + paths.foreach { path => + val input = HadoopInputFile.fromPath(new HadoopPath(path.stripPrefix("file:")), conf) + val reader = ParquetFileReader.open(input) + try { + val columns = reader.getFooter.getBlocks.asScala.flatMap(_.getColumns.asScala) + assert(columns.nonEmpty, s"expected at least one column chunk in $path") + columns.foreach { column => + val columnName = column.getPath.toDotString + val bloomFilter = reader.readBloomFilter(column) + if (enabledColumns.contains(columnName)) { + assert( + bloomFilter != null && bloomFilter.getBitsetSize > 0, + s"expected a non-empty bloom filter for $columnName in $path") + } else { + assert( + bloomFilter == null, + s"expected no bloom filter for unconfigured column $columnName in $path") + } + } + } finally { + reader.close() + } + } + } + /** Native acceleration shared assumption -- currently just the Iceberg-on-classpath check. */ private def assumeNativeAcceleration(): Unit = { assume(icebergAvailable, "Iceberg not available in classpath") diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala index a102d142c83..95d5688f737 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala @@ -387,16 +387,14 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes } } - test("fall-back: per-column bloom filter enabled") { + test("Compatible when a per-column bloom filter is enabled") { withDetectionCatalog { _ => + createTable( "bloom_col", partitionSpec = "", properties = Some("'write.parquet.bloom-filter-enabled.column.id'='true'")) - assertUnsupportedContains( - "bloom_col", - "write.parquet.bloom-filter-enabled.column.id", - "true") + assertSupportLevelIs[Compatible]("bloom_col") } } diff --git a/spark/src/test/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslationSuite.scala b/spark/src/test/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslationSuite.scala index f1dd09ccf7e..77795b46968 100644 --- a/spark/src/test/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslationSuite.scala +++ b/spark/src/test/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslationSuite.scala @@ -176,6 +176,14 @@ class IcebergWriteProtoTranslationSuite extends AnyFunSuite { assert(settings.getPageRowLimit == 1000) } + test("per-column bloom filter properties are translated deterministically") { + val prefix = Keys.ParquetBloomFilterColumnEnabledPrefix + val settings = buildParquetSettings( + Map(s"${prefix}region" -> "TRUE", s"${prefix}id" -> "true", s"${prefix}amount" -> "false"), + TestCreatedBy) + assert(settings.getBloomFilterEnabledColumnsList == java.util.Arrays.asList("id", "region")) + } + test("size properties are parsed with Java Integer.parseInt semantics") { // No trimming and no values past Int.MaxValue -- exactly what iceberg-java's // PropertyUtil.propertyAsInt would do. The eligibility gate declines these values From e24f5bd3cdcac83d302bd691da11538adb5ca99a Mon Sep 17 00:00:00 2001 From: Nikita Matskevich Date: Sun, 6 Sep 2026 19:47:09 +0200 Subject: [PATCH 02/32] feat: support Iceberg bloom filter write settings --- .../user-guide/latest/iceberg-writes.md | 42 +- .../src/execution/operators/iceberg_write.rs | 241 ++++++++++- native/proto/src/proto/operator.proto | 15 +- .../comet/iceberg/IcebergReflection.scala | 53 +++ .../operator/CometIcebergNativeWrite.scala | 171 +++++++- .../IcebergWriteProtoTranslation.scala | 85 +++- .../comet/CometIcebergWriteActionSuite.scala | 402 +++++++++++++++++- .../CometIcebergWriteDetectionSuite.scala | 154 ++++++- .../IcebergWriteProtoTranslationSuite.scala | 67 ++- 9 files changed, 1198 insertions(+), 32 deletions(-) diff --git a/docs/source/user-guide/latest/iceberg-writes.md b/docs/source/user-guide/latest/iceberg-writes.md index cdd077356d0..13d0cf74116 100644 --- a/docs/source/user-guide/latest/iceberg-writes.md +++ b/docs/source/user-guide/latest/iceberg-writes.md @@ -154,7 +154,9 @@ A write is eligible only when ALL of the following hold: | `write.parquet.page-version` | unset or `v1` | | `write.parquet.shred-variants` | unset or `false` (Spark 4.x / Iceberg 1.11 resolve this into every parquet write) | | `write.parquet.variant-inference-buffer-size` | any value (only meaningful when shredding, which is gated) | -| `write.parquet.bloom-filter-enabled.column.` | `true` or `false`; bloom filters are written only for columns set to `true` | +| `write.parquet.bloom-filter-enabled.column.` | `true` or `false`; an explicit NDV enables the column even when this value is `false`, matching Iceberg's property application order | +| `write.parquet.bloom-filter-fpp.column.` / `write.parquet.bloom-filter-ndv.column.` | For every column named by an `enabled` property, FPP must be a finite double strictly between 0 and 1 and NDV must be a positive Java long no greater than `Long.MAX_VALUE / 8`; the Iceberg FPP default is `0.01` | +| `write.parquet.bloom-filter-max-bytes` | unset (Iceberg default: 1 MiB), or a power of two from 32 bytes through 128 MiB inclusive; every other value falls back to iceberg-java. A 32-byte value also falls back when explicit NDV/FPP would request a larger filter, because Parquet Java ignores that boundary as a maximum. | | `write.metadata.metrics.*` | any value (manifest metrics are re-derived on the JVM with Iceberg's own logic) | | `write.spark.fanout.enabled` | any value (the native writer implements both clustered and fanout modes) | | `write.target-file-size-bytes` | any value (the two writers can choose different roll points; see accepted divergences) | @@ -164,7 +166,7 @@ A write is eligible only when ALL of the following hold: Within the namespaces that shape data-file bytes — `write.parquet.*` and `parquet.*` — everything not listed above must be absent: unvetted `write.parquet.*` keys (e.g. -`bloom-filter-max-bytes`, `stats-enabled.column.*`, keys added by future Iceberg versions), +`stats-enabled.column.*`, keys added by future Iceberg versions), any `parquet.*` table property (including `parquet.enable.dictionary`), and any `parquet.*` key in the session Hadoop configuration (with `HadoopFileIO`-backed output those reach iceberg-java's writer but not the native one). Also gated explicitly: any `encryption.*` key, @@ -242,6 +244,42 @@ no reader decision is based on them), differences visible in manifest metadata ( the write and feed later readers' pruning decisions, so each one is analyzed individually below), and one operational path-layout caveat. +### Parquet Bloom-filter sizing + +[Iceberg's documented write properties](https://iceberg.apache.org/docs/latest/configuration/#write-properties) +describe three related inputs. FPP is the requested false-positive probability (default `0.01`), +NDV is the expected number of distinct values when explicitly set, and `max-bytes` is an upper +bound (default 1 MiB). + +Apache Parquet Java permits arbitrary integer caps. When such a cap binds, it serializes exactly +that many bytes, although only complete 32-byte SBBF blocks are used and any trailing partial +block remains zero. The Apache Arrow Rust `parquet` crate requires a power-of-two block count so +its post-write folding remains valid. A non-power-of-two cap can therefore change the +hash-to-block mapping, making a filter that may have worse reader pruning than Parquet Java's +filter. + +Comet uses its native Iceberg writer only when the effective `max-bytes` value is a power of two +from 32 bytes through 128 MiB inclusive. If an explicit value is not a power of two or is outside +that range, `CometIcebergWriteExec` is not used for the write; Spark's default Iceberg Java writer +writes the table instead. The same fallback applies when `max-bytes=32` would bind an explicit +NDV/FPP request, because Parquet Java ignores exactly 32 bytes as a maximum, and when NDV is above +`Long.MAX_VALUE / 8`, where Parquet Java's sizing multiplication can overflow. + +For supported values, Apache Parquet Java applies the sizing properties as follows: + +- with no NDV, allocate the full `max-bytes` value; +- with an NDV, calculate a requested size from NDV and FPP, then cap it at `max-bytes`; +- when the cap binds, it takes precedence, so the requested FPP is not guaranteed; +- a large maximum never enlarges the allocation selected by an explicit, smaller NDV. + +For every write that is eligible for the native path, Comet applies exactly the same allocation +decision algorithm. + +After values are inserted, the Apache Arrow Rust `parquet` crate may fold a sparsely populated +filter to a smaller power-of-two filter while preserving the requested FPP. Parquet Java's +non-adaptive Iceberg path keeps its initial allocation. The native result can consequently use +less file space while remaining safe for every Parquet reader. + ### Physical file layout only (cosmetic) No Iceberg reader bases a planning or correctness decision on these; they change the bytes of diff --git a/native/core/src/execution/operators/iceberg_write.rs b/native/core/src/execution/operators/iceberg_write.rs index 6e8d45af587..d3a18e6270c 100644 --- a/native/core/src/execution/operators/iceberg_write.rs +++ b/native/core/src/execution/operators/iceberg_write.rs @@ -78,6 +78,7 @@ use iceberg::ErrorKind; use parquet::arrow::PARQUET_FIELD_ID_META_KEY; use parquet::basic::{BrotliLevel, Compression, GzipLevel, ZstdLevel}; use parquet::file::properties::{EnabledStatistics, WriterProperties}; +use parquet::schema::types::ColumnPath; use serde::{Deserialize, Serialize}; use datafusion_comet_proto::spark_operator::{ @@ -2312,11 +2313,146 @@ fn build_writer_properties(settings: &IcebergParquetWriteSettings) -> DFResult ColumnPath { + ColumnPath::from(path.split('.').map(str::to_owned).collect::>()) +} + +// Match Apache Parquet Java's BlockSplitBloomFilter implementation bounds: +// https://github.com/apache/parquet-java/blob/78a8d3230eb4769db93de5f2f2e18363c04cae81/parquet-column/src/main/java/org/apache/parquet/column/values/bloomfilter/BlockSplitBloomFilter.java#L40-L50 +const BLOOM_FILTER_MIN_BYTES: usize = 32; +const BLOOM_FILTER_MAX_BYTES: usize = 128 * 1024 * 1024; +const BLOOM_FILTER_HASH_PROBES: f64 = 8.0; +const ICEBERG_DEFAULT_BLOOM_FILTER_FPP: f64 = 0.01; +#[cfg(test)] +const ICEBERG_DEFAULT_BLOOM_FILTER_MAX_BYTES: usize = 1024 * 1024; + +/// The positive denominator obtained by solving the Bloom-filter false-positive equation +/// `fpp = (1 - exp(-k * ndv / bits))^k` for `bits`, with the Parquet SBBF's `k = 8` probes. +/// +/// See the Apache Arrow Rust `parquet` implementation and its cited paper: +/// https://github.com/apache/arrow-rs/blob/58.4.0/parquet/src/bloom_filter/mod.rs#L369-L376 +/// http://algo2.iti.kit.edu/documents/cacheefficientbloomfilters-jea.pdf +fn bloom_filter_fpp_denominator(fpp: f64) -> f64 { + -(1.0 - fpp.powf(1.0 / BLOOM_FILTER_HASH_PROBES)).ln() +} + +/// Reproduce parquet-mr's non-adaptive allocation decision before translating the resulting +/// power-of-two byte size into parquet-rs's NDV-shaped API. An absent NDV requests the full cap; +/// an explicit NDV sizes from NDV/FPP and then applies the cap. The native eligibility gate only +/// admits representable power-of-two caps. +/// +/// Apache Parquet Java implementation: +/// https://github.com/apache/parquet-java/blob/78a8d3230eb4769db93de5f2f2e18363c04cae81/parquet-column/src/main/java/org/apache/parquet/column/values/bloomfilter/BlockSplitBloomFilter.java#L277-L301 +/// https://github.com/apache/parquet-java/blob/78a8d3230eb4769db93de5f2f2e18363c04cae81/parquet-column/src/main/java/org/apache/parquet/column/values/bloomfilter/BlockSplitBloomFilter.java#L195-L218 +fn parquet_mr_bloom_filter_bytes(ndv: Option, fpp: f64, max_bytes: usize) -> usize { + let Some(ndv) = ndv else { + return max_bytes; + }; + + let calculated = BLOOM_FILTER_HASH_PROBES * ndv as f64 / bloom_filter_fpp_denominator(fpp); + let mut num_bits = calculated as i32; + let upper_bits = (BLOOM_FILTER_MAX_BYTES * 8) as i32; + if num_bits > upper_bits || calculated < 0.0 { + num_bits = upper_bits; + } + // This deliberately mirrors parquet-mr 1.17's integer expression, including its unusual + // mask, so allocation thresholds remain compatible rather than merely mathematically close. + num_bits = (num_bits + 255) & !256; + num_bits = num_bits.max((BLOOM_FILTER_MIN_BYTES * 8) as i32); + let requested = (num_bits as usize) / 8; + let allocated = requested + .clamp(BLOOM_FILTER_MIN_BYTES, BLOOM_FILTER_MAX_BYTES) + .next_power_of_two(); + // The eligibility gate excludes max=32 when this cap would change parquet-mr's allocation. + allocated.min(max_bytes) +} + +/// Mirror the NDV/FPP sizing and power-of-two allocation used by the Apache Arrow Rust `parquet` +/// crate. Its source derives the formula from the standard Bloom-filter false-positive equation +/// with eight hash probes and links the underlying cache-efficient Bloom-filter paper: +/// https://github.com/apache/arrow-rs/blob/58.4.0/parquet/src/bloom_filter/mod.rs#L363-L395 +fn parquet_rs_bloom_filter_bytes(ndv: u64, fpp: f64) -> usize { + let num_bits = + (BLOOM_FILTER_HASH_PROBES * ndv as f64 / bloom_filter_fpp_denominator(fpp)) as usize; + (num_bits / 8) + .clamp(BLOOM_FILTER_MIN_BYTES, BLOOM_FILTER_MAX_BYTES) + .next_power_of_two() +} + +/// Encode an exact power-of-two allocation using parquet-rs 58.x's public NDV/FPP setters. +/// +/// A target `B > 32` is selected by every raw byte count in `(B/2, B]`. Aim at `3B/4`, far from +/// either floating-point boundary, and verify using the exact parquet-rs sizing expression. The +/// binary-search fallback covers unusual but still representable FPP values without relying on +/// the inverse formula landing on a particular floating-point integer. +fn synthetic_ndv_for_bloom_filter_bytes(target_bytes: usize, fpp: f64) -> DFResult { + let fpp_denominator = bloom_filter_fpp_denominator(fpp); + // Any raw size in (B / 2, B] rounds up to the target power-of-two allocation B. Choose the + // midpoint of that interval to stay away from floating-point boundaries at either end. + let raw_target_bytes = target_bytes as f64 * 3.0 / 4.0; + let candidate = ((raw_target_bytes * fpp_denominator).round() as u64).max(1); + if parquet_rs_bloom_filter_bytes(candidate, fpp) == target_bytes { + return Ok(candidate); + } + + // Rust's standard binary-search helpers operate on materialized slices; this is a lower-bound + // search over the implicit NDV domain `1..=u64::MAX`, so keep the numeric search explicit. + let mut low = 1_u64; + let mut high = u64::MAX; + while low < high { + let mid = low + (high - low) / 2; + if parquet_rs_bloom_filter_bytes(mid, fpp) < target_bytes { + low = mid.saturating_add(1); + } else { + high = mid; + } + } + if parquet_rs_bloom_filter_bytes(low, fpp) == target_bytes { + Ok(low) + } else { + Err(DataFusionError::Internal(format!( + "FPP {fpp} cannot represent a {target_bytes}-byte parquet-rs Bloom filter" + ))) + } +} + +fn validate_bloom_filter_inputs(fpp: f64, bytes: usize) -> DFResult<()> { + if !fpp.is_finite() || !(0.0..1.0).contains(&fpp) { + return Err(DataFusionError::Internal(format!( + "Bloom filter FPP must be finite and strictly between 0 and 1, got {fpp}" + ))); + } + if !(BLOOM_FILTER_MIN_BYTES..=BLOOM_FILTER_MAX_BYTES).contains(&bytes) + || !bytes.is_power_of_two() + { + return Err(DataFusionError::Internal(format!( + "Bloom filter byte size must be a power of two in [{BLOOM_FILTER_MIN_BYTES}, {BLOOM_FILTER_MAX_BYTES}], got {bytes}" + ))); + } + Ok(()) +} + fn compression_from_proto(codec: i32, level: Option) -> DFResult { let codec = ProtoCompressionCodec::try_from(codec).map_err(|_| { DataFusionError::Internal(format!("Unknown CompressionCodec proto value: {codec}")) @@ -2485,6 +2621,9 @@ mod tests { page_row_limit: 20_000, created_by: "Apache Iceberg (Comet test)".to_string(), bloom_filter_enabled_columns: Vec::new(), + bloom_filter_max_bytes: ICEBERG_DEFAULT_BLOOM_FILTER_MAX_BYTES as u64, + bloom_filter_fpp_by_column: Default::default(), + bloom_filter_ndv_by_column: Default::default(), } } @@ -2567,13 +2706,102 @@ mod tests { #[test] fn enables_bloom_filters_only_for_configured_columns() { let mut settings = base_settings(); - settings.bloom_filter_enabled_columns = vec!["id".to_string(), "nested.value".to_string()]; + settings.bloom_filter_enabled_columns = vec![ + "id".to_string(), + "tags.list.element".to_string(), + "attrs.key_value.value".to_string(), + ]; let props = build_writer_properties(&settings).unwrap(); - assert!(props.bloom_filter_properties(&"id".into()).is_some()); assert!(props - .bloom_filter_properties(&"nested.value".into()) + .bloom_filter_properties(&parquet_column_path("id")) + .is_some()); + assert!(props + .bloom_filter_properties(&parquet_column_path("tags.list.element")) .is_some()); - assert!(props.bloom_filter_properties(&"other".into()).is_none()); + assert!(props + .bloom_filter_properties(&parquet_column_path("attrs.key_value.value")) + .is_some()); + assert!(props + .bloom_filter_properties(&parquet_column_path("other")) + .is_none()); + } + + #[test] + fn bloom_filter_defaults_match_iceberg() { + // Iceberg's documented write-property defaults: + // https://iceberg.apache.org/docs/latest/configuration/#write-properties + let mut settings = base_settings(); + settings.bloom_filter_enabled_columns = vec!["id".to_string()]; + let props = build_writer_properties(&settings).unwrap(); + let bloom = props.bloom_filter_properties(&"id".into()).unwrap(); + assert_eq!(bloom.fpp, ICEBERG_DEFAULT_BLOOM_FILTER_FPP); + assert_eq!( + parquet_rs_bloom_filter_bytes(bloom.ndv, bloom.fpp), + ICEBERG_DEFAULT_BLOOM_FILTER_MAX_BYTES + ); + } + + #[test] + fn synthetic_ndv_hits_every_supported_size_away_from_float_boundaries() { + for fpp in [0.0001, ICEBERG_DEFAULT_BLOOM_FILTER_FPP, 0.05, 0.5, 0.99] { + let mut bytes = BLOOM_FILTER_MIN_BYTES; + while bytes <= BLOOM_FILTER_MAX_BYTES { + let ndv = synthetic_ndv_for_bloom_filter_bytes(bytes, fpp).unwrap(); + assert_eq!(parquet_rs_bloom_filter_bytes(ndv, fpp), bytes); + bytes *= 2; + } + } + } + + #[test] + fn explicit_ndv_controls_requested_size_and_max_only_caps_it() { + let small = parquet_mr_bloom_filter_bytes( + Some(10), + ICEBERG_DEFAULT_BLOOM_FILTER_FPP, + 64 * 1024 * 1024, + ); + assert!(small < 64 * 1024 * 1024); + + let capped = parquet_mr_bloom_filter_bytes( + Some(100_000_000), + ICEBERG_DEFAULT_BLOOM_FILTER_FPP, + ICEBERG_DEFAULT_BLOOM_FILTER_MAX_BYTES, + ); + assert_eq!(capped, ICEBERG_DEFAULT_BLOOM_FILTER_MAX_BYTES); + let uncapped = + parquet_mr_bloom_filter_bytes(None, ICEBERG_DEFAULT_BLOOM_FILTER_FPP, 64 * 1024 * 1024); + assert_eq!(uncapped, 64 * 1024 * 1024); + } + + #[test] + fn minimum_bloom_filter_size_is_representable() { + assert_eq!( + parquet_mr_bloom_filter_bytes( + Some(1), + ICEBERG_DEFAULT_BLOOM_FILTER_FPP, + BLOOM_FILTER_MIN_BYTES, + ), + BLOOM_FILTER_MIN_BYTES + ); + let ndv = synthetic_ndv_for_bloom_filter_bytes( + BLOOM_FILTER_MIN_BYTES, + ICEBERG_DEFAULT_BLOOM_FILTER_FPP, + ) + .unwrap(); + assert_eq!( + parquet_rs_bloom_filter_bytes(ndv, ICEBERG_DEFAULT_BLOOM_FILTER_FPP), + BLOOM_FILTER_MIN_BYTES + ); + } + + #[test] + fn impossible_synthetic_ndv_is_rejected() { + let err = synthetic_ndv_for_bloom_filter_bytes( + ICEBERG_DEFAULT_BLOOM_FILTER_MAX_BYTES, + f64::MIN_POSITIVE, + ) + .unwrap_err(); + assert!(format!("{err}").contains("cannot represent")); } #[test] @@ -3052,6 +3280,9 @@ mod tests { page_row_limit: 20_000, created_by: "Apache Iceberg (Comet integration test)".to_string(), bloom_filter_enabled_columns: Vec::new(), + bloom_filter_max_bytes: ICEBERG_DEFAULT_BLOOM_FILTER_MAX_BYTES as u64, + bloom_filter_fpp_by_column: Default::default(), + bloom_filter_ndv_by_column: Default::default(), }; Arc::new(IcebergWriteCommon { catalog_properties: HashMap::new(), diff --git a/native/proto/src/proto/operator.proto b/native/proto/src/proto/operator.proto index 3c4c6ca12eb..21a09ae5d0b 100644 --- a/native/proto/src/proto/operator.proto +++ b/native/proto/src/proto/operator.proto @@ -807,10 +807,19 @@ message IcebergParquetWriteSettings { // String written into parquet file metadata. JVM-side default is // `"Apache Iceberg (Comet)"`. string created_by = 7; - // Iceberg columns whose `write.parquet.bloom-filter-enabled.column.` property resolves to - // true. Entries use Iceberg's dotted column-path syntax and are translated to parquet-rs - // `ColumnPath`s by the native writer. + // Physical Parquet leaf paths for Iceberg columns whose + // `write.parquet.bloom-filter-enabled.column.` property resolves to true. The JVM driver + // translates Iceberg logical names to the actual Parquet paths used by list/map encodings. repeated string bloom_filter_enabled_columns = 8; + // Iceberg `write.parquet.bloom-filter-max-bytes` (default 1 MiB). Native eligibility only + // admits powers of two in [32, 128 MiB], which parquet-rs can represent exactly. + uint64 bloom_filter_max_bytes = 9; + // Effective per-column FPP, including Iceberg's 0.01 default. A value is present for every + // enabled column so parquet-rs's different 0.05 default can never leak into Iceberg writes. + map bloom_filter_fpp_by_column = 10; + // User-provided Iceberg NDV values. Absence is significant: parquet-mr allocates the full + // max in that case, whereas an explicit NDV sizes the filter before applying the max as a cap. + map bloom_filter_ndv_by_column = 11; } // Broadcast payload -- one of these per write, identical for every task. diff --git a/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala b/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala index 33a0593568d..b5d78a9465f 100644 --- a/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala +++ b/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala @@ -58,6 +58,7 @@ object IcebergReflection extends Logging { val SPARK_BATCH_QUERY_SCAN = "org.apache.iceberg.spark.source.SparkBatchQueryScan" val SPARK_STAGED_SCAN = "org.apache.iceberg.spark.source.SparkStagedScan" val SPARK_SCHEMA_UTIL = "org.apache.iceberg.spark.SparkSchemaUtil" + val PARQUET_SCHEMA_UTIL = "org.apache.iceberg.parquet.ParquetSchemaUtil" val TABLE = "org.apache.iceberg.Table" val DEFAULT_LOCATION_PROVIDER = "org.apache.iceberg.LocationProviders$DefaultLocationProvider" @@ -687,6 +688,52 @@ object IcebergReflection extends Logging { } } + // scalastyle:off line.size.limit + /** + * Maps Iceberg's logical primitive-column names to their physical Parquet leaf paths. + * + * This mirrors the map Iceberg Java builds before applying per-column writer settings. In + * particular, Parquet's canonical three-level encodings insert `list` for array elements and + * `key_value` for map keys/values, so logical names such as `tags.element` and `attrs.value` + * cannot be passed directly to Parquet writer properties. + * + * See: + * https://github.com/apache/iceberg/blob/apache-iceberg-1.11.0/parquet/src/main/java/org/apache/iceberg/parquet/Parquet.java#L411-L421 + */ + // scalastyle:on line.size.limit + def getParquetPathByIcebergColumnName(schema: Any): Option[Map[String, String]] = { + import scala.jdk.CollectionConverters._ + try { + val parquetSchemaUtil = loadClass(ClassNames.PARQUET_SCHEMA_UTIL) + val parquetSchema = parquetSchemaUtil + .getMethod("convert", loadClass(ClassNames.SCHEMA), classOf[String]) + .invoke(null, schema.asInstanceOf[AnyRef], "table") + val columns = getMethod(parquetSchema.getClass, "getColumns") + .invoke(parquetSchema) + .asInstanceOf[java.util.List[AnyRef]] + val findColumnName = getMethod(schema.getClass, "findColumnName", classOf[Int]) + + Some(columns.asScala.flatMap { column => + val primitiveType = getMethod(column.getClass, "getPrimitiveType").invoke(column) + val parquetId = getMethod(primitiveType.getClass, "getId").invoke(primitiveType) + Option(parquetId).flatMap { id => + val fieldId = getMethod(id.getClass, "intValue").invoke(id).asInstanceOf[Int] + Option(findColumnName.invoke(schema, Int.box(fieldId))).map { icebergName => + val parquetPath = getMethod(column.getClass, "getPath") + .invoke(column) + .asInstanceOf[Array[String]] + .mkString(".") + icebergName.asInstanceOf[String] -> parquetPath + } + } + }.toMap) + } catch { + case e: Exception => + logError(s"Iceberg reflection failure: Parquet column paths: ${e.getMessage}") + None + } + } + /** * All schema versions a table has had (table.schemas().values()), for resolving field ids of * columns that have since been dropped -- mirrors Iceberg-Java's FieldLookup. table.schemas() @@ -1579,6 +1626,12 @@ object IcebergReflection extends Logging { def tablePropertyIntConstant(fieldName: String): Int = readTablePropertiesField(fieldName).asInstanceOf[Integer].intValue() + def tablePropertyDoubleConstantOpt(fieldName: String): Option[Double] = + tablePropertiesClassOpt.flatMap { cls => + try Some(cls.getField(fieldName).get(null).asInstanceOf[java.lang.Double].doubleValue()) + catch { case _: NoSuchFieldException => None } + } + /** * Like [[tablePropertyConstant]] but returns `None` when the constant is absent in the Iceberg * version on the classpath rather than throwing. Used to gate behaviour that only some Iceberg diff --git a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala index 6d96fba5d07..e106f5e1844 100644 --- a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala +++ b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala @@ -50,6 +50,10 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { IcebergReflection.tablePropertyConstant("WRITE_LOCATION_PROVIDER_IMPL") lazy val BloomFilterColumnEnabledPrefix: String = IcebergReflection.tablePropertyConstant("PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX") + lazy val ParquetBloomFilterMaxBytes: String = + IcebergReflection.tablePropertyConstant("PARQUET_BLOOM_FILTER_MAX_BYTES") + val ParquetBloomFilterColumnFppPrefix = "write.parquet.bloom-filter-fpp.column." + val ParquetBloomFilterColumnNdvPrefix = "write.parquet.bloom-filter-ndv.column." lazy val ParquetRowGroupCheckMinRecordCount: String = IcebergReflection.tablePropertyConstant("PARQUET_ROW_GROUP_CHECK_MIN_RECORD_COUNT") lazy val ParquetRowGroupCheckMinRecordCountDefault: Int = @@ -281,10 +285,14 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { PropertyKeys.ParquetRowGroupCheckMaxRecordCount, PropertyKeys.ParquetPageVersion, PropertyKeys.ParquetShredVariants, - PropertyKeys.ParquetVariantBufferSize) + PropertyKeys.ParquetVariantBufferSize, + PropertyKeys.ParquetBloomFilterMaxBytes) private lazy val vettedParquetWritePrefixes: Seq[String] = - Seq(PropertyKeys.BloomFilterColumnEnabledPrefix) + Seq( + PropertyKeys.BloomFilterColumnEnabledPrefix, + PropertyKeys.ParquetBloomFilterColumnFppPrefix, + PropertyKeys.ParquetBloomFilterColumnNdvPrefix) override def getSupportLevel(op: IcebergWriteExec): SupportLevel = try { @@ -422,6 +430,7 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { requireParquetPageVersionDefault, requireShredVariantsDisabled, requireNativeSupportedCompressionLevel, + requireNativeSupportedBloomFilterProperties, requireOnlyVettedParquetWriteProperties, requirePropertyAbsent( PropertyKeys.ParquetEnableDictionary, @@ -586,6 +595,149 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { private val requireNativeSupportedCompressionLevel: TriggerRule = ctx => IcebergWriteProtoTranslation.compressionLevelRejection(ctx.properties) + // These are Apache Parquet Java BlockSplitBloomFilter implementation bounds, not Iceberg + // TableProperties constants, so they cannot be obtained through IcebergReflection: + // scalastyle:off line.size.limit + // https://github.com/apache/parquet-java/blob/78a8d3230eb4769db93de5f2f2e18363c04cae81/parquet-column/src/main/java/org/apache/parquet/column/values/bloomfilter/BlockSplitBloomFilter.java#L40-L50 + // scalastyle:on line.size.limit + private val MinBloomFilterBytes = 32 + private val MaxBloomFilterBytes = 128 * 1024 * 1024 + private val BloomFilterHashProbes = 8 + private val MaxNonOverflowingBloomFilterNdv = Long.MaxValue / BloomFilterHashProbes + + /** + * parquet-rs 58.x represents Bloom filters as a power-of-two number of bytes. parquet-mr + * accepts arbitrary caps and, when one binds, serializes that exact length. Keep those writes + * on the classic path instead of silently changing the number of usable Bloom blocks. + */ + private val requireNativeSupportedBloomFilterProperties: TriggerRule = ctx => { + // The FPP constant is absent from Iceberg 1.5.2, and the NDV constant is absent through + // 1.10. Use the literal prefixes to detect the properties, then optional reflection to check + // capability: an older runtime ignores an explicit property, so that write must fall back. + val unavailableRuntimeProperty = Seq( + PropertyKeys.ParquetBloomFilterColumnFppPrefix -> + "PARQUET_BLOOM_FILTER_COLUMN_FPP_PREFIX", + PropertyKeys.ParquetBloomFilterColumnNdvPrefix -> + "PARQUET_BLOOM_FILTER_COLUMN_NDV_PREFIX").collectFirst { + case (prefix, constant) + if ctx.properties.keys.exists(_.startsWith(prefix)) && + IcebergReflection.tablePropertyConstantOpt(constant).isEmpty => + s"$prefix* is not interpreted by the Iceberg version on the classpath" + } + val maxRejection = + ctx.properties.get(PropertyKeys.ParquetBloomFilterMaxBytes).flatMap { raw => + scala.util.Try(java.lang.Integer.parseInt(raw)).toOption match { + case None => Some(s"${PropertyKeys.ParquetBloomFilterMaxBytes}=$raw is not a Java int") + case Some(value) + if value < MinBloomFilterBytes || value > MaxBloomFilterBytes || + (value & (value - 1)) != 0 => + Some( + s"${PropertyKeys.ParquetBloomFilterMaxBytes}=$value must be a power of two " + + s"in [$MinBloomFilterBytes, $MaxBloomFilterBytes] for native writes") + case Some(_) => None + } + } + + maxRejection.orElse(unavailableRuntimeProperty).orElse { + val maxBytes = ctx.properties + .get(PropertyKeys.ParquetBloomFilterMaxBytes) + .flatMap(raw => scala.util.Try(java.lang.Integer.parseInt(raw)).toOption) + .getOrElse(IcebergWriteProtoTranslation.Defaults.BloomFilterMaxBytes) + // Iceberg visits every enabled-prefix entry and applies enabled, FPP, then NDV. Validate + // the associated shape properties even for enabled=false; a valid NDV also re-enables the + // filter in parquet-mr. + val configured = ctx.properties.iterator.collect { + case (key, _) if key.startsWith(PropertyKeys.BloomFilterColumnEnabledPrefix) => + key.substring(PropertyKeys.BloomFilterColumnEnabledPrefix.length) + }.toSeq + configured.iterator + .flatMap { column => + val fppKey = PropertyKeys.ParquetBloomFilterColumnFppPrefix + column + val ndvKey = PropertyKeys.ParquetBloomFilterColumnNdvPrefix + column + val parsedFpp = ctx.properties.get(fppKey) match { + case Some(raw) => scala.util.Try(java.lang.Double.parseDouble(raw)).toOption + case None => Some(IcebergWriteProtoTranslation.Defaults.BloomFilterFpp) + } + val parsedNdv = ctx.properties + .get(ndvKey) + .flatMap(raw => scala.util.Try(java.lang.Long.parseLong(raw)).toOption) + val fppError = ctx.properties.get(fppKey).flatMap { raw => + parsedFpp match { + case Some(value) + if value > 0.0d && value < 1.0d && java.lang.Double.isFinite(value) && + bloomFilterSizesRepresentable(maxBytes, value) => + None + case Some(value) + if value > 0.0d && value < 1.0d && java.lang.Double.isFinite(value) => + Some(s"$fppKey=$raw cannot represent the configured native Bloom sizes") + case _ => Some(s"$fppKey=$raw must be a finite double strictly between 0 and 1") + } + } + val ndvError = ctx.properties.get(ndvKey).flatMap { raw => + parsedNdv match { + case Some(value) if value > 0L && value <= MaxNonOverflowingBloomFilterNdv => None + case Some(value) if value > MaxNonOverflowingBloomFilterNdv => + Some(s"$ndvKey=$raw exceeds $MaxNonOverflowingBloomFilterNdv; " + + "parquet-mr Bloom sizing may overflow") + case _ => Some(s"$ndvKey=$raw must be a positive Java long") + } + } + val ignoredMinimumCapError = parsedNdv.collect { + case ndv + if maxBytes == MinBloomFilterBytes && ndv > 0L && + ndv <= MaxNonOverflowingBloomFilterNdv && + parsedFpp.exists( + parquetMrRequestedBloomFilterBytes(ndv, _) > MinBloomFilterBytes) => + s"${PropertyKeys.ParquetBloomFilterMaxBytes}=$MinBloomFilterBytes is ignored by " + + s"parquet-mr for $ndvKey=$ndv" + } + Seq(fppError, ndvError, ignoredMinimumCapError).flatten + } + .toSeq + .headOption + } + } + + // Planning-time counterpart of the native inverse-NDV check. A target B is safely encoded by + // aiming at 3B/4, in the interior of parquet-rs's (B/2, B] round-up interval. Requiring every + // power-of-two through the configured cap is conservative and keeps pathological-but-valid + // floating-point FPPs on the JVM path rather than discovering them after task launch. + private def bloomFilterSizesRepresentable(maxBytes: Int, fpp: Double): Boolean = { + val denominator = + -Math.log(1.0d - Math.pow(fpp, 1.0d / BloomFilterHashProbes.toDouble)) + if (!java.lang.Double.isFinite(denominator) || denominator <= 0.0d) return false + + Iterator + .iterate(MinBloomFilterBytes)(_ * 2) + .takeWhile(_ <= maxBytes) + .forall { target => + val ndv = Math.max(1L, Math.round(target.toDouble * 0.75d * denominator)) + val calculatedBits = (-BloomFilterHashProbes.toDouble * ndv.toDouble / + Math.log(1.0d - Math.pow(fpp, 1.0d / BloomFilterHashProbes.toDouble))).toLong + val rawBytes = Math.max( + MinBloomFilterBytes.toLong, + Math.min(MaxBloomFilterBytes.toLong, calculatedBits / 8L)) + val allocated = java.lang.Long.highestOneBit(rawBytes - 1L) << 1 + allocated == target.toLong + } + } + + /** The uncapped byte count parquet-mr passes to its explicit-NDV constructor path. */ + private def parquetMrRequestedBloomFilterBytes(ndv: Long, fpp: Double): Int = { + // Keep the long multiplication before floating-point conversion to match parquet-mr: + // scalastyle:off line.size.limit + // https://github.com/apache/parquet-java/blob/78a8d3230eb4769db93de5f2f2e18363c04cae81/parquet-column/src/main/java/org/apache/parquet/column/values/bloomfilter/BlockSplitBloomFilter.java#L277-L301 + // scalastyle:on line.size.limit + val calculated = (-BloomFilterHashProbes.toLong * ndv).toDouble / + Math.log(1.0d - Math.pow(fpp, 1.0d / BloomFilterHashProbes.toDouble)) + val maxBits = MaxBloomFilterBytes << 3 + var bits = calculated.toInt + if (bits > maxBits || calculated < 0.0d) bits = maxBits + val bitsPerBlock = MinBloomFilterBytes << 3 + bits = (bits + bitsPerBlock - 1) & ~bitsPerBlock + Math.max(bits, bitsPerBlock) / 8 + } + private val requireOnlyVettedParquetWriteProperties: TriggerRule = ctx => ctx.properties .find { case (k, _) => @@ -1113,8 +1265,19 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { val resolvedWriteProperties = IcebergReflection.getWritePropertiesFromSparkWrite(sparkWrite).getOrElse(Map.empty) val effectiveProperties = properties ++ resolvedWriteProperties - val parquetSettings = - IcebergWriteProtoTranslation.buildParquetSettings(effectiveProperties, createdBy) + val parquetPathByIcebergColumnName = + if (IcebergWriteProtoTranslation.hasEnabledBloomFilters(effectiveProperties)) { + IcebergReflection.getParquetPathByIcebergColumnName(writeSchema).getOrElse { + withFallbackReason(op, "Could not resolve physical Parquet paths for Bloom filters") + return None + } + } else { + Map.empty[String, String] + } + val parquetSettings = IcebergWriteProtoTranslation.buildParquetSettings( + effectiveProperties, + createdBy, + parquetPathByIcebergColumnName) // `FileIO.properties()` misses configuration a HadoopFileIO carries through the Hadoop // Configuration instead (fs.s3a.* credentials, custom endpoint, path-style access), which diff --git a/spark/src/main/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslation.scala b/spark/src/main/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslation.scala index c2de086bd8d..4f98a21c8df 100644 --- a/spark/src/main/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslation.scala +++ b/spark/src/main/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslation.scala @@ -59,6 +59,12 @@ object IcebergWriteProtoTranslation { IcebergReflection.tablePropertyConstant("PARQUET_DICT_SIZE_BYTES") lazy val ParquetBloomFilterColumnEnabledPrefix: String = IcebergReflection.tablePropertyConstant("PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX") + lazy val ParquetBloomFilterMaxBytes: String = + IcebergReflection.tablePropertyConstant("PARQUET_BLOOM_FILTER_MAX_BYTES") + // These were added to Iceberg after bloom enablement/max-bytes. Literals let older runtimes + // keep using the defaults; detection rejects an explicit property they would ignore. + val ParquetBloomFilterColumnFppPrefix = "write.parquet.bloom-filter-fpp.column." + val ParquetBloomFilterColumnNdvPrefix = "write.parquet.bloom-filter-ndv.column." } /** Iceberg's numeric defaults, pulled at runtime so they stay in lock-step with the runtime. */ @@ -71,6 +77,14 @@ object IcebergWriteProtoTranslation { IcebergReflection.tablePropertyIntConstant("PARQUET_DICT_SIZE_BYTES_DEFAULT").toLong lazy val PageRowLimit: Int = IcebergReflection.tablePropertyIntConstant("PARQUET_PAGE_ROW_LIMIT_DEFAULT") + lazy val BloomFilterMaxBytes: Int = + IcebergReflection.tablePropertyIntConstant("PARQUET_BLOOM_FILTER_MAX_BYTES_DEFAULT") + // Iceberg introduced the public constant together with the FPP property. Keep the literal + // fallback for runtimes old enough not to expose it; 0.01 is also parquet-mr's default. + lazy val BloomFilterFpp: Double = + IcebergReflection + .tablePropertyDoubleConstantOpt("PARQUET_BLOOM_FILTER_COLUMN_FPP_DEFAULT") + .getOrElse(0.01d) } /** @@ -102,10 +116,43 @@ object IcebergWriteProtoTranslation { } } + private def configuredBloomFilterColumnNames(props: Map[String, String]): Seq[String] = + props.iterator + .collect { + case (key, _) if key.startsWith(Keys.ParquetBloomFilterColumnEnabledPrefix) => + key.substring(Keys.ParquetBloomFilterColumnEnabledPrefix.length) + } + .toSeq + .sorted + + private def enabledBloomFilterColumnNames(props: Map[String, String]): Seq[String] = + configuredBloomFilterColumnNames(props).filter { column => + val enabled = + java.lang.Boolean.valueOf(props(Keys.ParquetBloomFilterColumnEnabledPrefix + column)) + // Iceberg applies enabled, FPP, and NDV in that order. parquet-mr's NDV setter enables the + // column, so an explicit NDV wins over enabled=false. + enabled || props.contains(Keys.ParquetBloomFilterColumnNdvPrefix + column) + } + + def hasEnabledBloomFilters(props: Map[String, String]): Boolean = + enabledBloomFilterColumnNames(props).nonEmpty + + /** + * Test convenience for schemas where Iceberg logical names and physical Parquet paths are + * identical. Production translation must supply Iceberg Java's logical-to-physical path map. + */ + private[operator] def buildParquetSettings( + props: Map[String, String], + createdBy: String): IcebergParquetWriteSettings = { + val identityPaths = enabledBloomFilterColumnNames(props).map(name => name -> name).toMap + buildParquetSettings(props, createdBy, identityPaths) + } + /** Builds the parquet settings message. Pure: no SparkWrite or Iceberg `Table` access. */ def buildParquetSettings( props: Map[String, String], - createdBy: String): IcebergParquetWriteSettings = { + createdBy: String, + parquetPathByIcebergColumnName: Map[String, String]): IcebergParquetWriteSettings = { val rowGroupSize = parseJavaInt(props, Keys.ParquetRowGroupSizeBytes, Defaults.RowGroupSizeBytes.toInt).toLong val pageSize = @@ -114,15 +161,28 @@ object IcebergWriteProtoTranslation { parseJavaInt(props, Keys.ParquetDictSizeBytes, Defaults.DictSizeBytes.toInt).toLong val pageRowLimit = parseJavaInt(props, Keys.ParquetPageRowLimit, Defaults.PageRowLimit) val compression = resolveCompression(props) - val bloomFilterEnabledColumns = props.iterator - .collect { - case (key, value) - if key.startsWith(Keys.ParquetBloomFilterColumnEnabledPrefix) && - value.equalsIgnoreCase("true") => - key.substring(Keys.ParquetBloomFilterColumnEnabledPrefix.length) + // Iceberg properties use logical schema paths, while Parquet writer properties require the + // physical leaf path. Missing fields are skipped, matching Iceberg Java's writer behavior. + val bloomFilterColumns = enabledBloomFilterColumnNames(props) + .flatMap { icebergName => + parquetPathByIcebergColumnName.get(icebergName).map(icebergName -> _) } - .toSeq - .sorted + .sortBy(_._2) + val bloomFilterEnabledColumns = bloomFilterColumns.map(_._2) + val bloomFilterMaxBytes = + parseJavaInt(props, Keys.ParquetBloomFilterMaxBytes, Defaults.BloomFilterMaxBytes).toLong + val bloomFilterFppByColumn = bloomFilterColumns.map { case (icebergName, parquetPath) => + val value = props + .get(Keys.ParquetBloomFilterColumnFppPrefix + icebergName) + .map(java.lang.Double.parseDouble) + .getOrElse(Defaults.BloomFilterFpp) + parquetPath -> value + }.toMap + val bloomFilterNdvByColumn = bloomFilterColumns.flatMap { case (icebergName, parquetPath) => + props + .get(Keys.ParquetBloomFilterColumnNdvPrefix + icebergName) + .map(value => parquetPath -> java.lang.Long.parseLong(value)) + }.toMap val builder = IcebergParquetWriteSettings .newBuilder() .setCompression(compression) @@ -132,6 +192,13 @@ object IcebergWriteProtoTranslation { .setPageRowLimit(pageRowLimit) .setCreatedBy(createdBy) .addAllBloomFilterEnabledColumns(bloomFilterEnabledColumns.asJava) + .setBloomFilterMaxBytes(bloomFilterMaxBytes) + .putAllBloomFilterFppByColumn(bloomFilterFppByColumn.map { case (k, v) => + k -> Double.box(v) + }.asJava) + .putAllBloomFilterNdvByColumn(bloomFilterNdvByColumn.map { case (k, v) => + k -> Long.box(v) + }.asJava) resolveCompressionLevel(props, compression).foreach(builder.setCompressionLevel) diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala index bccf0826644..4dac2cc05b6 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala @@ -19,7 +19,7 @@ package org.apache.comet -import java.io.File +import java.io.{ByteArrayOutputStream, File} import java.nio.file.{FileAlreadyExistsException, Files} import java.sql.Timestamp import java.util.concurrent.{ConcurrentLinkedQueue, CountDownLatch, TimeUnit} @@ -1263,6 +1263,339 @@ class CometIcebergWriteActionSuite } } + test("explicit NDV re-enables a bloom filter after enabled=false") { + assumeNativeAcceleration() + assumeIcebergBloomShapeProperties() + withIcebergCatalog { warehouseDir => + val configuredFpp = 0.01 + val configuredNdv = 1000 + val properties = Some( + "'write.parquet.bloom-filter-enabled.column.id'='false', " + + s"'write.parquet.bloom-filter-fpp.column.id'='$configuredFpp', " + + s"'write.parquet.bloom-filter-ndv.column.id'='$configuredNdv'") + createTable(warehouseDir, "bloom_false_ndv_native", partitionSpec = "", properties) + createTable(warehouseDir, "bloom_false_ndv_jvm", partitionSpec = "", properties) + + def insert(table: String): Unit = spark.sql( + s"INSERT INTO cat.db.$table " + + s"SELECT CAST(id AS INT), 'region', CAST(id AS DOUBLE) " + + s"FROM range(0, $configuredNdv, 1, 1)") + + assertNativeWriteEngages("bloom_false_ndv_native", 0 until configuredNdv) { + insert("bloom_false_ndv_native") + } + withSQLConf(CometConf.COMET_ICEBERG_NATIVE_WRITE_ENABLED.key -> "false") { + insert("bloom_false_ndv_jvm") + } + + val native = parquetBloomFilterBytes("bloom_false_ndv_native", "id") + val jvm = parquetBloomFilterBytes("bloom_false_ndv_jvm", "id") + assert(native.map(_.length) == jvm.map(_.length)) + assert(native.zip(jvm).forall { case (left, right) => + java.util.Arrays.equals(left, right) + }) + } + } + + test("enabled=false preserves JVM validation errors for malformed FPP and NDV") { + assumeNativeAcceleration() + assumeIcebergBloomShapeProperties() + withIcebergCatalog { warehouseDir => + Seq( + "fpp" -> "'write.parquet.bloom-filter-fpp.column.id'='garbage'", + "ndv" -> "'write.parquet.bloom-filter-ndv.column.id'='garbage'").foreach { + case (suffix, malformedProperty) => + val table = s"bloom_false_bad_$suffix" + createTable( + warehouseDir, + table, + partitionSpec = "", + properties = + Some(s"'write.parquet.bloom-filter-enabled.column.id'='false', $malformedProperty")) + + def insert(): Unit = + spark.sql(s"INSERT INTO cat.db.$table VALUES (1, 'region', 1.0)") + + val withComet = intercept[Throwable] { + withNativeEnabled(insert()) + } + val withJvm = intercept[Throwable] { + withSQLConf(CometConf.COMET_ICEBERG_NATIVE_WRITE_ENABLED.key -> "false")(insert()) + } + val cometCause = exceptionChain(withComet).last + val jvmCause = exceptionChain(withJvm).last + assert(cometCause.getClass == jvmCause.getClass) + assert(cometCause.getMessage == jvmCause.getMessage) + } + } + } + + test("max-bytes=32 falls back only when parquet-mr ignores the cap") { + assumeNativeAcceleration() + assumeIcebergBloomShapeProperties() + withIcebergCatalog { warehouseDir => + val minimumBytes = 32 + val naturallyMinimumNdv = 1 + val bindingNdv = 1000000 + val bindingFpp = 0.0001 + val parquetMrBindingBytes = 4 * 1024 * 1024 + val enabled = "'write.parquet.bloom-filter-enabled.column.id'='true', " + createTable( + warehouseDir, + "bloom_minimum_native", + partitionSpec = "", + properties = Some( + enabled + s"'write.parquet.bloom-filter-ndv.column.id'='$naturallyMinimumNdv', " + + s"'write.parquet.bloom-filter-max-bytes'='$minimumBytes'")) + createTable( + warehouseDir, + "bloom_minimum_fallback", + partitionSpec = "", + properties = Some( + enabled + s"'write.parquet.bloom-filter-fpp.column.id'='$bindingFpp', " + + s"'write.parquet.bloom-filter-ndv.column.id'='$bindingNdv', " + + s"'write.parquet.bloom-filter-max-bytes'='$minimumBytes'")) + + assertNativeWriteEngages("bloom_minimum_native", Seq(1)) { + spark.sql("INSERT INTO cat.db.bloom_minimum_native VALUES (1, 'region', 1.0)") + } + assertNativeWriteDoesNotEngage("bloom_minimum_fallback", Seq(1)) { + spark.sql("INSERT INTO cat.db.bloom_minimum_fallback VALUES (1, 'region', 1.0)") + } + + assert( + parquetBloomFilterBytes("bloom_minimum_native", "id").forall(_.length == minimumBytes)) + assert( + parquetBloomFilterBytes("bloom_minimum_fallback", "id") + .forall(_.length == parquetMrBindingBytes)) + } + } + + test("native bloom filters resolve list and map leaves to physical Parquet paths") { + assumeNativeAcceleration() + withIcebergCatalog { _ => + spark.sql(s""" + CREATE TABLE $catalog.$ns.native_nested_bloom ( + id INT, + tags ARRAY, + attrs MAP + ) USING iceberg + TBLPROPERTIES ( + 'write.parquet.bloom-filter-enabled.column.tags.element'='true', + 'write.parquet.bloom-filter-enabled.column.attrs.key'='true', + 'write.parquet.bloom-filter-enabled.column.attrs.value'='true' + ) + """) + + assertNativeWriteEngages("native_nested_bloom", Seq(1, 2)) { + spark.sql(""" + INSERT INTO cat.db.native_nested_bloom VALUES + (1, array('red', 'green'), map('small', 10, 'large', 20)), + (2, array('blue'), map('medium', 30)) + """) + } + + assertParquetBloomFilters( + "native_nested_bloom", + enabledColumns = Set("tags.list.element", "attrs.key_value.key", "attrs.key_value.value")) + } + } + + test("native bloom filter is byte-identical to parquet-mr when max-bytes binds") { + assumeNativeAcceleration() + assumeIcebergBloomShapeProperties() + withIcebergCatalog { warehouseDir => + // This NDV/FPP pair requests a 128 MiB allocation before max-bytes is applied. The 4 KiB + // maximum must therefore bind on both writers rather than merely coinciding with the + // naturally selected size. + val configuredNdv = 100000000L + val bindingMaxBytes = 4 * 1024 + val properties = Some( + "'write.parquet.bloom-filter-enabled.column.id'='true', " + + "'write.parquet.bloom-filter-fpp.column.id'='0.01', " + + s"'write.parquet.bloom-filter-ndv.column.id'='$configuredNdv', " + + s"'write.parquet.bloom-filter-max-bytes'='$bindingMaxBytes'") + createTable(warehouseDir, "bloom_identity_native", partitionSpec = "", properties) + createTable(warehouseDir, "bloom_identity_jvm", partitionSpec = "", properties) + + def insert(table: String): Unit = spark.sql( + s"INSERT INTO cat.db.$table " + + "SELECT CAST(id AS INT), 'region', CAST(id AS DOUBLE) FROM range(0, 10000, 1, 1)") + + assertNativeWriteEngages("bloom_identity_native", 0 until 10000) { + insert("bloom_identity_native") + } + withSQLConf(CometConf.COMET_ICEBERG_NATIVE_WRITE_ENABLED.key -> "false") { + insert("bloom_identity_jvm") + } + + val native = parquetBloomFilterBytes("bloom_identity_native", "id") + val jvm = parquetBloomFilterBytes("bloom_identity_jvm", "id") + assert( + native.nonEmpty && native.forall(_.length == bindingMaxBytes), + s"native Bloom filter must be capped at $bindingMaxBytes bytes") + assert( + jvm.nonEmpty && jvm.forall(_.length == bindingMaxBytes), + s"JVM Bloom filter must be capped at $bindingMaxBytes bytes") + assert(native.size == jvm.size, "native and JVM writes must produce the same file count") + assert( + native.zip(jvm).forall { case (left, right) => java.util.Arrays.equals(left, right) }, + "expected byte-identical capped SBBF bitsets for identical values and allocation") + } + } + + test("native bloom sizing covers FPP and NDV presence combinations and binding caps") { + assumeNativeAcceleration() + assumeIcebergBloomShapeProperties() + withIcebergCatalog { warehouseDir => + val enabled = "'write.parquet.bloom-filter-enabled.column.id'='true'" + val cases: Seq[(String, String)] = Seq( + ("fpp_only", s"$enabled, 'write.parquet.bloom-filter-fpp.column.id'='0.02'"), + ("ndv_only", s"$enabled, 'write.parquet.bloom-filter-ndv.column.id'='1000'"), + ( + "both", + s"$enabled, 'write.parquet.bloom-filter-fpp.column.id'='0.005', " + + "'write.parquet.bloom-filter-ndv.column.id'='1000'"), + // The requested NDV/FPP needs far more than 64 bytes. Like parquet-mr, max wins and the + // target FPP becomes impossible to guarantee, but membership must remain correct. + ( + "binding_cap", + s"$enabled, 'write.parquet.bloom-filter-fpp.column.id'='0.0001', " + + "'write.parquet.bloom-filter-ndv.column.id'='1000000', " + + "'write.parquet.bloom-filter-max-bytes'='64'")) + + cases.zipWithIndex.foreach { case ((suffix, properties), index) => + val table = s"bloom_shape_${suffix}" + createTable(warehouseDir, table, partitionSpec = "", properties = Some(properties)) + val ids = index * 256 until (index + 1) * 256 + assertNativeWriteEngages(table, ids) { + spark.sql( + s"INSERT INTO cat.db.$table SELECT CAST(id AS INT), 'region', " + + s"CAST(id AS DOUBLE) FROM range(${ids.start}, ${ids.end}, 1, 1)") + } + val bytes = parquetBloomFilterBytes(table, "id") + assert(bytes.nonEmpty && bytes.forall(_.nonEmpty)) + if (suffix == "binding_cap") { + assert(bytes.forall(_.length == 64), s"expected binding 64-byte cap for $table") + assertParquetBloomContainsInts(table, "id", ids) + } + } + } + } + + test("explicit underestimated NDV retains parquet-mr allocation precedence") { + assumeNativeAcceleration() + assumeIcebergBloomShapeProperties() + withIcebergCatalog { warehouseDir => + // max-bytes is only a cap in parquet-mr; it does not enlarge a filter whose explicit NDV + // was underestimated. This comparison prevents Comet from silently replacing the user's + // NDV with an artificial NDV derived from the much larger maximum. + val properties = Some( + "'write.parquet.bloom-filter-enabled.column.id'='true', " + + "'write.parquet.bloom-filter-fpp.column.id'='0.01', " + + "'write.parquet.bloom-filter-ndv.column.id'='10', " + + "'write.parquet.bloom-filter-max-bytes'='67108864'") + createTable(warehouseDir, "bloom_low_ndv_native", partitionSpec = "", properties) + createTable(warehouseDir, "bloom_low_ndv_jvm", partitionSpec = "", properties) + + def insert(table: String): Unit = spark.sql( + s"INSERT INTO cat.db.$table " + + "SELECT CAST(id AS INT), 'region', CAST(id AS DOUBLE) FROM range(0, 4096, 1, 1)") + + assertNativeWriteEngages("bloom_low_ndv_native", 0 until 4096) { + insert("bloom_low_ndv_native") + } + insert("bloom_low_ndv_jvm") + + val native = parquetBloomFilterBytes("bloom_low_ndv_native", "id") + val jvm = parquetBloomFilterBytes("bloom_low_ndv_jvm", "id") + assert(native.map(_.length) == jvm.map(_.length)) + assert(native.zip(jvm).forall { case (left, right) => + java.util.Arrays.equals(left, right) + }) + } + } + + test("large representable max folds natively while adjacent non-power-of-two falls back") { + assumeNativeAcceleration() + withIcebergCatalog { warehouseDir => + val powerOfTwo = 64 * 1024 * 1024 + val enabled = "'write.parquet.bloom-filter-enabled.column.id'='true', " + createTable( + warehouseDir, + "bloom_large_native", + partitionSpec = "", + properties = Some(enabled + s"'write.parquet.bloom-filter-max-bytes'='$powerOfTwo'")) + createTable( + warehouseDir, + "bloom_large_jvm", + partitionSpec = "", + properties = + Some(enabled + s"'write.parquet.bloom-filter-max-bytes'='${powerOfTwo + 1}'")) + + def insert(table: String): Unit = spark.sql( + s"INSERT INTO cat.db.$table " + + "SELECT CAST(id AS INT), 'region', CAST(id AS DOUBLE) FROM range(0, 256, 1, 1)") + + assertNativeWriteEngages("bloom_large_native", 0 until 256) { + insert("bloom_large_native") + } + assertNativeWriteDoesNotEngage("bloom_large_jvm", 0 until 256) { + insert("bloom_large_jvm") + } + + val nativeBytes = parquetBloomFilterBytes("bloom_large_native", "id") + val nativeSize = nativeBytes.head.length + val jvmSize = parquetBloomFilterBytes("bloom_large_jvm", "id").head.length + assert(nativeSize < 1024 * 1024, s"expected folding, got $nativeSize bytes") + // Iceberg runtimes bundle different Parquet Java versions: newer versions retain the exact + // cap while older ones round it upward. Both demonstrate that the fallback avoids folding + // this deliberately oversized filter down to the native allocation. + assert(jvmSize > powerOfTwo, s"expected an oversized JVM filter, got $jvmSize bytes") + assertParquetBloomContainsInts("bloom_large_native", "id", 0 until 256) + + // The adjacent non-power-of-two JVM filter is intentionally much larger, so comparing its + // bytes with the folded native filter would be meaningless. Instead, write the same values + // through the JVM writer with a cap equal to the observed folded size. Power-of-two folding + // must produce exactly the same SBBF bitset as hashing directly into that final allocation. + createTable( + warehouseDir, + "bloom_large_folded_jvm", + partitionSpec = "", + properties = Some(enabled + s"'write.parquet.bloom-filter-max-bytes'='$nativeSize'")) + withSQLConf(CometConf.COMET_ICEBERG_NATIVE_WRITE_ENABLED.key -> "false") { + insert("bloom_large_folded_jvm") + } + assertRows("bloom_large_folded_jvm", 0 until 256) + val foldedJvmBytes = parquetBloomFilterBytes("bloom_large_folded_jvm", "id") + assert(nativeBytes.map(_.length) == foldedJvmBytes.map(_.length)) + assert( + nativeBytes.zip(foldedJvmBytes).forall { case (left, right) => + java.util.Arrays.equals(left, right) + }, + "folded native SBBF must be byte-identical to a same-sized JVM SBBF") + } + } + + test("out-of-range bloom max uses the classic writer") { + assumeNativeAcceleration() + withIcebergCatalog { warehouseDir => + // Leave Bloom filters disabled so the stock writer can demonstrate planning fallback + // without allocating a 128 MiB filter for the oversized case. + Seq("31", "134217729").zipWithIndex.foreach { case (max, index) => + val table = s"bloom_range_fallback_$index" + createTable( + warehouseDir, + table, + partitionSpec = "", + properties = Some(s"'write.parquet.bloom-filter-max-bytes'='$max'")) + assertNativeWriteDoesNotEngage(table, Seq(index)) { + spark.sql(s"INSERT INTO cat.db.$table VALUES ($index, 'region', 1.0)") + } + } + } + } + // What Iceberg's Spark writer stamps for `sort_order_id` on appended files changed across // releases: through 1.10 `SparkWrite$WriterFactory` never wires the table sort order (files // get 0 even on a sorted table); 1.11 added `SparkWriteConf.outputSortOrderId` and stamps the @@ -4882,11 +5215,78 @@ class CometIcebergWriteActionSuite } } + private def parquetBloomFilterBytes(tableName: String, columnName: String): Seq[Array[Byte]] = { + val paths = spark + .sql(s"SELECT file_path FROM $catalog.$ns.$tableName.data_files ORDER BY file_path") + .collect() + .map(_.getString(0)) + val conf = spark.sparkContext.hadoopConfiguration + paths.toSeq.flatMap { path => + val input = HadoopInputFile.fromPath(new Path(path.stripPrefix("file:")), conf) + val reader = ParquetFileReader.open(input) + try { + reader.getFooter.getBlocks.asScala.flatMap { block => + block.getColumns.asScala + .filter(_.getPath.toDotString == columnName) + .map { column => + val bloom = reader.readBloomFilter(column) + assert(bloom != null, s"expected Bloom filter for $columnName in $path") + val out = new ByteArrayOutputStream(bloom.getBitsetSize) + bloom.writeTo(out) + out.toByteArray + } + } + } finally { + reader.close() + } + } + } + + private def assertParquetBloomContainsInts( + tableName: String, + columnName: String, + values: Seq[Int]): Unit = { + val paths = spark + .sql(s"SELECT file_path FROM $catalog.$ns.$tableName.data_files") + .collect() + .map(_.getString(0)) + val conf = spark.sparkContext.hadoopConfiguration + paths.foreach { path => + val input = HadoopInputFile.fromPath(new Path(path.stripPrefix("file:")), conf) + val reader = ParquetFileReader.open(input) + try { + reader.getFooter.getBlocks.asScala.foreach { block => + block.getColumns.asScala + .filter(_.getPath.toDotString == columnName) + .foreach { column => + val bloom = reader.readBloomFilter(column) + assert(bloom != null, s"expected Bloom filter for $columnName in $path") + values.foreach { value => + assert( + bloom.findHash(bloom.hash(value)), + s"Bloom filter false negative for $columnName=$value in $path") + } + } + } + } finally { + reader.close() + } + } + } + /** Native acceleration shared assumption -- currently just the Iceberg-on-classpath check. */ private def assumeNativeAcceleration(): Unit = { assume(icebergAvailable, "Iceberg not available in classpath") } + private def assumeIcebergBloomShapeProperties(): Unit = { + assume( + org.apache.comet.iceberg.IcebergReflection + .tablePropertyConstantOpt("PARQUET_BLOOM_FILTER_COLUMN_FPP_PREFIX") + .isDefined, + "Iceberg runtime does not interpret per-column Bloom FPP/NDV properties") + } + /** * Every row-consuming operator must receive row-based input. Spark guarantees that by inserting * `ColumnarToRow` transitions in `ApplyColumnarRulesAndInsertTransitions`; an operator that diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala index 95d5688f737..5fc91e65c52 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala @@ -377,13 +377,147 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes } } - test("fall-back: write.parquet.bloom-filter-max-bytes set") { - withDetectionCatalog { _ => + test("bloom-filter max-bytes accepts only representable power-of-two values") { + withDetectionCatalog { dir => + Seq("32", "524288", "1048576", "134217728").zipWithIndex.foreach { case (value, index) => + val table = s"bloom_max_ok_$index" + createTable( + dir, + table, + partitionSpec = "", + properties = Some(s"'write.parquet.bloom-filter-max-bytes'='$value'")) + assertSupportLevelIs[Compatible](table) + } + + Seq("31", "33", "100", "134217729", "0", "-1", " 32", "garbage", "2147483648").zipWithIndex + .foreach { case (value, index) => + val table = s"bloom_max_bad_$index" + createTable( + dir, + table, + partitionSpec = "", + properties = Some(s"'write.parquet.bloom-filter-max-bytes'='$value'")) + assertUnsupportedContainsAllowingWriteFailure( + table, + "write.parquet.bloom-filter-max-bytes") + } + } + } + + test("bloom-filter max-bytes=32 falls back only when parquet-mr ignores the cap") { + assumeIcebergBloomShapeProperties() + withDetectionCatalog { dir => + val minimumBytes = 32 + val naturallyMinimumNdv = 1 + val bindingFpp = 0.0001 + val bindingNdv = 1000000 + Seq( + ("without_ndv", s"'write.parquet.bloom-filter-max-bytes'='$minimumBytes'"), + ( + "natural_minimum", + "'write.parquet.bloom-filter-enabled.column.id'='true', " + + s"'write.parquet.bloom-filter-ndv.column.id'='$naturallyMinimumNdv', " + + s"'write.parquet.bloom-filter-max-bytes'='$minimumBytes'")).foreach { + case (table, properties) => + createTable(dir, table, partitionSpec = "", properties = Some(properties)) + assertSupportLevelIs[Compatible](table) + } + + createTable( + dir, + "ignored_minimum_cap", + partitionSpec = "", + properties = Some( + "'write.parquet.bloom-filter-enabled.column.id'='true', " + + s"'write.parquet.bloom-filter-fpp.column.id'='$bindingFpp', " + + s"'write.parquet.bloom-filter-ndv.column.id'='$bindingNdv', " + + s"'write.parquet.bloom-filter-max-bytes'='$minimumBytes'")) + assertUnsupportedContainsAllowingWriteFailure( + "ignored_minimum_cap", + "write.parquet.bloom-filter-max-bytes") + } + } + + test("fall-back: bloom-filter NDV values that overflow parquet-mr sizing arithmetic") { + assumeIcebergBloomShapeProperties() + withDetectionCatalog { dir => + val largestNonOverflowingNdv = Long.MaxValue / 8L + val cases = Seq( + (largestNonOverflowingNdv, true), + (largestNonOverflowingNdv + 1L, false), + (1L << 61, false), + (Long.MaxValue, false)) + + cases.zipWithIndex.foreach { case ((ndv, compatible), index) => + val table = s"bloom_ndv_overflow_$index" + createTable( + dir, + table, + partitionSpec = "", + properties = Some( + "'write.parquet.bloom-filter-enabled.column.id'='true', " + + s"'write.parquet.bloom-filter-ndv.column.id'='$ndv'")) + if (compatible) { + assertSupportLevelIs[Compatible](table) + } else { + assertUnsupportedContainsAllowingWriteFailure( + table, + "write.parquet.bloom-filter-ndv.column.id") + } + } + } + } + + test("bloom-filter enabled=false still applies and validates FPP and NDV") { + assumeIcebergBloomShapeProperties() + withDetectionCatalog { dir => + val configuredNdv = 1000 createTable( - "bloom_max", + dir, + "false_with_ndv", partitionSpec = "", - properties = Some("'write.parquet.bloom-filter-max-bytes'='524288'")) - assertUnsupportedContains("bloom_max", "write.parquet.bloom-filter-max-bytes") + properties = Some( + "'write.parquet.bloom-filter-enabled.column.id'='false', " + + s"'write.parquet.bloom-filter-ndv.column.id'='$configuredNdv'")) + assertSupportLevelIs[Compatible]("false_with_ndv") + + Seq( + "'write.parquet.bloom-filter-fpp.column.id'='garbage'", + "'write.parquet.bloom-filter-ndv.column.id'='garbage'").zipWithIndex.foreach { + case (property, index) => + val table = s"false_with_bad_shape_$index" + createTable( + dir, + table, + partitionSpec = "", + properties = + Some(s"'write.parquet.bloom-filter-enabled.column.id'='false', $property")) + assertUnsupportedContainsAllowingWriteFailure(table, "write.parquet.bloom-filter") + } + } + } + + test("fall-back: invalid enabled-column FPP and NDV") { + withDetectionCatalog { dir => + Seq( + "'write.parquet.bloom-filter-fpp.column.id'='0'", + "'write.parquet.bloom-filter-fpp.column.id'='1'", + "'write.parquet.bloom-filter-fpp.column.id'='NaN'", + // Positive and below one, but too small for any integer NDV to encode the requested + // power-of-two allocation through parquet-rs's NDV/FPP API. + "'write.parquet.bloom-filter-fpp.column.id'='4.9E-324'", + "'write.parquet.bloom-filter-ndv.column.id'='0'", + "'write.parquet.bloom-filter-ndv.column.id'='garbage'").zipWithIndex.foreach { + case (property, index) => + val table = s"bloom_shape_bad_$index" + createTable( + dir, + table, + partitionSpec = "", + properties = + Some(s"'write.parquet.bloom-filter-enabled.column.id'='true', $property")) + assertUnsupportedContainsAllowingWriteFailure(table, "write.parquet.bloom-filter") + } } } @@ -1776,6 +1910,14 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes } } + private def assumeIcebergBloomShapeProperties(): Unit = { + assume( + IcebergReflection + .tablePropertyConstantOpt("PARQUET_BLOOM_FILTER_COLUMN_FPP_PREFIX") + .isDefined, + "Iceberg runtime does not interpret per-column Bloom FPP/NDV properties") + } + /** * Runs Spark's transition insertion followed by [[EliminateRedundantTransitions]] over a * hand-built `CometIcebergWriteExec -> CometSparkToColumnarExec -> source` plan and returns the @@ -1846,8 +1988,6 @@ case class TransitionProbeLeaf(columnar: Boolean) extends LeafExecNode { throw new UnsupportedOperationException("planning-only node") override protected def doExecuteColumnar(): RDD[ColumnarBatch] = throw new UnsupportedOperationException("planning-only node") -} - /** * A FileIO that works normally (delegating to HadoopFileIO) but whose class is not on Comet's * recognized-FileIO allowlist. Composition rather than inheritance is the point: a HadoopFileIO diff --git a/spark/src/test/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslationSuite.scala b/spark/src/test/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslationSuite.scala index 77795b46968..03e56eb81a0 100644 --- a/spark/src/test/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslationSuite.scala +++ b/spark/src/test/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslationSuite.scala @@ -179,9 +179,74 @@ class IcebergWriteProtoTranslationSuite extends AnyFunSuite { test("per-column bloom filter properties are translated deterministically") { val prefix = Keys.ParquetBloomFilterColumnEnabledPrefix val settings = buildParquetSettings( - Map(s"${prefix}region" -> "TRUE", s"${prefix}id" -> "true", s"${prefix}amount" -> "false"), + Map( + s"${prefix}region" -> "TRUE", + s"${prefix}id" -> "true", + s"${prefix}amount" -> "false", + s"${Keys.ParquetBloomFilterColumnFppPrefix}region" -> "0.02", + s"${Keys.ParquetBloomFilterColumnNdvPrefix}id" -> "1234"), TestCreatedBy) assert(settings.getBloomFilterEnabledColumnsList == java.util.Arrays.asList("id", "region")) + assert(settings.getBloomFilterMaxBytes == 1024L * 1024L) + assert(settings.getBloomFilterFppByColumnMap.get("id") == 0.01d) + assert(settings.getBloomFilterFppByColumnMap.get("region") == 0.02d) + assert(settings.getBloomFilterNdvByColumnMap.get("id") == 1234L) + assert(!settings.getBloomFilterNdvByColumnMap.containsKey("region")) + assert(!settings.getBloomFilterFppByColumnMap.containsKey("amount")) + } + + test("an explicit NDV re-enables a bloom filter after enabled=false") { + val prefix = Keys.ParquetBloomFilterColumnEnabledPrefix + val configuredFpp = 0.02d + val configuredNdv = 1234L + val settings = buildParquetSettings( + Map( + s"${prefix}id" -> "false", + s"${Keys.ParquetBloomFilterColumnFppPrefix}id" -> configuredFpp.toString, + s"${Keys.ParquetBloomFilterColumnNdvPrefix}id" -> configuredNdv.toString), + TestCreatedBy) + + assert(settings.getBloomFilterEnabledColumnsList == java.util.Arrays.asList("id")) + assert(settings.getBloomFilterFppByColumnMap.get("id") == configuredFpp) + assert(settings.getBloomFilterNdvByColumnMap.get("id") == configuredNdv) + } + + test("bloom filter properties use physical Parquet paths for list and map leaves") { + val prefix = Keys.ParquetBloomFilterColumnEnabledPrefix + val settings = buildParquetSettings( + Map( + s"${prefix}tags.element" -> "true", + s"${prefix}attrs.key" -> "true", + s"${prefix}attrs.value" -> "true", + s"${prefix}missing" -> "true", + s"${Keys.ParquetBloomFilterColumnFppPrefix}tags.element" -> "0.02", + s"${Keys.ParquetBloomFilterColumnNdvPrefix}attrs.value" -> "1234"), + TestCreatedBy, + Map( + "tags.element" -> "tags.list.element", + "attrs.key" -> "attrs.key_value.key", + "attrs.value" -> "attrs.key_value.value")) + + assert( + settings.getBloomFilterEnabledColumnsList == java.util.Arrays + .asList("attrs.key_value.key", "attrs.key_value.value", "tags.list.element")) + assert(settings.getBloomFilterFppByColumnMap.get("tags.list.element") == 0.02d) + assert(settings.getBloomFilterNdvByColumnMap.get("attrs.key_value.value") == 1234L) + assert(!settings.getBloomFilterEnabledColumnsList.contains("missing")) + } + + test("Iceberg bloom filter defaults and explicit max are translated exactly") { + val enabled = Keys.ParquetBloomFilterColumnEnabledPrefix + "id" + val defaults = buildParquetSettings(Map(enabled -> "true"), TestCreatedBy) + assert(defaults.getBloomFilterMaxBytes == Defaults.BloomFilterMaxBytes) + assert(Defaults.BloomFilterMaxBytes == 1024 * 1024) + assert(defaults.getBloomFilterFppByColumnMap.get("id") == Defaults.BloomFilterFpp) + assert(Defaults.BloomFilterFpp == 0.01d) + + val explicit = buildParquetSettings( + Map(enabled -> "true", Keys.ParquetBloomFilterMaxBytes -> "67108864"), + TestCreatedBy) + assert(explicit.getBloomFilterMaxBytes == 64L * 1024L * 1024L) } test("size properties are parsed with Java Integer.parseInt semantics") { From f15f4769d62a8a3321c6debed0ccfb814616c4d8 Mon Sep 17 00:00:00 2001 From: Nikita Matskevich Date: Tue, 8 Sep 2026 10:30:18 +0200 Subject: [PATCH 03/32] fix: fall back for sanitized Iceberg bloom paths --- .../comet/iceberg/IcebergReflection.scala | 38 ++++++++++++++++--- .../operator/CometIcebergNativeWrite.scala | 17 ++++++++- .../IcebergWriteProtoTranslation.scala | 2 +- .../comet/CometIcebergWriteActionSuite.scala | 30 +++++++++++++++ 4 files changed, 79 insertions(+), 8 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala b/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala index b5d78a9465f..876ecb6c241 100644 --- a/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala +++ b/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala @@ -39,6 +39,10 @@ import org.apache.comet.util.ClassLoaders */ object IcebergReflection extends Logging { + case class ParquetPathResolution( + pathByIcebergColumnName: Map[String, String], + renamedIcebergColumnNames: Set[String]) + /** * Iceberg class names used throughout Comet. */ @@ -701,7 +705,7 @@ object IcebergReflection extends Logging { * https://github.com/apache/iceberg/blob/apache-iceberg-1.11.0/parquet/src/main/java/org/apache/iceberg/parquet/Parquet.java#L411-L421 */ // scalastyle:on line.size.limit - def getParquetPathByIcebergColumnName(schema: Any): Option[Map[String, String]] = { + def getParquetPathResolution(schema: Any): Option[ParquetPathResolution] = { import scala.jdk.CollectionConverters._ try { val parquetSchemaUtil = loadClass(ClassNames.PARQUET_SCHEMA_UTIL) @@ -712,21 +716,43 @@ object IcebergReflection extends Logging { .invoke(parquetSchema) .asInstanceOf[java.util.List[AnyRef]] val findColumnName = getMethod(schema.getClass, "findColumnName", classOf[Int]) + val findField = getMethod(schema.getClass, "findField", classOf[Int]) - Some(columns.asScala.flatMap { column => + val resolved = columns.asScala.flatMap { column => val primitiveType = getMethod(column.getClass, "getPrimitiveType").invoke(column) val parquetId = getMethod(primitiveType.getClass, "getId").invoke(primitiveType) Option(parquetId).flatMap { id => val fieldId = getMethod(id.getClass, "intValue").invoke(id).asInstanceOf[Int] Option(findColumnName.invoke(schema, Int.box(fieldId))).map { icebergName => - val parquetPath = getMethod(column.getClass, "getPath") + val parquetPathParts = getMethod(column.getClass, "getPath") .invoke(column) .asInstanceOf[Array[String]] - .mkString(".") - icebergName.asInstanceOf[String] -> parquetPath + var parquetType = parquetSchema.asInstanceOf[AnyRef] + val renamed = parquetPathParts.exists { pathPart => + parquetType = getMethod(parquetType.getClass, "getType", classOf[String]) + .invoke(parquetType, pathPart) + .asInstanceOf[AnyRef] + Option(getMethod(parquetType.getClass, "getId").invoke(parquetType)).exists { + parquetFieldId => + val idValue = getMethod(parquetFieldId.getClass, "intValue") + .invoke(parquetFieldId) + .asInstanceOf[Int] + Option(findField.invoke(schema, Int.box(idValue))).exists { icebergField => + val icebergFieldName = getMethod(icebergField.getClass, "name") + .invoke(icebergField) + .asInstanceOf[String] + icebergFieldName != pathPart + } + } + } + (icebergName.asInstanceOf[String], parquetPathParts.mkString("."), renamed) } } - }.toMap) + }.toSeq + Some( + ParquetPathResolution( + resolved.map { case (name, path, _) => name -> path }.toMap, + resolved.collect { case (name, _, true) => name }.toSet)) } catch { case e: Exception => logError(s"Iceberg reflection failure: Parquet column paths: ${e.getMessage}") diff --git a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala index e106f5e1844..c87b1f0d31f 100644 --- a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala +++ b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala @@ -1267,10 +1267,25 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { val effectiveProperties = properties ++ resolvedWriteProperties val parquetPathByIcebergColumnName = if (IcebergWriteProtoTranslation.hasEnabledBloomFilters(effectiveProperties)) { - IcebergReflection.getParquetPathByIcebergColumnName(writeSchema).getOrElse { + val resolution = IcebergReflection.getParquetPathResolution(writeSchema).getOrElse { withFallbackReason(op, "Could not resolve physical Parquet paths for Bloom filters") return None } + val renamedColumns = + IcebergWriteProtoTranslation + .enabledBloomFilterColumnNames(effectiveProperties) + .filter(resolution.renamedIcebergColumnNames) + if (renamedColumns.nonEmpty) { + // Iceberg Java sanitizes these Parquet names, while the pinned iceberg-rust Arrow + // conversion preserves them. Passing the Java path to parquet-rs would silently miss + // the native writer column, so retain the JVM writer until paths travel structurally. + withFallbackReason( + op, + s"Bloom-filter columns are renamed in Iceberg Java's Parquet schema: " + + renamedColumns.mkString(", ")) + return None + } + resolution.pathByIcebergColumnName } else { Map.empty[String, String] } diff --git a/spark/src/main/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslation.scala b/spark/src/main/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslation.scala index 4f98a21c8df..336a5b733f4 100644 --- a/spark/src/main/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslation.scala +++ b/spark/src/main/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslation.scala @@ -125,7 +125,7 @@ object IcebergWriteProtoTranslation { .toSeq .sorted - private def enabledBloomFilterColumnNames(props: Map[String, String]): Seq[String] = + private[operator] def enabledBloomFilterColumnNames(props: Map[String, String]): Seq[String] = configuredBloomFilterColumnNames(props).filter { column => val enabled = java.lang.Boolean.valueOf(props(Keys.ParquetBloomFilterColumnEnabledPrefix + column)) diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala index 4dac2cc05b6..855384d550f 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala @@ -1401,6 +1401,36 @@ class CometIcebergWriteActionSuite } } + test("quoted bloom-filter columns renamed by Iceberg fall back with a written filter") { + assumeNativeAcceleration() + withIcebergCatalog { _ => + spark.sql(s""" + CREATE TABLE $catalog.$ns.bloom_quoted_name ( + `order id` INT + ) USING iceberg + TBLPROPERTIES ( + 'write.parquet.bloom-filter-enabled.column.order id'='true' + ) + """) + + val snapshot = withNativeEnabled { + captureWrite("bloom_quoted_name") { + spark.sql(s"INSERT INTO $catalog.$ns.bloom_quoted_name VALUES (1), (2)") + } + } + assertExactlyOneCommit(snapshot) + val nativeExecs = snapshot.plans.flatMap { plan => + collectWithSubqueries(plan) { case exec: CometIcebergWriteExec => exec } + } + assert( + nativeExecs.isEmpty, + s"expected the sanitized Bloom-filter path to fall back, plans:\n${snapshot.plans.mkString("\n--\n")}") + assert( + parquetBloomFilterBytes("bloom_quoted_name", "order_x20id").nonEmpty, + "expected the fallback Parquet Java writer to emit the requested Bloom filter") + } + } + test("native bloom filter is byte-identical to parquet-mr when max-bytes binds") { assumeNativeAcceleration() assumeIcebergBloomShapeProperties() From 07a77c17e1a7542da3db6306f6c04cb3bcda69f2 Mon Sep 17 00:00:00 2001 From: Nikita Matskevich Date: Tue, 8 Sep 2026 10:37:40 +0200 Subject: [PATCH 04/32] test: gate Iceberg bloom shape properties independently --- .../CometIcebergWriteDetectionSuite.scala | 109 ++++++++++++------ 1 file changed, 71 insertions(+), 38 deletions(-) diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala index 5fc91e65c52..e5e6355a9e5 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala @@ -405,22 +405,30 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes } test("bloom-filter max-bytes=32 falls back only when parquet-mr ignores the cap") { - assumeIcebergBloomShapeProperties() withDetectionCatalog { dir => val minimumBytes = 32 val naturallyMinimumNdv = 1 val bindingFpp = 0.0001 val bindingNdv = 1000000 - Seq( - ("without_ndv", s"'write.parquet.bloom-filter-max-bytes'='$minimumBytes'"), - ( - "natural_minimum", + createTable( + dir, + "without_ndv", + partitionSpec = "", + properties = Some(s"'write.parquet.bloom-filter-max-bytes'='$minimumBytes'")) + assertSupportLevelIs[Compatible]("without_ndv") + + createTable( + dir, + "natural_minimum", + partitionSpec = "", + properties = Some( "'write.parquet.bloom-filter-enabled.column.id'='true', " + s"'write.parquet.bloom-filter-ndv.column.id'='$naturallyMinimumNdv', " + - s"'write.parquet.bloom-filter-max-bytes'='$minimumBytes'")).foreach { - case (table, properties) => - createTable(dir, table, partitionSpec = "", properties = Some(properties)) - assertSupportLevelIs[Compatible](table) + s"'write.parquet.bloom-filter-max-bytes'='$minimumBytes'")) + if (icebergSupportsBloomNdv) { + assertSupportLevelIs[Compatible]("natural_minimum") + } else { + assertUnsupportedContains("natural_minimum", "is not interpreted") } createTable( @@ -432,14 +440,17 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes s"'write.parquet.bloom-filter-fpp.column.id'='$bindingFpp', " + s"'write.parquet.bloom-filter-ndv.column.id'='$bindingNdv', " + s"'write.parquet.bloom-filter-max-bytes'='$minimumBytes'")) - assertUnsupportedContainsAllowingWriteFailure( - "ignored_minimum_cap", - "write.parquet.bloom-filter-max-bytes") + if (icebergSupportsBloomFpp && icebergSupportsBloomNdv) { + assertUnsupportedContainsAllowingWriteFailure( + "ignored_minimum_cap", + "write.parquet.bloom-filter-max-bytes") + } else { + assertUnsupportedContainsAllowingWriteFailure("ignored_minimum_cap", "is not interpreted") + } } } test("fall-back: bloom-filter NDV values that overflow parquet-mr sizing arithmetic") { - assumeIcebergBloomShapeProperties() withDetectionCatalog { dir => val largestNonOverflowingNdv = Long.MaxValue / 8L val cases = Seq( @@ -457,19 +468,19 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes properties = Some( "'write.parquet.bloom-filter-enabled.column.id'='true', " + s"'write.parquet.bloom-filter-ndv.column.id'='$ndv'")) - if (compatible) { + if (compatible && icebergSupportsBloomNdv) { assertSupportLevelIs[Compatible](table) } else { assertUnsupportedContainsAllowingWriteFailure( table, - "write.parquet.bloom-filter-ndv.column.id") + if (icebergSupportsBloomNdv) "write.parquet.bloom-filter-ndv.column.id" + else "is not interpreted") } } } } - test("bloom-filter enabled=false still applies and validates FPP and NDV") { - assumeIcebergBloomShapeProperties() + test("bloom-filter enabled=false with NDV remains enabled and validates NDV") { withDetectionCatalog { dir => val configuredNdv = 1000 createTable( @@ -479,21 +490,39 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes properties = Some( "'write.parquet.bloom-filter-enabled.column.id'='false', " + s"'write.parquet.bloom-filter-ndv.column.id'='$configuredNdv'")) - assertSupportLevelIs[Compatible]("false_with_ndv") - - Seq( - "'write.parquet.bloom-filter-fpp.column.id'='garbage'", - "'write.parquet.bloom-filter-ndv.column.id'='garbage'").zipWithIndex.foreach { - case (property, index) => - val table = s"false_with_bad_shape_$index" - createTable( - dir, - table, - partitionSpec = "", - properties = - Some(s"'write.parquet.bloom-filter-enabled.column.id'='false', $property")) - assertUnsupportedContainsAllowingWriteFailure(table, "write.parquet.bloom-filter") + if (icebergSupportsBloomNdv) { + assertSupportLevelIs[Compatible]("false_with_ndv") + } else { + assertUnsupportedContains("false_with_ndv", "is not interpreted") } + + createTable( + dir, + "false_with_bad_ndv", + partitionSpec = "", + properties = Some( + "'write.parquet.bloom-filter-enabled.column.id'='false', " + + "'write.parquet.bloom-filter-ndv.column.id'='garbage'")) + assertUnsupportedContainsAllowingWriteFailure( + "false_with_bad_ndv", + if (icebergSupportsBloomNdv) "write.parquet.bloom-filter-ndv" + else "is not interpreted") + } + } + + test("bloom-filter enabled=false still validates FPP") { + withDetectionCatalog { dir => + createTable( + dir, + "false_with_bad_fpp", + partitionSpec = "", + properties = Some( + "'write.parquet.bloom-filter-enabled.column.id'='false', " + + "'write.parquet.bloom-filter-fpp.column.id'='garbage'")) + assertUnsupportedContainsAllowingWriteFailure( + "false_with_bad_fpp", + if (icebergSupportsBloomFpp) "write.parquet.bloom-filter-fpp" + else "is not interpreted") } } @@ -1910,13 +1939,15 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes } } - private def assumeIcebergBloomShapeProperties(): Unit = { - assume( - IcebergReflection - .tablePropertyConstantOpt("PARQUET_BLOOM_FILTER_COLUMN_FPP_PREFIX") - .isDefined, - "Iceberg runtime does not interpret per-column Bloom FPP/NDV properties") - } + private def icebergSupportsBloomFpp: Boolean = + IcebergReflection + .tablePropertyConstantOpt("PARQUET_BLOOM_FILTER_COLUMN_FPP_PREFIX") + .isDefined + + private def icebergSupportsBloomNdv: Boolean = + IcebergReflection + .tablePropertyConstantOpt("PARQUET_BLOOM_FILTER_COLUMN_NDV_PREFIX") + .isDefined /** * Runs Spark's transition insertion followed by [[EliminateRedundantTransitions]] over a @@ -1988,6 +2019,8 @@ case class TransitionProbeLeaf(columnar: Boolean) extends LeafExecNode { throw new UnsupportedOperationException("planning-only node") override protected def doExecuteColumnar(): RDD[ColumnarBatch] = throw new UnsupportedOperationException("planning-only node") +} + /** * A FileIO that works normally (delegating to HadoopFileIO) but whose class is not on Comet's * recognized-FileIO allowlist. Composition rather than inheritance is the point: a HadoopFileIO From e15137daf536b6793460404b89bd385a718be753 Mon Sep 17 00:00:00 2001 From: Nikita Matskevich Date: Tue, 8 Sep 2026 11:41:20 +0200 Subject: [PATCH 05/32] fix: match Iceberg-version bloom property support --- .../operator/CometIcebergNativeWrite.scala | 52 +++-- .../IcebergWriteProtoTranslation.scala | 5 +- .../comet/CometIcebergWriteActionSuite.scala | 220 +++++++++++++----- .../CometIcebergWriteDetectionSuite.scala | 67 +++--- 4 files changed, 225 insertions(+), 119 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala index c87b1f0d31f..0f7b42ae14a 100644 --- a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala +++ b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala @@ -606,26 +606,33 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { private val MaxNonOverflowingBloomFilterNdv = Long.MaxValue / BloomFilterHashProbes /** - * parquet-rs 58.x represents Bloom filters as a power-of-two number of bytes. parquet-mr - * accepts arbitrary caps and, when one binds, serializes that exact length. Keep those writes - * on the classic path instead of silently changing the number of usable Bloom blocks. + * Keep only Bloom shape properties interpreted by the Iceberg runtime on the classpath. Older + * Iceberg releases leave these table properties untouched but do not pass them to parquet-mr. + * Ignoring them here preserves that version's JVM-writer behavior while allowing the remaining + * supported Bloom configuration to execute natively. */ - private val requireNativeSupportedBloomFilterProperties: TriggerRule = ctx => { - // The FPP constant is absent from Iceberg 1.5.2, and the NDV constant is absent through - // 1.10. Use the literal prefixes to detect the properties, then optional reflection to check - // capability: an older runtime ignores an explicit property, so that write must fall back. - val unavailableRuntimeProperty = Seq( + private def interpretedBloomFilterProperties( + properties: Map[String, String]): Map[String, String] = { + val unsupportedPrefixes = Seq( PropertyKeys.ParquetBloomFilterColumnFppPrefix -> "PARQUET_BLOOM_FILTER_COLUMN_FPP_PREFIX", PropertyKeys.ParquetBloomFilterColumnNdvPrefix -> - "PARQUET_BLOOM_FILTER_COLUMN_NDV_PREFIX").collectFirst { - case (prefix, constant) - if ctx.properties.keys.exists(_.startsWith(prefix)) && - IcebergReflection.tablePropertyConstantOpt(constant).isEmpty => - s"$prefix* is not interpreted by the Iceberg version on the classpath" + "PARQUET_BLOOM_FILTER_COLUMN_NDV_PREFIX").collect { + case (prefix, constant) if IcebergReflection.tablePropertyConstantOpt(constant).isEmpty => + prefix } + properties.filterNot { case (key, _) => unsupportedPrefixes.exists(key.startsWith) } + } + + /** + * parquet-rs 58.x represents Bloom filters as a power-of-two number of bytes. parquet-mr + * accepts arbitrary caps and, when one binds, serializes that exact length. Keep those writes + * on the classic path instead of silently changing the number of usable Bloom blocks. + */ + private val requireNativeSupportedBloomFilterProperties: TriggerRule = ctx => { + val properties = interpretedBloomFilterProperties(ctx.properties) val maxRejection = - ctx.properties.get(PropertyKeys.ParquetBloomFilterMaxBytes).flatMap { raw => + properties.get(PropertyKeys.ParquetBloomFilterMaxBytes).flatMap { raw => scala.util.Try(java.lang.Integer.parseInt(raw)).toOption match { case None => Some(s"${PropertyKeys.ParquetBloomFilterMaxBytes}=$raw is not a Java int") case Some(value) @@ -638,15 +645,15 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { } } - maxRejection.orElse(unavailableRuntimeProperty).orElse { - val maxBytes = ctx.properties + maxRejection.orElse { + val maxBytes = properties .get(PropertyKeys.ParquetBloomFilterMaxBytes) .flatMap(raw => scala.util.Try(java.lang.Integer.parseInt(raw)).toOption) .getOrElse(IcebergWriteProtoTranslation.Defaults.BloomFilterMaxBytes) // Iceberg visits every enabled-prefix entry and applies enabled, FPP, then NDV. Validate // the associated shape properties even for enabled=false; a valid NDV also re-enables the // filter in parquet-mr. - val configured = ctx.properties.iterator.collect { + val configured = properties.iterator.collect { case (key, _) if key.startsWith(PropertyKeys.BloomFilterColumnEnabledPrefix) => key.substring(PropertyKeys.BloomFilterColumnEnabledPrefix.length) }.toSeq @@ -654,14 +661,14 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { .flatMap { column => val fppKey = PropertyKeys.ParquetBloomFilterColumnFppPrefix + column val ndvKey = PropertyKeys.ParquetBloomFilterColumnNdvPrefix + column - val parsedFpp = ctx.properties.get(fppKey) match { + val parsedFpp = properties.get(fppKey) match { case Some(raw) => scala.util.Try(java.lang.Double.parseDouble(raw)).toOption case None => Some(IcebergWriteProtoTranslation.Defaults.BloomFilterFpp) } - val parsedNdv = ctx.properties + val parsedNdv = properties .get(ndvKey) .flatMap(raw => scala.util.Try(java.lang.Long.parseLong(raw)).toOption) - val fppError = ctx.properties.get(fppKey).flatMap { raw => + val fppError = properties.get(fppKey).flatMap { raw => parsedFpp match { case Some(value) if value > 0.0d && value < 1.0d && java.lang.Double.isFinite(value) && @@ -673,7 +680,7 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { case _ => Some(s"$fppKey=$raw must be a finite double strictly between 0 and 1") } } - val ndvError = ctx.properties.get(ndvKey).flatMap { raw => + val ndvError = properties.get(ndvKey).flatMap { raw => parsedNdv match { case Some(value) if value > 0L && value <= MaxNonOverflowingBloomFilterNdv => None case Some(value) if value > MaxNonOverflowingBloomFilterNdv => @@ -1264,7 +1271,8 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { // `option("write-parquet-compression-codec", "gzip")`) survive into the native writer. val resolvedWriteProperties = IcebergReflection.getWritePropertiesFromSparkWrite(sparkWrite).getOrElse(Map.empty) - val effectiveProperties = properties ++ resolvedWriteProperties + val effectiveProperties = + interpretedBloomFilterProperties(properties ++ resolvedWriteProperties) val parquetPathByIcebergColumnName = if (IcebergWriteProtoTranslation.hasEnabledBloomFilters(effectiveProperties)) { val resolution = IcebergReflection.getParquetPathResolution(writeSchema).getOrElse { diff --git a/spark/src/main/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslation.scala b/spark/src/main/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslation.scala index 336a5b733f4..108b6b38844 100644 --- a/spark/src/main/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslation.scala +++ b/spark/src/main/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslation.scala @@ -61,8 +61,9 @@ object IcebergWriteProtoTranslation { IcebergReflection.tablePropertyConstant("PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX") lazy val ParquetBloomFilterMaxBytes: String = IcebergReflection.tablePropertyConstant("PARQUET_BLOOM_FILTER_MAX_BYTES") - // These were added to Iceberg after bloom enablement/max-bytes. Literals let older runtimes - // keep using the defaults; detection rejects an explicit property they would ignore. + // These were added to Iceberg after bloom enablement/max-bytes. Literals let the translation + // support them without a hard binary dependency; the caller removes either prefix when the + // Iceberg runtime does not interpret it, matching that runtime's JVM writer. val ParquetBloomFilterColumnFppPrefix = "write.parquet.bloom-filter-fpp.column." val ParquetBloomFilterColumnNdvPrefix = "write.parquet.bloom-filter-ndv.column." } diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala index 855384d550f..d9d9b62fdea 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala @@ -1263,9 +1263,8 @@ class CometIcebergWriteActionSuite } } - test("explicit NDV re-enables a bloom filter after enabled=false") { + test("enabled=false plus NDV matches JVM behavior for the Iceberg runtime") { assumeNativeAcceleration() - assumeIcebergBloomShapeProperties() withIcebergCatalog { warehouseDir => val configuredFpp = 0.01 val configuredNdv = 1000 @@ -1288,45 +1287,65 @@ class CometIcebergWriteActionSuite insert("bloom_false_ndv_jvm") } - val native = parquetBloomFilterBytes("bloom_false_ndv_native", "id") - val jvm = parquetBloomFilterBytes("bloom_false_ndv_jvm", "id") - assert(native.map(_.length) == jvm.map(_.length)) - assert(native.zip(jvm).forall { case (left, right) => - java.util.Arrays.equals(left, right) - }) + val nativeHasBloom = parquetBloomFilterPresent("bloom_false_ndv_native", "id") + val jvmHasBloom = parquetBloomFilterPresent("bloom_false_ndv_jvm", "id") + assert(nativeHasBloom == jvmHasBloom) + if (icebergSupportsBloomNdv) { + assert(nativeHasBloom, "an interpreted NDV must re-enable the Bloom filter") + val native = parquetBloomFilterBytes("bloom_false_ndv_native", "id") + val jvm = parquetBloomFilterBytes("bloom_false_ndv_jvm", "id") + assert(native.map(_.length) == jvm.map(_.length)) + assert(native.zip(jvm).forall { case (left, right) => + java.util.Arrays.equals(left, right) + }) + } else { + assert(!nativeHasBloom, "an uninterpreted NDV must not override enabled=false") + } } } test("enabled=false preserves JVM validation errors for malformed FPP and NDV") { assumeNativeAcceleration() - assumeIcebergBloomShapeProperties() withIcebergCatalog { warehouseDir => Seq( - "fpp" -> "'write.parquet.bloom-filter-fpp.column.id'='garbage'", - "ndv" -> "'write.parquet.bloom-filter-ndv.column.id'='garbage'").foreach { - case (suffix, malformedProperty) => - val table = s"bloom_false_bad_$suffix" - createTable( - warehouseDir, - table, - partitionSpec = "", - properties = - Some(s"'write.parquet.bloom-filter-enabled.column.id'='false', $malformedProperty")) + ("fpp", "'write.parquet.bloom-filter-fpp.column.id'='garbage'", icebergSupportsBloomFpp), + ("ndv", "'write.parquet.bloom-filter-ndv.column.id'='garbage'", icebergSupportsBloomNdv)) + .foreach { case (suffix, malformedProperty, interpreted) => + val properties = + Some(s"'write.parquet.bloom-filter-enabled.column.id'='false', $malformedProperty") + val nativeTable = s"bloom_false_bad_${suffix}_native" + val jvmTable = s"bloom_false_bad_${suffix}_jvm" + Seq(nativeTable, jvmTable).foreach { table => + createTable(warehouseDir, table, partitionSpec = "", properties = properties) + } - def insert(): Unit = + def insert(table: String): Unit = spark.sql(s"INSERT INTO cat.db.$table VALUES (1, 'region', 1.0)") - val withComet = intercept[Throwable] { - withNativeEnabled(insert()) - } - val withJvm = intercept[Throwable] { - withSQLConf(CometConf.COMET_ICEBERG_NATIVE_WRITE_ENABLED.key -> "false")(insert()) + if (interpreted) { + val withComet = intercept[Throwable] { + withNativeEnabled(insert(nativeTable)) + } + val withJvm = intercept[Throwable] { + withSQLConf(CometConf.COMET_ICEBERG_NATIVE_WRITE_ENABLED.key -> "false") { + insert(jvmTable) + } + } + val cometCause = exceptionChain(withComet).last + val jvmCause = exceptionChain(withJvm).last + assert(cometCause.getClass == jvmCause.getClass) + assert(cometCause.getMessage == jvmCause.getMessage) + } else { + assertNativeWriteEngages(nativeTable, Seq(1))(insert(nativeTable)) + withSQLConf(CometConf.COMET_ICEBERG_NATIVE_WRITE_ENABLED.key -> "false") { + insert(jvmTable) + } + assert( + !parquetBloomFilterPresent(nativeTable, "id") && + !parquetBloomFilterPresent(jvmTable, "id"), + s"uninterpreted $suffix must not enable a Bloom filter") } - val cometCause = exceptionChain(withComet).last - val jvmCause = exceptionChain(withJvm).last - assert(cometCause.getClass == jvmCause.getClass) - assert(cometCause.getMessage == jvmCause.getMessage) - } + } } } @@ -1401,17 +1420,19 @@ class CometIcebergWriteActionSuite } } - test("quoted bloom-filter columns renamed by Iceberg fall back with a written filter") { + test("quoted bloom-filter columns renamed by Iceberg fall back to the JVM footer") { assumeNativeAcceleration() withIcebergCatalog { _ => - spark.sql(s""" - CREATE TABLE $catalog.$ns.bloom_quoted_name ( - `order id` INT - ) USING iceberg - TBLPROPERTIES ( - 'write.parquet.bloom-filter-enabled.column.order id'='true' - ) - """) + Seq("bloom_quoted_name", "bloom_quoted_name_jvm").foreach { table => + spark.sql(s""" + CREATE TABLE $catalog.$ns.$table ( + `order id` INT + ) USING iceberg + TBLPROPERTIES ( + 'write.parquet.bloom-filter-enabled.column.order id'='true' + ) + """) + } val snapshot = withNativeEnabled { captureWrite("bloom_quoted_name") { @@ -1425,9 +1446,17 @@ class CometIcebergWriteActionSuite assert( nativeExecs.isEmpty, s"expected the sanitized Bloom-filter path to fall back, plans:\n${snapshot.plans.mkString("\n--\n")}") + withSQLConf(CometConf.COMET_ICEBERG_NATIVE_WRITE_ENABLED.key -> "false") { + spark.sql(s"INSERT INTO $catalog.$ns.bloom_quoted_name_jvm VALUES (1), (2)") + } + val fallbackHasBloom = parquetBloomFilterPresent("bloom_quoted_name", "order_x20id") + val jvmHasBloom = parquetBloomFilterPresent("bloom_quoted_name_jvm", "order_x20id") assert( - parquetBloomFilterBytes("bloom_quoted_name", "order_x20id").nonEmpty, - "expected the fallback Parquet Java writer to emit the requested Bloom filter") + fallbackHasBloom == jvmHasBloom, + "expected the fallback write to match the Parquet Java footer") + if (icebergSupportsBloomFpp) { + assert(jvmHasBloom, "expected this Iceberg version to write the quoted-column filter") + } } } @@ -1474,40 +1503,74 @@ class CometIcebergWriteActionSuite } } - test("native bloom sizing covers FPP and NDV presence combinations and binding caps") { + test("native bloom sizing matches JVM for shape properties supported by the Iceberg runtime") { assumeNativeAcceleration() - assumeIcebergBloomShapeProperties() withIcebergCatalog { warehouseDir => val enabled = "'write.parquet.bloom-filter-enabled.column.id'='true'" - val cases: Seq[(String, String)] = Seq( - ("fpp_only", s"$enabled, 'write.parquet.bloom-filter-fpp.column.id'='0.02'"), - ("ndv_only", s"$enabled, 'write.parquet.bloom-filter-ndv.column.id'='1000'"), + val cases: Seq[(String, String, Boolean, Boolean)] = Seq( + ( + "fpp_only", + s"$enabled, 'write.parquet.bloom-filter-fpp.column.id'='0.02'", + icebergSupportsBloomFpp, + false), + ( + "ndv_only", + s"$enabled, 'write.parquet.bloom-filter-ndv.column.id'='1000'", + icebergSupportsBloomNdv, + true), ( "both", s"$enabled, 'write.parquet.bloom-filter-fpp.column.id'='0.005', " + - "'write.parquet.bloom-filter-ndv.column.id'='1000'"), + "'write.parquet.bloom-filter-ndv.column.id'='1000'", + icebergSupportsBloomFpp && icebergSupportsBloomNdv, + true), // The requested NDV/FPP needs far more than 64 bytes. Like parquet-mr, max wins and the // target FPP becomes impossible to guarantee, but membership must remain correct. ( "binding_cap", s"$enabled, 'write.parquet.bloom-filter-fpp.column.id'='0.0001', " + "'write.parquet.bloom-filter-ndv.column.id'='1000000', " + - "'write.parquet.bloom-filter-max-bytes'='64'")) + "'write.parquet.bloom-filter-max-bytes'='64'", + icebergSupportsBloomFpp && icebergSupportsBloomNdv, + true)) - cases.zipWithIndex.foreach { case ((suffix, properties), index) => - val table = s"bloom_shape_${suffix}" - createTable(warehouseDir, table, partitionSpec = "", properties = Some(properties)) - val ids = index * 256 until (index + 1) * 256 - assertNativeWriteEngages(table, ids) { - spark.sql( - s"INSERT INTO cat.db.$table SELECT CAST(id AS INT), 'region', " + - s"CAST(id AS DOUBLE) FROM range(${ids.start}, ${ids.end}, 1, 1)") + cases.filter(_._3).foreach { case (suffix, properties, _, expectByteIdentity) => + val nativeTable = s"bloom_shape_${suffix}_native" + val jvmTable = s"bloom_shape_${suffix}_jvm" + Seq(nativeTable, jvmTable).foreach { table => + createTable(warehouseDir, table, partitionSpec = "", properties = Some(properties)) + } + // Keep the explicit-NDV cases at their estimated cardinality so parquet-rs does not + // fold their allocation before byte-parity is checked. + val insertedCardinality = if (suffix == "ndv_only" || suffix == "both") 1000 else 256 + val ids = 0 until insertedCardinality + def insert(table: String): Unit = + spark.sql(s"INSERT INTO cat.db.$table SELECT CAST(id AS INT), 'region', " + + s"CAST(id AS DOUBLE) FROM range(${ids.start}, ${ids.end}, 1, 1)") + + assertNativeWriteEngages(nativeTable, ids) { + insert(nativeTable) + } + withSQLConf(CometConf.COMET_ICEBERG_NATIVE_WRITE_ENABLED.key -> "false") { + insert(jvmTable) + } + val native = parquetBloomFilterBytes(nativeTable, "id") + val jvm = parquetBloomFilterBytes(jvmTable, "id") + if (expectByteIdentity) { + assert(native.map(_.length) == jvm.map(_.length)) + assert(native.zip(jvm).forall { case (left, right) => + java.util.Arrays.equals(left, right) + }) + } else { + // Without explicit NDV, parquet-rs can fold to the observed cardinality while Parquet + // Java retains its initial maximum allocation. Folding is safe for readers, so require + // only that native uses no more space and still contains every inserted value. + assert(native.zip(jvm).forall { case (left, right) => left.length <= right.length }) + assertParquetBloomContainsInts(nativeTable, "id", ids) } - val bytes = parquetBloomFilterBytes(table, "id") - assert(bytes.nonEmpty && bytes.forall(_.nonEmpty)) if (suffix == "binding_cap") { - assert(bytes.forall(_.length == 64), s"expected binding 64-byte cap for $table") - assertParquetBloomContainsInts(table, "id", ids) + assert(native.forall(_.length == 64), s"expected binding 64-byte cap for $nativeTable") + assertParquetBloomContainsInts(nativeTable, "id", ids) } } } @@ -5272,6 +5335,29 @@ class CometIcebergWriteActionSuite } } + private def parquetBloomFilterPresent(tableName: String, columnName: String): Boolean = { + val paths = spark + .sql(s"SELECT file_path FROM $catalog.$ns.$tableName.data_files ORDER BY file_path") + .collect() + .map(_.getString(0)) + val conf = spark.sparkContext.hadoopConfiguration + val present = paths.toSeq.flatMap { path => + val input = HadoopInputFile.fromPath(new Path(path.stripPrefix("file:")), conf) + val reader = ParquetFileReader.open(input) + try { + reader.getFooter.getBlocks.asScala.flatMap { block => + block.getColumns.asScala + .filter(_.getPath.toDotString == columnName) + .map(column => reader.readBloomFilter(column) != null) + } + } finally { + reader.close() + } + } + assert(present.nonEmpty, s"expected Parquet column $columnName in $tableName") + present.forall(identity) + } + private def assertParquetBloomContainsInts( tableName: String, columnName: String, @@ -5311,12 +5397,20 @@ class CometIcebergWriteActionSuite private def assumeIcebergBloomShapeProperties(): Unit = { assume( - org.apache.comet.iceberg.IcebergReflection - .tablePropertyConstantOpt("PARQUET_BLOOM_FILTER_COLUMN_FPP_PREFIX") - .isDefined, + icebergSupportsBloomFpp && icebergSupportsBloomNdv, "Iceberg runtime does not interpret per-column Bloom FPP/NDV properties") } + private def icebergSupportsBloomFpp: Boolean = + org.apache.comet.iceberg.IcebergReflection + .tablePropertyConstantOpt("PARQUET_BLOOM_FILTER_COLUMN_FPP_PREFIX") + .isDefined + + private def icebergSupportsBloomNdv: Boolean = + org.apache.comet.iceberg.IcebergReflection + .tablePropertyConstantOpt("PARQUET_BLOOM_FILTER_COLUMN_NDV_PREFIX") + .isDefined + /** * Every row-consuming operator must receive row-based input. Spark guarantees that by inserting * `ColumnarToRow` transitions in `ApplyColumnarRulesAndInsertTransitions`; an operator that diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala index e5e6355a9e5..985aed64a5c 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala @@ -425,11 +425,7 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes "'write.parquet.bloom-filter-enabled.column.id'='true', " + s"'write.parquet.bloom-filter-ndv.column.id'='$naturallyMinimumNdv', " + s"'write.parquet.bloom-filter-max-bytes'='$minimumBytes'")) - if (icebergSupportsBloomNdv) { - assertSupportLevelIs[Compatible]("natural_minimum") - } else { - assertUnsupportedContains("natural_minimum", "is not interpreted") - } + assertSupportLevelIs[Compatible]("natural_minimum") createTable( dir, @@ -445,7 +441,7 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes "ignored_minimum_cap", "write.parquet.bloom-filter-max-bytes") } else { - assertUnsupportedContainsAllowingWriteFailure("ignored_minimum_cap", "is not interpreted") + assertSupportLevelIs[Compatible]("ignored_minimum_cap") } } } @@ -468,19 +464,18 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes properties = Some( "'write.parquet.bloom-filter-enabled.column.id'='true', " + s"'write.parquet.bloom-filter-ndv.column.id'='$ndv'")) - if (compatible && icebergSupportsBloomNdv) { + if (compatible || !icebergSupportsBloomNdv) { assertSupportLevelIs[Compatible](table) } else { assertUnsupportedContainsAllowingWriteFailure( table, - if (icebergSupportsBloomNdv) "write.parquet.bloom-filter-ndv.column.id" - else "is not interpreted") + "write.parquet.bloom-filter-ndv.column.id") } } } } - test("bloom-filter enabled=false with NDV remains enabled and validates NDV") { + test("bloom-filter enabled=false with NDV follows runtime support and validates NDV") { withDetectionCatalog { dir => val configuredNdv = 1000 createTable( @@ -490,11 +485,7 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes properties = Some( "'write.parquet.bloom-filter-enabled.column.id'='false', " + s"'write.parquet.bloom-filter-ndv.column.id'='$configuredNdv'")) - if (icebergSupportsBloomNdv) { - assertSupportLevelIs[Compatible]("false_with_ndv") - } else { - assertUnsupportedContains("false_with_ndv", "is not interpreted") - } + assertSupportLevelIs[Compatible]("false_with_ndv") createTable( dir, @@ -503,10 +494,13 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes properties = Some( "'write.parquet.bloom-filter-enabled.column.id'='false', " + "'write.parquet.bloom-filter-ndv.column.id'='garbage'")) - assertUnsupportedContainsAllowingWriteFailure( - "false_with_bad_ndv", - if (icebergSupportsBloomNdv) "write.parquet.bloom-filter-ndv" - else "is not interpreted") + if (icebergSupportsBloomNdv) { + assertUnsupportedContainsAllowingWriteFailure( + "false_with_bad_ndv", + "write.parquet.bloom-filter-ndv") + } else { + assertSupportLevelIs[Compatible]("false_with_bad_ndv") + } } } @@ -519,25 +513,30 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes properties = Some( "'write.parquet.bloom-filter-enabled.column.id'='false', " + "'write.parquet.bloom-filter-fpp.column.id'='garbage'")) - assertUnsupportedContainsAllowingWriteFailure( - "false_with_bad_fpp", - if (icebergSupportsBloomFpp) "write.parquet.bloom-filter-fpp" - else "is not interpreted") + if (icebergSupportsBloomFpp) { + assertUnsupportedContainsAllowingWriteFailure( + "false_with_bad_fpp", + "write.parquet.bloom-filter-fpp") + } else { + assertSupportLevelIs[Compatible]("false_with_bad_fpp") + } } } test("fall-back: invalid enabled-column FPP and NDV") { withDetectionCatalog { dir => Seq( - "'write.parquet.bloom-filter-fpp.column.id'='0'", - "'write.parquet.bloom-filter-fpp.column.id'='1'", - "'write.parquet.bloom-filter-fpp.column.id'='NaN'", + ("'write.parquet.bloom-filter-fpp.column.id'='0'", icebergSupportsBloomFpp), + ("'write.parquet.bloom-filter-fpp.column.id'='1'", icebergSupportsBloomFpp), + ("'write.parquet.bloom-filter-fpp.column.id'='NaN'", icebergSupportsBloomFpp), // Positive and below one, but too small for any integer NDV to encode the requested // power-of-two allocation through parquet-rs's NDV/FPP API. - "'write.parquet.bloom-filter-fpp.column.id'='4.9E-324'", - "'write.parquet.bloom-filter-ndv.column.id'='0'", - "'write.parquet.bloom-filter-ndv.column.id'='garbage'").zipWithIndex.foreach { - case (property, index) => + ("'write.parquet.bloom-filter-fpp.column.id'='4.9E-324'", icebergSupportsBloomFpp), + ("'write.parquet.bloom-filter-ndv.column.id'='0'", icebergSupportsBloomNdv), + ( + "'write.parquet.bloom-filter-ndv.column.id'='garbage'", + icebergSupportsBloomNdv)).zipWithIndex + .foreach { case ((property, interpreted), index) => val table = s"bloom_shape_bad_$index" createTable( dir, @@ -545,8 +544,12 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes partitionSpec = "", properties = Some(s"'write.parquet.bloom-filter-enabled.column.id'='true', $property")) - assertUnsupportedContainsAllowingWriteFailure(table, "write.parquet.bloom-filter") - } + if (interpreted) { + assertUnsupportedContainsAllowingWriteFailure(table, "write.parquet.bloom-filter") + } else { + assertSupportLevelIs[Compatible](table) + } + } } } From 90ac05dbc11a279fe68506a155059e362cd774c8 Mon Sep 17 00:00:00 2001 From: Nikita Matskevich Date: Tue, 8 Sep 2026 14:05:51 +0200 Subject: [PATCH 06/32] fix: use Parquet 59 bloom filter APIs --- native/core/src/execution/operators/iceberg_write.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/native/core/src/execution/operators/iceberg_write.rs b/native/core/src/execution/operators/iceberg_write.rs index d3a18e6270c..07c32dde665 100644 --- a/native/core/src/execution/operators/iceberg_write.rs +++ b/native/core/src/execution/operators/iceberg_write.rs @@ -2327,7 +2327,7 @@ fn build_writer_properties(settings: &IcebergParquetWriteSettings) -> DFResult Date: Tue, 8 Sep 2026 15:44:28 +0200 Subject: [PATCH 07/32] fix: satisfy formatting checks --- docs/source/user-guide/latest/iceberg-writes.md | 6 +++--- .../comet/serde/operator/CometIcebergNativeWrite.scala | 2 +- .../org/apache/comet/CometIcebergWriteActionSuite.scala | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/source/user-guide/latest/iceberg-writes.md b/docs/source/user-guide/latest/iceberg-writes.md index 13d0cf74116..9464176a421 100644 --- a/docs/source/user-guide/latest/iceberg-writes.md +++ b/docs/source/user-guide/latest/iceberg-writes.md @@ -154,9 +154,9 @@ A write is eligible only when ALL of the following hold: | `write.parquet.page-version` | unset or `v1` | | `write.parquet.shred-variants` | unset or `false` (Spark 4.x / Iceberg 1.11 resolve this into every parquet write) | | `write.parquet.variant-inference-buffer-size` | any value (only meaningful when shredding, which is gated) | -| `write.parquet.bloom-filter-enabled.column.` | `true` or `false`; an explicit NDV enables the column even when this value is `false`, matching Iceberg's property application order | -| `write.parquet.bloom-filter-fpp.column.` / `write.parquet.bloom-filter-ndv.column.` | For every column named by an `enabled` property, FPP must be a finite double strictly between 0 and 1 and NDV must be a positive Java long no greater than `Long.MAX_VALUE / 8`; the Iceberg FPP default is `0.01` | -| `write.parquet.bloom-filter-max-bytes` | unset (Iceberg default: 1 MiB), or a power of two from 32 bytes through 128 MiB inclusive; every other value falls back to iceberg-java. A 32-byte value also falls back when explicit NDV/FPP would request a larger filter, because Parquet Java ignores that boundary as a maximum. | +| `write.parquet.bloom-filter-enabled.column.` | `true` or `false`; an explicit NDV enables the column even when this value is `false`, matching Iceberg's property application order | +| `write.parquet.bloom-filter-fpp.column.` / `write.parquet.bloom-filter-ndv.column.` | For every column named by an `enabled` property, FPP must be a finite double strictly between 0 and 1 and NDV must be a positive Java long no greater than `Long.MAX_VALUE / 8`; the Iceberg FPP default is `0.01` | +| `write.parquet.bloom-filter-max-bytes` | unset (Iceberg default: 1 MiB), or a power of two from 32 bytes through 128 MiB inclusive; every other value falls back to iceberg-java. A 32-byte value also falls back when explicit NDV/FPP would request a larger filter, because Parquet Java ignores that boundary as a maximum. | | `write.metadata.metrics.*` | any value (manifest metrics are re-derived on the JVM with Iceberg's own logic) | | `write.spark.fanout.enabled` | any value (the native writer implements both clustered and fanout modes) | | `write.target-file-size-bytes` | any value (the two writers can choose different roll points; see accepted divergences) | diff --git a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala index 0f7b42ae14a..bb6e828398b 100644 --- a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala +++ b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala @@ -1289,7 +1289,7 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { // the native writer column, so retain the JVM writer until paths travel structurally. withFallbackReason( op, - s"Bloom-filter columns are renamed in Iceberg Java's Parquet schema: " + + "Bloom-filter columns are renamed in Iceberg Java's Parquet schema: " + renamedColumns.mkString(", ")) return None } diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala index d9d9b62fdea..c49f1f0b0d5 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala @@ -1277,7 +1277,7 @@ class CometIcebergWriteActionSuite def insert(table: String): Unit = spark.sql( s"INSERT INTO cat.db.$table " + - s"SELECT CAST(id AS INT), 'region', CAST(id AS DOUBLE) " + + "SELECT CAST(id AS INT), 'region', CAST(id AS DOUBLE) " + s"FROM range(0, $configuredNdv, 1, 1)") assertNativeWriteEngages("bloom_false_ndv_native", 0 until configuredNdv) { From 15554e92a456ba419fee05f6a6fabc980b5df2d1 Mon Sep 17 00:00:00 2001 From: Nikita Matskevich Date: Sat, 12 Sep 2026 23:34:10 +0200 Subject: [PATCH 08/32] docs: describe Parquet 59 bloom folding --- docs/source/user-guide/latest/iceberg-writes.md | 2 +- .../src/execution/operators/iceberg_write.rs | 16 ++++++++++------ .../serde/operator/CometIcebergNativeWrite.scala | 7 ++++--- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/docs/source/user-guide/latest/iceberg-writes.md b/docs/source/user-guide/latest/iceberg-writes.md index 9464176a421..b5828c88881 100644 --- a/docs/source/user-guide/latest/iceberg-writes.md +++ b/docs/source/user-guide/latest/iceberg-writes.md @@ -155,7 +155,7 @@ A write is eligible only when ALL of the following hold: | `write.parquet.shred-variants` | unset or `false` (Spark 4.x / Iceberg 1.11 resolve this into every parquet write) | | `write.parquet.variant-inference-buffer-size` | any value (only meaningful when shredding, which is gated) | | `write.parquet.bloom-filter-enabled.column.` | `true` or `false`; an explicit NDV enables the column even when this value is `false`, matching Iceberg's property application order | -| `write.parquet.bloom-filter-fpp.column.` / `write.parquet.bloom-filter-ndv.column.` | For every column named by an `enabled` property, FPP must be a finite double strictly between 0 and 1 and NDV must be a positive Java long no greater than `Long.MAX_VALUE / 8`; the Iceberg FPP default is `0.01` | +| `write.parquet.bloom-filter-fpp.column.` / `write.parquet.bloom-filter-ndv.column.` | Interpretation depends on the Iceberg runtime: Iceberg 1.5.2 ignores both prefixes, Iceberg 1.8.1 through 1.10.x interprets FPP but ignores NDV, and Iceberg 1.11 interprets both. For every interpreted property on a column named by an `enabled` property, FPP must be a finite double strictly between 0 and 1 and NDV must be a positive Java long no greater than `Long.MAX_VALUE / 8`; the Iceberg FPP default is `0.01` | | `write.parquet.bloom-filter-max-bytes` | unset (Iceberg default: 1 MiB), or a power of two from 32 bytes through 128 MiB inclusive; every other value falls back to iceberg-java. A 32-byte value also falls back when explicit NDV/FPP would request a larger filter, because Parquet Java ignores that boundary as a maximum. | | `write.metadata.metrics.*` | any value (manifest metrics are re-derived on the JVM with Iceberg's own logic) | | `write.spark.fanout.enabled` | any value (the native writer implements both clustered and fanout modes) | diff --git a/native/core/src/execution/operators/iceberg_write.rs b/native/core/src/execution/operators/iceberg_write.rs index 07c32dde665..2c0313d08ba 100644 --- a/native/core/src/execution/operators/iceberg_write.rs +++ b/native/core/src/execution/operators/iceberg_write.rs @@ -2351,7 +2351,7 @@ const ICEBERG_DEFAULT_BLOOM_FILTER_MAX_BYTES: usize = 1024 * 1024; /// `fpp = (1 - exp(-k * ndv / bits))^k` for `bits`, with the Parquet SBBF's `k = 8` probes. /// /// See the Apache Arrow Rust `parquet` implementation and its cited paper: -/// https://github.com/apache/arrow-rs/blob/58.4.0/parquet/src/bloom_filter/mod.rs#L369-L376 +/// https://github.com/apache/arrow-rs/blob/59.3.0/parquet/src/bloom_filter/mod.rs#L363-L376 /// http://algo2.iti.kit.edu/documents/cacheefficientbloomfilters-jea.pdf fn bloom_filter_fpp_denominator(fpp: f64) -> f64 { -(1.0 - fpp.powf(1.0 / BLOOM_FILTER_HASH_PROBES)).ln() @@ -2388,10 +2388,13 @@ fn parquet_mr_bloom_filter_bytes(ndv: Option, fpp: f64, max_bytes: usize) - allocated.min(max_bytes) } -/// Mirror the NDV/FPP sizing and power-of-two allocation used by the Apache Arrow Rust `parquet` -/// crate. Its source derives the formula from the standard Bloom-filter false-positive equation -/// with eight hash probes and links the underlying cache-efficient Bloom-filter paper: -/// https://github.com/apache/arrow-rs/blob/58.4.0/parquet/src/bloom_filter/mod.rs#L363-L395 +/// Mirror the initial NDV/FPP sizing and power-of-two allocation used by version 59.3.0 of the +/// Apache Arrow Rust `parquet` crate. Its source derives the formula from the standard +/// Bloom-filter false-positive equation with eight hash probes and links the underlying +/// cache-efficient Bloom-filter paper. After values are inserted, the 59.3.0 writer folds a +/// sparse allocation to the smallest power-of-two size that maintains the target FPP: +/// https://github.com/apache/arrow-rs/blob/59.3.0/parquet/src/bloom_filter/mod.rs#L363-L395 +/// https://github.com/apache/arrow-rs/blob/59.3.0/parquet/src/bloom_filter/mod.rs#L618-L623 fn parquet_rs_bloom_filter_bytes(ndv: u64, fpp: f64) -> usize { let num_bits = (BLOOM_FILTER_HASH_PROBES * ndv as f64 / bloom_filter_fpp_denominator(fpp)) as usize; @@ -2400,7 +2403,8 @@ fn parquet_rs_bloom_filter_bytes(ndv: u64, fpp: f64) -> usize { .next_power_of_two() } -/// Encode an exact power-of-two allocation using parquet-rs 58.x's public NDV/FPP setters. +/// Encode an exact initial power-of-two allocation using parquet-rs 59.3.0's public max-NDV/FPP +/// setters. The writer may subsequently fold that allocation based on the values inserted. /// /// A target `B > 32` is selected by every raw byte count in `(B/2, B]`. Aim at `3B/4`, far from /// either floating-point boundary, and verify using the exact parquet-rs sizing expression. The diff --git a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala index bb6e828398b..761cb8d870b 100644 --- a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala +++ b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala @@ -625,9 +625,10 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { } /** - * parquet-rs 58.x represents Bloom filters as a power-of-two number of bytes. parquet-mr - * accepts arbitrary caps and, when one binds, serializes that exact length. Keep those writes - * on the classic path instead of silently changing the number of usable Bloom blocks. + * parquet-rs 59.3.0 initially represents Bloom filters as a power-of-two number of bytes and + * may fold that allocation after values are inserted. parquet-mr accepts arbitrary caps and, + * when one binds, serializes that exact length. Keep those writes on the classic path instead + * of silently changing the number of usable Bloom blocks. */ private val requireNativeSupportedBloomFilterProperties: TriggerRule = ctx => { val properties = interpretedBloomFilterProperties(ctx.properties) From 7419d02446de35308286945c086749b343a8bac5 Mon Sep 17 00:00:00 2001 From: Nikita Matskevich Date: Sat, 12 Sep 2026 23:47:31 +0200 Subject: [PATCH 09/32] fix: reject zero bloom filter FPP --- native/core/src/execution/operators/iceberg_write.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/native/core/src/execution/operators/iceberg_write.rs b/native/core/src/execution/operators/iceberg_write.rs index 2c0313d08ba..11bd07d0c38 100644 --- a/native/core/src/execution/operators/iceberg_write.rs +++ b/native/core/src/execution/operators/iceberg_write.rs @@ -2442,7 +2442,7 @@ fn synthetic_ndv_for_bloom_filter_bytes(target_bytes: usize, fpp: f64) -> DFResu } fn validate_bloom_filter_inputs(fpp: f64, bytes: usize) -> DFResult<()> { - if !fpp.is_finite() || !(0.0..1.0).contains(&fpp) { + if !fpp.is_finite() || !(fpp > 0.0 && fpp < 1.0) { return Err(DataFusionError::Internal(format!( "Bloom filter FPP must be finite and strictly between 0 and 1, got {fpp}" ))); @@ -2808,6 +2808,13 @@ mod tests { assert!(format!("{err}").contains("cannot represent")); } + #[test] + fn rejects_zero_bloom_filter_fpp() { + let err = + validate_bloom_filter_inputs(0.0, ICEBERG_DEFAULT_BLOOM_FILTER_MAX_BYTES).unwrap_err(); + assert!(format!("{err}").contains("strictly between 0 and 1")); + } + #[test] fn rejects_unknown_codec() { let mut settings = base_settings(); From f45fc9a0152f3a1fd8e4c42ae5ed33947c2ca220 Mon Sep 17 00:00:00 2001 From: Nikita Matskevich Date: Sun, 13 Sep 2026 00:21:46 +0200 Subject: [PATCH 10/32] fix: ignore unused bloom max byte caps --- .../operator/CometIcebergNativeWrite.scala | 134 ++++++++++-------- .../comet/CometIcebergWriteActionSuite.scala | 29 ++-- .../CometIcebergWriteDetectionSuite.scala | 16 ++- 3 files changed, 102 insertions(+), 77 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala index 761cb8d870b..4e16dbc0302 100644 --- a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala +++ b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala @@ -632,77 +632,85 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { */ private val requireNativeSupportedBloomFilterProperties: TriggerRule = ctx => { val properties = interpretedBloomFilterProperties(ctx.properties) - val maxRejection = - properties.get(PropertyKeys.ParquetBloomFilterMaxBytes).flatMap { raw => - scala.util.Try(java.lang.Integer.parseInt(raw)).toOption match { - case None => Some(s"${PropertyKeys.ParquetBloomFilterMaxBytes}=$raw is not a Java int") - case Some(value) + val configuredMaxBytes = properties.get(PropertyKeys.ParquetBloomFilterMaxBytes).map { raw => + raw -> scala.util.Try(java.lang.Integer.parseInt(raw)).toOption + } + val malformedMaxRejection = configuredMaxBytes.collect { case (raw, None) => + s"${PropertyKeys.ParquetBloomFilterMaxBytes}=$raw is not a Java int" + } + // Iceberg visits every enabled-prefix entry and applies enabled, FPP, then NDV. With no such + // entry, a parseable maximum cannot affect the file and must not force an otherwise-compatible + // write onto the classic path. Keep rejecting malformed values because protobuf translation + // parses this property before native execution. + val configured = properties.iterator.collect { + case (key, _) if key.startsWith(PropertyKeys.BloomFilterColumnEnabledPrefix) => + key.substring(PropertyKeys.BloomFilterColumnEnabledPrefix.length) + }.toSeq + if (configured.isEmpty) { + malformedMaxRejection + } else { + val maxRejection = malformedMaxRejection.orElse { + configuredMaxBytes.collect { + case (_, Some(value)) if value < MinBloomFilterBytes || value > MaxBloomFilterBytes || (value & (value - 1)) != 0 => - Some( - s"${PropertyKeys.ParquetBloomFilterMaxBytes}=$value must be a power of two " + - s"in [$MinBloomFilterBytes, $MaxBloomFilterBytes] for native writes") - case Some(_) => None + s"${PropertyKeys.ParquetBloomFilterMaxBytes}=$value must be a power of two " + + s"in [$MinBloomFilterBytes, $MaxBloomFilterBytes] for native writes" } } - maxRejection.orElse { - val maxBytes = properties - .get(PropertyKeys.ParquetBloomFilterMaxBytes) - .flatMap(raw => scala.util.Try(java.lang.Integer.parseInt(raw)).toOption) - .getOrElse(IcebergWriteProtoTranslation.Defaults.BloomFilterMaxBytes) - // Iceberg visits every enabled-prefix entry and applies enabled, FPP, then NDV. Validate - // the associated shape properties even for enabled=false; a valid NDV also re-enables the - // filter in parquet-mr. - val configured = properties.iterator.collect { - case (key, _) if key.startsWith(PropertyKeys.BloomFilterColumnEnabledPrefix) => - key.substring(PropertyKeys.BloomFilterColumnEnabledPrefix.length) - }.toSeq - configured.iterator - .flatMap { column => - val fppKey = PropertyKeys.ParquetBloomFilterColumnFppPrefix + column - val ndvKey = PropertyKeys.ParquetBloomFilterColumnNdvPrefix + column - val parsedFpp = properties.get(fppKey) match { - case Some(raw) => scala.util.Try(java.lang.Double.parseDouble(raw)).toOption - case None => Some(IcebergWriteProtoTranslation.Defaults.BloomFilterFpp) - } - val parsedNdv = properties - .get(ndvKey) - .flatMap(raw => scala.util.Try(java.lang.Long.parseLong(raw)).toOption) - val fppError = properties.get(fppKey).flatMap { raw => - parsedFpp match { - case Some(value) - if value > 0.0d && value < 1.0d && java.lang.Double.isFinite(value) && - bloomFilterSizesRepresentable(maxBytes, value) => - None - case Some(value) - if value > 0.0d && value < 1.0d && java.lang.Double.isFinite(value) => - Some(s"$fppKey=$raw cannot represent the configured native Bloom sizes") - case _ => Some(s"$fppKey=$raw must be a finite double strictly between 0 and 1") + maxRejection.orElse { + val maxBytes = configuredMaxBytes + .flatMap(_._2) + .getOrElse(IcebergWriteProtoTranslation.Defaults.BloomFilterMaxBytes) + // Validate associated shape properties even for enabled=false; a valid NDV re-enables + // the filter in parquet-mr. + configured.iterator + .flatMap { column => + val fppKey = PropertyKeys.ParquetBloomFilterColumnFppPrefix + column + val ndvKey = PropertyKeys.ParquetBloomFilterColumnNdvPrefix + column + val parsedFpp = properties.get(fppKey) match { + case Some(raw) => scala.util.Try(java.lang.Double.parseDouble(raw)).toOption + case None => Some(IcebergWriteProtoTranslation.Defaults.BloomFilterFpp) } - } - val ndvError = properties.get(ndvKey).flatMap { raw => - parsedNdv match { - case Some(value) if value > 0L && value <= MaxNonOverflowingBloomFilterNdv => None - case Some(value) if value > MaxNonOverflowingBloomFilterNdv => - Some(s"$ndvKey=$raw exceeds $MaxNonOverflowingBloomFilterNdv; " + - "parquet-mr Bloom sizing may overflow") - case _ => Some(s"$ndvKey=$raw must be a positive Java long") + val parsedNdv = properties + .get(ndvKey) + .flatMap(raw => scala.util.Try(java.lang.Long.parseLong(raw)).toOption) + val fppError = properties.get(fppKey).flatMap { raw => + parsedFpp match { + case Some(value) + if value > 0.0d && value < 1.0d && java.lang.Double.isFinite(value) && + bloomFilterSizesRepresentable(maxBytes, value) => + None + case Some(value) + if value > 0.0d && value < 1.0d && java.lang.Double.isFinite(value) => + Some(s"$fppKey=$raw cannot represent the configured native Bloom sizes") + case _ => Some(s"$fppKey=$raw must be a finite double strictly between 0 and 1") + } } + val ndvError = properties.get(ndvKey).flatMap { raw => + parsedNdv match { + case Some(value) if value > 0L && value <= MaxNonOverflowingBloomFilterNdv => None + case Some(value) if value > MaxNonOverflowingBloomFilterNdv => + Some(s"$ndvKey=$raw exceeds $MaxNonOverflowingBloomFilterNdv; " + + "parquet-mr Bloom sizing may overflow") + case _ => Some(s"$ndvKey=$raw must be a positive Java long") + } + } + val ignoredMinimumCapError = parsedNdv.collect { + case ndv + if maxBytes == MinBloomFilterBytes && ndv > 0L && + ndv <= MaxNonOverflowingBloomFilterNdv && + parsedFpp.exists( + parquetMrRequestedBloomFilterBytes(ndv, _) > MinBloomFilterBytes) => + s"${PropertyKeys.ParquetBloomFilterMaxBytes}=$MinBloomFilterBytes is ignored by " + + s"parquet-mr for $ndvKey=$ndv" + } + Seq(fppError, ndvError, ignoredMinimumCapError).flatten } - val ignoredMinimumCapError = parsedNdv.collect { - case ndv - if maxBytes == MinBloomFilterBytes && ndv > 0L && - ndv <= MaxNonOverflowingBloomFilterNdv && - parsedFpp.exists( - parquetMrRequestedBloomFilterBytes(ndv, _) > MinBloomFilterBytes) => - s"${PropertyKeys.ParquetBloomFilterMaxBytes}=$MinBloomFilterBytes is ignored by " + - s"parquet-mr for $ndvKey=$ndv" - } - Seq(fppError, ndvError, ignoredMinimumCapError).flatten - } - .toSeq - .headOption + .toSeq + .headOption + } } } diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala index c49f1f0b0d5..c94b4119e03 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala @@ -1670,21 +1670,24 @@ class CometIcebergWriteActionSuite } } - test("out-of-range bloom max uses the classic writer") { + test("out-of-range bloom max uses the classic writer when a bloom column is configured") { assumeNativeAcceleration() + assumeIcebergBloomShapeProperties() withIcebergCatalog { warehouseDir => - // Leave Bloom filters disabled so the stock writer can demonstrate planning fallback - // without allocating a 128 MiB filter for the oversized case. - Seq("31", "134217729").zipWithIndex.foreach { case (max, index) => - val table = s"bloom_range_fallback_$index" - createTable( - warehouseDir, - table, - partitionSpec = "", - properties = Some(s"'write.parquet.bloom-filter-max-bytes'='$max'")) - assertNativeWriteDoesNotEngage(table, Seq(index)) { - spark.sql(s"INSERT INTO cat.db.$table VALUES ($index, 'region', 1.0)") - } + // A tiny explicit NDV keeps the fallback writer's allocation small even for the oversized + // maximum. Without a configured Bloom column, the maximum is unused and stays native. + val enabledWithSmallNdv = + "'write.parquet.bloom-filter-enabled.column.id'='true', " + + "'write.parquet.bloom-filter-ndv.column.id'='1', " + val max = "134217729" + val table = "bloom_range_fallback" + createTable( + warehouseDir, + table, + partitionSpec = "", + properties = Some(enabledWithSmallNdv + s"'write.parquet.bloom-filter-max-bytes'='$max'")) + assertNativeWriteDoesNotEngage(table, Seq(1)) { + spark.sql(s"INSERT INTO cat.db.$table VALUES (1, 'region', 1.0)") } } } diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala index 985aed64a5c..edc81d23f53 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala @@ -396,7 +396,9 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes dir, table, partitionSpec = "", - properties = Some(s"'write.parquet.bloom-filter-max-bytes'='$value'")) + properties = Some( + "'write.parquet.bloom-filter-enabled.column.id'='true', " + + s"'write.parquet.bloom-filter-max-bytes'='$value'")) assertUnsupportedContainsAllowingWriteFailure( table, "write.parquet.bloom-filter-max-bytes") @@ -404,6 +406,18 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes } } + test("unused bloom-filter max-bytes does not force fallback") { + withDetectionCatalog { dir => + val nonPowerOfTwoMaxBytes = 100 + createTable( + dir, + "unused_bloom_max", + partitionSpec = "", + properties = Some(s"'write.parquet.bloom-filter-max-bytes'='$nonPowerOfTwoMaxBytes'")) + assertSupportLevelIs[Compatible]("unused_bloom_max") + } + } + test("bloom-filter max-bytes=32 falls back only when parquet-mr ignores the cap") { withDetectionCatalog { dir => val minimumBytes = 32 From c47ef21a3bad257a0438ee4291671980ce153917 Mon Sep 17 00:00:00 2001 From: Nikita Matskevich Date: Sun, 13 Sep 2026 01:08:54 +0200 Subject: [PATCH 11/32] test: document bloom folding after writes --- .../comet/CometIcebergWriteActionSuite.scala | 40 ++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala index c94b4119e03..1ea5291ed30 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala @@ -1598,7 +1598,9 @@ class CometIcebergWriteActionSuite assertNativeWriteEngages("bloom_low_ndv_native", 0 until 4096) { insert("bloom_low_ndv_native") } - insert("bloom_low_ndv_jvm") + withSQLConf(CometConf.COMET_ICEBERG_NATIVE_WRITE_ENABLED.key -> "false") { + insert("bloom_low_ndv_jvm") + } val native = parquetBloomFilterBytes("bloom_low_ndv_native", "id") val jvm = parquetBloomFilterBytes("bloom_low_ndv_jvm", "id") @@ -1609,6 +1611,42 @@ class CometIcebergWriteActionSuite } } + test("explicit NDV allocation folds to observed cardinality") { + assumeNativeAcceleration() + assumeIcebergBloomShapeProperties() + withIcebergCatalog { warehouseDir => + val configuredNdv = 1000000L + val insertedIds = 0 until 2 + val properties = Some( + "'write.parquet.bloom-filter-enabled.column.id'='true', " + + s"'write.parquet.bloom-filter-ndv.column.id'='$configuredNdv'") + createTable(warehouseDir, "bloom_explicit_ndv_fold_native", partitionSpec = "", properties) + createTable(warehouseDir, "bloom_explicit_ndv_fold_jvm", partitionSpec = "", properties) + + def insert(table: String): Unit = spark.sql( + s"INSERT INTO cat.db.$table " + + s"SELECT CAST(id AS INT), 'region', CAST(id AS DOUBLE) " + + s"FROM range(${insertedIds.start}, ${insertedIds.end}, 1, 1)") + + assertNativeWriteEngages("bloom_explicit_ndv_fold_native", insertedIds) { + insert("bloom_explicit_ndv_fold_native") + } + withSQLConf(CometConf.COMET_ICEBERG_NATIVE_WRITE_ENABLED.key -> "false") { + insert("bloom_explicit_ndv_fold_jvm") + } + + val native = parquetBloomFilterBytes("bloom_explicit_ndv_fold_native", "id") + val jvm = parquetBloomFilterBytes("bloom_explicit_ndv_fold_jvm", "id") + assert(native.nonEmpty && native.size == jvm.size) + assert( + native.zip(jvm).forall { case (nativeBytes, jvmBytes) => + nativeBytes.length < jvmBytes.length + }, + "expected parquet-rs to fold the explicit-NDV allocation after observing two values") + assertParquetBloomContainsInts("bloom_explicit_ndv_fold_native", "id", insertedIds) + } + } + test("large representable max folds natively while adjacent non-power-of-two falls back") { assumeNativeAcceleration() withIcebergCatalog { warehouseDir => From 1f4f4b0446fd390651377de665a9ed14f2a6c1c6 Mon Sep 17 00:00:00 2001 From: Nikita Matskevich Date: Sun, 13 Sep 2026 01:13:36 +0200 Subject: [PATCH 12/32] test: lock bloom sizing parity across runtimes --- .../src/execution/operators/iceberg_write.rs | 50 +++++++++++++++++++ .../operator/CometIcebergNativeWrite.scala | 23 +++++++-- .../IcebergWriteProtoTranslationSuite.scala | 29 +++++++++++ 3 files changed, 98 insertions(+), 4 deletions(-) diff --git a/native/core/src/execution/operators/iceberg_write.rs b/native/core/src/execution/operators/iceberg_write.rs index 11bd07d0c38..cbd715f2a37 100644 --- a/native/core/src/execution/operators/iceberg_write.rs +++ b/native/core/src/execution/operators/iceberg_write.rs @@ -2615,6 +2615,17 @@ mod tests { } use datafusion_comet_proto::spark_operator::CompressionCodec as ProtoCodec; + // Keep this table in sync with the cases in IcebergWriteProtoTranslationSuite. These fixed + // expectations make JVM planning and native task sizing fail independently if either drifts. + const BLOOM_FILTER_SIZING_CASES: &[(Option, f64, usize, usize)] = &[ + (None, 0.01, 1024 * 1024, 1024 * 1024), + (Some(1), 0.01, 1024 * 1024, 32), + (Some(1_000), 0.01, 1024 * 1024, 2 * 1024), + (Some(1_000), 0.005, 1024 * 1024, 2 * 1024), + (Some(1_000_000), 0.0001, 64, 64), + (Some(100_000_000), 0.01, 4 * 1024, 4 * 1024), + ]; + fn base_settings() -> IcebergParquetWriteSettings { IcebergParquetWriteSettings { compression: ProtoCodec::Zstd as i32, @@ -2745,6 +2756,23 @@ mod tests { ); } + #[test] + fn bloom_sizing_arithmetic_stays_aligned_with_jvm_planning() { + for &(ndv, fpp, max_bytes, expected_bytes) in BLOOM_FILTER_SIZING_CASES { + assert_eq!( + parquet_mr_bloom_filter_bytes(ndv, fpp, max_bytes), + expected_bytes, + "ndv={ndv:?} fpp={fpp} max_bytes={max_bytes}" + ); + let synthetic_ndv = synthetic_ndv_for_bloom_filter_bytes(expected_bytes, fpp).unwrap(); + assert_eq!( + parquet_rs_bloom_filter_bytes(synthetic_ndv, fpp), + expected_bytes, + "ndv={ndv:?} fpp={fpp} max_bytes={max_bytes}" + ); + } + } + #[test] fn synthetic_ndv_hits_every_supported_size_away_from_float_boundaries() { for fpp in [0.0001, ICEBERG_DEFAULT_BLOOM_FILTER_FPP, 0.05, 0.5, 0.99] { @@ -2808,6 +2836,28 @@ mod tests { assert!(format!("{err}").contains("cannot represent")); } + #[test] + fn binary_search_finds_representable_ndv_when_inverse_candidate_misses() { + const FPP_WITH_ROUNDING_SENSITIVE_INVERSE: f64 = f64::from_bits(0x3c6e_f871_5805_0dee); + const TARGET_BYTES: usize = 256; + + let denominator = bloom_filter_fpp_denominator(FPP_WITH_ROUNDING_SENSITIVE_INVERSE); + let inverse_candidate = + ((TARGET_BYTES as f64 * 3.0 / 4.0 * denominator).round() as u64).max(1); + assert_ne!( + parquet_rs_bloom_filter_bytes(inverse_candidate, FPP_WITH_ROUNDING_SENSITIVE_INVERSE), + TARGET_BYTES + ); + + let searched = + synthetic_ndv_for_bloom_filter_bytes(TARGET_BYTES, FPP_WITH_ROUNDING_SENSITIVE_INVERSE) + .unwrap(); + assert_eq!( + parquet_rs_bloom_filter_bytes(searched, FPP_WITH_ROUNDING_SENSITIVE_INVERSE), + TARGET_BYTES + ); + } + #[test] fn rejects_zero_bloom_filter_fpp() { let err = diff --git a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala index 4e16dbc0302..9fd4ff6879f 100644 --- a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala +++ b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala @@ -600,8 +600,8 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { // scalastyle:off line.size.limit // https://github.com/apache/parquet-java/blob/78a8d3230eb4769db93de5f2f2e18363c04cae81/parquet-column/src/main/java/org/apache/parquet/column/values/bloomfilter/BlockSplitBloomFilter.java#L40-L50 // scalastyle:on line.size.limit - private val MinBloomFilterBytes = 32 - private val MaxBloomFilterBytes = 128 * 1024 * 1024 + private[operator] val MinBloomFilterBytes = 32 + private[operator] val MaxBloomFilterBytes = 128 * 1024 * 1024 private val BloomFilterHashProbes = 8 private val MaxNonOverflowingBloomFilterNdv = Long.MaxValue / BloomFilterHashProbes @@ -718,7 +718,7 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { // aiming at 3B/4, in the interior of parquet-rs's (B/2, B] round-up interval. Requiring every // power-of-two through the configured cap is conservative and keeps pathological-but-valid // floating-point FPPs on the JVM path rather than discovering them after task launch. - private def bloomFilterSizesRepresentable(maxBytes: Int, fpp: Double): Boolean = { + private[operator] def bloomFilterSizesRepresentable(maxBytes: Int, fpp: Double): Boolean = { val denominator = -Math.log(1.0d - Math.pow(fpp, 1.0d / BloomFilterHashProbes.toDouble)) if (!java.lang.Double.isFinite(denominator) || denominator <= 0.0d) return false @@ -739,7 +739,7 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { } /** The uncapped byte count parquet-mr passes to its explicit-NDV constructor path. */ - private def parquetMrRequestedBloomFilterBytes(ndv: Long, fpp: Double): Int = { + private[operator] def parquetMrRequestedBloomFilterBytes(ndv: Long, fpp: Double): Int = { // Keep the long multiplication before floating-point conversion to match parquet-mr: // scalastyle:off line.size.limit // https://github.com/apache/parquet-java/blob/78a8d3230eb4769db93de5f2f2e18363c04cae81/parquet-column/src/main/java/org/apache/parquet/column/values/bloomfilter/BlockSplitBloomFilter.java#L277-L301 @@ -754,6 +754,21 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { Math.max(bits, bitsPerBlock) / 8 } + /** The power-of-two byte allocation Comet asks parquet-rs to make for parquet-mr parity. */ + private[operator] def parquetMrBloomFilterBytes( + ndv: Option[Long], + fpp: Double, + maxBytes: Int): Int = { + ndv match { + case None => maxBytes + case Some(value) => + val requested = parquetMrRequestedBloomFilterBytes(value, fpp) + val bounded = Math.max(MinBloomFilterBytes, Math.min(MaxBloomFilterBytes, requested)) + val allocated = java.lang.Integer.highestOneBit(bounded - 1) << 1 + Math.min(allocated, maxBytes) + } + } + private val requireOnlyVettedParquetWriteProperties: TriggerRule = ctx => ctx.properties .find { case (k, _) => diff --git a/spark/src/test/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslationSuite.scala b/spark/src/test/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslationSuite.scala index 03e56eb81a0..62a28235db4 100644 --- a/spark/src/test/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslationSuite.scala +++ b/spark/src/test/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslationSuite.scala @@ -249,6 +249,35 @@ class IcebergWriteProtoTranslationSuite extends AnyFunSuite { assert(explicit.getBloomFilterMaxBytes == 64L * 1024L * 1024L) } + test("Bloom sizing arithmetic stays aligned with the native writer") { + import CometIcebergNativeWrite.{bloomFilterSizesRepresentable, MinBloomFilterBytes, parquetMrBloomFilterBytes} + + val kibibyte = 1024 + val mebibyte = kibibyte * kibibyte + val twiceMinimum = 2 * MinBloomFilterBytes + // Keep this table in sync with BLOOM_FILTER_SIZING_CASES in iceberg_write.rs. These fixed + // expectations make JVM planning and native task sizing fail independently if either drifts. + val cases = Seq( + (None, 0.01d, mebibyte, mebibyte), + (Some(1L), 0.01d, mebibyte, MinBloomFilterBytes), + (Some(1000L), 0.01d, mebibyte, 2 * kibibyte), + (Some(1000L), 0.005d, mebibyte, 2 * kibibyte), + (Some(1000000L), 0.0001d, twiceMinimum, twiceMinimum), + (Some(100000000L), 0.01d, 4 * kibibyte, 4 * kibibyte)) + + cases.foreach { case (ndv, fpp, maxBytes, expectedBytes) => + assert( + parquetMrBloomFilterBytes(ndv, fpp, maxBytes) == expectedBytes, + s"ndv=$ndv fpp=$fpp maxBytes=$maxBytes") + assert( + bloomFilterSizesRepresentable(maxBytes, fpp), + s"ndv=$ndv fpp=$fpp maxBytes=$maxBytes") + } + assert( + !bloomFilterSizesRepresentable(mebibyte, java.lang.Double.MIN_NORMAL), + "the gate must reject a target for which native synthetic-NDV sizing fails") + } + test("size properties are parsed with Java Integer.parseInt semantics") { // No trimming and no values past Int.MaxValue -- exactly what iceberg-java's // PropertyUtil.propertyAsInt would do. The eligibility gate declines these values From 485d798afbee08d5e627c0992642befcb17e9000 Mon Sep 17 00:00:00 2001 From: Nikita Matskevich Date: Sun, 13 Sep 2026 23:48:49 +0200 Subject: [PATCH 13/32] refactor: group Iceberg bloom column settings --- .../src/execution/operators/iceberg_write.rs | 49 ++++++++++++------- native/proto/src/proto/operator.proto | 21 ++++---- .../IcebergWriteProtoTranslation.scala | 31 +++++------- .../IcebergWriteProtoTranslationSuite.scala | 36 ++++++++------ 4 files changed, 75 insertions(+), 62 deletions(-) diff --git a/native/core/src/execution/operators/iceberg_write.rs b/native/core/src/execution/operators/iceberg_write.rs index cbd715f2a37..86a0e8bb967 100644 --- a/native/core/src/execution/operators/iceberg_write.rs +++ b/native/core/src/execution/operators/iceberg_write.rs @@ -81,6 +81,8 @@ use parquet::file::properties::{EnabledStatistics, WriterProperties}; use parquet::schema::types::ColumnPath; use serde::{Deserialize, Serialize}; +#[cfg(test)] +use datafusion_comet_proto::spark_operator::IcebergColumnBloomFilterProps; use datafusion_comet_proto::spark_operator::{ CompressionCodec as ProtoCompressionCodec, IcebergParquetWriteSettings, IcebergWrite, IcebergWriteCommon, IcebergWriterMode as ProtoIcebergWriterMode, @@ -2312,14 +2314,10 @@ fn build_writer_properties(settings: &IcebergParquetWriteSettings) -> DFResult ColumnPath { const BLOOM_FILTER_MIN_BYTES: usize = 32; const BLOOM_FILTER_MAX_BYTES: usize = 128 * 1024 * 1024; const BLOOM_FILTER_HASH_PROBES: f64 = 8.0; +#[cfg(test)] const ICEBERG_DEFAULT_BLOOM_FILTER_FPP: f64 = 0.01; #[cfg(test)] const ICEBERG_DEFAULT_BLOOM_FILTER_MAX_BYTES: usize = 1024 * 1024; @@ -2635,10 +2634,8 @@ mod tests { dict_size_bytes: 2 * 1024 * 1024, page_row_limit: 20_000, created_by: "Apache Iceberg (Comet test)".to_string(), - bloom_filter_enabled_columns: Vec::new(), + bloom_filters: Vec::new(), bloom_filter_max_bytes: ICEBERG_DEFAULT_BLOOM_FILTER_MAX_BYTES as u64, - bloom_filter_fpp_by_column: Default::default(), - bloom_filter_ndv_by_column: Default::default(), } } @@ -2721,10 +2718,22 @@ mod tests { #[test] fn enables_bloom_filters_only_for_configured_columns() { let mut settings = base_settings(); - settings.bloom_filter_enabled_columns = vec![ - "id".to_string(), - "tags.list.element".to_string(), - "attrs.key_value.value".to_string(), + settings.bloom_filters = vec![ + IcebergColumnBloomFilterProps { + parquet_path: "id".to_string(), + fpp: ICEBERG_DEFAULT_BLOOM_FILTER_FPP, + ndv: None, + }, + IcebergColumnBloomFilterProps { + parquet_path: "tags.list.element".to_string(), + fpp: ICEBERG_DEFAULT_BLOOM_FILTER_FPP, + ndv: None, + }, + IcebergColumnBloomFilterProps { + parquet_path: "attrs.key_value.value".to_string(), + fpp: ICEBERG_DEFAULT_BLOOM_FILTER_FPP, + ndv: None, + }, ]; let props = build_writer_properties(&settings).unwrap(); assert!(props @@ -2746,7 +2755,11 @@ mod tests { // Iceberg's documented write-property defaults: // https://iceberg.apache.org/docs/latest/configuration/#write-properties let mut settings = base_settings(); - settings.bloom_filter_enabled_columns = vec!["id".to_string()]; + settings.bloom_filters = vec![IcebergColumnBloomFilterProps { + parquet_path: "id".to_string(), + fpp: ICEBERG_DEFAULT_BLOOM_FILTER_FPP, + ndv: None, + }]; let props = build_writer_properties(&settings).unwrap(); let bloom = props.bloom_filter_properties(&"id".into()).unwrap(); assert_eq!(bloom.fpp(), ICEBERG_DEFAULT_BLOOM_FILTER_FPP); @@ -3340,10 +3353,8 @@ mod tests { dict_size_bytes: 2 * 1024 * 1024, page_row_limit: 20_000, created_by: "Apache Iceberg (Comet integration test)".to_string(), - bloom_filter_enabled_columns: Vec::new(), + bloom_filters: Vec::new(), bloom_filter_max_bytes: ICEBERG_DEFAULT_BLOOM_FILTER_MAX_BYTES as u64, - bloom_filter_fpp_by_column: Default::default(), - bloom_filter_ndv_by_column: Default::default(), }; Arc::new(IcebergWriteCommon { catalog_properties: HashMap::new(), diff --git a/native/proto/src/proto/operator.proto b/native/proto/src/proto/operator.proto index 21a09ae5d0b..bfe4bb6a489 100644 --- a/native/proto/src/proto/operator.proto +++ b/native/proto/src/proto/operator.proto @@ -782,6 +782,14 @@ enum IcebergWriterMode { ICEBERG_WRITER_CLUSTERED = 2; } +message IcebergColumnBloomFilterProps { + string parquet_path = 1; + // Effective FPP, with the Iceberg default resolved on the JVM. + double fpp = 2; + // Absent means allocate the full max-bytes value initially. + optional uint64 ndv = 3; +} + // Parquet-rs `WriterProperties` knobs derived from Iceberg table properties on the JVM driver. // All fields use Iceberg defaults when absent in the table properties; the rust side does not // re-apply defaults. @@ -807,19 +815,12 @@ message IcebergParquetWriteSettings { // String written into parquet file metadata. JVM-side default is // `"Apache Iceberg (Comet)"`. string created_by = 7; - // Physical Parquet leaf paths for Iceberg columns whose - // `write.parquet.bloom-filter-enabled.column.` property resolves to true. The JVM driver - // translates Iceberg logical names to the actual Parquet paths used by list/map encodings. - repeated string bloom_filter_enabled_columns = 8; + // Per-column Bloom-filter settings. The JVM driver translates Iceberg logical names to the + // physical Parquet leaf paths used by list/map encodings and resolves effective defaults. + repeated IcebergColumnBloomFilterProps bloom_filters = 8; // Iceberg `write.parquet.bloom-filter-max-bytes` (default 1 MiB). Native eligibility only // admits powers of two in [32, 128 MiB], which parquet-rs can represent exactly. uint64 bloom_filter_max_bytes = 9; - // Effective per-column FPP, including Iceberg's 0.01 default. A value is present for every - // enabled column so parquet-rs's different 0.05 default can never leak into Iceberg writes. - map bloom_filter_fpp_by_column = 10; - // User-provided Iceberg NDV values. Absence is significant: parquet-mr allocates the full - // max in that case, whereas an explicit NDV sizes the filter before applying the max as a cap. - map bloom_filter_ndv_by_column = 11; } // Broadcast payload -- one of these per write, identical for every task. diff --git a/spark/src/main/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslation.scala b/spark/src/main/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslation.scala index 108b6b38844..c51778617bd 100644 --- a/spark/src/main/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslation.scala +++ b/spark/src/main/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslation.scala @@ -169,21 +169,22 @@ object IcebergWriteProtoTranslation { parquetPathByIcebergColumnName.get(icebergName).map(icebergName -> _) } .sortBy(_._2) - val bloomFilterEnabledColumns = bloomFilterColumns.map(_._2) val bloomFilterMaxBytes = parseJavaInt(props, Keys.ParquetBloomFilterMaxBytes, Defaults.BloomFilterMaxBytes).toLong - val bloomFilterFppByColumn = bloomFilterColumns.map { case (icebergName, parquetPath) => - val value = props - .get(Keys.ParquetBloomFilterColumnFppPrefix + icebergName) - .map(java.lang.Double.parseDouble) - .getOrElse(Defaults.BloomFilterFpp) - parquetPath -> value - }.toMap - val bloomFilterNdvByColumn = bloomFilterColumns.flatMap { case (icebergName, parquetPath) => + val bloomFilters = bloomFilterColumns.map { case (icebergName, parquetPath) => + val builder = IcebergColumnBloomFilterProps + .newBuilder() + .setParquetPath(parquetPath) + .setFpp( + props + .get(Keys.ParquetBloomFilterColumnFppPrefix + icebergName) + .map(java.lang.Double.parseDouble) + .getOrElse(Defaults.BloomFilterFpp)) props .get(Keys.ParquetBloomFilterColumnNdvPrefix + icebergName) - .map(value => parquetPath -> java.lang.Long.parseLong(value)) - }.toMap + .foreach(value => builder.setNdv(java.lang.Long.parseLong(value))) + builder.build() + } val builder = IcebergParquetWriteSettings .newBuilder() .setCompression(compression) @@ -192,14 +193,8 @@ object IcebergWriteProtoTranslation { .setDictSizeBytes(dictSize) .setPageRowLimit(pageRowLimit) .setCreatedBy(createdBy) - .addAllBloomFilterEnabledColumns(bloomFilterEnabledColumns.asJava) + .addAllBloomFilters(bloomFilters.asJava) .setBloomFilterMaxBytes(bloomFilterMaxBytes) - .putAllBloomFilterFppByColumn(bloomFilterFppByColumn.map { case (k, v) => - k -> Double.box(v) - }.asJava) - .putAllBloomFilterNdvByColumn(bloomFilterNdvByColumn.map { case (k, v) => - k -> Long.box(v) - }.asJava) resolveCompressionLevel(props, compression).foreach(builder.setCompressionLevel) diff --git a/spark/src/test/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslationSuite.scala b/spark/src/test/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslationSuite.scala index 62a28235db4..2cea3fb42d0 100644 --- a/spark/src/test/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslationSuite.scala +++ b/spark/src/test/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslationSuite.scala @@ -19,6 +19,8 @@ package org.apache.comet.serde.operator +import scala.jdk.CollectionConverters._ + import org.scalatest.funsuite.AnyFunSuite import org.apache.comet.serde.OperatorOuterClass.{CompressionCodec, IcebergWriterMode} @@ -186,13 +188,15 @@ class IcebergWriteProtoTranslationSuite extends AnyFunSuite { s"${Keys.ParquetBloomFilterColumnFppPrefix}region" -> "0.02", s"${Keys.ParquetBloomFilterColumnNdvPrefix}id" -> "1234"), TestCreatedBy) - assert(settings.getBloomFilterEnabledColumnsList == java.util.Arrays.asList("id", "region")) + assert(settings.getBloomFiltersList.asScala.map(_.getParquetPath) == Seq("id", "region")) + val bloomFilters = settings.getBloomFiltersList.asScala.map(p => p.getParquetPath -> p).toMap + assert(bloomFilters.keySet == Set("id", "region")) assert(settings.getBloomFilterMaxBytes == 1024L * 1024L) - assert(settings.getBloomFilterFppByColumnMap.get("id") == 0.01d) - assert(settings.getBloomFilterFppByColumnMap.get("region") == 0.02d) - assert(settings.getBloomFilterNdvByColumnMap.get("id") == 1234L) - assert(!settings.getBloomFilterNdvByColumnMap.containsKey("region")) - assert(!settings.getBloomFilterFppByColumnMap.containsKey("amount")) + assert(bloomFilters("id").getFpp == 0.01d) + assert(bloomFilters("region").getFpp == 0.02d) + assert(bloomFilters("id").getNdv == 1234L) + assert(!bloomFilters("region").hasNdv) + assert(!bloomFilters.contains("amount")) } test("an explicit NDV re-enables a bloom filter after enabled=false") { @@ -206,9 +210,10 @@ class IcebergWriteProtoTranslationSuite extends AnyFunSuite { s"${Keys.ParquetBloomFilterColumnNdvPrefix}id" -> configuredNdv.toString), TestCreatedBy) - assert(settings.getBloomFilterEnabledColumnsList == java.util.Arrays.asList("id")) - assert(settings.getBloomFilterFppByColumnMap.get("id") == configuredFpp) - assert(settings.getBloomFilterNdvByColumnMap.get("id") == configuredNdv) + val bloomFilter = settings.getBloomFilters(0) + assert(bloomFilter.getParquetPath == "id") + assert(bloomFilter.getFpp == configuredFpp) + assert(bloomFilter.getNdv == configuredNdv) } test("bloom filter properties use physical Parquet paths for list and map leaves") { @@ -227,12 +232,13 @@ class IcebergWriteProtoTranslationSuite extends AnyFunSuite { "attrs.key" -> "attrs.key_value.key", "attrs.value" -> "attrs.key_value.value")) + val bloomFilters = settings.getBloomFiltersList.asScala.map(p => p.getParquetPath -> p).toMap assert( - settings.getBloomFilterEnabledColumnsList == java.util.Arrays - .asList("attrs.key_value.key", "attrs.key_value.value", "tags.list.element")) - assert(settings.getBloomFilterFppByColumnMap.get("tags.list.element") == 0.02d) - assert(settings.getBloomFilterNdvByColumnMap.get("attrs.key_value.value") == 1234L) - assert(!settings.getBloomFilterEnabledColumnsList.contains("missing")) + bloomFilters.keySet == + Set("attrs.key_value.key", "attrs.key_value.value", "tags.list.element")) + assert(bloomFilters("tags.list.element").getFpp == 0.02d) + assert(bloomFilters("attrs.key_value.value").getNdv == 1234L) + assert(!bloomFilters.contains("missing")) } test("Iceberg bloom filter defaults and explicit max are translated exactly") { @@ -240,7 +246,7 @@ class IcebergWriteProtoTranslationSuite extends AnyFunSuite { val defaults = buildParquetSettings(Map(enabled -> "true"), TestCreatedBy) assert(defaults.getBloomFilterMaxBytes == Defaults.BloomFilterMaxBytes) assert(Defaults.BloomFilterMaxBytes == 1024 * 1024) - assert(defaults.getBloomFilterFppByColumnMap.get("id") == Defaults.BloomFilterFpp) + assert(defaults.getBloomFilters(0).getFpp == Defaults.BloomFilterFpp) assert(Defaults.BloomFilterFpp == 0.01d) val explicit = buildParquetSettings( From d9343fa008f4a0790e6cca9cbaafc53fd2ac2950 Mon Sep 17 00:00:00 2001 From: Nikita Matskevich Date: Sun, 13 Sep 2026 23:55:16 +0200 Subject: [PATCH 14/32] fix: match Iceberg bloom path resolution by version --- .../comet/iceberg/IcebergReflection.scala | 39 ++++- .../operator/CometIcebergNativeWrite.scala | 36 ++++- .../comet/CometIcebergWriteActionSuite.scala | 143 +++++++++++++++--- 3 files changed, 191 insertions(+), 27 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala b/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala index 876ecb6c241..0c50cc0172e 100644 --- a/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala +++ b/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala @@ -41,7 +41,9 @@ object IcebergReflection extends Logging { case class ParquetPathResolution( pathByIcebergColumnName: Map[String, String], - renamedIcebergColumnNames: Set[String]) + pathByFieldId: Map[Int, String], + physicalPaths: Set[String], + renamedPhysicalPaths: Set[String]) /** * Iceberg class names used throughout Comet. @@ -745,14 +747,16 @@ object IcebergReflection extends Logging { } } } - (icebergName.asInstanceOf[String], parquetPathParts.mkString("."), renamed) + (fieldId, icebergName.asInstanceOf[String], parquetPathParts.mkString("."), renamed) } } }.toSeq Some( ParquetPathResolution( - resolved.map { case (name, path, _) => name -> path }.toMap, - resolved.collect { case (name, _, true) => name }.toSet)) + resolved.map { case (_, name, path, _) => name -> path }.toMap, + resolved.map { case (fieldId, _, path, _) => fieldId -> path }.toMap, + resolved.map { case (_, _, path, _) => path }.toSet, + resolved.collect { case (_, _, path, true) => path }.toSet)) } catch { case e: Exception => logError(s"Iceberg reflection failure: Parquet column paths: ${e.getMessage}") @@ -869,7 +873,34 @@ object IcebergReflection extends Logging { } /** +<<<<<<< HEAD * The fields of `schema` at any depth, as id, name, and the type of a field with no children. +======= + * Returns the field id resolved by Iceberg's name lookup, including runtime-supported aliases. + */ + def findFieldIdByName(schema: Any, name: String): Option[Int] = { + try { + val findFieldMethod = getMethod(schema.getClass, "findField", classOf[String]) + Option(findFieldMethod.invoke(schema, name)).map { field => + getMethod(field.getClass, "fieldId").invoke(field).asInstanceOf[Int] + } + } catch { + case _: Exception => None + } + } + + /** + * Returns a schema equal to `baseSchema` but guaranteed to contain `requiredFieldIds`. Any id + * not already present is resolved from the table's schema history (`table.schemas()`) and + * appended. + * + * Two callers need this: an equality delete may be keyed on a column since dropped from the + * current schema (mirroring Iceberg-Java's `DeleteFilter.fileProjection`), and a partition + * spec's source columns must be present for iceberg-rust to resolve `partition_type` when the + * query projects them out. Called at serialization time, so it throws on failure (a required id + * that cannot be resolved, or any reflection error) rather than silently degrading. + * CometScanRule is responsible for falling back before we get here. +>>>>>>> eab9eb47e (fix: match Iceberg bloom path resolution by version) */ private def fieldsAtAnyDepth(schema: Any): Seq[(Int, String, Option[Any])] = { import scala.jdk.CollectionConverters._ diff --git a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala index 9fd4ff6879f..a9ce6509975 100644 --- a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala +++ b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala @@ -1303,10 +1303,36 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { withFallbackReason(op, "Could not resolve physical Parquet paths for Bloom filters") return None } - val renamedColumns = - IcebergWriteProtoTranslation - .enabledBloomFilterColumnNames(effectiveProperties) - .filter(resolution.renamedIcebergColumnNames) + val configuredColumns = + IcebergWriteProtoTranslation.enabledBloomFilterColumnNames(effectiveProperties) + val icebergVersion = IcebergReflection.icebergVersion() + val versionParts = icebergVersion.split("[.-]", 3) + val version = + if (versionParts.length >= 2) { + scala.util.Try((versionParts(0).toInt, versionParts(1).toInt)).toOption + } else { + None + } + val resolvedPaths = version match { + case Some((1, minor)) if minor < 8 => + // Iceberg 1.5 passes the configured name straight to parquet-mr. + configuredColumns.filter(resolution.physicalPaths).map(path => path -> path).toMap + case Some((1, minor)) if minor < 10 => + // Iceberg 1.8 resolves canonical and short collection aliases through field ids. + configuredColumns.flatMap { configuredName => + IcebergReflection + .findFieldIdByName(writeSchema, configuredName) + .flatMap(resolution.pathByFieldId.get) + .map(configuredName -> _) + }.toMap + case _ => + // Iceberg 1.10+ keys writer properties by canonical findColumnName output. + resolution.pathByIcebergColumnName + } + val renamedColumns = resolvedPaths.collect { + case (configuredName, path) if resolution.renamedPhysicalPaths.contains(path) => + configuredName + } if (renamedColumns.nonEmpty) { // Iceberg Java sanitizes these Parquet names, while the pinned iceberg-rust Arrow // conversion preserves them. Passing the Java path to parquet-rs would silently miss @@ -1317,7 +1343,7 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { renamedColumns.mkString(", ")) return None } - resolution.pathByIcebergColumnName + resolvedPaths } else { Map.empty[String, String] } diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala index 1ea5291ed30..bf5c47f82b2 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala @@ -32,6 +32,7 @@ import scala.concurrent.duration.DurationInt import scala.jdk.CollectionConverters._ import scala.util.control.NonFatal +<<<<<<< HEAD import org.json4s.{DefaultFormats, Formats} import org.json4s.jackson.JsonMethods.parse @@ -40,6 +41,12 @@ import org.apache.parquet.hadoop.ParquetFileReader import org.apache.parquet.hadoop.util.HadoopInputFile import org.apache.spark.{CometListenerBusUtils, SparkConf, SparkException, SparkThrowable, Success, TaskContext} import org.apache.spark.rdd.RDD +======= +import org.apache.hadoop.fs.Path +import org.apache.parquet.hadoop.ParquetFileReader +import org.apache.parquet.hadoop.util.HadoopInputFile +import org.apache.spark.{SparkConf, Success} +>>>>>>> eab9eb47e (fix: match Iceberg bloom path resolution by version) import org.apache.spark.scheduler.{SparkListener, SparkListenerTaskEnd} import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.DataFrame @@ -1393,30 +1400,130 @@ class CometIcebergWriteActionSuite test("native bloom filters resolve list and map leaves to physical Parquet paths") { assumeNativeAcceleration() withIcebergCatalog { _ => - spark.sql(s""" - CREATE TABLE $catalog.$ns.native_nested_bloom ( - id INT, - tags ARRAY, - attrs MAP - ) USING iceberg - TBLPROPERTIES ( - 'write.parquet.bloom-filter-enabled.column.tags.element'='true', - 'write.parquet.bloom-filter-enabled.column.attrs.key'='true', - 'write.parquet.bloom-filter-enabled.column.attrs.value'='true' - ) + Seq("native_nested_bloom", "native_nested_bloom_jvm").foreach { table => + spark.sql(s""" + CREATE TABLE $catalog.$ns.$table ( + id INT, + tags ARRAY>, + attrs MAP> + ) USING iceberg + TBLPROPERTIES ( + 'write.parquet.bloom-filter-enabled.column.tags.element.a'='true', + 'write.parquet.bloom-filter-enabled.column.attrs.key'='true', + 'write.parquet.bloom-filter-enabled.column.attrs.value.b'='true' + ) + """) + } + + def insert(table: String): Unit = spark.sql(s""" + INSERT INTO $catalog.$ns.$table VALUES + (1, array(named_struct('a', 10), named_struct('a', 20)), + map('small', named_struct('b', 30), 'large', named_struct('b', 40))), + (2, array(named_struct('a', 50)), map('medium', named_struct('b', 60))) """) assertNativeWriteEngages("native_nested_bloom", Seq(1, 2)) { - spark.sql(""" - INSERT INTO cat.db.native_nested_bloom VALUES - (1, array('red', 'green'), map('small', 10, 'large', 20)), - (2, array('blue'), map('medium', 30)) + insert("native_nested_bloom") + } + withSQLConf(CometConf.COMET_ICEBERG_NATIVE_WRITE_ENABLED.key -> "false") { + insert("native_nested_bloom_jvm") + } + + val physicalColumns = + Set("tags.list.element.a", "attrs.key_value.key", "attrs.key_value.value.b") + // Iceberg 1.5 passes the configured logical names directly to Parquet Java, where they do + // not match these physical list/map paths. Iceberg 1.8+ resolves canonical names first. + val expectedColumns = + if (icebergVersionAtLeast(1, 8)) physicalColumns else Set.empty[String] + assertParquetBloomFilters("native_nested_bloom", enabledColumns = expectedColumns) + assertParquetBloomFilters("native_nested_bloom_jvm", enabledColumns = expectedColumns) + } + } + + test("short nested bloom aliases match the Iceberg runtime footer") { + assumeNativeAcceleration() + withIcebergCatalog { _ => + Seq("nested_alias_native", "nested_alias_jvm").foreach { table => + spark.sql(s""" + CREATE TABLE $catalog.$ns.$table ( + id INT, + tags ARRAY>, + attrs MAP> + ) USING iceberg + TBLPROPERTIES ( + 'write.parquet.bloom-filter-enabled.column.tags.a'='true', + 'write.parquet.bloom-filter-enabled.column.attrs.b'='true' + ) """) } - assertParquetBloomFilters( - "native_nested_bloom", - enabledColumns = Set("tags.list.element", "attrs.key_value.key", "attrs.key_value.value")) + def insert(table: String): Unit = spark.sql(s""" + INSERT INTO $catalog.$ns.$table VALUES + (1, array(named_struct('a', 10)), map('first', named_struct('b', 20))), + (2, array(named_struct('a', 30)), map('second', named_struct('b', 40))) + """) + + assertNativeWriteEngages("nested_alias_native", Seq(1, 2)) { + insert("nested_alias_native") + } + withSQLConf(CometConf.COMET_ICEBERG_NATIVE_WRITE_ENABLED.key -> "false") { + insert("nested_alias_jvm") + } + + val icebergUsesFieldIdResolution = + icebergVersionAtLeast(1, 8) && !icebergVersionAtLeast(1, 10) + Seq("tags.list.element.a", "attrs.key_value.value.b").foreach { physicalColumn => + val nativeHasBloom = parquetBloomFilterPresent("nested_alias_native", physicalColumn) + val jvmHasBloom = parquetBloomFilterPresent("nested_alias_jvm", physicalColumn) + assert(nativeHasBloom == jvmHasBloom, s"native and JVM footer differ for $physicalColumn") + assert( + jvmHasBloom == icebergUsesFieldIdResolution, + s"unexpected short-alias behavior for $physicalColumn on this Iceberg runtime") + } + } + } + + test("Iceberg 1.5 physical nested bloom paths execute natively and match the JVM footer") { + assumeNativeAcceleration() + assume( + !icebergVersionAtLeast(1, 8), + "physical-path Bloom properties are specific to Iceberg 1.5") + withIcebergCatalog { _ => + Seq("physical_path_native", "physical_path_jvm").foreach { table => + spark.sql(s""" + CREATE TABLE $catalog.$ns.$table ( + id INT, + tags ARRAY>, + attrs MAP> + ) USING iceberg + TBLPROPERTIES ( + 'write.parquet.bloom-filter-enabled.column.tags.list.element.a'='true', + 'write.parquet.bloom-filter-enabled.column.attrs.key_value.value.b'='true' + ) + """) + } + + def insert(table: String): Unit = spark.sql(s""" + INSERT INTO $catalog.$ns.$table VALUES + (1, array(named_struct('a', 10)), map('first', named_struct('b', 20))), + (2, array(named_struct('a', 30)), map('second', named_struct('b', 40))) + """) + + assertNativeWriteEngages("physical_path_native", Seq(1, 2)) { + insert("physical_path_native") + } + withSQLConf(CometConf.COMET_ICEBERG_NATIVE_WRITE_ENABLED.key -> "false") { + insert("physical_path_jvm") + } + + Seq("tags.list.element.a", "attrs.key_value.value.b").foreach { physicalColumn => + assert( + parquetBloomFilterPresent("physical_path_native", physicalColumn), + s"expected native Bloom filter for $physicalColumn") + assert( + parquetBloomFilterPresent("physical_path_jvm", physicalColumn), + s"expected JVM Bloom filter for $physicalColumn") + } } } From 6b8c274bb27b97d2f81418593b7f8fc66d8bb05a Mon Sep 17 00:00:00 2001 From: Nikita Matskevich Date: Mon, 14 Sep 2026 05:32:02 +0200 Subject: [PATCH 15/32] feat: make native Iceberg bloom folding opt-in --- .../user-guide/latest/iceberg-writes.md | 23 +++- .../scala/org/apache/comet/CometConf.scala | 11 ++ .../comet/iceberg/IcebergReflection.scala | 16 +-- .../operator/CometIcebergNativeWrite.scala | 20 +++- .../comet/CometIcebergWriteActionSuite.scala | 105 +++++++++++++++--- .../CometIcebergWriteDetectionSuite.scala | 17 +++ 6 files changed, 156 insertions(+), 36 deletions(-) diff --git a/docs/source/user-guide/latest/iceberg-writes.md b/docs/source/user-guide/latest/iceberg-writes.md index b5828c88881..50756000c6e 100644 --- a/docs/source/user-guide/latest/iceberg-writes.md +++ b/docs/source/user-guide/latest/iceberg-writes.md @@ -265,6 +265,11 @@ writes the table instead. The same fallback applies when `max-bytes=32` would bi NDV/FPP request, because Parquet Java ignores exactly 32 bytes as a maximum, and when NDV is above `Long.MAX_VALUE / 8`, where Parquet Java's sizing multiplication can overflow. +Bloom-filter-enabled writes also require +`spark.comet.iceberg.write.bloomFilterFolding.enabled=true`. The option is off by default. When it +is false, Comet leaves the write to Spark's default Iceberg Java writer because the Apache Arrow +Rust `parquet` writer does not expose an option to disable its post-insert folding. + For supported values, Apache Parquet Java applies the sizing properties as follows: - with no NDV, allocate the full `max-bytes` value; @@ -276,9 +281,21 @@ For every write that is eligible for the native path, Comet applies exactly the decision algorithm. After values are inserted, the Apache Arrow Rust `parquet` crate may fold a sparsely populated -filter to a smaller power-of-two filter while preserving the requested FPP. Parquet Java's -non-adaptive Iceberg path keeps its initial allocation. The native result can consequently use -less file space while remaining safe for every Parquet reader. +filter to a smaller power-of-two allocation. Sparse filters occur in ordinary workloads when NDV +is overestimated, row-group cardinalities are uneven, or a final row group is only partly full. +Parquet Java's non-adaptive Iceberg path retains its initial allocation: its realized FPP can be +much lower than requested, at the cost of serializing the larger filter. The Rust writer folds +until another fold would exceed the configured FPP, so it serializes fewer bytes but can have a +higher realized FPP and weaker pruning than the Java writer. Unless `max-bytes` already makes the +requested FPP impossible, the result remains within the configured target. Folding cannot +introduce false negatives or incorrect query results. + +The option therefore controls a serialized-file-size versus pruning tradeoff. It does not imply +lower peak writer memory: the Rust writer retains the original vector capacity while folding. +The test `overestimated explicit NDV folds and exposes the realized-FPP tradeoff` pins this +behavior with matching native/Java membership checks, serialized Bloom sizes, and measured false +positives. `large representable max folds natively while adjacent non-power-of-two falls back` +covers the extreme case of a 64-MiB configured maximum and only 256 distinct values. ### Physical file layout only (cosmetic) diff --git a/spark/src/main/scala/org/apache/comet/CometConf.scala b/spark/src/main/scala/org/apache/comet/CometConf.scala index 4f6e14bebf1..792c6cf16eb 100644 --- a/spark/src/main/scala/org/apache/comet/CometConf.scala +++ b/spark/src/main/scala/org/apache/comet/CometConf.scala @@ -144,6 +144,17 @@ object CometConf extends ShimCometConf { .booleanConf .createWithDefault(false) + val COMET_ICEBERG_NATIVE_WRITE_BLOOM_FILTER_FOLDING_ENABLED: ConfigEntry[Boolean] = + conf("spark.comet.iceberg.write.bloomFilterFolding.enabled") + .category(CATEGORY_TESTING) + .doc( + "Whether Bloom-filter-enabled Iceberg writes may use Comet's native writer. The " + + "Apache Arrow Rust parquet writer folds sparse Bloom filters after insertion, which " + + "can reduce serialized size while raising realized FPP toward the configured target. " + + "When false, these writes use Iceberg's Java writer. Off by default.") + .booleanConf + .createWithDefault(false) + val COMET_ICEBERG_DATA_FILE_CONCURRENCY_LIMIT: ConfigEntry[Int] = conf("spark.comet.scan.icebergNative.dataFileConcurrencyLimit") .category(CATEGORY_SCAN) diff --git a/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala b/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala index 0c50cc0172e..7c52ef15074 100644 --- a/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala +++ b/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala @@ -873,9 +873,6 @@ object IcebergReflection extends Logging { } /** -<<<<<<< HEAD - * The fields of `schema` at any depth, as id, name, and the type of a field with no children. -======= * Returns the field id resolved by Iceberg's name lookup, including runtime-supported aliases. */ def findFieldIdByName(schema: Any, name: String): Option[Int] = { @@ -890,17 +887,8 @@ object IcebergReflection extends Logging { } /** - * Returns a schema equal to `baseSchema` but guaranteed to contain `requiredFieldIds`. Any id - * not already present is resolved from the table's schema history (`table.schemas()`) and - * appended. - * - * Two callers need this: an equality delete may be keyed on a column since dropped from the - * current schema (mirroring Iceberg-Java's `DeleteFilter.fileProjection`), and a partition - * spec's source columns must be present for iceberg-rust to resolve `partition_type` when the - * query projects them out. Called at serialization time, so it throws on failure (a required id - * that cannot be resolved, or any reflection error) rather than silently degrading. - * CometScanRule is responsible for falling back before we get here. ->>>>>>> eab9eb47e (fix: match Iceberg bloom path resolution by version) + * The fields of `schema` at any depth, as id, name, and the type of a field with no children. + */ private def fieldsAtAnyDepth(schema: Any): Seq[(Int, String, Option[Any])] = { import scala.jdk.CollectionConverters._ diff --git a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala index a9ce6509975..c960fcd83c3 100644 --- a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala +++ b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala @@ -336,7 +336,9 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { () => op.session.sessionState.newHadoopConf(), sessionConf => effectiveHadoopConf(op, table, sessionConf)) val context = - TriggerContext(table, tableProperties ++ writeProperties, sparkWrite, hadoopConfs) + TriggerContext( + table, tableProperties ++ writeProperties, sparkWrite, hadoopConfs, + CometConf.COMET_ICEBERG_NATIVE_WRITE_BLOOM_FILTER_FOLDING_ENABLED.get(op.conf)) triggers.iterator.map(rule => rule(context)).collectFirst { case Some(reason) => reason } } @@ -344,7 +346,8 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { table: Any, properties: Map[String, String], sparkWrite: Any, - hadoopConfs: LazyHadoopConfs) { + hadoopConfs: LazyHadoopConfs, + bloomFilterFoldingEnabled: Boolean) { def hadoopConf: Configuration = hadoopConfs.session def s3HadoopConf: Configuration = hadoopConfs.effective } @@ -431,6 +434,7 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { requireShredVariantsDisabled, requireNativeSupportedCompressionLevel, requireNativeSupportedBloomFilterProperties, + requireBloomFilterFoldingOptIn, requireOnlyVettedParquetWriteProperties, requirePropertyAbsent( PropertyKeys.ParquetEnableDictionary, @@ -714,6 +718,18 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { } } + private val requireBloomFilterFoldingOptIn: TriggerRule = ctx => { + val properties = interpretedBloomFilterProperties(ctx.properties) + if (IcebergWriteProtoTranslation.hasEnabledBloomFilters(properties) && + !ctx.bloomFilterFoldingEnabled) { + Some( + s"${CometConf.COMET_ICEBERG_NATIVE_WRITE_BLOOM_FILTER_FOLDING_ENABLED.key}=false " + + "(native Parquet Bloom-filter folding is not enabled)") + } else { + None + } + } + // Planning-time counterpart of the native inverse-NDV check. A target B is safely encoded by // aiming at 3B/4, in the interior of parquet-rs's (B/2, B] round-up interval. Requiring every // power-of-two through the configured cap is conservative and keeps pathological-but-valid diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala index bf5c47f82b2..ec99151de1b 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala @@ -32,7 +32,6 @@ import scala.concurrent.duration.DurationInt import scala.jdk.CollectionConverters._ import scala.util.control.NonFatal -<<<<<<< HEAD import org.json4s.{DefaultFormats, Formats} import org.json4s.jackson.JsonMethods.parse @@ -41,12 +40,6 @@ import org.apache.parquet.hadoop.ParquetFileReader import org.apache.parquet.hadoop.util.HadoopInputFile import org.apache.spark.{CometListenerBusUtils, SparkConf, SparkException, SparkThrowable, Success, TaskContext} import org.apache.spark.rdd.RDD -======= -import org.apache.hadoop.fs.Path -import org.apache.parquet.hadoop.ParquetFileReader -import org.apache.parquet.hadoop.util.HadoopInputFile -import org.apache.spark.{SparkConf, Success} ->>>>>>> eab9eb47e (fix: match Iceberg bloom path resolution by version) import org.apache.spark.scheduler.{SparkListener, SparkListenerTaskEnd} import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.DataFrame @@ -1270,6 +1263,27 @@ class CometIcebergWriteActionSuite } } + test("Bloom-filter folding config selects the native or JVM Iceberg writer") { + assumeNativeAcceleration() + withIcebergCatalog { warehouseDir => + val properties = Some("'write.parquet.bloom-filter-enabled.column.id'='true'") + createTable(warehouseDir, "bloom_folding_enabled", partitionSpec = "", properties) + createTable(warehouseDir, "bloom_folding_disabled", partitionSpec = "", properties) + + assertNativeWriteEngages("bloom_folding_enabled", Seq(1)) { + spark.sql("INSERT INTO cat.db.bloom_folding_enabled VALUES (1, 'region', 1.0)") + } + assertNativeWriteDoesNotEngage( + "bloom_folding_disabled", + Seq(1), + bloomFilterFoldingEnabled = false) { + spark.sql("INSERT INTO cat.db.bloom_folding_disabled VALUES (1, 'region', 1.0)") + } + assertParquetBloomFilters("bloom_folding_enabled", enabledColumns = Set("id")) + assertParquetBloomFilters("bloom_folding_disabled", enabledColumns = Set("id")) + } + } + test("enabled=false plus NDV matches JVM behavior for the Iceberg runtime") { assumeNativeAcceleration() withIcebergCatalog { warehouseDir => @@ -1718,14 +1732,20 @@ class CometIcebergWriteActionSuite } } - test("explicit NDV allocation folds to observed cardinality") { + test("overestimated explicit NDV folds and exposes the realized-FPP tradeoff") { assumeNativeAcceleration() assumeIcebergBloomShapeProperties() withIcebergCatalog { warehouseDir => - val configuredNdv = 1000000L - val insertedIds = 0 until 2 + val configuredNdv = 1000L + val configuredFpp = 0.01 + val insertedIds = 0 until 100 + val absentProbeValues = 10000000 until 11000000 + val expectedNativeBytes = 128 + val expectedJvmBytes = 2048 + val expectedNativeFalsePositives = 9088L val properties = Some( "'write.parquet.bloom-filter-enabled.column.id'='true', " + + s"'write.parquet.bloom-filter-fpp.column.id'='$configuredFpp', " + s"'write.parquet.bloom-filter-ndv.column.id'='$configuredNdv'") createTable(warehouseDir, "bloom_explicit_ndv_fold_native", partitionSpec = "", properties) createTable(warehouseDir, "bloom_explicit_ndv_fold_jvm", partitionSpec = "", properties) @@ -1746,11 +1766,22 @@ class CometIcebergWriteActionSuite val jvm = parquetBloomFilterBytes("bloom_explicit_ndv_fold_jvm", "id") assert(native.nonEmpty && native.size == jvm.size) assert( - native.zip(jvm).forall { case (nativeBytes, jvmBytes) => - nativeBytes.length < jvmBytes.length - }, - "expected parquet-rs to fold the explicit-NDV allocation after observing two values") + native.forall(_.length == expectedNativeBytes), + s"expected folded native filters of $expectedNativeBytes bytes") + assert( + jvm.forall(_.length == expectedJvmBytes), + s"expected Parquet Java filters of $expectedJvmBytes bytes") assertParquetBloomContainsInts("bloom_explicit_ndv_fold_native", "id", insertedIds) + assertParquetBloomContainsInts("bloom_explicit_ndv_fold_jvm", "id", insertedIds) + + val nativeFalsePositives = + parquetBloomFalsePositiveCount("bloom_explicit_ndv_fold_native", "id", absentProbeValues) + val jvmFalsePositives = + parquetBloomFalsePositiveCount("bloom_explicit_ndv_fold_jvm", "id", absentProbeValues) + assert(nativeFalsePositives == expectedNativeFalsePositives) + assert(jvmFalsePositives == 0L) + assert(nativeFalsePositives > jvmFalsePositives) + assert(nativeFalsePositives.toDouble / absentProbeValues.size <= configuredFpp) } } @@ -5506,6 +5537,36 @@ class CometIcebergWriteActionSuite present.forall(identity) } + private def parquetBloomFalsePositiveCount( + tableName: String, + columnName: String, + absentValues: Seq[Int]): Long = { + val paths = spark + .sql(s"SELECT file_path FROM $catalog.$ns.$tableName.data_files ORDER BY file_path") + .collect() + .map(_.getString(0)) + val conf = spark.sparkContext.hadoopConfiguration + val counts = paths.toSeq.flatMap { path => + val input = HadoopInputFile.fromPath(new Path(path.stripPrefix("file:")), conf) + val reader = ParquetFileReader.open(input) + try { + reader.getFooter.getBlocks.asScala.flatMap { block => + block.getColumns.asScala + .filter(_.getPath.toDotString == columnName) + .map { column => + val bloom = reader.readBloomFilter(column) + assert(bloom != null, s"expected Bloom filter for $columnName in $path") + absentValues.count(value => bloom.findHash(bloom.hash(value))).toLong + } + } + } finally { + reader.close() + } + } + assert(counts.size == 1, s"expected one Bloom filter for $columnName, got ${counts.size}") + counts.head + } + private def assertParquetBloomContainsInts( tableName: String, columnName: String, @@ -5595,10 +5656,12 @@ class CometIcebergWriteActionSuite * version / session-state combinations where `withSQLConf` loses the override before the rule * fires. */ - private def withNativeEnabled[T](action: => T): T = + private def withNativeEnabled[T](action: => T, bloomFilterFoldingEnabled: Boolean = true): T = withSessionConf( CometConf.COMET_ICEBERG_NATIVE_WRITE_ENABLED.key -> Some("true"), - CometConf.COMET_EXEC_LOCAL_TABLE_SCAN_ENABLED.key -> Some("true"))(action) + CometConf.COMET_EXEC_LOCAL_TABLE_SCAN_ENABLED.key -> Some("true"), + CometConf.COMET_ICEBERG_NATIVE_WRITE_BLOOM_FILTER_FOLDING_ENABLED.key -> + Some(bloomFilterFoldingEnabled.toString))(action) /** * Sets each key to its value, or unsets it for `None`, for the duration of `action`, then puts @@ -5826,7 +5889,15 @@ class CometIcebergWriteActionSuite */ private def assertNativeWriteDoesNotEngage(tableName: String, expectedIds: Seq[Int])( action: => Unit): Unit = { - val snapshot = withNativeEnabled { captureWrite(tableName)(action) } + assertNativeWriteDoesNotEngage(tableName, expectedIds, bloomFilterFoldingEnabled = true)( + action) + } + + private def assertNativeWriteDoesNotEngage( + tableName: String, + expectedIds: Seq[Int], + bloomFilterFoldingEnabled: Boolean)(action: => Unit): Unit = { + val snapshot = withNativeEnabled(captureWrite(tableName)(action), bloomFilterFoldingEnabled) assertExactlyOneCommit(snapshot) val nativeExecs = snapshot.plans.flatMap { p => collectWithSubqueries(p) { case e: CometIcebergWriteExec => e } diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala index edc81d23f53..1f3cc2f8f88 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala @@ -54,6 +54,7 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes override protected def sparkConf: SparkConf = { super.sparkConf .set(CometConf.COMET_ICEBERG_NATIVE_WRITE_ENABLED.key, "true") + .set(CometConf.COMET_ICEBERG_NATIVE_WRITE_BLOOM_FILTER_FOLDING_ENABLED.key, "true") } override protected def test(testName: String, testTags: Tag*)(testFun: => Any)(implicit @@ -578,6 +579,22 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes } } + test("Bloom-filter folding must be explicitly enabled for native writes") { + withDetectionCatalog { dir => + createTable( + dir, + "bloom_folding_opt_in", + partitionSpec = "", + properties = Some("'write.parquet.bloom-filter-enabled.column.id'='true'")) + withSQLConf( + CometConf.COMET_ICEBERG_NATIVE_WRITE_BLOOM_FILTER_FOLDING_ENABLED.key -> "false") { + assertUnsupportedContains( + "bloom_folding_opt_in", + CometConf.COMET_ICEBERG_NATIVE_WRITE_BLOOM_FILTER_FOLDING_ENABLED.key) + } + } + } + test("Compatible when the schema exceeds max-inferred-column-defaults") { withDetectionCatalog { _ => createTable( From ae68947c637ff11f3ce37f7446c58445ff1a93e3 Mon Sep 17 00:00:00 2001 From: Nikita Matskevich Date: Sat, 24 Oct 2026 14:07:55 +0200 Subject: [PATCH 16/32] fix: integrate Bloom write opt-in with current Iceberg writer --- .../user-guide/latest/iceberg-writes.md | 380 ------------------ docs/source/user-guide/latest/iceberg.md | 53 ++- native/proto/src/proto/operator.proto | 3 +- .../scala/org/apache/comet/CometConf.scala | 16 +- .../comet/iceberg/IcebergReflection.scala | 1 - .../operator/CometIcebergNativeWrite.scala | 18 +- .../comet/CometIcebergWriteActionSuite.scala | 39 +- .../CometIcebergWriteDetectionSuite.scala | 28 +- 8 files changed, 105 insertions(+), 433 deletions(-) delete mode 100644 docs/source/user-guide/latest/iceberg-writes.md diff --git a/docs/source/user-guide/latest/iceberg-writes.md b/docs/source/user-guide/latest/iceberg-writes.md deleted file mode 100644 index 50756000c6e..00000000000 --- a/docs/source/user-guide/latest/iceberg-writes.md +++ /dev/null @@ -1,380 +0,0 @@ - - -# Iceberg Writes: Comet's Split-Operator Plan (Experimental) - -**This feature is experimental and disabled by default.** Enable it only after validating it -against your own workloads. - -## Overview - -Spark writes an Iceberg table through a single physical operator that combines data-file -writing with metadata writing, committing, and catalog validation. Because that operator sits -outside Spark's Adaptive Query Execution (AQE), the sub-query feeding the write — the scans, -projects, sorts, and exchanges producing the rows — cannot be re-planned at runtime. - -When `spark.comet.write.iceberg.splitOperator.enabled=true`, Comet rewrites eligible Iceberg -writes into two operators: - -1. **`IcebergWrite`** — writes the data files on the executors, exactly as iceberg-java does - today, and returns each task's serialized commit message. This operator and the sub-query - feeding it run inside AQE. -2. **`IcebergCommit`** — collects the commit messages on the driver and performs the normal - Iceberg commit (including commit-time validation), outside AQE, exactly once. - -With only the split plan enabled, data files are still written by iceberg-java; only the plan -shape changes. The split makes the write's input visible to AQE and to Comet's columnar rules, -and it is the foundation for the second toggle: when -`spark.comet.iceberg.write.enabled=true` and the write passes the eligibility check below, the -`IcebergWrite` operator's per-task Parquet write is delegated to -[iceberg-rust](https://github.com/apache/iceberg-rust) via Comet's native execution pipeline -([#5308](https://github.com/apache/datafusion-comet/issues/5308)). - -## How the native write works - -The JVM-side planner marshals everything iceberg-rust needs — the write schema and partition -spec as JSON, the data location, the resolved parquet writer settings, the writer mode -(unpartitioned / fanout / clustered, mirroring `SparkWrite`'s own choice), object-store -configuration (the table's `FileIO` properties, e.g. REST-vended credentials, merged over -`fs.s3a.*` settings translated from the session Hadoop configuration — the same translation -the native scan uses, since `HadoopFileIO` carries its S3 configuration in the Hadoop -Configuration rather than in `FileIO` properties), and per-task IDs — into the serialized -native plan. On each task, iceberg-rust writes the Parquet files and -returns its `DataFile` metadata packed as a single in-memory Iceberg V2 data manifest; the JVM -decodes those bytes with Iceberg's own `ManifestFiles.read`, re-derives each file's manifest -metrics from the written Parquet footer with Iceberg's `MetricsConfig` logic (so metrics modes, -truncation, and bounds decisions are iceberg-java's by construction), and wraps the result in -the same `TaskCommit` message the JVM writer would have produced. Everything iceberg-java does -post-write — snapshot assignment, manifest-list aggregation, commit validation and retries — -is untouched: `IcebergCommit` performs the normal `BatchWrite.commit`. - -## Configuration - -Standard Comet + Iceberg setup (see [`iceberg.md`](iceberg.md)) plus the write-side toggle: - -``` -# Standard Comet / Iceberg wiring -spark.plugins=org.apache.spark.CometPlugin -spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions -spark.sql.catalog.=org.apache.iceberg.spark.SparkCatalog -spark.sql.catalog..type=hadoop # or hive / glue / rest / ... -spark.sql.catalog..warehouse=... - -# Split-operator plan (experimental, off by default) -spark.comet.write.iceberg.splitOperator.enabled=true - -# Native-write eligibility detection (experimental, off by default; requires the split plan) -spark.comet.iceberg.write.enabled=true -``` - -## Supported operations - -The split-operator plan supports the following operations on every Spark version Comet -supports: - -- `INSERT INTO` / DataFrame `append` (`AppendData`) -- `INSERT OVERWRITE`, static and dynamic (`OverwriteByExpression`, - `OverwritePartitionsDynamic`) -- Copy-on-write `DELETE` / `UPDATE` / `MERGE` (`ReplaceData`) - -The mechanism behind row-level DML differs by Spark version: on Spark 4.0+ the analyzer emits -operation-coded rows that Comet's writer dispatches through `ReplaceData`'s projections, while -on Spark 3.4/3.5 the rewritten rows are written as a plain row stream. The supported set of -operations is the same either way. - -On Spark 4.1+ the split plan matches two further stock-Spark behaviours: MERGE metrics are -forwarded to the writer's commit (Iceberg 1.11+ records them in the snapshot summary), and -cached catalog tables are recached by name after a write so cache entries survive schema -changes. - -## When Comet falls back to Spark's write operator - -The rewrite is skipped — and the write runs through Spark's stock combined operator — when: - -- `spark.comet.write.iceberg.splitOperator.enabled` is `false` (the default); -- the write is not an Iceberg `SparkWrite` (any other V2 data source); -- the table uses merge-on-read: delta writes (Iceberg `WriteDelta`) are not intercepted; -- the statement is CTAS / RTAS on Spark 3.4, where the staged exec writes inline; on Spark - 3.5+ those statements re-plan their inner append, which is intercepted normally; -- the write requires Spark's commit coordinator, which Comet's per-task commit protocol does - not use; -- Comet cannot reflect the Iceberg internals needed to build the two-operator plan (for - example an unrecognised write class or a `ReplaceData` projection it cannot map). - -In every fallback case the write is planned as if Comet were absent; there is no correctness -trade-off, only no plan change. - -## Native Parquet write eligibility - -When `spark.comet.iceberg.write.enabled=true` -([#5308](https://github.com/apache/datafusion-comet/issues/5308)), the `IcebergWrite` operator's -per-task Parquet write is delegated to [iceberg-rust](https://github.com/apache/iceberg-rust). -The native writer must produce the same outcome as iceberg-java — the same Parquet features, -statistics, and manifest metadata — so a write is only eligible when every table property it -depends on is one the native path reproduces exactly, and additionally only when the plan -feeding the write is fully Comet-native. For a partitioned table that plan includes the hash -distribution and local sort Iceberg requests on its partition transforms; those stay native -because the transforms themselves have native implementations (see -[Iceberg system functions](iceberg.md)). Ineligible writes run through iceberg-java unchanged, -with the reason reported as a fall-back reason in Comet's extended EXPLAIN output. - -**Most Iceberg write settings are not supported.** Detection is an allowlist: a write is -eligible only when its entire effective configuration matches the table below, and anything -else — any other write-affecting property, any key added by a future Iceberg version, any -value outside the supported set, any reflection failure while inspecting the write — falls -back to iceberg-java with a reason reported in extended EXPLAIN. Checks run on the effective -configuration: table properties overlaid with `SparkWrite.writeProperties`, which is where -iceberg-java resolves per-write options and `spark.sql.iceberg.*` session overrides. - -A write is eligible only when ALL of the following hold: - -| Setting | Supported values | -| ------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| resolved write format (`write-format` option overlaid on `write.format.default`) | `parquet` | -| `format-version` | `1` or `2` | -| `write.parquet.compression-codec` / `compression-level` / `row-group-size-bytes` / `page-size-bytes` / `page-row-limit` / `dict-size-bytes` | the sizes/limit must be positive Java ints (`Integer.parseInt` semantics — no trimming, no values past `Int.MaxValue` — matching iceberg-java, whose writer fails on anything else); `compression-level` must be a Java int within the native writer's per-codec range (zstd 1–22, gzip 0–9, brotli 0–11; ignored by both writers for snappy/lz4/none) — iceberg-java never validates the level, so an out-of-range value falls back rather than becoming a native task failure | -| `write.parquet.row-group-check-min-record-count` | unset or `100` (the default) | -| `write.parquet.row-group-check-max-record-count` | unset or `10000` (the default) | -| `write.parquet.page-version` | unset or `v1` | -| `write.parquet.shred-variants` | unset or `false` (Spark 4.x / Iceberg 1.11 resolve this into every parquet write) | -| `write.parquet.variant-inference-buffer-size` | any value (only meaningful when shredding, which is gated) | -| `write.parquet.bloom-filter-enabled.column.` | `true` or `false`; an explicit NDV enables the column even when this value is `false`, matching Iceberg's property application order | -| `write.parquet.bloom-filter-fpp.column.` / `write.parquet.bloom-filter-ndv.column.` | Interpretation depends on the Iceberg runtime: Iceberg 1.5.2 ignores both prefixes, Iceberg 1.8.1 through 1.10.x interprets FPP but ignores NDV, and Iceberg 1.11 interprets both. For every interpreted property on a column named by an `enabled` property, FPP must be a finite double strictly between 0 and 1 and NDV must be a positive Java long no greater than `Long.MAX_VALUE / 8`; the Iceberg FPP default is `0.01` | -| `write.parquet.bloom-filter-max-bytes` | unset (Iceberg default: 1 MiB), or a power of two from 32 bytes through 128 MiB inclusive; every other value falls back to iceberg-java. A 32-byte value also falls back when explicit NDV/FPP would request a larger filter, because Parquet Java ignores that boundary as a maximum. | -| `write.metadata.metrics.*` | any value (manifest metrics are re-derived on the JVM with Iceberg's own logic) | -| `write.spark.fanout.enabled` | any value (the native writer implements both clustered and fanout modes) | -| `write.target-file-size-bytes` | any value (the two writers can choose different roll points; see accepted divergences) | -| data location URI scheme | `file`, `memory`, `s3`, `s3a`, `gs` | -| partition spec | any | -| column types | any except `uuid` (Spark plans it as a string; no Arrow cast reaches `fixed(16)`) | - -Within the namespaces that shape data-file bytes — `write.parquet.*` and `parquet.*` — -everything not listed above must be absent: unvetted `write.parquet.*` keys (e.g. -`stats-enabled.column.*`, keys added by future Iceberg versions), -any `parquet.*` table property (including `parquet.enable.dictionary`), and any `parquet.*` -key in the session Hadoop configuration (with `HadoopFileIO`-backed output those reach -iceberg-java's writer but not the native one). Also gated explicitly: any `encryption.*` key, -`write.object-storage.enabled=true`, `write.location-provider.impl`, and `io-impl`. - -Two checks look past properties at the table's instantiated state, because both can be -configured at the catalog level (or by a custom `TableOperations`) without any table or write -property changing: `table.io()` must be a recognized `FileIO` (the same allowlist the native -scan uses, minus the `EncryptingFileIO` family — the native writer produces plaintext files, -so an encrypting `FileIO` is rejected on the write side), and `table.encryption()` must be -Iceberg's `PlaintextEncryptionManager`. Anything else falls back. - -Other `write.*` properties are intentionally not gated because they cannot make the native -writer produce different data files: distribution and ordering settings shape the Spark plan -identically on both paths, WAP / branch / snapshot properties act on the JVM committer, -`write.avro.*` / `write.orc.*` apply only to formats already excluded, and merge-on-read -settings route the write through `WriteDelta`, which the split plan never intercepts. Every -rule is pinned by `CometIcebergWriteDetectionSuite`. - -Manifest `DataFile` metrics are assembled on the JVM before commit: each written file's -metrics are re-derived from its parquet footer through the version-matched -`ParquetUtil.footerMetrics` and `MetricsConfig.forTable`, with float/double NaN counts and -bounds carried over from the native writer's tracked state. iceberg-java's metadata decisions -— metrics modes, the inferred-column cap -(`write.metadata.metrics.max-inferred-column-defaults`), bound truncation, and list/map bounds -suppression — are therefore applied by iceberg-java's own code regardless of what the native -writer reports. This costs one footer-sized ranged read per written file at write time. - -## Failure handling - -Eligibility is decided entirely at plan time. That includes the reflection surface: every -iceberg-java class, method, and constructor the executor-side commit-message assembly uses is -eagerly resolved by the eligibility gate on the driver, so an Iceberg release that moves any -of them declines the native path with a fall-back reason instead of failing tasks mid-write. -Once planned, the physical plan is fixed — there is no per-task re-decision or runtime switch -back to the JVM writer. - -When a native write fails partway through a task (an object-store error, a data-dependent cast -failure), the error propagates as an ordinary Spark task failure and Spark's task retry -re-executes it — through the native writer again. Retries cannot collide: each attempt's task -attempt id is embedded in its data file names. - -Partial results are never committed. The commit set is exactly the commit messages returned by -successful tasks — a failed task contributes none — and if the job fails, the driver-side -commit operator aborts without committing anything. A failed task attempt also deletes the -data files it created, as iceberg-java's writer abort does. The native writer records every -location it hands to a file writer, and exactly one side owns deleting them at any moment: the -native writer owns them until its output batch reaches the JVM (so it cleans up a failed write, -a task torn down before the write completed — for example because the operator feeding it threw -— and a failure encoding the manifest or building that batch), and the JVM owns them from then -on through a task failure listener. The handoff does not depend on decoding the manifest: the -native side reports the locations in the output batch alongside it, and the listener is handed -them before the manifest is decoded, so a failure in that decode still cleans up. Both -deletions are best-effort and never mask the original failure; anything they miss is invisible -to every reader, since readers resolve files through committed manifests only, and is reclaimed -by Iceberg's normal `remove_orphan_files` maintenance. - -When one task fails, the tasks that had already completed leave committed-nothing data files -too. The committer collects each task's commit message as that task finishes, so on a job -failure it aborts with the completed messages and deletes their data files through the table -`FileIO`. (Iceberg's own `SparkWrite.abort` skips cleanup unless a commit failed with a -cleanable error, so on the stock path those files are left for `remove_orphan_files`.) A -failure during the driver-side commit itself behaves exactly as on the stock path: the commit -messages carry genuine `SparkWrite$TaskCommit` objects, so Iceberg's own `SparkWrite.abort` -cleanup (which deletes the files listed in the commit messages for cleanable failures) applies -unchanged. - -## Accepted divergences behind the toggle - -Some differences between parquet-mr and the pinned parquet-rs / iceberg-rust are unconditional — -they apply to every native write and cannot be configured away. Enabling -`spark.comet.iceberg.write.enabled` accepts them. They fall into three classes with very -different blast radius: differences confined to the physical bytes of a data file (cosmetic — -no reader decision is based on them), differences visible in manifest metadata (these outlive -the write and feed later readers' pruning decisions, so each one is analyzed individually -below), and one operational path-layout caveat. - -### Parquet Bloom-filter sizing - -[Iceberg's documented write properties](https://iceberg.apache.org/docs/latest/configuration/#write-properties) -describe three related inputs. FPP is the requested false-positive probability (default `0.01`), -NDV is the expected number of distinct values when explicitly set, and `max-bytes` is an upper -bound (default 1 MiB). - -Apache Parquet Java permits arbitrary integer caps. When such a cap binds, it serializes exactly -that many bytes, although only complete 32-byte SBBF blocks are used and any trailing partial -block remains zero. The Apache Arrow Rust `parquet` crate requires a power-of-two block count so -its post-write folding remains valid. A non-power-of-two cap can therefore change the -hash-to-block mapping, making a filter that may have worse reader pruning than Parquet Java's -filter. - -Comet uses its native Iceberg writer only when the effective `max-bytes` value is a power of two -from 32 bytes through 128 MiB inclusive. If an explicit value is not a power of two or is outside -that range, `CometIcebergWriteExec` is not used for the write; Spark's default Iceberg Java writer -writes the table instead. The same fallback applies when `max-bytes=32` would bind an explicit -NDV/FPP request, because Parquet Java ignores exactly 32 bytes as a maximum, and when NDV is above -`Long.MAX_VALUE / 8`, where Parquet Java's sizing multiplication can overflow. - -Bloom-filter-enabled writes also require -`spark.comet.iceberg.write.bloomFilterFolding.enabled=true`. The option is off by default. When it -is false, Comet leaves the write to Spark's default Iceberg Java writer because the Apache Arrow -Rust `parquet` writer does not expose an option to disable its post-insert folding. - -For supported values, Apache Parquet Java applies the sizing properties as follows: - -- with no NDV, allocate the full `max-bytes` value; -- with an NDV, calculate a requested size from NDV and FPP, then cap it at `max-bytes`; -- when the cap binds, it takes precedence, so the requested FPP is not guaranteed; -- a large maximum never enlarges the allocation selected by an explicit, smaller NDV. - -For every write that is eligible for the native path, Comet applies exactly the same allocation -decision algorithm. - -After values are inserted, the Apache Arrow Rust `parquet` crate may fold a sparsely populated -filter to a smaller power-of-two allocation. Sparse filters occur in ordinary workloads when NDV -is overestimated, row-group cardinalities are uneven, or a final row group is only partly full. -Parquet Java's non-adaptive Iceberg path retains its initial allocation: its realized FPP can be -much lower than requested, at the cost of serializing the larger filter. The Rust writer folds -until another fold would exceed the configured FPP, so it serializes fewer bytes but can have a -higher realized FPP and weaker pruning than the Java writer. Unless `max-bytes` already makes the -requested FPP impossible, the result remains within the configured target. Folding cannot -introduce false negatives or incorrect query results. - -The option therefore controls a serialized-file-size versus pruning tradeoff. It does not imply -lower peak writer memory: the Rust writer retains the original vector capacity while folding. -The test `overestimated explicit NDV folds and exposes the realized-FPP tradeoff` pins this -behavior with matching native/Java membership checks, serialized Bloom sizes, and measured false -positives. `large representable max folds natively while adjacent non-power-of-two falls back` -covers the extreme case of a 64-MiB configured maximum and only 256 distinct values. - -### Physical file layout only (cosmetic) - -No Iceberg reader bases a planning or correctness decision on these; they change the bytes of -a data file but not what any reader computes from it: - -- Footer key-value metadata differs: native files carry an `ARROW:schema` entry and no - `iceberg.schema` entry; iceberg-java files are the opposite. -- The Parquet root schema element is named `arrow_schema` (iceberg-java: `table`). -- `created_by` identifies parquet-rs, not parquet-mr. -- No page CRC checksums and no page-header statistics (parquet-mr writes both by default). - Absent page-header statistics can only make a reader scan more pages, never skip pages it - should have read; page pruning uses the column index, which the native writer does produce. -- Dictionary-encoded pages are labeled `RLE_DICTIONARY` (parquet-mr v1 files: `PLAIN_DICTIONARY`). -- Fixed-length binary columns (`uuid`, `fixed`, decimals with precision > 18) are not - dictionary-encoded (parquet-mr dictionary-encodes them). -- Row-group boundaries: parquet-mr flushes by byte size at a record-count check cadence, - parquet-rs buffers by row count. File naming follows the same cadence-style difference - (iceberg-java names files `---`; iceberg-rust uses a - process-local counter). -- Partition directory names match iceberg-java 1.8+'s `PartitionSpec.partitionToPath` for every - partition type except `float` and `double`, where the value is rendered with Rust's shortest - representation instead of `Float.toString` / `Double.toString` (`f=1` where iceberg-java writes - `f=1.0`). On Iceberg 1.5.x, which the Spark 3.4 profile pins, iceberg-java itself spelled - `timestamp` and `timestamptz` directories with `LocalDateTime.toString()` / - `OffsetDateTime.toString()` (`ts=1969-12-31T23:59:58.500Z`) and left the partition field name - unescaped; Comet uses the 1.8+ spelling on every profile. Distinct partition values still get - distinct directories in all cases, and no reader parses these names — files are resolved through - committed manifests. Iceberg deprecated float and double partitioning in 1.3. -- File rolling lands on the same row grid as iceberg-java but not necessarily on the same row. - Both writers re-check the current file's size against `write.target-file-size-bytes` once - every 1000 rows of that file (iceberg-java's `RollingFileWriter.ROWS_DIVISOR`; Comet hands the - iceberg-rust writer rows in 1000-row units, per partition file, to get the same grid), so each - writer rolls only on a 1000-row boundary of its own file. - The shared grid is all that is shared. What each writer compares against the target differs — - flushed bytes plus parquet-rs's estimate of the open row group, versus parquet-mr's file - position plus its buffered size — and the two use different threshold comparisons. These are - independent size estimates, so nothing bounds how far apart the two writers' roll points are: - they may cross the target several grid steps apart, and the resulting files can differ in row - count by an arbitrary number of 1000-row blocks. Do not rely on file-layout parity between the - two writers; rely only on each file rolling on its own 1000-row boundary. -- A fanout write lists a task's data files in file-path order, where iceberg-java lists them in - its own `StructLikeMap` iteration order. Both are stable across runs, and neither is a - documented ordering, but the manifest entry order becomes the scan-task order and so the row - order of an unordered `SELECT *`. Only the sorted order is reproducible on the native path: - iceberg-rust's `FanoutWriter` closes its per-partition writers out of a `HashMap`, which under - Rust's per-process `RandomState` would otherwise give a different order on every run. Clustered - and unpartitioned writes append in creation order on both paths and are unaffected. -- Compressed page bytes are implementation-defined: the codec and any explicit level are - translated, but parquet-rs and parquet-mr embed different encoder implementations and - defaults (zstd default levels, LZ4 framing), so byte-identical output is not achievable even - for a default `zstd` table. The decompressed data is identical. For the same reason, - codec-level side channels (`zlib.compress.level`, `compression.brotli.quality`, - `io.compression.codec.zstd.level` — the last is present in every Hadoop configuration by - default) are not gated: they can only shift compressed bytes, which are already accepted as - divergent. - -### Manifest metadata visible to later readers - -Manifest metrics drive partition- and file-level pruning for every future reader of the table, -so a divergence here would outlive the write. This class is deliberately kept almost empty: -`DataFile` metrics are not taken from the native writer's manifest but re-derived on the JVM -from each written file's parquet footer through iceberg-java's own `ParquetUtil.footerMetrics` -and `MetricsConfig.forTable` (see above). Metrics modes, lower/upper bound truncation, the -null-count conventions, and list/map bounds suppression are therefore iceberg-java's code -making iceberg-java's decisions, and the parity suite compares committed manifests -byte-for-byte against JVM-written ones. Two footer-derived values can still differ from what -iceberg-java's _writer-tracked_ state would have recorded, and both are analyzed safe: - -- Float/double bounds involving zero may differ in sign: parquet-rs normalises footer - statistics to min `-0.0` / max `+0.0` (the parquet-format recommendation), while - iceberg-java's writer-tracked bounds preserve the exact sign it saw. The native path's - manifest bounds inherit the normalised values — a strictly conservative widening that cannot - change pruning decisions. -- On Iceberg 1.10+, manifest `value_counts` / `null_value_counts` for float/double columns - nested under a nullable struct count rows whose parent struct is null (they come from the - parquet footer), while iceberg-java's writer-tracked counts do not. Both counts inflate by - the same amount, so the derived null ratios and `IS NULL` / `IS NOT NULL` pruning decisions - are unaffected. - -All content not listed above — the logical data, encodings for non-FLBA columns, statistics -values, and manifest metadata — must match iceberg-java exactly, or the write falls back. diff --git a/docs/source/user-guide/latest/iceberg.md b/docs/source/user-guide/latest/iceberg.md index 972ceea6fbf..7539aa35ce7 100644 --- a/docs/source/user-guide/latest/iceberg.md +++ b/docs/source/user-guide/latest/iceberg.md @@ -485,9 +485,11 @@ A write is eligible only when ALL of the following hold: | `write.parquet.page-version` | unset or `v1` | | `write.parquet.shred-variants` | unset or `false` (Spark 4.x / Iceberg 1.11 resolve this into every parquet write) | | `write.parquet.variant-inference-buffer-size` | any value (only meaningful when shredding, which is gated) | -| `write.parquet.bloom-filter-enabled.column.` | unset or `false` | | `write.metadata.metrics.*` | any value (manifest metrics are re-derived on the JVM with Iceberg's own logic) | | `write.spark.fanout.enabled` | any value (the native writer implements both clustered and fanout modes) | +| `write.parquet.bloom-filter-enabled.column.` | `true` or `false`; on Iceberg 1.11+, an explicit NDV enables the column even when this value is `false`. Enabled columns use the native writer when `spark.comet.write.iceberg.bloomFilterFolding.enabled=true` | +| `write.parquet.bloom-filter-fpp.column.` / `write.parquet.bloom-filter-ndv.column.` | Interpretation depends on the Iceberg runtime: Iceberg 1.5.2 ignores both prefixes, Iceberg 1.8.1 through 1.10.x interprets FPP but ignores NDV, and Iceberg 1.11 interprets both. For every interpreted property on a column named by an `enabled` property, FPP must be a finite double strictly between 0 and 1 and NDV must be a positive Java long no greater than `Long.MAX_VALUE / 8`; the Iceberg FPP default is `0.01` | +| `write.parquet.bloom-filter-max-bytes` | With a configured Bloom column: unset (1 MiB) or a power of two from 32 bytes through 128 MiB. Java writes other settings, including a 32-byte cap that it ignores for a larger explicit NDV/FPP request. With no configured Bloom column: any positive Java int. | | `write.target-file-size-bytes` | any value (the two writers can choose different roll points; see accepted divergences) | | data location URI scheme | `file`, `memory`, `s3`, `s3a`, `gs`, matched case-sensitively (`S3://` falls back). `s3`, `s3a` and `gs` need a bucket in the authority (`s3://bucket/...`), so a hostless form such as `s3:/bucket/key` falls back. `gs` only when the `FileIO` opening the data location is a `GCSFileIO`; see below | | resolved `table.locationProvider()` | Iceberg's built-in `DefaultLocationProvider` | @@ -498,7 +500,7 @@ A write is eligible only when ALL of the following hold: Within the namespaces that shape data-file bytes — `write.parquet.*` and `parquet.*` — everything not listed above must be absent: unvetted `write.parquet.*` keys (e.g. -`bloom-filter-max-bytes`, `stats-enabled.column.*`, keys added by future Iceberg versions), +`stats-enabled.column.*`, keys added by future Iceberg versions), any `parquet.*` table property (including `parquet.enable.dictionary`), and any `parquet.*` key in the session Hadoop configuration other than the reader-only `parquet.hadoop.vectored.io.enabled` (with `HadoopFileIO`-backed output those reach @@ -651,6 +653,53 @@ messages carry genuine `SparkWrite$TaskCommit` objects, so Iceberg's own `SparkW cleanup (which deletes the files listed in the commit messages for cleanable failures) applies unchanged. +### Native Bloom-filter writes + +Set `spark.comet.write.iceberg.bloomFilterFolding.enabled=true` together with +`spark.comet.write.iceberg.enabled=true` to use native writes for Bloom-enabled tables. The +Bloom option defaults to `false`, which selects Iceberg's Java writer when a Bloom column is +enabled. It selects the writer; it cannot turn folding off inside the pinned parquet-rs writer. + +Native Bloom filters offer a file-size versus pruning tradeoff. After inserting values, +parquet-rs can fold a sparse filter into fewer bytes while targeting the configured false-positive +probability (FPP). Iceberg's non-adaptive Java writer retains its initial allocation, which can +have a lower realized FPP and prune more row groups. Sparse filters can arise from ordinary skew, +an overestimated distinct-value count (NDV), or a partially filled final row group. Keeping the +option off can be useful when preserving Java's stronger pruning for those row groups matters +more than reducing serialized filter size. Turning it on accepts that tradeoff. Folding preserves +inserted-value membership; it does not introduce false negatives. It retains the original buffer +capacity, so smaller serialized filters do not imply lower peak writer memory. + +Boolean filters also differ: parquet-java leaves their bitsets empty, whereas parquet-rs inserts +boolean values. This difference is covered by the same opt-in. Applications comparing Bloom +bytes across writers should account for it separately from folding. + +Java also omits Bloom filters for column chunks whose data pages are entirely dictionary encoded, +while the pinned Rust writer can retain them. This changes filter presence and serialized space +independently of folding. + +Comet matches Java's initial allocation decision for native-eligible settings: without NDV it +allocates `write.parquet.bloom-filter-max-bytes` (default 1 MiB); with NDV it sizes from NDV and FPP +(default `0.01`) and applies the cap. A binding cap can make the requested FPP unattainable on +either writer. Iceberg 1.10 interprets per-column FPP and ignores NDV; Iceberg 1.11 interprets both. +Comet follows the loaded Iceberg runtime, including its defaults and ignored properties. + +The eligibility table above describes settings that use the native writer. Configured Bloom +columns use power-of-two caps from 32 bytes to 128 MiB; other caps use Java's writer, as do NDVs +above `Long.MAX_VALUE / 8` and explicit NDV/FPP requests for which Java ignores a 32-byte cap. +With no configured Bloom column, any positive Java-int cap is accepted because it does not affect +the file. Malformed or non-positive caps follow Java's validation through fallback. + +Iceberg 1.10 and newer versions use canonical Iceberg column names. Earlier Iceberg runtimes +need no separate user setting: Comet resolves raw physical Parquet paths before Iceberg 1.8, +field-name aliases in 1.8/1.9, and canonical names in 1.10 and newer, matching each Java writer. + +The upstream [folding option request](https://github.com/apache/arrow-rs/issues/11891) tracks a +way to retain the initial allocation in parquet-rs, implemented in +[arrow-rs #11892](https://github.com/apache/arrow-rs/pull/11892). Disabling folding addresses one source of +byte differences; physical value encodings, initial sizing, and other writer settings still +need to match for byte identity. + ### Accepted divergences Some differences between parquet-mr and the pinned parquet-rs / iceberg-rust are unconditional — diff --git a/native/proto/src/proto/operator.proto b/native/proto/src/proto/operator.proto index bfe4bb6a489..8d1443ade39 100644 --- a/native/proto/src/proto/operator.proto +++ b/native/proto/src/proto/operator.proto @@ -819,7 +819,8 @@ message IcebergParquetWriteSettings { // physical Parquet leaf paths used by list/map encodings and resolves effective defaults. repeated IcebergColumnBloomFilterProps bloom_filters = 8; // Iceberg `write.parquet.bloom-filter-max-bytes` (default 1 MiB). Native eligibility only - // admits powers of two in [32, 128 MiB], which parquet-rs can represent exactly. + // admits powers of two in [32, 128 MiB] when a Bloom column is configured. With no Bloom + // columns, any positive Java int is accepted because it does not affect the file. uint64 bloom_filter_max_bytes = 9; } diff --git a/spark/src/main/scala/org/apache/comet/CometConf.scala b/spark/src/main/scala/org/apache/comet/CometConf.scala index 792c6cf16eb..e327fd6fb30 100644 --- a/spark/src/main/scala/org/apache/comet/CometConf.scala +++ b/spark/src/main/scala/org/apache/comet/CometConf.scala @@ -145,13 +145,15 @@ object CometConf extends ShimCometConf { .createWithDefault(false) val COMET_ICEBERG_NATIVE_WRITE_BLOOM_FILTER_FOLDING_ENABLED: ConfigEntry[Boolean] = - conf("spark.comet.iceberg.write.bloomFilterFolding.enabled") - .category(CATEGORY_TESTING) - .doc( - "Whether Bloom-filter-enabled Iceberg writes may use Comet's native writer. The " + - "Apache Arrow Rust parquet writer folds sparse Bloom filters after insertion, which " + - "can reduce serialized size while raising realized FPP toward the configured target. " + - "When false, these writes use Iceberg's Java writer. Off by default.") + conf("spark.comet.write.iceberg.bloomFilterFolding.enabled") + .category(CATEGORY_EXEC) + .doc("Whether Bloom-filter-enabled Iceberg writes may use Comet's native writer. The " + + "Apache Arrow Rust parquet writer folds sparse Bloom filters after insertion, which " + + "can reduce serialized size while raising realized FPP toward the configured target. " + + "When false, these writes use Iceberg's Java writer. This option selects the writer; " + + "it cannot disable folding inside parquet-rs. Boolean filters also differ: Java leaves " + + "them empty while Rust inserts values. Requires spark.comet.write.iceberg.enabled. " + + "Off by default.") .booleanConf .createWithDefault(false) diff --git a/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala b/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala index 7c52ef15074..eab9d5a11c3 100644 --- a/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala +++ b/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala @@ -888,7 +888,6 @@ object IcebergReflection extends Logging { /** * The fields of `schema` at any depth, as id, name, and the type of a field with no children. - */ private def fieldsAtAnyDepth(schema: Any): Seq[(Int, String, Option[Any])] = { import scala.jdk.CollectionConverters._ diff --git a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala index c960fcd83c3..fe1c0f1bdca 100644 --- a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala +++ b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala @@ -337,7 +337,10 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { sessionConf => effectiveHadoopConf(op, table, sessionConf)) val context = TriggerContext( - table, tableProperties ++ writeProperties, sparkWrite, hadoopConfs, + table, + tableProperties ++ writeProperties, + sparkWrite, + hadoopConfs, CometConf.COMET_ICEBERG_NATIVE_WRITE_BLOOM_FILTER_FOLDING_ENABLED.get(op.conf)) triggers.iterator.map(rule => rule(context)).collectFirst { case Some(reason) => reason } } @@ -639,13 +642,16 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { val configuredMaxBytes = properties.get(PropertyKeys.ParquetBloomFilterMaxBytes).map { raw => raw -> scala.util.Try(java.lang.Integer.parseInt(raw)).toOption } - val malformedMaxRejection = configuredMaxBytes.collect { case (raw, None) => - s"${PropertyKeys.ParquetBloomFilterMaxBytes}=$raw is not a Java int" + val malformedMaxRejection = configuredMaxBytes.collect { + case (raw, None) => + s"${PropertyKeys.ParquetBloomFilterMaxBytes}=$raw is not a Java int" + case (raw, Some(value)) if value <= 0 => + s"${PropertyKeys.ParquetBloomFilterMaxBytes}=$raw must be positive" } // Iceberg visits every enabled-prefix entry and applies enabled, FPP, then NDV. With no such - // entry, a parseable maximum cannot affect the file and must not force an otherwise-compatible - // write onto the classic path. Keep rejecting malformed values because protobuf translation - // parses this property before native execution. + // entry, a positive, parseable maximum cannot affect the file and must not force an + // otherwise-compatible write onto the classic path. Keep rejecting malformed and + // non-positive values because Iceberg validates them even with no Bloom column. val configured = properties.iterator.collect { case (key, _) if key.startsWith(PropertyKeys.BloomFilterColumnEnabledPrefix) => key.substring(PropertyKeys.BloomFilterColumnEnabledPrefix.length) diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala index ec99151de1b..ba6eeae12e7 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala @@ -1245,7 +1245,6 @@ class CometIcebergWriteActionSuite assumeNativeAcceleration() withIcebergCatalog { warehouseDir => createTable( - warehouseDir, "native_bloom", partitionSpec = "", properties = Some("'write.parquet.bloom-filter-enabled.column.id'='true'")) @@ -1267,8 +1266,8 @@ class CometIcebergWriteActionSuite assumeNativeAcceleration() withIcebergCatalog { warehouseDir => val properties = Some("'write.parquet.bloom-filter-enabled.column.id'='true'") - createTable(warehouseDir, "bloom_folding_enabled", partitionSpec = "", properties) - createTable(warehouseDir, "bloom_folding_disabled", partitionSpec = "", properties) + createTable("bloom_folding_enabled", partitionSpec = "", properties) + createTable("bloom_folding_disabled", partitionSpec = "", properties) assertNativeWriteEngages("bloom_folding_enabled", Seq(1)) { spark.sql("INSERT INTO cat.db.bloom_folding_enabled VALUES (1, 'region', 1.0)") @@ -1293,8 +1292,8 @@ class CometIcebergWriteActionSuite "'write.parquet.bloom-filter-enabled.column.id'='false', " + s"'write.parquet.bloom-filter-fpp.column.id'='$configuredFpp', " + s"'write.parquet.bloom-filter-ndv.column.id'='$configuredNdv'") - createTable(warehouseDir, "bloom_false_ndv_native", partitionSpec = "", properties) - createTable(warehouseDir, "bloom_false_ndv_jvm", partitionSpec = "", properties) + createTable("bloom_false_ndv_native", partitionSpec = "", properties) + createTable("bloom_false_ndv_jvm", partitionSpec = "", properties) def insert(table: String): Unit = spark.sql( s"INSERT INTO cat.db.$table " + @@ -1337,7 +1336,7 @@ class CometIcebergWriteActionSuite val nativeTable = s"bloom_false_bad_${suffix}_native" val jvmTable = s"bloom_false_bad_${suffix}_jvm" Seq(nativeTable, jvmTable).foreach { table => - createTable(warehouseDir, table, partitionSpec = "", properties = properties) + createTable(table, partitionSpec = "", properties = properties) } def insert(table: String): Unit = @@ -1381,14 +1380,12 @@ class CometIcebergWriteActionSuite val parquetMrBindingBytes = 4 * 1024 * 1024 val enabled = "'write.parquet.bloom-filter-enabled.column.id'='true', " createTable( - warehouseDir, "bloom_minimum_native", partitionSpec = "", properties = Some( enabled + s"'write.parquet.bloom-filter-ndv.column.id'='$naturallyMinimumNdv', " + s"'write.parquet.bloom-filter-max-bytes'='$minimumBytes'")) createTable( - warehouseDir, "bloom_minimum_fallback", partitionSpec = "", properties = Some( @@ -1595,8 +1592,8 @@ class CometIcebergWriteActionSuite "'write.parquet.bloom-filter-fpp.column.id'='0.01', " + s"'write.parquet.bloom-filter-ndv.column.id'='$configuredNdv', " + s"'write.parquet.bloom-filter-max-bytes'='$bindingMaxBytes'") - createTable(warehouseDir, "bloom_identity_native", partitionSpec = "", properties) - createTable(warehouseDir, "bloom_identity_jvm", partitionSpec = "", properties) + createTable("bloom_identity_native", partitionSpec = "", properties) + createTable("bloom_identity_jvm", partitionSpec = "", properties) def insert(table: String): Unit = spark.sql( s"INSERT INTO cat.db.$table " + @@ -1659,7 +1656,7 @@ class CometIcebergWriteActionSuite val nativeTable = s"bloom_shape_${suffix}_native" val jvmTable = s"bloom_shape_${suffix}_jvm" Seq(nativeTable, jvmTable).foreach { table => - createTable(warehouseDir, table, partitionSpec = "", properties = Some(properties)) + createTable(table, partitionSpec = "", properties = Some(properties)) } // Keep the explicit-NDV cases at their estimated cardinality so parquet-rs does not // fold their allocation before byte-parity is checked. @@ -1709,8 +1706,8 @@ class CometIcebergWriteActionSuite "'write.parquet.bloom-filter-fpp.column.id'='0.01', " + "'write.parquet.bloom-filter-ndv.column.id'='10', " + "'write.parquet.bloom-filter-max-bytes'='67108864'") - createTable(warehouseDir, "bloom_low_ndv_native", partitionSpec = "", properties) - createTable(warehouseDir, "bloom_low_ndv_jvm", partitionSpec = "", properties) + createTable("bloom_low_ndv_native", partitionSpec = "", properties) + createTable("bloom_low_ndv_jvm", partitionSpec = "", properties) def insert(table: String): Unit = spark.sql( s"INSERT INTO cat.db.$table " + @@ -1747,8 +1744,8 @@ class CometIcebergWriteActionSuite "'write.parquet.bloom-filter-enabled.column.id'='true', " + s"'write.parquet.bloom-filter-fpp.column.id'='$configuredFpp', " + s"'write.parquet.bloom-filter-ndv.column.id'='$configuredNdv'") - createTable(warehouseDir, "bloom_explicit_ndv_fold_native", partitionSpec = "", properties) - createTable(warehouseDir, "bloom_explicit_ndv_fold_jvm", partitionSpec = "", properties) + createTable("bloom_explicit_ndv_fold_native", partitionSpec = "", properties) + createTable("bloom_explicit_ndv_fold_jvm", partitionSpec = "", properties) def insert(table: String): Unit = spark.sql( s"INSERT INTO cat.db.$table " + @@ -1791,12 +1788,10 @@ class CometIcebergWriteActionSuite val powerOfTwo = 64 * 1024 * 1024 val enabled = "'write.parquet.bloom-filter-enabled.column.id'='true', " createTable( - warehouseDir, "bloom_large_native", partitionSpec = "", properties = Some(enabled + s"'write.parquet.bloom-filter-max-bytes'='$powerOfTwo'")) createTable( - warehouseDir, "bloom_large_jvm", partitionSpec = "", properties = @@ -1828,7 +1823,6 @@ class CometIcebergWriteActionSuite // through the JVM writer with a cap equal to the observed folded size. Power-of-two folding // must produce exactly the same SBBF bitset as hashing directly into that final allocation. createTable( - warehouseDir, "bloom_large_folded_jvm", partitionSpec = "", properties = Some(enabled + s"'write.parquet.bloom-filter-max-bytes'='$nativeSize'")) @@ -1858,7 +1852,6 @@ class CometIcebergWriteActionSuite val max = "134217729" val table = "bloom_range_fallback" createTable( - warehouseDir, table, partitionSpec = "", properties = Some(enabledWithSmallNdv + s"'write.parquet.bloom-filter-max-bytes'='$max'")) @@ -5494,7 +5487,7 @@ class CometIcebergWriteActionSuite .map(_.getString(0)) val conf = spark.sparkContext.hadoopConfiguration paths.toSeq.flatMap { path => - val input = HadoopInputFile.fromPath(new Path(path.stripPrefix("file:")), conf) + val input = HadoopInputFile.fromPath(new HadoopPath(path.stripPrefix("file:")), conf) val reader = ParquetFileReader.open(input) try { reader.getFooter.getBlocks.asScala.flatMap { block => @@ -5521,7 +5514,7 @@ class CometIcebergWriteActionSuite .map(_.getString(0)) val conf = spark.sparkContext.hadoopConfiguration val present = paths.toSeq.flatMap { path => - val input = HadoopInputFile.fromPath(new Path(path.stripPrefix("file:")), conf) + val input = HadoopInputFile.fromPath(new HadoopPath(path.stripPrefix("file:")), conf) val reader = ParquetFileReader.open(input) try { reader.getFooter.getBlocks.asScala.flatMap { block => @@ -5547,7 +5540,7 @@ class CometIcebergWriteActionSuite .map(_.getString(0)) val conf = spark.sparkContext.hadoopConfiguration val counts = paths.toSeq.flatMap { path => - val input = HadoopInputFile.fromPath(new Path(path.stripPrefix("file:")), conf) + val input = HadoopInputFile.fromPath(new HadoopPath(path.stripPrefix("file:")), conf) val reader = ParquetFileReader.open(input) try { reader.getFooter.getBlocks.asScala.flatMap { block => @@ -5577,7 +5570,7 @@ class CometIcebergWriteActionSuite .map(_.getString(0)) val conf = spark.sparkContext.hadoopConfiguration paths.foreach { path => - val input = HadoopInputFile.fromPath(new Path(path.stripPrefix("file:")), conf) + val input = HadoopInputFile.fromPath(new HadoopPath(path.stripPrefix("file:")), conf) val reader = ParquetFileReader.open(input) try { reader.getFooter.getBlocks.asScala.foreach { block => diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala index 1f3cc2f8f88..50790aeb064 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala @@ -383,7 +383,6 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes Seq("32", "524288", "1048576", "134217728").zipWithIndex.foreach { case (value, index) => val table = s"bloom_max_ok_$index" createTable( - dir, table, partitionSpec = "", properties = Some(s"'write.parquet.bloom-filter-max-bytes'='$value'")) @@ -394,7 +393,6 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes .foreach { case (value, index) => val table = s"bloom_max_bad_$index" createTable( - dir, table, partitionSpec = "", properties = Some( @@ -411,7 +409,6 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes withDetectionCatalog { dir => val nonPowerOfTwoMaxBytes = 100 createTable( - dir, "unused_bloom_max", partitionSpec = "", properties = Some(s"'write.parquet.bloom-filter-max-bytes'='$nonPowerOfTwoMaxBytes'")) @@ -419,6 +416,21 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes } } + test("unused bloom-filter max-bytes preserves Java validation") { + withDetectionCatalog { _ => + Seq("0", "-1").zipWithIndex.foreach { case (value, index) => + val table = s"unused_bloom_invalid_$index" + createTable( + table, + partitionSpec = "", + properties = Some(s"'write.parquet.bloom-filter-max-bytes'='$value'")) + assertUnsupportedContainsAllowingWriteFailure( + table, + "write.parquet.bloom-filter-max-bytes") + } + } + } + test("bloom-filter max-bytes=32 falls back only when parquet-mr ignores the cap") { withDetectionCatalog { dir => val minimumBytes = 32 @@ -426,14 +438,12 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes val bindingFpp = 0.0001 val bindingNdv = 1000000 createTable( - dir, "without_ndv", partitionSpec = "", properties = Some(s"'write.parquet.bloom-filter-max-bytes'='$minimumBytes'")) assertSupportLevelIs[Compatible]("without_ndv") createTable( - dir, "natural_minimum", partitionSpec = "", properties = Some( @@ -443,7 +453,6 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes assertSupportLevelIs[Compatible]("natural_minimum") createTable( - dir, "ignored_minimum_cap", partitionSpec = "", properties = Some( @@ -473,7 +482,6 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes cases.zipWithIndex.foreach { case ((ndv, compatible), index) => val table = s"bloom_ndv_overflow_$index" createTable( - dir, table, partitionSpec = "", properties = Some( @@ -494,7 +502,6 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes withDetectionCatalog { dir => val configuredNdv = 1000 createTable( - dir, "false_with_ndv", partitionSpec = "", properties = Some( @@ -503,7 +510,6 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes assertSupportLevelIs[Compatible]("false_with_ndv") createTable( - dir, "false_with_bad_ndv", partitionSpec = "", properties = Some( @@ -522,7 +528,6 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes test("bloom-filter enabled=false still validates FPP") { withDetectionCatalog { dir => createTable( - dir, "false_with_bad_fpp", partitionSpec = "", properties = Some( @@ -554,7 +559,6 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes .foreach { case ((property, interpreted), index) => val table = s"bloom_shape_bad_$index" createTable( - dir, table, partitionSpec = "", properties = @@ -570,7 +574,6 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes test("Compatible when a per-column bloom filter is enabled") { withDetectionCatalog { _ => - createTable( "bloom_col", partitionSpec = "", @@ -582,7 +585,6 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes test("Bloom-filter folding must be explicitly enabled for native writes") { withDetectionCatalog { dir => createTable( - dir, "bloom_folding_opt_in", partitionSpec = "", properties = Some("'write.parquet.bloom-filter-enabled.column.id'='true'")) From 19ef2d529bc51b2821946f7fa3145cad62ca3e15 Mon Sep 17 00:00:00 2001 From: Nikita Matskevich Date: Sun, 25 Oct 2026 01:02:15 +0200 Subject: [PATCH 17/32] docs: clarify native Iceberg Bloom filter tradeoffs --- docs/source/user-guide/latest/iceberg.md | 79 ++++++++++-------------- 1 file changed, 34 insertions(+), 45 deletions(-) diff --git a/docs/source/user-guide/latest/iceberg.md b/docs/source/user-guide/latest/iceberg.md index 7539aa35ce7..5e4de26dbfd 100644 --- a/docs/source/user-guide/latest/iceberg.md +++ b/docs/source/user-guide/latest/iceberg.md @@ -488,7 +488,7 @@ A write is eligible only when ALL of the following hold: | `write.metadata.metrics.*` | any value (manifest metrics are re-derived on the JVM with Iceberg's own logic) | | `write.spark.fanout.enabled` | any value (the native writer implements both clustered and fanout modes) | | `write.parquet.bloom-filter-enabled.column.` | `true` or `false`; on Iceberg 1.11+, an explicit NDV enables the column even when this value is `false`. Enabled columns use the native writer when `spark.comet.write.iceberg.bloomFilterFolding.enabled=true` | -| `write.parquet.bloom-filter-fpp.column.` / `write.parquet.bloom-filter-ndv.column.` | Interpretation depends on the Iceberg runtime: Iceberg 1.5.2 ignores both prefixes, Iceberg 1.8.1 through 1.10.x interprets FPP but ignores NDV, and Iceberg 1.11 interprets both. For every interpreted property on a column named by an `enabled` property, FPP must be a finite double strictly between 0 and 1 and NDV must be a positive Java long no greater than `Long.MAX_VALUE / 8`; the Iceberg FPP default is `0.01` | +| `write.parquet.bloom-filter-fpp.column.` / `write.parquet.bloom-filter-ndv.column.` | Iceberg 1.10 interprets FPP but ignores NDV; 1.11 interprets both. For interpreted properties on a column named by an `enabled` property, FPP is finite and strictly between 0 and 1 (default `0.01`), and NDV is a positive Java long no greater than `Long.MAX_VALUE / 8`. | | `write.parquet.bloom-filter-max-bytes` | With a configured Bloom column: unset (1 MiB) or a power of two from 32 bytes through 128 MiB. Java writes other settings, including a 32-byte cap that it ignores for a larger explicit NDV/FPP request. With no configured Bloom column: any positive Java int. | | `write.target-file-size-bytes` | any value (the two writers can choose different roll points; see accepted divergences) | | data location URI scheme | `file`, `memory`, `s3`, `s3a`, `gs`, matched case-sensitively (`S3://` falls back). `s3`, `s3a` and `gs` need a bucket in the authority (`s3://bucket/...`), so a hostless form such as `s3:/bucket/key` falls back. `gs` only when the `FileIO` opening the data location is a `GCSFileIO`; see below | @@ -655,50 +655,39 @@ unchanged. ### Native Bloom-filter writes -Set `spark.comet.write.iceberg.bloomFilterFolding.enabled=true` together with -`spark.comet.write.iceberg.enabled=true` to use native writes for Bloom-enabled tables. The -Bloom option defaults to `false`, which selects Iceberg's Java writer when a Bloom column is -enabled. It selects the writer; it cannot turn folding off inside the pinned parquet-rs writer. - -Native Bloom filters offer a file-size versus pruning tradeoff. After inserting values, -parquet-rs can fold a sparse filter into fewer bytes while targeting the configured false-positive -probability (FPP). Iceberg's non-adaptive Java writer retains its initial allocation, which can -have a lower realized FPP and prune more row groups. Sparse filters can arise from ordinary skew, -an overestimated distinct-value count (NDV), or a partially filled final row group. Keeping the -option off can be useful when preserving Java's stronger pruning for those row groups matters -more than reducing serialized filter size. Turning it on accepts that tradeoff. Folding preserves -inserted-value membership; it does not introduce false negatives. It retains the original buffer -capacity, so smaller serialized filters do not imply lower peak writer memory. - -Boolean filters also differ: parquet-java leaves their bitsets empty, whereas parquet-rs inserts -boolean values. This difference is covered by the same opt-in. Applications comparing Bloom -bytes across writers should account for it separately from folding. - -Java also omits Bloom filters for column chunks whose data pages are entirely dictionary encoded, -while the pinned Rust writer can retain them. This changes filter presence and serialized space -independently of folding. - -Comet matches Java's initial allocation decision for native-eligible settings: without NDV it -allocates `write.parquet.bloom-filter-max-bytes` (default 1 MiB); with NDV it sizes from NDV and FPP -(default `0.01`) and applies the cap. A binding cap can make the requested FPP unattainable on -either writer. Iceberg 1.10 interprets per-column FPP and ignores NDV; Iceberg 1.11 interprets both. -Comet follows the loaded Iceberg runtime, including its defaults and ignored properties. - -The eligibility table above describes settings that use the native writer. Configured Bloom -columns use power-of-two caps from 32 bytes to 128 MiB; other caps use Java's writer, as do NDVs -above `Long.MAX_VALUE / 8` and explicit NDV/FPP requests for which Java ignores a 32-byte cap. -With no configured Bloom column, any positive Java-int cap is accepted because it does not affect -the file. Malformed or non-positive caps follow Java's validation through fallback. - -Iceberg 1.10 and newer versions use canonical Iceberg column names. Earlier Iceberg runtimes -need no separate user setting: Comet resolves raw physical Parquet paths before Iceberg 1.8, -field-name aliases in 1.8/1.9, and canonical names in 1.10 and newer, matching each Java writer. - -The upstream [folding option request](https://github.com/apache/arrow-rs/issues/11891) tracks a -way to retain the initial allocation in parquet-rs, implemented in -[arrow-rs #11892](https://github.com/apache/arrow-rs/pull/11892). Disabling folding addresses one source of -byte differences; physical value encodings, initial sizing, and other writer settings still -need to match for byte identity. +Comet's native Iceberg writer supports tables with column Bloom filters and can produce +byte-identical filters to Java for compatible settings. Other setups produce different filters +because parquet-rs enables folding by default, while Iceberg's non-adaptive parquet-java writer +does not. + +Folding automatically shrinks a filter when its initial NDV- or max-bytes-based allocation is +larger than the actual values need at the requested false-positive probability (FPP). It saves +serialized space by shrinking toward the target FPP; Java retains the larger filter. Folding +retains the original buffer capacity, so this does not imply lower peak writer memory. + +Enable `spark.comet.write.iceberg.bloomFilterFolding.enabled=true` together with +`spark.comet.write.iceberg.enabled=true` to use native Bloom writes. The Bloom option defaults to +`false`, selecting Java's writer. Enabling it accepts byte differences and the pruning tradeoff +below; it cannot turn folding off inside the pinned parquet-rs writer. Enabling it is generally +safe when the requested FPP meets your pruning needs: folding preserves inserted-value membership +and introduces no false negatives. + +Pruning can regress when existing Java-written filters are oversized for their actual values +and therefore have a much lower FPP than requested. For example, with no explicit NDV, a 64-MiB +maximum and only 256 distinct values, Java can retain a 64-MiB filter with extra pruning accuracy. +Rust can shrink that filter toward the target FPP and produce more false positives. Leave the +option off if your workload relies on that extra accuracy. This can happen with an overestimated +NDV, skew, or a partly filled row group. With explicit NDV, `max-bytes` is a ceiling; a large cap alone does not enlarge the allocation. + +Even with the option enabled, unsupported settings use Java's writer, preserving Bloom filters. +For configured Bloom columns, native caps are powers of two from 32 bytes through 128 MiB; +other caps and unsupported NDV/FPP combinations fall back as described in the eligibility table. + +Byte parity also requires matching inserted values, physical encodings, and initial sizes. +Boolean filters differ because Java does not populate them; dictionary-only chunks can have a +native filter where Java omits it. Comet follows the loaded runtime's properties: Iceberg 1.10 +ignores NDV, while 1.11 interprets it. The upstream [folding option PR](https://github.com/apache/arrow-rs/pull/11892) +([issue](https://github.com/apache/arrow-rs/issues/11891)) addresses retaining the initial Rust allocation. ### Accepted divergences From 1af729bb952968ed7563affd8176e77094945523 Mon Sep 17 00:00:00 2001 From: Nikita Matskevich Date: Sun, 25 Oct 2026 01:08:57 +0200 Subject: [PATCH 18/32] docs: simplify Bloom filter compatibility introduction --- docs/source/user-guide/latest/iceberg.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/source/user-guide/latest/iceberg.md b/docs/source/user-guide/latest/iceberg.md index 5e4de26dbfd..a630a33e16b 100644 --- a/docs/source/user-guide/latest/iceberg.md +++ b/docs/source/user-guide/latest/iceberg.md @@ -655,10 +655,9 @@ unchanged. ### Native Bloom-filter writes -Comet's native Iceberg writer supports tables with column Bloom filters and can produce -byte-identical filters to Java for compatible settings. Other setups produce different filters -because parquet-rs enables folding by default, while Iceberg's non-adaptive parquet-java writer -does not. +Comet's native Iceberg writer supports tables with column Bloom filters. However, byte-identical +behavior to classic Spark cannot be guaranteed for all settings: parquet-rs enables folding by +default, while Spark's non-adaptive parquet-java writer does not. Folding automatically shrinks a filter when its initial NDV- or max-bytes-based allocation is larger than the actual values need at the requested false-positive probability (FPP). It saves From 2901d470875c5a9392551c17bdb5e56fdb963afe Mon Sep 17 00:00:00 2001 From: Nikita Matskevich Date: Sun, 25 Oct 2026 01:11:31 +0200 Subject: [PATCH 19/32] docs: list exceptional Bloom filter fallback cases --- docs/source/user-guide/latest/iceberg.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/docs/source/user-guide/latest/iceberg.md b/docs/source/user-guide/latest/iceberg.md index a630a33e16b..c3c7ca1ba0e 100644 --- a/docs/source/user-guide/latest/iceberg.md +++ b/docs/source/user-guide/latest/iceberg.md @@ -678,9 +678,16 @@ Rust can shrink that filter toward the target FPP and produce more false positiv option off if your workload relies on that extra accuracy. This can happen with an overestimated NDV, skew, or a partly filled row group. With explicit NDV, `max-bytes` is a ceiling; a large cap alone does not enlarge the allocation. -Even with the option enabled, unsupported settings use Java's writer, preserving Bloom filters. -For configured Bloom columns, native caps are powers of two from 32 bytes through 128 MiB; -other caps and unsupported NDV/FPP combinations fall back as described in the eligibility table. +For configured Bloom columns, Comet uses Java's writer even with the option enabled when: + +- `max-bytes` is not a power of two from 32 bytes through 128 MiB. +- `max-bytes=32` and explicit NDV/FPP settings request a larger filter, because Java ignores + that boundary as a maximum. +- NDV is not a positive Java long or exceeds `Long.MAX_VALUE / 8`. +- FPP is not finite and strictly between 0 and 1, or is too extreme to represent the required + native filter sizes. + +Java's normal validation applies to invalid values and can still fail the write. Byte parity also requires matching inserted values, physical encodings, and initial sizes. Boolean filters differ because Java does not populate them; dictionary-only chunks can have a From 120dd0eee4e69db3a4521985fd98b780383af11d Mon Sep 17 00:00:00 2001 From: Nikita Matskevich Date: Sun, 25 Oct 2026 01:14:02 +0200 Subject: [PATCH 20/32] docs: remove upstream folding work from user guide --- docs/source/user-guide/latest/iceberg.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/source/user-guide/latest/iceberg.md b/docs/source/user-guide/latest/iceberg.md index c3c7ca1ba0e..65449fa0cf2 100644 --- a/docs/source/user-guide/latest/iceberg.md +++ b/docs/source/user-guide/latest/iceberg.md @@ -692,8 +692,7 @@ Java's normal validation applies to invalid values and can still fail the write. Byte parity also requires matching inserted values, physical encodings, and initial sizes. Boolean filters differ because Java does not populate them; dictionary-only chunks can have a native filter where Java omits it. Comet follows the loaded runtime's properties: Iceberg 1.10 -ignores NDV, while 1.11 interprets it. The upstream [folding option PR](https://github.com/apache/arrow-rs/pull/11892) -([issue](https://github.com/apache/arrow-rs/issues/11891)) addresses retaining the initial Rust allocation. +ignores NDV, while 1.11 interprets it. ### Accepted divergences From d24684d2b252034f73027a64e265f4e71a8f5e4f Mon Sep 17 00:00:00 2001 From: Nikita Matskevich Date: Sun, 25 Oct 2026 01:19:03 +0200 Subject: [PATCH 21/32] docs: group Bloom write prerequisites under accepted divergences --- docs/source/user-guide/latest/iceberg.md | 59 ++++++++++++------------ 1 file changed, 29 insertions(+), 30 deletions(-) diff --git a/docs/source/user-guide/latest/iceberg.md b/docs/source/user-guide/latest/iceberg.md index 65449fa0cf2..75af10699a9 100644 --- a/docs/source/user-guide/latest/iceberg.md +++ b/docs/source/user-guide/latest/iceberg.md @@ -653,57 +653,56 @@ messages carry genuine `SparkWrite$TaskCommit` objects, so Iceberg's own `SparkW cleanup (which deletes the files listed in the commit messages for cleanable failures) applies unchanged. -### Native Bloom-filter writes +### Accepted divergences + +Enabling `spark.comet.write.iceberg.enabled` accepts the physical file-layout and manifest-metadata +differences described below. Bloom-filter differences require the separate opt-in described here, +because they can affect pruning. + +#### Bloom filters Comet's native Iceberg writer supports tables with column Bloom filters. However, byte-identical behavior to classic Spark cannot be guaranteed for all settings: parquet-rs enables folding by default, while Spark's non-adaptive parquet-java writer does not. -Folding automatically shrinks a filter when its initial NDV- or max-bytes-based allocation is +Folding automatically shrinks a filter when its initial allocation from NDV or `max-bytes` is larger than the actual values need at the requested false-positive probability (FPP). It saves serialized space by shrinking toward the target FPP; Java retains the larger filter. Folding retains the original buffer capacity, so this does not imply lower peak writer memory. -Enable `spark.comet.write.iceberg.bloomFilterFolding.enabled=true` together with -`spark.comet.write.iceberg.enabled=true` to use native Bloom writes. The Bloom option defaults to -`false`, selecting Java's writer. Enabling it accepts byte differences and the pruning tradeoff -below; it cannot turn folding off inside the pinned parquet-rs writer. Enabling it is generally -safe when the requested FPP meets your pruning needs: folding preserves inserted-value membership -and introduces no false negatives. +In addition to the [general native-write eligibility conditions](#native-parquet-write-eligibility), +native writes for configured Bloom columns require all of the following: + +- `spark.comet.write.iceberg.enabled=true`. +- `spark.comet.write.iceberg.bloomFilterFolding.enabled=true`. +- The effective `write.parquet.bloom-filter-max-bytes` (default 1 MiB) is a power of two from + 32 bytes through 128 MiB. +- An interpreted NDV is absent or a positive Java long no greater than `Long.MAX_VALUE / 8`. +- Each interpreted FPP is finite, strictly between 0 and 1, and representable at the required + native filter sizes. +- With a 32-byte cap and interpreted explicit NDV, the NDV/FPP request fits in 32 bytes, because + Java ignores that boundary as a maximum for larger requests. +- Bloom-filter columns use physical Parquet paths supported by the native writer. + +The Bloom option defaults to `false`, selecting Java's writer. Enabling it accepts byte differences +and the pruning tradeoff below; it cannot turn folding off inside the pinned parquet-rs writer. +Enabling it is generally safe when the requested FPP meets your pruning needs: folding preserves +inserted-value membership and introduces no false negatives. Unsupported settings select Java's +writer, where normal property validation applies. Pruning can regress when existing Java-written filters are oversized for their actual values and therefore have a much lower FPP than requested. For example, with no explicit NDV, a 64-MiB maximum and only 256 distinct values, Java can retain a 64-MiB filter with extra pruning accuracy. Rust can shrink that filter toward the target FPP and produce more false positives. Leave the option off if your workload relies on that extra accuracy. This can happen with an overestimated -NDV, skew, or a partly filled row group. With explicit NDV, `max-bytes` is a ceiling; a large cap alone does not enlarge the allocation. - -For configured Bloom columns, Comet uses Java's writer even with the option enabled when: - -- `max-bytes` is not a power of two from 32 bytes through 128 MiB. -- `max-bytes=32` and explicit NDV/FPP settings request a larger filter, because Java ignores - that boundary as a maximum. -- NDV is not a positive Java long or exceeds `Long.MAX_VALUE / 8`. -- FPP is not finite and strictly between 0 and 1, or is too extreme to represent the required - native filter sizes. - -Java's normal validation applies to invalid values and can still fail the write. +NDV, skew, or a partly filled row group. With explicit NDV, `max-bytes` is a ceiling; a large cap +alone does not enlarge the allocation. Byte parity also requires matching inserted values, physical encodings, and initial sizes. Boolean filters differ because Java does not populate them; dictionary-only chunks can have a native filter where Java omits it. Comet follows the loaded runtime's properties: Iceberg 1.10 ignores NDV, while 1.11 interprets it. -### Accepted divergences - -Some differences between parquet-mr and the pinned parquet-rs / iceberg-rust are unconditional — -they apply to every native write and cannot be configured away. Enabling -`spark.comet.write.iceberg.enabled` accepts them. They fall into two classes with very -different blast radius: differences confined to the physical bytes of a data file (cosmetic — -no reader decision is based on them), and differences visible in manifest metadata (these -outlive the write and feed later readers' pruning decisions, so each one is analyzed -individually below). - #### Physical file layout only (cosmetic) No Iceberg reader bases a planning or correctness decision on these; they change the bytes of From 0b2b6e5296e2ff6ccc0cad755abfe8cd3e78dac6 Mon Sep 17 00:00:00 2001 From: Nikita Matskevich Date: Sun, 25 Oct 2026 19:37:34 +0100 Subject: [PATCH 22/32] fix: fall back for floating-point Iceberg Bloom writes --- docs/source/user-guide/latest/iceberg.md | 6 +- .../scala/org/apache/comet/CometConf.scala | 2 +- .../comet/iceberg/IcebergReflection.scala | 23 +++-- .../operator/CometIcebergNativeWrite.scala | 87 ++++++++++++++----- .../comet/CometIcebergWriteActionSuite.scala | 63 ++++++++++++++ .../CometIcebergWriteDetectionSuite.scala | 61 +++++++++++++ 6 files changed, 210 insertions(+), 32 deletions(-) diff --git a/docs/source/user-guide/latest/iceberg.md b/docs/source/user-guide/latest/iceberg.md index 75af10699a9..2b47cc050e3 100644 --- a/docs/source/user-guide/latest/iceberg.md +++ b/docs/source/user-guide/latest/iceberg.md @@ -683,12 +683,16 @@ native writes for configured Bloom columns require all of the following: - With a 32-byte cap and interpreted explicit NDV, the NDV/FPP request fits in 32 bytes, because Java ignores that boundary as a maximum for larger requests. - Bloom-filter columns use physical Parquet paths supported by the native writer. +- Bloom-filter columns are not FLOAT or DOUBLE, including nested leaves. Floating-point columns + without Bloom filters remain eligible for native writes. The Bloom option defaults to `false`, selecting Java's writer. Enabling it accepts byte differences and the pruning tradeoff below; it cannot turn folding off inside the pinned parquet-rs writer. Enabling it is generally safe when the requested FPP meets your pruning needs: folding preserves inserted-value membership and introduces no false negatives. Unsupported settings select Java's -writer, where normal property validation applies. +writer, where normal property validation applies. FLOAT/DOUBLE Bloom writes use Java until native +writes handle NaN equality consistently with Spark. The fallback is decided from the schema and +effective write properties before execution, and its reason identifies the affected columns. Pruning can regress when existing Java-written filters are oversized for their actual values and therefore have a much lower FPP than requested. For example, with no explicit NDV, a 64-MiB diff --git a/spark/src/main/scala/org/apache/comet/CometConf.scala b/spark/src/main/scala/org/apache/comet/CometConf.scala index e327fd6fb30..166f8721fdb 100644 --- a/spark/src/main/scala/org/apache/comet/CometConf.scala +++ b/spark/src/main/scala/org/apache/comet/CometConf.scala @@ -153,7 +153,7 @@ object CometConf extends ShimCometConf { "When false, these writes use Iceberg's Java writer. This option selects the writer; " + "it cannot disable folding inside parquet-rs. Boolean filters also differ: Java leaves " + "them empty while Rust inserts values. Requires spark.comet.write.iceberg.enabled. " + - "Off by default.") + "FLOAT/DOUBLE Bloom columns use Java to preserve NaN equality. Off by default.") .booleanConf .createWithDefault(false) diff --git a/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala b/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala index eab9d5a11c3..58011bad706 100644 --- a/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala +++ b/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala @@ -43,7 +43,8 @@ object IcebergReflection extends Logging { pathByIcebergColumnName: Map[String, String], pathByFieldId: Map[Int, String], physicalPaths: Set[String], - renamedPhysicalPaths: Set[String]) + renamedPhysicalPaths: Set[String], + floatingPointPhysicalPaths: Set[String]) /** * Iceberg class names used throughout Comet. @@ -722,6 +723,10 @@ object IcebergReflection extends Logging { val resolved = columns.asScala.flatMap { column => val primitiveType = getMethod(column.getClass, "getPrimitiveType").invoke(column) + val floatingPoint = Set("FLOAT", "DOUBLE").contains( + getMethod(primitiveType.getClass, "getPrimitiveTypeName") + .invoke(primitiveType) + .toString) val parquetId = getMethod(primitiveType.getClass, "getId").invoke(primitiveType) Option(parquetId).flatMap { id => val fieldId = getMethod(id.getClass, "intValue").invoke(id).asInstanceOf[Int] @@ -747,16 +752,22 @@ object IcebergReflection extends Logging { } } } - (fieldId, icebergName.asInstanceOf[String], parquetPathParts.mkString("."), renamed) + ( + fieldId, + icebergName.asInstanceOf[String], + parquetPathParts.mkString("."), + renamed, + floatingPoint) } } }.toSeq Some( ParquetPathResolution( - resolved.map { case (_, name, path, _) => name -> path }.toMap, - resolved.map { case (fieldId, _, path, _) => fieldId -> path }.toMap, - resolved.map { case (_, _, path, _) => path }.toSet, - resolved.collect { case (_, _, path, true) => path }.toSet)) + resolved.map { case (_, name, path, _, _) => name -> path }.toMap, + resolved.map { case (fieldId, _, path, _, _) => fieldId -> path }.toMap, + resolved.map { case (_, _, path, _, _) => path }.toSet, + resolved.collect { case (_, _, path, true, _) => path }.toSet, + resolved.collect { case (_, _, path, _, true) => path }.toSet)) } catch { case e: Exception => logError(s"Iceberg reflection failure: Parquet column paths: ${e.getMessage}") diff --git a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala index fe1c0f1bdca..c34a0eea1fb 100644 --- a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala +++ b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala @@ -438,6 +438,7 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { requireNativeSupportedCompressionLevel, requireNativeSupportedBloomFilterProperties, requireBloomFilterFoldingOptIn, + requireNoFloatingPointBloomFilters, requireOnlyVettedParquetWriteProperties, requirePropertyAbsent( PropertyKeys.ParquetEnableDictionary, @@ -724,6 +725,67 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { } } + private def bloomFilterParquetPaths( + configuredColumns: Seq[String], + schema: Any, + resolution: IcebergReflection.ParquetPathResolution): Map[String, String] = { + val icebergVersion = IcebergReflection.icebergVersion() + val versionParts = icebergVersion.split("[.-]", 3) + val version = + if (versionParts.length >= 2) { + scala.util.Try((versionParts(0).toInt, versionParts(1).toInt)).toOption + } else { + None + } + version match { + case Some((1, minor)) if minor < 8 => + // Iceberg 1.5 passes the configured name straight to parquet-mr. + configuredColumns.filter(resolution.physicalPaths).map(path => path -> path).toMap + case Some((1, minor)) if minor < 10 => + // Iceberg 1.8 resolves canonical and short collection aliases through field ids. + configuredColumns.flatMap { configuredName => + IcebergReflection + .findFieldIdByName(schema, configuredName) + .flatMap(resolution.pathByFieldId.get) + .map(configuredName -> _) + }.toMap + case _ => + // Iceberg 1.10+ keys writer properties by canonical findColumnName output. + resolution.pathByIcebergColumnName + } + } + + // A floating-point Bloom filter can reject a SQL-equal NaN with a different bit payload. + // Until native writes canonicalize these values, preserve Java's writer for affected leaves. + private val requireNoFloatingPointBloomFilters: TriggerRule = ctx => { + val properties = interpretedBloomFilterProperties(ctx.properties) + val columns = IcebergWriteProtoTranslation.enabledBloomFilterColumnNames(properties) + if (columns.isEmpty) { + None + } else { + val schema = IcebergReflection + .getWriteSchemaFromSparkWrite(ctx.sparkWrite) + .getOrElse( + throw new IllegalStateException("could not resolve schema for Bloom type checking")) + val resolution = IcebergReflection + .getParquetPathResolution(schema) + .getOrElse( + throw new IllegalStateException( + "could not resolve Parquet paths for Bloom type checking")) + val paths = bloomFilterParquetPaths(columns, schema, resolution) + val affected = columns.filter { column => + paths.get(column).exists(resolution.floatingPointPhysicalPaths.contains) + } + if (affected.isEmpty) { + None + } else { + Some( + s"Bloom filters on FLOAT/DOUBLE columns ${affected.sorted.mkString(", ")} " + + "require the Java writer to preserve NaN equality") + } + } + } + private val requireBloomFilterFoldingOptIn: TriggerRule = ctx => { val properties = interpretedBloomFilterProperties(ctx.properties) if (IcebergWriteProtoTranslation.hasEnabledBloomFilters(properties) && @@ -1327,30 +1389,7 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { } val configuredColumns = IcebergWriteProtoTranslation.enabledBloomFilterColumnNames(effectiveProperties) - val icebergVersion = IcebergReflection.icebergVersion() - val versionParts = icebergVersion.split("[.-]", 3) - val version = - if (versionParts.length >= 2) { - scala.util.Try((versionParts(0).toInt, versionParts(1).toInt)).toOption - } else { - None - } - val resolvedPaths = version match { - case Some((1, minor)) if minor < 8 => - // Iceberg 1.5 passes the configured name straight to parquet-mr. - configuredColumns.filter(resolution.physicalPaths).map(path => path -> path).toMap - case Some((1, minor)) if minor < 10 => - // Iceberg 1.8 resolves canonical and short collection aliases through field ids. - configuredColumns.flatMap { configuredName => - IcebergReflection - .findFieldIdByName(writeSchema, configuredName) - .flatMap(resolution.pathByFieldId.get) - .map(configuredName -> _) - }.toMap - case _ => - // Iceberg 1.10+ keys writer properties by canonical findColumnName output. - resolution.pathByIcebergColumnName - } + val resolvedPaths = bloomFilterParquetPaths(configuredColumns, writeSchema, resolution) val renamedColumns = resolvedPaths.collect { case (configuredName, path) if resolution.renamedPhysicalPaths.contains(path) => configuredName diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala index ba6eeae12e7..536d15a5412 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala @@ -1283,6 +1283,69 @@ class CometIcebergWriteActionSuite } } + test("floating-point Bloom writes fall back and preserve NaN predicate results") { + assumeNativeAcceleration() + withIcebergCatalog { _ => + val table = "floating_bloom_nan" + spark.sql(s""" + CREATE TABLE $catalog.$ns.$table ( + id INT, region STRING, amount DOUBLE, f FLOAT + ) USING iceberg TBLPROPERTIES ( + 'write.parquet.bloom-filter-enabled.column.amount'='true', + 'write.parquet.bloom-filter-enabled.column.f'='true', + 'write.parquet.dict-size-bytes'='1' + ) + """) + val count = 4096 + assertNativeWriteDoesNotEngage(table, 0 until count) { + spark.sql(s""" + INSERT INTO $catalog.$ns.$table + SELECT CAST(id AS INT), 'region', CAST('NaN' AS DOUBLE), CAST('NaN' AS FLOAT) + FROM range(0, $count, 1, 1) + """) + } + val paths = spark + .sql(s"SELECT file_path FROM $catalog.$ns.$table.data_files") + .collect() + .map(_.getString(0)) + assert(paths.nonEmpty) + val conf = spark.sparkContext.hadoopConfiguration + val bloomKey = "parquet.filter.bloom.enabled" + val previous = Option(conf.get(bloomKey)) + try { + withSQLConf(CometConf.COMET_ENABLED.key -> "false") { + Seq(false, true).foreach { enabled => + conf.setBoolean(bloomKey, enabled) + val written = spark.read.parquet(paths: _*) + assert(written.filter("amount = CAST('NaN' AS DOUBLE)").count() == count) + assert(written.filter("f = CAST('NaN' AS FLOAT)").count() == count) + } + } + } finally { + previous match { + case Some(value) => conf.set(bloomKey, value) + case None => conf.unset(bloomKey) + } + } + } + } + + test("floating-point columns without Bloom filters keep native writes") { + assumeNativeAcceleration() + withIcebergCatalog { _ => + createTable( + "unfiltered_floating_column", + partitionSpec = "", + properties = Some("'write.parquet.bloom-filter-enabled.column.id'='true'")) + assertNativeWriteEngages("unfiltered_floating_column", Seq(1)) { + spark.sql( + s"INSERT INTO $catalog.$ns.unfiltered_floating_column " + + "VALUES (1, 'region', CAST('NaN' AS DOUBLE))") + } + assertParquetBloomFilters("unfiltered_floating_column", enabledColumns = Set("id")) + } + } + test("enabled=false plus NDV matches JVM behavior for the Iceberg runtime") { assumeNativeAcceleration() withIcebergCatalog { warehouseDir => diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala index 50790aeb064..eefe5ea5fde 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala @@ -582,6 +582,53 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes } } + test("floating-point Bloom detection reports top-level and nested affected leaves") { + withDetectionCatalog { _ => + val columns = Seq( + "amount" -> "amount", + "f" -> "f", + "detail.value" -> "detail.value", + "samples.element" -> "samples.list.element", + "attrs.value" -> "attrs.key_value.value", + "lookup.key" -> "lookup.key_value.key") + columns.zipWithIndex.foreach { case ((logical, physical), index) => + val column = if (icebergSupportsBloomFpp) logical else physical + val table = s"floating_bloom_$index" + val write = floatingBloomWriteExec( + table, + s"'write.parquet.bloom-filter-enabled.column.$column'='true'") + assertUnsupportedContains(write, table, "FLOAT/DOUBLE", column, "NaN equality") + } + } + } + + test("floating-point Bloom detection preserves native eligibility for unaffected columns") { + withDetectionCatalog { _ => + val stringKey = if (icebergSupportsBloomFpp) "attrs.key" else "attrs.key_value.key" + val integerValue = if (icebergSupportsBloomFpp) "lookup.value" else "lookup.key_value.value" + val cases = Seq( + "'write.parquet.bloom-filter-enabled.column.id'='true'", + s"'write.parquet.bloom-filter-enabled.column.$stringKey'='true'", + s"'write.parquet.bloom-filter-enabled.column.$integerValue'='true'", + "'write.parquet.bloom-filter-enabled.column.missing'='true'", + "'write.parquet.bloom-filter-enabled.column.amount'='false'") + cases.zipWithIndex.foreach { case (properties, index) => + val write = floatingBloomWriteExec(s"unaffected_bloom_$index", properties) + assert(CometIcebergNativeWrite.getSupportLevel(write).isInstanceOf[Compatible]) + } + val table = "floating_bloom_false_ndv" + val write = floatingBloomWriteExec( + table, + "'write.parquet.bloom-filter-enabled.column.amount'='false', " + + "'write.parquet.bloom-filter-ndv.column.amount'='1000'") + if (icebergSupportsBloomNdv) { + assertUnsupportedContains(write, table, "FLOAT/DOUBLE", "amount", "NaN equality") + } else { + assert(CometIcebergNativeWrite.getSupportLevel(write).isInstanceOf[Compatible]) + } + } + } + test("Bloom-filter folding must be explicitly enabled for native writes") { withDetectionCatalog { dir => createTable( @@ -1792,6 +1839,20 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes } } + private def floatingBloomWriteExec(tableName: String, properties: String): IcebergWriteExec = { + spark.sql(s""" + CREATE TABLE $catalog.$ns.$tableName ( + id INT, region STRING, amount DOUBLE, f FLOAT, + detail STRUCT, samples ARRAY, + attrs MAP, lookup MAP + ) USING iceberg TBLPROPERTIES ($properties) + """) + planInsertWriteExec( + s"$catalog.$ns.$tableName", + "(1, 'region', 1.0, CAST(2.0 AS FLOAT), named_struct('value', 3.0D), " + + "array(CAST(4.0 AS FLOAT)), map('key', 5.0D), map(CAST(6.0 AS FLOAT), 1))") + } + private def createTable( tableName: String, partitionSpec: String, From 418b2deba66ab216c7d12fe84ff4f76761bd5bd6 Mon Sep 17 00:00:00 2001 From: Nikita Matskevich Date: Sun, 25 Oct 2026 20:52:16 +0100 Subject: [PATCH 23/32] fix: ignore unconfigured columns in Iceberg Bloom path checks --- .../operator/CometIcebergNativeWrite.scala | 4 +++- .../comet/CometIcebergWriteActionSuite.scala | 21 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala index c34a0eea1fb..9d147e7308b 100644 --- a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala +++ b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala @@ -751,7 +751,9 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { }.toMap case _ => // Iceberg 1.10+ keys writer properties by canonical findColumnName output. - resolution.pathByIcebergColumnName + resolution.pathByIcebergColumnName.filter { case (column, _) => + configuredColumns.contains(column) + } } } diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala index 536d15a5412..a996930dabf 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala @@ -1601,6 +1601,27 @@ class CometIcebergWriteActionSuite } } + test("unconfigured quoted columns preserve native Bloom writes") { + assumeNativeAcceleration() + withIcebergCatalog { _ => + spark.sql(s""" + CREATE TABLE $catalog.$ns.bloom_unconfigured_quoted ( + id INT, `order id` INT + ) USING iceberg + TBLPROPERTIES ('write.parquet.bloom-filter-enabled.column.id'='true') + """) + assertNativeWriteEngages("bloom_unconfigured_quoted", 0 until 1000) { + spark.sql(s""" + INSERT INTO $catalog.$ns.bloom_unconfigured_quoted + SELECT CAST(id AS INT), CAST(id AS INT) FROM range(0, 1000, 1, 1) + """) + } + assert(parquetBloomFilterPresent("bloom_unconfigured_quoted", "id")) + assertParquetBloomContainsInts("bloom_unconfigured_quoted", "id", 0 until 1000) + assert(!parquetBloomFilterPresent("bloom_unconfigured_quoted", "order id")) + } + } + test("quoted bloom-filter columns renamed by Iceberg fall back to the JVM footer") { assumeNativeAcceleration() withIcebergCatalog { _ => From 4f2bec2c4193629e3201a7994f55907531da4e7d Mon Sep 17 00:00:00 2001 From: Nikita Matskevich Date: Sun, 25 Oct 2026 20:53:54 +0100 Subject: [PATCH 24/32] fix: fall back for unidentified Iceberg Bloom naming conventions --- docs/source/user-guide/latest/iceberg.md | 1 + .../operator/CometIcebergNativeWrite.scala | 17 +++++--- .../IcebergWriteProtoTranslationSuite.scala | 43 +++++++++++++++++++ 3 files changed, 55 insertions(+), 6 deletions(-) diff --git a/docs/source/user-guide/latest/iceberg.md b/docs/source/user-guide/latest/iceberg.md index 2b47cc050e3..56f9f50da76 100644 --- a/docs/source/user-guide/latest/iceberg.md +++ b/docs/source/user-guide/latest/iceberg.md @@ -675,6 +675,7 @@ native writes for configured Bloom columns require all of the following: - `spark.comet.write.iceberg.enabled=true`. - `spark.comet.write.iceberg.bloomFilterFolding.enabled=true`. +- The Iceberg runtime reports an identifiable version so Comet can match its column-name rules. - The effective `write.parquet.bloom-filter-max-bytes` (default 1 MiB) is a power of two from 32 bytes through 128 MiB. - An interpreted NDV is absent or a positive Java long no greater than `Long.MAX_VALUE / 8`. diff --git a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala index 9d147e7308b..97f2efdbaf0 100644 --- a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala +++ b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala @@ -725,14 +725,14 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { } } - private def bloomFilterParquetPaths( + private[operator] def bloomFilterParquetPaths( configuredColumns: Seq[String], schema: Any, - resolution: IcebergReflection.ParquetPathResolution): Map[String, String] = { - val icebergVersion = IcebergReflection.icebergVersion() - val versionParts = icebergVersion.split("[.-]", 3) + resolution: IcebergReflection.ParquetPathResolution, + icebergVersion: String = IcebergReflection.icebergVersion()): Map[String, String] = { val version = - if (versionParts.length >= 2) { + if (Option(icebergVersion).exists(_.matches("\\d+\\.\\d+\\.\\d+(?:[-+].+)?"))) { + val versionParts = icebergVersion.split("[.-]", 3) scala.util.Try((versionParts(0).toInt, versionParts(1).toInt)).toOption } else { None @@ -749,11 +749,16 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { .flatMap(resolution.pathByFieldId.get) .map(configuredName -> _) }.toMap - case _ => + case Some((1, minor)) if minor >= 10 => // Iceberg 1.10+ keys writer properties by canonical findColumnName output. resolution.pathByIcebergColumnName.filter { case (column, _) => configuredColumns.contains(column) } + case _ => + // getSupportLevel catches this during planning and keeps the write on Java. Guessing + // canonical naming for repackaged runtimes could silently omit requested filters. + throw new IllegalArgumentException( + s"Cannot determine Iceberg Bloom column-name convention for version '$icebergVersion'") } } diff --git a/spark/src/test/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslationSuite.scala b/spark/src/test/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslationSuite.scala index 2cea3fb42d0..b824114746c 100644 --- a/spark/src/test/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslationSuite.scala +++ b/spark/src/test/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslationSuite.scala @@ -23,6 +23,7 @@ import scala.jdk.CollectionConverters._ import org.scalatest.funsuite.AnyFunSuite +import org.apache.comet.iceberg.IcebergReflection import org.apache.comet.serde.OperatorOuterClass.{CompressionCodec, IcebergWriterMode} /** @@ -35,6 +36,48 @@ class IcebergWriteProtoTranslationSuite extends AnyFunSuite { private val TestCreatedBy = "Apache DataFusion Comet (test)" + test("Bloom path resolution rejects unidentified Iceberg naming conventions") { + val resolution = IcebergReflection.ParquetPathResolution( + Map("tags.element.a" -> "tags.list.element.a"), + Map(1 -> "tags.list.element.a"), + Set("tags.list.element.a"), + Set.empty, + Set.empty) + Seq("unknown", "", "vendor-build", "1.8.bad", "1.10", "2.0.0").foreach { version => + val error = intercept[IllegalArgumentException] { + CometIcebergNativeWrite.bloomFilterParquetPaths(Seq("tags.a"), null, resolution, version) + } + assert(error.getMessage.contains("Cannot determine Iceberg Bloom column-name convention")) + assert(error.getMessage.contains(s"'$version'")) + } + } + + test("Bloom path resolution preserves recognized runtime naming conventions") { + val physicalPath = "tags.list.element.a" + val resolution = IcebergReflection.ParquetPathResolution( + Map("tags.element.a" -> physicalPath, "order id" -> "order_x20id"), + Map(1 -> physicalPath), + Set(physicalPath, "order_x20id"), + Set("order_x20id"), + Set.empty) + Seq("1.5.2", "1.5.2-SNAPSHOT").foreach { version => + assert( + CometIcebergNativeWrite.bloomFilterParquetPaths( + Seq(physicalPath, "tags.element.a"), + null, + resolution, + version) == Map(physicalPath -> physicalPath)) + } + Seq("1.10.0", "1.11.0", "1.11.0-SNAPSHOT", "1.11.0+vendor").foreach { version => + assert( + CometIcebergNativeWrite.bloomFilterParquetPaths( + Seq("tags.element.a", "tags.a"), + null, + resolution, + version) == Map("tags.element.a" -> physicalPath)) + } + } + // -- compression ----------------------------------------------------------- test("compression defaults to zstd at parquet-mr's level 3 when no property is set") { From 77857aaed661911aef684f2b3544c3c6d98d66af Mon Sep 17 00:00:00 2001 From: Nikita Matskevich Date: Sun, 25 Oct 2026 21:06:18 +0100 Subject: [PATCH 25/32] docs: clarify Iceberg Bloom opt-in and unused caps --- docs/source/user-guide/latest/iceberg.md | 23 ++++++++++++------- native/proto/src/proto/operator.proto | 8 ++++--- .../scala/org/apache/comet/CometConf.scala | 16 +++++++------ 3 files changed, 29 insertions(+), 18 deletions(-) diff --git a/docs/source/user-guide/latest/iceberg.md b/docs/source/user-guide/latest/iceberg.md index 56f9f50da76..4db4cece104 100644 --- a/docs/source/user-guide/latest/iceberg.md +++ b/docs/source/user-guide/latest/iceberg.md @@ -489,7 +489,7 @@ A write is eligible only when ALL of the following hold: | `write.spark.fanout.enabled` | any value (the native writer implements both clustered and fanout modes) | | `write.parquet.bloom-filter-enabled.column.` | `true` or `false`; on Iceberg 1.11+, an explicit NDV enables the column even when this value is `false`. Enabled columns use the native writer when `spark.comet.write.iceberg.bloomFilterFolding.enabled=true` | | `write.parquet.bloom-filter-fpp.column.` / `write.parquet.bloom-filter-ndv.column.` | Iceberg 1.10 interprets FPP but ignores NDV; 1.11 interprets both. For interpreted properties on a column named by an `enabled` property, FPP is finite and strictly between 0 and 1 (default `0.01`), and NDV is a positive Java long no greater than `Long.MAX_VALUE / 8`. | -| `write.parquet.bloom-filter-max-bytes` | With a configured Bloom column: unset (1 MiB) or a power of two from 32 bytes through 128 MiB. Java writes other settings, including a 32-byte cap that it ignores for a larger explicit NDV/FPP request. With no configured Bloom column: any positive Java int. | +| `write.parquet.bloom-filter-max-bytes` | `write.parquet.bloom-filter-enabled.column.*` present (even `false`): unset (1 MiB) or a power of two from 32 bytes through 128 MiB; larger explicit NDV/FPP requests fall back at 32 bytes because Java ignores that cap. With no such key: unset or any positive Java int, including `100`. Malformed, zero, or negative caps use Java and fail its validation. | | `write.target-file-size-bytes` | any value (the two writers can choose different roll points; see accepted divergences) | | data location URI scheme | `file`, `memory`, `s3`, `s3a`, `gs`, matched case-sensitively (`S3://` falls back). `s3`, `s3a` and `gs` need a bucket in the authority (`s3://bucket/...`), so a hostless form such as `s3:/bucket/key` falls back. `gs` only when the `FileIO` opening the data location is a `GCSFileIO`; see below | | resolved `table.locationProvider()` | Iceberg's built-in `DefaultLocationProvider` | @@ -656,8 +656,8 @@ unchanged. ### Accepted divergences Enabling `spark.comet.write.iceberg.enabled` accepts the physical file-layout and manifest-metadata -differences described below. Bloom-filter differences require the separate opt-in described here, -because they can affect pruning. +differences described below. Bloom-filter differences require the additional +`spark.comet.write.iceberg.bloomFilterFolding.enabled` opt-in, because they can affect pruning. #### Bloom filters @@ -687,8 +687,14 @@ native writes for configured Bloom columns require all of the following: - Bloom-filter columns are not FLOAT or DOUBLE, including nested leaves. Floating-point columns without Bloom filters remain eligible for native writes. -The Bloom option defaults to `false`, selecting Java's writer. Enabling it accepts byte differences -and the pruning tradeoff below; it cannot turn folding off inside the pinned parquet-rs writer. +The cap range applies whenever a `write.parquet.bloom-filter-enabled.column.*` property is +present, even if set to `false`. With no such properties, the cap is unused: unset or any +positive Java int, such as `100`, keeps an otherwise eligible write native. Malformed, zero, +or negative caps select Java, which rejects them through its normal validation. + +The Bloom option defaults to `false`, selecting Java's writer for Bloom-enabled tables. Enabling +it accepts byte differences and the pruning tradeoff below; it cannot turn folding off inside +the pinned parquet-rs writer. Enabling it is generally safe when the requested FPP meets your pruning needs: folding preserves inserted-value membership and introduces no false negatives. Unsupported settings select Java's writer, where normal property validation applies. FLOAT/DOUBLE Bloom writes use Java until native @@ -704,9 +710,10 @@ NDV, skew, or a partly filled row group. With explicit NDV, `max-bytes` is a cei alone does not enlarge the allocation. Byte parity also requires matching inserted values, physical encodings, and initial sizes. -Boolean filters differ because Java does not populate them; dictionary-only chunks can have a -native filter where Java omits it. Comet follows the loaded runtime's properties: Iceberg 1.10 -ignores NDV, while 1.11 interprets it. +The Bloom opt-in also accepts boolean-filter differences: parquet-java writes an all-zero +filter because it does not insert boolean values, while parquet-rs writes a populated filter. +Dictionary-only chunks can have a native filter where Java omits it. Comet follows the loaded +runtime's properties: Iceberg 1.10 ignores NDV, while 1.11 interprets it. #### Physical file layout only (cosmetic) diff --git a/native/proto/src/proto/operator.proto b/native/proto/src/proto/operator.proto index 8d1443ade39..fab7d6d01a3 100644 --- a/native/proto/src/proto/operator.proto +++ b/native/proto/src/proto/operator.proto @@ -818,9 +818,11 @@ message IcebergParquetWriteSettings { // Per-column Bloom-filter settings. The JVM driver translates Iceberg logical names to the // physical Parquet leaf paths used by list/map encodings and resolves effective defaults. repeated IcebergColumnBloomFilterProps bloom_filters = 8; - // Iceberg `write.parquet.bloom-filter-max-bytes` (default 1 MiB). Native eligibility only - // admits powers of two in [32, 128 MiB] when a Bloom column is configured. With no Bloom - // columns, any positive Java int is accepted because it does not affect the file. + // Iceberg `write.parquet.bloom-filter-max-bytes` (default 1 MiB). If any + // `write.parquet.bloom-filter-enabled.column.*` key is present (even false), native + // eligibility admits only powers of two in [32, 128 MiB]. With no such key, an unset cap + // or any positive Java int (including 100) is accepted because it does not affect the file. + // Malformed or non-positive caps fall back to Java, which rejects them during validation. uint64 bloom_filter_max_bytes = 9; } diff --git a/spark/src/main/scala/org/apache/comet/CometConf.scala b/spark/src/main/scala/org/apache/comet/CometConf.scala index 166f8721fdb..509ede7db7a 100644 --- a/spark/src/main/scala/org/apache/comet/CometConf.scala +++ b/spark/src/main/scala/org/apache/comet/CometConf.scala @@ -147,13 +147,15 @@ object CometConf extends ShimCometConf { val COMET_ICEBERG_NATIVE_WRITE_BLOOM_FILTER_FOLDING_ENABLED: ConfigEntry[Boolean] = conf("spark.comet.write.iceberg.bloomFilterFolding.enabled") .category(CATEGORY_EXEC) - .doc("Whether Bloom-filter-enabled Iceberg writes may use Comet's native writer. The " + - "Apache Arrow Rust parquet writer folds sparse Bloom filters after insertion, which " + - "can reduce serialized size while raising realized FPP toward the configured target. " + - "When false, these writes use Iceberg's Java writer. This option selects the writer; " + - "it cannot disable folding inside parquet-rs. Boolean filters also differ: Java leaves " + - "them empty while Rust inserts values. Requires spark.comet.write.iceberg.enabled. " + - "FLOAT/DOUBLE Bloom columns use Java to preserve NaN equality. Off by default.") + .doc( + "Whether Bloom-filter-enabled Iceberg writes may use Comet's native writer. The " + + "Apache Arrow Rust parquet writer folds sparse Bloom filters after insertion, which " + + "can reduce serialized size while raising realized FPP toward the configured target. " + + "When false, these writes use Iceberg's Java writer. This option selects the writer; " + + "it cannot disable folding inside parquet-rs. The opt-in also accepts boolean filter " + + "differences: Java writes all-zero filters while Rust populates them. " + + "Requires spark.comet.write.iceberg.enabled. " + + "FLOAT/DOUBLE Bloom columns use Java to preserve NaN equality. Off by default.") .booleanConf .createWithDefault(false) From e28e33ccd7b20bfef85d7f099d328e6219a57551 Mon Sep 17 00:00:00 2001 From: Nikita Matskevich Date: Sun, 25 Oct 2026 21:14:07 +0100 Subject: [PATCH 26/32] test: cover Iceberg Bloom parity across physical types --- .../comet/CometIcebergWriteActionSuite.scala | 154 ++++++++++++++++++ 1 file changed, 154 insertions(+) diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala index a996930dabf..406e45eb8d0 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala @@ -20,6 +20,7 @@ package org.apache.comet import java.io.{ByteArrayOutputStream, File} +import java.nio.ByteBuffer import java.nio.file.{FileAlreadyExistsException, Files} import java.sql.Timestamp import java.util.concurrent.{ConcurrentLinkedQueue, CountDownLatch, TimeUnit} @@ -38,6 +39,7 @@ import org.json4s.jackson.JsonMethods.parse import org.apache.hadoop.fs.{Path => HadoopPath} import org.apache.parquet.hadoop.ParquetFileReader import org.apache.parquet.hadoop.util.HadoopInputFile +import org.apache.parquet.io.api.Binary import org.apache.spark.{CometListenerBusUtils, SparkConf, SparkException, SparkThrowable, Success, TaskContext} import org.apache.spark.rdd.RDD import org.apache.spark.scheduler.{SparkListener, SparkListenerTaskEnd} @@ -1705,6 +1707,122 @@ class CometIcebergWriteActionSuite } } + test("native bloom filters match Java bytes and membership across physical types") { + assumeNativeAcceleration() + withIcebergCatalog { _ => + case class BloomColumn( + name: String, + sqlType: String, + expression: String, + physicalType: String, + encodedValue: Int => Any) + val columns = Seq( + BloomColumn( + "big", + "BIGINT", + "(id - 5000) * 1000000007", + "INT64", + id => (id - 5000).toLong * 1000000007L), + BloomColumn( + "text", + "STRING", + "concat('bloom-', cast(id - 5000 AS STRING), 'é')", + "BINARY", + id => Binary.fromString(s"bloom-${id - 5000}é")), + BloomColumn( + "bytes", + "BINARY", + "unhex(concat('00FF', lpad(hex(id), 8, '0')))", + "BINARY", + id => + Binary.fromConstantByteArray( + ByteBuffer + .allocate(6) + .put(0.toByte) + .put(0xff.toByte) + .putInt(id) + .array())), + BloomColumn( + "day", + "DATE", + "date_add(DATE '1970-01-01', cast(id - 5000 AS INT))", + "INT32", + id => id - 5000), + BloomColumn( + "ts", + "TIMESTAMP", + "timestamp_micros((id - 5000) * 1000003)", + "INT64", + id => (id - 5000).toLong * 1000003L), + BloomColumn( + "decimal_int32", + "DECIMAL(9, 2)", + "cast((id - 5000) / 100.0 AS DECIMAL(9, 2))", + "INT32", + id => id - 5000), + BloomColumn( + "decimal_int64", + "DECIMAL(18, 2)", + "cast((id - 5000) * 10000000000 AS DECIMAL(18, 2))", + "INT64", + id => (id - 5000).toLong * 1000000000000L), + BloomColumn( + "decimal_fixed", + "DECIMAL(38, 2)", + "cast((id - 5000) * 10000000000000000000000 AS DECIMAL(38, 2))", + "FIXED_LEN_BYTE_ARRAY", + id => { + val unscaled = BigInt(id - 5000) * BigInt(10).pow(24) + val bytes = unscaled.toByteArray + val padding = Array.fill[Byte](16 - bytes.length)( + if (unscaled.signum < 0) 0xff.toByte else 0.toByte) + Binary.fromConstantByteArray(padding ++ bytes) + })) + // Java can suppress filters for dictionary-only chunks. Force plain pages so every + // configured type contributes a filter to the cross-writer comparison. + val properties = (Seq( + "'write.parquet.bloom-filter-max-bytes'='4096'", + "'write.parquet.dict-size-bytes'='1'") ++ columns.map { column => + s"'write.parquet.bloom-filter-enabled.column.${column.name}'='true'" + }).mkString(", ") + Seq("bloom_types_native", "bloom_types_jvm").foreach { table => + val fields = columns.map(column => s"${column.name} ${column.sqlType}").mkString(", ") + spark.sql( + s"CREATE TABLE $catalog.$ns.$table (id INT, $fields) " + + s"USING iceberg TBLPROPERTIES ($properties)") + } + def insert(table: String): Unit = { + val values = columns.map(_.expression).mkString(", ") + spark.sql( + s"INSERT INTO $catalog.$ns.$table SELECT cast(id AS INT), $values " + + "FROM range(0, 10000, 1, 1)") + } + // No NDV/FPP properties: this fixture also runs on Iceberg 1.5.2. Ten thousand distinct + // values keep the 4 KiB filters dense enough to prevent Rust's post-insert folding. + // FLOAT/DOUBLE have separate fallback coverage because their Bloom writes use Java. + assertNativeWriteEngages("bloom_types_native", 0 until 10000)(insert("bloom_types_native")) + withSQLConf(CometConf.COMET_ICEBERG_NATIVE_WRITE_ENABLED.key -> "false") { + insert("bloom_types_jvm") + } + columns.foreach { column => + val native = parquetBloomFilterBytes("bloom_types_native", column.name) + val jvm = parquetBloomFilterBytes("bloom_types_jvm", column.name) + assert(native.size == 1 && jvm.size == 1, s"expected one filter for ${column.name}") + assert(native.head.length == 4096 && jvm.head.length == 4096) + assert( + java.util.Arrays.equals(native.head, jvm.head), + s"expected byte-identical Bloom filters for ${column.name}") + Seq("bloom_types_native", "bloom_types_jvm").foreach { table => + assertParquetBloomContainsValues( + table, + column.name, + column.physicalType, + (0 until 10000).map(column.encodedValue)) + } + } + } + } + test("native bloom sizing matches JVM for shape properties supported by the Iceberg runtime") { assumeNativeAcceleration() withIcebergCatalog { warehouseDir => @@ -5644,6 +5762,42 @@ class CometIcebergWriteActionSuite counts.head } + private def assertParquetBloomContainsValues( + tableName: String, + columnName: String, + physicalType: String, + values: Seq[Any]): Unit = { + val paths = spark + .sql(s"SELECT file_path FROM $catalog.$ns.$tableName.data_files") + .collect() + .map(_.getString(0)) + assert(paths.length == 1, s"expected one data file for $tableName") + val conf = spark.sparkContext.hadoopConfiguration + val input = HadoopInputFile.fromPath(new HadoopPath(paths.head.stripPrefix("file:")), conf) + val reader = ParquetFileReader.open(input) + try { + val chunks = reader.getFooter.getBlocks.asScala + .flatMap(_.getColumns.asScala) + .filter(_.getPath.toDotString == columnName) + assert(chunks.size == 1, s"expected one column chunk for $columnName in $tableName") + val column = chunks.head + assert(column.getPrimitiveType.getPrimitiveTypeName.name() == physicalType) + val bloom = reader.readBloomFilter(column) + assert(bloom != null, s"expected Bloom filter for $columnName in $tableName") + values.foreach { value => + val hash = value match { + case v: Int => bloom.hash(v) + case v: Long => bloom.hash(v) + case v: Binary => bloom.hash(v) + case other => fail(s"unsupported Bloom membership value: $other") + } + assert(bloom.findHash(hash), s"Bloom false negative for $columnName=$value in $tableName") + } + } finally { + reader.close() + } + } + private def assertParquetBloomContainsInts( tableName: String, columnName: String, From 7554402e182fec2f674ae5d56680440ef54b04d9 Mon Sep 17 00:00:00 2001 From: Nikita Matskevich Date: Sun, 25 Oct 2026 21:27:14 +0100 Subject: [PATCH 27/32] fix: bound Iceberg Bloom synthetic NDV search --- native/core/src/execution/operators/iceberg_write.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/native/core/src/execution/operators/iceberg_write.rs b/native/core/src/execution/operators/iceberg_write.rs index 86a0e8bb967..604cf68f4cd 100644 --- a/native/core/src/execution/operators/iceberg_write.rs +++ b/native/core/src/execution/operators/iceberg_write.rs @@ -2420,9 +2420,9 @@ fn synthetic_ndv_for_bloom_filter_bytes(target_bytes: usize, fpp: f64) -> DFResu } // Rust's standard binary-search helpers operate on materialized slices; this is a lower-bound - // search over the implicit NDV domain `1..=u64::MAX`, so keep the numeric search explicit. + // search over the implicit NDV domain `1..=u64::MAX / 8`, so keep the numeric search explicit. let mut low = 1_u64; - let mut high = u64::MAX; + let mut high = u64::MAX >> 3; while low < high { let mid = low + (high - low) / 2; if parquet_rs_bloom_filter_bytes(mid, fpp) < target_bytes { From d0167b4012345a3b07023bcafc9a34c6f6084093 Mon Sep 17 00:00:00 2001 From: Nikita Matskevich Date: Sun, 25 Oct 2026 21:48:59 +0100 Subject: [PATCH 28/32] bench: compare native Iceberg Bloom writes --- benchmarks/micro/iceberg_bloom_writes.py | 150 ++++++++++++++++++ docs/source/contributor-guide/benchmarking.md | 35 ++++ .../CometIcebergWriteBenchmark.scala | 112 ++++++++++++- 3 files changed, 290 insertions(+), 7 deletions(-) create mode 100755 benchmarks/micro/iceberg_bloom_writes.py diff --git a/benchmarks/micro/iceberg_bloom_writes.py b/benchmarks/micro/iceberg_bloom_writes.py new file mode 100755 index 00000000000..4c58bbf754d --- /dev/null +++ b/benchmarks/micro/iceberg_bloom_writes.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Run native Iceberg Bloom off/on in separate JVMs, reporting timing and process peak RSS.""" + +import argparse +import filecmp +import json +import os +import platform +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +import run as micro + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--spark-profile", default="spark-4.1") + parser.add_argument("--rows", type=int, default=4 * 1024 * 1024) + parser.add_argument("--iterations", type=int, default=5) + parser.add_argument("--heap", default=micro.DEFAULT_HEAP) + parser.add_argument("--output-dir", type=Path) + parser.add_argument( + "--allow-debug", + action="store_true", + help="smoke checks only, not performance results", + ) + args = parser.parse_args() + if sys.platform not in ("darwin", "linux"): + parser.error("peak-RSS collection supports Linux and macOS") + if not 8 <= args.rows <= 2147483647 or args.iterations < 1: + parser.error("rows must be in [8, 2147483647], and iterations must be positive") + repo = Path(__file__).resolve().parents[2] + profile = "-P" + args.spark_profile + env = micro.base_env(repo) + micro.require_supported_jdk(env) + if not args.allow_debug: + library = "libcomet.dylib" if sys.platform == "darwin" else "libcomet.so" + release = repo / "native/target/release" / library + packaged = (repo / "spark/target/classes/org/apache/comet").rglob(library) + if not release.is_file() or not any( + filecmp.cmp(release, path, shallow=False) for path in packaged + ): + parser.error( + "build/install release artifacts first with make release PROFILES=" + + profile + ) + bench = micro.benchmark_args(repo, env, profile, args.heap) + env["MAVEN_OPTS"] = bench["MAVEN_OPTS"] + # Each arm's result table stays in its own log instead of overwriting the suite result file. + env.pop("SPARK_GENERATE_BENCHMARK_FILES", None) + output = args.output_dir or Path(tempfile.mkdtemp(prefix="comet-iceberg-bloom-")) + output = output.resolve() + output.mkdir(parents=True, exist_ok=True) + results = [] + for arm in ("off", "on"): + log = output / ("bloom-" + arm + ".log") + command = micro.suite_command( + "CometIcebergWriteBenchmark", bench["MVN_ARGS"], profile + ) + command.append(f"-Dexec.args=bloom-{arm} {args.rows} {args.iterations}") + started = time.monotonic() + with log.open("w") as stream: + process = subprocess.Popen( + command, + cwd=repo / "spark", + env=env, + stdout=stream, + stderr=subprocess.STDOUT, + ) + try: + _, status, usage = os.wait4(process.pid, 0) + process.returncode = os.waitstatus_to_exitcode(status) + finally: + if process.returncode is None: + process.terminate() + process.wait() + if process.returncode: + raise SystemExit(f"Bloom {arm} failed; see {log}") + # wait4 returns bytes on macOS and KiB on Linux, independently for each child JVM. + peak_bytes = int(usage.ru_maxrss * (1 if sys.platform == "darwin" else 1024)) + label = "Native Bloom " + arm + timing = [ + line.strip() + for line in log.read_text().splitlines() + if line.strip().startswith(label) + ] + if not timing: + raise SystemExit(f"Bloom {arm} produced no benchmark timing row; see {log}") + fields = timing[-1][len(label) :].split() + try: + best_ms, average_ms, stdev_ms = map(float, fields[:3]) + except ValueError: + raise SystemExit(f"Cannot parse Spark benchmark timing columns; see {log}") + results.append( + { + "bloom": arm, + "write_best_ms": best_ms, + "write_average_ms": average_ms, + "write_stdev_ms": stdev_ms, + "peak_rss_bytes": peak_bytes, + "timing_row": timing[-1], + "process_seconds": round(time.monotonic() - started, 3), + "log": str(log), + } + ) + print( + f"Bloom {arm}: write average {average_ms:.1f} ms " + f"(best {best_ms:.1f}, stdev {stdev_ms:.1f})", + flush=True, + ) + print(f"Whole-process peak RSS: {peak_bytes / 1024**2:.1f} MiB", flush=True) + summary = { + "spark_profile": args.spark_profile, + "rows": args.rows, + "minimum_iterations": args.iterations, + "heap": args.heap, + "smoke_only": args.allow_debug, + "platform": platform.platform(), + "git": micro.git_info(repo), + "results": results, + "rss_scope": "Whole JVM process, including Maven, source generation, validation and warmup; not per-write allocation", + } + path = output / "summary.json" + path.write_text(json.dumps(summary, indent=2) + "\n") + print("Results: " + str(path), flush=True) + + +if __name__ == "__main__": + main() diff --git a/docs/source/contributor-guide/benchmarking.md b/docs/source/contributor-guide/benchmarking.md index c3318515e4f..b446e6cb3c7 100644 --- a/docs/source/contributor-guide/benchmarking.md +++ b/docs/source/contributor-guide/benchmarking.md @@ -43,6 +43,41 @@ These can also be run as a suite on a dedicated machine, see [Micro Benchmarking on AWS EC2](benchmarking_micro_ec2.md). Published results are in [benchmarks/results/micro](https://github.com/apache/datafusion-comet/tree/main/benchmarks/results/micro). +## Iceberg Bloom-filter writes + +`CometIcebergWriteBenchmark` compares Spark and Comet Iceberg writers. Its `bloom` mode compares +native writes with Bloom filters off/on, using the same prepared Parquet source in eight files. Both cases enable native writes and the Bloom folding opt-in; only the table's Bloom +properties differ. Filters cover the bigint, string, and binary columns at the default 1 MiB cap, +without explicit NDV/FPP. The benchmark checks the native plan, row count, and filter presence +before measuring. Table creation and source preparation are outside the timed insert; the Iceberg +commit is included. + +Build once, then run the comparison in separate JVMs to collect peak RSS: + +```shell +make release PROFILES=-Pspark-4.1 +python3 benchmarks/micro/iceberg_bloom_writes.py --spark-profile spark-4.1 --heap 8g +``` + +The runner reuses the micro-benchmark runner's Maven/JVM settings without rebuilding. Each case +warms up and measures at least five writes of 4,194,304 rows. It prints Spark's timing row and +whole-process peak RSS, and writes individual logs and `summary.json` to the reported directory. +Use `--output-dir` to choose that directory. RSS includes Maven, source generation, validation, +warmup, and writes; it is not the writer's allocation or its peak alone. The cases run in fresh +processes because RSS high-water marks and retained native buffers cannot be reset between cases. +The warehouse is local, so these results do not include object-store latency. + +For a quick functional check after a development build, pass +`--allow-debug --rows 32768 --iterations 1`. These runs are marked as smoke checks and do not +provide release-performance measurements. For timing alone, pass `bloom`, `bloom-off`, or +`bloom-on` to the existing benchmark target, optionally followed by rows and minimum iterations: + +```shell +make benchmark-org.apache.spark.sql.benchmark.CometIcebergWriteBenchmark bloom +``` + +Do not measure RSS around that `make` command: its release-build prerequisite includes compilation. + ```{toctree} :hidden: diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometIcebergWriteBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometIcebergWriteBenchmark.scala index 3ca224ec1a4..168c13536e0 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometIcebergWriteBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometIcebergWriteBenchmark.scala @@ -20,7 +20,11 @@ package org.apache.spark.sql.benchmark import scala.concurrent.duration._ +import scala.jdk.CollectionConverters._ +import org.apache.hadoop.fs.{Path => HadoopPath} +import org.apache.parquet.hadoop.ParquetFileReader +import org.apache.parquet.hadoop.util.HadoopInputFile import org.apache.spark.SparkConf import org.apache.spark.benchmark.Benchmark import org.apache.spark.sql.SparkSession @@ -61,8 +65,8 @@ import org.apache.comet.CometSparkSessionExtensions * pay equally. It dilutes the writer's factor rather than inflating it. * - The warehouse is a local temporary directory, so no object-store latency is included. * - The two writers choose different file roll points (see the accepted divergences in - * `docs/source/user-guide/latest/iceberg-writes.md`), so the resulting file layouts are not - * expected to match. Only wall clock and row counts are compared. + * `docs/source/user-guide/latest/iceberg.md`), so the resulting file layouts are not expected + * to match. Only wall clock and row counts are compared. * - `Rate` and `Per Row` are always computed against the corpus size, so for the `DELETE` case * they describe the rows the statement passed over, not the far smaller number it removed. * - The corpus below is this benchmark's own, not the one #5361 measured. The factors are the @@ -72,7 +76,10 @@ import org.apache.comet.CometSparkSessionExtensions * {{{ * SPARK_GENERATE_BENCHMARK_FILES=1 make benchmark-org.apache.spark.sql.benchmark.CometIcebergWriteBenchmark * }}} - * Results will be written to "spark/benchmarks/CometIcebergWriteBenchmark-**results.txt". + * Results will be written to "spark/benchmarks/CometIcebergWriteBenchmark-**results.txt". Pass + * `bloom`, `bloom-off`, or `bloom-on`, followed optionally by row count and iteration count, to + * compare native writes without and with Bloom filters. For isolated peak-RSS measurements, use + * `benchmarks/micro/iceberg_bloom_writes.py` after building Comet. */ object CometIcebergWriteBenchmark extends CometBenchmarkBase { @@ -285,6 +292,16 @@ object CometIcebergWriteBenchmark extends CometBenchmarkBase { } override def runCometBenchmark(mainArgs: Array[String]): Unit = { + val mode = mainArgs.headOption.getOrElse("all") + require( + Set("all", "bloom", "bloom-off", "bloom-on").contains(mode), + "Expected all, bloom, bloom-off, or bloom-on, optionally followed by rows and iterations") + require(mainArgs.length <= 3, "Expected at most mode, rows, and iterations") + val rows = mainArgs.lift(1).map(_.toInt).getOrElse(4 * 1024 * 1024) + val iterations = mainArgs.lift(2).map(_.toInt).getOrElse(5) + require( + rows >= sourceFiles && iterations > 0, + "Need at least eight rows and positive iterations") if (!icebergOnClasspath) { // scalastyle:off println println("Iceberg is not on the classpath; skipping. Build with an Iceberg-enabled profile.") @@ -295,14 +312,18 @@ object CometIcebergWriteBenchmark extends CometBenchmarkBase { withTempPath { warehouse => configureIcebergHadoopCatalog(warehouse, catalog) - runBenchmarkWithTable("Iceberg write", 4 * 1024 * 1024) { values => + runBenchmarkWithTable("Iceberg write", rows) { values => withTempPath { dir => withTempTable("parquetV1Table") { saveAsParquetV1Table( spark.sql(corpusQuery).repartition(sourceFiles).write, dir.getCanonicalPath + "/parquetV1") try { - workloads(values).foreach(runWorkload(_, values)) + if (mode == "all") { + workloads(values).foreach(runWorkload(_, values, iterations)) + } else { + runBloomWorkload(values, iterations, mode) + } } finally { spark.sql(s"DROP TABLE IF EXISTS $targetTable") } @@ -312,7 +333,84 @@ object CometIcebergWriteBenchmark extends CometBenchmarkBase { } } - private def runWorkload(workload: Workload, values: Int): Unit = { + private val bloomColumns = Set("c_long", "c_str", "c_bin") + + private def runBloomWorkload(values: Int, iterations: Int, mode: String): Unit = { + val benchmark = new Benchmark( + "Native Iceberg writes, default Bloom cap (1 MiB per column)", + values.toLong, + minNumIters = iterations, + warmupTime = 500.millis, + minTime = 500.millis, + output = output) + Seq(false, true) + .filter(enabled => mode == "bloom" || mode == s"bloom-${if (enabled) "on" else "off"}") + .foreach { enabled => + val arm = arms.last.copy( + name = s"Native Bloom ${if (enabled) "on" else "off"}", + // Keep native writes on in BOTH arms. Turning this option off selects Java for + // Bloom-enabled tables, which would measure a different writer rather than hashing. + confs = arms.last.confs :+ + (CometConf.COMET_ICEBERG_NATIVE_WRITE_BLOOM_FILTER_FOLDING_ENABLED.key -> "true")) + val workload = Workload( + title = "unpartitioned INSERT with native Bloom filters", + partitionSpec = "", + properties = if (enabled) bloomColumns.toSeq.sorted.map { column => + s"'write.parquet.bloom-filter-enabled.column.$column'='true'" + } + else Nil, + orderBy = None, + prePopulate = false, + statement = s"INSERT INTO $targetTable SELECT * FROM parquetV1Table", + expectedRowsAfter = values.toLong, + expectShuffle = None, + expectSort = None) + val label = verifyArm(benchmark, arm, workload) + require(label == arm.name, "Bloom benchmark requires native writes in both arms") + verifyBloomFilters(if (enabled) bloomColumns else Set.empty) + benchmark.addTimerCase(label) { timer => + resetTable(workload) + withSQLConf(arm.confs: _*) { + timer.startTiming() + spark.sql(workload.statement) + timer.stopTiming() + } + } + } + benchmark.run() + } + + private def verifyBloomFilters(expected: Set[String]): Unit = { + val files = spark.sql(s"SELECT file_path FROM $targetTable.data_files").collect() + require(files.nonEmpty, "Bloom benchmark must produce data files") + files.foreach { file => + val input = HadoopInputFile.fromPath( + new HadoopPath(file.getString(0)), + spark.sparkContext.hadoopConfiguration) + val reader = ParquetFileReader.open(input) + try { + val columns = reader.getFooter.getBlocks.asScala.flatMap(_.getColumns.asScala) + require( + expected.subsetOf(columns.map(_.getPath.toDotString).toSet), + "Benchmark output is missing configured Bloom columns") + columns.foreach { column => + val bloom = reader.readBloomFilter(column) + require( + (bloom != null) == expected.contains(column.getPath.toDotString), + s"Unexpected Bloom filter presence for ${column.getPath} in ${file.getString(0)}") + if (bloom != null) { + require( + bloom.getBitsetSize > 0 && bloom.getBitsetSize <= 1024 * 1024, + "Bloom filter must fit the default 1 MiB cap") + } + } + } finally { + reader.close() + } + } + } + + private def runWorkload(workload: Workload, values: Int, iterations: Int): Unit = { // `warmupTime` is a deadline and `minTime` a total across the measured iterations, so both are // floors on a case's total cost rather than on one write. The floors are set below the fastest // write the corpus produces - the native writer is around half a second here - so that every @@ -326,7 +424,7 @@ object CometIcebergWriteBenchmark extends CometBenchmarkBase { // its own runtime as spread between iterations, against a few percent for the two Comet arms // - and it is also the divisor of every `Relative` figure. Five iterations rather than the // usual three are cheap here and steady the number the table is read for. - minNumIters = 5, + minNumIters = iterations, warmupTime = 500.millis, minTime = 500.millis, output = output) From 38e896b816fd55c4727cb2be75c16f98b1cb76c8 Mon Sep 17 00:00:00 2001 From: Nikita Matskevich Date: Sun, 25 Oct 2026 22:40:26 +0100 Subject: [PATCH 29/32] bench: remove one-off Bloom measurement runner --- benchmarks/micro/iceberg_bloom_writes.py | 150 ------------------ docs/source/contributor-guide/benchmarking.md | 38 +++-- .../CometIcebergWriteBenchmark.scala | 4 +- 3 files changed, 24 insertions(+), 168 deletions(-) delete mode 100755 benchmarks/micro/iceberg_bloom_writes.py diff --git a/benchmarks/micro/iceberg_bloom_writes.py b/benchmarks/micro/iceberg_bloom_writes.py deleted file mode 100755 index 4c58bbf754d..00000000000 --- a/benchmarks/micro/iceberg_bloom_writes.py +++ /dev/null @@ -1,150 +0,0 @@ -#!/usr/bin/env python3 -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -"""Run native Iceberg Bloom off/on in separate JVMs, reporting timing and process peak RSS.""" - -import argparse -import filecmp -import json -import os -import platform -import subprocess -import sys -import tempfile -import time -from pathlib import Path - -import run as micro - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--spark-profile", default="spark-4.1") - parser.add_argument("--rows", type=int, default=4 * 1024 * 1024) - parser.add_argument("--iterations", type=int, default=5) - parser.add_argument("--heap", default=micro.DEFAULT_HEAP) - parser.add_argument("--output-dir", type=Path) - parser.add_argument( - "--allow-debug", - action="store_true", - help="smoke checks only, not performance results", - ) - args = parser.parse_args() - if sys.platform not in ("darwin", "linux"): - parser.error("peak-RSS collection supports Linux and macOS") - if not 8 <= args.rows <= 2147483647 or args.iterations < 1: - parser.error("rows must be in [8, 2147483647], and iterations must be positive") - repo = Path(__file__).resolve().parents[2] - profile = "-P" + args.spark_profile - env = micro.base_env(repo) - micro.require_supported_jdk(env) - if not args.allow_debug: - library = "libcomet.dylib" if sys.platform == "darwin" else "libcomet.so" - release = repo / "native/target/release" / library - packaged = (repo / "spark/target/classes/org/apache/comet").rglob(library) - if not release.is_file() or not any( - filecmp.cmp(release, path, shallow=False) for path in packaged - ): - parser.error( - "build/install release artifacts first with make release PROFILES=" - + profile - ) - bench = micro.benchmark_args(repo, env, profile, args.heap) - env["MAVEN_OPTS"] = bench["MAVEN_OPTS"] - # Each arm's result table stays in its own log instead of overwriting the suite result file. - env.pop("SPARK_GENERATE_BENCHMARK_FILES", None) - output = args.output_dir or Path(tempfile.mkdtemp(prefix="comet-iceberg-bloom-")) - output = output.resolve() - output.mkdir(parents=True, exist_ok=True) - results = [] - for arm in ("off", "on"): - log = output / ("bloom-" + arm + ".log") - command = micro.suite_command( - "CometIcebergWriteBenchmark", bench["MVN_ARGS"], profile - ) - command.append(f"-Dexec.args=bloom-{arm} {args.rows} {args.iterations}") - started = time.monotonic() - with log.open("w") as stream: - process = subprocess.Popen( - command, - cwd=repo / "spark", - env=env, - stdout=stream, - stderr=subprocess.STDOUT, - ) - try: - _, status, usage = os.wait4(process.pid, 0) - process.returncode = os.waitstatus_to_exitcode(status) - finally: - if process.returncode is None: - process.terminate() - process.wait() - if process.returncode: - raise SystemExit(f"Bloom {arm} failed; see {log}") - # wait4 returns bytes on macOS and KiB on Linux, independently for each child JVM. - peak_bytes = int(usage.ru_maxrss * (1 if sys.platform == "darwin" else 1024)) - label = "Native Bloom " + arm - timing = [ - line.strip() - for line in log.read_text().splitlines() - if line.strip().startswith(label) - ] - if not timing: - raise SystemExit(f"Bloom {arm} produced no benchmark timing row; see {log}") - fields = timing[-1][len(label) :].split() - try: - best_ms, average_ms, stdev_ms = map(float, fields[:3]) - except ValueError: - raise SystemExit(f"Cannot parse Spark benchmark timing columns; see {log}") - results.append( - { - "bloom": arm, - "write_best_ms": best_ms, - "write_average_ms": average_ms, - "write_stdev_ms": stdev_ms, - "peak_rss_bytes": peak_bytes, - "timing_row": timing[-1], - "process_seconds": round(time.monotonic() - started, 3), - "log": str(log), - } - ) - print( - f"Bloom {arm}: write average {average_ms:.1f} ms " - f"(best {best_ms:.1f}, stdev {stdev_ms:.1f})", - flush=True, - ) - print(f"Whole-process peak RSS: {peak_bytes / 1024**2:.1f} MiB", flush=True) - summary = { - "spark_profile": args.spark_profile, - "rows": args.rows, - "minimum_iterations": args.iterations, - "heap": args.heap, - "smoke_only": args.allow_debug, - "platform": platform.platform(), - "git": micro.git_info(repo), - "results": results, - "rss_scope": "Whole JVM process, including Maven, source generation, validation and warmup; not per-write allocation", - } - path = output / "summary.json" - path.write_text(json.dumps(summary, indent=2) + "\n") - print("Results: " + str(path), flush=True) - - -if __name__ == "__main__": - main() diff --git a/docs/source/contributor-guide/benchmarking.md b/docs/source/contributor-guide/benchmarking.md index b446e6cb3c7..7195251e72f 100644 --- a/docs/source/contributor-guide/benchmarking.md +++ b/docs/source/contributor-guide/benchmarking.md @@ -52,31 +52,37 @@ without explicit NDV/FPP. The benchmark checks the native plan, row count, and f before measuring. Table creation and source preparation are outside the timed insert; the Iceberg commit is included. -Build once, then run the comparison in separate JVMs to collect peak RSS: +Use the existing benchmark target for timing, optionally followed by rows and minimum iterations: ```shell -make release PROFILES=-Pspark-4.1 -python3 benchmarks/micro/iceberg_bloom_writes.py --spark-profile spark-4.1 --heap 8g +make benchmark-org.apache.spark.sql.benchmark.CometIcebergWriteBenchmark bloom ``` -The runner reuses the micro-benchmark runner's Maven/JVM settings without rebuilding. Each case -warms up and measures at least five writes of 4,194,304 rows. It prints Spark's timing row and -whole-process peak RSS, and writes individual logs and `summary.json` to the reported directory. -Use `--output-dir` to choose that directory. RSS includes Maven, source generation, validation, -warmup, and writes; it is not the writer's allocation or its peak alone. The cases run in fresh -processes because RSS high-water marks and retained native buffers cannot be reset between cases. -The warehouse is local, so these results do not include object-store latency. +Each case warms up and measures at least five writes of 4,194,304 rows by default. Use +`bloom-off` or `bloom-on` to select a single case, or append `32768 1` for a quick functional +check. Performance comparisons require a release build. -For a quick functional check after a development build, pass -`--allow-debug --rows 32768 --iterations 1`. These runs are marked as smoke checks and do not -provide release-performance measurements. For timing alone, pass `bloom`, `bloom-off`, or -`bloom-on` to the existing benchmark target, optionally followed by rows and minimum iterations: +To collect peak RSS, build once and run each case in a separate JVM using the existing Maven +invocation. Obtain the shared JVM options from the Makefile: ```shell -make benchmark-org.apache.spark.sql.benchmark.CometIcebergWriteBenchmark bloom +make release PROFILES=-Pspark-4.1 +export COMET_CONF_DIR="$PWD/conf" +export MAVEN_OPTS="$(make --no-print-directory print-benchmark-args PROFILES=-Pspark-4.1 BENCH_HEAP=8g | sed -n 's/^MAVEN_OPTS=//p')" +cd spark +/usr/bin/time -v ../mvnw exec:java -Dexec.classpathScope=test \ + -Dexec.cleanupDaemonThreads=false \ + -Dexec.mainClass=org.apache.spark.sql.benchmark.CometIcebergWriteBenchmark \ + -Dexec.args="bloom-off" -Pspark-4.1 ``` -Do not measure RSS around that `make` command: its release-build prerequisite includes compilation. +Repeat with `bloom-on`. On macOS, use `/usr/bin/time -l` instead of `-v`; its maximum RSS is in +bytes, while GNU time reports KiB. The reported RSS includes Maven, source generation, validation, +warmup, and writes; it is not the writer's allocation or its peak alone. Fresh processes avoid +carrying an earlier case's RSS high-water mark and retained native buffers into the next case. +The warehouse is local, so these results do not include object-store latency. + +Do not measure RSS around `make benchmark-*`: its release-build prerequisite includes compilation. ```{toctree} :hidden: diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometIcebergWriteBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometIcebergWriteBenchmark.scala index 168c13536e0..750cb81b018 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometIcebergWriteBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometIcebergWriteBenchmark.scala @@ -78,8 +78,8 @@ import org.apache.comet.CometSparkSessionExtensions * }}} * Results will be written to "spark/benchmarks/CometIcebergWriteBenchmark-**results.txt". Pass * `bloom`, `bloom-off`, or `bloom-on`, followed optionally by row count and iteration count, to - * compare native writes without and with Bloom filters. For isolated peak-RSS measurements, use - * `benchmarks/micro/iceberg_bloom_writes.py` after building Comet. + * compare native writes without and with Bloom filters. The contributor benchmarking guide + * describes collecting peak RSS in separate JVMs with the operating system's time utility. */ object CometIcebergWriteBenchmark extends CometBenchmarkBase { From 4b0878d733b10ebd6c64b673cdda40654f78d046 Mon Sep 17 00:00:00 2001 From: Nikita Matskevich Date: Sun, 25 Oct 2026 23:36:54 +0100 Subject: [PATCH 30/32] fix: limit native Bloom writes to modern Iceberg runtimes --- docs/source/contributor-guide/benchmarking.md | 2 +- .../contributor-guide/iceberg-writes.md | 5 ++ docs/source/user-guide/latest/iceberg.md | 16 ++++- .../scala/org/apache/comet/CometConf.scala | 17 +++-- .../comet/iceberg/IcebergReflection.scala | 14 ----- .../operator/CometIcebergNativeWrite.scala | 62 ++++++++++--------- .../comet/CometIcebergWriteActionSuite.scala | 42 ++++++++++++- .../CometIcebergWriteDetectionSuite.scala | 48 +++++++++++++- .../IcebergWriteProtoTranslationSuite.scala | 20 +++--- 9 files changed, 158 insertions(+), 68 deletions(-) diff --git a/docs/source/contributor-guide/benchmarking.md b/docs/source/contributor-guide/benchmarking.md index 7195251e72f..420f78abd9f 100644 --- a/docs/source/contributor-guide/benchmarking.md +++ b/docs/source/contributor-guide/benchmarking.md @@ -46,7 +46,7 @@ These can also be run as a suite on a dedicated machine, see ## Iceberg Bloom-filter writes `CometIcebergWriteBenchmark` compares Spark and Comet Iceberg writers. Its `bloom` mode compares -native writes with Bloom filters off/on, using the same prepared Parquet source in eight files. Both cases enable native writes and the Bloom folding opt-in; only the table's Bloom +native writes with Bloom filters off/on (requiring Iceberg 1.10+), using the same prepared Parquet source in eight files. Both cases enable native writes and the Bloom folding opt-in; only the table's Bloom properties differ. Filters cover the bigint, string, and binary columns at the default 1 MiB cap, without explicit NDV/FPP. The benchmark checks the native plan, row count, and filter presence before measuring. Table creation and source preparation are outside the timed insert; the Iceberg diff --git a/docs/source/contributor-guide/iceberg-writes.md b/docs/source/contributor-guide/iceberg-writes.md index 373f3c91df5..a299f2b2cba 100644 --- a/docs/source/contributor-guide/iceberg-writes.md +++ b/docs/source/contributor-guide/iceberg-writes.md @@ -458,6 +458,11 @@ The Comet suites run against the Iceberg version each Spark profile pins in `spa for Spark 3.4, 1.8.1 for 3.5, 1.10.0 for 4.0, and 1.11.0 for 4.1. No Iceberg runtime is published for Spark 4.2, so the 4.2 profile builds against the 4.0 runtime and the Iceberg suites skip there (`icebergAvailable` is false on Spark 4.2). Only the default profile runs on every pull request. + +Bloom-enabled native writes additionally require an identifiable Iceberg version >= 1.10. +On the 3.4 and 3.5 profiles, any per-column Bloom property (including `false` enablement and standalone FPP/NDV) selects Java, +even when the Bloom folding opt-in is enabled. Other native-write eligibility is unchanged. + Behaviour that differs between Iceberg versions, such as reflection targets, metrics conventions and partition path spelling, needs a test that runs on each. diff --git a/docs/source/user-guide/latest/iceberg.md b/docs/source/user-guide/latest/iceberg.md index 4db4cece104..bca71b660db 100644 --- a/docs/source/user-guide/latest/iceberg.md +++ b/docs/source/user-guide/latest/iceberg.md @@ -487,7 +487,7 @@ A write is eligible only when ALL of the following hold: | `write.parquet.variant-inference-buffer-size` | any value (only meaningful when shredding, which is gated) | | `write.metadata.metrics.*` | any value (manifest metrics are re-derived on the JVM with Iceberg's own logic) | | `write.spark.fanout.enabled` | any value (the native writer implements both clustered and fanout modes) | -| `write.parquet.bloom-filter-enabled.column.` | `true` or `false`; on Iceberg 1.11+, an explicit NDV enables the column even when this value is `false`. Enabled columns use the native writer when `spark.comet.write.iceberg.bloomFilterFolding.enabled=true` | +| `write.parquet.bloom-filter-enabled.column.` | `true` or `false`; native eligibility requires identifiable Iceberg 1.10+. Older or unidentified runtimes use Java for any per-column Bloom property, including `false` enablement and standalone FPP/NDV. On Iceberg 1.11+, an explicit NDV enables the column even when this value is `false`. Enabled columns use the native writer when `spark.comet.write.iceberg.bloomFilterFolding.enabled=true` | | `write.parquet.bloom-filter-fpp.column.` / `write.parquet.bloom-filter-ndv.column.` | Iceberg 1.10 interprets FPP but ignores NDV; 1.11 interprets both. For interpreted properties on a column named by an `enabled` property, FPP is finite and strictly between 0 and 1 (default `0.01`), and NDV is a positive Java long no greater than `Long.MAX_VALUE / 8`. | | `write.parquet.bloom-filter-max-bytes` | `write.parquet.bloom-filter-enabled.column.*` present (even `false`): unset (1 MiB) or a power of two from 32 bytes through 128 MiB; larger explicit NDV/FPP requests fall back at 32 bytes because Java ignores that cap. With no such key: unset or any positive Java int, including `100`. Malformed, zero, or negative caps use Java and fail its validation. | | `write.target-file-size-bytes` | any value (the two writers can choose different roll points; see accepted divergences) | @@ -675,7 +675,9 @@ native writes for configured Bloom columns require all of the following: - `spark.comet.write.iceberg.enabled=true`. - `spark.comet.write.iceberg.bloomFilterFolding.enabled=true`. -- The Iceberg runtime reports an identifiable version so Comet can match its column-name rules. +- The Iceberg runtime reports an identifiable version of 1.10 or later. Earlier or + unidentified runtimes use Java whenever a per-column Bloom property is configured, + including `false` enablement and standalone FPP/NDV properties. The opt-in cannot enable native Bloom writes there. - The effective `write.parquet.bloom-filter-max-bytes` (default 1 MiB) is a power of two from 32 bytes through 128 MiB. - An interpreted NDV is absent or a positive Java long no greater than `Long.MAX_VALUE / 8`. @@ -687,6 +689,16 @@ native writes for configured Bloom columns require all of the following: - Bloom-filter columns are not FLOAT or DOUBLE, including nested leaves. Floating-point columns without Bloom filters remain eligible for native writes. +The default Spark profiles have the following Bloom-write eligibility. The loaded Iceberg +version determines eligibility; Spark's version alone does not enable this feature. + +| Spark profile | Pinned Iceberg | Bloom opt-in behavior | +| ------------- | -------------- | --------------------------------------------- | +| 3.4 | 1.5.2 | Java writer, even with the opt-in enabled | +| 3.5 | 1.8.1 | Java writer, even with the opt-in enabled | +| 4.0 | 1.10.0 | Native eligible; FPP interpreted, NDV ignored | +| 4.1 | 1.11.0 | Native eligible; FPP and NDV interpreted | + The cap range applies whenever a `write.parquet.bloom-filter-enabled.column.*` property is present, even if set to `false`. With no such properties, the cap is unused: unset or any positive Java int, such as `100`, keeps an otherwise eligible write native. Malformed, zero, diff --git a/spark/src/main/scala/org/apache/comet/CometConf.scala b/spark/src/main/scala/org/apache/comet/CometConf.scala index 509ede7db7a..fa2278e16e2 100644 --- a/spark/src/main/scala/org/apache/comet/CometConf.scala +++ b/spark/src/main/scala/org/apache/comet/CometConf.scala @@ -147,15 +147,14 @@ object CometConf extends ShimCometConf { val COMET_ICEBERG_NATIVE_WRITE_BLOOM_FILTER_FOLDING_ENABLED: ConfigEntry[Boolean] = conf("spark.comet.write.iceberg.bloomFilterFolding.enabled") .category(CATEGORY_EXEC) - .doc( - "Whether Bloom-filter-enabled Iceberg writes may use Comet's native writer. The " + - "Apache Arrow Rust parquet writer folds sparse Bloom filters after insertion, which " + - "can reduce serialized size while raising realized FPP toward the configured target. " + - "When false, these writes use Iceberg's Java writer. This option selects the writer; " + - "it cannot disable folding inside parquet-rs. The opt-in also accepts boolean filter " + - "differences: Java writes all-zero filters while Rust populates them. " + - "Requires spark.comet.write.iceberg.enabled. " + - "FLOAT/DOUBLE Bloom columns use Java to preserve NaN equality. Off by default.") + .doc("Whether Bloom-filter-enabled Iceberg writes may use Comet's native writer. The " + + "Apache Arrow Rust parquet writer folds sparse Bloom filters after insertion, which " + + "can reduce serialized size while raising realized FPP toward the configured target. " + + "When false, these writes use Iceberg's Java writer. This option selects the writer; " + + "it cannot disable folding inside parquet-rs. The opt-in also accepts boolean filter " + + "differences: Java writes all-zero filters while Rust populates them. " + + "Requires spark.comet.write.iceberg.enabled and an identifiable Iceberg version >= 1.10. " + + "FLOAT/DOUBLE Bloom columns use Java to preserve NaN equality. Off by default.") .booleanConf .createWithDefault(false) diff --git a/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala b/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala index 58011bad706..1c4f8df77a8 100644 --- a/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala +++ b/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala @@ -883,20 +883,6 @@ object IcebergReflection extends Logging { .toMap } - /** - * Returns the field id resolved by Iceberg's name lookup, including runtime-supported aliases. - */ - def findFieldIdByName(schema: Any, name: String): Option[Int] = { - try { - val findFieldMethod = getMethod(schema.getClass, "findField", classOf[String]) - Option(findFieldMethod.invoke(schema, name)).map { field => - getMethod(field.getClass, "fieldId").invoke(field).asInstanceOf[Int] - } - } catch { - case _: Exception => None - } - } - /** * The fields of `schema` at any depth, as id, name, and the type of a field with no children. */ diff --git a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala index 97f2efdbaf0..21e5828e435 100644 --- a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala +++ b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala @@ -436,6 +436,7 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { requireParquetPageVersionDefault, requireShredVariantsDisabled, requireNativeSupportedCompressionLevel, + requireSupportedBloomFilterRuntime, requireNativeSupportedBloomFilterProperties, requireBloomFilterFoldingOptIn, requireNoFloatingPointBloomFilters, @@ -725,43 +726,48 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { } } - private[operator] def bloomFilterParquetPaths( - configuredColumns: Seq[String], - schema: Any, - resolution: IcebergReflection.ParquetPathResolution, - icebergVersion: String = IcebergReflection.icebergVersion()): Map[String, String] = { + private val requireSupportedBloomFilterRuntime: TriggerRule = ctx => { + val prefixes = Seq( + PropertyKeys.BloomFilterColumnEnabledPrefix, + PropertyKeys.ParquetBloomFilterColumnFppPrefix, + PropertyKeys.ParquetBloomFilterColumnNdvPrefix) + if (ctx.properties.keys.exists(key => prefixes.exists(key.startsWith))) { + bloomFilterRuntimeRejection(IcebergReflection.icebergVersion()) + } else { + None + } + } + + private[operator] def bloomFilterRuntimeRejection(icebergVersion: String): Option[String] = { val version = if (Option(icebergVersion).exists(_.matches("\\d+\\.\\d+\\.\\d+(?:[-+].+)?"))) { - val versionParts = icebergVersion.split("[.-]", 3) - scala.util.Try((versionParts(0).toInt, versionParts(1).toInt)).toOption + val parts = icebergVersion.split("[.-]", 3) + scala.util.Try((parts(0).toInt, parts(1).toInt)).toOption } else { None } version match { - case Some((1, minor)) if minor < 8 => - // Iceberg 1.5 passes the configured name straight to parquet-mr. - configuredColumns.filter(resolution.physicalPaths).map(path => path -> path).toMap - case Some((1, minor)) if minor < 10 => - // Iceberg 1.8 resolves canonical and short collection aliases through field ids. - configuredColumns.flatMap { configuredName => - IcebergReflection - .findFieldIdByName(schema, configuredName) - .flatMap(resolution.pathByFieldId.get) - .map(configuredName -> _) - }.toMap - case Some((1, minor)) if minor >= 10 => - // Iceberg 1.10+ keys writer properties by canonical findColumnName output. - resolution.pathByIcebergColumnName.filter { case (column, _) => - configuredColumns.contains(column) - } + case Some((1, minor)) if minor >= 10 => None + case Some((1, _)) => + Some(s"Native Bloom writes require Iceberg 1.10 or later; found '$icebergVersion'") case _ => - // getSupportLevel catches this during planning and keeps the write on Java. Guessing - // canonical naming for repackaged runtimes could silently omit requested filters. - throw new IllegalArgumentException( + Some( s"Cannot determine Iceberg Bloom column-name convention for version '$icebergVersion'") } } + private[operator] def bloomFilterParquetPaths( + configuredColumns: Seq[String], + resolution: IcebergReflection.ParquetPathResolution, + icebergVersion: String = IcebergReflection.icebergVersion()): Map[String, String] = { + bloomFilterRuntimeRejection(icebergVersion).foreach { reason => + throw new IllegalArgumentException(reason) + } + resolution.pathByIcebergColumnName.filter { case (column, _) => + configuredColumns.contains(column) + } + } + // A floating-point Bloom filter can reject a SQL-equal NaN with a different bit payload. // Until native writes canonicalize these values, preserve Java's writer for affected leaves. private val requireNoFloatingPointBloomFilters: TriggerRule = ctx => { @@ -779,7 +785,7 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { .getOrElse( throw new IllegalStateException( "could not resolve Parquet paths for Bloom type checking")) - val paths = bloomFilterParquetPaths(columns, schema, resolution) + val paths = bloomFilterParquetPaths(columns, resolution) val affected = columns.filter { column => paths.get(column).exists(resolution.floatingPointPhysicalPaths.contains) } @@ -1396,7 +1402,7 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { } val configuredColumns = IcebergWriteProtoTranslation.enabledBloomFilterColumnNames(effectiveProperties) - val resolvedPaths = bloomFilterParquetPaths(configuredColumns, writeSchema, resolution) + val resolvedPaths = bloomFilterParquetPaths(configuredColumns, resolution) val renamedColumns = resolvedPaths.collect { case (configuredName, path) if resolution.renamedPhysicalPaths.contains(path) => configuredName diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala index 406e45eb8d0..922149b4034 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala @@ -1243,7 +1243,25 @@ class CometIcebergWriteActionSuite } } + test("legacy Bloom settings use Java despite the native folding opt-in") { + assumeNativeAcceleration() + assume(!icebergVersionAtLeast(1, 10), "legacy runtime only") + withIcebergCatalog { _ => + Seq("true", "false").zipWithIndex.foreach { case (enabled, index) => + val table = s"legacy_bloom_$index" + createTable( + table, + partitionSpec = "", + properties = Some(s"'write.parquet.bloom-filter-enabled.column.id'='$enabled'")) + assertNativeWriteDoesNotEngage(table, Seq(1)) { + spark.sql(s"INSERT INTO $catalog.$ns.$table VALUES (1, 'region', 1.0)") + } + } + } + } + test("native acceleration: writes configured Iceberg Parquet bloom filters") { + assumeNativeBloomRuntime() assumeNativeAcceleration() withIcebergCatalog { warehouseDir => createTable( @@ -1265,6 +1283,7 @@ class CometIcebergWriteActionSuite } test("Bloom-filter folding config selects the native or JVM Iceberg writer") { + assumeNativeBloomRuntime() assumeNativeAcceleration() withIcebergCatalog { warehouseDir => val properties = Some("'write.parquet.bloom-filter-enabled.column.id'='true'") @@ -1333,6 +1352,7 @@ class CometIcebergWriteActionSuite } test("floating-point columns without Bloom filters keep native writes") { + assumeNativeBloomRuntime() assumeNativeAcceleration() withIcebergCatalog { _ => createTable( @@ -1349,6 +1369,7 @@ class CometIcebergWriteActionSuite } test("enabled=false plus NDV matches JVM behavior for the Iceberg runtime") { + assumeNativeBloomRuntime() assumeNativeAcceleration() withIcebergCatalog { warehouseDir => val configuredFpp = 0.01 @@ -1390,6 +1411,7 @@ class CometIcebergWriteActionSuite } test("enabled=false preserves JVM validation errors for malformed FPP and NDV") { + assumeNativeBloomRuntime() assumeNativeAcceleration() withIcebergCatalog { warehouseDir => Seq( @@ -1435,6 +1457,7 @@ class CometIcebergWriteActionSuite } test("max-bytes=32 falls back only when parquet-mr ignores the cap") { + assumeNativeBloomRuntime() assumeNativeAcceleration() assumeIcebergBloomShapeProperties() withIcebergCatalog { warehouseDir => @@ -1474,6 +1497,7 @@ class CometIcebergWriteActionSuite } test("native bloom filters resolve list and map leaves to physical Parquet paths") { + assumeNativeBloomRuntime() assumeNativeAcceleration() withIcebergCatalog { _ => Seq("native_nested_bloom", "native_nested_bloom_jvm").foreach { table => @@ -1517,6 +1541,7 @@ class CometIcebergWriteActionSuite } test("short nested bloom aliases match the Iceberg runtime footer") { + assumeNativeBloomRuntime() assumeNativeAcceleration() withIcebergCatalog { _ => Seq("nested_alias_native", "nested_alias_jvm").foreach { table => @@ -1559,7 +1584,7 @@ class CometIcebergWriteActionSuite } } - test("Iceberg 1.5 physical nested bloom paths execute natively and match the JVM footer") { + test("Iceberg 1.5 physical nested bloom paths use the JVM writer") { assumeNativeAcceleration() assume( !icebergVersionAtLeast(1, 8), @@ -1585,7 +1610,7 @@ class CometIcebergWriteActionSuite (2, array(named_struct('a', 30)), map('second', named_struct('b', 40))) """) - assertNativeWriteEngages("physical_path_native", Seq(1, 2)) { + assertNativeWriteDoesNotEngage("physical_path_native", Seq(1, 2)) { insert("physical_path_native") } withSQLConf(CometConf.COMET_ICEBERG_NATIVE_WRITE_ENABLED.key -> "false") { @@ -1595,7 +1620,7 @@ class CometIcebergWriteActionSuite Seq("tags.list.element.a", "attrs.key_value.value.b").foreach { physicalColumn => assert( parquetBloomFilterPresent("physical_path_native", physicalColumn), - s"expected native Bloom filter for $physicalColumn") + s"expected preserved Bloom filter for $physicalColumn") assert( parquetBloomFilterPresent("physical_path_jvm", physicalColumn), s"expected JVM Bloom filter for $physicalColumn") @@ -1604,6 +1629,7 @@ class CometIcebergWriteActionSuite } test("unconfigured quoted columns preserve native Bloom writes") { + assumeNativeBloomRuntime() assumeNativeAcceleration() withIcebergCatalog { _ => spark.sql(s""" @@ -1665,6 +1691,7 @@ class CometIcebergWriteActionSuite } test("native bloom filter is byte-identical to parquet-mr when max-bytes binds") { + assumeNativeBloomRuntime() assumeNativeAcceleration() assumeIcebergBloomShapeProperties() withIcebergCatalog { warehouseDir => @@ -1708,6 +1735,7 @@ class CometIcebergWriteActionSuite } test("native bloom filters match Java bytes and membership across physical types") { + assumeNativeBloomRuntime() assumeNativeAcceleration() withIcebergCatalog { _ => case class BloomColumn( @@ -1824,6 +1852,7 @@ class CometIcebergWriteActionSuite } test("native bloom sizing matches JVM for shape properties supported by the Iceberg runtime") { + assumeNativeBloomRuntime() assumeNativeAcceleration() withIcebergCatalog { warehouseDir => val enabled = "'write.parquet.bloom-filter-enabled.column.id'='true'" @@ -1897,6 +1926,7 @@ class CometIcebergWriteActionSuite } test("explicit underestimated NDV retains parquet-mr allocation precedence") { + assumeNativeBloomRuntime() assumeNativeAcceleration() assumeIcebergBloomShapeProperties() withIcebergCatalog { warehouseDir => @@ -1932,6 +1962,7 @@ class CometIcebergWriteActionSuite } test("overestimated explicit NDV folds and exposes the realized-FPP tradeoff") { + assumeNativeBloomRuntime() assumeNativeAcceleration() assumeIcebergBloomShapeProperties() withIcebergCatalog { warehouseDir => @@ -1985,6 +2016,7 @@ class CometIcebergWriteActionSuite } test("large representable max folds natively while adjacent non-power-of-two falls back") { + assumeNativeBloomRuntime() assumeNativeAcceleration() withIcebergCatalog { warehouseDir => val powerOfTwo = 64 * 1024 * 1024 @@ -2043,6 +2075,7 @@ class CometIcebergWriteActionSuite } test("out-of-range bloom max uses the classic writer when a bloom column is configured") { + assumeNativeBloomRuntime() assumeNativeAcceleration() assumeIcebergBloomShapeProperties() withIcebergCatalog { warehouseDir => @@ -5835,6 +5868,9 @@ class CometIcebergWriteActionSuite assume(icebergAvailable, "Iceberg not available in classpath") } + private def assumeNativeBloomRuntime(): Unit = + assume(icebergVersionAtLeast(1, 10), "Native Bloom writes require Iceberg 1.10 or later") + private def assumeIcebergBloomShapeProperties(): Unit = { assume( icebergSupportsBloomFpp && icebergSupportsBloomNdv, diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala index eefe5ea5fde..681b6da0de9 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala @@ -378,6 +378,39 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes } } + test("legacy runtime rejects every configured Bloom entry before filtering disabled names") { + assume(!icebergVersionAtLeast(1, 10), "legacy runtime only") + withDetectionCatalog { _ => + Seq( + ("id", "true"), + ("id", "false"), + ("", "true"), + ("", "false"), + ("missing", "true")).zipWithIndex.foreach { case ((column, enabled), index) => + val table = s"legacy_bloom_setting_$index" + createTable( + table, + partitionSpec = "", + properties = Some(s"'write.parquet.bloom-filter-enabled.column.$column'='$enabled'")) + assertUnsupportedContainsAllowingWriteFailure(table, "Iceberg 1.10 or later") + } + } + } + + test("legacy runtime also falls back for standalone Bloom shape properties") { + assume(!icebergVersionAtLeast(1, 10), "legacy runtime only") + withDetectionCatalog { _ => + Seq("fpp", "ndv").zipWithIndex.foreach { case (shape, index) => + val table = s"legacy_bloom_shape_$index" + createTable( + table, + partitionSpec = "", + properties = Some(s"'write.parquet.bloom-filter-$shape.column.id'='garbage'")) + assertUnsupportedContainsAllowingWriteFailure(table, "Iceberg 1.10 or later") + } + } + } + test("bloom-filter max-bytes accepts only representable power-of-two values") { withDetectionCatalog { dir => Seq("32", "524288", "1048576", "134217728").zipWithIndex.foreach { case (value, index) => @@ -400,7 +433,8 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes s"'write.parquet.bloom-filter-max-bytes'='$value'")) assertUnsupportedContainsAllowingWriteFailure( table, - "write.parquet.bloom-filter-max-bytes") + if (icebergVersionAtLeast(1, 10)) "write.parquet.bloom-filter-max-bytes" + else "Iceberg 1.10 or later") } } } @@ -432,6 +466,7 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes } test("bloom-filter max-bytes=32 falls back only when parquet-mr ignores the cap") { + assumeNativeBloomRuntime() withDetectionCatalog { dir => val minimumBytes = 32 val naturallyMinimumNdv = 1 @@ -471,6 +506,7 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes } test("fall-back: bloom-filter NDV values that overflow parquet-mr sizing arithmetic") { + assumeNativeBloomRuntime() withDetectionCatalog { dir => val largestNonOverflowingNdv = Long.MaxValue / 8L val cases = Seq( @@ -499,6 +535,7 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes } test("bloom-filter enabled=false with NDV follows runtime support and validates NDV") { + assumeNativeBloomRuntime() withDetectionCatalog { dir => val configuredNdv = 1000 createTable( @@ -526,6 +563,7 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes } test("bloom-filter enabled=false still validates FPP") { + assumeNativeBloomRuntime() withDetectionCatalog { dir => createTable( "false_with_bad_fpp", @@ -544,6 +582,7 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes } test("fall-back: invalid enabled-column FPP and NDV") { + assumeNativeBloomRuntime() withDetectionCatalog { dir => Seq( ("'write.parquet.bloom-filter-fpp.column.id'='0'", icebergSupportsBloomFpp), @@ -573,6 +612,7 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes } test("Compatible when a per-column bloom filter is enabled") { + assumeNativeBloomRuntime() withDetectionCatalog { _ => createTable( "bloom_col", @@ -583,6 +623,7 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes } test("floating-point Bloom detection reports top-level and nested affected leaves") { + assumeNativeBloomRuntime() withDetectionCatalog { _ => val columns = Seq( "amount" -> "amount", @@ -603,6 +644,7 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes } test("floating-point Bloom detection preserves native eligibility for unaffected columns") { + assumeNativeBloomRuntime() withDetectionCatalog { _ => val stringKey = if (icebergSupportsBloomFpp) "attrs.key" else "attrs.key_value.key" val integerValue = if (icebergSupportsBloomFpp) "lookup.value" else "lookup.key_value.value" @@ -630,6 +672,7 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes } test("Bloom-filter folding must be explicitly enabled for native writes") { + assumeNativeBloomRuntime() withDetectionCatalog { dir => createTable( "bloom_folding_opt_in", @@ -2036,6 +2079,9 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes } } + private def assumeNativeBloomRuntime(): Unit = + assume(icebergVersionAtLeast(1, 10), "Native Bloom writes require Iceberg 1.10 or later") + private def icebergSupportsBloomFpp: Boolean = IcebergReflection .tablePropertyConstantOpt("PARQUET_BLOOM_FILTER_COLUMN_FPP_PREFIX") diff --git a/spark/src/test/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslationSuite.scala b/spark/src/test/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslationSuite.scala index b824114746c..cecf245e8c4 100644 --- a/spark/src/test/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslationSuite.scala +++ b/spark/src/test/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslationSuite.scala @@ -45,7 +45,7 @@ class IcebergWriteProtoTranslationSuite extends AnyFunSuite { Set.empty) Seq("unknown", "", "vendor-build", "1.8.bad", "1.10", "2.0.0").foreach { version => val error = intercept[IllegalArgumentException] { - CometIcebergNativeWrite.bloomFilterParquetPaths(Seq("tags.a"), null, resolution, version) + CometIcebergNativeWrite.bloomFilterParquetPaths(Seq("tags.a"), resolution, version) } assert(error.getMessage.contains("Cannot determine Iceberg Bloom column-name convention")) assert(error.getMessage.contains(s"'$version'")) @@ -60,24 +60,24 @@ class IcebergWriteProtoTranslationSuite extends AnyFunSuite { Set(physicalPath, "order_x20id"), Set("order_x20id"), Set.empty) - Seq("1.5.2", "1.5.2-SNAPSHOT").foreach { version => - assert( - CometIcebergNativeWrite.bloomFilterParquetPaths( - Seq(physicalPath, "tags.element.a"), - null, - resolution, - version) == Map(physicalPath -> physicalPath)) - } Seq("1.10.0", "1.11.0", "1.11.0-SNAPSHOT", "1.11.0+vendor").foreach { version => assert( CometIcebergNativeWrite.bloomFilterParquetPaths( Seq("tags.element.a", "tags.a"), - null, resolution, version) == Map("tags.element.a" -> physicalPath)) } } + test("Bloom writes reject older runtimes even when the version is identifiable") { + Seq("1.5.2", "1.8.1", "1.9.1", "1.8.1-SNAPSHOT").foreach { version => + assert( + CometIcebergNativeWrite + .bloomFilterRuntimeRejection(version) + .exists(_.contains("Iceberg 1.10 or later"))) + } + } + // -- compression ----------------------------------------------------------- test("compression defaults to zstd at parquet-mr's level 3 when no property is set") { From 9c468b65708fba344ddaa1530630cd556d4c787c Mon Sep 17 00:00:00 2001 From: Nikita Matskevich Date: Mon, 26 Oct 2026 10:18:19 +0100 Subject: [PATCH 31/32] fix: remove redundant interpolation in Bloom write test --- .../scala/org/apache/comet/CometIcebergWriteActionSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala index 922149b4034..a2e40fa9dab 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala @@ -1982,7 +1982,7 @@ class CometIcebergWriteActionSuite def insert(table: String): Unit = spark.sql( s"INSERT INTO cat.db.$table " + - s"SELECT CAST(id AS INT), 'region', CAST(id AS DOUBLE) " + + "SELECT CAST(id AS INT), 'region', CAST(id AS DOUBLE) " + s"FROM range(${insertedIds.start}, ${insertedIds.end}, 1, 1)") assertNativeWriteEngages("bloom_explicit_ndv_fold_native", insertedIds) { From 01c19f9f0ff9cb5e0c43e1786798bd1eb0005346 Mon Sep 17 00:00:00 2001 From: Nikita Matskevich Date: Tue, 27 Oct 2026 18:00:15 +0100 Subject: [PATCH 32/32] fix: remove unused Bloom test callback parameters --- .../comet/CometIcebergWriteActionSuite.scala | 22 +++++++++---------- .../CometIcebergWriteDetectionSuite.scala | 16 +++++++------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala index a2e40fa9dab..a399e379912 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala @@ -1263,7 +1263,7 @@ class CometIcebergWriteActionSuite test("native acceleration: writes configured Iceberg Parquet bloom filters") { assumeNativeBloomRuntime() assumeNativeAcceleration() - withIcebergCatalog { warehouseDir => + withIcebergCatalog { _ => createTable( "native_bloom", partitionSpec = "", @@ -1285,7 +1285,7 @@ class CometIcebergWriteActionSuite test("Bloom-filter folding config selects the native or JVM Iceberg writer") { assumeNativeBloomRuntime() assumeNativeAcceleration() - withIcebergCatalog { warehouseDir => + withIcebergCatalog { _ => val properties = Some("'write.parquet.bloom-filter-enabled.column.id'='true'") createTable("bloom_folding_enabled", partitionSpec = "", properties) createTable("bloom_folding_disabled", partitionSpec = "", properties) @@ -1371,7 +1371,7 @@ class CometIcebergWriteActionSuite test("enabled=false plus NDV matches JVM behavior for the Iceberg runtime") { assumeNativeBloomRuntime() assumeNativeAcceleration() - withIcebergCatalog { warehouseDir => + withIcebergCatalog { _ => val configuredFpp = 0.01 val configuredNdv = 1000 val properties = Some( @@ -1413,7 +1413,7 @@ class CometIcebergWriteActionSuite test("enabled=false preserves JVM validation errors for malformed FPP and NDV") { assumeNativeBloomRuntime() assumeNativeAcceleration() - withIcebergCatalog { warehouseDir => + withIcebergCatalog { _ => Seq( ("fpp", "'write.parquet.bloom-filter-fpp.column.id'='garbage'", icebergSupportsBloomFpp), ("ndv", "'write.parquet.bloom-filter-ndv.column.id'='garbage'", icebergSupportsBloomNdv)) @@ -1460,7 +1460,7 @@ class CometIcebergWriteActionSuite assumeNativeBloomRuntime() assumeNativeAcceleration() assumeIcebergBloomShapeProperties() - withIcebergCatalog { warehouseDir => + withIcebergCatalog { _ => val minimumBytes = 32 val naturallyMinimumNdv = 1 val bindingNdv = 1000000 @@ -1694,7 +1694,7 @@ class CometIcebergWriteActionSuite assumeNativeBloomRuntime() assumeNativeAcceleration() assumeIcebergBloomShapeProperties() - withIcebergCatalog { warehouseDir => + withIcebergCatalog { _ => // This NDV/FPP pair requests a 128 MiB allocation before max-bytes is applied. The 4 KiB // maximum must therefore bind on both writers rather than merely coinciding with the // naturally selected size. @@ -1854,7 +1854,7 @@ class CometIcebergWriteActionSuite test("native bloom sizing matches JVM for shape properties supported by the Iceberg runtime") { assumeNativeBloomRuntime() assumeNativeAcceleration() - withIcebergCatalog { warehouseDir => + withIcebergCatalog { _ => val enabled = "'write.parquet.bloom-filter-enabled.column.id'='true'" val cases: Seq[(String, String, Boolean, Boolean)] = Seq( ( @@ -1929,7 +1929,7 @@ class CometIcebergWriteActionSuite assumeNativeBloomRuntime() assumeNativeAcceleration() assumeIcebergBloomShapeProperties() - withIcebergCatalog { warehouseDir => + withIcebergCatalog { _ => // max-bytes is only a cap in parquet-mr; it does not enlarge a filter whose explicit NDV // was underestimated. This comparison prevents Comet from silently replacing the user's // NDV with an artificial NDV derived from the much larger maximum. @@ -1965,7 +1965,7 @@ class CometIcebergWriteActionSuite assumeNativeBloomRuntime() assumeNativeAcceleration() assumeIcebergBloomShapeProperties() - withIcebergCatalog { warehouseDir => + withIcebergCatalog { _ => val configuredNdv = 1000L val configuredFpp = 0.01 val insertedIds = 0 until 100 @@ -2018,7 +2018,7 @@ class CometIcebergWriteActionSuite test("large representable max folds natively while adjacent non-power-of-two falls back") { assumeNativeBloomRuntime() assumeNativeAcceleration() - withIcebergCatalog { warehouseDir => + withIcebergCatalog { _ => val powerOfTwo = 64 * 1024 * 1024 val enabled = "'write.parquet.bloom-filter-enabled.column.id'='true', " createTable( @@ -2078,7 +2078,7 @@ class CometIcebergWriteActionSuite assumeNativeBloomRuntime() assumeNativeAcceleration() assumeIcebergBloomShapeProperties() - withIcebergCatalog { warehouseDir => + withIcebergCatalog { _ => // A tiny explicit NDV keeps the fallback writer's allocation small even for the oversized // maximum. Without a configured Bloom column, the maximum is unused and stays native. val enabledWithSmallNdv = diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala index 681b6da0de9..2b724e487b0 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala @@ -412,7 +412,7 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes } test("bloom-filter max-bytes accepts only representable power-of-two values") { - withDetectionCatalog { dir => + withDetectionCatalog { _ => Seq("32", "524288", "1048576", "134217728").zipWithIndex.foreach { case (value, index) => val table = s"bloom_max_ok_$index" createTable( @@ -440,7 +440,7 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes } test("unused bloom-filter max-bytes does not force fallback") { - withDetectionCatalog { dir => + withDetectionCatalog { _ => val nonPowerOfTwoMaxBytes = 100 createTable( "unused_bloom_max", @@ -467,7 +467,7 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes test("bloom-filter max-bytes=32 falls back only when parquet-mr ignores the cap") { assumeNativeBloomRuntime() - withDetectionCatalog { dir => + withDetectionCatalog { _ => val minimumBytes = 32 val naturallyMinimumNdv = 1 val bindingFpp = 0.0001 @@ -507,7 +507,7 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes test("fall-back: bloom-filter NDV values that overflow parquet-mr sizing arithmetic") { assumeNativeBloomRuntime() - withDetectionCatalog { dir => + withDetectionCatalog { _ => val largestNonOverflowingNdv = Long.MaxValue / 8L val cases = Seq( (largestNonOverflowingNdv, true), @@ -536,7 +536,7 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes test("bloom-filter enabled=false with NDV follows runtime support and validates NDV") { assumeNativeBloomRuntime() - withDetectionCatalog { dir => + withDetectionCatalog { _ => val configuredNdv = 1000 createTable( "false_with_ndv", @@ -564,7 +564,7 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes test("bloom-filter enabled=false still validates FPP") { assumeNativeBloomRuntime() - withDetectionCatalog { dir => + withDetectionCatalog { _ => createTable( "false_with_bad_fpp", partitionSpec = "", @@ -583,7 +583,7 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes test("fall-back: invalid enabled-column FPP and NDV") { assumeNativeBloomRuntime() - withDetectionCatalog { dir => + withDetectionCatalog { _ => Seq( ("'write.parquet.bloom-filter-fpp.column.id'='0'", icebergSupportsBloomFpp), ("'write.parquet.bloom-filter-fpp.column.id'='1'", icebergSupportsBloomFpp), @@ -673,7 +673,7 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes test("Bloom-filter folding must be explicitly enabled for native writes") { assumeNativeBloomRuntime() - withDetectionCatalog { dir => + withDetectionCatalog { _ => createTable( "bloom_folding_opt_in", partitionSpec = "",