Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions docs/source/api/file_index.rst
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,15 @@ File Index
Interface
=========

.. doxygenclass:: paimon::FileIndexRead
:members:

.. doxygenstruct:: paimon::DataSplit::SimpleDataFileMeta
:members:

.. doxygenstruct:: paimon::FileRowSelection
:members:

.. doxygenstruct:: paimon::FullTextMatch
:members:

Expand Down
3 changes: 3 additions & 0 deletions docs/source/api/write.rst
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@ Write
Interface
=========

.. doxygenclass:: paimon::DeletionVectorWrite
:members:

.. doxygenclass:: paimon::FileStoreWrite
:members:
:undoc-members:
Expand Down
28 changes: 28 additions & 0 deletions docs/source/user_guide/write.rst
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,34 @@ Parquet, and Avro, thereby improving write throughput.
the files it ignored. See :doc:`compaction` for the ``full-compaction`` and
``lookup`` producers.

Updates with Append and Deletion Vectors
---------------------------------------

For fixed-bucket or unaware-bucket managed append tables, enable
``deletion-vectors.enabled``.

1. Plan batch splits at a base snapshot without limits or pruning by individual
keys. Create ``FileIndexRead`` per split and reuse ``Query(predicate)`` to return
files and physical positions excluding historical DV, or an empty result.
Metadata statistics may exclude whole files. The precise read path uses indexes
and data filtering to find exact positions. Index readers and historical DV are reused.
2. Compute replacement rows externally and append through ``FileStoreWrite``
with ``write-only=true``.
3. Call ``DeletionVectorWrite::PrepareCommit()`` with the table path, branch,
options, ``base_snapshot_id``, all deletion positions, file system, memory pool,
and executor. The entire batch uses one snapshot; passing ``std::nullopt``
selects the latest snapshot for this call.
4. Combine the returned DV messages with ``FileStoreWrite::PrepareCommit()`` messages in one
``FileStoreCommit::Commit()`` call. Deletes alone need only the DV messages.

Different buckets may be updated concurrently. For the same partition and bucket,
wait for the previous update to commit before selecting the next base snapshot.
Do not run compaction or overwrite on that bucket during an update.

Both APIs load table metadata from disk and support bitmap32 only, including
historical DV. Primary-key, Data Evolution, real-time, query-authorized, and
format tables are unsupported.

External Data Paths
-------------------

Expand Down
2 changes: 2 additions & 0 deletions include/paimon/api.h
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,9 @@

#include "paimon/commit_context.h" // IWYU pragma: export
#include "paimon/defs.h" // IWYU pragma: export
#include "paimon/deletion_vector_write.h" // IWYU pragma: export
#include "paimon/factories/factory.h" // IWYU pragma: export
#include "paimon/file_index/file_index_read.h" // IWYU pragma: export
#include "paimon/file_store_commit.h" // IWYU pragma: export
#include "paimon/file_store_write.h" // IWYU pragma: export
#include "paimon/fs/file_system_factory.h" // IWYU pragma: export
Expand Down
58 changes: 58 additions & 0 deletions include/paimon/deletion_vector_write.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#pragma once

#include <cstdint>
#include <map>
#include <memory>
#include <optional>
#include <string>
#include <vector>

#include "paimon/commit_message.h"
#include "paimon/result.h"
#include "paimon/table/source/file_row_selection.h"
#include "paimon/visibility.h"

namespace paimon {
class Executor;
class FileSystem;
class MemoryPool;

/// Merges append-table DV updates. Different buckets may run concurrently.
/// Serialize updates, compaction and overwrite within each partition/bucket,
/// from snapshot selection through commit.
class PAIMON_EXPORT DeletionVectorWrite {
public:
DeletionVectorWrite() = delete;
~DeletionVectorWrite() = delete;

/// Uses one snapshot for the batch; std::nullopt selects latest.
/// Requires a managed fixed/unaware append table with bitmap32 DV.
/// Commit returned messages with append messages; callers handle recovery and orphan cleanup.
static Result<std::vector<std::shared_ptr<CommitMessage>>> PrepareCommit(
const std::string& table_path, const std::string& branch,
const std::map<std::string, std::string>& options,
const std::optional<int64_t>& base_snapshot_id,
const std::vector<FileRowSelection>& deletions,
const std::shared_ptr<FileSystem>& file_system, const std::shared_ptr<MemoryPool>& pool,
const std::shared_ptr<Executor>& executor);
};

} // namespace paimon
54 changes: 54 additions & 0 deletions include/paimon/file_index/file_index_read.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#pragma once

#include <map>
#include <memory>
#include <string>
#include <vector>

#include "paimon/result.h"
#include "paimon/table/source/file_row_selection.h"
#include "paimon/visibility.h"

namespace paimon {
class Predicate;
class FileSystem;
class MemoryPool;
class Split;

/// Queries live physical positions in a managed append-table batch split; bitmap32 only.
/// Uses indexes and data filtering to return exact positions.
/// Caches index readers and historical DV. Not thread-safe.
class PAIMON_EXPORT FileIndexRead {
public:
/// Binds a split; options override table options.
static Result<std::unique_ptr<FileIndexRead>> Create(
const std::string& table_path, const std::string& branch,
const std::map<std::string, std::string>& options, const std::shared_ptr<Split>& split,
const std::shared_ptr<FileSystem>& file_system, const std::shared_ptr<MemoryPool>& pool);
virtual ~FileIndexRead() = default;

/// Returns matching files and positions; empty for an empty split or no matching rows.
/// Predicate must be non-null.
virtual Result<std::vector<FileRowSelection>> Query(
const std::shared_ptr<Predicate>& predicate) = 0;
};

} // namespace paimon
2 changes: 2 additions & 0 deletions include/paimon/table/source/data_split.h
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ class PAIMON_EXPORT DataSplit : public Split {
int64_t schema_id;
int32_t level;
Timestamp creation_time;
/// Snapshot delete count: DV cardinality or stored metadata; nullopt if unknown.
std::optional<int64_t> delete_row_count;

bool operator==(const SimpleDataFileMeta& other) const;
Expand All @@ -81,6 +82,7 @@ class PAIMON_EXPORT DataSplit : public Split {
virtual int32_t Bucket() const = 0;

/// Get the list of metadata for all data files in this split.
/// Uses snapshot DV metadata without reading DV files or changing stored metadata.
/// @note This method will be removed in future versions and is only used for append tables.
virtual std::vector<SimpleDataFileMeta> GetFileList() const = 0;
};
Expand Down
40 changes: 40 additions & 0 deletions include/paimon/table/source/file_row_selection.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#pragma once

#include <cstdint>
#include <map>
#include <string>

#include "paimon/table/source/data_split.h"
#include "paimon/utils/roaring_bitmap32.h"
#include "paimon/visibility.h"

namespace paimon {

/// Zero-based physical file positions, never renumbered after filtering.
struct PAIMON_EXPORT FileRowSelection {
DataSplit::SimpleDataFileMeta file;
/// String partition values; NULL uses partition.default-name.
std::map<std::string, std::string> partition;
int32_t bucket = 0;
RoaringBitmap32 positions;
};

} // namespace paimon
4 changes: 4 additions & 0 deletions src/paimon/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -419,6 +419,9 @@ set(PAIMON_CORE_SRCS
core/operation/orphan_files_cleaner_impl.cpp
core/operation/raw_file_split_read.cpp
core/operation/precise_file_index_read.cpp
core/operation/append_dv_context.cpp
core/operation/file_index_read.cpp
core/operation/deletion_vector_write.cpp
core/operation/read_context.cpp
core/operation/scan_context.cpp
core/manifest/snapshot_live_manifest_entries.cpp
Expand Down Expand Up @@ -1015,6 +1018,7 @@ if(PAIMON_BUILD_TESTS)
core/operation/commit/sequence_snapshot_properties_test.cpp
core/operation/key_value_file_store_write_test.cpp
core/operation/precise_file_index_read_test.cpp
core/operation/append_dv_test.cpp
core/operation/internal_read_context_test.cpp
core/operation/abstract_split_read_test.cpp
core/operation/append_only_file_store_write_test.cpp
Expand Down
30 changes: 30 additions & 0 deletions src/paimon/core/deletionvectors/bucketed_dv_maintainer.h
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,15 @@
#include <memory>
#include <optional>
#include <string>
#include <utility>

#include "paimon/core/deletionvectors/bitmap64_deletion_vector.h"
#include "paimon/core/deletionvectors/bitmap_deletion_vector.h"
#include "paimon/core/deletionvectors/deletion_vector.h"
#include "paimon/core/deletionvectors/deletion_vectors_index_file.h"
#include "paimon/core/index/index_file_handler.h"
#include "paimon/core/index/index_file_meta.h"
#include "paimon/utils/roaring_bitmap32.h"

namespace paimon {

Expand Down Expand Up @@ -62,6 +64,34 @@ class BucketedDvMaintainer {
return Status::OK();
}

Status NotifyNewDeletion(const std::string& file_name, const RoaringBitmap32& positions) {
if (positions.IsEmpty()) {
return Status::OK();
}
auto bitmap = std::make_shared<BitmapDeletionVector>(positions);
auto iter = deletion_vectors_.find(file_name);
std::shared_ptr<DeletionVector> incoming = bitmap;
bool use_bitmap64 =
iter == deletion_vectors_.end()
? bitmap64_
: dynamic_cast<Bitmap64DeletionVector*>(iter->second.get()) != nullptr;
if (use_bitmap64) {
incoming = Bitmap64DeletionVector::FromBitmapDeletionVector(*bitmap);
}
if (iter == deletion_vectors_.end()) {
deletion_vectors_.emplace(file_name, std::move(incoming));
modified_ = true;
return Status::OK();
}
PAIMON_ASSIGN_OR_RAISE(int64_t before, iter->second->GetCardinality());
PAIMON_RETURN_NOT_OK(iter->second->Merge(incoming));
PAIMON_ASSIGN_OR_RAISE(int64_t after, iter->second->GetCardinality());
if (after != before) {
modified_ = true;
}
return Status::OK();
}

std::optional<std::shared_ptr<DeletionVector>> DeletionVectorOf(
const std::string& file_name) const {
if (auto it = deletion_vectors_.find(file_name); it != deletion_vectors_.end()) {
Expand Down
61 changes: 61 additions & 0 deletions src/paimon/core/deletionvectors/bucketed_dv_maintainer_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,67 @@ TEST(BucketedDvMaintainerTest, TestNotifyNewDeletionOnExistingVectorOnlyMarksWhe
ASSERT_TRUE(modified_write.has_value());
}

TEST(BucketedDvMaintainerTest, TestNotifyNewDeletionBitmap) {
for (bool restored_bitmap64 : {false, true}) {
auto dir = UniqueTestDirectory::Create();
ASSERT_TRUE(dir);
auto index = CreateDvIndexFile(dir->Str(), !restored_bitmap64);
auto bitmap = std::make_shared<BitmapDeletionVector>(RoaringBitmap32::From({0, 1, 2}));
std::shared_ptr<DeletionVector> existing = bitmap;
if (restored_bitmap64) {
existing = Bitmap64DeletionVector::FromBitmapDeletionVector(*bitmap);
}
BucketedDvMaintainer maintainer(index, {{"existing", existing}});

ASSERT_OK(maintainer.NotifyNewDeletion("empty", RoaringBitmap32()));
ASSERT_FALSE(maintainer.DeletionVectorOf("empty").has_value());
ASSERT_OK(maintainer.NotifyNewDeletion("existing", RoaringBitmap32::From({1, 2})));
ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<IndexFileMeta>> unchanged,
maintainer.WriteDeletionVectorsIndex());
ASSERT_FALSE(unchanged.has_value());

auto positions = RoaringBitmap32::From({2, 7, 65537});
ASSERT_OK(maintainer.NotifyNewDeletion("existing", positions));
ASSERT_OK(maintainer.NotifyNewDeletion("new", positions));
ASSERT_EQ(positions, RoaringBitmap32::From({2, 7, 65537}));
ASSERT_OK_AND_ASSIGN(int64_t cardinality, existing->GetCardinality());
ASSERT_EQ(cardinality, 5);
ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<IndexFileMeta>> written,
maintainer.WriteDeletionVectorsIndex());
ASSERT_TRUE(written.has_value());
std::map<std::string, std::shared_ptr<DeletionVector>> restored;
ASSERT_OK_AND_ASSIGN(restored, index->ReadAllDeletionVectors(*written));
ASSERT_EQ(restored.size(), 2);
ASSERT_EQ(dynamic_cast<Bitmap64DeletionVector*>(restored.at("existing").get()) != nullptr,
restored_bitmap64);
ASSERT_EQ(dynamic_cast<Bitmap64DeletionVector*>(restored.at("new").get()) != nullptr,
!restored_bitmap64);
for (int32_t position : {0, 1, 2, 7, 65537}) {
ASSERT_OK_AND_ASSIGN(bool deleted, restored.at("existing")->IsDeleted(position));
ASSERT_TRUE(deleted);
}
ASSERT_OK_AND_ASSIGN(cardinality, restored.at("new")->GetCardinality());
ASSERT_EQ(cardinality, 3);
for (int32_t position : {2, 7, 65537}) {
ASSERT_OK_AND_ASSIGN(bool deleted, restored.at("new")->IsDeleted(position));
ASSERT_TRUE(deleted);
}

ASSERT_OK(maintainer.NotifyNewDeletion("existing", positions));
ASSERT_OK(maintainer.NotifyNewDeletion("new", positions));
ASSERT_OK(maintainer.NotifyNewDeletion("existing", RoaringBitmap32()));
ASSERT_OK_AND_ASSIGN(unchanged, maintainer.WriteDeletionVectorsIndex());
ASSERT_FALSE(unchanged.has_value());
positions.Add(9);
ASSERT_OK_AND_ASSIGN(bool deleted, existing->IsDeleted(9));
ASSERT_FALSE(deleted);
auto created = maintainer.DeletionVectorOf("new");
ASSERT_TRUE(created.has_value());
ASSERT_OK_AND_ASSIGN(deleted, (*created)->IsDeleted(9));
ASSERT_FALSE(deleted);
}
}

TEST(BucketedDvMaintainerTest, MixedFormatsAfterConfigurationChanges) {
for (bool initial_bitmap64 : {false, true}) {
auto dir = UniqueTestDirectory::Create();
Expand Down
Loading
Loading