Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
302 changes: 279 additions & 23 deletions docs/source/user_guide/catalog.rst

Large diffs are not rendered by default.

6 changes: 6 additions & 0 deletions include/paimon/defs.h
Original file line number Diff line number Diff line change
Expand Up @@ -324,6 +324,12 @@ struct PAIMON_EXPORT Options {
/// "rest.commit.lock-enabled" on the table. Default value is "false".
static const char REST_COMMIT_ACQUIRE_LOCK[];

/// "query-auth.enabled" - Whether the catalog enforces row filters and column masks on reads
/// of the table. A scan of such a table asks the catalog built into its context for the rules
/// and attaches them to its splits, and a read applies them; a read path that would not apply
/// them is refused. The options of a read cannot disable it. Default value is "false".
static const char QUERY_AUTH_ENABLED[];

/// "compaction.max-size-amplification-percent" - The size amplification is defined as the
/// amount (in percentage) of additional storage needed to store a single byte of data in the
/// merge tree for changelog mode table. Default value is 200.
Expand Down
21 changes: 16 additions & 5 deletions include/paimon/read_context.h
Original file line number Diff line number Diff line change
Expand Up @@ -408,6 +408,8 @@ class PAIMON_EXPORT ReadContextBuilder {

/// Whether to perform precise filtering according to predicates for data read from format
/// reader.
/// @note On a table with `query-auth.enabled=true` the predicate is evaluated on the masked
/// values, and its conjuncts on masked columns are evaluated even when this is false.
/// @param enabled Whether to enable precise filtering (default: false)
/// @return Reference to this builder for method chaining.
ReadContextBuilder& EnablePredicateFilter(bool enabled);
Expand Down Expand Up @@ -566,11 +568,20 @@ class PAIMON_EXPORT ReadContextBuilder {
/// @note If not set, use default file system (configured in `Options::FILE_SYSTEM`)
ReadContextBuilder& WithFileSystem(const std::shared_ptr<FileSystem>& file_system);

/// Reads a native table through its own file system - including the per-table temporary
/// credentials a catalog that issues them hands out through `Catalog::GetTableFileSystem`.
/// This is a shorthand for `WithFileSystem(catalog->GetTableFileSystem(identifier))`; an
/// explicit `WithFileSystem()` takes precedence, so the catalog is not asked.
/// @param catalog Non-null catalog, read when `Finish()` builds the context.
/// Reads a native table through its catalog, which supplies its file system:
/// `catalog->GetTableFileSystem(identifier)`, with any per-table temporary credentials; an
/// explicit `WithFileSystem()` takes precedence, so the catalog is not asked for one.
/// `TableRead::Create()` reads `query-auth.enabled` from the schema the catalog holds for the
/// branch read. A branch named by a system table path is resolved before requesting the table's
/// file system and must agree with the identifier, `branch` option and `WithBranch()`.
/// The fields the read reads with still come from `SetTableSchema()` on the main
/// branch or from the latest schema under the table path. When the catalog does not enable it,
/// the read path must be `catalog->GetTableLocation()` of the branch read.
/// A `sys` identifier requires a global system table path. A branch the catalog
/// would read as another object, such as `MAIN` or one holding a `$`, is rejected before the
/// catalog is asked for anything.
/// @param catalog Non-null catalog, read when `Finish()` builds the context and when
/// `TableRead::Create()` runs.
/// @param identifier The native table to read.
/// @return Reference to this builder for method chaining.
ReadContextBuilder& WithCatalog(const std::shared_ptr<Catalog>& catalog,
Expand Down
48 changes: 43 additions & 5 deletions include/paimon/scan_context.h
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,22 @@ class PAIMON_EXPORT ScanContext {
const std::shared_ptr<Catalog>& catalog,
const std::optional<Identifier>& identifier);

/// Also takes the columns the query reads, see `ScanContextBuilder::SetReadFieldNames()`.
ScanContext(const std::string& path, bool is_streaming_mode, std::optional<int32_t> limit,
const std::shared_ptr<ScanFilter>& scan_filter,
const std::shared_ptr<GlobalIndexResult>& global_index_result,
const std::shared_ptr<RealtimeContext>& realtime_context,
const std::shared_ptr<MemoryPool>& memory_pool,
const std::shared_ptr<Executor>& executor,
const std::shared_ptr<FileSystem>& specific_file_system,
const std::optional<std::string>& table_schema,
const std::map<std::string, std::string>& options,
const std::shared_ptr<Cache>& cache,
const std::shared_ptr<FormatTable>& format_table,
const std::shared_ptr<Catalog>& catalog,
const std::optional<Identifier>& identifier,
const std::optional<std::vector<std::string>>& read_field_names);

~ScanContext();

const std::string& GetPath() const {
Expand Down Expand Up @@ -127,6 +143,10 @@ class PAIMON_EXPORT ScanContext {
return identifier_;
}

const std::optional<std::vector<std::string>>& GetReadFieldNames() const {
return read_field_names_;
}

private:
std::string path_;
bool is_streaming_mode_;
Expand All @@ -143,6 +163,7 @@ class PAIMON_EXPORT ScanContext {
std::shared_ptr<FormatTable> format_table_;
std::shared_ptr<Catalog> catalog_;
std::optional<Identifier> identifier_;
std::optional<std::vector<std::string>> read_field_names_;
};

/// Filter configuration for table scan operations
Expand Down Expand Up @@ -197,6 +218,11 @@ class PAIMON_EXPORT ScanContextBuilder {
const std::vector<std::map<std::string, std::string>>& partition_filters);
/// Set a predicate for filtering data.
ScanContextBuilder& SetPredicate(const std::shared_ptr<Predicate>& predicate);
/// Set the columns the query reads. A table with `query-auth.enabled=true` asks its catalog
/// for the rules of these columns plus those of the predicate and the partition filter; by
/// default it asks for all columns. An empty list asks for no other column, unlike the empty
/// list of `ReadContextBuilder::SetReadFieldNames()`, which reads every column.
ScanContextBuilder& SetReadFieldNames(const std::vector<std::string>& read_field_names);

/// Sets the result of a global index search (e.g., row ids (may with scores) from a distributed
/// index lookup). This is used to push down index-filtered row ids into the scan for efficient
Expand Down Expand Up @@ -241,11 +267,23 @@ class PAIMON_EXPORT ScanContextBuilder {
/// @note If not set, use default file system (configured in `Options::FILE_SYSTEM`)
ScanContextBuilder& WithFileSystem(const std::shared_ptr<FileSystem>& file_system);

/// Plans a native table through its own file system - including the per-table temporary
/// credentials a catalog that issues them hands out through `Catalog::GetTableFileSystem`.
/// This is a shorthand for `WithFileSystem(catalog->GetTableFileSystem(identifier))`; an
/// explicit `WithFileSystem()` takes precedence, so the catalog is not asked.
/// @param catalog Non-null catalog, read when `Finish()` builds the context.
/// Plans a native table through its catalog, which supplies its file system and, for a table
/// with `query-auth.enabled`, its rules. The file system is
/// `catalog->GetTableFileSystem(identifier)`, with any per-table temporary credentials; an
/// explicit `WithFileSystem()` takes precedence, so the catalog is not asked for one.
/// `TableScan::Create()` reads `query-auth.enabled` from the schema the catalog holds for the
/// branch scanned; the fields the scan plans with still come from `SetTableSchema()` on the
/// main branch or from the latest schema under the table path. The scan path must be
/// `catalog->GetTableLocation()` of the branch scanned, so the catalog's setting is taken only
/// for the table it keeps there. A `sys` identifier requires a global system table path.
/// A query-auth scan checks the location again on every plan, along with whether its selected
/// and rule columns keep their field ids in the catalog's current schema.
/// A branch the `branch` option or a system table path names is carried into the identifier,
/// as `tbl$branch_dev`, before requesting the table's file system; an
/// identifier naming another branch, or a branch the catalog would read as another object, such
/// as `MAIN` or one holding a `$`, is rejected before the catalog is asked for anything.
/// @param catalog Non-null catalog, read when `Finish()` builds the context and when
/// `TableScan::Create()` runs.
/// @param identifier The native table to scan.
/// @return Reference to this builder for method chaining.
ScanContextBuilder& WithCatalog(const std::shared_ptr<Catalog>& catalog,
Expand Down
3 changes: 2 additions & 1 deletion include/paimon/table/source/split.h
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,8 @@ class MemoryPool;
/// This split can be a `DataSplit` (for direct data file reads), an `IndexedSplit` (for reads
/// leveraging global indexes), or a `FormatDataSplit` (for a format table's plain data files).
/// Only the first two are serializable; a `FormatDataSplit` is in-memory only and `Serialize()`
/// refuses it.
/// refuses it. A split of a table with `query-auth.enabled=true` wraps one of the first two with
/// the row filters and column masks it is read with.
class PAIMON_EXPORT Split {
public:
virtual ~Split() = default;
Expand Down
6 changes: 6 additions & 0 deletions src/paimon/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,7 @@ set(PAIMON_CORE_SRCS
core/catalog/catalog_utils.cpp
core/catalog/file_system_catalog.cpp
core/catalog/identifier.cpp
core/catalog/table_query_auth_result.cpp
core/core_options.cpp
core/deletionvectors/deletion_vector.cpp
core/deletionvectors/deletion_file_writer.cpp
Expand Down Expand Up @@ -485,6 +486,8 @@ set(PAIMON_CORE_SRCS
core/table/source/merge_tree_split_generator.cpp
core/table/source/data_evolution_split_generator.cpp
core/table/source/plan_impl.cpp
core/table/source/query_auth_table_read.cpp
core/table/source/query_auth_table_scan.cpp
core/table/source/realtime_table_scan.cpp
core/table/source/snapshot/snapshot_reader.cpp
core/table/source/startup_mode.cpp
Expand Down Expand Up @@ -852,6 +855,7 @@ if(PAIMON_BUILD_TESTS)
core/catalog/file_system_catalog_test.cpp
core/catalog/catalog_test.cpp
core/catalog/identifier_test.cpp
core/catalog/table_query_auth_result_test.cpp
core/compact/compact_deletion_file_test.cpp
core/compact/compact_future_manager_test.cpp
core/compact/noop_compact_manager_test.cpp
Expand Down Expand Up @@ -1046,6 +1050,8 @@ if(PAIMON_BUILD_TESTS)
core/table/sink/commit_message_impl_test.cpp
core/table/source/fallback_data_split_test.cpp
core/table/source/primary_key_sorted_index_scan_test.cpp
core/table/source/query_auth_split_test.cpp
core/table/source/query_auth_table_read_test.cpp
core/table/source/table_read_test.cpp
core/table/source/append_count_reader_test.cpp
core/table/source/pk_count_reader_test.cpp
Expand Down
1 change: 1 addition & 0 deletions src/paimon/common/defs.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ const char Options::COMMIT_MAX_RETRY_WAIT[] = "commit.max-retry-wait";
const char Options::COMMIT_DISCARD_DUPLICATE_FILES[] = "commit.discard-duplicate-files";
const char Options::REST_COMMIT_LOCK_ENABLED[] = "rest.commit.lock-enabled";
const char Options::REST_COMMIT_ACQUIRE_LOCK[] = "rest.commit.acquire-lock";
const char Options::QUERY_AUTH_ENABLED[] = "query-auth.enabled";
const char Options::SEQUENCE_FIELD[] = "sequence.field";
const char Options::SEQUENCE_FIELD_SORT_ORDER[] = "sequence.field.sort-order";
const char Options::MERGE_ENGINE[] = "merge-engine";
Expand Down
8 changes: 2 additions & 6 deletions src/paimon/common/predicate/like.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -44,11 +44,6 @@ inline size_t Utf8CodePointLength(unsigned char leading_byte) {
return 1; // invalid continuation byte, treat as single byte
}

inline bool IsJavaRegexLineTerminator(const std::string& code_point) {
return code_point == "\n" || code_point == "\r" || code_point == "\xC2\x85" ||
code_point == "\xE2\x80\xA8" || code_point == "\xE2\x80\xA9";
}

} // namespace

Result<bool> Like::MatchString(const std::string& field, const std::string& pattern) {
Expand Down Expand Up @@ -153,7 +148,8 @@ Result<bool> Like::MatchString(const std::string& field, const std::string& patt
if (wild && pc == "%") {
dp[j] = dp[j - 1] || dp[j];
} else if (wild && pc == "_") {
dp[j] = prev && !IsJavaRegexLineTerminator(field_char);
// As Java `Like`, which matches `_` with `(?s:.)`: any one code point.
dp[j] = prev;
} else {
dp[j] = (pc == field_char) ? prev : false;
}
Expand Down
54 changes: 48 additions & 6 deletions src/paimon/common/predicate/literal_converter.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
#include "arrow/type.h"
#include "arrow/type_traits.h"
#include "arrow/util/decimal.h"
#include "arrow/util/int_util_overflow.h"
#include "fmt/format.h"
#include "paimon/common/data/binary_string.h"
#include "paimon/common/data/internal_row.h"
Expand Down Expand Up @@ -379,6 +380,48 @@ Result<std::shared_ptr<arrow::Array>> BuildDecimalArray(const std::vector<Litera
return array;
}

// The number of `unit`s since the epoch `timestamp` names, or an error when that number does not
// fit int64. Epoch nanoseconds only reach the years 1677 to 2262, so a timestamp outside them
// still has milliseconds but no nanoseconds to write.
Result<int64_t> TimestampToCheckedInteger(const Timestamp& timestamp, arrow::TimeUnit::type unit) {
int64_t millis = timestamp.GetMillisecond();
const int64_t nanos = timestamp.GetNanoOfMillisecond();
int64_t factor = 1;
int64_t remainder = 0;
switch (unit) {
case arrow::TimeUnit::MILLI:
return millis;
case arrow::TimeUnit::MICRO:
factor = 1000;
remainder = nanos / 1000;
break;
case arrow::TimeUnit::NANO:
factor = 1000000;
remainder = nanos;
break;
default:
return Status::Invalid(fmt::format("Not support writing a timestamp literal in {}",
arrow::timestamp(unit)->ToString()));
}
// The earliest values that fit have a negative millisecond whose product alone does not, so a
// negative millisecond lends one `factor` to the remainder first.
if (millis < 0 && remainder > 0) {
++millis;
remainder -= factor;
}
int64_t scaled = 0;
int64_t value = 0;
if (arrow::internal::MultiplyWithOverflow(millis, factor, &scaled) ||
arrow::internal::AddWithOverflow(scaled, remainder, &value)) {
// `Timestamp::ToString` cannot format an instant this far from the epoch.
return Status::Invalid(fmt::format(
"Timestamp literal of {} milliseconds and {} nanoseconds since the epoch does not fit "
"int64 in {}",
timestamp.GetMillisecond(), nanos, arrow::timestamp(unit)->ToString()));
}
return value;
}

// Writes the literals to a timestamp array of the finest time unit that keeps every value, one
// `is_in` can only compare against a column of that very unit. One array holds one arrow type, so
// a non null literal is needed to settle the unit.
Expand All @@ -396,18 +439,17 @@ Result<std::shared_ptr<arrow::Array>> BuildTimestampArray(const std::vector<Lite
"Not support converting literals of TIMESTAMP type to an arrow array without a non "
"null literal to take the time unit from");
}
arrow::TimestampBuilder builder(
arrow::timestamp(LiteralConverter::MinRequiredTimeUnit(literals)), pool);
const DateTimeUtils::TimeType time_type = DateTimeUtils::GetTimeTypeFromArrowType(
checked_pointer_cast<arrow::TimestampType>(builder.type()));
const arrow::TimeUnit::type unit = LiteralConverter::MinRequiredTimeUnit(literals);
arrow::TimestampBuilder builder(arrow::timestamp(unit), pool);
PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Reserve(static_cast<int64_t>(literals.size())));
for (const auto& literal : literals) {
if (literal.IsNull()) {
PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.AppendNull());
continue;
}
PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Append(
DateTimeUtils::TimestampToInteger(literal.GetValue<Timestamp>(), time_type)));
PAIMON_ASSIGN_OR_RAISE(int64_t value,
TimestampToCheckedInteger(literal.GetValue<Timestamp>(), unit));
PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Append(value));
}
std::shared_ptr<arrow::Array> array;
PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Finish(&array));
Expand Down
3 changes: 2 additions & 1 deletion src/paimon/common/predicate/literal_converter.h
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,8 @@ class PAIMON_EXPORT LiteralConverter {
/// precision and the scale its literals carry, so at least one literal has to be non
/// null and every non null one has to carry the same pair. `TIMESTAMP` is written with
/// the finest time unit that keeps every value, so at least one literal has to be non
/// null to settle the unit.
/// null to settle the unit, and a literal whose value in that unit does not fit int64
/// fails.
static Result<std::shared_ptr<arrow::Array>> ConvertLiteralsToArray(
const FieldType& field_type, const std::vector<Literal>& literals, arrow::MemoryPool* pool);

Expand Down
33 changes: 33 additions & 0 deletions src/paimon/common/predicate/literal_converter_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
#include <algorithm>
#include <cmath>
#include <cstddef>
#include <limits>
#include <variant>

#include "arrow/api.h"
Expand Down Expand Up @@ -597,6 +598,38 @@ TEST_F(LiteralConverterTest, TestLiteralsToArrayWithoutValue) {
arrow::ipc::internal::json::ArrayFromJSON(arrow::utf8(), R"([null, null])").ValueOrDie());
}

TEST_F(LiteralConverterTest, TestLiteralsToArrayRejectsTimestampsOutOfRange) {
// INT64_MAX and INT64_MIN epoch nanoseconds
CheckLiteralsToArray(
FieldType::TIMESTAMP,
{Literal(Timestamp(9223372036854l, 775807)), Literal(Timestamp(-9223372036855l, 224192))},
arrow::ipc::internal::json::ArrayFromJSON(arrow::timestamp(arrow::TimeUnit::NANO),
R"([9223372036854775807, -9223372036854775808])")
.ValueOrDie());
ASSERT_NOK_WITH_MSG(LiteralConverter::ConvertLiteralsToArray(
FieldType::TIMESTAMP, {Literal(Timestamp(9223372036854l, 775808))},
arrow::default_memory_pool()),
"does not fit int64");
ASSERT_NOK_WITH_MSG(LiteralConverter::ConvertLiteralsToArray(
FieldType::TIMESTAMP, {Literal(Timestamp(-9223372036855l, 224191))},
arrow::default_memory_pool()),
"does not fit int64");
// 2500-01-01T00:00:00.000000001, beyond epoch nanoseconds
ASSERT_NOK_WITH_MSG(LiteralConverter::ConvertLiteralsToArray(
FieldType::TIMESTAMP, {Literal(Timestamp(16725225600000l, 1))},
arrow::default_memory_pool()),
"does not fit int64");
CheckLiteralsToArray(FieldType::TIMESTAMP, {Literal(Timestamp(16725225600000l, 1000))},
arrow::ipc::internal::json::ArrayFromJSON(
arrow::timestamp(arrow::TimeUnit::MICRO), R"([16725225600000001])")
.ValueOrDie());
ASSERT_NOK_WITH_MSG(
LiteralConverter::ConvertLiteralsToArray(
FieldType::TIMESTAMP, {Literal(Timestamp(std::numeric_limits<int64_t>::max(), 1000))},
arrow::default_memory_pool()),
"does not fit int64");
}

TEST_F(LiteralConverterTest, TestLiteralsToArrayUnsupportedType) {
// a timestamp takes its arrow type from a literal, so it needs one that is not null
ASSERT_NOK_WITH_MSG(LiteralConverter::ConvertLiteralsToArray(FieldType::TIMESTAMP, {},
Expand Down
Loading
Loading